=== RUN kuttl harness.go:460: starting setup harness.go:258: running tests using configured kubeconfig. harness.go:281: Successful connection to cluster at: https://34.60.180.255 harness.go:366: running tests harness.go:77: going to run test suite with timeout of 600 seconds for each step harness.go:378: testsuite: e2e-tests/tests has 59 tests === RUN kuttl/harness === RUN kuttl/harness/major-upgrade-logical-replica === PAUSE kuttl/harness/major-upgrade-logical-replica === CONT kuttl/harness/major-upgrade-logical-replica logger.go:42: 20:36:45 | major-upgrade-logical-replica | Ignoring "conf": does not begin with a number followed by a dash. logger.go:42: 20:36:45 | major-upgrade-logical-replica | Creating namespace "kuttl-test-intent-zebra" logger.go:42: 20:36:46 | major-upgrade-logical-replica/0-deploy-operator | starting test step 0-deploy-operator logger.go:42: 20:36:46 | major-upgrade-logical-replica/0-deploy-operator | running command: [sh -c set -o errexit set -o xtrace source ../../functions init_temp_dir # do this only in the first TestStep deploy_operator deploy_client deploy_s3_secrets] logger.go:42: 20:36:46 | major-upgrade-logical-replica/0-deploy-operator | + source ../../functions logger.go:42: 20:36:46 | major-upgrade-logical-replica/0-deploy-operator | +++ realpath ../../.. logger.go:42: 20:36:46 | major-upgrade-logical-replica/0-deploy-operator | ++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1727 logger.go:42: 20:36:46 | major-upgrade-logical-replica/0-deploy-operator | ++ CERT_MANAGER_VER=1.20.2 logger.go:42: 20:36:46 | major-upgrade-logical-replica/0-deploy-operator | ++ CERT_MANAGER_TEST_LABEL=e2e.percona.com/managed-by=percona-postgresql-operator-tests logger.go:42: 20:36:46 | major-upgrade-logical-replica/0-deploy-operator | ++ CHAOS_MESH_VER=2.5.1 logger.go:42: 20:36:46 | major-upgrade-logical-replica/0-deploy-operator | ++ BUSYBOX_VER=1.36 logger.go:42: 20:36:46 | major-upgrade-logical-replica/0-deploy-operator | ++++ pwd logger.go:42: 20:36:46 | major-upgrade-logical-replica/0-deploy-operator | +++ basename /mnt/jenkins/workspace/cloud-pg-operator_PR-1727/e2e-tests/tests/major-upgrade-logical-replica logger.go:42: 20:36:46 | major-upgrade-logical-replica/0-deploy-operator | ++ test_name=major-upgrade-logical-replica logger.go:42: 20:36:46 | major-upgrade-logical-replica/0-deploy-operator | ++ source /mnt/jenkins/workspace/cloud-pg-operator_PR-1727/e2e-tests/vars.sh logger.go:42: 20:36:46 | major-upgrade-logical-replica/0-deploy-operator | +++ export ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1727 logger.go:42: 20:36:46 | major-upgrade-logical-replica/0-deploy-operator | +++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1727 logger.go:42: 20:36:46 | major-upgrade-logical-replica/0-deploy-operator | +++ export DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1727/deploy logger.go:42: 20:36:46 | major-upgrade-logical-replica/0-deploy-operator | +++ DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1727/deploy logger.go:42: 20:36:46 | major-upgrade-logical-replica/0-deploy-operator | +++ export TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1727/e2e-tests logger.go:42: 20:36:46 | major-upgrade-logical-replica/0-deploy-operator | +++ TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1727/e2e-tests logger.go:42: 20:36:46 | major-upgrade-logical-replica/0-deploy-operator | +++ export TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1727/e2e-tests/conf logger.go:42: 20:36:46 | major-upgrade-logical-replica/0-deploy-operator | +++ TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1727/e2e-tests/conf logger.go:42: 20:36:46 | major-upgrade-logical-replica/0-deploy-operator | +++ export TEST_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1727/e2e-tests/tests/major-upgrade-logical-replica/conf logger.go:42: 20:36:46 | major-upgrade-logical-replica/0-deploy-operator | +++ TEST_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1727/e2e-tests/tests/major-upgrade-logical-replica/conf logger.go:42: 20:36:46 | major-upgrade-logical-replica/0-deploy-operator | +++ export TEMP_DIR=/tmp/kuttl/pg/major-upgrade-logical-replica logger.go:42: 20:36:46 | major-upgrade-logical-replica/0-deploy-operator | +++ TEMP_DIR=/tmp/kuttl/pg/major-upgrade-logical-replica logger.go:42: 20:36:46 | major-upgrade-logical-replica/0-deploy-operator | ++++ git rev-parse --abbrev-ref HEAD logger.go:42: 20:36:46 | major-upgrade-logical-replica/0-deploy-operator | +++ export GIT_BRANCH=PR-1727 logger.go:42: 20:36:46 | major-upgrade-logical-replica/0-deploy-operator | +++ GIT_BRANCH=PR-1727 logger.go:42: 20:36:46 | major-upgrade-logical-replica/0-deploy-operator | +++ export VERSION=PR-1727-5c0d1c600 logger.go:42: 20:36:46 | major-upgrade-logical-replica/0-deploy-operator | +++ VERSION=PR-1727-5c0d1c600 logger.go:42: 20:36:46 | major-upgrade-logical-replica/0-deploy-operator | +++ export SKIP_TEST_WARNINGS=true logger.go:42: 20:36:46 | major-upgrade-logical-replica/0-deploy-operator | +++ SKIP_TEST_WARNINGS=true logger.go:42: 20:36:46 | major-upgrade-logical-replica/0-deploy-operator | +++ export K8S_UPGRADE_PLATFORM=gke logger.go:42: 20:36:46 | major-upgrade-logical-replica/0-deploy-operator | +++ K8S_UPGRADE_PLATFORM=gke logger.go:42: 20:36:46 | major-upgrade-logical-replica/0-deploy-operator | +++ export K8S_UPGRADE_REGION=us-central1-a logger.go:42: 20:36:46 | major-upgrade-logical-replica/0-deploy-operator | +++ K8S_UPGRADE_REGION=us-central1-a logger.go:42: 20:36:46 | major-upgrade-logical-replica/0-deploy-operator | +++ export K8S_UPGRADE_INITIAL_VERSION= logger.go:42: 20:36:46 | major-upgrade-logical-replica/0-deploy-operator | +++ K8S_UPGRADE_INITIAL_VERSION= logger.go:42: 20:36:46 | major-upgrade-logical-replica/0-deploy-operator | +++ export K8S_UPGRADE_FINAL_VERSION= logger.go:42: 20:36:46 | major-upgrade-logical-replica/0-deploy-operator | +++ K8S_UPGRADE_FINAL_VERSION= logger.go:42: 20:36:46 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ -z '' ]] logger.go:42: 20:36:46 | major-upgrade-logical-replica/0-deploy-operator | ++++ detect_k8s_platform logger.go:42: 20:36:46 | major-upgrade-logical-replica/0-deploy-operator | ++++ local detected_platform logger.go:42: 20:36:46 | major-upgrade-logical-replica/0-deploy-operator | ++++ local xtrace_enabled=false logger.go:42: 20:36:46 | major-upgrade-logical-replica/0-deploy-operator | ++++ [[ -o xtrace ]] logger.go:42: 20:36:46 | major-upgrade-logical-replica/0-deploy-operator | ++++ xtrace_enabled=true logger.go:42: 20:36:46 | major-upgrade-logical-replica/0-deploy-operator | ++++ set +o xtrace logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ PLATFORM=gke logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ export PLATFORM logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE=perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE=perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ ! -n '' ]] logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ -n '' ]] logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ export PG_VER=18 logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ PG_VER=18 logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ export PG_DISTRIBUTION= logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ PG_DISTRIBUTION= logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ '' == community ]] logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ export BUCKET=pg-operator-testing logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ BUCKET=pg-operator-testing logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ export PMM_SERVER_VERSION=9.9.9 logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ PMM_SERVER_VERSION=9.9.9 logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE_LOGCOLLECTOR=perconalab/fluentbit:main-logcollector logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE_LOGCOLLECTOR=perconalab/fluentbit:main-logcollector logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ export PGOV1_TAG=1.4.0 logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ PGOV1_TAG=1.4.0 logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ export PGOV1_VER=14 logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ PGOV1_VER=14 logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ export CPGO_VERSION=5.8.7 logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ CPGO_VERSION=5.8.7 logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ export MINIO_VER=5.4.0 logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ MINIO_VER=5.4.0 logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ export VAULT_VER=0.32.0 logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ VAULT_VER=0.32.0 logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ export REGISTRY_NAME=docker.io logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ REGISTRY_NAME=docker.io logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ export REGISTRY_NAME_FULL=docker.io/ logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ REGISTRY_NAME_FULL=docker.io/ logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | ++++ printenv logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | ++++ grep -E '^IMAGE' logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | ++++ awk -F= '{print $1}' logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | ++++ eval 'echo $IMAGE_BACKREST' logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++++ echo perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ var_value=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == docker.io/* ]] logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == percona/* ]] logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == perconalab/* ]] logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ echo IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ echo perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | ++++ eval 'echo $IMAGE' logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++++ echo perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ var_value=perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 == docker.io/* ]] logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 == percona/* ]] logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 == perconalab/* ]] logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ new_value=docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ echo IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | ++++ eval 'echo $IMAGE_UPGRADE' logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++++ echo perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ var_value=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == docker.io/* ]] logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == percona/* ]] logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == perconalab/* ]] logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ echo IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | ++++ eval 'echo $IMAGE_LOGCOLLECTOR' logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++++ echo perconalab/fluentbit:main-logcollector logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ var_value=perconalab/fluentbit:main-logcollector logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/fluentbit:main-logcollector == docker.io/* ]] logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/fluentbit:main-logcollector == percona/* ]] logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/fluentbit:main-logcollector == perconalab/* ]] logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ new_value=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ echo IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | ++++ eval 'echo $IMAGE_BASE' logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++++ echo perconalab/percona-postgresql-operator logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ var_value=perconalab/percona-postgresql-operator logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator == docker.io/* ]] logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator == percona/* ]] logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator == perconalab/* ]] logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ new_value=docker.io/perconalab/percona-postgresql-operator logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ echo IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | ++++ eval 'echo $IMAGE_PGBOUNCER' logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++++ echo perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ var_value=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == docker.io/* ]] logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == percona/* ]] logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == perconalab/* ]] logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ echo IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | ++++ eval 'echo $IMAGE_POSTGRESQL' logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++++ echo perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ var_value=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == docker.io/* ]] logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == percona/* ]] logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == perconalab/* ]] logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ echo IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | ++++ eval 'echo $IMAGE_PMM_SERVER' logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++++ echo perconalab/pmm-server:3-dev-latest logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ var_value=perconalab/pmm-server:3-dev-latest logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/pmm-server:3-dev-latest == docker.io/* ]] logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/pmm-server:3-dev-latest == percona/* ]] logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/pmm-server:3-dev-latest == perconalab/* ]] logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ new_value=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ echo IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | ++++ eval 'echo $IMAGE_PMM_CLIENT' logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++++ echo perconalab/pmm-client:3-dev-latest logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ var_value=perconalab/pmm-client:3-dev-latest logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/pmm-client:3-dev-latest == docker.io/* ]] logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/pmm-client:3-dev-latest == percona/* ]] logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/pmm-client:3-dev-latest == perconalab/* ]] logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ new_value=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ echo IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | ++++ which gdate logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | which: no gdate in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1727/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | ++++ which date logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ date=/usr/sbin/date logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | ++++ which gsed logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | which: no gsed in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1727/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | ++++ which sed logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | +++ sed=/usr/sbin/sed logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | + init_temp_dir logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | + rm -rf /tmp/kuttl/pg/major-upgrade-logical-replica logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | + mkdir -p /tmp/kuttl/pg/major-upgrade-logical-replica logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | + deploy_operator logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | + local cw_prefix= logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | + destroy_operator logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | + kubectl -n pg-operator delete deployment percona-postgresql-operator --force --grace-period=0 logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | Error from server (NotFound): deployments.apps "percona-postgresql-operator" not found logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | + true logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | + [[ -n pg-operator ]] logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | + kubectl delete namespace pg-operator --force --grace-period=0 --ignore-not-found logger.go:42: 20:36:48 | major-upgrade-logical-replica/0-deploy-operator | Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. logger.go:42: 20:36:49 | major-upgrade-logical-replica/0-deploy-operator | + [[ -n pg-operator ]] logger.go:42: 20:36:49 | major-upgrade-logical-replica/0-deploy-operator | + create_namespace pg-operator logger.go:42: 20:36:49 | major-upgrade-logical-replica/0-deploy-operator | + local namespace=pg-operator logger.go:42: 20:36:49 | major-upgrade-logical-replica/0-deploy-operator | + [[ gke == openshift ]] logger.go:42: 20:36:49 | major-upgrade-logical-replica/0-deploy-operator | + kubectl delete namespace pg-operator --ignore-not-found logger.go:42: 20:36:50 | major-upgrade-logical-replica/0-deploy-operator | + kubectl wait --for=delete namespace pg-operator logger.go:42: 20:36:50 | major-upgrade-logical-replica/0-deploy-operator | + kubectl create namespace pg-operator logger.go:42: 20:36:51 | major-upgrade-logical-replica/0-deploy-operator | namespace/pg-operator created logger.go:42: 20:36:51 | major-upgrade-logical-replica/0-deploy-operator | + cw_prefix=cw- logger.go:42: 20:36:51 | major-upgrade-logical-replica/0-deploy-operator | + kubectl -n pg-operator apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pg-operator_PR-1727/deploy/crd.yaml logger.go:42: 20:36:52 | major-upgrade-logical-replica/0-deploy-operator | customresourcedefinition.apiextensions.k8s.io/crunchybridgeclusters.upstream.pgv2.percona.com serverside-applied logger.go:42: 20:36:52 | major-upgrade-logical-replica/0-deploy-operator | customresourcedefinition.apiextensions.k8s.io/perconapgbackups.pgv2.percona.com serverside-applied logger.go:42: 20:36:54 | major-upgrade-logical-replica/0-deploy-operator | customresourcedefinition.apiextensions.k8s.io/perconapgclusters.pgv2.percona.com serverside-applied logger.go:42: 20:36:55 | major-upgrade-logical-replica/0-deploy-operator | customresourcedefinition.apiextensions.k8s.io/perconapgrestores.pgv2.percona.com serverside-applied logger.go:42: 20:36:55 | major-upgrade-logical-replica/0-deploy-operator | customresourcedefinition.apiextensions.k8s.io/perconapgupgrades.pgv2.percona.com serverside-applied logger.go:42: 20:36:55 | major-upgrade-logical-replica/0-deploy-operator | customresourcedefinition.apiextensions.k8s.io/pgadmins.upstream.pgv2.percona.com serverside-applied logger.go:42: 20:36:56 | major-upgrade-logical-replica/0-deploy-operator | customresourcedefinition.apiextensions.k8s.io/pgupgrades.upstream.pgv2.percona.com serverside-applied logger.go:42: 20:36:57 | major-upgrade-logical-replica/0-deploy-operator | customresourcedefinition.apiextensions.k8s.io/postgresclusters.upstream.pgv2.percona.com serverside-applied logger.go:42: 20:36:57 | major-upgrade-logical-replica/0-deploy-operator | + kubectl -n pg-operator apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pg-operator_PR-1727/deploy/cw-rbac.yaml logger.go:42: 20:36:58 | major-upgrade-logical-replica/0-deploy-operator | serviceaccount/percona-postgresql-operator serverside-applied logger.go:42: 20:36:58 | major-upgrade-logical-replica/0-deploy-operator | clusterrole.rbac.authorization.k8s.io/percona-postgresql-operator serverside-applied logger.go:42: 20:36:59 | major-upgrade-logical-replica/0-deploy-operator | clusterrolebinding.rbac.authorization.k8s.io/percona-postgresql-operator serverside-applied logger.go:42: 20:36:59 | major-upgrade-logical-replica/0-deploy-operator | + local disable_telemetry=true logger.go:42: 20:36:59 | major-upgrade-logical-replica/0-deploy-operator | + '[' major-upgrade-logical-replica == telemetry-transfer ']' logger.go:42: 20:36:59 | major-upgrade-logical-replica/0-deploy-operator | + yq eval '.spec.template.spec.containers[0].image = "docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600"' /mnt/jenkins/workspace/cloud-pg-operator_PR-1727/deploy/cw-operator.yaml logger.go:42: 20:36:59 | major-upgrade-logical-replica/0-deploy-operator | + yq eval '(.spec.template.spec.containers[] | select(.name=="operator") | .env[] | select(.name=="DISABLE_TELEMETRY") | .value) = "true"' - logger.go:42: 20:36:59 | major-upgrade-logical-replica/0-deploy-operator | + yq eval '(.spec.template.spec.containers[] | select(.name=="operator") | .env[] | select(.name=="LOG_LEVEL") | .value) = "DEBUG"' - logger.go:42: 20:36:59 | major-upgrade-logical-replica/0-deploy-operator | + yq eval '(.spec.template.spec.containers[] | select(.name=="operator") | .env[] | select(.name=="PGO_FEATURE_GATES") | .value) = ""' - logger.go:42: 20:36:59 | major-upgrade-logical-replica/0-deploy-operator | + kubectl -n pg-operator apply -f - logger.go:42: 20:37:00 | major-upgrade-logical-replica/0-deploy-operator | deployment.apps/percona-postgresql-operator created logger.go:42: 20:37:00 | major-upgrade-logical-replica/0-deploy-operator | + deploy_client logger.go:42: 20:37:00 | major-upgrade-logical-replica/0-deploy-operator | + kubectl -n kuttl-test-intent-zebra apply -f /mnt/jenkins/workspace/cloud-pg-operator_PR-1727/e2e-tests/conf/client.yaml logger.go:42: 20:37:01 | major-upgrade-logical-replica/0-deploy-operator | deployment.apps/pg-client created logger.go:42: 20:37:01 | major-upgrade-logical-replica/0-deploy-operator | + deploy_s3_secrets logger.go:42: 20:37:01 | major-upgrade-logical-replica/0-deploy-operator | + set +o xtrace logger.go:42: 20:37:02 | major-upgrade-logical-replica/0-deploy-operator | secret/major-upgrade-logical-replica-pgbackrest-secrets created logger.go:42: 20:37:03 | major-upgrade-logical-replica/0-deploy-operator | secret/aws-s3-secret created logger.go:42: 20:37:04 | major-upgrade-logical-replica/0-deploy-operator | secret/do-spaces-secret created logger.go:42: 20:37:04 | major-upgrade-logical-replica/0-deploy-operator | secret/gcp-cs-secret created logger.go:42: 20:37:04 | major-upgrade-logical-replica/0-deploy-operator | secret/azure-secret created logger.go:42: 20:37:06 | major-upgrade-logical-replica/0-deploy-operator | secret/minio-secret created logger.go:42: 20:37:06 | major-upgrade-logical-replica/0-deploy-operator | running command: [sh -c kubectl assert exist-enhanced deployment percona-postgresql-operator -n ${OPERATOR_NS:-$NAMESPACE} --field-selector status.readyReplicas=1] logger.go:42: 20:37:06 | major-upgrade-logical-replica/0-deploy-operator | ASSERT deployment percona-postgresql-operator matching field criteria 'status.readyReplicas=1' should exist. logger.go:42: 20:37:07 | major-upgrade-logical-replica/0-deploy-operator | INFO Found 1 resource(s). logger.go:42: 20:37:07 | major-upgrade-logical-replica/0-deploy-operator | NAME NAMESPACE COL0 logger.go:42: 20:37:07 | major-upgrade-logical-replica/0-deploy-operator | percona-postgresql-operator pg-operator 1 logger.go:42: 20:37:07 | major-upgrade-logical-replica/0-deploy-operator | ASSERT PASS logger.go:42: 20:37:07 | major-upgrade-logical-replica/0-deploy-operator | test step completed 0-deploy-operator logger.go:42: 20:37:07 | major-upgrade-logical-replica/1-create-cluster | starting test step 1-create-cluster logger.go:42: 20:37:07 | major-upgrade-logical-replica/1-create-cluster | running command: [sh -c set -o errexit set -o xtrace source ../../functions postgres_image=$(get_container_image "postgres" 17) pgbouncer_image=$(get_container_image "pgbouncer" 17) pgbackrest_image=$(get_container_image "pgbackrest" 17) get_cr \ | yq eval " .spec.postgresVersion = 17 | .spec.image = \"${postgres_image}\" | .spec.proxy.pgBouncer.image = \"${pgbouncer_image}\" | .spec.backups.pgbackrest.image = \"${pgbackrest_image}\" | .spec.patroni.removeDataDirectoryOnDivergedTimelines = true" \ | kubectl -n "${NAMESPACE}" apply -f -] logger.go:42: 20:37:07 | major-upgrade-logical-replica/1-create-cluster | + source ../../functions logger.go:42: 20:37:07 | major-upgrade-logical-replica/1-create-cluster | +++ realpath ../../.. logger.go:42: 20:37:07 | major-upgrade-logical-replica/1-create-cluster | ++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1727 logger.go:42: 20:37:07 | major-upgrade-logical-replica/1-create-cluster | ++ CERT_MANAGER_VER=1.20.2 logger.go:42: 20:37:07 | major-upgrade-logical-replica/1-create-cluster | ++ CERT_MANAGER_TEST_LABEL=e2e.percona.com/managed-by=percona-postgresql-operator-tests logger.go:42: 20:37:07 | major-upgrade-logical-replica/1-create-cluster | ++ CHAOS_MESH_VER=2.5.1 logger.go:42: 20:37:07 | major-upgrade-logical-replica/1-create-cluster | ++ BUSYBOX_VER=1.36 logger.go:42: 20:37:07 | major-upgrade-logical-replica/1-create-cluster | ++++ pwd logger.go:42: 20:37:07 | major-upgrade-logical-replica/1-create-cluster | +++ basename /mnt/jenkins/workspace/cloud-pg-operator_PR-1727/e2e-tests/tests/major-upgrade-logical-replica logger.go:42: 20:37:07 | major-upgrade-logical-replica/1-create-cluster | ++ test_name=major-upgrade-logical-replica logger.go:42: 20:37:07 | major-upgrade-logical-replica/1-create-cluster | ++ source /mnt/jenkins/workspace/cloud-pg-operator_PR-1727/e2e-tests/vars.sh logger.go:42: 20:37:07 | major-upgrade-logical-replica/1-create-cluster | +++ export ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1727 logger.go:42: 20:37:07 | major-upgrade-logical-replica/1-create-cluster | +++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1727 logger.go:42: 20:37:07 | major-upgrade-logical-replica/1-create-cluster | +++ export DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1727/deploy logger.go:42: 20:37:07 | major-upgrade-logical-replica/1-create-cluster | +++ DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1727/deploy logger.go:42: 20:37:07 | major-upgrade-logical-replica/1-create-cluster | +++ export TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1727/e2e-tests logger.go:42: 20:37:07 | major-upgrade-logical-replica/1-create-cluster | +++ TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1727/e2e-tests logger.go:42: 20:37:07 | major-upgrade-logical-replica/1-create-cluster | +++ export TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1727/e2e-tests/conf logger.go:42: 20:37:07 | major-upgrade-logical-replica/1-create-cluster | +++ TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1727/e2e-tests/conf logger.go:42: 20:37:07 | major-upgrade-logical-replica/1-create-cluster | +++ export TEST_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1727/e2e-tests/tests/major-upgrade-logical-replica/conf logger.go:42: 20:37:07 | major-upgrade-logical-replica/1-create-cluster | +++ TEST_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1727/e2e-tests/tests/major-upgrade-logical-replica/conf logger.go:42: 20:37:07 | major-upgrade-logical-replica/1-create-cluster | +++ export TEMP_DIR=/tmp/kuttl/pg/major-upgrade-logical-replica logger.go:42: 20:37:07 | major-upgrade-logical-replica/1-create-cluster | +++ TEMP_DIR=/tmp/kuttl/pg/major-upgrade-logical-replica logger.go:42: 20:37:07 | major-upgrade-logical-replica/1-create-cluster | ++++ git rev-parse --abbrev-ref HEAD logger.go:42: 20:37:07 | major-upgrade-logical-replica/1-create-cluster | +++ export GIT_BRANCH=PR-1727 logger.go:42: 20:37:07 | major-upgrade-logical-replica/1-create-cluster | +++ GIT_BRANCH=PR-1727 logger.go:42: 20:37:07 | major-upgrade-logical-replica/1-create-cluster | +++ export VERSION=PR-1727-5c0d1c600 logger.go:42: 20:37:07 | major-upgrade-logical-replica/1-create-cluster | +++ VERSION=PR-1727-5c0d1c600 logger.go:42: 20:37:07 | major-upgrade-logical-replica/1-create-cluster | +++ export SKIP_TEST_WARNINGS=true logger.go:42: 20:37:07 | major-upgrade-logical-replica/1-create-cluster | +++ SKIP_TEST_WARNINGS=true logger.go:42: 20:37:07 | major-upgrade-logical-replica/1-create-cluster | +++ export K8S_UPGRADE_PLATFORM=gke logger.go:42: 20:37:07 | major-upgrade-logical-replica/1-create-cluster | +++ K8S_UPGRADE_PLATFORM=gke logger.go:42: 20:37:07 | major-upgrade-logical-replica/1-create-cluster | +++ export K8S_UPGRADE_REGION=us-central1-a logger.go:42: 20:37:07 | major-upgrade-logical-replica/1-create-cluster | +++ K8S_UPGRADE_REGION=us-central1-a logger.go:42: 20:37:07 | major-upgrade-logical-replica/1-create-cluster | +++ export K8S_UPGRADE_INITIAL_VERSION= logger.go:42: 20:37:07 | major-upgrade-logical-replica/1-create-cluster | +++ K8S_UPGRADE_INITIAL_VERSION= logger.go:42: 20:37:07 | major-upgrade-logical-replica/1-create-cluster | +++ export K8S_UPGRADE_FINAL_VERSION= logger.go:42: 20:37:07 | major-upgrade-logical-replica/1-create-cluster | +++ K8S_UPGRADE_FINAL_VERSION= logger.go:42: 20:37:07 | major-upgrade-logical-replica/1-create-cluster | +++ [[ -z '' ]] logger.go:42: 20:37:07 | major-upgrade-logical-replica/1-create-cluster | ++++ detect_k8s_platform logger.go:42: 20:37:07 | major-upgrade-logical-replica/1-create-cluster | ++++ local detected_platform logger.go:42: 20:37:07 | major-upgrade-logical-replica/1-create-cluster | ++++ local xtrace_enabled=false logger.go:42: 20:37:07 | major-upgrade-logical-replica/1-create-cluster | ++++ [[ -o xtrace ]] logger.go:42: 20:37:07 | major-upgrade-logical-replica/1-create-cluster | ++++ xtrace_enabled=true logger.go:42: 20:37:07 | major-upgrade-logical-replica/1-create-cluster | ++++ set +o xtrace logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ PLATFORM=gke logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ export PLATFORM logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE=perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE=perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ [[ ! -n '' ]] logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ [[ -n '' ]] logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ export PG_VER=18 logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ PG_VER=18 logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ export PG_DISTRIBUTION= logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ PG_DISTRIBUTION= logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ [[ '' == community ]] logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ export BUCKET=pg-operator-testing logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ BUCKET=pg-operator-testing logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ export PMM_SERVER_VERSION=9.9.9 logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ PMM_SERVER_VERSION=9.9.9 logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE_LOGCOLLECTOR=perconalab/fluentbit:main-logcollector logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE_LOGCOLLECTOR=perconalab/fluentbit:main-logcollector logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ export PGOV1_TAG=1.4.0 logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ PGOV1_TAG=1.4.0 logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ export PGOV1_VER=14 logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ PGOV1_VER=14 logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ export CPGO_VERSION=5.8.7 logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ CPGO_VERSION=5.8.7 logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ export MINIO_VER=5.4.0 logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ MINIO_VER=5.4.0 logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ export VAULT_VER=0.32.0 logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ VAULT_VER=0.32.0 logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ export REGISTRY_NAME=docker.io logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ REGISTRY_NAME=docker.io logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ export REGISTRY_NAME_FULL=docker.io/ logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ REGISTRY_NAME_FULL=docker.io/ logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | ++++ printenv logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | ++++ grep -E '^IMAGE' logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | ++++ awk -F= '{print $1}' logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | ++++ eval 'echo $IMAGE_BACKREST' logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++++ echo perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ var_value=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == docker.io/* ]] logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == percona/* ]] logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == perconalab/* ]] logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ echo IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ echo perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | ++++ eval 'echo $IMAGE' logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++++ echo perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ var_value=perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 == docker.io/* ]] logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 == percona/* ]] logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 == perconalab/* ]] logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ new_value=docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ echo IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | ++++ eval 'echo $IMAGE_UPGRADE' logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++++ echo perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ var_value=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == docker.io/* ]] logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == percona/* ]] logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == perconalab/* ]] logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ echo IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | ++++ eval 'echo $IMAGE_LOGCOLLECTOR' logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++++ echo perconalab/fluentbit:main-logcollector logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ var_value=perconalab/fluentbit:main-logcollector logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/fluentbit:main-logcollector == docker.io/* ]] logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/fluentbit:main-logcollector == percona/* ]] logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/fluentbit:main-logcollector == perconalab/* ]] logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ new_value=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ echo IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | ++++ eval 'echo $IMAGE_BASE' logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++++ echo perconalab/percona-postgresql-operator logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ var_value=perconalab/percona-postgresql-operator logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator == docker.io/* ]] logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator == percona/* ]] logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator == perconalab/* ]] logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ new_value=docker.io/perconalab/percona-postgresql-operator logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ echo IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | ++++ eval 'echo $IMAGE_PGBOUNCER' logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++++ echo perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ var_value=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == docker.io/* ]] logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == percona/* ]] logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == perconalab/* ]] logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ echo IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | ++++ eval 'echo $IMAGE_POSTGRESQL' logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++++ echo perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ var_value=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == docker.io/* ]] logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == percona/* ]] logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == perconalab/* ]] logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ echo IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | ++++ eval 'echo $IMAGE_PMM_SERVER' logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++++ echo perconalab/pmm-server:3-dev-latest logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ var_value=perconalab/pmm-server:3-dev-latest logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/pmm-server:3-dev-latest == docker.io/* ]] logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/pmm-server:3-dev-latest == percona/* ]] logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/pmm-server:3-dev-latest == perconalab/* ]] logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ new_value=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ echo IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | ++++ eval 'echo $IMAGE_PMM_CLIENT' logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++++ echo perconalab/pmm-client:3-dev-latest logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ var_value=perconalab/pmm-client:3-dev-latest logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/pmm-client:3-dev-latest == docker.io/* ]] logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/pmm-client:3-dev-latest == percona/* ]] logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/pmm-client:3-dev-latest == perconalab/* ]] logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ new_value=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ echo IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | ++++ which gdate logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | which: no gdate in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1727/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | ++++ which date logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ date=/usr/sbin/date logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | ++++ which gsed logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | which: no gsed in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1727/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | ++++ which sed logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | +++ sed=/usr/sbin/sed logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | ++ get_container_image postgres 17 logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | ++ local component=postgres logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | ++ local pgVersion=17 logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | ++ [[ docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 == percona/* ]] logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | ++ [[ docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 == docker.io/percona/* ]] logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | ++ [[ -n '' ]] logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | ++ local operatorVersion=PR-1727-5c0d1c600 logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | ++ [[ ! PR-1727-5c0d1c600 =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | ++ operatorVersion=main logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | ++ [[ '' == community ]] logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | ++ [[ postgres == pgbouncer ]] logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | ++ [[ postgres == pgbackrest ]] logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | ++ [[ postgres == upgrade ]] logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | ++ [[ postgres == postgis ]] logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | ++ echo docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | + postgres_image=docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | ++ get_container_image pgbouncer 17 logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | ++ local component=pgbouncer logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | ++ local pgVersion=17 logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | ++ [[ docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 == percona/* ]] logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | ++ [[ docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 == docker.io/percona/* ]] logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | ++ [[ -n '' ]] logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | ++ local operatorVersion=PR-1727-5c0d1c600 logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | ++ [[ ! PR-1727-5c0d1c600 =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | ++ operatorVersion=main logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | ++ [[ '' == community ]] logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | ++ [[ pgbouncer == pgbouncer ]] logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | ++ echo docker.io/perconalab/percona-postgresql-operator:main-pgbouncer17 logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | + pgbouncer_image=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer17 logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | ++ get_container_image pgbackrest 17 logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | ++ local component=pgbackrest logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | ++ local pgVersion=17 logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | ++ [[ docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 == percona/* ]] logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | ++ [[ docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 == docker.io/percona/* ]] logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | ++ [[ -n '' ]] logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | ++ local operatorVersion=PR-1727-5c0d1c600 logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | ++ [[ ! PR-1727-5c0d1c600 =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | ++ operatorVersion=main logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | ++ [[ '' == community ]] logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | ++ [[ pgbackrest == pgbouncer ]] logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | ++ [[ pgbackrest == pgbackrest ]] logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | ++ echo docker.io/perconalab/percona-postgresql-operator:main-pgbackrest17 logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | + pgbackrest_image=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest17 logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | + get_cr logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | + local cr_name=major-upgrade-logical-replica logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | + local repo_path= logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | + local source_path= logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | + local platform=gke logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | + local cr_file=/mnt/jenkins/workspace/cloud-pg-operator_PR-1727/e2e-tests/tests/major-upgrade-logical-replica/conf/major-upgrade-logical-replica.yaml logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | + yq eval $'\n .spec.postgresVersion = 17 |\n .spec.image = "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" |\n .spec.proxy.pgBouncer.image = "docker.io/perconalab/percona-postgresql-operator:main-pgbouncer17" |\n .spec.backups.pgbackrest.image = "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest17" |\n .spec.patroni.removeDataDirectoryOnDivergedTimelines = true' logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | + local platform_cr_file=/mnt/jenkins/workspace/cloud-pg-operator_PR-1727/e2e-tests/tests/major-upgrade-logical-replica/conf/major-upgrade-logical-replica-gke.yaml logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | + local spaces_cr_file=/mnt/jenkins/workspace/cloud-pg-operator_PR-1727/e2e-tests/tests/major-upgrade-logical-replica/conf/major-upgrade-logical-replica-spaces.yaml logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | + [[ -f /mnt/jenkins/workspace/cloud-pg-operator_PR-1727/e2e-tests/tests/major-upgrade-logical-replica/conf/major-upgrade-logical-replica-gke.yaml ]] logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | + [[ gke == doks ]] logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | + crs=('/mnt/jenkins/workspace/cloud-pg-operator_PR-1727/deploy/cr.yaml') logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | + kubectl -n kuttl-test-intent-zebra apply -f - logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | + local crs logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | + [[ -f /mnt/jenkins/workspace/cloud-pg-operator_PR-1727/e2e-tests/tests/major-upgrade-logical-replica/conf/major-upgrade-logical-replica.yaml ]] logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | + crs+=($cr_file) logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | + yq eval-all $'\n\t\tselect(fileIndex == 0) * (select(fileIndex == 1) // {}) |\n\t\t.metadata.name = "major-upgrade-logical-replica" |\n\t\t.metadata.labels = {"e2e":"major-upgrade-logical-replica"} |\n\t\t.spec.postgresVersion = 18 |\n\t\t.spec.users += [{"name":"postgres","password":{"type":"AlphaNumeric"}}] |\n\t\t.spec.users += [{"name":"major-upgrade-logical-replica","password":{"type":"AlphaNumeric"}}] |\n\t\t.spec.image = "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" |\n\t\t.spec.initContainer.image = "docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600" |\n\t\t.spec.backups.pgbackrest.image = "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18" |\n\t\t.spec.proxy.pgBouncer.image = "docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18" |\n\t\t.spec.logcollector.image = "docker.io/perconalab/fluentbit:main-logcollector" |\n\t\t.spec.pmm.image = "docker.io/perconalab/pmm-client:3-dev-latest" |\n\t\t.spec.pmm.secret = "major-upgrade-logical-replica-pmm-secret" |\n\t\t.spec.pmm.customClusterName = "major-upgrade-logical-replica-pmm-custom-name" |\n\t\t.spec.pmm.postgresParams = "--environment=dev-postgres"\n\t' /mnt/jenkins/workspace/cloud-pg-operator_PR-1727/deploy/cr.yaml /mnt/jenkins/workspace/cloud-pg-operator_PR-1727/e2e-tests/tests/major-upgrade-logical-replica/conf/major-upgrade-logical-replica.yaml logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | + /usr/sbin/sed -i 's|||g' /tmp/kuttl/pg/major-upgrade-logical-replica/cr.yaml logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | + /usr/sbin/sed -i 's|||g' /tmp/kuttl/pg/major-upgrade-logical-replica/cr.yaml logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | + /usr/sbin/sed -i 's||pg-operator-testing|g' /tmp/kuttl/pg/major-upgrade-logical-replica/cr.yaml logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | + /usr/sbin/sed -i 's||docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600|g' /tmp/kuttl/pg/major-upgrade-logical-replica/cr.yaml logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | + [[ gke == openshift ]] logger.go:42: 20:37:09 | major-upgrade-logical-replica/1-create-cluster | + cat /tmp/kuttl/pg/major-upgrade-logical-replica/cr.yaml logger.go:42: 20:37:11 | major-upgrade-logical-replica/1-create-cluster | perconapgcluster.pgv2.percona.com/major-upgrade-logical-replica created logger.go:42: 20:38:04 | major-upgrade-logical-replica/1-create-cluster | test step completed 1-create-cluster logger.go:42: 20:38:04 | major-upgrade-logical-replica/2-write-data | starting test step 2-write-data logger.go:42: 20:38:04 | major-upgrade-logical-replica/2-write-data | running command: [sh -c set -o errexit set -o xtrace source ../../functions PASS=$(get_psql_user_pass major-upgrade-logical-replica-pguser-postgres) HOST=$(get_psql_user_host major-upgrade-logical-replica-pguser-postgres) run_psql_local \ 'CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);' \ "postgres:${PASS}@${HOST}" run_psql_local \ '\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)' \ "postgres:${PASS}@${HOST}"] logger.go:42: 20:38:04 | major-upgrade-logical-replica/2-write-data | + source ../../functions logger.go:42: 20:38:04 | major-upgrade-logical-replica/2-write-data | +++ realpath ../../.. logger.go:42: 20:38:04 | major-upgrade-logical-replica/2-write-data | ++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1727 logger.go:42: 20:38:04 | major-upgrade-logical-replica/2-write-data | ++ CERT_MANAGER_VER=1.20.2 logger.go:42: 20:38:04 | major-upgrade-logical-replica/2-write-data | ++ CERT_MANAGER_TEST_LABEL=e2e.percona.com/managed-by=percona-postgresql-operator-tests logger.go:42: 20:38:04 | major-upgrade-logical-replica/2-write-data | ++ CHAOS_MESH_VER=2.5.1 logger.go:42: 20:38:04 | major-upgrade-logical-replica/2-write-data | ++ BUSYBOX_VER=1.36 logger.go:42: 20:38:04 | major-upgrade-logical-replica/2-write-data | ++++ pwd logger.go:42: 20:38:04 | major-upgrade-logical-replica/2-write-data | +++ basename /mnt/jenkins/workspace/cloud-pg-operator_PR-1727/e2e-tests/tests/major-upgrade-logical-replica logger.go:42: 20:38:04 | major-upgrade-logical-replica/2-write-data | ++ test_name=major-upgrade-logical-replica logger.go:42: 20:38:04 | major-upgrade-logical-replica/2-write-data | ++ source /mnt/jenkins/workspace/cloud-pg-operator_PR-1727/e2e-tests/vars.sh logger.go:42: 20:38:04 | major-upgrade-logical-replica/2-write-data | +++ export ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1727 logger.go:42: 20:38:04 | major-upgrade-logical-replica/2-write-data | +++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1727 logger.go:42: 20:38:04 | major-upgrade-logical-replica/2-write-data | +++ export DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1727/deploy logger.go:42: 20:38:04 | major-upgrade-logical-replica/2-write-data | +++ DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1727/deploy logger.go:42: 20:38:04 | major-upgrade-logical-replica/2-write-data | +++ export TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1727/e2e-tests logger.go:42: 20:38:04 | major-upgrade-logical-replica/2-write-data | +++ TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1727/e2e-tests logger.go:42: 20:38:04 | major-upgrade-logical-replica/2-write-data | +++ export TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1727/e2e-tests/conf logger.go:42: 20:38:04 | major-upgrade-logical-replica/2-write-data | +++ TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1727/e2e-tests/conf logger.go:42: 20:38:04 | major-upgrade-logical-replica/2-write-data | +++ export TEST_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1727/e2e-tests/tests/major-upgrade-logical-replica/conf logger.go:42: 20:38:04 | major-upgrade-logical-replica/2-write-data | +++ TEST_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1727/e2e-tests/tests/major-upgrade-logical-replica/conf logger.go:42: 20:38:04 | major-upgrade-logical-replica/2-write-data | +++ export TEMP_DIR=/tmp/kuttl/pg/major-upgrade-logical-replica logger.go:42: 20:38:04 | major-upgrade-logical-replica/2-write-data | +++ TEMP_DIR=/tmp/kuttl/pg/major-upgrade-logical-replica logger.go:42: 20:38:04 | major-upgrade-logical-replica/2-write-data | ++++ git rev-parse --abbrev-ref HEAD logger.go:42: 20:38:04 | major-upgrade-logical-replica/2-write-data | +++ export GIT_BRANCH=PR-1727 logger.go:42: 20:38:04 | major-upgrade-logical-replica/2-write-data | +++ GIT_BRANCH=PR-1727 logger.go:42: 20:38:04 | major-upgrade-logical-replica/2-write-data | +++ export VERSION=PR-1727-5c0d1c600 logger.go:42: 20:38:04 | major-upgrade-logical-replica/2-write-data | +++ VERSION=PR-1727-5c0d1c600 logger.go:42: 20:38:04 | major-upgrade-logical-replica/2-write-data | +++ export SKIP_TEST_WARNINGS=true logger.go:42: 20:38:04 | major-upgrade-logical-replica/2-write-data | +++ SKIP_TEST_WARNINGS=true logger.go:42: 20:38:04 | major-upgrade-logical-replica/2-write-data | +++ export K8S_UPGRADE_PLATFORM=gke logger.go:42: 20:38:04 | major-upgrade-logical-replica/2-write-data | +++ K8S_UPGRADE_PLATFORM=gke logger.go:42: 20:38:04 | major-upgrade-logical-replica/2-write-data | +++ export K8S_UPGRADE_REGION=us-central1-a logger.go:42: 20:38:04 | major-upgrade-logical-replica/2-write-data | +++ K8S_UPGRADE_REGION=us-central1-a logger.go:42: 20:38:04 | major-upgrade-logical-replica/2-write-data | +++ export K8S_UPGRADE_INITIAL_VERSION= logger.go:42: 20:38:04 | major-upgrade-logical-replica/2-write-data | +++ K8S_UPGRADE_INITIAL_VERSION= logger.go:42: 20:38:04 | major-upgrade-logical-replica/2-write-data | +++ export K8S_UPGRADE_FINAL_VERSION= logger.go:42: 20:38:04 | major-upgrade-logical-replica/2-write-data | +++ K8S_UPGRADE_FINAL_VERSION= logger.go:42: 20:38:04 | major-upgrade-logical-replica/2-write-data | +++ [[ -z '' ]] logger.go:42: 20:38:04 | major-upgrade-logical-replica/2-write-data | ++++ detect_k8s_platform logger.go:42: 20:38:04 | major-upgrade-logical-replica/2-write-data | ++++ local detected_platform logger.go:42: 20:38:04 | major-upgrade-logical-replica/2-write-data | ++++ local xtrace_enabled=false logger.go:42: 20:38:04 | major-upgrade-logical-replica/2-write-data | ++++ [[ -o xtrace ]] logger.go:42: 20:38:04 | major-upgrade-logical-replica/2-write-data | ++++ xtrace_enabled=true logger.go:42: 20:38:04 | major-upgrade-logical-replica/2-write-data | ++++ set +o xtrace logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ PLATFORM=gke logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ export PLATFORM logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE=perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ IMAGE=perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ [[ ! -n '' ]] logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ [[ -n '' ]] logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ export PG_VER=18 logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ PG_VER=18 logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ export PG_DISTRIBUTION= logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ PG_DISTRIBUTION= logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ [[ '' == community ]] logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ export BUCKET=pg-operator-testing logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ BUCKET=pg-operator-testing logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ export PMM_SERVER_VERSION=9.9.9 logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ PMM_SERVER_VERSION=9.9.9 logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE_LOGCOLLECTOR=perconalab/fluentbit:main-logcollector logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ IMAGE_LOGCOLLECTOR=perconalab/fluentbit:main-logcollector logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ export PGOV1_TAG=1.4.0 logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ PGOV1_TAG=1.4.0 logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ export PGOV1_VER=14 logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ PGOV1_VER=14 logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ export CPGO_VERSION=5.8.7 logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ CPGO_VERSION=5.8.7 logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ export MINIO_VER=5.4.0 logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ MINIO_VER=5.4.0 logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ export VAULT_VER=0.32.0 logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ VAULT_VER=0.32.0 logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ export REGISTRY_NAME=docker.io logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ REGISTRY_NAME=docker.io logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ export REGISTRY_NAME_FULL=docker.io/ logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ REGISTRY_NAME_FULL=docker.io/ logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | ++++ printenv logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | ++++ grep -E '^IMAGE' logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | ++++ awk -F= '{print $1}' logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | ++++ eval 'echo $IMAGE_BACKREST' logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++++ echo perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ var_value=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == docker.io/* ]] logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == percona/* ]] logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == perconalab/* ]] logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ echo IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ echo perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | ++++ eval 'echo $IMAGE' logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++++ echo perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ var_value=perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 == docker.io/* ]] logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 == percona/* ]] logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 == perconalab/* ]] logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ new_value=docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ echo IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | ++++ eval 'echo $IMAGE_UPGRADE' logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++++ echo perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ var_value=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == docker.io/* ]] logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == percona/* ]] logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == perconalab/* ]] logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ echo IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | ++++ eval 'echo $IMAGE_LOGCOLLECTOR' logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++++ echo perconalab/fluentbit:main-logcollector logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ var_value=perconalab/fluentbit:main-logcollector logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/fluentbit:main-logcollector == docker.io/* ]] logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/fluentbit:main-logcollector == percona/* ]] logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/fluentbit:main-logcollector == perconalab/* ]] logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ new_value=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ echo IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | ++++ eval 'echo $IMAGE_BASE' logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++++ echo perconalab/percona-postgresql-operator logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ var_value=perconalab/percona-postgresql-operator logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator == docker.io/* ]] logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator == percona/* ]] logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator == perconalab/* ]] logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ new_value=docker.io/perconalab/percona-postgresql-operator logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ echo IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | ++++ eval 'echo $IMAGE_PGBOUNCER' logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++++ echo perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ var_value=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == docker.io/* ]] logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == percona/* ]] logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == perconalab/* ]] logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ echo IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | ++++ eval 'echo $IMAGE_POSTGRESQL' logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++++ echo perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ var_value=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == docker.io/* ]] logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == percona/* ]] logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == perconalab/* ]] logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ echo IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | ++++ eval 'echo $IMAGE_PMM_SERVER' logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++++ echo perconalab/pmm-server:3-dev-latest logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ var_value=perconalab/pmm-server:3-dev-latest logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/pmm-server:3-dev-latest == docker.io/* ]] logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/pmm-server:3-dev-latest == percona/* ]] logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/pmm-server:3-dev-latest == perconalab/* ]] logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ new_value=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ echo IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | ++++ eval 'echo $IMAGE_PMM_CLIENT' logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++++ echo perconalab/pmm-client:3-dev-latest logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ var_value=perconalab/pmm-client:3-dev-latest logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/pmm-client:3-dev-latest == docker.io/* ]] logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/pmm-client:3-dev-latest == percona/* ]] logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/pmm-client:3-dev-latest == perconalab/* ]] logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ new_value=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ echo IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | ++++ which gdate logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | which: no gdate in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1727/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | ++++ which date logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ date=/usr/sbin/date logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | ++++ which gsed logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | which: no gsed in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1727/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | ++++ which sed logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | +++ sed=/usr/sbin/sed logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | ++ get_psql_user_pass major-upgrade-logical-replica-pguser-postgres logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | ++ local secret_name=major-upgrade-logical-replica-pguser-postgres logger.go:42: 20:38:06 | major-upgrade-logical-replica/2-write-data | ++ kubectl -n kuttl-test-intent-zebra get secret/major-upgrade-logical-replica-pguser-postgres '--template={{.data.password | base64decode}}' logger.go:42: 20:38:07 | major-upgrade-logical-replica/2-write-data | + PASS=2zjZOVoLkCxNalqBwszLWiyn logger.go:42: 20:38:07 | major-upgrade-logical-replica/2-write-data | ++ get_psql_user_host major-upgrade-logical-replica-pguser-postgres logger.go:42: 20:38:07 | major-upgrade-logical-replica/2-write-data | ++ local secret_name=major-upgrade-logical-replica-pguser-postgres logger.go:42: 20:38:07 | major-upgrade-logical-replica/2-write-data | ++ kubectl -n kuttl-test-intent-zebra get secret/major-upgrade-logical-replica-pguser-postgres '--template={{.data.host | base64decode }}' logger.go:42: 20:38:07 | major-upgrade-logical-replica/2-write-data | + HOST=major-upgrade-logical-replica-primary.kuttl-test-intent-zebra.svc logger.go:42: 20:38:07 | major-upgrade-logical-replica/2-write-data | + run_psql_local 'CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);' postgres:2zjZOVoLkCxNalqBwszLWiyn@major-upgrade-logical-replica-primary.kuttl-test-intent-zebra.svc logger.go:42: 20:38:07 | major-upgrade-logical-replica/2-write-data | + local 'command=CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);' logger.go:42: 20:38:07 | major-upgrade-logical-replica/2-write-data | + local uri=postgres:2zjZOVoLkCxNalqBwszLWiyn@major-upgrade-logical-replica-primary.kuttl-test-intent-zebra.svc logger.go:42: 20:38:07 | major-upgrade-logical-replica/2-write-data | + local driver=postgres logger.go:42: 20:38:07 | major-upgrade-logical-replica/2-write-data | ++ get_client_pod logger.go:42: 20:38:07 | major-upgrade-logical-replica/2-write-data | ++ kubectl -n kuttl-test-intent-zebra get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' logger.go:42: 20:38:08 | major-upgrade-logical-replica/2-write-data | + kubectl -n kuttl-test-intent-zebra exec pg-client-d7c7bb45b-8nwnb -- bash -c 'printf '\''CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://'\''postgres:2zjZOVoLkCxNalqBwszLWiyn@major-upgrade-logical-replica-primary.kuttl-test-intent-zebra.svc'\''' logger.go:42: 20:38:09 | major-upgrade-logical-replica/2-write-data | + run_psql_local '\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)' postgres:2zjZOVoLkCxNalqBwszLWiyn@major-upgrade-logical-replica-primary.kuttl-test-intent-zebra.svc logger.go:42: 20:38:09 | major-upgrade-logical-replica/2-write-data | + local 'command=\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)' logger.go:42: 20:38:09 | major-upgrade-logical-replica/2-write-data | + local uri=postgres:2zjZOVoLkCxNalqBwszLWiyn@major-upgrade-logical-replica-primary.kuttl-test-intent-zebra.svc logger.go:42: 20:38:09 | major-upgrade-logical-replica/2-write-data | + local driver=postgres logger.go:42: 20:38:09 | major-upgrade-logical-replica/2-write-data | ++ get_client_pod logger.go:42: 20:38:09 | major-upgrade-logical-replica/2-write-data | ++ kubectl -n kuttl-test-intent-zebra get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' logger.go:42: 20:38:10 | major-upgrade-logical-replica/2-write-data | + kubectl -n kuttl-test-intent-zebra exec pg-client-d7c7bb45b-8nwnb -- bash -c 'printf '\''\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://'\''postgres:2zjZOVoLkCxNalqBwszLWiyn@major-upgrade-logical-replica-primary.kuttl-test-intent-zebra.svc'\''' logger.go:42: 20:38:11 | major-upgrade-logical-replica/2-write-data | test step completed 2-write-data logger.go:42: 20:38:11 | major-upgrade-logical-replica/3-create-seed-backup | starting test step 3-create-seed-backup logger.go:42: 20:38:11 | major-upgrade-logical-replica/3-create-seed-backup | running command: [sh -c set -o errexit set -o xtrace source ../../functions # The pgbackrest bootstrap Job restores the latest full backup, and the # one taken at cluster creation predates the rows written above. Take a # new one so the seeded data is actually in the backup being restored. cat </dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 20:41:54 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 20:41:55 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 20:41:57 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 20:41:59 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 20:42:01 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 20:42:03 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 20:42:05 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 20:42:07 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 20:42:09 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 20:42:11 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 20:42:11 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 20:42:12 | major-upgrade-logical-replica/5-upgrade | Error from server (BadRequest): container "database" in pod "17-to-18-pgdata-8xjv4" is waiting to start: PodInitializing logger.go:42: 20:42:15 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 20:42:15 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 20:42:16 | major-upgrade-logical-replica/5-upgrade | Error from server (BadRequest): container "database" in pod "17-to-18-pgdata-8xjv4" is waiting to start: PodInitializing logger.go:42: 20:42:18 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 20:42:19 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 20:42:20 | major-upgrade-logical-replica/5-upgrade | Error from server (BadRequest): container "database" in pod "17-to-18-pgdata-8xjv4" is waiting to start: PodInitializing logger.go:42: 20:42:22 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 20:42:23 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 20:42:24 | major-upgrade-logical-replica/5-upgrade | Error from server (BadRequest): container "database" in pod "17-to-18-pgdata-8xjv4" is waiting to start: PodInitializing logger.go:42: 20:42:26 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 20:42:27 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 20:42:28 | major-upgrade-logical-replica/5-upgrade | Error from server (BadRequest): container "database" in pod "17-to-18-pgdata-8xjv4" is waiting to start: PodInitializing logger.go:42: 20:42:30 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 20:42:31 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 20:42:32 | major-upgrade-logical-replica/5-upgrade | Error from server (BadRequest): container "database" in pod "17-to-18-pgdata-8xjv4" is waiting to start: PodInitializing logger.go:42: 20:42:34 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 20:42:35 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 20:42:36 | major-upgrade-logical-replica/5-upgrade | Error from server (BadRequest): container "database" in pod "17-to-18-pgdata-8xjv4" is waiting to start: PodInitializing logger.go:42: 20:42:38 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 20:42:39 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 20:42:40 | major-upgrade-logical-replica/5-upgrade | Error from server (BadRequest): container "database" in pod "17-to-18-pgdata-8xjv4" is waiting to start: PodInitializing logger.go:42: 20:42:42 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 20:42:43 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 20:42:44 | major-upgrade-logical-replica/5-upgrade | Error from server (BadRequest): container "database" in pod "17-to-18-pgdata-8xjv4" is waiting to start: PodInitializing logger.go:42: 20:42:46 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 20:42:47 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 20:42:48 | major-upgrade-logical-replica/5-upgrade | Error from server (BadRequest): container "database" in pod "17-to-18-pgdata-8xjv4" is waiting to start: PodInitializing logger.go:42: 20:42:50 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 20:42:51 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 20:42:52 | major-upgrade-logical-replica/5-upgrade | Performing PostgreSQL upgrade from version "17" to "18" ... logger.go:42: 20:42:52 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:42:52 | major-upgrade-logical-replica/5-upgrade | Step 1: Making new pgdata directory... logger.go:42: 20:42:52 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:42:52 | major-upgrade-logical-replica/5-upgrade | Step 2: Initializing new pgdata directory... logger.go:42: 20:42:52 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:42:52 | major-upgrade-logical-replica/5-upgrade | The files belonging to this database system will be owned by user "postgres". logger.go:42: 20:42:52 | major-upgrade-logical-replica/5-upgrade | This user must also own the server process. logger.go:42: 20:42:52 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:42:52 | major-upgrade-logical-replica/5-upgrade | The database cluster will be initialized with locale "en_US.utf-8". logger.go:42: 20:42:52 | major-upgrade-logical-replica/5-upgrade | The default database encoding has accordingly been set to "UTF8". logger.go:42: 20:42:52 | major-upgrade-logical-replica/5-upgrade | The default text search configuration will be set to "english". logger.go:42: 20:42:52 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:42:52 | major-upgrade-logical-replica/5-upgrade | Data page checksums are enabled. logger.go:42: 20:42:52 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:42:52 | major-upgrade-logical-replica/5-upgrade | fixing permissions on existing directory /pgdata/pg18 ... ok logger.go:42: 20:42:52 | major-upgrade-logical-replica/5-upgrade | creating subdirectories ... ok logger.go:42: 20:42:52 | major-upgrade-logical-replica/5-upgrade | selecting dynamic shared memory implementation ... posix logger.go:42: 20:42:52 | major-upgrade-logical-replica/5-upgrade | selecting default "max_connections" ... 100 logger.go:42: 20:42:52 | major-upgrade-logical-replica/5-upgrade | selecting default "shared_buffers" ... 128MB logger.go:42: 20:42:52 | major-upgrade-logical-replica/5-upgrade | selecting default time zone ... UTC logger.go:42: 20:42:52 | major-upgrade-logical-replica/5-upgrade | creating configuration files ... ok logger.go:42: 20:42:52 | major-upgrade-logical-replica/5-upgrade | running bootstrap script ... ok logger.go:42: 20:42:52 | major-upgrade-logical-replica/5-upgrade | performing post-bootstrap initialization ... ok logger.go:42: 20:42:52 | major-upgrade-logical-replica/5-upgrade | syncing data to disk ... ok logger.go:42: 20:42:52 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:42:52 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:42:52 | major-upgrade-logical-replica/5-upgrade | Success. You can now start the database server using: logger.go:42: 20:42:52 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:42:52 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/pg_ctl -D /pgdata/pg18 -l logfile start logger.go:42: 20:42:52 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:42:52 | major-upgrade-logical-replica/5-upgrade | initdb: warning: enabling "trust" authentication for local connections logger.go:42: 20:42:52 | major-upgrade-logical-replica/5-upgrade | initdb: hint: You can change this by editing pg_hba.conf or using the option -A, or --auth-local and --auth-host, the next time you run initdb. logger.go:42: 20:42:52 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:42:52 | major-upgrade-logical-replica/5-upgrade | Step 3: Setting the expected permissions on the old pgdata directory... logger.go:42: 20:42:52 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:42:52 | major-upgrade-logical-replica/5-upgrade | Step 4: Copying shared_preload_libraries setting to new postgresql.conf file... logger.go:42: 20:42:52 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:42:52 | major-upgrade-logical-replica/5-upgrade | Step 4b: Copying logical replication settings to new postgresql.conf file... logger.go:42: 20:42:52 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:42:52 | major-upgrade-logical-replica/5-upgrade | Step 5: Running pg_upgrade check... logger.go:42: 20:42:52 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:42:52 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 20:42:52 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 20:42:52 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 20:42:52 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 20:42:52 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 20:42:52 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 20:42:52 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 20:42:52 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 20:42:52 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 20:42:52 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 20:42:52 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 20:42:52 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 20:42:54 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 20:42:55 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 20:42:56 | major-upgrade-logical-replica/5-upgrade | Performing PostgreSQL upgrade from version "17" to "18" ... logger.go:42: 20:42:56 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:42:56 | major-upgrade-logical-replica/5-upgrade | Step 1: Making new pgdata directory... logger.go:42: 20:42:56 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:42:56 | major-upgrade-logical-replica/5-upgrade | Step 2: Initializing new pgdata directory... logger.go:42: 20:42:56 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:42:56 | major-upgrade-logical-replica/5-upgrade | The files belonging to this database system will be owned by user "postgres". logger.go:42: 20:42:56 | major-upgrade-logical-replica/5-upgrade | This user must also own the server process. logger.go:42: 20:42:56 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:42:56 | major-upgrade-logical-replica/5-upgrade | The database cluster will be initialized with locale "en_US.utf-8". logger.go:42: 20:42:56 | major-upgrade-logical-replica/5-upgrade | The default database encoding has accordingly been set to "UTF8". logger.go:42: 20:42:56 | major-upgrade-logical-replica/5-upgrade | The default text search configuration will be set to "english". logger.go:42: 20:42:56 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:42:56 | major-upgrade-logical-replica/5-upgrade | Data page checksums are enabled. logger.go:42: 20:42:56 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:42:56 | major-upgrade-logical-replica/5-upgrade | fixing permissions on existing directory /pgdata/pg18 ... ok logger.go:42: 20:42:56 | major-upgrade-logical-replica/5-upgrade | creating subdirectories ... ok logger.go:42: 20:42:56 | major-upgrade-logical-replica/5-upgrade | selecting dynamic shared memory implementation ... posix logger.go:42: 20:42:56 | major-upgrade-logical-replica/5-upgrade | selecting default "max_connections" ... 100 logger.go:42: 20:42:56 | major-upgrade-logical-replica/5-upgrade | selecting default "shared_buffers" ... 128MB logger.go:42: 20:42:56 | major-upgrade-logical-replica/5-upgrade | selecting default time zone ... UTC logger.go:42: 20:42:56 | major-upgrade-logical-replica/5-upgrade | creating configuration files ... ok logger.go:42: 20:42:56 | major-upgrade-logical-replica/5-upgrade | running bootstrap script ... ok logger.go:42: 20:42:56 | major-upgrade-logical-replica/5-upgrade | performing post-bootstrap initialization ... ok logger.go:42: 20:42:56 | major-upgrade-logical-replica/5-upgrade | syncing data to disk ... ok logger.go:42: 20:42:56 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:42:56 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:42:56 | major-upgrade-logical-replica/5-upgrade | Success. You can now start the database server using: logger.go:42: 20:42:56 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:42:56 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/pg_ctl -D /pgdata/pg18 -l logfile start logger.go:42: 20:42:56 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:42:56 | major-upgrade-logical-replica/5-upgrade | initdb: warning: enabling "trust" authentication for local connections logger.go:42: 20:42:56 | major-upgrade-logical-replica/5-upgrade | initdb: hint: You can change this by editing pg_hba.conf or using the option -A, or --auth-local and --auth-host, the next time you run initdb. logger.go:42: 20:42:56 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:42:56 | major-upgrade-logical-replica/5-upgrade | Step 3: Setting the expected permissions on the old pgdata directory... logger.go:42: 20:42:56 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:42:56 | major-upgrade-logical-replica/5-upgrade | Step 4: Copying shared_preload_libraries setting to new postgresql.conf file... logger.go:42: 20:42:56 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:42:56 | major-upgrade-logical-replica/5-upgrade | Step 4b: Copying logical replication settings to new postgresql.conf file... logger.go:42: 20:42:56 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:42:56 | major-upgrade-logical-replica/5-upgrade | Step 5: Running pg_upgrade check... logger.go:42: 20:42:56 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:42:56 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 20:42:56 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 20:42:56 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 20:42:56 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 20:42:56 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 20:42:56 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 20:42:56 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 20:42:56 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 20:42:56 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 20:42:56 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 20:42:56 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 20:42:56 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 20:42:56 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 20:42:56 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 20:42:56 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 20:42:56 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 20:42:56 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 20:42:56 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:42:56 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 20:42:56 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:42:56 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 20:42:56 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:42:56 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:42:56 | major-upgrade-logical-replica/5-upgrade | real 0m1.877s logger.go:42: 20:42:56 | major-upgrade-logical-replica/5-upgrade | user 0m0.078s logger.go:42: 20:42:56 | major-upgrade-logical-replica/5-upgrade | sys 0m0.111s logger.go:42: 20:42:56 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 20:42:56 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 20:42:56 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 20:42:56 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 20:42:56 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 20:42:56 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 20:42:56 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 20:42:56 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 20:42:56 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 20:42:56 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 20:42:56 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 20:42:56 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 20:42:56 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 20:42:56 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 20:42:56 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 20:42:56 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 20:42:56 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 20:42:56 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 20:42:56 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 20:42:56 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:42:56 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 20:42:56 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 20:42:56 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:42:56 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 20:42:56 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 20:42:56 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 20:42:56 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 20:42:56 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 20:42:56 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 20:42:56 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 20:42:56 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 20:42:58 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 20:42:59 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 20:42:59 | major-upgrade-logical-replica/5-upgrade | Step 2: Initializing new pgdata directory... logger.go:42: 20:42:59 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:42:59 | major-upgrade-logical-replica/5-upgrade | The files belonging to this database system will be owned by user "postgres". logger.go:42: 20:42:59 | major-upgrade-logical-replica/5-upgrade | This user must also own the server process. logger.go:42: 20:42:59 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:42:59 | major-upgrade-logical-replica/5-upgrade | The database cluster will be initialized with locale "en_US.utf-8". logger.go:42: 20:42:59 | major-upgrade-logical-replica/5-upgrade | The default database encoding has accordingly been set to "UTF8". logger.go:42: 20:42:59 | major-upgrade-logical-replica/5-upgrade | The default text search configuration will be set to "english". logger.go:42: 20:42:59 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:42:59 | major-upgrade-logical-replica/5-upgrade | Data page checksums are enabled. logger.go:42: 20:42:59 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:42:59 | major-upgrade-logical-replica/5-upgrade | fixing permissions on existing directory /pgdata/pg18 ... ok logger.go:42: 20:42:59 | major-upgrade-logical-replica/5-upgrade | creating subdirectories ... ok logger.go:42: 20:42:59 | major-upgrade-logical-replica/5-upgrade | selecting dynamic shared memory implementation ... posix logger.go:42: 20:42:59 | major-upgrade-logical-replica/5-upgrade | selecting default "max_connections" ... 100 logger.go:42: 20:42:59 | major-upgrade-logical-replica/5-upgrade | selecting default "shared_buffers" ... 128MB logger.go:42: 20:42:59 | major-upgrade-logical-replica/5-upgrade | selecting default time zone ... UTC logger.go:42: 20:42:59 | major-upgrade-logical-replica/5-upgrade | creating configuration files ... ok logger.go:42: 20:42:59 | major-upgrade-logical-replica/5-upgrade | running bootstrap script ... ok logger.go:42: 20:42:59 | major-upgrade-logical-replica/5-upgrade | performing post-bootstrap initialization ... ok logger.go:42: 20:42:59 | major-upgrade-logical-replica/5-upgrade | syncing data to disk ... ok logger.go:42: 20:42:59 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:42:59 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:42:59 | major-upgrade-logical-replica/5-upgrade | Success. You can now start the database server using: logger.go:42: 20:42:59 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:42:59 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/pg_ctl -D /pgdata/pg18 -l logfile start logger.go:42: 20:42:59 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:42:59 | major-upgrade-logical-replica/5-upgrade | initdb: warning: enabling "trust" authentication for local connections logger.go:42: 20:42:59 | major-upgrade-logical-replica/5-upgrade | initdb: hint: You can change this by editing pg_hba.conf or using the option -A, or --auth-local and --auth-host, the next time you run initdb. logger.go:42: 20:42:59 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:42:59 | major-upgrade-logical-replica/5-upgrade | Step 3: Setting the expected permissions on the old pgdata directory... logger.go:42: 20:42:59 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:42:59 | major-upgrade-logical-replica/5-upgrade | Step 4: Copying shared_preload_libraries setting to new postgresql.conf file... logger.go:42: 20:42:59 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:42:59 | major-upgrade-logical-replica/5-upgrade | Step 4b: Copying logical replication settings to new postgresql.conf file... logger.go:42: 20:42:59 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:42:59 | major-upgrade-logical-replica/5-upgrade | Step 5: Running pg_upgrade check... logger.go:42: 20:42:59 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:42:59 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 20:42:59 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 20:42:59 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 20:42:59 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 20:42:59 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 20:42:59 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 20:42:59 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 20:42:59 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 20:42:59 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 20:42:59 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 20:42:59 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 20:42:59 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 20:42:59 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 20:42:59 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 20:42:59 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 20:42:59 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 20:42:59 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 20:42:59 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:42:59 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 20:42:59 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:42:59 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 20:42:59 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:42:59 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:42:59 | major-upgrade-logical-replica/5-upgrade | real 0m1.877s logger.go:42: 20:42:59 | major-upgrade-logical-replica/5-upgrade | user 0m0.078s logger.go:42: 20:42:59 | major-upgrade-logical-replica/5-upgrade | sys 0m0.111s logger.go:42: 20:42:59 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 20:42:59 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 20:42:59 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 20:42:59 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 20:42:59 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 20:42:59 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 20:42:59 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 20:42:59 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 20:42:59 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 20:42:59 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 20:42:59 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 20:42:59 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 20:42:59 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 20:42:59 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 20:42:59 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 20:42:59 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 20:42:59 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 20:42:59 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 20:42:59 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 20:42:59 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:42:59 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 20:42:59 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 20:42:59 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:42:59 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 20:42:59 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 20:42:59 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 20:42:59 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 20:42:59 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 20:42:59 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 20:42:59 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 20:42:59 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 20:42:59 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 20:42:59 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 20:42:59 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 20:42:59 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 20:42:59 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 20:43:02 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 20:43:02 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 20:43:03 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 20:43:03 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 20:43:03 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 20:43:03 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 20:43:03 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 20:43:03 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 20:43:03 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 20:43:03 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 20:43:03 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 20:43:03 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 20:43:03 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 20:43:03 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 20:43:03 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 20:43:03 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 20:43:03 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:03 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 20:43:03 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:03 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 20:43:03 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:03 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:03 | major-upgrade-logical-replica/5-upgrade | real 0m1.877s logger.go:42: 20:43:03 | major-upgrade-logical-replica/5-upgrade | user 0m0.078s logger.go:42: 20:43:03 | major-upgrade-logical-replica/5-upgrade | sys 0m0.111s logger.go:42: 20:43:03 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 20:43:03 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 20:43:03 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 20:43:03 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 20:43:03 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 20:43:03 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 20:43:03 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 20:43:03 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 20:43:03 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 20:43:03 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 20:43:03 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 20:43:03 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 20:43:03 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 20:43:03 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 20:43:03 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 20:43:03 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 20:43:03 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 20:43:03 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 20:43:03 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 20:43:03 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:03 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 20:43:03 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 20:43:03 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:03 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 20:43:03 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 20:43:03 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 20:43:03 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 20:43:03 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 20:43:03 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 20:43:03 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 20:43:03 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 20:43:03 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 20:43:03 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 20:43:03 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 20:43:03 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 20:43:03 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 20:43:03 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 20:43:03 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 20:43:03 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 20:43:03 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 20:43:03 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 20:43:03 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 20:43:03 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:03 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 20:43:03 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 20:43:03 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 20:43:03 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 20:43:03 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 20:43:03 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 20:43:03 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 20:43:03 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 20:43:03 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 20:43:03 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 20:43:03 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:03 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 20:43:03 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 20:43:03 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 20:43:03 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 20:43:03 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 20:43:03 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:03 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 20:43:03 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 20:43:03 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 20:43:03 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 20:43:03 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 20:43:03 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 20:43:03 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 20:43:03 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 20:43:03 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:03 | major-upgrade-logical-replica/5-upgrade | real 0m9.786s logger.go:42: 20:43:03 | major-upgrade-logical-replica/5-upgrade | user 0m0.258s logger.go:42: 20:43:03 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:03 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 20:43:03 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:03 | major-upgrade-logical-replica/5-upgrade | sys 0m0.395s logger.go:42: 20:43:03 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:03 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 20:43:06 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 20:43:06 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 20:43:08 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 20:43:08 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 20:43:08 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 20:43:08 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 20:43:08 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 20:43:08 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 20:43:08 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 20:43:08 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 20:43:08 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 20:43:08 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 20:43:08 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 20:43:08 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 20:43:08 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 20:43:08 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 20:43:08 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:08 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 20:43:08 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:08 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 20:43:08 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:08 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:08 | major-upgrade-logical-replica/5-upgrade | real 0m1.877s logger.go:42: 20:43:08 | major-upgrade-logical-replica/5-upgrade | user 0m0.078s logger.go:42: 20:43:08 | major-upgrade-logical-replica/5-upgrade | sys 0m0.111s logger.go:42: 20:43:08 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 20:43:08 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 20:43:08 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 20:43:08 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 20:43:08 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 20:43:08 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 20:43:08 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 20:43:08 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 20:43:08 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 20:43:08 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 20:43:08 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 20:43:08 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 20:43:08 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 20:43:08 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 20:43:08 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 20:43:08 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 20:43:08 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 20:43:08 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 20:43:08 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 20:43:08 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:08 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 20:43:08 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 20:43:08 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:08 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 20:43:08 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 20:43:08 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 20:43:08 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 20:43:08 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 20:43:08 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 20:43:08 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 20:43:08 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 20:43:08 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 20:43:08 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 20:43:08 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 20:43:08 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 20:43:08 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 20:43:08 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 20:43:08 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 20:43:08 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 20:43:08 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 20:43:08 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 20:43:08 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 20:43:08 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:08 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 20:43:08 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 20:43:08 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 20:43:08 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 20:43:08 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 20:43:08 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 20:43:08 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 20:43:08 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 20:43:08 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 20:43:08 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 20:43:08 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:08 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 20:43:08 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 20:43:08 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 20:43:08 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 20:43:08 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 20:43:08 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:08 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 20:43:08 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 20:43:08 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 20:43:08 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 20:43:08 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 20:43:08 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 20:43:08 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 20:43:08 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 20:43:08 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:08 | major-upgrade-logical-replica/5-upgrade | real 0m9.786s logger.go:42: 20:43:08 | major-upgrade-logical-replica/5-upgrade | user 0m0.258s logger.go:42: 20:43:08 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:08 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 20:43:08 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:08 | major-upgrade-logical-replica/5-upgrade | sys 0m0.395s logger.go:42: 20:43:08 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:08 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 20:43:10 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 20:43:11 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 20:43:12 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 20:43:12 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 20:43:12 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 20:43:12 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 20:43:12 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 20:43:12 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 20:43:12 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 20:43:12 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 20:43:12 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 20:43:12 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 20:43:12 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 20:43:12 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 20:43:12 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 20:43:12 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 20:43:12 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:12 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 20:43:12 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:12 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 20:43:12 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:12 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:12 | major-upgrade-logical-replica/5-upgrade | real 0m1.877s logger.go:42: 20:43:12 | major-upgrade-logical-replica/5-upgrade | user 0m0.078s logger.go:42: 20:43:12 | major-upgrade-logical-replica/5-upgrade | sys 0m0.111s logger.go:42: 20:43:12 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 20:43:12 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 20:43:12 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 20:43:12 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 20:43:12 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 20:43:12 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 20:43:12 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 20:43:12 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 20:43:12 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 20:43:12 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 20:43:12 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 20:43:12 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 20:43:12 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 20:43:12 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 20:43:12 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 20:43:12 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 20:43:12 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 20:43:12 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 20:43:12 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 20:43:12 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:12 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 20:43:12 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 20:43:12 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:12 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 20:43:12 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 20:43:12 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 20:43:12 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 20:43:12 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 20:43:12 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 20:43:12 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 20:43:12 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 20:43:12 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 20:43:12 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 20:43:12 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 20:43:12 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 20:43:12 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 20:43:12 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 20:43:12 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 20:43:12 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 20:43:12 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 20:43:12 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 20:43:12 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 20:43:12 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:12 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 20:43:12 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 20:43:12 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 20:43:12 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 20:43:12 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 20:43:12 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 20:43:12 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 20:43:12 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 20:43:12 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 20:43:12 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 20:43:12 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:12 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 20:43:12 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 20:43:12 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 20:43:12 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 20:43:12 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 20:43:12 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:12 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 20:43:12 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 20:43:12 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 20:43:12 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 20:43:12 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 20:43:12 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 20:43:12 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 20:43:12 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 20:43:12 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:12 | major-upgrade-logical-replica/5-upgrade | real 0m9.786s logger.go:42: 20:43:12 | major-upgrade-logical-replica/5-upgrade | user 0m0.258s logger.go:42: 20:43:12 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:12 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 20:43:12 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:12 | major-upgrade-logical-replica/5-upgrade | sys 0m0.395s logger.go:42: 20:43:12 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:12 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 20:43:14 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 20:43:15 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 20:43:16 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 20:43:16 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 20:43:16 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 20:43:16 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 20:43:16 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 20:43:16 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 20:43:16 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 20:43:16 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 20:43:16 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 20:43:16 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 20:43:16 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 20:43:16 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 20:43:16 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 20:43:16 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 20:43:16 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:16 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 20:43:16 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:16 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 20:43:16 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:16 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:16 | major-upgrade-logical-replica/5-upgrade | real 0m1.877s logger.go:42: 20:43:16 | major-upgrade-logical-replica/5-upgrade | user 0m0.078s logger.go:42: 20:43:16 | major-upgrade-logical-replica/5-upgrade | sys 0m0.111s logger.go:42: 20:43:16 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 20:43:16 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 20:43:16 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 20:43:16 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 20:43:16 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 20:43:16 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 20:43:16 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 20:43:16 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 20:43:16 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 20:43:16 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 20:43:16 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 20:43:16 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 20:43:16 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 20:43:16 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 20:43:16 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 20:43:16 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 20:43:16 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 20:43:16 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 20:43:16 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 20:43:16 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:16 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 20:43:16 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 20:43:16 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:16 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 20:43:16 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 20:43:16 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 20:43:16 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 20:43:16 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 20:43:16 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 20:43:16 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 20:43:16 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 20:43:16 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 20:43:16 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 20:43:16 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 20:43:16 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 20:43:16 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 20:43:16 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 20:43:16 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 20:43:16 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 20:43:16 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 20:43:16 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 20:43:16 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 20:43:16 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:16 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 20:43:16 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 20:43:16 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 20:43:16 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 20:43:16 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 20:43:16 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 20:43:16 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 20:43:16 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 20:43:16 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 20:43:16 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 20:43:16 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:16 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 20:43:16 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 20:43:16 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 20:43:16 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 20:43:16 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 20:43:16 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:16 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 20:43:16 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 20:43:16 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 20:43:16 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 20:43:16 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 20:43:16 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 20:43:16 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 20:43:16 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 20:43:16 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:16 | major-upgrade-logical-replica/5-upgrade | real 0m9.786s logger.go:42: 20:43:16 | major-upgrade-logical-replica/5-upgrade | user 0m0.258s logger.go:42: 20:43:16 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:16 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 20:43:16 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:16 | major-upgrade-logical-replica/5-upgrade | sys 0m0.395s logger.go:42: 20:43:16 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:16 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 20:43:18 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 20:43:19 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 20:43:20 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 20:43:20 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 20:43:20 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 20:43:20 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 20:43:20 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 20:43:20 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 20:43:20 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 20:43:20 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 20:43:20 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 20:43:20 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 20:43:20 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 20:43:20 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 20:43:20 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 20:43:20 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 20:43:20 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:20 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 20:43:20 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:20 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 20:43:20 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:20 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:20 | major-upgrade-logical-replica/5-upgrade | real 0m1.877s logger.go:42: 20:43:20 | major-upgrade-logical-replica/5-upgrade | user 0m0.078s logger.go:42: 20:43:20 | major-upgrade-logical-replica/5-upgrade | sys 0m0.111s logger.go:42: 20:43:20 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 20:43:20 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 20:43:20 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 20:43:20 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 20:43:20 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 20:43:20 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 20:43:20 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 20:43:20 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 20:43:20 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 20:43:20 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 20:43:20 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 20:43:20 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 20:43:20 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 20:43:20 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 20:43:20 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 20:43:20 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 20:43:20 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 20:43:20 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 20:43:20 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 20:43:20 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:20 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 20:43:20 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 20:43:20 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:20 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 20:43:20 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 20:43:20 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 20:43:20 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 20:43:20 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 20:43:20 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 20:43:20 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 20:43:20 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 20:43:20 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 20:43:20 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 20:43:20 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 20:43:20 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 20:43:20 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 20:43:20 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 20:43:20 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 20:43:20 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 20:43:20 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 20:43:20 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 20:43:20 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 20:43:20 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:20 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 20:43:20 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 20:43:20 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 20:43:20 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 20:43:20 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 20:43:20 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 20:43:20 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 20:43:20 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 20:43:20 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 20:43:20 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 20:43:20 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:20 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 20:43:20 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 20:43:20 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 20:43:20 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 20:43:20 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 20:43:20 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:20 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 20:43:20 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 20:43:20 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 20:43:20 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 20:43:20 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 20:43:20 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 20:43:20 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 20:43:20 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 20:43:20 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:20 | major-upgrade-logical-replica/5-upgrade | real 0m9.786s logger.go:42: 20:43:20 | major-upgrade-logical-replica/5-upgrade | user 0m0.258s logger.go:42: 20:43:20 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:20 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 20:43:20 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:20 | major-upgrade-logical-replica/5-upgrade | sys 0m0.395s logger.go:42: 20:43:20 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:20 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 20:43:22 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 20:43:23 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 20:43:24 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 20:43:24 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 20:43:24 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 20:43:24 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 20:43:24 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 20:43:24 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 20:43:24 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 20:43:24 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 20:43:24 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 20:43:24 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 20:43:24 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 20:43:24 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 20:43:24 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 20:43:24 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 20:43:24 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:24 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 20:43:24 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:24 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 20:43:24 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:24 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:24 | major-upgrade-logical-replica/5-upgrade | real 0m1.877s logger.go:42: 20:43:24 | major-upgrade-logical-replica/5-upgrade | user 0m0.078s logger.go:42: 20:43:24 | major-upgrade-logical-replica/5-upgrade | sys 0m0.111s logger.go:42: 20:43:24 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 20:43:24 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 20:43:24 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 20:43:24 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 20:43:24 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 20:43:24 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 20:43:24 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 20:43:24 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 20:43:24 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 20:43:24 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 20:43:24 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 20:43:24 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 20:43:24 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 20:43:24 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 20:43:24 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 20:43:24 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 20:43:24 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 20:43:24 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 20:43:24 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 20:43:24 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:24 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 20:43:24 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 20:43:24 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:24 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 20:43:24 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 20:43:24 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 20:43:24 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 20:43:24 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 20:43:24 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 20:43:24 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 20:43:24 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 20:43:24 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 20:43:24 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 20:43:24 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 20:43:24 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 20:43:24 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 20:43:24 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 20:43:24 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 20:43:24 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 20:43:24 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 20:43:24 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 20:43:24 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 20:43:24 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:24 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 20:43:24 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 20:43:24 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 20:43:24 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 20:43:24 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 20:43:24 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 20:43:24 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 20:43:24 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 20:43:24 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 20:43:24 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 20:43:24 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:24 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 20:43:24 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 20:43:24 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 20:43:24 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 20:43:24 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 20:43:24 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:24 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 20:43:24 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 20:43:24 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 20:43:24 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 20:43:24 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 20:43:24 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 20:43:24 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 20:43:24 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 20:43:24 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:24 | major-upgrade-logical-replica/5-upgrade | real 0m9.786s logger.go:42: 20:43:24 | major-upgrade-logical-replica/5-upgrade | user 0m0.258s logger.go:42: 20:43:24 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:24 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 20:43:24 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:24 | major-upgrade-logical-replica/5-upgrade | sys 0m0.395s logger.go:42: 20:43:24 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:24 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 20:43:26 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 20:43:27 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 20:43:28 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 20:43:28 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 20:43:28 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 20:43:28 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 20:43:28 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 20:43:28 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 20:43:28 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 20:43:28 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 20:43:28 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 20:43:28 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 20:43:28 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 20:43:28 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 20:43:28 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 20:43:28 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 20:43:28 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:28 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 20:43:28 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:28 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 20:43:28 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:28 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:28 | major-upgrade-logical-replica/5-upgrade | real 0m1.877s logger.go:42: 20:43:28 | major-upgrade-logical-replica/5-upgrade | user 0m0.078s logger.go:42: 20:43:28 | major-upgrade-logical-replica/5-upgrade | sys 0m0.111s logger.go:42: 20:43:28 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 20:43:28 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 20:43:28 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 20:43:28 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 20:43:28 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 20:43:28 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 20:43:28 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 20:43:28 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 20:43:28 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 20:43:28 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 20:43:28 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 20:43:28 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 20:43:28 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 20:43:28 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 20:43:28 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 20:43:28 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 20:43:28 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 20:43:28 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 20:43:28 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 20:43:28 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:28 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 20:43:28 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 20:43:28 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:28 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 20:43:28 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 20:43:28 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 20:43:28 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 20:43:28 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 20:43:28 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 20:43:28 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 20:43:28 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 20:43:28 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 20:43:28 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 20:43:28 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 20:43:28 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 20:43:28 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 20:43:28 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 20:43:28 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 20:43:28 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 20:43:28 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 20:43:28 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 20:43:28 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 20:43:28 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:28 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 20:43:28 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 20:43:28 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 20:43:28 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 20:43:28 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 20:43:28 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 20:43:28 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 20:43:28 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 20:43:28 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 20:43:28 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 20:43:28 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:28 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 20:43:28 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 20:43:28 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 20:43:28 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 20:43:28 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 20:43:28 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:28 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 20:43:28 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 20:43:28 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 20:43:28 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 20:43:28 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 20:43:28 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 20:43:28 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 20:43:28 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 20:43:28 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:28 | major-upgrade-logical-replica/5-upgrade | real 0m9.786s logger.go:42: 20:43:28 | major-upgrade-logical-replica/5-upgrade | user 0m0.258s logger.go:42: 20:43:28 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:28 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 20:43:28 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:28 | major-upgrade-logical-replica/5-upgrade | sys 0m0.395s logger.go:42: 20:43:28 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:28 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 20:43:30 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 20:43:30 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 20:43:31 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 20:43:31 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 20:43:31 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 20:43:31 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 20:43:31 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 20:43:31 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 20:43:31 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 20:43:31 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 20:43:31 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 20:43:31 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 20:43:31 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 20:43:31 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 20:43:31 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 20:43:31 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 20:43:31 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:31 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 20:43:31 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:31 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 20:43:31 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:31 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:31 | major-upgrade-logical-replica/5-upgrade | real 0m1.877s logger.go:42: 20:43:31 | major-upgrade-logical-replica/5-upgrade | user 0m0.078s logger.go:42: 20:43:31 | major-upgrade-logical-replica/5-upgrade | sys 0m0.111s logger.go:42: 20:43:31 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 20:43:31 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 20:43:31 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 20:43:31 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 20:43:31 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 20:43:31 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 20:43:31 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 20:43:31 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 20:43:31 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 20:43:31 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 20:43:31 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 20:43:31 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 20:43:31 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 20:43:31 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 20:43:31 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 20:43:31 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 20:43:31 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 20:43:31 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 20:43:31 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 20:43:31 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:31 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 20:43:31 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 20:43:31 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:31 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 20:43:31 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 20:43:31 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 20:43:31 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 20:43:31 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 20:43:31 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 20:43:31 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 20:43:31 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 20:43:31 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 20:43:31 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 20:43:31 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 20:43:31 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 20:43:31 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 20:43:31 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 20:43:31 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 20:43:31 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 20:43:31 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 20:43:31 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 20:43:31 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 20:43:31 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:31 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 20:43:31 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 20:43:31 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 20:43:31 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 20:43:31 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 20:43:31 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 20:43:31 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 20:43:31 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 20:43:31 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 20:43:31 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 20:43:31 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:31 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 20:43:31 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 20:43:31 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 20:43:31 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 20:43:31 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 20:43:31 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:31 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 20:43:31 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 20:43:31 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 20:43:31 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 20:43:31 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 20:43:31 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 20:43:31 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 20:43:31 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 20:43:31 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:31 | major-upgrade-logical-replica/5-upgrade | real 0m9.786s logger.go:42: 20:43:31 | major-upgrade-logical-replica/5-upgrade | user 0m0.258s logger.go:42: 20:43:31 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:31 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 20:43:31 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:31 | major-upgrade-logical-replica/5-upgrade | sys 0m0.395s logger.go:42: 20:43:31 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:31 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 20:43:34 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 20:43:35 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 20:43:36 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 20:43:36 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 20:43:36 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 20:43:36 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 20:43:36 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 20:43:36 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 20:43:36 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 20:43:36 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 20:43:36 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 20:43:36 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 20:43:36 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 20:43:36 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 20:43:36 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 20:43:36 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 20:43:36 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:36 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 20:43:36 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:36 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 20:43:36 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:36 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:36 | major-upgrade-logical-replica/5-upgrade | real 0m1.877s logger.go:42: 20:43:36 | major-upgrade-logical-replica/5-upgrade | user 0m0.078s logger.go:42: 20:43:36 | major-upgrade-logical-replica/5-upgrade | sys 0m0.111s logger.go:42: 20:43:36 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 20:43:36 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 20:43:36 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 20:43:36 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 20:43:36 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 20:43:36 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 20:43:36 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 20:43:36 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 20:43:36 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 20:43:36 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 20:43:36 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 20:43:36 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 20:43:36 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 20:43:36 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 20:43:36 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 20:43:36 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 20:43:36 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 20:43:36 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 20:43:36 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 20:43:36 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:36 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 20:43:36 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 20:43:36 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:36 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 20:43:36 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 20:43:36 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 20:43:36 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 20:43:36 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 20:43:36 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 20:43:36 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 20:43:36 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 20:43:36 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 20:43:36 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 20:43:36 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 20:43:36 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 20:43:36 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 20:43:36 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 20:43:36 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 20:43:36 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 20:43:36 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 20:43:36 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 20:43:36 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 20:43:36 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:36 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 20:43:36 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 20:43:36 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 20:43:36 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 20:43:36 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 20:43:36 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 20:43:36 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 20:43:36 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 20:43:36 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 20:43:36 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 20:43:36 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:36 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 20:43:36 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 20:43:36 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 20:43:36 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 20:43:36 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 20:43:36 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:36 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 20:43:36 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 20:43:36 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 20:43:36 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 20:43:36 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 20:43:36 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 20:43:36 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 20:43:36 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 20:43:36 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:36 | major-upgrade-logical-replica/5-upgrade | real 0m9.786s logger.go:42: 20:43:36 | major-upgrade-logical-replica/5-upgrade | user 0m0.258s logger.go:42: 20:43:36 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:36 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 20:43:36 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:36 | major-upgrade-logical-replica/5-upgrade | sys 0m0.395s logger.go:42: 20:43:36 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:36 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 20:43:39 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 20:43:40 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 20:43:41 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 20:43:41 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 20:43:41 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 20:43:41 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 20:43:41 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 20:43:41 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 20:43:41 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 20:43:41 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 20:43:41 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 20:43:41 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 20:43:41 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 20:43:41 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 20:43:41 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 20:43:41 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 20:43:41 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:41 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 20:43:41 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:41 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 20:43:41 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:41 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:41 | major-upgrade-logical-replica/5-upgrade | real 0m1.877s logger.go:42: 20:43:41 | major-upgrade-logical-replica/5-upgrade | user 0m0.078s logger.go:42: 20:43:41 | major-upgrade-logical-replica/5-upgrade | sys 0m0.111s logger.go:42: 20:43:41 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 20:43:41 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 20:43:41 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 20:43:41 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 20:43:41 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 20:43:41 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 20:43:41 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 20:43:41 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 20:43:41 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 20:43:41 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 20:43:41 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 20:43:41 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 20:43:41 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 20:43:41 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 20:43:41 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 20:43:41 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 20:43:41 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 20:43:41 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 20:43:41 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 20:43:41 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:41 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 20:43:41 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 20:43:41 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:41 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 20:43:41 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 20:43:41 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 20:43:41 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 20:43:41 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 20:43:41 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 20:43:41 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 20:43:41 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 20:43:41 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 20:43:41 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 20:43:41 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 20:43:41 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 20:43:41 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 20:43:41 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 20:43:41 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 20:43:41 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 20:43:41 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 20:43:41 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 20:43:41 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 20:43:41 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:41 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 20:43:41 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 20:43:41 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 20:43:41 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 20:43:41 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 20:43:41 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 20:43:41 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 20:43:41 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 20:43:41 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 20:43:41 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 20:43:41 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:41 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 20:43:41 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 20:43:41 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 20:43:41 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 20:43:41 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 20:43:41 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:41 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 20:43:41 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 20:43:41 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 20:43:41 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 20:43:41 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 20:43:41 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 20:43:41 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 20:43:41 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 20:43:41 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:41 | major-upgrade-logical-replica/5-upgrade | real 0m9.786s logger.go:42: 20:43:41 | major-upgrade-logical-replica/5-upgrade | user 0m0.258s logger.go:42: 20:43:41 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:41 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 20:43:41 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:41 | major-upgrade-logical-replica/5-upgrade | sys 0m0.395s logger.go:42: 20:43:41 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:41 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 20:43:43 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 20:43:44 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 20:43:45 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 20:43:45 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 20:43:45 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 20:43:45 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 20:43:45 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 20:43:45 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 20:43:45 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 20:43:45 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 20:43:45 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 20:43:45 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 20:43:45 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 20:43:45 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 20:43:45 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 20:43:45 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 20:43:45 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:45 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 20:43:45 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:45 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 20:43:45 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:45 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:45 | major-upgrade-logical-replica/5-upgrade | real 0m1.877s logger.go:42: 20:43:45 | major-upgrade-logical-replica/5-upgrade | user 0m0.078s logger.go:42: 20:43:45 | major-upgrade-logical-replica/5-upgrade | sys 0m0.111s logger.go:42: 20:43:45 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 20:43:45 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 20:43:45 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 20:43:45 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 20:43:45 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 20:43:45 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 20:43:45 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 20:43:45 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 20:43:45 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 20:43:45 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 20:43:45 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 20:43:45 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 20:43:45 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 20:43:45 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 20:43:45 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 20:43:45 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 20:43:45 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 20:43:45 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 20:43:45 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 20:43:45 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:45 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 20:43:45 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 20:43:45 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:45 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 20:43:45 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 20:43:45 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 20:43:45 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 20:43:45 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 20:43:45 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 20:43:45 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 20:43:45 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 20:43:45 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 20:43:45 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 20:43:45 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 20:43:45 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 20:43:45 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 20:43:45 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 20:43:45 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 20:43:45 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 20:43:45 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 20:43:45 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 20:43:45 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 20:43:45 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:45 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 20:43:45 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 20:43:45 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 20:43:45 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 20:43:45 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 20:43:45 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 20:43:45 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 20:43:45 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 20:43:45 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 20:43:45 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 20:43:45 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:45 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 20:43:45 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 20:43:45 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 20:43:45 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 20:43:45 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 20:43:45 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:45 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 20:43:45 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 20:43:45 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 20:43:45 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 20:43:45 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 20:43:45 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 20:43:45 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 20:43:45 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 20:43:45 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:45 | major-upgrade-logical-replica/5-upgrade | real 0m9.786s logger.go:42: 20:43:45 | major-upgrade-logical-replica/5-upgrade | user 0m0.258s logger.go:42: 20:43:45 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:45 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 20:43:45 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:45 | major-upgrade-logical-replica/5-upgrade | sys 0m0.395s logger.go:42: 20:43:45 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:45 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 20:43:47 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 20:43:48 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 20:43:49 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 20:43:49 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 20:43:49 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 20:43:49 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 20:43:49 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 20:43:49 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 20:43:49 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 20:43:49 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 20:43:49 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 20:43:49 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 20:43:49 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 20:43:49 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 20:43:49 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 20:43:49 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 20:43:49 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:49 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 20:43:49 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:49 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 20:43:49 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:49 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:49 | major-upgrade-logical-replica/5-upgrade | real 0m1.877s logger.go:42: 20:43:49 | major-upgrade-logical-replica/5-upgrade | user 0m0.078s logger.go:42: 20:43:49 | major-upgrade-logical-replica/5-upgrade | sys 0m0.111s logger.go:42: 20:43:49 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 20:43:49 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 20:43:49 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 20:43:49 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 20:43:49 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 20:43:49 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 20:43:49 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 20:43:49 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 20:43:49 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 20:43:49 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 20:43:49 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 20:43:49 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 20:43:49 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 20:43:49 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 20:43:49 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 20:43:49 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 20:43:49 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 20:43:49 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 20:43:49 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 20:43:49 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:49 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 20:43:49 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 20:43:49 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:49 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 20:43:49 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 20:43:49 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 20:43:49 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 20:43:49 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 20:43:49 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 20:43:49 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 20:43:49 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 20:43:49 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 20:43:49 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 20:43:49 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 20:43:49 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 20:43:49 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 20:43:49 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 20:43:49 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 20:43:49 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 20:43:49 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 20:43:49 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 20:43:49 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 20:43:49 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:49 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 20:43:49 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 20:43:49 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 20:43:49 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 20:43:49 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 20:43:49 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 20:43:49 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 20:43:49 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 20:43:49 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 20:43:49 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 20:43:49 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:49 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 20:43:49 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 20:43:49 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 20:43:49 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 20:43:49 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 20:43:49 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:49 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 20:43:49 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 20:43:49 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 20:43:49 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 20:43:49 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 20:43:49 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 20:43:49 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 20:43:49 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 20:43:49 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:49 | major-upgrade-logical-replica/5-upgrade | real 0m9.786s logger.go:42: 20:43:49 | major-upgrade-logical-replica/5-upgrade | user 0m0.258s logger.go:42: 20:43:49 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:49 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 20:43:49 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:49 | major-upgrade-logical-replica/5-upgrade | sys 0m0.395s logger.go:42: 20:43:49 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:49 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 20:43:51 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 20:43:52 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 20:43:53 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 20:43:53 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 20:43:53 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 20:43:53 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 20:43:53 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 20:43:53 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 20:43:53 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 20:43:53 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 20:43:53 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 20:43:53 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 20:43:53 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 20:43:53 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 20:43:53 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 20:43:53 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 20:43:53 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:53 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 20:43:53 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:53 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 20:43:53 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:53 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:53 | major-upgrade-logical-replica/5-upgrade | real 0m1.877s logger.go:42: 20:43:53 | major-upgrade-logical-replica/5-upgrade | user 0m0.078s logger.go:42: 20:43:53 | major-upgrade-logical-replica/5-upgrade | sys 0m0.111s logger.go:42: 20:43:53 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 20:43:53 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 20:43:53 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 20:43:53 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 20:43:53 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 20:43:53 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 20:43:53 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 20:43:53 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 20:43:53 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 20:43:53 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 20:43:53 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 20:43:53 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 20:43:53 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 20:43:53 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 20:43:53 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 20:43:53 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 20:43:53 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 20:43:53 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 20:43:53 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 20:43:53 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:53 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 20:43:53 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 20:43:53 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:53 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 20:43:53 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 20:43:53 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 20:43:53 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 20:43:53 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 20:43:53 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 20:43:53 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 20:43:53 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 20:43:53 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 20:43:53 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 20:43:53 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 20:43:53 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 20:43:53 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 20:43:53 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 20:43:53 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 20:43:53 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 20:43:53 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 20:43:53 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 20:43:53 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 20:43:53 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:53 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 20:43:53 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 20:43:53 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 20:43:53 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 20:43:53 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 20:43:53 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 20:43:53 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 20:43:53 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 20:43:53 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 20:43:53 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 20:43:53 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:53 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 20:43:53 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 20:43:53 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 20:43:53 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 20:43:53 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 20:43:53 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:53 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 20:43:53 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 20:43:53 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 20:43:53 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 20:43:53 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 20:43:53 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 20:43:53 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 20:43:53 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 20:43:53 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:53 | major-upgrade-logical-replica/5-upgrade | real 0m9.786s logger.go:42: 20:43:53 | major-upgrade-logical-replica/5-upgrade | user 0m0.258s logger.go:42: 20:43:53 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:53 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 20:43:53 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:53 | major-upgrade-logical-replica/5-upgrade | sys 0m0.395s logger.go:42: 20:43:53 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:53 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 20:43:56 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 20:43:56 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 20:43:57 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 20:43:57 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 20:43:57 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 20:43:57 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 20:43:57 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 20:43:57 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 20:43:57 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 20:43:57 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 20:43:57 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 20:43:57 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 20:43:57 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 20:43:57 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 20:43:57 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 20:43:57 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 20:43:57 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:57 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 20:43:57 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:57 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 20:43:57 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:57 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:57 | major-upgrade-logical-replica/5-upgrade | real 0m1.877s logger.go:42: 20:43:57 | major-upgrade-logical-replica/5-upgrade | user 0m0.078s logger.go:42: 20:43:57 | major-upgrade-logical-replica/5-upgrade | sys 0m0.111s logger.go:42: 20:43:57 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 20:43:57 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 20:43:57 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 20:43:57 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 20:43:57 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 20:43:57 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 20:43:57 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 20:43:57 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 20:43:57 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 20:43:57 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 20:43:57 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 20:43:57 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 20:43:57 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 20:43:57 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 20:43:57 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 20:43:57 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 20:43:57 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 20:43:57 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 20:43:57 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 20:43:57 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:57 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 20:43:57 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 20:43:57 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:57 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 20:43:57 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 20:43:57 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 20:43:57 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 20:43:57 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 20:43:57 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 20:43:57 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 20:43:57 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 20:43:57 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 20:43:57 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 20:43:57 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 20:43:57 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 20:43:57 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 20:43:57 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 20:43:57 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 20:43:57 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 20:43:57 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 20:43:57 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 20:43:57 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 20:43:57 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:57 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 20:43:57 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 20:43:57 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 20:43:57 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 20:43:57 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 20:43:57 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 20:43:57 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 20:43:57 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 20:43:57 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 20:43:57 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 20:43:57 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:57 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 20:43:57 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 20:43:57 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 20:43:57 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 20:43:57 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 20:43:57 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:57 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 20:43:57 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 20:43:57 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 20:43:57 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 20:43:57 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 20:43:57 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 20:43:57 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 20:43:57 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 20:43:57 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:57 | major-upgrade-logical-replica/5-upgrade | real 0m9.786s logger.go:42: 20:43:57 | major-upgrade-logical-replica/5-upgrade | user 0m0.258s logger.go:42: 20:43:57 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:57 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 20:43:57 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:57 | major-upgrade-logical-replica/5-upgrade | sys 0m0.395s logger.go:42: 20:43:57 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:43:57 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 20:44:00 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 20:44:00 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 20:44:01 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 20:44:01 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 20:44:01 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 20:44:01 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 20:44:01 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 20:44:01 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 20:44:01 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 20:44:01 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 20:44:01 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 20:44:01 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 20:44:01 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 20:44:01 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 20:44:01 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 20:44:01 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 20:44:01 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:44:01 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 20:44:01 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:44:01 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 20:44:01 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:44:01 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:44:01 | major-upgrade-logical-replica/5-upgrade | real 0m1.877s logger.go:42: 20:44:01 | major-upgrade-logical-replica/5-upgrade | user 0m0.078s logger.go:42: 20:44:01 | major-upgrade-logical-replica/5-upgrade | sys 0m0.111s logger.go:42: 20:44:01 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 20:44:01 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 20:44:01 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 20:44:01 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 20:44:01 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 20:44:01 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 20:44:01 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 20:44:01 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 20:44:01 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 20:44:01 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 20:44:01 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 20:44:01 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 20:44:01 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 20:44:01 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 20:44:01 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 20:44:01 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 20:44:01 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 20:44:01 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 20:44:01 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 20:44:01 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:44:01 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 20:44:01 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 20:44:01 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:44:01 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 20:44:01 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 20:44:01 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 20:44:01 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 20:44:01 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 20:44:01 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 20:44:01 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 20:44:01 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 20:44:01 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 20:44:01 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 20:44:01 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 20:44:01 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 20:44:01 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 20:44:01 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 20:44:01 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 20:44:01 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 20:44:01 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 20:44:01 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 20:44:01 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 20:44:01 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:44:01 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 20:44:01 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 20:44:01 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 20:44:01 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 20:44:01 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 20:44:01 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 20:44:01 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 20:44:01 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 20:44:01 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 20:44:01 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 20:44:01 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:44:01 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 20:44:01 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 20:44:01 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 20:44:01 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 20:44:01 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 20:44:01 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:44:01 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 20:44:01 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 20:44:01 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 20:44:01 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 20:44:01 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 20:44:01 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 20:44:01 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 20:44:01 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 20:44:01 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:44:01 | major-upgrade-logical-replica/5-upgrade | real 0m9.786s logger.go:42: 20:44:01 | major-upgrade-logical-replica/5-upgrade | user 0m0.258s logger.go:42: 20:44:01 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:44:01 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 20:44:01 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:44:01 | major-upgrade-logical-replica/5-upgrade | sys 0m0.395s logger.go:42: 20:44:01 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:44:01 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 20:44:04 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 20:44:04 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 20:44:05 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 20:44:05 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 20:44:05 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 20:44:05 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 20:44:05 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 20:44:05 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 20:44:05 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 20:44:05 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 20:44:05 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 20:44:05 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 20:44:05 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 20:44:05 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 20:44:05 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 20:44:05 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 20:44:05 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:44:05 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 20:44:05 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:44:05 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 20:44:05 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:44:05 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:44:05 | major-upgrade-logical-replica/5-upgrade | real 0m1.877s logger.go:42: 20:44:05 | major-upgrade-logical-replica/5-upgrade | user 0m0.078s logger.go:42: 20:44:05 | major-upgrade-logical-replica/5-upgrade | sys 0m0.111s logger.go:42: 20:44:05 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 20:44:05 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 20:44:05 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 20:44:05 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 20:44:05 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 20:44:05 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 20:44:05 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 20:44:05 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 20:44:05 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 20:44:05 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 20:44:05 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 20:44:05 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 20:44:05 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 20:44:05 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 20:44:05 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 20:44:05 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 20:44:05 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 20:44:05 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 20:44:05 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 20:44:05 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:44:05 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 20:44:05 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 20:44:05 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:44:05 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 20:44:05 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 20:44:05 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 20:44:05 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 20:44:05 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 20:44:05 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 20:44:05 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 20:44:05 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 20:44:05 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 20:44:05 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 20:44:05 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 20:44:05 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 20:44:05 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 20:44:05 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 20:44:05 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 20:44:05 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 20:44:05 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 20:44:05 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 20:44:05 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 20:44:05 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:44:05 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 20:44:05 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 20:44:05 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 20:44:05 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 20:44:05 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 20:44:05 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 20:44:05 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 20:44:05 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 20:44:05 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 20:44:05 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 20:44:05 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:44:05 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 20:44:05 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 20:44:05 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 20:44:05 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 20:44:05 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 20:44:05 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:44:05 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 20:44:05 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 20:44:05 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 20:44:05 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 20:44:05 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 20:44:05 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 20:44:05 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 20:44:05 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 20:44:05 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:44:05 | major-upgrade-logical-replica/5-upgrade | real 0m9.786s logger.go:42: 20:44:05 | major-upgrade-logical-replica/5-upgrade | user 0m0.258s logger.go:42: 20:44:05 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:44:05 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 20:44:05 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:44:05 | major-upgrade-logical-replica/5-upgrade | sys 0m0.395s logger.go:42: 20:44:05 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:44:05 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 20:44:08 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 20:44:08 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 20:44:09 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 20:44:09 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 20:44:09 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 20:44:09 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 20:44:09 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 20:44:09 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 20:44:09 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 20:44:09 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 20:44:09 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 20:44:09 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 20:44:09 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 20:44:09 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 20:44:09 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 20:44:09 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 20:44:09 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:44:09 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 20:44:09 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:44:09 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 20:44:09 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:44:09 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:44:09 | major-upgrade-logical-replica/5-upgrade | real 0m1.877s logger.go:42: 20:44:09 | major-upgrade-logical-replica/5-upgrade | user 0m0.078s logger.go:42: 20:44:09 | major-upgrade-logical-replica/5-upgrade | sys 0m0.111s logger.go:42: 20:44:09 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 20:44:09 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 20:44:09 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 20:44:09 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 20:44:09 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 20:44:09 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 20:44:09 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 20:44:09 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 20:44:09 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 20:44:09 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 20:44:09 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 20:44:09 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 20:44:09 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 20:44:09 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 20:44:09 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 20:44:09 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 20:44:09 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 20:44:09 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 20:44:09 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 20:44:09 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:44:09 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 20:44:09 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 20:44:09 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:44:09 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 20:44:09 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 20:44:09 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 20:44:09 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 20:44:09 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 20:44:09 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 20:44:09 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 20:44:09 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 20:44:09 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 20:44:09 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 20:44:09 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 20:44:09 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 20:44:09 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 20:44:09 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 20:44:09 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 20:44:09 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 20:44:09 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 20:44:09 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 20:44:09 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 20:44:09 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:44:09 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 20:44:09 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 20:44:09 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 20:44:09 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 20:44:09 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 20:44:09 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 20:44:09 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 20:44:09 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 20:44:09 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 20:44:09 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 20:44:09 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:44:09 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 20:44:09 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 20:44:09 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 20:44:09 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 20:44:09 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 20:44:09 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:44:09 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 20:44:09 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 20:44:09 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 20:44:09 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 20:44:09 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 20:44:09 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 20:44:09 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 20:44:09 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 20:44:09 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:44:09 | major-upgrade-logical-replica/5-upgrade | real 0m9.786s logger.go:42: 20:44:09 | major-upgrade-logical-replica/5-upgrade | user 0m0.258s logger.go:42: 20:44:09 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:44:09 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 20:44:09 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:44:09 | major-upgrade-logical-replica/5-upgrade | sys 0m0.395s logger.go:42: 20:44:09 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 20:44:09 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 20:44:10 | major-upgrade-logical-replica/5-upgrade | test step completed 5-upgrade logger.go:42: 20:44:10 | major-upgrade-logical-replica/6-verify-replica-invalidated | starting test step 6-verify-replica-invalidated logger.go:42: 20:44:10 | major-upgrade-logical-replica/6-verify-replica-invalidated | running command: [sh -c set -o errexit set -o xtrace source ../../functions primary=$(get_pod_by_role major-upgrade-logical-replica primary name) kubectl -n "${NAMESPACE}" exec "${primary}" -- /pgdata/delete_old_cluster.sh # pg_upgrade only rewrites the data directories of the instance sets, so the # replica still holds a PostgreSQL 17 one. It has to be reported as broken # rather than handed the new binary and left to crash. retry 60 10 bash -c ' state=$(kubectl -n "'"${NAMESPACE}"'" get pg major-upgrade-logical-replica \ -o jsonpath="{.status.logicalReplicas[?(@.name==\"analytics\")].state}") reason=$(kubectl -n "'"${NAMESPACE}"'" get pg major-upgrade-logical-replica \ -o jsonpath="{.status.logicalReplicas[?(@.name==\"analytics\")].reason}") echo "logical replica analytics: ${state}/${reason}" [[ ${state} == "broken" && ${reason} == "SourceUpgraded" ]] ' message=$(kubectl -n "${NAMESPACE}" get pg major-upgrade-logical-replica \ -o jsonpath='{.status.logicalReplicas[?(@.name=="analytics")].message}') echo "message: ${message}" echo "${message}" | grep -q "remove it from spec.logicalReplicas" # The StatefulSet is stopped, so nothing is left crash-looping against a data # directory the new binary cannot open. retry 30 5 bash -c ' replicas=$(kubectl -n "'"${NAMESPACE}"'" get statefulset \ major-upgrade-logical-replica-lr-analytics -o jsonpath="{.spec.replicas}") echo "logical replica statefulset replicas: ${replicas}" [[ ${replicas} == "0" ]] ' # pg_upgrade carries the logical slots over to the new cluster. Nothing will # ever read from them again, so they must not be left pinning WAL. PASS=$(get_psql_user_pass major-upgrade-logical-replica-pguser-postgres) HOST=$(get_psql_user_host major-upgrade-logical-replica-pguser-postgres) no_slots_left() { local got got=$(run_psql \ "SELECT count(*) FROM pg_replication_slots WHERE plugin IS NOT NULL;" \ "postgres://postgres:${PASS}@${HOST}" | xargs) echo "logical slots left on the primary: ${got}" [[ ${got} == "0" ]] } retry 30 5 no_slots_left] logger.go:42: 20:44:10 | major-upgrade-logical-replica/6-verify-replica-invalidated | + source ../../functions logger.go:42: 20:44:10 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ realpath ../../.. logger.go:42: 20:44:10 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1727 logger.go:42: 20:44:10 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ CERT_MANAGER_VER=1.20.2 logger.go:42: 20:44:10 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ CERT_MANAGER_TEST_LABEL=e2e.percona.com/managed-by=percona-postgresql-operator-tests logger.go:42: 20:44:10 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ CHAOS_MESH_VER=2.5.1 logger.go:42: 20:44:10 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ BUSYBOX_VER=1.36 logger.go:42: 20:44:10 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ pwd logger.go:42: 20:44:10 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ basename /mnt/jenkins/workspace/cloud-pg-operator_PR-1727/e2e-tests/tests/major-upgrade-logical-replica logger.go:42: 20:44:10 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ test_name=major-upgrade-logical-replica logger.go:42: 20:44:10 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ source /mnt/jenkins/workspace/cloud-pg-operator_PR-1727/e2e-tests/vars.sh logger.go:42: 20:44:10 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1727 logger.go:42: 20:44:10 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1727 logger.go:42: 20:44:10 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1727/deploy logger.go:42: 20:44:10 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1727/deploy logger.go:42: 20:44:10 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1727/e2e-tests logger.go:42: 20:44:10 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1727/e2e-tests logger.go:42: 20:44:10 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1727/e2e-tests/conf logger.go:42: 20:44:10 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1727/e2e-tests/conf logger.go:42: 20:44:10 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export TEST_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1727/e2e-tests/tests/major-upgrade-logical-replica/conf logger.go:42: 20:44:10 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ TEST_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1727/e2e-tests/tests/major-upgrade-logical-replica/conf logger.go:42: 20:44:10 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export TEMP_DIR=/tmp/kuttl/pg/major-upgrade-logical-replica logger.go:42: 20:44:10 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ TEMP_DIR=/tmp/kuttl/pg/major-upgrade-logical-replica logger.go:42: 20:44:10 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ git rev-parse --abbrev-ref HEAD logger.go:42: 20:44:10 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export GIT_BRANCH=PR-1727 logger.go:42: 20:44:10 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ GIT_BRANCH=PR-1727 logger.go:42: 20:44:10 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export VERSION=PR-1727-5c0d1c600 logger.go:42: 20:44:10 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ VERSION=PR-1727-5c0d1c600 logger.go:42: 20:44:10 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export SKIP_TEST_WARNINGS=true logger.go:42: 20:44:10 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ SKIP_TEST_WARNINGS=true logger.go:42: 20:44:10 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export K8S_UPGRADE_PLATFORM=gke logger.go:42: 20:44:10 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ K8S_UPGRADE_PLATFORM=gke logger.go:42: 20:44:10 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export K8S_UPGRADE_REGION=us-central1-a logger.go:42: 20:44:10 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ K8S_UPGRADE_REGION=us-central1-a logger.go:42: 20:44:10 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export K8S_UPGRADE_INITIAL_VERSION= logger.go:42: 20:44:10 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ K8S_UPGRADE_INITIAL_VERSION= logger.go:42: 20:44:10 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export K8S_UPGRADE_FINAL_VERSION= logger.go:42: 20:44:10 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ K8S_UPGRADE_FINAL_VERSION= logger.go:42: 20:44:10 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ -z '' ]] logger.go:42: 20:44:10 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ detect_k8s_platform logger.go:42: 20:44:10 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ local detected_platform logger.go:42: 20:44:10 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ local xtrace_enabled=false logger.go:42: 20:44:10 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ [[ -o xtrace ]] logger.go:42: 20:44:10 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ xtrace_enabled=true logger.go:42: 20:44:10 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ set +o xtrace logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ PLATFORM=gke logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export PLATFORM logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE=perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE=perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ ! -n '' ]] logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ -n '' ]] logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export PG_VER=18 logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ PG_VER=18 logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export PG_DISTRIBUTION= logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ PG_DISTRIBUTION= logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ '' == community ]] logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export BUCKET=pg-operator-testing logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ BUCKET=pg-operator-testing logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export PMM_SERVER_VERSION=9.9.9 logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ PMM_SERVER_VERSION=9.9.9 logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE_LOGCOLLECTOR=perconalab/fluentbit:main-logcollector logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE_LOGCOLLECTOR=perconalab/fluentbit:main-logcollector logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export PGOV1_TAG=1.4.0 logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ PGOV1_TAG=1.4.0 logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export PGOV1_VER=14 logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ PGOV1_VER=14 logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export CPGO_VERSION=5.8.7 logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ CPGO_VERSION=5.8.7 logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export MINIO_VER=5.4.0 logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ MINIO_VER=5.4.0 logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export VAULT_VER=0.32.0 logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ VAULT_VER=0.32.0 logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export REGISTRY_NAME=docker.io logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ REGISTRY_NAME=docker.io logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export REGISTRY_NAME_FULL=docker.io/ logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ REGISTRY_NAME_FULL=docker.io/ logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ printenv logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ grep -E '^IMAGE' logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ awk -F= '{print $1}' logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ eval 'echo $IMAGE_BACKREST' logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++++ echo perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ var_value=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == docker.io/* ]] logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == percona/* ]] logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == perconalab/* ]] logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ eval 'echo $IMAGE' logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++++ echo perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ var_value=perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 == docker.io/* ]] logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 == percona/* ]] logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 == perconalab/* ]] logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ new_value=docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ eval 'echo $IMAGE_UPGRADE' logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++++ echo perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ var_value=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == docker.io/* ]] logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == percona/* ]] logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == perconalab/* ]] logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ eval 'echo $IMAGE_LOGCOLLECTOR' logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++++ echo perconalab/fluentbit:main-logcollector logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ var_value=perconalab/fluentbit:main-logcollector logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/fluentbit:main-logcollector == docker.io/* ]] logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/fluentbit:main-logcollector == percona/* ]] logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/fluentbit:main-logcollector == perconalab/* ]] logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ new_value=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ eval 'echo $IMAGE_BASE' logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++++ echo perconalab/percona-postgresql-operator logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ var_value=perconalab/percona-postgresql-operator logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator == docker.io/* ]] logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator == percona/* ]] logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator == perconalab/* ]] logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ new_value=docker.io/perconalab/percona-postgresql-operator logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ eval 'echo $IMAGE_PGBOUNCER' logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++++ echo perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ var_value=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == docker.io/* ]] logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == percona/* ]] logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == perconalab/* ]] logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ eval 'echo $IMAGE_POSTGRESQL' logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++++ echo perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ var_value=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == docker.io/* ]] logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == percona/* ]] logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == perconalab/* ]] logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ eval 'echo $IMAGE_PMM_SERVER' logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++++ echo perconalab/pmm-server:3-dev-latest logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ var_value=perconalab/pmm-server:3-dev-latest logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/pmm-server:3-dev-latest == docker.io/* ]] logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/pmm-server:3-dev-latest == percona/* ]] logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/pmm-server:3-dev-latest == perconalab/* ]] logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ new_value=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ eval 'echo $IMAGE_PMM_CLIENT' logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++++ echo perconalab/pmm-client:3-dev-latest logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ var_value=perconalab/pmm-client:3-dev-latest logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/pmm-client:3-dev-latest == docker.io/* ]] logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/pmm-client:3-dev-latest == percona/* ]] logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/pmm-client:3-dev-latest == perconalab/* ]] logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ new_value=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ which gdate logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | which: no gdate in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1727/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ which date logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ date=/usr/sbin/date logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ which gsed logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | which: no gsed in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1727/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ which sed logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ sed=/usr/sbin/sed logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ get_pod_by_role major-upgrade-logical-replica primary name logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ local cluster=major-upgrade-logical-replica logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ local role=primary logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ local parameter=name logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ case ${parameter} in logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ local 'jsonpath={.items[].metadata.name}' logger.go:42: 20:44:12 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ kubectl get pods --namespace kuttl-test-intent-zebra --selector=postgres-operator.crunchydata.com/role=primary,postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica -o 'jsonpath={.items[].metadata.name}' logger.go:42: 20:44:13 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ echo major-upgrade-logical-replica-instance1-n9h5-0 logger.go:42: 20:44:13 | major-upgrade-logical-replica/6-verify-replica-invalidated | + primary=major-upgrade-logical-replica-instance1-n9h5-0 logger.go:42: 20:44:13 | major-upgrade-logical-replica/6-verify-replica-invalidated | + kubectl -n kuttl-test-intent-zebra exec major-upgrade-logical-replica-instance1-n9h5-0 -- /pgdata/delete_old_cluster.sh logger.go:42: 20:44:15 | major-upgrade-logical-replica/6-verify-replica-invalidated | + retry 60 10 bash -c $'\n state=$(kubectl -n "kuttl-test-intent-zebra" get pg major-upgrade-logical-replica \\\n -o jsonpath="{.status.logicalReplicas[?(@.name==\\"analytics\\")].state}")\n reason=$(kubectl -n "kuttl-test-intent-zebra" get pg major-upgrade-logical-replica \\\n -o jsonpath="{.status.logicalReplicas[?(@.name==\\"analytics\\")].reason}")\n echo "logical replica analytics: ${state}/${reason}"\n [[ ${state} == "broken" && ${reason} == "SourceUpgraded" ]]\n' logger.go:42: 20:44:15 | major-upgrade-logical-replica/6-verify-replica-invalidated | + local max=60 logger.go:42: 20:44:15 | major-upgrade-logical-replica/6-verify-replica-invalidated | + local delay=10 logger.go:42: 20:44:15 | major-upgrade-logical-replica/6-verify-replica-invalidated | + shift 2 logger.go:42: 20:44:15 | major-upgrade-logical-replica/6-verify-replica-invalidated | + local n=1 logger.go:42: 20:44:15 | major-upgrade-logical-replica/6-verify-replica-invalidated | + bash -c $'\n state=$(kubectl -n "kuttl-test-intent-zebra" get pg major-upgrade-logical-replica \\\n -o jsonpath="{.status.logicalReplicas[?(@.name==\\"analytics\\")].state}")\n reason=$(kubectl -n "kuttl-test-intent-zebra" get pg major-upgrade-logical-replica \\\n -o jsonpath="{.status.logicalReplicas[?(@.name==\\"analytics\\")].reason}")\n echo "logical replica analytics: ${state}/${reason}"\n [[ ${state} == "broken" && ${reason} == "SourceUpgraded" ]]\n' logger.go:42: 20:44:16 | major-upgrade-logical-replica/6-verify-replica-invalidated | logical replica analytics: broken/SourceUpgraded logger.go:42: 20:44:16 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ kubectl -n kuttl-test-intent-zebra get pg major-upgrade-logical-replica -o 'jsonpath={.status.logicalReplicas[?(@.name=="analytics")].message}' logger.go:42: 20:44:17 | major-upgrade-logical-replica/6-verify-replica-invalidated | + message='this replica holds a PostgreSQL 17 data directory and the cluster now runs 18; the pg_upgrade job only rewrites the data directories of the instance sets, so this one cannot be started again; remove it from spec.logicalReplicas, wait for its entry to leave status.logicalReplicas, and add it back to seed it again' logger.go:42: 20:44:17 | major-upgrade-logical-replica/6-verify-replica-invalidated | + echo 'message: this replica holds a PostgreSQL 17 data directory and the cluster now runs 18; the pg_upgrade job only rewrites the data directories of the instance sets, so this one cannot be started again; remove it from spec.logicalReplicas, wait for its entry to leave status.logicalReplicas, and add it back to seed it again' logger.go:42: 20:44:17 | major-upgrade-logical-replica/6-verify-replica-invalidated | message: this replica holds a PostgreSQL 17 data directory and the cluster now runs 18; the pg_upgrade job only rewrites the data directories of the instance sets, so this one cannot be started again; remove it from spec.logicalReplicas, wait for its entry to leave status.logicalReplicas, and add it back to seed it again logger.go:42: 20:44:17 | major-upgrade-logical-replica/6-verify-replica-invalidated | + echo 'this replica holds a PostgreSQL 17 data directory and the cluster now runs 18; the pg_upgrade job only rewrites the data directories of the instance sets, so this one cannot be started again; remove it from spec.logicalReplicas, wait for its entry to leave status.logicalReplicas, and add it back to seed it again' logger.go:42: 20:44:17 | major-upgrade-logical-replica/6-verify-replica-invalidated | + grep -q 'remove it from spec.logicalReplicas' logger.go:42: 20:44:17 | major-upgrade-logical-replica/6-verify-replica-invalidated | + retry 30 5 bash -c $'\n replicas=$(kubectl -n "kuttl-test-intent-zebra" get statefulset \\\n major-upgrade-logical-replica-lr-analytics -o jsonpath="{.spec.replicas}")\n echo "logical replica statefulset replicas: ${replicas}"\n [[ ${replicas} == "0" ]]\n' logger.go:42: 20:44:17 | major-upgrade-logical-replica/6-verify-replica-invalidated | + local max=30 logger.go:42: 20:44:17 | major-upgrade-logical-replica/6-verify-replica-invalidated | + local delay=5 logger.go:42: 20:44:17 | major-upgrade-logical-replica/6-verify-replica-invalidated | + shift 2 logger.go:42: 20:44:17 | major-upgrade-logical-replica/6-verify-replica-invalidated | + local n=1 logger.go:42: 20:44:17 | major-upgrade-logical-replica/6-verify-replica-invalidated | + bash -c $'\n replicas=$(kubectl -n "kuttl-test-intent-zebra" get statefulset \\\n major-upgrade-logical-replica-lr-analytics -o jsonpath="{.spec.replicas}")\n echo "logical replica statefulset replicas: ${replicas}"\n [[ ${replicas} == "0" ]]\n' logger.go:42: 20:44:17 | major-upgrade-logical-replica/6-verify-replica-invalidated | logical replica statefulset replicas: 0 logger.go:42: 20:44:17 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ get_psql_user_pass major-upgrade-logical-replica-pguser-postgres logger.go:42: 20:44:17 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ local secret_name=major-upgrade-logical-replica-pguser-postgres logger.go:42: 20:44:17 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ kubectl -n kuttl-test-intent-zebra get secret/major-upgrade-logical-replica-pguser-postgres '--template={{.data.password | base64decode}}' logger.go:42: 20:44:18 | major-upgrade-logical-replica/6-verify-replica-invalidated | + PASS=2zjZOVoLkCxNalqBwszLWiyn logger.go:42: 20:44:18 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ get_psql_user_host major-upgrade-logical-replica-pguser-postgres logger.go:42: 20:44:18 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ local secret_name=major-upgrade-logical-replica-pguser-postgres logger.go:42: 20:44:18 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ kubectl -n kuttl-test-intent-zebra get secret/major-upgrade-logical-replica-pguser-postgres '--template={{.data.host | base64decode }}' logger.go:42: 20:44:18 | major-upgrade-logical-replica/6-verify-replica-invalidated | + HOST=major-upgrade-logical-replica-primary.kuttl-test-intent-zebra.svc logger.go:42: 20:44:18 | major-upgrade-logical-replica/6-verify-replica-invalidated | + retry 30 5 no_slots_left logger.go:42: 20:44:18 | major-upgrade-logical-replica/6-verify-replica-invalidated | + local max=30 logger.go:42: 20:44:18 | major-upgrade-logical-replica/6-verify-replica-invalidated | + local delay=5 logger.go:42: 20:44:18 | major-upgrade-logical-replica/6-verify-replica-invalidated | + shift 2 logger.go:42: 20:44:18 | major-upgrade-logical-replica/6-verify-replica-invalidated | + local n=1 logger.go:42: 20:44:18 | major-upgrade-logical-replica/6-verify-replica-invalidated | + no_slots_left logger.go:42: 20:44:18 | major-upgrade-logical-replica/6-verify-replica-invalidated | + local got logger.go:42: 20:44:18 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ run_psql 'SELECT count(*) FROM pg_replication_slots WHERE plugin IS NOT NULL;' postgres://postgres:2zjZOVoLkCxNalqBwszLWiyn@major-upgrade-logical-replica-primary.kuttl-test-intent-zebra.svc logger.go:42: 20:44:18 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ local 'command=SELECT count(*) FROM pg_replication_slots WHERE plugin IS NOT NULL;' logger.go:42: 20:44:18 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ local uri=postgres://postgres:2zjZOVoLkCxNalqBwszLWiyn@major-upgrade-logical-replica-primary.kuttl-test-intent-zebra.svc logger.go:42: 20:44:18 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ local password= logger.go:42: 20:44:18 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ xargs logger.go:42: 20:44:18 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ get_client_pod logger.go:42: 20:44:18 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ kubectl -n kuttl-test-intent-zebra get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' logger.go:42: 20:44:19 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ kubectl -n kuttl-test-intent-zebra exec pg-client-d7c7bb45b-8nwnb -- bash -c 'printf '\''SELECT count(*) FROM pg_replication_slots WHERE plugin IS NOT NULL;\n'\'' | PGPASSWORD='\'''\'' psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:2zjZOVoLkCxNalqBwszLWiyn@major-upgrade-logical-replica-primary.kuttl-test-intent-zebra.svc' logger.go:42: 20:44:20 | major-upgrade-logical-replica/6-verify-replica-invalidated | + got=0 logger.go:42: 20:44:20 | major-upgrade-logical-replica/6-verify-replica-invalidated | + echo 'logical slots left on the primary: 0' logger.go:42: 20:44:20 | major-upgrade-logical-replica/6-verify-replica-invalidated | logical slots left on the primary: 0 logger.go:42: 20:44:20 | major-upgrade-logical-replica/6-verify-replica-invalidated | + [[ 0 == 0 ]] logger.go:42: 20:44:20 | major-upgrade-logical-replica/6-verify-replica-invalidated | test step completed 6-verify-replica-invalidated logger.go:42: 20:44:20 | major-upgrade-logical-replica/7-create-reseed-backup | starting test step 7-create-reseed-backup logger.go:42: 20:44:20 | major-upgrade-logical-replica/7-create-reseed-backup | running command: [sh -c set -o errexit set -o xtrace source ../../functions # pg_upgrade left the repository holding only pre-upgrade backups, so the # bootstrap Job of the reseed below would restore a PostgreSQL 17 data # directory. Take a full backup of the upgraded cluster first. cat </dev/null 2>&1 && return 1 kubectl -n "${NAMESPACE}" get pvc major-upgrade-logical-replica-lr-analytics-pgdata >/dev/null 2>&1 && return 1 return 0 } retry 60 5 status_gone retry 60 5 objects_gone kubectl -n "${NAMESPACE}" patch pg major-upgrade-logical-replica --type=merge --patch '{ "spec": { "logicalReplicas": [ { "name": "analytics", "databases": ["myapp"], "bootstrapMethod": "pgbackrest", "dataVolumeClaimSpec": { "accessModes": ["ReadWriteOnce"], "resources": {"requests": {"storage": "1Gi"}} } } ] } }'] logger.go:42: 20:44:56 | major-upgrade-logical-replica/8-reseed-logical-replica | + source ../../functions logger.go:42: 20:44:56 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ realpath ../../.. logger.go:42: 20:44:56 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1727 logger.go:42: 20:44:56 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ CERT_MANAGER_VER=1.20.2 logger.go:42: 20:44:56 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ CERT_MANAGER_TEST_LABEL=e2e.percona.com/managed-by=percona-postgresql-operator-tests logger.go:42: 20:44:56 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ CHAOS_MESH_VER=2.5.1 logger.go:42: 20:44:56 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ BUSYBOX_VER=1.36 logger.go:42: 20:44:56 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ pwd logger.go:42: 20:44:56 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ basename /mnt/jenkins/workspace/cloud-pg-operator_PR-1727/e2e-tests/tests/major-upgrade-logical-replica logger.go:42: 20:44:56 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ test_name=major-upgrade-logical-replica logger.go:42: 20:44:56 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ source /mnt/jenkins/workspace/cloud-pg-operator_PR-1727/e2e-tests/vars.sh logger.go:42: 20:44:56 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1727 logger.go:42: 20:44:56 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1727 logger.go:42: 20:44:56 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1727/deploy logger.go:42: 20:44:56 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1727/deploy logger.go:42: 20:44:56 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1727/e2e-tests logger.go:42: 20:44:56 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1727/e2e-tests logger.go:42: 20:44:56 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1727/e2e-tests/conf logger.go:42: 20:44:56 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1727/e2e-tests/conf logger.go:42: 20:44:56 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export TEST_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1727/e2e-tests/tests/major-upgrade-logical-replica/conf logger.go:42: 20:44:56 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ TEST_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1727/e2e-tests/tests/major-upgrade-logical-replica/conf logger.go:42: 20:44:56 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export TEMP_DIR=/tmp/kuttl/pg/major-upgrade-logical-replica logger.go:42: 20:44:56 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ TEMP_DIR=/tmp/kuttl/pg/major-upgrade-logical-replica logger.go:42: 20:44:56 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ git rev-parse --abbrev-ref HEAD logger.go:42: 20:44:56 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export GIT_BRANCH=PR-1727 logger.go:42: 20:44:56 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ GIT_BRANCH=PR-1727 logger.go:42: 20:44:56 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export VERSION=PR-1727-5c0d1c600 logger.go:42: 20:44:56 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ VERSION=PR-1727-5c0d1c600 logger.go:42: 20:44:56 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export SKIP_TEST_WARNINGS=true logger.go:42: 20:44:56 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ SKIP_TEST_WARNINGS=true logger.go:42: 20:44:56 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export K8S_UPGRADE_PLATFORM=gke logger.go:42: 20:44:56 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ K8S_UPGRADE_PLATFORM=gke logger.go:42: 20:44:56 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export K8S_UPGRADE_REGION=us-central1-a logger.go:42: 20:44:56 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ K8S_UPGRADE_REGION=us-central1-a logger.go:42: 20:44:56 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export K8S_UPGRADE_INITIAL_VERSION= logger.go:42: 20:44:56 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ K8S_UPGRADE_INITIAL_VERSION= logger.go:42: 20:44:56 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export K8S_UPGRADE_FINAL_VERSION= logger.go:42: 20:44:56 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ K8S_UPGRADE_FINAL_VERSION= logger.go:42: 20:44:56 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ -z '' ]] logger.go:42: 20:44:56 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ detect_k8s_platform logger.go:42: 20:44:56 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ local detected_platform logger.go:42: 20:44:56 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ local xtrace_enabled=false logger.go:42: 20:44:56 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ [[ -o xtrace ]] logger.go:42: 20:44:56 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ xtrace_enabled=true logger.go:42: 20:44:56 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ set +o xtrace logger.go:42: 20:44:58 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ PLATFORM=gke logger.go:42: 20:44:58 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export PLATFORM logger.go:42: 20:44:58 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 20:44:58 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 20:44:58 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE=perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:44:58 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE=perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:44:58 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ ! -n '' ]] logger.go:42: 20:44:58 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ -n '' ]] logger.go:42: 20:44:58 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export PG_VER=18 logger.go:42: 20:44:58 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ PG_VER=18 logger.go:42: 20:44:58 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export PG_DISTRIBUTION= logger.go:42: 20:44:58 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ PG_DISTRIBUTION= logger.go:42: 20:44:58 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ '' == community ]] logger.go:42: 20:44:58 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 20:44:58 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 20:44:58 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 20:44:58 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 20:44:58 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 20:44:58 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 20:44:58 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 20:44:58 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 20:44:58 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export BUCKET=pg-operator-testing logger.go:42: 20:44:58 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ BUCKET=pg-operator-testing logger.go:42: 20:44:58 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export PMM_SERVER_VERSION=9.9.9 logger.go:42: 20:44:58 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ PMM_SERVER_VERSION=9.9.9 logger.go:42: 20:44:58 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 20:44:58 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 20:44:58 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 20:44:58 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 20:44:58 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_LOGCOLLECTOR=perconalab/fluentbit:main-logcollector logger.go:42: 20:44:58 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_LOGCOLLECTOR=perconalab/fluentbit:main-logcollector logger.go:42: 20:44:58 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export PGOV1_TAG=1.4.0 logger.go:42: 20:44:58 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ PGOV1_TAG=1.4.0 logger.go:42: 20:44:58 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export PGOV1_VER=14 logger.go:42: 20:44:58 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ PGOV1_VER=14 logger.go:42: 20:44:58 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export CPGO_VERSION=5.8.7 logger.go:42: 20:44:58 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ CPGO_VERSION=5.8.7 logger.go:42: 20:44:58 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export MINIO_VER=5.4.0 logger.go:42: 20:44:58 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ MINIO_VER=5.4.0 logger.go:42: 20:44:58 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export VAULT_VER=0.32.0 logger.go:42: 20:44:58 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ VAULT_VER=0.32.0 logger.go:42: 20:44:58 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export REGISTRY_NAME=docker.io logger.go:42: 20:44:58 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ REGISTRY_NAME=docker.io logger.go:42: 20:44:58 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export REGISTRY_NAME_FULL=docker.io/ logger.go:42: 20:44:58 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ REGISTRY_NAME_FULL=docker.io/ logger.go:42: 20:44:58 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ printenv logger.go:42: 20:44:58 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ grep -E '^IMAGE' logger.go:42: 20:44:58 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ awk -F= '{print $1}' logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ eval 'echo $IMAGE_BACKREST' logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | +++++ echo perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ var_value=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == docker.io/* ]] logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == percona/* ]] logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == perconalab/* ]] logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ eval 'echo $IMAGE' logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | +++++ echo perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ var_value=perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 == docker.io/* ]] logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 == percona/* ]] logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 == perconalab/* ]] logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ new_value=docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ eval 'echo $IMAGE_UPGRADE' logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | +++++ echo perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ var_value=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == docker.io/* ]] logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == percona/* ]] logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == perconalab/* ]] logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ eval 'echo $IMAGE_LOGCOLLECTOR' logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | +++++ echo perconalab/fluentbit:main-logcollector logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ var_value=perconalab/fluentbit:main-logcollector logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/fluentbit:main-logcollector == docker.io/* ]] logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/fluentbit:main-logcollector == percona/* ]] logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/fluentbit:main-logcollector == perconalab/* ]] logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ new_value=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ eval 'echo $IMAGE_BASE' logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | +++++ echo perconalab/percona-postgresql-operator logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ var_value=perconalab/percona-postgresql-operator logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator == docker.io/* ]] logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator == percona/* ]] logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator == perconalab/* ]] logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ new_value=docker.io/perconalab/percona-postgresql-operator logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ eval 'echo $IMAGE_PGBOUNCER' logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | +++++ echo perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ var_value=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == docker.io/* ]] logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == percona/* ]] logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == perconalab/* ]] logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ eval 'echo $IMAGE_POSTGRESQL' logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | +++++ echo perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ var_value=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == docker.io/* ]] logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == percona/* ]] logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == perconalab/* ]] logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ eval 'echo $IMAGE_PMM_SERVER' logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | +++++ echo perconalab/pmm-server:3-dev-latest logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ var_value=perconalab/pmm-server:3-dev-latest logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/pmm-server:3-dev-latest == docker.io/* ]] logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/pmm-server:3-dev-latest == percona/* ]] logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/pmm-server:3-dev-latest == perconalab/* ]] logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ new_value=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ eval 'echo $IMAGE_PMM_CLIENT' logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | +++++ echo perconalab/pmm-client:3-dev-latest logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ var_value=perconalab/pmm-client:3-dev-latest logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/pmm-client:3-dev-latest == docker.io/* ]] logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/pmm-client:3-dev-latest == percona/* ]] logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/pmm-client:3-dev-latest == perconalab/* ]] logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ new_value=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ which gdate logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | which: no gdate in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1727/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ which date logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ date=/usr/sbin/date logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ which gsed logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | which: no gsed in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1727/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ which sed logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ sed=/usr/sbin/sed logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | + kubectl -n kuttl-test-intent-zebra patch pg major-upgrade-logical-replica --type=json --patch '[{"op": "remove", "path": "/spec/logicalReplicas"}]' logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | perconapgcluster.pgv2.percona.com/major-upgrade-logical-replica patched logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | + retry 60 5 status_gone logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | + local max=60 logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | + local delay=5 logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | + shift 2 logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | + local n=1 logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | + status_gone logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | + local got logger.go:42: 20:44:59 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ kubectl -n kuttl-test-intent-zebra get pg major-upgrade-logical-replica -o 'jsonpath={.status.logicalReplicas}' logger.go:42: 20:45:00 | major-upgrade-logical-replica/8-reseed-logical-replica | + got='[{"invalidatedAt":"2026-09-03T20:43:43Z","message":"this replica holds a PostgreSQL 17 data directory and the cluster now runs 18; the pg_upgrade job only rewrites the data directories of the instance sets, so this one cannot be started again; remove it from spec.logicalReplicas, wait for its entry to leave status.logicalReplicas, and add it back to seed it again","name":"analytics","postgresVersion":17,"reason":"SourceUpgraded","seededAt":"2026-09-03T20:41:18Z","state":"broken"}]' logger.go:42: 20:45:00 | major-upgrade-logical-replica/8-reseed-logical-replica | + echo 'status.logicalReplicas: '\''[{"invalidatedAt":"2026-09-03T20:43:43Z","message":"this replica holds a PostgreSQL 17 data directory and the cluster now runs 18; the pg_upgrade job only rewrites the data directories of the instance sets, so this one cannot be started again; remove it from spec.logicalReplicas, wait for its entry to leave status.logicalReplicas, and add it back to seed it again","name":"analytics","postgresVersion":17,"reason":"SourceUpgraded","seededAt":"2026-09-03T20:41:18Z","state":"broken"}]'\''' logger.go:42: 20:45:00 | major-upgrade-logical-replica/8-reseed-logical-replica | status.logicalReplicas: '[{"invalidatedAt":"2026-09-03T20:43:43Z","message":"this replica holds a PostgreSQL 17 data directory and the cluster now runs 18; the pg_upgrade job only rewrites the data directories of the instance sets, so this one cannot be started again; remove it from spec.logicalReplicas, wait for its entry to leave status.logicalReplicas, and add it back to seed it again","name":"analytics","postgresVersion":17,"reason":"SourceUpgraded","seededAt":"2026-09-03T20:41:18Z","state":"broken"}]' logger.go:42: 20:45:00 | major-upgrade-logical-replica/8-reseed-logical-replica | + [[ -z [{"invalidatedAt":"2026-09-03T20:43:43Z","message":"this replica holds a PostgreSQL 17 data directory and the cluster now runs 18; the pg_upgrade job only rewrites the data directories of the instance sets, so this one cannot be started again; remove it from spec.logicalReplicas, wait for its entry to leave status.logicalReplicas, and add it back to seed it again","name":"analytics","postgresVersion":17,"reason":"SourceUpgraded","seededAt":"2026-09-03T20:41:18Z","state":"broken"}] ]] logger.go:42: 20:45:00 | major-upgrade-logical-replica/8-reseed-logical-replica | + [[ [{"invalidatedAt":"2026-09-03T20:43:43Z","message":"this replica holds a PostgreSQL 17 data directory and the cluster now runs 18; the pg_upgrade job only rewrites the data directories of the instance sets, so this one cannot be started again; remove it from spec.logicalReplicas, wait for its entry to leave status.logicalReplicas, and add it back to seed it again","name":"analytics","postgresVersion":17,"reason":"SourceUpgraded","seededAt":"2026-09-03T20:41:18Z","state":"broken"}] == \[\] ]] logger.go:42: 20:45:00 | major-upgrade-logical-replica/8-reseed-logical-replica | + [[ 1 -ge 60 ]] logger.go:42: 20:45:00 | major-upgrade-logical-replica/8-reseed-logical-replica | + (( n += 1 )) logger.go:42: 20:45:00 | major-upgrade-logical-replica/8-reseed-logical-replica | + sleep 5 logger.go:42: 20:45:05 | major-upgrade-logical-replica/8-reseed-logical-replica | + status_gone logger.go:42: 20:45:05 | major-upgrade-logical-replica/8-reseed-logical-replica | + local got logger.go:42: 20:45:05 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ kubectl -n kuttl-test-intent-zebra get pg major-upgrade-logical-replica -o 'jsonpath={.status.logicalReplicas}' logger.go:42: 20:45:06 | major-upgrade-logical-replica/8-reseed-logical-replica | + got= logger.go:42: 20:45:06 | major-upgrade-logical-replica/8-reseed-logical-replica | + echo 'status.logicalReplicas: '\'''\''' logger.go:42: 20:45:06 | major-upgrade-logical-replica/8-reseed-logical-replica | status.logicalReplicas: '' logger.go:42: 20:45:06 | major-upgrade-logical-replica/8-reseed-logical-replica | + [[ -z '' ]] logger.go:42: 20:45:06 | major-upgrade-logical-replica/8-reseed-logical-replica | + retry 60 5 objects_gone logger.go:42: 20:45:06 | major-upgrade-logical-replica/8-reseed-logical-replica | + local max=60 logger.go:42: 20:45:06 | major-upgrade-logical-replica/8-reseed-logical-replica | + local delay=5 logger.go:42: 20:45:06 | major-upgrade-logical-replica/8-reseed-logical-replica | + shift 2 logger.go:42: 20:45:06 | major-upgrade-logical-replica/8-reseed-logical-replica | + local n=1 logger.go:42: 20:45:06 | major-upgrade-logical-replica/8-reseed-logical-replica | + objects_gone logger.go:42: 20:45:06 | major-upgrade-logical-replica/8-reseed-logical-replica | + kubectl -n kuttl-test-intent-zebra get statefulset major-upgrade-logical-replica-lr-analytics logger.go:42: 20:45:06 | major-upgrade-logical-replica/8-reseed-logical-replica | + kubectl -n kuttl-test-intent-zebra get pvc major-upgrade-logical-replica-lr-analytics-pgdata logger.go:42: 20:45:07 | major-upgrade-logical-replica/8-reseed-logical-replica | + return 0 logger.go:42: 20:45:07 | major-upgrade-logical-replica/8-reseed-logical-replica | + kubectl -n kuttl-test-intent-zebra patch pg major-upgrade-logical-replica --type=merge --patch $'{\n "spec": {\n "logicalReplicas": [\n {\n "name": "analytics",\n "databases": ["myapp"],\n "bootstrapMethod": "pgbackrest",\n "dataVolumeClaimSpec": {\n "accessModes": ["ReadWriteOnce"],\n "resources": {"requests": {"storage": "1Gi"}}\n }\n }\n ]\n }\n}' logger.go:42: 20:45:08 | major-upgrade-logical-replica/8-reseed-logical-replica | perconapgcluster.pgv2.percona.com/major-upgrade-logical-replica patched logger.go:42: 20:45:08 | major-upgrade-logical-replica/8-reseed-logical-replica | running command: [sh -c set -o errexit set -o xtrace source ../../functions retry 90 10 bash -c ' state=$(kubectl -n "'"${NAMESPACE}"'" get pg major-upgrade-logical-replica \ -o jsonpath="{.status.logicalReplicas[?(@.name==\"analytics\")].state}") echo "logical replica analytics state: ${state}" [[ ${state} == "ready" ]] ' # Seeded from the upgraded cluster, so it is stamped with the new version. seeded=$(kubectl -n "${NAMESPACE}" get pg major-upgrade-logical-replica \ -o jsonpath='{.status.logicalReplicas[?(@.name=="analytics")].postgresVersion}') echo "seeded at PostgreSQL: ${seeded}" [[ ${seeded} == "18" ]] kubectl -n "${NAMESPACE}" wait --for=condition=Ready pod \ -l "pgv2.percona.com/logical-replica=analytics" --timeout=300s PASS=$(get_psql_user_pass major-upgrade-logical-replica-pguser-postgres) HOST=$(get_psql_user_host major-upgrade-logical-replica-pguser-postgres) REPLICA="major-upgrade-logical-replica-lr-analytics.${NAMESPACE}.svc" # Replication actually streams again on the upgraded cluster. run_psql \ '\c myapp \\\ INSERT INTO myApp (id) VALUES (100501);' \ "postgres://postgres:${PASS}@${HOST}" row_streamed() { local got got=$(run_psql \ '\c myapp \\\ SELECT count(*) FROM myApp WHERE id = 100501;' \ "postgres://postgres:${PASS}@${REPLICA}" | xargs) echo "streamed rows on the replica: ${got}" [[ ${got} == "1" ]] } retry 30 5 row_streamed] logger.go:42: 20:45:08 | major-upgrade-logical-replica/8-reseed-logical-replica | + source ../../functions logger.go:42: 20:45:08 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ realpath ../../.. logger.go:42: 20:45:08 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1727 logger.go:42: 20:45:08 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ CERT_MANAGER_VER=1.20.2 logger.go:42: 20:45:08 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ CERT_MANAGER_TEST_LABEL=e2e.percona.com/managed-by=percona-postgresql-operator-tests logger.go:42: 20:45:08 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ CHAOS_MESH_VER=2.5.1 logger.go:42: 20:45:08 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ BUSYBOX_VER=1.36 logger.go:42: 20:45:08 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ pwd logger.go:42: 20:45:08 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ basename /mnt/jenkins/workspace/cloud-pg-operator_PR-1727/e2e-tests/tests/major-upgrade-logical-replica logger.go:42: 20:45:08 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ test_name=major-upgrade-logical-replica logger.go:42: 20:45:08 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ source /mnt/jenkins/workspace/cloud-pg-operator_PR-1727/e2e-tests/vars.sh logger.go:42: 20:45:08 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1727 logger.go:42: 20:45:08 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1727 logger.go:42: 20:45:08 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1727/deploy logger.go:42: 20:45:08 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1727/deploy logger.go:42: 20:45:08 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1727/e2e-tests logger.go:42: 20:45:08 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1727/e2e-tests logger.go:42: 20:45:08 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1727/e2e-tests/conf logger.go:42: 20:45:08 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1727/e2e-tests/conf logger.go:42: 20:45:08 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export TEST_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1727/e2e-tests/tests/major-upgrade-logical-replica/conf logger.go:42: 20:45:08 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ TEST_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1727/e2e-tests/tests/major-upgrade-logical-replica/conf logger.go:42: 20:45:08 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export TEMP_DIR=/tmp/kuttl/pg/major-upgrade-logical-replica logger.go:42: 20:45:08 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ TEMP_DIR=/tmp/kuttl/pg/major-upgrade-logical-replica logger.go:42: 20:45:08 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ git rev-parse --abbrev-ref HEAD logger.go:42: 20:45:08 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export GIT_BRANCH=PR-1727 logger.go:42: 20:45:08 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ GIT_BRANCH=PR-1727 logger.go:42: 20:45:08 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export VERSION=PR-1727-5c0d1c600 logger.go:42: 20:45:08 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ VERSION=PR-1727-5c0d1c600 logger.go:42: 20:45:08 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export SKIP_TEST_WARNINGS=true logger.go:42: 20:45:08 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ SKIP_TEST_WARNINGS=true logger.go:42: 20:45:08 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export K8S_UPGRADE_PLATFORM=gke logger.go:42: 20:45:08 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ K8S_UPGRADE_PLATFORM=gke logger.go:42: 20:45:08 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export K8S_UPGRADE_REGION=us-central1-a logger.go:42: 20:45:08 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ K8S_UPGRADE_REGION=us-central1-a logger.go:42: 20:45:08 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export K8S_UPGRADE_INITIAL_VERSION= logger.go:42: 20:45:08 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ K8S_UPGRADE_INITIAL_VERSION= logger.go:42: 20:45:08 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export K8S_UPGRADE_FINAL_VERSION= logger.go:42: 20:45:08 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ K8S_UPGRADE_FINAL_VERSION= logger.go:42: 20:45:08 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ -z '' ]] logger.go:42: 20:45:08 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ detect_k8s_platform logger.go:42: 20:45:08 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ local detected_platform logger.go:42: 20:45:08 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ local xtrace_enabled=false logger.go:42: 20:45:08 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ [[ -o xtrace ]] logger.go:42: 20:45:08 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ xtrace_enabled=true logger.go:42: 20:45:08 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ set +o xtrace logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ PLATFORM=gke logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export PLATFORM logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE=perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE=perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ ! -n '' ]] logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ -n '' ]] logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export PG_VER=18 logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ PG_VER=18 logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export PG_DISTRIBUTION= logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ PG_DISTRIBUTION= logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ '' == community ]] logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export BUCKET=pg-operator-testing logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ BUCKET=pg-operator-testing logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export PMM_SERVER_VERSION=9.9.9 logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ PMM_SERVER_VERSION=9.9.9 logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_LOGCOLLECTOR=perconalab/fluentbit:main-logcollector logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_LOGCOLLECTOR=perconalab/fluentbit:main-logcollector logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export PGOV1_TAG=1.4.0 logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ PGOV1_TAG=1.4.0 logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export PGOV1_VER=14 logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ PGOV1_VER=14 logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export CPGO_VERSION=5.8.7 logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ CPGO_VERSION=5.8.7 logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export MINIO_VER=5.4.0 logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ MINIO_VER=5.4.0 logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export VAULT_VER=0.32.0 logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ VAULT_VER=0.32.0 logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export REGISTRY_NAME=docker.io logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ REGISTRY_NAME=docker.io logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export REGISTRY_NAME_FULL=docker.io/ logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ REGISTRY_NAME_FULL=docker.io/ logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ printenv logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ grep -E '^IMAGE' logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ awk -F= '{print $1}' logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ eval 'echo $IMAGE_BACKREST' logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++++ echo perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ var_value=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == docker.io/* ]] logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == percona/* ]] logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == perconalab/* ]] logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ eval 'echo $IMAGE' logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++++ echo perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ var_value=perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 == docker.io/* ]] logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 == percona/* ]] logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 == perconalab/* ]] logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ new_value=docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ eval 'echo $IMAGE_UPGRADE' logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++++ echo perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ var_value=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == docker.io/* ]] logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == percona/* ]] logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == perconalab/* ]] logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ eval 'echo $IMAGE_LOGCOLLECTOR' logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++++ echo perconalab/fluentbit:main-logcollector logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ var_value=perconalab/fluentbit:main-logcollector logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/fluentbit:main-logcollector == docker.io/* ]] logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/fluentbit:main-logcollector == percona/* ]] logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/fluentbit:main-logcollector == perconalab/* ]] logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ new_value=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ eval 'echo $IMAGE_BASE' logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++++ echo perconalab/percona-postgresql-operator logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ var_value=perconalab/percona-postgresql-operator logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator == docker.io/* ]] logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator == percona/* ]] logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator == perconalab/* ]] logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ new_value=docker.io/perconalab/percona-postgresql-operator logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ eval 'echo $IMAGE_PGBOUNCER' logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++++ echo perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ var_value=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == docker.io/* ]] logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == percona/* ]] logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == perconalab/* ]] logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ eval 'echo $IMAGE_POSTGRESQL' logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++++ echo perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ var_value=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == docker.io/* ]] logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == percona/* ]] logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == perconalab/* ]] logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ eval 'echo $IMAGE_PMM_SERVER' logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++++ echo perconalab/pmm-server:3-dev-latest logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ var_value=perconalab/pmm-server:3-dev-latest logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/pmm-server:3-dev-latest == docker.io/* ]] logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/pmm-server:3-dev-latest == percona/* ]] logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/pmm-server:3-dev-latest == perconalab/* ]] logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ new_value=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ eval 'echo $IMAGE_PMM_CLIENT' logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++++ echo perconalab/pmm-client:3-dev-latest logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ var_value=perconalab/pmm-client:3-dev-latest logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/pmm-client:3-dev-latest == docker.io/* ]] logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/pmm-client:3-dev-latest == percona/* ]] logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/pmm-client:3-dev-latest == perconalab/* ]] logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ new_value=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ which gdate logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | which: no gdate in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1727/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ which date logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ date=/usr/sbin/date logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ which gsed logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | which: no gsed in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1727/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ which sed logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ sed=/usr/sbin/sed logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | + retry 90 10 bash -c $'\n state=$(kubectl -n "kuttl-test-intent-zebra" get pg major-upgrade-logical-replica \\\n -o jsonpath="{.status.logicalReplicas[?(@.name==\\"analytics\\")].state}")\n echo "logical replica analytics state: ${state}"\n [[ ${state} == "ready" ]]\n' logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | + local max=90 logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | + local delay=10 logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | + shift 2 logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | + local n=1 logger.go:42: 20:45:10 | major-upgrade-logical-replica/8-reseed-logical-replica | + bash -c $'\n state=$(kubectl -n "kuttl-test-intent-zebra" get pg major-upgrade-logical-replica \\\n -o jsonpath="{.status.logicalReplicas[?(@.name==\\"analytics\\")].state}")\n echo "logical replica analytics state: ${state}"\n [[ ${state} == "ready" ]]\n' logger.go:42: 20:45:11 | major-upgrade-logical-replica/8-reseed-logical-replica | logical replica analytics state: bootstrapping logger.go:42: 20:45:11 | major-upgrade-logical-replica/8-reseed-logical-replica | + [[ 1 -ge 90 ]] logger.go:42: 20:45:11 | major-upgrade-logical-replica/8-reseed-logical-replica | + (( n += 1 )) logger.go:42: 20:45:11 | major-upgrade-logical-replica/8-reseed-logical-replica | + sleep 10 logger.go:42: 20:45:21 | major-upgrade-logical-replica/8-reseed-logical-replica | + bash -c $'\n state=$(kubectl -n "kuttl-test-intent-zebra" get pg major-upgrade-logical-replica \\\n -o jsonpath="{.status.logicalReplicas[?(@.name==\\"analytics\\")].state}")\n echo "logical replica analytics state: ${state}"\n [[ ${state} == "ready" ]]\n' logger.go:42: 20:45:21 | major-upgrade-logical-replica/8-reseed-logical-replica | logical replica analytics state: bootstrapping logger.go:42: 20:45:21 | major-upgrade-logical-replica/8-reseed-logical-replica | + [[ 2 -ge 90 ]] logger.go:42: 20:45:21 | major-upgrade-logical-replica/8-reseed-logical-replica | + (( n += 1 )) logger.go:42: 20:45:21 | major-upgrade-logical-replica/8-reseed-logical-replica | + sleep 10 logger.go:42: 20:45:31 | major-upgrade-logical-replica/8-reseed-logical-replica | + bash -c $'\n state=$(kubectl -n "kuttl-test-intent-zebra" get pg major-upgrade-logical-replica \\\n -o jsonpath="{.status.logicalReplicas[?(@.name==\\"analytics\\")].state}")\n echo "logical replica analytics state: ${state}"\n [[ ${state} == "ready" ]]\n' logger.go:42: 20:45:32 | major-upgrade-logical-replica/8-reseed-logical-replica | logical replica analytics state: bootstrapping logger.go:42: 20:45:32 | major-upgrade-logical-replica/8-reseed-logical-replica | + [[ 3 -ge 90 ]] logger.go:42: 20:45:32 | major-upgrade-logical-replica/8-reseed-logical-replica | + (( n += 1 )) logger.go:42: 20:45:32 | major-upgrade-logical-replica/8-reseed-logical-replica | + sleep 10 logger.go:42: 20:45:42 | major-upgrade-logical-replica/8-reseed-logical-replica | + bash -c $'\n state=$(kubectl -n "kuttl-test-intent-zebra" get pg major-upgrade-logical-replica \\\n -o jsonpath="{.status.logicalReplicas[?(@.name==\\"analytics\\")].state}")\n echo "logical replica analytics state: ${state}"\n [[ ${state} == "ready" ]]\n' logger.go:42: 20:45:43 | major-upgrade-logical-replica/8-reseed-logical-replica | logical replica analytics state: bootstrapping logger.go:42: 20:45:43 | major-upgrade-logical-replica/8-reseed-logical-replica | + [[ 4 -ge 90 ]] logger.go:42: 20:45:43 | major-upgrade-logical-replica/8-reseed-logical-replica | + (( n += 1 )) logger.go:42: 20:45:43 | major-upgrade-logical-replica/8-reseed-logical-replica | + sleep 10 logger.go:42: 20:45:53 | major-upgrade-logical-replica/8-reseed-logical-replica | + bash -c $'\n state=$(kubectl -n "kuttl-test-intent-zebra" get pg major-upgrade-logical-replica \\\n -o jsonpath="{.status.logicalReplicas[?(@.name==\\"analytics\\")].state}")\n echo "logical replica analytics state: ${state}"\n [[ ${state} == "ready" ]]\n' logger.go:42: 20:45:53 | major-upgrade-logical-replica/8-reseed-logical-replica | logical replica analytics state: bootstrapping logger.go:42: 20:45:53 | major-upgrade-logical-replica/8-reseed-logical-replica | + [[ 5 -ge 90 ]] logger.go:42: 20:45:53 | major-upgrade-logical-replica/8-reseed-logical-replica | + (( n += 1 )) logger.go:42: 20:45:53 | major-upgrade-logical-replica/8-reseed-logical-replica | + sleep 10 logger.go:42: 20:46:03 | major-upgrade-logical-replica/8-reseed-logical-replica | + bash -c $'\n state=$(kubectl -n "kuttl-test-intent-zebra" get pg major-upgrade-logical-replica \\\n -o jsonpath="{.status.logicalReplicas[?(@.name==\\"analytics\\")].state}")\n echo "logical replica analytics state: ${state}"\n [[ ${state} == "ready" ]]\n' logger.go:42: 20:46:04 | major-upgrade-logical-replica/8-reseed-logical-replica | logical replica analytics state: ready logger.go:42: 20:46:04 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ kubectl -n kuttl-test-intent-zebra get pg major-upgrade-logical-replica -o 'jsonpath={.status.logicalReplicas[?(@.name=="analytics")].postgresVersion}' logger.go:42: 20:46:05 | major-upgrade-logical-replica/8-reseed-logical-replica | + seeded=18 logger.go:42: 20:46:05 | major-upgrade-logical-replica/8-reseed-logical-replica | + echo 'seeded at PostgreSQL: 18' logger.go:42: 20:46:05 | major-upgrade-logical-replica/8-reseed-logical-replica | seeded at PostgreSQL: 18 logger.go:42: 20:46:05 | major-upgrade-logical-replica/8-reseed-logical-replica | + [[ 18 == 18 ]] logger.go:42: 20:46:05 | major-upgrade-logical-replica/8-reseed-logical-replica | + kubectl -n kuttl-test-intent-zebra wait --for=condition=Ready pod -l pgv2.percona.com/logical-replica=analytics --timeout=300s logger.go:42: 20:46:05 | major-upgrade-logical-replica/8-reseed-logical-replica | pod/major-upgrade-logical-replica-lr-analytics-0 condition met logger.go:42: 20:46:05 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ get_psql_user_pass major-upgrade-logical-replica-pguser-postgres logger.go:42: 20:46:05 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ local secret_name=major-upgrade-logical-replica-pguser-postgres logger.go:42: 20:46:05 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ kubectl -n kuttl-test-intent-zebra get secret/major-upgrade-logical-replica-pguser-postgres '--template={{.data.password | base64decode}}' logger.go:42: 20:46:06 | major-upgrade-logical-replica/8-reseed-logical-replica | + PASS=2zjZOVoLkCxNalqBwszLWiyn logger.go:42: 20:46:06 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ get_psql_user_host major-upgrade-logical-replica-pguser-postgres logger.go:42: 20:46:06 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ local secret_name=major-upgrade-logical-replica-pguser-postgres logger.go:42: 20:46:06 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ kubectl -n kuttl-test-intent-zebra get secret/major-upgrade-logical-replica-pguser-postgres '--template={{.data.host | base64decode }}' logger.go:42: 20:46:06 | major-upgrade-logical-replica/8-reseed-logical-replica | + HOST=major-upgrade-logical-replica-primary.kuttl-test-intent-zebra.svc logger.go:42: 20:46:06 | major-upgrade-logical-replica/8-reseed-logical-replica | + REPLICA=major-upgrade-logical-replica-lr-analytics.kuttl-test-intent-zebra.svc logger.go:42: 20:46:06 | major-upgrade-logical-replica/8-reseed-logical-replica | + run_psql '\c myapp \\\ INSERT INTO myApp (id) VALUES (100501);' postgres://postgres:2zjZOVoLkCxNalqBwszLWiyn@major-upgrade-logical-replica-primary.kuttl-test-intent-zebra.svc logger.go:42: 20:46:06 | major-upgrade-logical-replica/8-reseed-logical-replica | + local 'command=\c myapp \\\ INSERT INTO myApp (id) VALUES (100501);' logger.go:42: 20:46:06 | major-upgrade-logical-replica/8-reseed-logical-replica | + local uri=postgres://postgres:2zjZOVoLkCxNalqBwszLWiyn@major-upgrade-logical-replica-primary.kuttl-test-intent-zebra.svc logger.go:42: 20:46:06 | major-upgrade-logical-replica/8-reseed-logical-replica | + local password= logger.go:42: 20:46:06 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ get_client_pod logger.go:42: 20:46:06 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ kubectl -n kuttl-test-intent-zebra get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' logger.go:42: 20:46:07 | major-upgrade-logical-replica/8-reseed-logical-replica | + kubectl -n kuttl-test-intent-zebra exec pg-client-d7c7bb45b-8nwnb -- bash -c 'printf '\''\c myapp \\\ INSERT INTO myApp (id) VALUES (100501);\n'\'' | PGPASSWORD='\'''\'' psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:2zjZOVoLkCxNalqBwszLWiyn@major-upgrade-logical-replica-primary.kuttl-test-intent-zebra.svc' logger.go:42: 20:46:08 | major-upgrade-logical-replica/8-reseed-logical-replica | + retry 30 5 row_streamed logger.go:42: 20:46:08 | major-upgrade-logical-replica/8-reseed-logical-replica | + local max=30 logger.go:42: 20:46:08 | major-upgrade-logical-replica/8-reseed-logical-replica | + local delay=5 logger.go:42: 20:46:08 | major-upgrade-logical-replica/8-reseed-logical-replica | + shift 2 logger.go:42: 20:46:08 | major-upgrade-logical-replica/8-reseed-logical-replica | + local n=1 logger.go:42: 20:46:08 | major-upgrade-logical-replica/8-reseed-logical-replica | + row_streamed logger.go:42: 20:46:08 | major-upgrade-logical-replica/8-reseed-logical-replica | + local got logger.go:42: 20:46:08 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ run_psql '\c myapp \\\ SELECT count(*) FROM myApp WHERE id = 100501;' postgres://postgres:2zjZOVoLkCxNalqBwszLWiyn@major-upgrade-logical-replica-lr-analytics.kuttl-test-intent-zebra.svc logger.go:42: 20:46:08 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ xargs logger.go:42: 20:46:08 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ local 'command=\c myapp \\\ SELECT count(*) FROM myApp WHERE id = 100501;' logger.go:42: 20:46:08 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ local uri=postgres://postgres:2zjZOVoLkCxNalqBwszLWiyn@major-upgrade-logical-replica-lr-analytics.kuttl-test-intent-zebra.svc logger.go:42: 20:46:08 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ local password= logger.go:42: 20:46:08 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ get_client_pod logger.go:42: 20:46:08 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ kubectl -n kuttl-test-intent-zebra get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' logger.go:42: 20:46:09 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ kubectl -n kuttl-test-intent-zebra exec pg-client-d7c7bb45b-8nwnb -- bash -c 'printf '\''\c myapp \\\ SELECT count(*) FROM myApp WHERE id = 100501;\n'\'' | PGPASSWORD='\'''\'' psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:2zjZOVoLkCxNalqBwszLWiyn@major-upgrade-logical-replica-lr-analytics.kuttl-test-intent-zebra.svc' logger.go:42: 20:46:10 | major-upgrade-logical-replica/8-reseed-logical-replica | + got=1 logger.go:42: 20:46:10 | major-upgrade-logical-replica/8-reseed-logical-replica | + echo 'streamed rows on the replica: 1' logger.go:42: 20:46:10 | major-upgrade-logical-replica/8-reseed-logical-replica | streamed rows on the replica: 1 logger.go:42: 20:46:10 | major-upgrade-logical-replica/8-reseed-logical-replica | + [[ 1 == 1 ]] logger.go:42: 20:46:10 | major-upgrade-logical-replica/8-reseed-logical-replica | test step completed 8-reseed-logical-replica logger.go:42: 20:46:10 | major-upgrade-logical-replica/99-remove-cluster-gracefully | starting test step 99-remove-cluster-gracefully logger.go:42: 20:46:19 | major-upgrade-logical-replica/99-remove-cluster-gracefully | running command: [sh -c set -o errexit set -o xtrace source ../../functions remove_all_finalizers check_operator_panic destroy_operator] logger.go:42: 20:46:19 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + source ../../functions logger.go:42: 20:46:19 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ realpath ../../.. logger.go:42: 20:46:19 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1727 logger.go:42: 20:46:19 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++ CERT_MANAGER_VER=1.20.2 logger.go:42: 20:46:19 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++ CERT_MANAGER_TEST_LABEL=e2e.percona.com/managed-by=percona-postgresql-operator-tests logger.go:42: 20:46:19 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++ CHAOS_MESH_VER=2.5.1 logger.go:42: 20:46:19 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++ BUSYBOX_VER=1.36 logger.go:42: 20:46:19 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ pwd logger.go:42: 20:46:19 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ basename /mnt/jenkins/workspace/cloud-pg-operator_PR-1727/e2e-tests/tests/major-upgrade-logical-replica logger.go:42: 20:46:19 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++ test_name=major-upgrade-logical-replica logger.go:42: 20:46:19 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++ source /mnt/jenkins/workspace/cloud-pg-operator_PR-1727/e2e-tests/vars.sh logger.go:42: 20:46:19 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1727 logger.go:42: 20:46:19 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1727 logger.go:42: 20:46:19 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1727/deploy logger.go:42: 20:46:19 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1727/deploy logger.go:42: 20:46:19 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1727/e2e-tests logger.go:42: 20:46:19 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1727/e2e-tests logger.go:42: 20:46:19 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1727/e2e-tests/conf logger.go:42: 20:46:19 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1727/e2e-tests/conf logger.go:42: 20:46:19 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export TEST_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1727/e2e-tests/tests/major-upgrade-logical-replica/conf logger.go:42: 20:46:19 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ TEST_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1727/e2e-tests/tests/major-upgrade-logical-replica/conf logger.go:42: 20:46:19 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export TEMP_DIR=/tmp/kuttl/pg/major-upgrade-logical-replica logger.go:42: 20:46:19 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ TEMP_DIR=/tmp/kuttl/pg/major-upgrade-logical-replica logger.go:42: 20:46:19 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ git rev-parse --abbrev-ref HEAD logger.go:42: 20:46:19 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export GIT_BRANCH=PR-1727 logger.go:42: 20:46:19 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ GIT_BRANCH=PR-1727 logger.go:42: 20:46:19 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export VERSION=PR-1727-5c0d1c600 logger.go:42: 20:46:19 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ VERSION=PR-1727-5c0d1c600 logger.go:42: 20:46:19 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export SKIP_TEST_WARNINGS=true logger.go:42: 20:46:19 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ SKIP_TEST_WARNINGS=true logger.go:42: 20:46:19 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export K8S_UPGRADE_PLATFORM=gke logger.go:42: 20:46:19 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ K8S_UPGRADE_PLATFORM=gke logger.go:42: 20:46:19 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export K8S_UPGRADE_REGION=us-central1-a logger.go:42: 20:46:19 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ K8S_UPGRADE_REGION=us-central1-a logger.go:42: 20:46:19 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export K8S_UPGRADE_INITIAL_VERSION= logger.go:42: 20:46:19 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ K8S_UPGRADE_INITIAL_VERSION= logger.go:42: 20:46:19 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export K8S_UPGRADE_FINAL_VERSION= logger.go:42: 20:46:19 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ K8S_UPGRADE_FINAL_VERSION= logger.go:42: 20:46:19 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ -z '' ]] logger.go:42: 20:46:19 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ detect_k8s_platform logger.go:42: 20:46:19 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ local detected_platform logger.go:42: 20:46:19 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ local xtrace_enabled=false logger.go:42: 20:46:19 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ [[ -o xtrace ]] logger.go:42: 20:46:19 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ xtrace_enabled=true logger.go:42: 20:46:19 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ set +o xtrace logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ PLATFORM=gke logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export PLATFORM logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export IMAGE=perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ IMAGE=perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ ! -n '' ]] logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ -n '' ]] logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export PG_VER=18 logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ PG_VER=18 logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export PG_DISTRIBUTION= logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ PG_DISTRIBUTION= logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ '' == community ]] logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export BUCKET=pg-operator-testing logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ BUCKET=pg-operator-testing logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export PMM_SERVER_VERSION=9.9.9 logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ PMM_SERVER_VERSION=9.9.9 logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export IMAGE_LOGCOLLECTOR=perconalab/fluentbit:main-logcollector logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ IMAGE_LOGCOLLECTOR=perconalab/fluentbit:main-logcollector logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export PGOV1_TAG=1.4.0 logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ PGOV1_TAG=1.4.0 logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export PGOV1_VER=14 logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ PGOV1_VER=14 logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export CPGO_VERSION=5.8.7 logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ CPGO_VERSION=5.8.7 logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export MINIO_VER=5.4.0 logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ MINIO_VER=5.4.0 logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export VAULT_VER=0.32.0 logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ VAULT_VER=0.32.0 logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export REGISTRY_NAME=docker.io logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ REGISTRY_NAME=docker.io logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export REGISTRY_NAME_FULL=docker.io/ logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ REGISTRY_NAME_FULL=docker.io/ logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ printenv logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ grep -E '^IMAGE' logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ awk -F= '{print $1}' logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ eval 'echo $IMAGE_BACKREST' logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++++ echo perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ var_value=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == docker.io/* ]] logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == percona/* ]] logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == perconalab/* ]] logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ echo IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ echo perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ eval 'echo $IMAGE' logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++++ echo perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ var_value=perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 == docker.io/* ]] logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 == percona/* ]] logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 == perconalab/* ]] logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ new_value=docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ echo IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ eval 'echo $IMAGE_UPGRADE' logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++++ echo perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ var_value=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == docker.io/* ]] logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == percona/* ]] logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == perconalab/* ]] logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ echo IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ eval 'echo $IMAGE_LOGCOLLECTOR' logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++++ echo perconalab/fluentbit:main-logcollector logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ var_value=perconalab/fluentbit:main-logcollector logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/fluentbit:main-logcollector == docker.io/* ]] logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/fluentbit:main-logcollector == percona/* ]] logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/fluentbit:main-logcollector == perconalab/* ]] logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ new_value=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ echo IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ eval 'echo $IMAGE_BASE' logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++++ echo perconalab/percona-postgresql-operator logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ var_value=perconalab/percona-postgresql-operator logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/percona-postgresql-operator == docker.io/* ]] logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/percona-postgresql-operator == percona/* ]] logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/percona-postgresql-operator == perconalab/* ]] logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ new_value=docker.io/perconalab/percona-postgresql-operator logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ echo IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ eval 'echo $IMAGE_PGBOUNCER' logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++++ echo perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ var_value=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == docker.io/* ]] logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == percona/* ]] logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == perconalab/* ]] logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ echo IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ eval 'echo $IMAGE_POSTGRESQL' logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++++ echo perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ var_value=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == docker.io/* ]] logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == percona/* ]] logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == perconalab/* ]] logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ echo IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ eval 'echo $IMAGE_PMM_SERVER' logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++++ echo perconalab/pmm-server:3-dev-latest logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ var_value=perconalab/pmm-server:3-dev-latest logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/pmm-server:3-dev-latest == docker.io/* ]] logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/pmm-server:3-dev-latest == percona/* ]] logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/pmm-server:3-dev-latest == perconalab/* ]] logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ new_value=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ echo IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ eval 'echo $IMAGE_PMM_CLIENT' logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++++ echo perconalab/pmm-client:3-dev-latest logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ var_value=perconalab/pmm-client:3-dev-latest logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/pmm-client:3-dev-latest == docker.io/* ]] logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/pmm-client:3-dev-latest == percona/* ]] logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/pmm-client:3-dev-latest == perconalab/* ]] logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ new_value=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ echo IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600 logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ which gdate logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | which: no gdate in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1727/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ which date logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ date=/usr/sbin/date logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ which gsed logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | which: no gsed in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1727/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ which sed logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ sed=/usr/sbin/sed logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + remove_all_finalizers logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + resource_types=("pg-restore" "pg-backup" "pg") logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + for resource in "${resource_types[@]}" logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + echo 'removing all finalizers for pg-restore resources' logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | removing all finalizers for pg-restore resources logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + kubectl -n kuttl-test-intent-zebra get pg-restore -o json logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + jq '.items[] | .metadata.name' -r logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + IFS= logger.go:42: 20:46:21 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + read -r name logger.go:42: 20:46:22 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + for resource in "${resource_types[@]}" logger.go:42: 20:46:22 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + echo 'removing all finalizers for pg-backup resources' logger.go:42: 20:46:22 | major-upgrade-logical-replica/99-remove-cluster-gracefully | removing all finalizers for pg-backup resources logger.go:42: 20:46:22 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + kubectl -n kuttl-test-intent-zebra get pg-backup -o json logger.go:42: 20:46:22 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + jq '.items[] | .metadata.name' -r logger.go:42: 20:46:22 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + IFS= logger.go:42: 20:46:22 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + read -r name logger.go:42: 20:46:22 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++ kubectl -n kuttl-test-intent-zebra get pg-backup major-upgrade-logical-replica-backup-6dm9-pk7h9 -o yaml logger.go:42: 20:46:22 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++ yq '.metadata.finalizers | length' logger.go:42: 20:46:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + [[ 0 == 0 ]] logger.go:42: 20:46:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + continue logger.go:42: 20:46:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + IFS= logger.go:42: 20:46:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + read -r name logger.go:42: 20:46:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++ kubectl -n kuttl-test-intent-zebra get pg-backup major-upgrade-logical-replica-backup-ngzv-bvggz -o yaml logger.go:42: 20:46:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++ yq '.metadata.finalizers | length' logger.go:42: 20:46:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + [[ 0 == 0 ]] logger.go:42: 20:46:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + continue logger.go:42: 20:46:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + IFS= logger.go:42: 20:46:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + read -r name logger.go:42: 20:46:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++ kubectl -n kuttl-test-intent-zebra get pg-backup major-upgrade-logical-replica-reseed-repo1 -o yaml logger.go:42: 20:46:23 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++ yq '.metadata.finalizers | length' logger.go:42: 20:46:24 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + [[ 0 == 0 ]] logger.go:42: 20:46:24 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + continue logger.go:42: 20:46:24 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + IFS= logger.go:42: 20:46:24 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + read -r name logger.go:42: 20:46:24 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++ kubectl -n kuttl-test-intent-zebra get pg-backup major-upgrade-logical-replica-seed-repo1 -o yaml logger.go:42: 20:46:24 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++ yq '.metadata.finalizers | length' logger.go:42: 20:46:24 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + [[ 0 == 0 ]] logger.go:42: 20:46:24 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + continue logger.go:42: 20:46:24 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + IFS= logger.go:42: 20:46:24 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + read -r name logger.go:42: 20:46:24 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + for resource in "${resource_types[@]}" logger.go:42: 20:46:24 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + echo 'removing all finalizers for pg resources' logger.go:42: 20:46:24 | major-upgrade-logical-replica/99-remove-cluster-gracefully | removing all finalizers for pg resources logger.go:42: 20:46:24 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + kubectl -n kuttl-test-intent-zebra get pg -o json logger.go:42: 20:46:24 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + jq '.items[] | .metadata.name' -r logger.go:42: 20:46:24 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + IFS= logger.go:42: 20:46:24 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + read -r name logger.go:42: 20:46:25 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + check_operator_panic logger.go:42: 20:46:25 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++ get_operator_pod logger.go:42: 20:46:25 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ kubectl get pods -n pg-operator --selector=app.kubernetes.io/name=percona-postgresql-operator -o 'jsonpath={.items[].metadata.name}' logger.go:42: 20:46:25 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++ echo percona-postgresql-operator-6b8c4d8b4-cprtq logger.go:42: 20:46:25 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + local operator_pod=percona-postgresql-operator-6b8c4d8b4-cprtq logger.go:42: 20:46:25 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + local panic_log logger.go:42: 20:46:25 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++ kubectl logs -n pg-operator percona-postgresql-operator-6b8c4d8b4-cprtq -c operator logger.go:42: 20:46:25 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++ grep -A 100 'Observed a panic' logger.go:42: 20:46:27 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++ true logger.go:42: 20:46:27 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + panic_log= logger.go:42: 20:46:27 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + '[' -n '' ']' logger.go:42: 20:46:27 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + destroy_operator logger.go:42: 20:46:27 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + kubectl -n pg-operator delete deployment percona-postgresql-operator --force --grace-period=0 logger.go:42: 20:46:27 | major-upgrade-logical-replica/99-remove-cluster-gracefully | Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. logger.go:42: 20:46:27 | major-upgrade-logical-replica/99-remove-cluster-gracefully | deployment.apps "percona-postgresql-operator" force deleted from pg-operator namespace logger.go:42: 20:46:27 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + [[ -n pg-operator ]] logger.go:42: 20:46:27 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + kubectl delete namespace pg-operator --force --grace-period=0 --ignore-not-found logger.go:42: 20:46:27 | major-upgrade-logical-replica/99-remove-cluster-gracefully | Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. logger.go:42: 20:46:28 | major-upgrade-logical-replica/99-remove-cluster-gracefully | namespace "pg-operator" force deleted logger.go:42: 20:46:40 | major-upgrade-logical-replica/99-remove-cluster-gracefully | test step completed 99-remove-cluster-gracefully logger.go:42: 20:46:40 | major-upgrade-logical-replica | major-upgrade-logical-replica events from ns kuttl-test-intent-zebra: logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:01 +0000 UTC Normal Pod pg-client-d7c7bb45b-8nwnb Binding Scheduled Successfully assigned kuttl-test-intent-zebra/pg-client-d7c7bb45b-8nwnb to gke-jen-pg-1727-5c0d1c60-default-pool-dbebd946-36k6 default-scheduler logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:01 +0000 UTC Normal ReplicaSet.apps pg-client-d7c7bb45b SuccessfulCreate Created pod: pg-client-d7c7bb45b-8nwnb replicaset-controller logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:01 +0000 UTC Normal Deployment.apps pg-client ScalingReplicaSet Scaled up replica set pg-client-d7c7bb45b from 0 to 1 deployment-controller logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:02 +0000 UTC Normal Pod pg-client-d7c7bb45b-8nwnb.spec.containers{pg-client} Pulling Pulling image "perconalab/percona-distribution-postgresql:16" kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:02 +0000 UTC Normal Pod pg-client-d7c7bb45b-8nwnb.spec.containers{pg-client} Pulled Successfully pulled image "perconalab/percona-distribution-postgresql:16" in 141ms (141ms including waiting). Image size: 445181537 bytes. kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:02 +0000 UTC Normal Pod pg-client-d7c7bb45b-8nwnb.spec.containers{pg-client} Created Created container: pg-client kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:02 +0000 UTC Normal Pod pg-client-d7c7bb45b-8nwnb.spec.containers{pg-client} Started Started container pg-client kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:12 +0000 UTC Normal PersistentVolumeClaim major-upgrade-logical-replica-instance1-98gr-pgdata WaitForFirstConsumer waiting for first consumer to be created before binding persistentvolume-controller logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:12 +0000 UTC Normal PersistentVolumeClaim major-upgrade-logical-replica-instance1-98gr-pgdata ExternalProvisioning Waiting for a volume to be created either by the external provisioner 'pd.csi.storage.gke.io' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. persistentvolume-controller logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:12 +0000 UTC Normal PersistentVolumeClaim major-upgrade-logical-replica-instance1-98gr-pgdata Provisioning External provisioner is provisioning volume for claim "kuttl-test-intent-zebra/major-upgrade-logical-replica-instance1-98gr-pgdata" pd.csi.storage.gke.io_gke-ecf57bd162d444b6af7c-67c8-40b8-vm_b1f31345-bea8-4cb3-b226-8a4f71e8820f logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:12 +0000 UTC Normal StatefulSet.apps major-upgrade-logical-replica-instance1-98gr SuccessfulCreate create Pod major-upgrade-logical-replica-instance1-98gr-0 in StatefulSet major-upgrade-logical-replica-instance1-98gr successful statefulset-controller logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:12 +0000 UTC Normal PersistentVolumeClaim major-upgrade-logical-replica-instance1-n9h5-pgdata WaitForFirstConsumer waiting for first consumer to be created before binding persistentvolume-controller logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:12 +0000 UTC Normal StatefulSet.apps major-upgrade-logical-replica-instance1-n9h5 SuccessfulCreate create Pod major-upgrade-logical-replica-instance1-n9h5-0 in StatefulSet major-upgrade-logical-replica-instance1-n9h5 successful statefulset-controller logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:13 +0000 UTC Normal PersistentVolumeClaim major-upgrade-logical-replica-instance1-n9h5-pgdata ExternalProvisioning Waiting for a volume to be created either by the external provisioner 'pd.csi.storage.gke.io' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. persistentvolume-controller logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:13 +0000 UTC Normal PersistentVolumeClaim major-upgrade-logical-replica-instance1-n9h5-pgdata Provisioning External provisioner is provisioning volume for claim "kuttl-test-intent-zebra/major-upgrade-logical-replica-instance1-n9h5-pgdata" pd.csi.storage.gke.io_gke-ecf57bd162d444b6af7c-67c8-40b8-vm_b1f31345-bea8-4cb3-b226-8a4f71e8820f logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:13 +0000 UTC Normal StatefulSet.apps major-upgrade-logical-replica-repo-host SuccessfulCreate create Pod major-upgrade-logical-replica-repo-host-0 in StatefulSet major-upgrade-logical-replica-repo-host successful statefulset-controller logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:13 +0000 UTC Normal PersistentVolumeClaim major-upgrade-logical-replica-repo1 WaitForFirstConsumer waiting for first consumer to be created before binding persistentvolume-controller logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:13 +0000 UTC Normal PersistentVolumeClaim major-upgrade-logical-replica-repo1 ExternalProvisioning Waiting for a volume to be created either by the external provisioner 'pd.csi.storage.gke.io' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. persistentvolume-controller logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:13 +0000 UTC Normal PersistentVolumeClaim major-upgrade-logical-replica-repo1 Provisioning External provisioner is provisioning volume for claim "kuttl-test-intent-zebra/major-upgrade-logical-replica-repo1" pd.csi.storage.gke.io_gke-ecf57bd162d444b6af7c-67c8-40b8-vm_b1f31345-bea8-4cb3-b226-8a4f71e8820f logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:13 +0000 UTC Normal PostgresCluster.upstream.pgv2.percona.com major-upgrade-logical-replica RepoHostCreated created pgBackRest repository host StatefulSet/major-upgrade-logical-replica-repo-host postgrescluster-controller logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:14 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-98gr-0 Binding Scheduled Successfully assigned kuttl-test-intent-zebra/major-upgrade-logical-replica-instance1-98gr-0 to gke-jen-pg-1727-5c0d1c60-default-pool-dbebd946-36k6 default-scheduler logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:14 +0000 UTC Normal PersistentVolumeClaim major-upgrade-logical-replica-instance1-98gr-pgdata ProvisioningSucceeded Successfully provisioned volume pvc-8e3d75c8-ffc4-48db-8334-60bc400a90bf pd.csi.storage.gke.io_gke-ecf57bd162d444b6af7c-67c8-40b8-vm_b1f31345-bea8-4cb3-b226-8a4f71e8820f logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:14 +0000 UTC Normal PersistentVolumeClaim major-upgrade-logical-replica-instance1-n9h5-pgdata ProvisioningSucceeded Successfully provisioned volume pvc-48304494-39e0-4a65-b580-d56f472afcf1 pd.csi.storage.gke.io_gke-ecf57bd162d444b6af7c-67c8-40b8-vm_b1f31345-bea8-4cb3-b226-8a4f71e8820f logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:14 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-55767dc549-tj5zg Binding Scheduled Successfully assigned kuttl-test-intent-zebra/major-upgrade-logical-replica-pgbouncer-55767dc549-tj5zg to gke-jen-pg-1727-5c0d1c60-default-pool-dbebd946-svlt default-scheduler logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:14 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-55767dc549-tj5zg.spec.initContainers{pgbouncer-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600" kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:14 +0000 UTC Normal ReplicaSet.apps major-upgrade-logical-replica-pgbouncer-55767dc549 SuccessfulCreate Created pod: major-upgrade-logical-replica-pgbouncer-55767dc549-tj5zg replicaset-controller logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:14 +0000 UTC Normal Deployment.apps major-upgrade-logical-replica-pgbouncer ScalingReplicaSet Scaled up replica set major-upgrade-logical-replica-pgbouncer-55767dc549 from 0 to 1 deployment-controller logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:15 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-n9h5-0 Binding Scheduled Successfully assigned kuttl-test-intent-zebra/major-upgrade-logical-replica-instance1-n9h5-0 to gke-jen-pg-1727-5c0d1c60-default-pool-dbebd946-7mdw default-scheduler logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:15 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-55767dc549-tj5zg.spec.initContainers{pgbouncer-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600" in 156ms (156ms including waiting). Image size: 87061965 bytes. kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:15 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-55767dc549-tj5zg.spec.initContainers{pgbouncer-init} Created Created container: pgbouncer-init kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:15 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-55767dc549-tj5zg.spec.initContainers{pgbouncer-init} Started Started container pgbouncer-init kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:15 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0 Binding Scheduled Successfully assigned kuttl-test-intent-zebra/major-upgrade-logical-replica-repo-host-0 to gke-jen-pg-1727-5c0d1c60-default-pool-dbebd946-svlt default-scheduler logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:15 +0000 UTC Normal PersistentVolumeClaim major-upgrade-logical-replica-repo1 ProvisioningSucceeded Successfully provisioned volume pvc-cf88d012-9654-493f-938e-8e50aaf10ccc pd.csi.storage.gke.io_gke-ecf57bd162d444b6af7c-67c8-40b8-vm_b1f31345-bea8-4cb3-b226-8a4f71e8820f logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:16 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-55767dc549-tj5zg.spec.containers{pgbouncer} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbouncer17" kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:18 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-55767dc549-tj5zg.spec.containers{pgbouncer} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbouncer17" in 2.559s (2.559s including waiting). Image size: 84606212 bytes. kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:18 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-55767dc549-tj5zg.spec.containers{pgbouncer} Created Created container: pgbouncer kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:18 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-55767dc549-tj5zg.spec.containers{pgbouncer} Started Started container pgbouncer kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:19 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-55767dc549-tj5zg.spec.containers{pgbouncer-config} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbouncer17" kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:19 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-55767dc549-tj5zg.spec.containers{pgbouncer-config} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbouncer17" in 148ms (148ms including waiting). Image size: 84606212 bytes. kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:19 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-55767dc549-tj5zg.spec.containers{pgbouncer-config} Created Created container: pgbouncer-config kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:19 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-55767dc549-tj5zg.spec.containers{pgbouncer-config} Started Started container pgbouncer-config kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:20 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-n9h5-0 SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-48304494-39e0-4a65-b580-d56f472afcf1" attachdetach-controller logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:20 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0 SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-cf88d012-9654-493f-938e-8e50aaf10ccc" attachdetach-controller logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:21 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-98gr-0 SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-8e3d75c8-ffc4-48db-8334-60bc400a90bf" attachdetach-controller logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:21 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-n9h5-0.spec.initContainers{database-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600" kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:21 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-n9h5-0.spec.initContainers{database-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600" in 185ms (185ms including waiting). Image size: 87061965 bytes. kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:21 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-n9h5-0.spec.initContainers{database-init} Created Created container: database-init kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:21 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-n9h5-0.spec.initContainers{database-init} Started Started container database-init kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:22 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.initContainers{pgbackrest-log-dir} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest17" kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:23 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-n9h5-0.spec.initContainers{postgres-startup} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:24 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-98gr-0.spec.initContainers{database-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600" kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:24 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-98gr-0.spec.initContainers{database-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600" in 138ms (138ms including waiting). Image size: 87061965 bytes. kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:24 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-98gr-0.spec.initContainers{database-init} Created Created container: database-init kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:25 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-98gr-0.spec.initContainers{database-init} Started Started container database-init kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:25 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-98gr-0.spec.initContainers{postgres-startup} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:26 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.initContainers{pgbackrest-log-dir} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest17" in 4.481s (4.481s including waiting). Image size: 122686994 bytes. kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:26 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.initContainers{pgbackrest-log-dir} Created Created container: pgbackrest-log-dir kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:26 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.initContainers{pgbackrest-log-dir} Started Started container pgbackrest-log-dir kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:30 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.initContainers{nss-wrapper-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest17" kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:30 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.initContainers{nss-wrapper-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest17" in 100ms (100ms including waiting). Image size: 122686994 bytes. kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:30 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.initContainers{nss-wrapper-init} Created Created container: nss-wrapper-init kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:30 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.initContainers{nss-wrapper-init} Started Started container nss-wrapper-init kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:31 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.containers{pgbackrest} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest17" kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:31 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.containers{pgbackrest} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest17" in 106ms (106ms including waiting). Image size: 122686994 bytes. kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:31 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.containers{pgbackrest} Created Created container: pgbackrest kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:31 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.containers{pgbackrest} Started Started container pgbackrest kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:31 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.containers{pgbackrest-config} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest17" kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:31 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.containers{pgbackrest-config} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest17" in 107ms (107ms including waiting). Image size: 122686994 bytes. kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:31 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.containers{pgbackrest-config} Created Created container: pgbackrest-config kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:31 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.containers{pgbackrest-config} Started Started container pgbackrest-config kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:39 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-n9h5-0.spec.initContainers{postgres-startup} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" in 16.915s (16.915s including waiting). Image size: 447509907 bytes. kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:40 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-n9h5-0.spec.initContainers{postgres-startup} Created Created container: postgres-startup kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:40 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-n9h5-0.spec.initContainers{postgres-startup} Started Started container postgres-startup kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:41 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-98gr-0.spec.initContainers{postgres-startup} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" in 16.22s (16.22s including waiting). Image size: 447509907 bytes. kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:41 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-98gr-0.spec.initContainers{postgres-startup} Created Created container: postgres-startup kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:42 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-98gr-0.spec.initContainers{postgres-startup} Started Started container postgres-startup kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:48 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-n9h5-0.spec.initContainers{nss-wrapper-init-logcollector} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:48 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-n9h5-0.spec.initContainers{nss-wrapper-init-logcollector} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" in 117ms (117ms including waiting). Image size: 447509907 bytes. kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:48 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-n9h5-0.spec.initContainers{nss-wrapper-init-logcollector} Created Created container: nss-wrapper-init-logcollector kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:48 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-n9h5-0.spec.initContainers{nss-wrapper-init-logcollector} Started Started container nss-wrapper-init-logcollector kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:49 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-98gr-0.spec.initContainers{nss-wrapper-init-logcollector} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:49 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-98gr-0.spec.initContainers{nss-wrapper-init-logcollector} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" in 129ms (129ms including waiting). Image size: 447509907 bytes. kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:49 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-98gr-0.spec.initContainers{nss-wrapper-init-logcollector} Created Created container: nss-wrapper-init-logcollector kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:49 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-n9h5-0.spec.initContainers{nss-wrapper-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:49 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-n9h5-0.spec.initContainers{nss-wrapper-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" in 144ms (144ms including waiting). Image size: 447509907 bytes. kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:49 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-n9h5-0.spec.initContainers{nss-wrapper-init} Created Created container: nss-wrapper-init kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:49 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-n9h5-0.spec.initContainers{nss-wrapper-init} Started Started container nss-wrapper-init kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:50 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-98gr-0.spec.initContainers{nss-wrapper-init-logcollector} Started Started container nss-wrapper-init-logcollector kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:50 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-98gr-0.spec.initContainers{nss-wrapper-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:50 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-98gr-0.spec.initContainers{nss-wrapper-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" in 191ms (191ms including waiting). Image size: 447509907 bytes. kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:50 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-n9h5-0.spec.containers{database} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:50 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-n9h5-0.spec.containers{database} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" in 120ms (120ms including waiting). Image size: 447509907 bytes. kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:50 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-n9h5-0.spec.containers{database} Created Created container: database kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:50 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-n9h5-0.spec.containers{database} Started Started container database kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:50 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-n9h5-0.spec.containers{replication-cert-copy} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:50 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-n9h5-0.spec.containers{replication-cert-copy} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" in 124ms (124ms including waiting). Image size: 447509907 bytes. kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:50 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-n9h5-0.spec.containers{replication-cert-copy} Created Created container: replication-cert-copy kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:50 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-n9h5-0.spec.containers{replication-cert-copy} Started Started container replication-cert-copy kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:50 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-n9h5-0.spec.containers{logs} Pulled Container image "docker.io/perconalab/fluentbit:main-logcollector" already present on machine kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:51 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-98gr-0.spec.initContainers{nss-wrapper-init} Created Created container: nss-wrapper-init kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:51 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-98gr-0.spec.initContainers{nss-wrapper-init} Started Started container nss-wrapper-init kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:51 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-98gr-0.spec.containers{database} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:51 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-98gr-0.spec.containers{database} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" in 166ms (166ms including waiting). Image size: 447509907 bytes. kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:52 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-98gr-0.spec.containers{database} Created Created container: database kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:52 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-98gr-0.spec.containers{database} Started Started container database kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:52 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-98gr-0.spec.containers{replication-cert-copy} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:52 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-98gr-0.spec.containers{replication-cert-copy} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" in 165ms (165ms including waiting). Image size: 447509907 bytes. kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:52 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-98gr-0.spec.containers{replication-cert-copy} Created Created container: replication-cert-copy kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:52 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-98gr-0.spec.containers{replication-cert-copy} Started Started container replication-cert-copy kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:52 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-98gr-0.spec.containers{logs} Pulled Container image "docker.io/perconalab/fluentbit:main-logcollector" already present on machine kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:55 +0000 UTC Normal PostgresCluster.upstream.pgv2.percona.com major-upgrade-logical-replica StanzasCreated pgBackRest stanza creation completed successfully postgrescluster-controller logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:56 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-6dm9-pwgmt Binding Scheduled Successfully assigned kuttl-test-intent-zebra/major-upgrade-logical-replica-backup-6dm9-pwgmt to gke-jen-pg-1727-5c0d1c60-default-pool-dbebd946-7mdw default-scheduler logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:56 +0000 UTC Normal Job.batch major-upgrade-logical-replica-backup-6dm9 SuccessfulCreate Created pod: major-upgrade-logical-replica-backup-6dm9-pwgmt job-controller logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:57 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-6dm9-pwgmt.spec.initContainers{pgbackrest-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600" kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:57 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-6dm9-pwgmt.spec.initContainers{pgbackrest-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600" in 138ms (138ms including waiting). Image size: 87061965 bytes. kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:57 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-6dm9-pwgmt.spec.initContainers{pgbackrest-init} Created Created container: pgbackrest-init kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:37:57 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-6dm9-pwgmt.spec.initContainers{pgbackrest-init} Started Started container pgbackrest-init kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:38:00 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-6dm9-pwgmt.spec.containers{pgbackrest} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest17" kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:38:00 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-6dm9-pwgmt.spec.containers{pgbackrest} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest17" in 129ms (129ms including waiting). Image size: 122686994 bytes. kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:38:00 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-6dm9-pwgmt.spec.containers{pgbackrest} Created Created container: pgbackrest kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:38:00 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-6dm9-pwgmt.spec.containers{pgbackrest} Started Started container pgbackrest kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:38:14 +0000 UTC Normal Job.batch major-upgrade-logical-replica-backup-6dm9 Completed Job completed job-controller logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:38:27 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-7p22-td7xg Binding Scheduled Successfully assigned kuttl-test-intent-zebra/major-upgrade-logical-replica-backup-7p22-td7xg to gke-jen-pg-1727-5c0d1c60-default-pool-dbebd946-36k6 default-scheduler logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:38:27 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-7p22-td7xg.spec.initContainers{pgbackrest-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600" kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:38:27 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-7p22-td7xg.spec.initContainers{pgbackrest-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600" in 200ms (200ms including waiting). Image size: 87061965 bytes. kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:38:27 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-7p22-td7xg.spec.initContainers{pgbackrest-init} Created Created container: pgbackrest-init kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:38:27 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-7p22-td7xg.spec.initContainers{pgbackrest-init} Started Started container pgbackrest-init kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:38:27 +0000 UTC Normal Job.batch major-upgrade-logical-replica-backup-7p22 SuccessfulCreate Created pod: major-upgrade-logical-replica-backup-7p22-td7xg job-controller logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:38:28 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-7p22-td7xg.spec.containers{pgbackrest} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest17" kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:38:29 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-7p22-td7xg.spec.containers{pgbackrest} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest17" in 105ms (105ms including waiting). Image size: 122686994 bytes. kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:38:29 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-7p22-td7xg.spec.containers{pgbackrest} Created Created container: pgbackrest kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:38:29 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-7p22-td7xg.spec.containers{pgbackrest} Started Started container pgbackrest kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:40:21 +0000 UTC Normal Job.batch major-upgrade-logical-replica-backup-7p22 Completed Job completed job-controller logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:40:47 +0000 UTC Normal PersistentVolumeClaim major-upgrade-logical-replica-lr-analytics-pgdata WaitForFirstConsumer waiting for first consumer to be created before binding persistentvolume-controller logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:40:48 +0000 UTC Normal Job.batch major-upgrade-logical-replica-lr-analytics-bootstrap SuccessfulCreate Created pod: major-upgrade-logical-replica-lr-analytics-bootstrap-r9m66 job-controller logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:40:48 +0000 UTC Normal PersistentVolumeClaim major-upgrade-logical-replica-lr-analytics-pgdata ExternalProvisioning Waiting for a volume to be created either by the external provisioner 'pd.csi.storage.gke.io' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. persistentvolume-controller logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:40:48 +0000 UTC Normal PersistentVolumeClaim major-upgrade-logical-replica-lr-analytics-pgdata Provisioning External provisioner is provisioning volume for claim "kuttl-test-intent-zebra/major-upgrade-logical-replica-lr-analytics-pgdata" pd.csi.storage.gke.io_gke-ecf57bd162d444b6af7c-67c8-40b8-vm_b1f31345-bea8-4cb3-b226-8a4f71e8820f logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:40:49 +0000 UTC Normal PersistentVolumeClaim major-upgrade-logical-replica-lr-analytics-pgdata ProvisioningSucceeded Successfully provisioned volume pvc-ee07f684-80bb-44b5-84ef-90d2c0221bf7 pd.csi.storage.gke.io_gke-ecf57bd162d444b6af7c-67c8-40b8-vm_b1f31345-bea8-4cb3-b226-8a4f71e8820f logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:40:50 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-r9m66 Binding Scheduled Successfully assigned kuttl-test-intent-zebra/major-upgrade-logical-replica-lr-analytics-bootstrap-r9m66 to gke-jen-pg-1727-5c0d1c60-default-pool-dbebd946-36k6 default-scheduler logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:40:58 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-r9m66 SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-ee07f684-80bb-44b5-84ef-90d2c0221bf7" attachdetach-controller logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:40:59 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-r9m66.spec.initContainers{database-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600" kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:40:59 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-r9m66.spec.initContainers{database-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600" in 284ms (284ms including waiting). Image size: 87061965 bytes. kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:40:59 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-r9m66.spec.initContainers{database-init} Created Created container: database-init kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:41:00 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-r9m66.spec.initContainers{database-init} Started Started container database-init kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:41:00 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-r9m66.spec.initContainers{nss-wrapper-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:41:00 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-r9m66.spec.initContainers{nss-wrapper-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" in 155ms (155ms including waiting). Image size: 447509907 bytes. kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:41:00 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-r9m66.spec.initContainers{nss-wrapper-init} Created Created container: nss-wrapper-init kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:41:00 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-r9m66.spec.initContainers{nss-wrapper-init} Started Started container nss-wrapper-init kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:41:01 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-r9m66.spec.containers{logical-replica-bootstrap} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:41:01 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-r9m66.spec.containers{logical-replica-bootstrap} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" in 136ms (136ms including waiting). Image size: 447509907 bytes. kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:41:01 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-r9m66.spec.containers{logical-replica-bootstrap} Created Created container: logical-replica-bootstrap kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:41:01 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-r9m66.spec.containers{logical-replica-bootstrap} Started Started container logical-replica-bootstrap kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:41:18 +0000 UTC Normal Job.batch major-upgrade-logical-replica-lr-analytics-bootstrap Completed Job completed job-controller logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:41:19 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0 Binding Scheduled Successfully assigned kuttl-test-intent-zebra/major-upgrade-logical-replica-lr-analytics-0 to gke-jen-pg-1727-5c0d1c60-default-pool-dbebd946-36k6 default-scheduler logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:41:19 +0000 UTC Normal StatefulSet.apps major-upgrade-logical-replica-lr-analytics SuccessfulCreate create Pod major-upgrade-logical-replica-lr-analytics-0 in StatefulSet major-upgrade-logical-replica-lr-analytics successful statefulset-controller logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:41:28 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0 SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-ee07f684-80bb-44b5-84ef-90d2c0221bf7" attachdetach-controller logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:41:30 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0.spec.initContainers{nss-wrapper-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:41:30 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0.spec.initContainers{nss-wrapper-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" in 202ms (202ms including waiting). Image size: 447509907 bytes. kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:41:30 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0.spec.initContainers{nss-wrapper-init} Created Created container: nss-wrapper-init kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:41:30 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0.spec.initContainers{nss-wrapper-init} Started Started container nss-wrapper-init kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:41:31 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0.spec.containers{database} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:41:31 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0.spec.containers{database} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" in 140ms (140ms including waiting). Image size: 447509907 bytes. kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:41:31 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0.spec.containers{database} Created Created container: database kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:41:31 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0.spec.containers{database} Started Started container database kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:41:56 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0.spec.containers{database} Killing Stopping container database kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:41:56 +0000 UTC Normal StatefulSet.apps major-upgrade-logical-replica-lr-analytics SuccessfulDelete delete Pod major-upgrade-logical-replica-lr-analytics-0 in StatefulSet major-upgrade-logical-replica-lr-analytics successful statefulset-controller logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:41:57 +0000 UTC Normal StatefulSet.apps major-upgrade-logical-replica-instance1-98gr SuccessfulDelete delete Pod major-upgrade-logical-replica-instance1-98gr-0 in StatefulSet major-upgrade-logical-replica-instance1-98gr successful statefulset-controller logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:41:58 +0000 UTC Normal Deployment.apps major-upgrade-logical-replica-pgbouncer ScalingReplicaSet Scaled down replica set major-upgrade-logical-replica-pgbouncer-55767dc549 from 1 to 0 deployment-controller logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:41:59 +0000 UTC Warning Pod major-upgrade-logical-replica-instance1-98gr-0.spec.containers{database} Unhealthy Readiness probe failed: kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:41:59 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-55767dc549-tj5zg.spec.containers{pgbouncer} Killing Stopping container pgbouncer kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:41:59 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-55767dc549-tj5zg.spec.containers{pgbouncer-config} Killing Stopping container pgbouncer-config kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:41:59 +0000 UTC Normal ReplicaSet.apps major-upgrade-logical-replica-pgbouncer-55767dc549 SuccessfulDelete Deleted pod: major-upgrade-logical-replica-pgbouncer-55767dc549-tj5zg replicaset-controller logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:42:03 +0000 UTC Normal StatefulSet.apps major-upgrade-logical-replica-instance1-n9h5 SuccessfulDelete delete Pod major-upgrade-logical-replica-instance1-n9h5-0 in StatefulSet major-upgrade-logical-replica-instance1-n9h5 successful statefulset-controller logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:42:06 +0000 UTC Normal PodDisruptionBudget.policy major-upgrade-logical-replica-set-instance1 NoPods No matching pods found controllermanager logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:42:09 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.containers{pgbackrest} Killing Stopping container pgbackrest kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:42:09 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.containers{pgbackrest-config} Killing Stopping container pgbackrest-config kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:42:09 +0000 UTC Normal StatefulSet.apps major-upgrade-logical-replica-repo-host SuccessfulDelete delete Pod major-upgrade-logical-replica-repo-host-0 in StatefulSet major-upgrade-logical-replica-repo-host successful statefulset-controller logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:42:11 +0000 UTC Normal Pod 17-to-18-pgdata-8xjv4 Binding Scheduled Successfully assigned kuttl-test-intent-zebra/17-to-18-pgdata-8xjv4 to gke-jen-pg-1727-5c0d1c60-default-pool-dbebd946-7mdw default-scheduler logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:42:11 +0000 UTC Normal Job.batch 17-to-18-pgdata SuccessfulCreate Created pod: 17-to-18-pgdata-8xjv4 job-controller logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:42:17 +0000 UTC Normal Pod 17-to-18-pgdata-8xjv4.spec.initContainers{database-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600" kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:42:17 +0000 UTC Normal Pod 17-to-18-pgdata-8xjv4.spec.initContainers{database-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600" in 131ms (131ms including waiting). Image size: 87061965 bytes. kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:42:17 +0000 UTC Normal Pod 17-to-18-pgdata-8xjv4.spec.initContainers{database-init} Created Created container: database-init kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:42:17 +0000 UTC Normal Pod 17-to-18-pgdata-8xjv4.spec.initContainers{database-init} Started Started container database-init kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:42:18 +0000 UTC Normal Pod 17-to-18-pgdata-8xjv4.spec.containers{database} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-upgrade" kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:42:48 +0000 UTC Normal Pod 17-to-18-pgdata-8xjv4.spec.containers{database} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-upgrade" in 30.8s (30.8s including waiting). Image size: 952996708 bytes. kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:42:49 +0000 UTC Normal Pod 17-to-18-pgdata-8xjv4.spec.containers{database} Created Created container: database kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:42:49 +0000 UTC Normal Pod 17-to-18-pgdata-8xjv4.spec.containers{database} Started Started container database kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:05 +0000 UTC Normal Pod 17-to-18-major-upgrade-logical-replica-instance1-98gr-n6zp2 Binding Scheduled Successfully assigned kuttl-test-intent-zebra/17-to-18-major-upgrade-logical-replica-instance1-98gr-n6zp2 to gke-jen-pg-1727-5c0d1c60-default-pool-dbebd946-7mdw default-scheduler logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:05 +0000 UTC Normal Job.batch 17-to-18-major-upgrade-logical-replica-instance1-98gr SuccessfulCreate Created pod: 17-to-18-major-upgrade-logical-replica-instance1-98gr-n6zp2 job-controller logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:05 +0000 UTC Normal Job.batch 17-to-18-pgdata Completed Job completed job-controller logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:13 +0000 UTC Normal Pod 17-to-18-major-upgrade-logical-replica-instance1-98gr-n6zp2 SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-8e3d75c8-ffc4-48db-8334-60bc400a90bf" attachdetach-controller logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:14 +0000 UTC Normal Pod 17-to-18-major-upgrade-logical-replica-instance1-98gr-n6zp2.spec.containers{database} Pulled Container image "docker.io/perconalab/percona-postgresql-operator:main-upgrade" already present on machine kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:14 +0000 UTC Normal Pod 17-to-18-major-upgrade-logical-replica-instance1-98gr-n6zp2.spec.containers{database} Created Created container: database kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:14 +0000 UTC Normal Pod 17-to-18-major-upgrade-logical-replica-instance1-98gr-n6zp2.spec.containers{database} Started Started container database kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:17 +0000 UTC Normal Job.batch 17-to-18-major-upgrade-logical-replica-instance1-98gr Completed Job completed job-controller logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:24 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-n9h5-0 Binding Scheduled Successfully assigned kuttl-test-intent-zebra/major-upgrade-logical-replica-instance1-n9h5-0 to gke-jen-pg-1727-5c0d1c60-default-pool-dbebd946-7mdw default-scheduler logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:25 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-5b889f6889-q6cwr Binding Scheduled Successfully assigned kuttl-test-intent-zebra/major-upgrade-logical-replica-pgbouncer-5b889f6889-q6cwr to gke-jen-pg-1727-5c0d1c60-default-pool-dbebd946-svlt default-scheduler logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:25 +0000 UTC Normal ReplicaSet.apps major-upgrade-logical-replica-pgbouncer-5b889f6889 SuccessfulCreate Created pod: major-upgrade-logical-replica-pgbouncer-5b889f6889-q6cwr replicaset-controller logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:25 +0000 UTC Normal Deployment.apps major-upgrade-logical-replica-pgbouncer ScalingReplicaSet Scaled up replica set major-upgrade-logical-replica-pgbouncer-5b889f6889 from 0 to 1 deployment-controller logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:25 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0 Binding Scheduled Successfully assigned kuttl-test-intent-zebra/major-upgrade-logical-replica-repo-host-0 to gke-jen-pg-1727-5c0d1c60-default-pool-dbebd946-36k6 default-scheduler logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:26 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-5b889f6889-q6cwr.spec.initContainers{pgbouncer-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600" kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:26 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-5b889f6889-q6cwr.spec.initContainers{pgbouncer-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600" in 180ms (180ms including waiting). Image size: 87061965 bytes. kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:26 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-5b889f6889-q6cwr.spec.initContainers{pgbouncer-init} Created Created container: pgbouncer-init kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:26 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-5b889f6889-q6cwr.spec.initContainers{pgbouncer-init} Started Started container pgbouncer-init kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:27 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-5b889f6889-q6cwr.spec.containers{pgbouncer} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18" kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:27 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-5b889f6889-q6cwr.spec.containers{pgbouncer} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18" in 130ms (130ms including waiting). Image size: 84619748 bytes. kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:27 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-5b889f6889-q6cwr.spec.containers{pgbouncer} Created Created container: pgbouncer kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:27 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-5b889f6889-q6cwr.spec.containers{pgbouncer} Started Started container pgbouncer kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:27 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-5b889f6889-q6cwr.spec.containers{pgbouncer-config} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18" kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:27 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-5b889f6889-q6cwr.spec.containers{pgbouncer-config} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18" in 106ms (106ms including waiting). Image size: 84619748 bytes. kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:27 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-5b889f6889-q6cwr.spec.containers{pgbouncer-config} Created Created container: pgbouncer-config kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:28 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-5b889f6889-q6cwr.spec.containers{pgbouncer-config} Started Started container pgbouncer-config kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:29 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-n9h5-0 SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-48304494-39e0-4a65-b580-d56f472afcf1" attachdetach-controller logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:30 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-n9h5-0.spec.initContainers{database-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600" kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:30 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-n9h5-0.spec.initContainers{database-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600" in 147ms (147ms including waiting). Image size: 87061965 bytes. kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:30 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-n9h5-0.spec.initContainers{database-init} Created Created container: database-init kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:30 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-n9h5-0.spec.initContainers{database-init} Started Started container database-init kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:30 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0 SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-cf88d012-9654-493f-938e-8e50aaf10ccc" attachdetach-controller logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:31 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-n9h5-0.spec.initContainers{postgres-startup} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:31 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-n9h5-0.spec.initContainers{postgres-startup} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" in 190ms (190ms including waiting). Image size: 449178529 bytes. kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:31 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-n9h5-0.spec.initContainers{postgres-startup} Created Created container: postgres-startup kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:31 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-n9h5-0.spec.initContainers{postgres-startup} Started Started container postgres-startup kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:32 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-n9h5-0.spec.initContainers{nss-wrapper-init-logcollector} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:32 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-n9h5-0.spec.initContainers{nss-wrapper-init-logcollector} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" in 178ms (178ms including waiting). Image size: 449178529 bytes. kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:32 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-n9h5-0.spec.initContainers{nss-wrapper-init-logcollector} Created Created container: nss-wrapper-init-logcollector kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:32 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-n9h5-0.spec.initContainers{nss-wrapper-init-logcollector} Started Started container nss-wrapper-init-logcollector kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:32 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.initContainers{pgbackrest-log-dir} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18" kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:32 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.initContainers{pgbackrest-log-dir} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18" in 160ms (160ms including waiting). Image size: 122700307 bytes. kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:32 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.initContainers{pgbackrest-log-dir} Created Created container: pgbackrest-log-dir kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:32 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.initContainers{pgbackrest-log-dir} Started Started container pgbackrest-log-dir kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:32 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.initContainers{nss-wrapper-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18" kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:33 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-n9h5-0.spec.initContainers{nss-wrapper-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:33 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-n9h5-0.spec.initContainers{nss-wrapper-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" in 142ms (142ms including waiting). Image size: 449178529 bytes. kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:33 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-n9h5-0.spec.initContainers{nss-wrapper-init} Created Created container: nss-wrapper-init kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:33 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-n9h5-0.spec.initContainers{nss-wrapper-init} Started Started container nss-wrapper-init kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:33 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.initContainers{nss-wrapper-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18" in 117ms (118ms including waiting). Image size: 122700307 bytes. kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:33 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.initContainers{nss-wrapper-init} Created Created container: nss-wrapper-init kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:33 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.initContainers{nss-wrapper-init} Started Started container nss-wrapper-init kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:33 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.containers{pgbackrest} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18" kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:34 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-n9h5-0.spec.containers{database} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:34 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-n9h5-0.spec.containers{database} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" in 111ms (111ms including waiting). Image size: 449178529 bytes. kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:34 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-n9h5-0.spec.containers{database} Created Created container: database kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:34 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-n9h5-0.spec.containers{database} Started Started container database kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:34 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-n9h5-0.spec.containers{replication-cert-copy} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:34 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-n9h5-0.spec.containers{replication-cert-copy} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" in 113ms (113ms including waiting). Image size: 449178529 bytes. kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:34 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-n9h5-0.spec.containers{replication-cert-copy} Created Created container: replication-cert-copy kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:34 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.containers{pgbackrest} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18" in 116ms (116ms including waiting). Image size: 122700307 bytes. kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:34 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.containers{pgbackrest} Created Created container: pgbackrest kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:34 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.containers{pgbackrest} Started Started container pgbackrest kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:34 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.containers{pgbackrest-config} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18" kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:34 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.containers{pgbackrest-config} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18" in 104ms (104ms including waiting). Image size: 122700307 bytes. kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:34 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.containers{pgbackrest-config} Created Created container: pgbackrest-config kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:34 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.containers{pgbackrest-config} Started Started container pgbackrest-config kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:35 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-n9h5-0.spec.containers{replication-cert-copy} Started Started container replication-cert-copy kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:35 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-n9h5-0.spec.containers{logs} Pulled Container image "docker.io/perconalab/fluentbit:main-logcollector" already present on machine kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:41 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-98gr-0 Binding Scheduled Successfully assigned kuttl-test-intent-zebra/major-upgrade-logical-replica-instance1-98gr-0 to gke-jen-pg-1727-5c0d1c60-default-pool-dbebd946-svlt default-scheduler logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:43 +0000 UTC Warning PerconaPGCluster.pgv2.percona.com major-upgrade-logical-replica LogicalReplicaInvalidated Logical replica "analytics" must be seeded again: the cluster was upgraded from PostgreSQL 17 to 18 after it was seeded perconapgcluster-controller logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:45 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-ngzv-q9kps Binding Scheduled Successfully assigned kuttl-test-intent-zebra/major-upgrade-logical-replica-backup-ngzv-q9kps to gke-jen-pg-1727-5c0d1c60-default-pool-dbebd946-36k6 default-scheduler logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:45 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-ngzv-q9kps.spec.initContainers{pgbackrest-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600" kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:45 +0000 UTC Normal Job.batch major-upgrade-logical-replica-backup-ngzv SuccessfulCreate Created pod: major-upgrade-logical-replica-backup-ngzv-q9kps job-controller logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:46 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-ngzv-q9kps.spec.initContainers{pgbackrest-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600" in 188ms (188ms including waiting). Image size: 87061965 bytes. kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:46 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-ngzv-q9kps.spec.initContainers{pgbackrest-init} Created Created container: pgbackrest-init kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:46 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-ngzv-q9kps.spec.initContainers{pgbackrest-init} Started Started container pgbackrest-init kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:47 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-ngzv-q9kps.spec.containers{pgbackrest} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18" kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:47 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-ngzv-q9kps.spec.containers{pgbackrest} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18" in 125ms (125ms including waiting). Image size: 122700307 bytes. kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:47 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-ngzv-q9kps.spec.containers{pgbackrest} Created Created container: pgbackrest kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:47 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-ngzv-q9kps.spec.containers{pgbackrest} Started Started container pgbackrest kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:48 +0000 UTC Warning PostgresCluster.upstream.pgv2.percona.com major-upgrade-logical-replica UnableToCreateStanzas repo1-path = /pgbackrest/repo1 2026-09-03 20:43:48.224 P00 ERROR: [050]: unable to acquire lock on file '/tmp/pgbackrest/db-archive-1.lock': Resource temporarily unavailable HINT: is another pgBackRest process running? 2026-09-03 20:43:48.420 P00 ERROR: [050]: unable to acquire lock on file '/tmp/pgbackrest/db-backup-1.lock': Resource temporarily unavailable HINT: is another pgBackRest process running? : command terminated with exit code 50 postgrescluster-controller logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:52 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-98gr-0 SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-8e3d75c8-ffc4-48db-8334-60bc400a90bf" attachdetach-controller logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:53 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-98gr-0.spec.initContainers{database-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600" kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:54 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-98gr-0.spec.initContainers{database-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600" in 177ms (177ms including waiting). Image size: 87061965 bytes. kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:54 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-98gr-0.spec.initContainers{database-init} Created Created container: database-init kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:54 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-98gr-0.spec.initContainers{database-init} Started Started container database-init kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:55 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-98gr-0.spec.initContainers{postgres-startup} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:55 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-98gr-0.spec.initContainers{postgres-startup} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" in 130ms (130ms including waiting). Image size: 449178529 bytes. kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:55 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-98gr-0.spec.initContainers{postgres-startup} Created Created container: postgres-startup kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:55 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-98gr-0.spec.initContainers{postgres-startup} Started Started container postgres-startup kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:56 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-98gr-0.spec.initContainers{nss-wrapper-init-logcollector} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:56 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-98gr-0.spec.initContainers{nss-wrapper-init-logcollector} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" in 132ms (132ms including waiting). Image size: 449178529 bytes. kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:56 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-98gr-0.spec.initContainers{nss-wrapper-init-logcollector} Created Created container: nss-wrapper-init-logcollector kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:56 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-98gr-0.spec.initContainers{nss-wrapper-init-logcollector} Started Started container nss-wrapper-init-logcollector kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:57 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-98gr-0.spec.initContainers{nss-wrapper-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:57 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-98gr-0.spec.initContainers{nss-wrapper-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" in 128ms (128ms including waiting). Image size: 449178529 bytes. kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:57 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-98gr-0.spec.initContainers{nss-wrapper-init} Created Created container: nss-wrapper-init kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:57 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-98gr-0.spec.initContainers{nss-wrapper-init} Started Started container nss-wrapper-init kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:58 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-98gr-0.spec.containers{database} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:58 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-98gr-0.spec.containers{database} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" in 136ms (136ms including waiting). Image size: 449178529 bytes. kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:58 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-98gr-0.spec.containers{database} Created Created container: database kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:58 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-98gr-0.spec.containers{database} Started Started container database kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:58 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-98gr-0.spec.containers{replication-cert-copy} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:59 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-98gr-0.spec.containers{replication-cert-copy} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" in 159ms (159ms including waiting). Image size: 449178529 bytes. kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:59 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-98gr-0.spec.containers{replication-cert-copy} Created Created container: replication-cert-copy kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:59 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-98gr-0.spec.containers{replication-cert-copy} Started Started container replication-cert-copy kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:43:59 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-98gr-0.spec.containers{logs} Pulled Container image "docker.io/perconalab/fluentbit:main-logcollector" already present on machine kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:44:01 +0000 UTC Normal Job.batch major-upgrade-logical-replica-backup-ngzv Completed Job completed job-controller logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:44:01 +0000 UTC Warning Pod major-upgrade-logical-replica-instance1-98gr-0.spec.containers{database} Unhealthy Readiness probe failed: kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:44:26 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-2swd-pb2qs Binding Scheduled Successfully assigned kuttl-test-intent-zebra/major-upgrade-logical-replica-backup-2swd-pb2qs to gke-jen-pg-1727-5c0d1c60-default-pool-dbebd946-36k6 default-scheduler logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:44:26 +0000 UTC Normal Job.batch major-upgrade-logical-replica-backup-2swd SuccessfulCreate Created pod: major-upgrade-logical-replica-backup-2swd-pb2qs job-controller logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:44:27 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-2swd-pb2qs.spec.initContainers{pgbackrest-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600" kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:44:27 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-2swd-pb2qs.spec.initContainers{pgbackrest-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600" in 143ms (143ms including waiting). Image size: 87061965 bytes. kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:44:27 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-2swd-pb2qs.spec.initContainers{pgbackrest-init} Created Created container: pgbackrest-init kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:44:27 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-2swd-pb2qs.spec.initContainers{pgbackrest-init} Started Started container pgbackrest-init kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:44:28 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-2swd-pb2qs.spec.containers{pgbackrest} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18" kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:44:28 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-2swd-pb2qs.spec.containers{pgbackrest} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18" in 97ms (97ms including waiting). Image size: 122700307 bytes. kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:44:28 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-2swd-pb2qs.spec.containers{pgbackrest} Created Created container: pgbackrest kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:44:28 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-2swd-pb2qs.spec.containers{pgbackrest} Started Started container pgbackrest kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:44:40 +0000 UTC Normal Job.batch major-upgrade-logical-replica-backup-2swd Completed Job completed job-controller logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:45:11 +0000 UTC Normal Job.batch major-upgrade-logical-replica-lr-analytics-bootstrap SuccessfulCreate Created pod: major-upgrade-logical-replica-lr-analytics-bootstrap-d2gzn job-controller logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:45:11 +0000 UTC Normal PersistentVolumeClaim major-upgrade-logical-replica-lr-analytics-pgdata WaitForFirstConsumer waiting for first consumer to be created before binding persistentvolume-controller logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:45:11 +0000 UTC Normal PersistentVolumeClaim major-upgrade-logical-replica-lr-analytics-pgdata ExternalProvisioning Waiting for a volume to be created either by the external provisioner 'pd.csi.storage.gke.io' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. persistentvolume-controller logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:45:11 +0000 UTC Normal PersistentVolumeClaim major-upgrade-logical-replica-lr-analytics-pgdata Provisioning External provisioner is provisioning volume for claim "kuttl-test-intent-zebra/major-upgrade-logical-replica-lr-analytics-pgdata" pd.csi.storage.gke.io_gke-ecf57bd162d444b6af7c-67c8-40b8-vm_b1f31345-bea8-4cb3-b226-8a4f71e8820f logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:45:13 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-d2gzn Binding Scheduled Successfully assigned kuttl-test-intent-zebra/major-upgrade-logical-replica-lr-analytics-bootstrap-d2gzn to gke-jen-pg-1727-5c0d1c60-default-pool-dbebd946-36k6 default-scheduler logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:45:13 +0000 UTC Normal PersistentVolumeClaim major-upgrade-logical-replica-lr-analytics-pgdata ProvisioningSucceeded Successfully provisioned volume pvc-b03abb8f-c3d7-468e-adf3-00a464e6c41b pd.csi.storage.gke.io_gke-ecf57bd162d444b6af7c-67c8-40b8-vm_b1f31345-bea8-4cb3-b226-8a4f71e8820f logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:45:19 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-d2gzn SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-b03abb8f-c3d7-468e-adf3-00a464e6c41b" attachdetach-controller logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:45:24 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-d2gzn.spec.initContainers{database-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600" kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:45:24 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-d2gzn.spec.initContainers{database-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1727-5c0d1c600" in 176ms (176ms including waiting). Image size: 87061965 bytes. kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:45:24 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-d2gzn.spec.initContainers{database-init} Created Created container: database-init kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:45:24 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-d2gzn.spec.initContainers{database-init} Started Started container database-init kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:45:25 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-d2gzn.spec.initContainers{nss-wrapper-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:45:25 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-d2gzn.spec.initContainers{nss-wrapper-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" in 144ms (144ms including waiting). Image size: 449178529 bytes. kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:45:25 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-d2gzn.spec.initContainers{nss-wrapper-init} Created Created container: nss-wrapper-init kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:45:25 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-d2gzn.spec.initContainers{nss-wrapper-init} Started Started container nss-wrapper-init kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:45:26 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-d2gzn.spec.containers{logical-replica-bootstrap} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:45:26 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-d2gzn.spec.containers{logical-replica-bootstrap} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" in 118ms (118ms including waiting). Image size: 449178529 bytes. kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:45:26 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-d2gzn.spec.containers{logical-replica-bootstrap} Created Created container: logical-replica-bootstrap kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:45:26 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-d2gzn.spec.containers{logical-replica-bootstrap} Started Started container logical-replica-bootstrap kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:45:40 +0000 UTC Normal Job.batch major-upgrade-logical-replica-lr-analytics-bootstrap Completed Job completed job-controller logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:45:41 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0 Binding Scheduled Successfully assigned kuttl-test-intent-zebra/major-upgrade-logical-replica-lr-analytics-0 to gke-jen-pg-1727-5c0d1c60-default-pool-dbebd946-36k6 default-scheduler logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:45:41 +0000 UTC Normal StatefulSet.apps major-upgrade-logical-replica-lr-analytics SuccessfulCreate create Pod major-upgrade-logical-replica-lr-analytics-0 in StatefulSet major-upgrade-logical-replica-lr-analytics successful statefulset-controller logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:45:44 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0.spec.initContainers{nss-wrapper-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:45:44 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0.spec.initContainers{nss-wrapper-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" in 130ms (130ms including waiting). Image size: 449178529 bytes. kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:45:44 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0.spec.initContainers{nss-wrapper-init} Created Created container: nss-wrapper-init kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:45:44 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0.spec.initContainers{nss-wrapper-init} Started Started container nss-wrapper-init kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:45:45 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0.spec.containers{database} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:45:45 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0.spec.containers{database} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" in 122ms (122ms including waiting). Image size: 449178529 bytes. kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:45:45 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0.spec.containers{database} Created Created container: database kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:45:45 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0.spec.containers{database} Started Started container database kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:46:12 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0.spec.containers{database} Killing Stopping container database kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:46:18 +0000 UTC Warning Pod major-upgrade-logical-replica-instance1-n9h5-0.spec.containers{database} Unhealthy Readiness probe failed: kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:46:20 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-5b889f6889-q6cwr.spec.containers{pgbouncer} Killing Stopping container pgbouncer kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:46:20 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-5b889f6889-q6cwr.spec.containers{pgbouncer-config} Killing Stopping container pgbouncer-config kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:46:20 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.containers{pgbackrest} Killing Stopping container pgbackrest kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | 2026-09-03 20:46:20 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.containers{pgbackrest-config} Killing Stopping container pgbackrest-config kubelet logger.go:42: 20:46:40 | major-upgrade-logical-replica | Deleting namespace "kuttl-test-intent-zebra" === NAME kuttl harness.go:404: run tests finished harness.go:511: cleaning up harness.go:568: removing temp folder: "" --- PASS: kuttl (659.65s) --- PASS: kuttl/harness (0.00s) --- PASS: kuttl/harness/major-upgrade-logical-replica (658.78s) PASS