=== RUN kuttl harness.go:469: starting setup harness.go:259: running tests using configured kubeconfig. harness.go:282: Successful connection to cluster at: https://35.232.35.196 harness.go:365: running tests harness.go:78: going to run test suite with timeout of 600 seconds for each step harness.go:377: testsuite: e2e-tests/tests has 60 tests === RUN kuttl/harness === RUN kuttl/harness/major-upgrade-logical-replica === PAUSE kuttl/harness/major-upgrade-logical-replica === CONT kuttl/harness/major-upgrade-logical-replica logger.go:42: 21:21:14 | major-upgrade-logical-replica | Ignoring "conf": does not begin with a number followed by a dash. logger.go:42: 21:21:14 | major-upgrade-logical-replica | Creating namespace "kuttl-test-crack-mammoth" logger.go:42: 21:21:15 | major-upgrade-logical-replica/0-deploy-operator | starting test step 0-deploy-operator logger.go:42: 21:21:15 | major-upgrade-logical-replica/0-deploy-operator | running command: [sh -c set -o errexit set -o xtrace source ../../functions init_temp_dir # do this only in the first TestStep deploy_operator deploy_client deploy_s3_secrets] logger.go:42: 21:21:15 | major-upgrade-logical-replica/0-deploy-operator | + source ../../functions logger.go:42: 21:21:15 | major-upgrade-logical-replica/0-deploy-operator | +++ realpath ../../.. logger.go:42: 21:21:15 | major-upgrade-logical-replica/0-deploy-operator | ++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812 logger.go:42: 21:21:15 | major-upgrade-logical-replica/0-deploy-operator | ++ CERT_MANAGER_VER=1.20.2 logger.go:42: 21:21:15 | major-upgrade-logical-replica/0-deploy-operator | ++ CERT_MANAGER_TEST_LABEL=e2e.percona.com/managed-by=percona-postgresql-operator-tests logger.go:42: 21:21:15 | major-upgrade-logical-replica/0-deploy-operator | ++ CHAOS_MESH_VER=2.5.1 logger.go:42: 21:21:15 | major-upgrade-logical-replica/0-deploy-operator | ++ BUSYBOX_VER=1.36 logger.go:42: 21:21:15 | major-upgrade-logical-replica/0-deploy-operator | ++++ pwd logger.go:42: 21:21:15 | major-upgrade-logical-replica/0-deploy-operator | +++ basename /mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests/tests/major-upgrade-logical-replica logger.go:42: 21:21:15 | major-upgrade-logical-replica/0-deploy-operator | ++ test_name=major-upgrade-logical-replica logger.go:42: 21:21:15 | major-upgrade-logical-replica/0-deploy-operator | ++ source /mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests/vars.sh logger.go:42: 21:21:15 | major-upgrade-logical-replica/0-deploy-operator | +++ export ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812 logger.go:42: 21:21:15 | major-upgrade-logical-replica/0-deploy-operator | +++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812 logger.go:42: 21:21:15 | major-upgrade-logical-replica/0-deploy-operator | +++ export DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/deploy logger.go:42: 21:21:15 | major-upgrade-logical-replica/0-deploy-operator | +++ DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/deploy logger.go:42: 21:21:15 | major-upgrade-logical-replica/0-deploy-operator | +++ export TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests logger.go:42: 21:21:15 | major-upgrade-logical-replica/0-deploy-operator | +++ TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests logger.go:42: 21:21:15 | major-upgrade-logical-replica/0-deploy-operator | +++ export TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests/conf logger.go:42: 21:21:15 | major-upgrade-logical-replica/0-deploy-operator | +++ TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests/conf logger.go:42: 21:21:15 | major-upgrade-logical-replica/0-deploy-operator | +++ export TEST_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests/tests/major-upgrade-logical-replica/conf logger.go:42: 21:21:15 | major-upgrade-logical-replica/0-deploy-operator | +++ TEST_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests/tests/major-upgrade-logical-replica/conf logger.go:42: 21:21:15 | major-upgrade-logical-replica/0-deploy-operator | +++ export TEMP_DIR=/tmp/kuttl/pg/major-upgrade-logical-replica logger.go:42: 21:21:15 | major-upgrade-logical-replica/0-deploy-operator | +++ TEMP_DIR=/tmp/kuttl/pg/major-upgrade-logical-replica logger.go:42: 21:21:15 | major-upgrade-logical-replica/0-deploy-operator | ++++ git rev-parse --abbrev-ref HEAD logger.go:42: 21:21:15 | major-upgrade-logical-replica/0-deploy-operator | +++ export GIT_BRANCH=PR-1812 logger.go:42: 21:21:15 | major-upgrade-logical-replica/0-deploy-operator | +++ GIT_BRANCH=PR-1812 logger.go:42: 21:21:15 | major-upgrade-logical-replica/0-deploy-operator | +++ export VERSION=PR-1812-09e4007fd logger.go:42: 21:21:15 | major-upgrade-logical-replica/0-deploy-operator | +++ VERSION=PR-1812-09e4007fd logger.go:42: 21:21:15 | major-upgrade-logical-replica/0-deploy-operator | +++ export SKIP_TEST_WARNINGS=true logger.go:42: 21:21:15 | major-upgrade-logical-replica/0-deploy-operator | +++ SKIP_TEST_WARNINGS=true logger.go:42: 21:21:15 | major-upgrade-logical-replica/0-deploy-operator | +++ export K8S_UPGRADE_PLATFORM=gke logger.go:42: 21:21:15 | major-upgrade-logical-replica/0-deploy-operator | +++ K8S_UPGRADE_PLATFORM=gke logger.go:42: 21:21:15 | major-upgrade-logical-replica/0-deploy-operator | +++ export K8S_UPGRADE_REGION=us-central1-a logger.go:42: 21:21:15 | major-upgrade-logical-replica/0-deploy-operator | +++ K8S_UPGRADE_REGION=us-central1-a logger.go:42: 21:21:15 | major-upgrade-logical-replica/0-deploy-operator | +++ export K8S_UPGRADE_INITIAL_VERSION= logger.go:42: 21:21:15 | major-upgrade-logical-replica/0-deploy-operator | +++ K8S_UPGRADE_INITIAL_VERSION= logger.go:42: 21:21:15 | major-upgrade-logical-replica/0-deploy-operator | +++ export K8S_UPGRADE_FINAL_VERSION= logger.go:42: 21:21:15 | major-upgrade-logical-replica/0-deploy-operator | +++ K8S_UPGRADE_FINAL_VERSION= logger.go:42: 21:21:15 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ -z '' ]] logger.go:42: 21:21:15 | major-upgrade-logical-replica/0-deploy-operator | ++++ detect_k8s_platform logger.go:42: 21:21:15 | major-upgrade-logical-replica/0-deploy-operator | ++++ local detected_platform logger.go:42: 21:21:15 | major-upgrade-logical-replica/0-deploy-operator | ++++ local xtrace_enabled=false logger.go:42: 21:21:15 | major-upgrade-logical-replica/0-deploy-operator | ++++ [[ -o xtrace ]] logger.go:42: 21:21:15 | major-upgrade-logical-replica/0-deploy-operator | ++++ xtrace_enabled=true logger.go:42: 21:21:15 | major-upgrade-logical-replica/0-deploy-operator | ++++ set +o xtrace logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ PLATFORM=gke logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ export PLATFORM logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE=perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE=perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ ! -n '' ]] logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ -n '' ]] logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ export PG_VER=18 logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ PG_VER=18 logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ export PG_DISTRIBUTION= logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ PG_DISTRIBUTION= logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ '' == community ]] logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ export BUCKET=pg-operator-testing logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ BUCKET=pg-operator-testing logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ export PMM_SERVER_VERSION=9.9.9 logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ PMM_SERVER_VERSION=9.9.9 logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE_LOGCOLLECTOR=perconalab/fluentbit:main-logcollector logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE_LOGCOLLECTOR=perconalab/fluentbit:main-logcollector logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ export PGOV1_TAG=1.4.0 logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ PGOV1_TAG=1.4.0 logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ export PGOV1_VER=14 logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ PGOV1_VER=14 logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ export CPGO_VERSION=5.8.7 logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ CPGO_VERSION=5.8.7 logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ export MINIO_VER=5.4.0 logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ MINIO_VER=5.4.0 logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ export VAULT_VER=0.32.0 logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ VAULT_VER=0.32.0 logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ export REGISTRY_NAME=docker.io logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ REGISTRY_NAME=docker.io logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ export REGISTRY_NAME_FULL=docker.io/ logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ REGISTRY_NAME_FULL=docker.io/ logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | ++++ printenv logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | ++++ grep -E '^IMAGE' logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | ++++ awk -F= '{print $1}' logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | ++++ eval 'echo $IMAGE_BACKREST' logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++++ echo perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ var_value=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == docker.io/* ]] logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == percona/* ]] logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == perconalab/* ]] logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ echo IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ echo perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | ++++ eval 'echo $IMAGE' logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++++ echo perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ var_value=perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:PR-1812-09e4007fd == docker.io/* ]] logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:PR-1812-09e4007fd == percona/* ]] logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:PR-1812-09e4007fd == perconalab/* ]] logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ new_value=docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ echo IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | ++++ eval 'echo $IMAGE_UPGRADE' logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++++ echo perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ var_value=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == docker.io/* ]] logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == percona/* ]] logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == perconalab/* ]] logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ echo IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | ++++ eval 'echo $IMAGE_LOGCOLLECTOR' logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++++ echo perconalab/fluentbit:main-logcollector logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ var_value=perconalab/fluentbit:main-logcollector logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/fluentbit:main-logcollector == docker.io/* ]] logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/fluentbit:main-logcollector == percona/* ]] logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/fluentbit:main-logcollector == perconalab/* ]] logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ new_value=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ echo IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | ++++ eval 'echo $IMAGE_BASE' logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++++ echo perconalab/percona-postgresql-operator logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ var_value=perconalab/percona-postgresql-operator logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator == docker.io/* ]] logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator == percona/* ]] logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator == perconalab/* ]] logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ new_value=docker.io/perconalab/percona-postgresql-operator logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ echo IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | ++++ eval 'echo $IMAGE_PGBOUNCER' logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++++ echo perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ var_value=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == docker.io/* ]] logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == percona/* ]] logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == perconalab/* ]] logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ echo IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | ++++ eval 'echo $IMAGE_POSTGRESQL' logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++++ echo perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ var_value=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == docker.io/* ]] logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == percona/* ]] logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == perconalab/* ]] logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ echo IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | ++++ eval 'echo $IMAGE_PMM_SERVER' logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++++ echo perconalab/pmm-server:3-dev-latest logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ var_value=perconalab/pmm-server:3-dev-latest logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/pmm-server:3-dev-latest == docker.io/* ]] logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/pmm-server:3-dev-latest == percona/* ]] logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/pmm-server:3-dev-latest == perconalab/* ]] logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ new_value=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ echo IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | ++++ eval 'echo $IMAGE_PMM_CLIENT' logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++++ echo perconalab/pmm-client:3-dev-latest logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ var_value=perconalab/pmm-client:3-dev-latest logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/pmm-client:3-dev-latest == docker.io/* ]] logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/pmm-client:3-dev-latest == percona/* ]] logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/pmm-client:3-dev-latest == perconalab/* ]] logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ new_value=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ echo IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | ++++ which gdate logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | which: no gdate in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | ++++ which date logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ date=/usr/sbin/date logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | ++++ which gsed logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | which: no gsed in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | ++++ which sed logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | +++ sed=/usr/sbin/sed logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | + init_temp_dir logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | + rm -rf /tmp/kuttl/pg/major-upgrade-logical-replica logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | + mkdir -p /tmp/kuttl/pg/major-upgrade-logical-replica logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | + deploy_operator logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | + local cw_prefix= logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | + destroy_operator logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | + kubectl -n pg-operator delete deployment percona-postgresql-operator --force --grace-period=0 logger.go:42: 21:21:17 | major-upgrade-logical-replica/0-deploy-operator | Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. logger.go:42: 21:21:18 | major-upgrade-logical-replica/0-deploy-operator | deployment.apps "percona-postgresql-operator" force deleted from pg-operator namespace logger.go:42: 21:21:18 | major-upgrade-logical-replica/0-deploy-operator | + [[ -n pg-operator ]] logger.go:42: 21:21:18 | major-upgrade-logical-replica/0-deploy-operator | + kubectl delete namespace pg-operator --force --grace-period=0 --ignore-not-found logger.go:42: 21:21:18 | major-upgrade-logical-replica/0-deploy-operator | Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. logger.go:42: 21:21:18 | major-upgrade-logical-replica/0-deploy-operator | namespace "pg-operator" force deleted logger.go:42: 21:21:30 | major-upgrade-logical-replica/0-deploy-operator | + [[ -n pg-operator ]] logger.go:42: 21:21:30 | major-upgrade-logical-replica/0-deploy-operator | + create_namespace pg-operator logger.go:42: 21:21:30 | major-upgrade-logical-replica/0-deploy-operator | + local namespace=pg-operator logger.go:42: 21:21:30 | major-upgrade-logical-replica/0-deploy-operator | + [[ gke == openshift ]] logger.go:42: 21:21:30 | major-upgrade-logical-replica/0-deploy-operator | + kubectl delete namespace pg-operator --ignore-not-found logger.go:42: 21:21:30 | major-upgrade-logical-replica/0-deploy-operator | + kubectl wait --for=delete namespace pg-operator logger.go:42: 21:21:31 | major-upgrade-logical-replica/0-deploy-operator | + kubectl create namespace pg-operator logger.go:42: 21:21:31 | major-upgrade-logical-replica/0-deploy-operator | namespace/pg-operator created logger.go:42: 21:21:31 | major-upgrade-logical-replica/0-deploy-operator | + cw_prefix=cw- logger.go:42: 21:21:31 | major-upgrade-logical-replica/0-deploy-operator | + kubectl -n pg-operator apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pg-operator_PR-1812/deploy/crd.yaml logger.go:42: 21:21:32 | major-upgrade-logical-replica/0-deploy-operator | customresourcedefinition.apiextensions.k8s.io/crunchybridgeclusters.upstream.pgv2.percona.com serverside-applied logger.go:42: 21:21:32 | major-upgrade-logical-replica/0-deploy-operator | customresourcedefinition.apiextensions.k8s.io/perconapgbackups.pgv2.percona.com serverside-applied logger.go:42: 21:21:34 | major-upgrade-logical-replica/0-deploy-operator | customresourcedefinition.apiextensions.k8s.io/perconapgclusters.pgv2.percona.com serverside-applied logger.go:42: 21:21:34 | major-upgrade-logical-replica/0-deploy-operator | customresourcedefinition.apiextensions.k8s.io/perconapgrestores.pgv2.percona.com serverside-applied logger.go:42: 21:21:35 | major-upgrade-logical-replica/0-deploy-operator | customresourcedefinition.apiextensions.k8s.io/perconapgupgrades.pgv2.percona.com serverside-applied logger.go:42: 21:21:35 | major-upgrade-logical-replica/0-deploy-operator | customresourcedefinition.apiextensions.k8s.io/pgadmins.upstream.pgv2.percona.com serverside-applied logger.go:42: 21:21:35 | major-upgrade-logical-replica/0-deploy-operator | customresourcedefinition.apiextensions.k8s.io/pgupgrades.upstream.pgv2.percona.com serverside-applied logger.go:42: 21:21:37 | major-upgrade-logical-replica/0-deploy-operator | customresourcedefinition.apiextensions.k8s.io/postgresclusters.upstream.pgv2.percona.com serverside-applied logger.go:42: 21:21:37 | major-upgrade-logical-replica/0-deploy-operator | + kubectl -n pg-operator apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pg-operator_PR-1812/deploy/cw-rbac.yaml logger.go:42: 21:21:38 | major-upgrade-logical-replica/0-deploy-operator | serviceaccount/percona-postgresql-operator serverside-applied logger.go:42: 21:21:38 | major-upgrade-logical-replica/0-deploy-operator | clusterrole.rbac.authorization.k8s.io/percona-postgresql-operator serverside-applied logger.go:42: 21:21:38 | major-upgrade-logical-replica/0-deploy-operator | clusterrolebinding.rbac.authorization.k8s.io/percona-postgresql-operator serverside-applied logger.go:42: 21:21:38 | major-upgrade-logical-replica/0-deploy-operator | + local disable_telemetry=true logger.go:42: 21:21:38 | major-upgrade-logical-replica/0-deploy-operator | + '[' major-upgrade-logical-replica == telemetry-transfer ']' logger.go:42: 21:21:38 | major-upgrade-logical-replica/0-deploy-operator | + yq eval '.spec.template.spec.containers[0].image = "docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd"' /mnt/jenkins/workspace/cloud-pg-operator_PR-1812/deploy/cw-operator.yaml logger.go:42: 21:21:38 | major-upgrade-logical-replica/0-deploy-operator | + yq eval '(.spec.template.spec.containers[] | select(.name=="operator") | .env[] | select(.name=="DISABLE_TELEMETRY") | .value) = "true"' - logger.go:42: 21:21:38 | major-upgrade-logical-replica/0-deploy-operator | + yq eval '(.spec.template.spec.containers[] | select(.name=="operator") | .env[] | select(.name=="LOG_LEVEL") | .value) = "DEBUG"' - logger.go:42: 21:21:38 | major-upgrade-logical-replica/0-deploy-operator | + yq eval '(.spec.template.spec.containers[] | select(.name=="operator") | .env[] | select(.name=="PGO_FEATURE_GATES") | .value) = ""' - logger.go:42: 21:21:38 | major-upgrade-logical-replica/0-deploy-operator | + kubectl -n pg-operator apply -f - logger.go:42: 21:21:39 | major-upgrade-logical-replica/0-deploy-operator | deployment.apps/percona-postgresql-operator created logger.go:42: 21:21:39 | major-upgrade-logical-replica/0-deploy-operator | + deploy_client logger.go:42: 21:21:39 | major-upgrade-logical-replica/0-deploy-operator | + kubectl -n kuttl-test-crack-mammoth apply -f /mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests/conf/client.yaml logger.go:42: 21:21:40 | major-upgrade-logical-replica/0-deploy-operator | deployment.apps/pg-client created logger.go:42: 21:21:40 | major-upgrade-logical-replica/0-deploy-operator | + deploy_s3_secrets logger.go:42: 21:21:40 | major-upgrade-logical-replica/0-deploy-operator | + set +o xtrace logger.go:42: 21:21:41 | major-upgrade-logical-replica/0-deploy-operator | secret/major-upgrade-logical-replica-pgbackrest-secrets created logger.go:42: 21:21:42 | major-upgrade-logical-replica/0-deploy-operator | secret/aws-s3-secret created logger.go:42: 21:21:42 | major-upgrade-logical-replica/0-deploy-operator | secret/do-spaces-secret created logger.go:42: 21:21:43 | major-upgrade-logical-replica/0-deploy-operator | secret/gcp-cs-secret created logger.go:42: 21:21:43 | major-upgrade-logical-replica/0-deploy-operator | secret/azure-secret created logger.go:42: 21:21:44 | major-upgrade-logical-replica/0-deploy-operator | secret/minio-secret created logger.go:42: 21:21:45 | major-upgrade-logical-replica/0-deploy-operator | running command: [sh -c kubectl assert exist-enhanced deployment percona-postgresql-operator -n ${OPERATOR_NS:-$NAMESPACE} --field-selector status.readyReplicas=1] logger.go:42: 21:21:45 | major-upgrade-logical-replica/0-deploy-operator | ASSERT deployment percona-postgresql-operator matching field criteria 'status.readyReplicas=1' should exist. logger.go:42: 21:21:45 | major-upgrade-logical-replica/0-deploy-operator | ASSERT FAIL Resource(s) not found. logger.go:42: 21:21:47 | major-upgrade-logical-replica/0-deploy-operator | running command: [sh -c kubectl assert exist-enhanced deployment percona-postgresql-operator -n ${OPERATOR_NS:-$NAMESPACE} --field-selector status.readyReplicas=1] logger.go:42: 21:21:47 | major-upgrade-logical-replica/0-deploy-operator | ASSERT deployment percona-postgresql-operator matching field criteria 'status.readyReplicas=1' should exist. logger.go:42: 21:21:47 | major-upgrade-logical-replica/0-deploy-operator | INFO Found 1 resource(s). logger.go:42: 21:21:47 | major-upgrade-logical-replica/0-deploy-operator | NAME NAMESPACE COL0 logger.go:42: 21:21:47 | major-upgrade-logical-replica/0-deploy-operator | percona-postgresql-operator pg-operator 1 logger.go:42: 21:21:47 | major-upgrade-logical-replica/0-deploy-operator | ASSERT PASS logger.go:42: 21:21:47 | major-upgrade-logical-replica/0-deploy-operator | test step completed 0-deploy-operator logger.go:42: 21:21:47 | major-upgrade-logical-replica/1-create-cluster | starting test step 1-create-cluster logger.go:42: 21:21:47 | major-upgrade-logical-replica/1-create-cluster | running command: [sh -c set -o errexit set -o xtrace source ../../functions postgres_image=$(get_container_image "postgres" 17) pgbouncer_image=$(get_container_image "pgbouncer" 17) pgbackrest_image=$(get_container_image "pgbackrest" 17) get_cr \ | yq eval " .spec.postgresVersion = 17 | .spec.image = \"${postgres_image}\" | .spec.proxy.pgBouncer.image = \"${pgbouncer_image}\" | .spec.backups.pgbackrest.image = \"${pgbackrest_image}\" | .spec.patroni.removeDataDirectoryOnDivergedTimelines = true" \ | kubectl -n "${NAMESPACE}" apply -f -] logger.go:42: 21:21:47 | major-upgrade-logical-replica/1-create-cluster | + source ../../functions logger.go:42: 21:21:47 | major-upgrade-logical-replica/1-create-cluster | +++ realpath ../../.. logger.go:42: 21:21:47 | major-upgrade-logical-replica/1-create-cluster | ++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812 logger.go:42: 21:21:47 | major-upgrade-logical-replica/1-create-cluster | ++ CERT_MANAGER_VER=1.20.2 logger.go:42: 21:21:47 | major-upgrade-logical-replica/1-create-cluster | ++ CERT_MANAGER_TEST_LABEL=e2e.percona.com/managed-by=percona-postgresql-operator-tests logger.go:42: 21:21:47 | major-upgrade-logical-replica/1-create-cluster | ++ CHAOS_MESH_VER=2.5.1 logger.go:42: 21:21:47 | major-upgrade-logical-replica/1-create-cluster | ++ BUSYBOX_VER=1.36 logger.go:42: 21:21:47 | major-upgrade-logical-replica/1-create-cluster | ++++ pwd logger.go:42: 21:21:47 | major-upgrade-logical-replica/1-create-cluster | +++ basename /mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests/tests/major-upgrade-logical-replica logger.go:42: 21:21:47 | major-upgrade-logical-replica/1-create-cluster | ++ test_name=major-upgrade-logical-replica logger.go:42: 21:21:47 | major-upgrade-logical-replica/1-create-cluster | ++ source /mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests/vars.sh logger.go:42: 21:21:47 | major-upgrade-logical-replica/1-create-cluster | +++ export ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812 logger.go:42: 21:21:47 | major-upgrade-logical-replica/1-create-cluster | +++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812 logger.go:42: 21:21:47 | major-upgrade-logical-replica/1-create-cluster | +++ export DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/deploy logger.go:42: 21:21:47 | major-upgrade-logical-replica/1-create-cluster | +++ DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/deploy logger.go:42: 21:21:47 | major-upgrade-logical-replica/1-create-cluster | +++ export TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests logger.go:42: 21:21:47 | major-upgrade-logical-replica/1-create-cluster | +++ TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests logger.go:42: 21:21:47 | major-upgrade-logical-replica/1-create-cluster | +++ export TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests/conf logger.go:42: 21:21:47 | major-upgrade-logical-replica/1-create-cluster | +++ TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests/conf logger.go:42: 21:21:47 | major-upgrade-logical-replica/1-create-cluster | +++ export TEST_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests/tests/major-upgrade-logical-replica/conf logger.go:42: 21:21:47 | major-upgrade-logical-replica/1-create-cluster | +++ TEST_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests/tests/major-upgrade-logical-replica/conf logger.go:42: 21:21:47 | major-upgrade-logical-replica/1-create-cluster | +++ export TEMP_DIR=/tmp/kuttl/pg/major-upgrade-logical-replica logger.go:42: 21:21:47 | major-upgrade-logical-replica/1-create-cluster | +++ TEMP_DIR=/tmp/kuttl/pg/major-upgrade-logical-replica logger.go:42: 21:21:47 | major-upgrade-logical-replica/1-create-cluster | ++++ git rev-parse --abbrev-ref HEAD logger.go:42: 21:21:47 | major-upgrade-logical-replica/1-create-cluster | +++ export GIT_BRANCH=PR-1812 logger.go:42: 21:21:47 | major-upgrade-logical-replica/1-create-cluster | +++ GIT_BRANCH=PR-1812 logger.go:42: 21:21:47 | major-upgrade-logical-replica/1-create-cluster | +++ export VERSION=PR-1812-09e4007fd logger.go:42: 21:21:47 | major-upgrade-logical-replica/1-create-cluster | +++ VERSION=PR-1812-09e4007fd logger.go:42: 21:21:47 | major-upgrade-logical-replica/1-create-cluster | +++ export SKIP_TEST_WARNINGS=true logger.go:42: 21:21:47 | major-upgrade-logical-replica/1-create-cluster | +++ SKIP_TEST_WARNINGS=true logger.go:42: 21:21:47 | major-upgrade-logical-replica/1-create-cluster | +++ export K8S_UPGRADE_PLATFORM=gke logger.go:42: 21:21:47 | major-upgrade-logical-replica/1-create-cluster | +++ K8S_UPGRADE_PLATFORM=gke logger.go:42: 21:21:47 | major-upgrade-logical-replica/1-create-cluster | +++ export K8S_UPGRADE_REGION=us-central1-a logger.go:42: 21:21:47 | major-upgrade-logical-replica/1-create-cluster | +++ K8S_UPGRADE_REGION=us-central1-a logger.go:42: 21:21:47 | major-upgrade-logical-replica/1-create-cluster | +++ export K8S_UPGRADE_INITIAL_VERSION= logger.go:42: 21:21:47 | major-upgrade-logical-replica/1-create-cluster | +++ K8S_UPGRADE_INITIAL_VERSION= logger.go:42: 21:21:47 | major-upgrade-logical-replica/1-create-cluster | +++ export K8S_UPGRADE_FINAL_VERSION= logger.go:42: 21:21:47 | major-upgrade-logical-replica/1-create-cluster | +++ K8S_UPGRADE_FINAL_VERSION= logger.go:42: 21:21:47 | major-upgrade-logical-replica/1-create-cluster | +++ [[ -z '' ]] logger.go:42: 21:21:47 | major-upgrade-logical-replica/1-create-cluster | ++++ detect_k8s_platform logger.go:42: 21:21:47 | major-upgrade-logical-replica/1-create-cluster | ++++ local detected_platform logger.go:42: 21:21:47 | major-upgrade-logical-replica/1-create-cluster | ++++ local xtrace_enabled=false logger.go:42: 21:21:47 | major-upgrade-logical-replica/1-create-cluster | ++++ [[ -o xtrace ]] logger.go:42: 21:21:47 | major-upgrade-logical-replica/1-create-cluster | ++++ xtrace_enabled=true logger.go:42: 21:21:47 | major-upgrade-logical-replica/1-create-cluster | ++++ set +o xtrace logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ PLATFORM=gke logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ export PLATFORM logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE=perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE=perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ [[ ! -n '' ]] logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ [[ -n '' ]] logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ export PG_VER=18 logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ PG_VER=18 logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ export PG_DISTRIBUTION= logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ PG_DISTRIBUTION= logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ [[ '' == community ]] logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ export BUCKET=pg-operator-testing logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ BUCKET=pg-operator-testing logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ export PMM_SERVER_VERSION=9.9.9 logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ PMM_SERVER_VERSION=9.9.9 logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE_LOGCOLLECTOR=perconalab/fluentbit:main-logcollector logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE_LOGCOLLECTOR=perconalab/fluentbit:main-logcollector logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ export PGOV1_TAG=1.4.0 logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ PGOV1_TAG=1.4.0 logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ export PGOV1_VER=14 logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ PGOV1_VER=14 logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ export CPGO_VERSION=5.8.7 logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ CPGO_VERSION=5.8.7 logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ export MINIO_VER=5.4.0 logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ MINIO_VER=5.4.0 logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ export VAULT_VER=0.32.0 logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ VAULT_VER=0.32.0 logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ export REGISTRY_NAME=docker.io logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ REGISTRY_NAME=docker.io logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ export REGISTRY_NAME_FULL=docker.io/ logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ REGISTRY_NAME_FULL=docker.io/ logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | ++++ printenv logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | ++++ grep -E '^IMAGE' logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | ++++ awk -F= '{print $1}' logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | ++++ eval 'echo $IMAGE_BACKREST' logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++++ echo perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ var_value=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == docker.io/* ]] logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == percona/* ]] logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == perconalab/* ]] logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ echo IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ echo perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | ++++ eval 'echo $IMAGE' logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++++ echo perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ var_value=perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:PR-1812-09e4007fd == docker.io/* ]] logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:PR-1812-09e4007fd == percona/* ]] logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:PR-1812-09e4007fd == perconalab/* ]] logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ new_value=docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ echo IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | ++++ eval 'echo $IMAGE_UPGRADE' logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++++ echo perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ var_value=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == docker.io/* ]] logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == percona/* ]] logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == perconalab/* ]] logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ echo IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | ++++ eval 'echo $IMAGE_LOGCOLLECTOR' logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++++ echo perconalab/fluentbit:main-logcollector logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ var_value=perconalab/fluentbit:main-logcollector logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/fluentbit:main-logcollector == docker.io/* ]] logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/fluentbit:main-logcollector == percona/* ]] logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/fluentbit:main-logcollector == perconalab/* ]] logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ new_value=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ echo IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | ++++ eval 'echo $IMAGE_BASE' logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++++ echo perconalab/percona-postgresql-operator logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ var_value=perconalab/percona-postgresql-operator logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator == docker.io/* ]] logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator == percona/* ]] logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator == perconalab/* ]] logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ new_value=docker.io/perconalab/percona-postgresql-operator logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ echo IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | ++++ eval 'echo $IMAGE_PGBOUNCER' logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++++ echo perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ var_value=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == docker.io/* ]] logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == percona/* ]] logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == perconalab/* ]] logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ echo IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | ++++ eval 'echo $IMAGE_POSTGRESQL' logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++++ echo perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ var_value=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == docker.io/* ]] logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == percona/* ]] logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == perconalab/* ]] logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ echo IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | ++++ eval 'echo $IMAGE_PMM_SERVER' logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++++ echo perconalab/pmm-server:3-dev-latest logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ var_value=perconalab/pmm-server:3-dev-latest logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/pmm-server:3-dev-latest == docker.io/* ]] logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/pmm-server:3-dev-latest == percona/* ]] logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/pmm-server:3-dev-latest == perconalab/* ]] logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ new_value=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ echo IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | ++++ eval 'echo $IMAGE_PMM_CLIENT' logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++++ echo perconalab/pmm-client:3-dev-latest logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ var_value=perconalab/pmm-client:3-dev-latest logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/pmm-client:3-dev-latest == docker.io/* ]] logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/pmm-client:3-dev-latest == percona/* ]] logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/pmm-client:3-dev-latest == perconalab/* ]] logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ new_value=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ echo IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | ++++ which gdate logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | which: no gdate in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | ++++ which date logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ date=/usr/sbin/date logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | ++++ which gsed logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | which: no gsed in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | ++++ which sed logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | +++ sed=/usr/sbin/sed logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | ++ get_container_image postgres 17 logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | ++ local component=postgres logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | ++ local pgVersion=17 logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | ++ [[ docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd == percona/* ]] logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | ++ [[ docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd == docker.io/percona/* ]] logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | ++ [[ -n '' ]] logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | ++ local operatorVersion=PR-1812-09e4007fd logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | ++ [[ ! PR-1812-09e4007fd =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | ++ operatorVersion=main logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | ++ [[ '' == community ]] logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | ++ [[ postgres == pgbouncer ]] logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | ++ [[ postgres == pgbackrest ]] logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | ++ [[ postgres == upgrade ]] logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | ++ [[ postgres == postgis ]] logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | ++ echo docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | + postgres_image=docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | ++ get_container_image pgbouncer 17 logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | ++ local component=pgbouncer logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | ++ local pgVersion=17 logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | ++ [[ docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd == percona/* ]] logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | ++ [[ docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd == docker.io/percona/* ]] logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | ++ [[ -n '' ]] logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | ++ local operatorVersion=PR-1812-09e4007fd logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | ++ [[ ! PR-1812-09e4007fd =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | ++ operatorVersion=main logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | ++ [[ '' == community ]] logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | ++ [[ pgbouncer == pgbouncer ]] logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | ++ echo docker.io/perconalab/percona-postgresql-operator:main-pgbouncer17 logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | + pgbouncer_image=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer17 logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | ++ get_container_image pgbackrest 17 logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | ++ local component=pgbackrest logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | ++ local pgVersion=17 logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | ++ [[ docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd == percona/* ]] logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | ++ [[ docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd == docker.io/percona/* ]] logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | ++ [[ -n '' ]] logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | ++ local operatorVersion=PR-1812-09e4007fd logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | ++ [[ ! PR-1812-09e4007fd =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | ++ operatorVersion=main logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | ++ [[ '' == community ]] logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | ++ [[ pgbackrest == pgbouncer ]] logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | ++ [[ pgbackrest == pgbackrest ]] logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | ++ echo docker.io/perconalab/percona-postgresql-operator:main-pgbackrest17 logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | + pgbackrest_image=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest17 logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | + get_cr logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | + local cr_name=major-upgrade-logical-replica logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | + local repo_path= logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | + local source_path= logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | + local platform=gke logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | + local cr_file=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests/tests/major-upgrade-logical-replica/conf/major-upgrade-logical-replica.yaml logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | + local platform_cr_file=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests/tests/major-upgrade-logical-replica/conf/major-upgrade-logical-replica-gke.yaml logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | + local spaces_cr_file=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests/tests/major-upgrade-logical-replica/conf/major-upgrade-logical-replica-spaces.yaml logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | + [[ -f /mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests/tests/major-upgrade-logical-replica/conf/major-upgrade-logical-replica-gke.yaml ]] logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | + yq eval $'\n .spec.postgresVersion = 17 |\n .spec.image = "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" |\n .spec.proxy.pgBouncer.image = "docker.io/perconalab/percona-postgresql-operator:main-pgbouncer17" |\n .spec.backups.pgbackrest.image = "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest17" |\n .spec.patroni.removeDataDirectoryOnDivergedTimelines = true' logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | + [[ gke == doks ]] logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | + crs=('/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/deploy/cr.yaml') logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | + local crs logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | + [[ -f /mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests/tests/major-upgrade-logical-replica/conf/major-upgrade-logical-replica.yaml ]] logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | + crs+=($cr_file) logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | + kubectl -n kuttl-test-crack-mammoth apply -f - logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | + yq eval-all $'\n\t\tselect(fileIndex == 0) * (select(fileIndex == 1) // {}) |\n\t\t.metadata.name = "major-upgrade-logical-replica" |\n\t\t.metadata.labels = {"e2e":"major-upgrade-logical-replica"} |\n\t\t.spec.postgresVersion = 18 |\n\t\t.spec.users += [{"name":"postgres","password":{"type":"AlphaNumeric"}}] |\n\t\t.spec.users += [{"name":"major-upgrade-logical-replica","password":{"type":"AlphaNumeric"}}] |\n\t\t.spec.image = "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" |\n\t\t.spec.initContainer.image = "docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd" |\n\t\t.spec.backups.pgbackrest.image = "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18" |\n\t\t.spec.proxy.pgBouncer.image = "docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18" |\n\t\t.spec.logcollector.image = "docker.io/perconalab/fluentbit:main-logcollector" |\n\t\t.spec.pmm.image = "docker.io/perconalab/pmm-client:3-dev-latest" |\n\t\t.spec.pmm.secret = "major-upgrade-logical-replica-pmm-secret" |\n\t\t.spec.pmm.customClusterName = "major-upgrade-logical-replica-pmm-custom-name" |\n\t\t.spec.pmm.postgresParams = "--environment=dev-postgres"\n\t' /mnt/jenkins/workspace/cloud-pg-operator_PR-1812/deploy/cr.yaml /mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests/tests/major-upgrade-logical-replica/conf/major-upgrade-logical-replica.yaml logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | + /usr/sbin/sed -i 's|||g' /tmp/kuttl/pg/major-upgrade-logical-replica/cr.yaml logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | + /usr/sbin/sed -i 's|||g' /tmp/kuttl/pg/major-upgrade-logical-replica/cr.yaml logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | + /usr/sbin/sed -i 's||pg-operator-testing|g' /tmp/kuttl/pg/major-upgrade-logical-replica/cr.yaml logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | + /usr/sbin/sed -i 's||docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd|g' /tmp/kuttl/pg/major-upgrade-logical-replica/cr.yaml logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | + [[ gke == openshift ]] logger.go:42: 21:21:49 | major-upgrade-logical-replica/1-create-cluster | + cat /tmp/kuttl/pg/major-upgrade-logical-replica/cr.yaml logger.go:42: 21:21:51 | major-upgrade-logical-replica/1-create-cluster | perconapgcluster.pgv2.percona.com/major-upgrade-logical-replica created logger.go:42: 21:23:01 | major-upgrade-logical-replica/1-create-cluster | test step completed 1-create-cluster logger.go:42: 21:23:01 | major-upgrade-logical-replica/2-write-data | starting test step 2-write-data logger.go:42: 21:23:01 | major-upgrade-logical-replica/2-write-data | running command: [sh -c set -o errexit set -o xtrace source ../../functions PASS=$(get_psql_user_pass major-upgrade-logical-replica-pguser-postgres) HOST=$(get_psql_user_host major-upgrade-logical-replica-pguser-postgres) run_psql_local \ 'CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);' \ "postgres:${PASS}@${HOST}" run_psql_local \ '\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)' \ "postgres:${PASS}@${HOST}"] logger.go:42: 21:23:01 | major-upgrade-logical-replica/2-write-data | + source ../../functions logger.go:42: 21:23:01 | major-upgrade-logical-replica/2-write-data | +++ realpath ../../.. logger.go:42: 21:23:01 | major-upgrade-logical-replica/2-write-data | ++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812 logger.go:42: 21:23:01 | major-upgrade-logical-replica/2-write-data | ++ CERT_MANAGER_VER=1.20.2 logger.go:42: 21:23:01 | major-upgrade-logical-replica/2-write-data | ++ CERT_MANAGER_TEST_LABEL=e2e.percona.com/managed-by=percona-postgresql-operator-tests logger.go:42: 21:23:01 | major-upgrade-logical-replica/2-write-data | ++ CHAOS_MESH_VER=2.5.1 logger.go:42: 21:23:01 | major-upgrade-logical-replica/2-write-data | ++ BUSYBOX_VER=1.36 logger.go:42: 21:23:01 | major-upgrade-logical-replica/2-write-data | ++++ pwd logger.go:42: 21:23:01 | major-upgrade-logical-replica/2-write-data | +++ basename /mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests/tests/major-upgrade-logical-replica logger.go:42: 21:23:01 | major-upgrade-logical-replica/2-write-data | ++ test_name=major-upgrade-logical-replica logger.go:42: 21:23:01 | major-upgrade-logical-replica/2-write-data | ++ source /mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests/vars.sh logger.go:42: 21:23:01 | major-upgrade-logical-replica/2-write-data | +++ export ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812 logger.go:42: 21:23:01 | major-upgrade-logical-replica/2-write-data | +++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812 logger.go:42: 21:23:01 | major-upgrade-logical-replica/2-write-data | +++ export DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/deploy logger.go:42: 21:23:01 | major-upgrade-logical-replica/2-write-data | +++ DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/deploy logger.go:42: 21:23:01 | major-upgrade-logical-replica/2-write-data | +++ export TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests logger.go:42: 21:23:01 | major-upgrade-logical-replica/2-write-data | +++ TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests logger.go:42: 21:23:01 | major-upgrade-logical-replica/2-write-data | +++ export TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests/conf logger.go:42: 21:23:01 | major-upgrade-logical-replica/2-write-data | +++ TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests/conf logger.go:42: 21:23:01 | major-upgrade-logical-replica/2-write-data | +++ export TEST_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests/tests/major-upgrade-logical-replica/conf logger.go:42: 21:23:01 | major-upgrade-logical-replica/2-write-data | +++ TEST_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests/tests/major-upgrade-logical-replica/conf logger.go:42: 21:23:01 | major-upgrade-logical-replica/2-write-data | +++ export TEMP_DIR=/tmp/kuttl/pg/major-upgrade-logical-replica logger.go:42: 21:23:01 | major-upgrade-logical-replica/2-write-data | +++ TEMP_DIR=/tmp/kuttl/pg/major-upgrade-logical-replica logger.go:42: 21:23:01 | major-upgrade-logical-replica/2-write-data | ++++ git rev-parse --abbrev-ref HEAD logger.go:42: 21:23:01 | major-upgrade-logical-replica/2-write-data | +++ export GIT_BRANCH=PR-1812 logger.go:42: 21:23:01 | major-upgrade-logical-replica/2-write-data | +++ GIT_BRANCH=PR-1812 logger.go:42: 21:23:01 | major-upgrade-logical-replica/2-write-data | +++ export VERSION=PR-1812-09e4007fd logger.go:42: 21:23:01 | major-upgrade-logical-replica/2-write-data | +++ VERSION=PR-1812-09e4007fd logger.go:42: 21:23:01 | major-upgrade-logical-replica/2-write-data | +++ export SKIP_TEST_WARNINGS=true logger.go:42: 21:23:01 | major-upgrade-logical-replica/2-write-data | +++ SKIP_TEST_WARNINGS=true logger.go:42: 21:23:01 | major-upgrade-logical-replica/2-write-data | +++ export K8S_UPGRADE_PLATFORM=gke logger.go:42: 21:23:01 | major-upgrade-logical-replica/2-write-data | +++ K8S_UPGRADE_PLATFORM=gke logger.go:42: 21:23:01 | major-upgrade-logical-replica/2-write-data | +++ export K8S_UPGRADE_REGION=us-central1-a logger.go:42: 21:23:01 | major-upgrade-logical-replica/2-write-data | +++ K8S_UPGRADE_REGION=us-central1-a logger.go:42: 21:23:01 | major-upgrade-logical-replica/2-write-data | +++ export K8S_UPGRADE_INITIAL_VERSION= logger.go:42: 21:23:01 | major-upgrade-logical-replica/2-write-data | +++ K8S_UPGRADE_INITIAL_VERSION= logger.go:42: 21:23:01 | major-upgrade-logical-replica/2-write-data | +++ export K8S_UPGRADE_FINAL_VERSION= logger.go:42: 21:23:01 | major-upgrade-logical-replica/2-write-data | +++ K8S_UPGRADE_FINAL_VERSION= logger.go:42: 21:23:01 | major-upgrade-logical-replica/2-write-data | +++ [[ -z '' ]] logger.go:42: 21:23:01 | major-upgrade-logical-replica/2-write-data | ++++ detect_k8s_platform logger.go:42: 21:23:01 | major-upgrade-logical-replica/2-write-data | ++++ local detected_platform logger.go:42: 21:23:01 | major-upgrade-logical-replica/2-write-data | ++++ local xtrace_enabled=false logger.go:42: 21:23:01 | major-upgrade-logical-replica/2-write-data | ++++ [[ -o xtrace ]] logger.go:42: 21:23:01 | major-upgrade-logical-replica/2-write-data | ++++ xtrace_enabled=true logger.go:42: 21:23:01 | major-upgrade-logical-replica/2-write-data | ++++ set +o xtrace logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ PLATFORM=gke logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ export PLATFORM logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE=perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ IMAGE=perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ [[ ! -n '' ]] logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ [[ -n '' ]] logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ export PG_VER=18 logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ PG_VER=18 logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ export PG_DISTRIBUTION= logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ PG_DISTRIBUTION= logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ [[ '' == community ]] logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ export BUCKET=pg-operator-testing logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ BUCKET=pg-operator-testing logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ export PMM_SERVER_VERSION=9.9.9 logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ PMM_SERVER_VERSION=9.9.9 logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE_LOGCOLLECTOR=perconalab/fluentbit:main-logcollector logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ IMAGE_LOGCOLLECTOR=perconalab/fluentbit:main-logcollector logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ export PGOV1_TAG=1.4.0 logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ PGOV1_TAG=1.4.0 logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ export PGOV1_VER=14 logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ PGOV1_VER=14 logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ export CPGO_VERSION=5.8.7 logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ CPGO_VERSION=5.8.7 logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ export MINIO_VER=5.4.0 logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ MINIO_VER=5.4.0 logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ export VAULT_VER=0.32.0 logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ VAULT_VER=0.32.0 logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ export REGISTRY_NAME=docker.io logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ REGISTRY_NAME=docker.io logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ export REGISTRY_NAME_FULL=docker.io/ logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ REGISTRY_NAME_FULL=docker.io/ logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | ++++ printenv logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | ++++ grep -E '^IMAGE' logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | ++++ awk -F= '{print $1}' logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | ++++ eval 'echo $IMAGE_BACKREST' logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++++ echo perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ var_value=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == docker.io/* ]] logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == percona/* ]] logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == perconalab/* ]] logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ echo IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ echo perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | ++++ eval 'echo $IMAGE' logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++++ echo perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ var_value=perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator:PR-1812-09e4007fd == docker.io/* ]] logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator:PR-1812-09e4007fd == percona/* ]] logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator:PR-1812-09e4007fd == perconalab/* ]] logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ new_value=docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ echo IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | ++++ eval 'echo $IMAGE_UPGRADE' logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++++ echo perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ var_value=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == docker.io/* ]] logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == percona/* ]] logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == perconalab/* ]] logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ echo IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | ++++ eval 'echo $IMAGE_LOGCOLLECTOR' logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++++ echo perconalab/fluentbit:main-logcollector logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ var_value=perconalab/fluentbit:main-logcollector logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/fluentbit:main-logcollector == docker.io/* ]] logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/fluentbit:main-logcollector == percona/* ]] logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/fluentbit:main-logcollector == perconalab/* ]] logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ new_value=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ echo IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | ++++ eval 'echo $IMAGE_BASE' logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++++ echo perconalab/percona-postgresql-operator logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ var_value=perconalab/percona-postgresql-operator logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator == docker.io/* ]] logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator == percona/* ]] logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator == perconalab/* ]] logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ new_value=docker.io/perconalab/percona-postgresql-operator logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ echo IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | ++++ eval 'echo $IMAGE_PGBOUNCER' logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++++ echo perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ var_value=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == docker.io/* ]] logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == percona/* ]] logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == perconalab/* ]] logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ echo IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | ++++ eval 'echo $IMAGE_POSTGRESQL' logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++++ echo perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ var_value=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == docker.io/* ]] logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == percona/* ]] logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == perconalab/* ]] logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ echo IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | ++++ eval 'echo $IMAGE_PMM_SERVER' logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++++ echo perconalab/pmm-server:3-dev-latest logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ var_value=perconalab/pmm-server:3-dev-latest logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/pmm-server:3-dev-latest == docker.io/* ]] logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/pmm-server:3-dev-latest == percona/* ]] logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/pmm-server:3-dev-latest == perconalab/* ]] logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ new_value=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ echo IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | ++++ eval 'echo $IMAGE_PMM_CLIENT' logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++++ echo perconalab/pmm-client:3-dev-latest logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ var_value=perconalab/pmm-client:3-dev-latest logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/pmm-client:3-dev-latest == docker.io/* ]] logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/pmm-client:3-dev-latest == percona/* ]] logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/pmm-client:3-dev-latest == perconalab/* ]] logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ new_value=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ echo IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | ++++ which gdate logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | which: no gdate in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | ++++ which date logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ date=/usr/sbin/date logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | ++++ which gsed logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | which: no gsed in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | ++++ which sed logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | +++ sed=/usr/sbin/sed logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | ++ get_psql_user_pass major-upgrade-logical-replica-pguser-postgres logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | ++ local secret_name=major-upgrade-logical-replica-pguser-postgres logger.go:42: 21:23:03 | major-upgrade-logical-replica/2-write-data | ++ kubectl -n kuttl-test-crack-mammoth get secret/major-upgrade-logical-replica-pguser-postgres '--template={{.data.password | base64decode}}' logger.go:42: 21:23:04 | major-upgrade-logical-replica/2-write-data | + PASS=sDe1VtqrVUd9fUX0dZMg5ebe logger.go:42: 21:23:04 | major-upgrade-logical-replica/2-write-data | ++ get_psql_user_host major-upgrade-logical-replica-pguser-postgres logger.go:42: 21:23:04 | major-upgrade-logical-replica/2-write-data | ++ local secret_name=major-upgrade-logical-replica-pguser-postgres logger.go:42: 21:23:04 | major-upgrade-logical-replica/2-write-data | ++ kubectl -n kuttl-test-crack-mammoth get secret/major-upgrade-logical-replica-pguser-postgres '--template={{.data.host | base64decode }}' logger.go:42: 21:23:04 | major-upgrade-logical-replica/2-write-data | + HOST=major-upgrade-logical-replica-primary.kuttl-test-crack-mammoth.svc logger.go:42: 21:23:04 | major-upgrade-logical-replica/2-write-data | + run_psql_local 'CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);' postgres:sDe1VtqrVUd9fUX0dZMg5ebe@major-upgrade-logical-replica-primary.kuttl-test-crack-mammoth.svc logger.go:42: 21:23:04 | major-upgrade-logical-replica/2-write-data | + local 'command=CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);' logger.go:42: 21:23:04 | major-upgrade-logical-replica/2-write-data | + local uri=postgres:sDe1VtqrVUd9fUX0dZMg5ebe@major-upgrade-logical-replica-primary.kuttl-test-crack-mammoth.svc logger.go:42: 21:23:04 | major-upgrade-logical-replica/2-write-data | + local driver=postgres logger.go:42: 21:23:04 | major-upgrade-logical-replica/2-write-data | ++ get_client_pod logger.go:42: 21:23:04 | major-upgrade-logical-replica/2-write-data | ++ kubectl -n kuttl-test-crack-mammoth get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' logger.go:42: 21:23:05 | major-upgrade-logical-replica/2-write-data | + kubectl -n kuttl-test-crack-mammoth exec pg-client-d7c7bb45b-xgndw -- bash -c 'printf '\''CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://'\''postgres:sDe1VtqrVUd9fUX0dZMg5ebe@major-upgrade-logical-replica-primary.kuttl-test-crack-mammoth.svc'\''' logger.go:42: 21:23:06 | major-upgrade-logical-replica/2-write-data | + run_psql_local '\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)' postgres:sDe1VtqrVUd9fUX0dZMg5ebe@major-upgrade-logical-replica-primary.kuttl-test-crack-mammoth.svc logger.go:42: 21:23:06 | major-upgrade-logical-replica/2-write-data | + local 'command=\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)' logger.go:42: 21:23:06 | major-upgrade-logical-replica/2-write-data | + local uri=postgres:sDe1VtqrVUd9fUX0dZMg5ebe@major-upgrade-logical-replica-primary.kuttl-test-crack-mammoth.svc logger.go:42: 21:23:06 | major-upgrade-logical-replica/2-write-data | + local driver=postgres logger.go:42: 21:23:06 | major-upgrade-logical-replica/2-write-data | ++ get_client_pod logger.go:42: 21:23:06 | major-upgrade-logical-replica/2-write-data | ++ kubectl -n kuttl-test-crack-mammoth get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' logger.go:42: 21:23:06 | major-upgrade-logical-replica/2-write-data | + kubectl -n kuttl-test-crack-mammoth exec pg-client-d7c7bb45b-xgndw -- bash -c 'printf '\''\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://'\''postgres:sDe1VtqrVUd9fUX0dZMg5ebe@major-upgrade-logical-replica-primary.kuttl-test-crack-mammoth.svc'\''' logger.go:42: 21:23:07 | major-upgrade-logical-replica/2-write-data | test step completed 2-write-data logger.go:42: 21:23:07 | major-upgrade-logical-replica/3-create-seed-backup | starting test step 3-create-seed-backup logger.go:42: 21:23:07 | major-upgrade-logical-replica/3-create-seed-backup | running command: [sh -c set -o errexit set -o xtrace source ../../functions # The pgbackrest bootstrap Job restores the latest full backup, and the # one taken at cluster creation predates the rows written above. Take a # new one so the seeded data is actually in the backup being restored. cat </dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 21:26:32 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 21:26:34 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 21:26:35 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 21:26:37 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 21:26:39 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 21:26:41 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 21:26:43 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 21:26:44 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 21:26:46 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 21:26:48 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 21:26:50 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 21:26:51 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 21:26:52 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 21:26:53 | major-upgrade-logical-replica/5-upgrade | Error from server (BadRequest): container "database" in pod "17-to-18-pgdata-qmrgm" is waiting to start: PodInitializing logger.go:42: 21:26:55 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 21:26:56 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 21:26:57 | major-upgrade-logical-replica/5-upgrade | Error from server (BadRequest): container "database" in pod "17-to-18-pgdata-qmrgm" is waiting to start: PodInitializing logger.go:42: 21:26:59 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 21:26:59 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 21:27:00 | major-upgrade-logical-replica/5-upgrade | Error from server (BadRequest): container "database" in pod "17-to-18-pgdata-qmrgm" is waiting to start: PodInitializing logger.go:42: 21:27:03 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 21:27:03 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 21:27:04 | major-upgrade-logical-replica/5-upgrade | Error from server (BadRequest): container "database" in pod "17-to-18-pgdata-qmrgm" is waiting to start: PodInitializing logger.go:42: 21:27:06 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 21:27:07 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 21:27:08 | major-upgrade-logical-replica/5-upgrade | Error from server (BadRequest): container "database" in pod "17-to-18-pgdata-qmrgm" is waiting to start: PodInitializing logger.go:42: 21:27:10 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 21:27:10 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 21:27:11 | major-upgrade-logical-replica/5-upgrade | Error from server (BadRequest): container "database" in pod "17-to-18-pgdata-qmrgm" is waiting to start: PodInitializing logger.go:42: 21:27:13 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 21:27:14 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 21:27:15 | major-upgrade-logical-replica/5-upgrade | Error from server (BadRequest): container "database" in pod "17-to-18-pgdata-qmrgm" is waiting to start: PodInitializing logger.go:42: 21:27:17 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 21:27:17 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 21:27:18 | major-upgrade-logical-replica/5-upgrade | Error from server (BadRequest): container "database" in pod "17-to-18-pgdata-qmrgm" is waiting to start: PodInitializing logger.go:42: 21:27:20 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 21:27:21 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 21:27:22 | major-upgrade-logical-replica/5-upgrade | Error from server (BadRequest): container "database" in pod "17-to-18-pgdata-qmrgm" is waiting to start: PodInitializing logger.go:42: 21:27:24 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 21:27:24 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 21:27:25 | major-upgrade-logical-replica/5-upgrade | Error from server (BadRequest): container "database" in pod "17-to-18-pgdata-qmrgm" is waiting to start: PodInitializing logger.go:42: 21:27:27 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 21:27:28 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 21:27:29 | major-upgrade-logical-replica/5-upgrade | Error from server (BadRequest): container "database" in pod "17-to-18-pgdata-qmrgm" is waiting to start: PodInitializing logger.go:42: 21:27:31 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 21:27:31 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 21:27:32 | major-upgrade-logical-replica/5-upgrade | Error from server (BadRequest): container "database" in pod "17-to-18-pgdata-qmrgm" is waiting to start: PodInitializing logger.go:42: 21:27:34 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 21:27:35 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 21:27:36 | major-upgrade-logical-replica/5-upgrade | Error from server (BadRequest): container "database" in pod "17-to-18-pgdata-qmrgm" is waiting to start: PodInitializing logger.go:42: 21:27:38 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 21:27:39 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 21:27:39 | major-upgrade-logical-replica/5-upgrade | Performing PostgreSQL upgrade from version "17" to "18" ... logger.go:42: 21:27:39 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:27:39 | major-upgrade-logical-replica/5-upgrade | Step 1: Making new pgdata directory... logger.go:42: 21:27:39 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:27:39 | major-upgrade-logical-replica/5-upgrade | Step 2: Initializing new pgdata directory... logger.go:42: 21:27:39 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:27:39 | major-upgrade-logical-replica/5-upgrade | The files belonging to this database system will be owned by user "postgres". logger.go:42: 21:27:39 | major-upgrade-logical-replica/5-upgrade | This user must also own the server process. logger.go:42: 21:27:39 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:27:39 | major-upgrade-logical-replica/5-upgrade | The database cluster will be initialized with locale "en_US.utf-8". logger.go:42: 21:27:39 | major-upgrade-logical-replica/5-upgrade | The default database encoding has accordingly been set to "UTF8". logger.go:42: 21:27:39 | major-upgrade-logical-replica/5-upgrade | The default text search configuration will be set to "english". logger.go:42: 21:27:39 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:27:39 | major-upgrade-logical-replica/5-upgrade | Data page checksums are enabled. logger.go:42: 21:27:39 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:27:39 | major-upgrade-logical-replica/5-upgrade | fixing permissions on existing directory /pgdata/pg18 ... ok logger.go:42: 21:27:39 | major-upgrade-logical-replica/5-upgrade | creating subdirectories ... ok logger.go:42: 21:27:39 | major-upgrade-logical-replica/5-upgrade | selecting dynamic shared memory implementation ... posix logger.go:42: 21:27:39 | major-upgrade-logical-replica/5-upgrade | selecting default "max_connections" ... 100 logger.go:42: 21:27:39 | major-upgrade-logical-replica/5-upgrade | selecting default "shared_buffers" ... 128MB logger.go:42: 21:27:39 | major-upgrade-logical-replica/5-upgrade | selecting default time zone ... UTC logger.go:42: 21:27:39 | major-upgrade-logical-replica/5-upgrade | creating configuration files ... ok logger.go:42: 21:27:39 | major-upgrade-logical-replica/5-upgrade | running bootstrap script ... ok logger.go:42: 21:27:39 | major-upgrade-logical-replica/5-upgrade | performing post-bootstrap initialization ... ok logger.go:42: 21:27:39 | major-upgrade-logical-replica/5-upgrade | syncing data to disk ... ok logger.go:42: 21:27:39 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:27:39 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:27:39 | major-upgrade-logical-replica/5-upgrade | Success. You can now start the database server using: logger.go:42: 21:27:39 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:27:39 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/pg_ctl -D /pgdata/pg18 -l logfile start logger.go:42: 21:27:39 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:27:39 | major-upgrade-logical-replica/5-upgrade | initdb: warning: enabling "trust" authentication for local connections logger.go:42: 21:27:39 | major-upgrade-logical-replica/5-upgrade | initdb: hint: You can change this by editing pg_hba.conf or using the option -A, or --auth-local and --auth-host, the next time you run initdb. logger.go:42: 21:27:39 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:27:39 | major-upgrade-logical-replica/5-upgrade | Step 3: Setting the expected permissions on the old pgdata directory... logger.go:42: 21:27:39 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:27:39 | major-upgrade-logical-replica/5-upgrade | Step 4: Copying shared_preload_libraries setting to new postgresql.conf file... logger.go:42: 21:27:39 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:27:39 | major-upgrade-logical-replica/5-upgrade | Step 4b: Copying logical replication settings to new postgresql.conf file... logger.go:42: 21:27:39 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:27:39 | major-upgrade-logical-replica/5-upgrade | Step 5: Running pg_upgrade check... logger.go:42: 21:27:39 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:27:39 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 21:27:39 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 21:27:39 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 21:27:39 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 21:27:39 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:27:39 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:27:39 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 21:27:39 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 21:27:39 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 21:27:39 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 21:27:42 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 21:27:42 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 21:27:43 | major-upgrade-logical-replica/5-upgrade | Performing PostgreSQL upgrade from version "17" to "18" ... logger.go:42: 21:27:43 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:27:43 | major-upgrade-logical-replica/5-upgrade | Step 1: Making new pgdata directory... logger.go:42: 21:27:43 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:27:43 | major-upgrade-logical-replica/5-upgrade | Step 2: Initializing new pgdata directory... logger.go:42: 21:27:43 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:27:43 | major-upgrade-logical-replica/5-upgrade | The files belonging to this database system will be owned by user "postgres". logger.go:42: 21:27:43 | major-upgrade-logical-replica/5-upgrade | This user must also own the server process. logger.go:42: 21:27:43 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:27:43 | major-upgrade-logical-replica/5-upgrade | The database cluster will be initialized with locale "en_US.utf-8". logger.go:42: 21:27:43 | major-upgrade-logical-replica/5-upgrade | The default database encoding has accordingly been set to "UTF8". logger.go:42: 21:27:43 | major-upgrade-logical-replica/5-upgrade | The default text search configuration will be set to "english". logger.go:42: 21:27:43 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:27:43 | major-upgrade-logical-replica/5-upgrade | Data page checksums are enabled. logger.go:42: 21:27:43 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:27:43 | major-upgrade-logical-replica/5-upgrade | fixing permissions on existing directory /pgdata/pg18 ... ok logger.go:42: 21:27:43 | major-upgrade-logical-replica/5-upgrade | creating subdirectories ... ok logger.go:42: 21:27:43 | major-upgrade-logical-replica/5-upgrade | selecting dynamic shared memory implementation ... posix logger.go:42: 21:27:43 | major-upgrade-logical-replica/5-upgrade | selecting default "max_connections" ... 100 logger.go:42: 21:27:43 | major-upgrade-logical-replica/5-upgrade | selecting default "shared_buffers" ... 128MB logger.go:42: 21:27:43 | major-upgrade-logical-replica/5-upgrade | selecting default time zone ... UTC logger.go:42: 21:27:43 | major-upgrade-logical-replica/5-upgrade | creating configuration files ... ok logger.go:42: 21:27:43 | major-upgrade-logical-replica/5-upgrade | running bootstrap script ... ok logger.go:42: 21:27:43 | major-upgrade-logical-replica/5-upgrade | performing post-bootstrap initialization ... ok logger.go:42: 21:27:43 | major-upgrade-logical-replica/5-upgrade | syncing data to disk ... ok logger.go:42: 21:27:43 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:27:43 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:27:43 | major-upgrade-logical-replica/5-upgrade | Success. You can now start the database server using: logger.go:42: 21:27:43 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:27:43 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/pg_ctl -D /pgdata/pg18 -l logfile start logger.go:42: 21:27:43 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:27:43 | major-upgrade-logical-replica/5-upgrade | initdb: warning: enabling "trust" authentication for local connections logger.go:42: 21:27:43 | major-upgrade-logical-replica/5-upgrade | initdb: hint: You can change this by editing pg_hba.conf or using the option -A, or --auth-local and --auth-host, the next time you run initdb. logger.go:42: 21:27:43 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:27:43 | major-upgrade-logical-replica/5-upgrade | Step 3: Setting the expected permissions on the old pgdata directory... logger.go:42: 21:27:43 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:27:43 | major-upgrade-logical-replica/5-upgrade | Step 4: Copying shared_preload_libraries setting to new postgresql.conf file... logger.go:42: 21:27:43 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:27:43 | major-upgrade-logical-replica/5-upgrade | Step 4b: Copying logical replication settings to new postgresql.conf file... logger.go:42: 21:27:43 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:27:43 | major-upgrade-logical-replica/5-upgrade | Step 5: Running pg_upgrade check... logger.go:42: 21:27:43 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:27:43 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 21:27:43 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 21:27:43 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 21:27:43 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 21:27:43 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:27:43 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:27:43 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 21:27:43 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 21:27:43 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 21:27:43 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 21:27:43 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 21:27:43 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 21:27:43 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 21:27:43 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:27:43 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:27:43 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 21:27:43 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 21:27:43 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:27:43 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 21:27:43 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:27:43 | major-upgrade-logical-replica/5-upgrade | real 0m1.940s logger.go:42: 21:27:43 | major-upgrade-logical-replica/5-upgrade | user 0m0.088s logger.go:42: 21:27:43 | major-upgrade-logical-replica/5-upgrade | sys 0m0.098s logger.go:42: 21:27:43 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:27:43 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 21:27:43 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:27:43 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 21:27:43 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 21:27:43 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 21:27:43 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 21:27:43 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:27:43 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:27:43 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 21:27:43 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 21:27:43 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 21:27:43 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 21:27:43 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 21:27:43 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 21:27:43 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 21:27:43 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 21:27:43 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 21:27:43 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:27:43 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:27:43 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 21:27:43 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 21:27:43 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:27:43 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 21:27:43 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 21:27:43 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:27:43 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 21:27:43 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 21:27:43 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 21:27:43 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 21:27:43 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 21:27:43 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 21:27:43 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 21:27:45 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 21:27:46 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 21:27:46 | major-upgrade-logical-replica/5-upgrade | Performing PostgreSQL upgrade from version "17" to "18" ... logger.go:42: 21:27:46 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:27:46 | major-upgrade-logical-replica/5-upgrade | Step 1: Making new pgdata directory... logger.go:42: 21:27:46 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:27:46 | major-upgrade-logical-replica/5-upgrade | Step 2: Initializing new pgdata directory... logger.go:42: 21:27:46 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:27:46 | major-upgrade-logical-replica/5-upgrade | The files belonging to this database system will be owned by user "postgres". logger.go:42: 21:27:46 | major-upgrade-logical-replica/5-upgrade | This user must also own the server process. logger.go:42: 21:27:46 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:27:46 | major-upgrade-logical-replica/5-upgrade | The database cluster will be initialized with locale "en_US.utf-8". logger.go:42: 21:27:46 | major-upgrade-logical-replica/5-upgrade | The default database encoding has accordingly been set to "UTF8". logger.go:42: 21:27:46 | major-upgrade-logical-replica/5-upgrade | The default text search configuration will be set to "english". logger.go:42: 21:27:46 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:27:46 | major-upgrade-logical-replica/5-upgrade | Data page checksums are enabled. logger.go:42: 21:27:46 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:27:46 | major-upgrade-logical-replica/5-upgrade | fixing permissions on existing directory /pgdata/pg18 ... ok logger.go:42: 21:27:46 | major-upgrade-logical-replica/5-upgrade | creating subdirectories ... ok logger.go:42: 21:27:46 | major-upgrade-logical-replica/5-upgrade | selecting dynamic shared memory implementation ... posix logger.go:42: 21:27:46 | major-upgrade-logical-replica/5-upgrade | selecting default "max_connections" ... 100 logger.go:42: 21:27:46 | major-upgrade-logical-replica/5-upgrade | selecting default "shared_buffers" ... 128MB logger.go:42: 21:27:46 | major-upgrade-logical-replica/5-upgrade | selecting default time zone ... UTC logger.go:42: 21:27:46 | major-upgrade-logical-replica/5-upgrade | creating configuration files ... ok logger.go:42: 21:27:46 | major-upgrade-logical-replica/5-upgrade | running bootstrap script ... ok logger.go:42: 21:27:46 | major-upgrade-logical-replica/5-upgrade | performing post-bootstrap initialization ... ok logger.go:42: 21:27:46 | major-upgrade-logical-replica/5-upgrade | syncing data to disk ... ok logger.go:42: 21:27:46 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:27:46 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:27:46 | major-upgrade-logical-replica/5-upgrade | Success. You can now start the database server using: logger.go:42: 21:27:46 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:27:46 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/pg_ctl -D /pgdata/pg18 -l logfile start logger.go:42: 21:27:46 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:27:46 | major-upgrade-logical-replica/5-upgrade | initdb: warning: enabling "trust" authentication for local connections logger.go:42: 21:27:46 | major-upgrade-logical-replica/5-upgrade | initdb: hint: You can change this by editing pg_hba.conf or using the option -A, or --auth-local and --auth-host, the next time you run initdb. logger.go:42: 21:27:46 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:27:46 | major-upgrade-logical-replica/5-upgrade | Step 3: Setting the expected permissions on the old pgdata directory... logger.go:42: 21:27:46 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:27:46 | major-upgrade-logical-replica/5-upgrade | Step 4: Copying shared_preload_libraries setting to new postgresql.conf file... logger.go:42: 21:27:46 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:27:46 | major-upgrade-logical-replica/5-upgrade | Step 4b: Copying logical replication settings to new postgresql.conf file... logger.go:42: 21:27:46 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:27:46 | major-upgrade-logical-replica/5-upgrade | Step 5: Running pg_upgrade check... logger.go:42: 21:27:46 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:27:46 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 21:27:46 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 21:27:46 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 21:27:46 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 21:27:46 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:27:46 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:27:46 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 21:27:46 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 21:27:46 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 21:27:46 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 21:27:46 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 21:27:46 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 21:27:46 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 21:27:46 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:27:46 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:27:46 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 21:27:46 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 21:27:46 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:27:46 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 21:27:46 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:27:46 | major-upgrade-logical-replica/5-upgrade | real 0m1.940s logger.go:42: 21:27:46 | major-upgrade-logical-replica/5-upgrade | user 0m0.088s logger.go:42: 21:27:46 | major-upgrade-logical-replica/5-upgrade | sys 0m0.098s logger.go:42: 21:27:46 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:27:46 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 21:27:46 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:27:46 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 21:27:46 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 21:27:46 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 21:27:46 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 21:27:46 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:27:46 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:27:46 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 21:27:46 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 21:27:46 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 21:27:46 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 21:27:46 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 21:27:46 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 21:27:46 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 21:27:46 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 21:27:46 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 21:27:46 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:27:46 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:27:46 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 21:27:46 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 21:27:46 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:27:46 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 21:27:46 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 21:27:46 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:27:46 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 21:27:46 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 21:27:46 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 21:27:46 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 21:27:46 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 21:27:46 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 21:27:46 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 21:27:46 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 21:27:49 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 21:27:49 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 21:27:50 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 21:27:50 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:27:50 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:27:50 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 21:27:50 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 21:27:50 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 21:27:50 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 21:27:50 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 21:27:50 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 21:27:50 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 21:27:50 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:27:50 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:27:50 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 21:27:50 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 21:27:50 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:27:50 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 21:27:50 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:27:50 | major-upgrade-logical-replica/5-upgrade | real 0m1.940s logger.go:42: 21:27:50 | major-upgrade-logical-replica/5-upgrade | user 0m0.088s logger.go:42: 21:27:50 | major-upgrade-logical-replica/5-upgrade | sys 0m0.098s logger.go:42: 21:27:50 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:27:50 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 21:27:50 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:27:50 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 21:27:50 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 21:27:50 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 21:27:50 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 21:27:50 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:27:50 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:27:50 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 21:27:50 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 21:27:50 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 21:27:50 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 21:27:50 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 21:27:50 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 21:27:50 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 21:27:50 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 21:27:50 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 21:27:50 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:27:50 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:27:50 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 21:27:50 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 21:27:50 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:27:50 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 21:27:50 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 21:27:50 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:27:50 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 21:27:50 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 21:27:50 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 21:27:50 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 21:27:50 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 21:27:50 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 21:27:50 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 21:27:50 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 21:27:50 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 21:27:50 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 21:27:50 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 21:27:50 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 21:27:50 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 21:27:50 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 21:27:50 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 21:27:50 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 21:27:50 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 21:27:50 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 21:27:50 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 21:27:50 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:27:50 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 21:27:50 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 21:27:50 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 21:27:50 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 21:27:50 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 21:27:50 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 21:27:50 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 21:27:50 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 21:27:50 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 21:27:50 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 21:27:50 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:27:50 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 21:27:50 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 21:27:50 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 21:27:50 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 21:27:50 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 21:27:50 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:27:50 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 21:27:50 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 21:27:50 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 21:27:50 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 21:27:50 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 21:27:50 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 21:27:50 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 21:27:50 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 21:27:50 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:27:50 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 21:27:50 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:27:50 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:27:50 | major-upgrade-logical-replica/5-upgrade | real 0m9.551s logger.go:42: 21:27:50 | major-upgrade-logical-replica/5-upgrade | user 0m0.247s logger.go:42: 21:27:50 | major-upgrade-logical-replica/5-upgrade | sys 0m0.461s logger.go:42: 21:27:50 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:27:50 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 21:27:52 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 21:27:53 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 21:27:54 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 21:27:54 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:27:54 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:27:54 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 21:27:54 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 21:27:54 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 21:27:54 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 21:27:54 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 21:27:54 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 21:27:54 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 21:27:54 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:27:54 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:27:54 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 21:27:54 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 21:27:54 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:27:54 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 21:27:54 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:27:54 | major-upgrade-logical-replica/5-upgrade | real 0m1.940s logger.go:42: 21:27:54 | major-upgrade-logical-replica/5-upgrade | user 0m0.088s logger.go:42: 21:27:54 | major-upgrade-logical-replica/5-upgrade | sys 0m0.098s logger.go:42: 21:27:54 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:27:54 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 21:27:54 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:27:54 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 21:27:54 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 21:27:54 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 21:27:54 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 21:27:54 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:27:54 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:27:54 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 21:27:54 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 21:27:54 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 21:27:54 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 21:27:54 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 21:27:54 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 21:27:54 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 21:27:54 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 21:27:54 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 21:27:54 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:27:54 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:27:54 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 21:27:54 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 21:27:54 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:27:54 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 21:27:54 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 21:27:54 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:27:54 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 21:27:54 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 21:27:54 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 21:27:54 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 21:27:54 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 21:27:54 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 21:27:54 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 21:27:54 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 21:27:54 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 21:27:54 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 21:27:54 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 21:27:54 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 21:27:54 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 21:27:54 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 21:27:54 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 21:27:54 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 21:27:54 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 21:27:54 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 21:27:54 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 21:27:54 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:27:54 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 21:27:54 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 21:27:54 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 21:27:54 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 21:27:54 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 21:27:54 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 21:27:54 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 21:27:54 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 21:27:54 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 21:27:54 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 21:27:54 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:27:54 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 21:27:54 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 21:27:54 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 21:27:54 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 21:27:54 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 21:27:54 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:27:54 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 21:27:54 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 21:27:54 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 21:27:54 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 21:27:54 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 21:27:54 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 21:27:54 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 21:27:54 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 21:27:54 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:27:54 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 21:27:54 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:27:54 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:27:54 | major-upgrade-logical-replica/5-upgrade | real 0m9.551s logger.go:42: 21:27:54 | major-upgrade-logical-replica/5-upgrade | user 0m0.247s logger.go:42: 21:27:54 | major-upgrade-logical-replica/5-upgrade | sys 0m0.461s logger.go:42: 21:27:54 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:27:54 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 21:27:56 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 21:27:56 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 21:27:57 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 21:27:57 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:27:57 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:27:57 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 21:27:57 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 21:27:57 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 21:27:57 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 21:27:57 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 21:27:57 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 21:27:57 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 21:27:57 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:27:57 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:27:57 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 21:27:57 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 21:27:57 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:27:57 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 21:27:57 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:27:57 | major-upgrade-logical-replica/5-upgrade | real 0m1.940s logger.go:42: 21:27:57 | major-upgrade-logical-replica/5-upgrade | user 0m0.088s logger.go:42: 21:27:57 | major-upgrade-logical-replica/5-upgrade | sys 0m0.098s logger.go:42: 21:27:57 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:27:57 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 21:27:57 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:27:57 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 21:27:57 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 21:27:57 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 21:27:57 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 21:27:57 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:27:57 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:27:57 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 21:27:57 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 21:27:57 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 21:27:57 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 21:27:57 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 21:27:57 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 21:27:57 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 21:27:57 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 21:27:57 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 21:27:57 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:27:57 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:27:57 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 21:27:57 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 21:27:57 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:27:57 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 21:27:57 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 21:27:57 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:27:57 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 21:27:57 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 21:27:57 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 21:27:57 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 21:27:57 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 21:27:57 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 21:27:57 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 21:27:57 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 21:27:57 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 21:27:57 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 21:27:57 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 21:27:57 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 21:27:57 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 21:27:57 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 21:27:57 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 21:27:57 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 21:27:57 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 21:27:57 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 21:27:57 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 21:27:57 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:27:57 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 21:27:57 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 21:27:57 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 21:27:57 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 21:27:57 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 21:27:57 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 21:27:57 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 21:27:57 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 21:27:57 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 21:27:57 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 21:27:57 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:27:57 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 21:27:57 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 21:27:57 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 21:27:57 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 21:27:57 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 21:27:57 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:27:57 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 21:27:57 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 21:27:57 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 21:27:57 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 21:27:57 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 21:27:57 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 21:27:57 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 21:27:57 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 21:27:57 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:27:57 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 21:27:57 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:27:57 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:27:57 | major-upgrade-logical-replica/5-upgrade | real 0m9.551s logger.go:42: 21:27:57 | major-upgrade-logical-replica/5-upgrade | user 0m0.247s logger.go:42: 21:27:57 | major-upgrade-logical-replica/5-upgrade | sys 0m0.461s logger.go:42: 21:27:57 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:27:57 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 21:27:59 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 21:28:00 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 21:28:01 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 21:28:01 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:28:01 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:28:01 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 21:28:01 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 21:28:01 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 21:28:01 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 21:28:01 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 21:28:01 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 21:28:01 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 21:28:01 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:28:01 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:28:01 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 21:28:01 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 21:28:01 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:01 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 21:28:01 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:01 | major-upgrade-logical-replica/5-upgrade | real 0m1.940s logger.go:42: 21:28:01 | major-upgrade-logical-replica/5-upgrade | user 0m0.088s logger.go:42: 21:28:01 | major-upgrade-logical-replica/5-upgrade | sys 0m0.098s logger.go:42: 21:28:01 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:01 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 21:28:01 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:01 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 21:28:01 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 21:28:01 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 21:28:01 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 21:28:01 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:28:01 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:28:01 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 21:28:01 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 21:28:01 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 21:28:01 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 21:28:01 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 21:28:01 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 21:28:01 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 21:28:01 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 21:28:01 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 21:28:01 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:28:01 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:28:01 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 21:28:01 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 21:28:01 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:01 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 21:28:01 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 21:28:01 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:01 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 21:28:01 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 21:28:01 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 21:28:01 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 21:28:01 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 21:28:01 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 21:28:01 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 21:28:01 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 21:28:01 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 21:28:01 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 21:28:01 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 21:28:01 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 21:28:01 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 21:28:01 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 21:28:01 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 21:28:01 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 21:28:01 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 21:28:01 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 21:28:01 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 21:28:01 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:01 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 21:28:01 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 21:28:01 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 21:28:01 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 21:28:01 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 21:28:01 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 21:28:01 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 21:28:01 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 21:28:01 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 21:28:01 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 21:28:01 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:01 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 21:28:01 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 21:28:01 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 21:28:01 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 21:28:01 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 21:28:01 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:01 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 21:28:01 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 21:28:01 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 21:28:01 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 21:28:01 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 21:28:01 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 21:28:01 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 21:28:01 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 21:28:01 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:01 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 21:28:01 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:01 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:01 | major-upgrade-logical-replica/5-upgrade | real 0m9.551s logger.go:42: 21:28:01 | major-upgrade-logical-replica/5-upgrade | user 0m0.247s logger.go:42: 21:28:01 | major-upgrade-logical-replica/5-upgrade | sys 0m0.461s logger.go:42: 21:28:01 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:01 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 21:28:03 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 21:28:04 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 21:28:05 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 21:28:05 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:28:05 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:28:05 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 21:28:05 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 21:28:05 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 21:28:05 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 21:28:05 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 21:28:05 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 21:28:05 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 21:28:05 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:28:05 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:28:05 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 21:28:05 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 21:28:05 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:05 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 21:28:05 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:05 | major-upgrade-logical-replica/5-upgrade | real 0m1.940s logger.go:42: 21:28:05 | major-upgrade-logical-replica/5-upgrade | user 0m0.088s logger.go:42: 21:28:05 | major-upgrade-logical-replica/5-upgrade | sys 0m0.098s logger.go:42: 21:28:05 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:05 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 21:28:05 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:05 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 21:28:05 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 21:28:05 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 21:28:05 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 21:28:05 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:28:05 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:28:05 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 21:28:05 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 21:28:05 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 21:28:05 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 21:28:05 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 21:28:05 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 21:28:05 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 21:28:05 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 21:28:05 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 21:28:05 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:28:05 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:28:05 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 21:28:05 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 21:28:05 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:05 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 21:28:05 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 21:28:05 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:05 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 21:28:05 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 21:28:05 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 21:28:05 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 21:28:05 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 21:28:05 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 21:28:05 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 21:28:05 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 21:28:05 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 21:28:05 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 21:28:05 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 21:28:05 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 21:28:05 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 21:28:05 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 21:28:05 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 21:28:05 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 21:28:05 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 21:28:05 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 21:28:05 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 21:28:05 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:05 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 21:28:05 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 21:28:05 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 21:28:05 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 21:28:05 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 21:28:05 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 21:28:05 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 21:28:05 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 21:28:05 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 21:28:05 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 21:28:05 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:05 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 21:28:05 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 21:28:05 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 21:28:05 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 21:28:05 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 21:28:05 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:05 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 21:28:05 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 21:28:05 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 21:28:05 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 21:28:05 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 21:28:05 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 21:28:05 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 21:28:05 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 21:28:05 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:05 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 21:28:05 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:05 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:05 | major-upgrade-logical-replica/5-upgrade | real 0m9.551s logger.go:42: 21:28:05 | major-upgrade-logical-replica/5-upgrade | user 0m0.247s logger.go:42: 21:28:05 | major-upgrade-logical-replica/5-upgrade | sys 0m0.461s logger.go:42: 21:28:05 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:05 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 21:28:07 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 21:28:07 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 21:28:09 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 21:28:09 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:28:09 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:28:09 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 21:28:09 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 21:28:09 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 21:28:09 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 21:28:09 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 21:28:09 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 21:28:09 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 21:28:09 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:28:09 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:28:09 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 21:28:09 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 21:28:09 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:09 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 21:28:09 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:09 | major-upgrade-logical-replica/5-upgrade | real 0m1.940s logger.go:42: 21:28:09 | major-upgrade-logical-replica/5-upgrade | user 0m0.088s logger.go:42: 21:28:09 | major-upgrade-logical-replica/5-upgrade | sys 0m0.098s logger.go:42: 21:28:09 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:09 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 21:28:09 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:09 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 21:28:09 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 21:28:09 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 21:28:09 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 21:28:09 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:28:09 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:28:09 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 21:28:09 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 21:28:09 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 21:28:09 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 21:28:09 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 21:28:09 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 21:28:09 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 21:28:09 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 21:28:09 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 21:28:09 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:28:09 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:28:09 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 21:28:09 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 21:28:09 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:09 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 21:28:09 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 21:28:09 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:09 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 21:28:09 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 21:28:09 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 21:28:09 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 21:28:09 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 21:28:09 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 21:28:09 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 21:28:09 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 21:28:09 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 21:28:09 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 21:28:09 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 21:28:09 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 21:28:09 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 21:28:09 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 21:28:09 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 21:28:09 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 21:28:09 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 21:28:09 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 21:28:09 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 21:28:09 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:09 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 21:28:09 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 21:28:09 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 21:28:09 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 21:28:09 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 21:28:09 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 21:28:09 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 21:28:09 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 21:28:09 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 21:28:09 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 21:28:09 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:09 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 21:28:09 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 21:28:09 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 21:28:09 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 21:28:09 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 21:28:09 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:09 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 21:28:09 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 21:28:09 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 21:28:09 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 21:28:09 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 21:28:09 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 21:28:09 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 21:28:09 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 21:28:09 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:09 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 21:28:09 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:09 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:09 | major-upgrade-logical-replica/5-upgrade | real 0m9.551s logger.go:42: 21:28:09 | major-upgrade-logical-replica/5-upgrade | user 0m0.247s logger.go:42: 21:28:09 | major-upgrade-logical-replica/5-upgrade | sys 0m0.461s logger.go:42: 21:28:09 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:09 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 21:28:11 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 21:28:11 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 21:28:12 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 21:28:12 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:28:12 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:28:12 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 21:28:12 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 21:28:12 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 21:28:12 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 21:28:12 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 21:28:12 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 21:28:12 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 21:28:12 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:28:12 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:28:12 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 21:28:12 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 21:28:12 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:12 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 21:28:12 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:12 | major-upgrade-logical-replica/5-upgrade | real 0m1.940s logger.go:42: 21:28:12 | major-upgrade-logical-replica/5-upgrade | user 0m0.088s logger.go:42: 21:28:12 | major-upgrade-logical-replica/5-upgrade | sys 0m0.098s logger.go:42: 21:28:12 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:12 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 21:28:12 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:12 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 21:28:12 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 21:28:12 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 21:28:12 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 21:28:12 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:28:12 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:28:12 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 21:28:12 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 21:28:12 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 21:28:12 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 21:28:12 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 21:28:12 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 21:28:12 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 21:28:12 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 21:28:12 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 21:28:12 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:28:12 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:28:12 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 21:28:12 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 21:28:12 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:12 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 21:28:12 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 21:28:12 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:12 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 21:28:12 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 21:28:12 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 21:28:12 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 21:28:12 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 21:28:12 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 21:28:12 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 21:28:12 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 21:28:12 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 21:28:12 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 21:28:12 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 21:28:12 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 21:28:12 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 21:28:12 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 21:28:12 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 21:28:12 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 21:28:12 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 21:28:12 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 21:28:12 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 21:28:12 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:12 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 21:28:12 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 21:28:12 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 21:28:12 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 21:28:12 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 21:28:12 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 21:28:12 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 21:28:12 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 21:28:12 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 21:28:12 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 21:28:12 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:12 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 21:28:12 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 21:28:12 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 21:28:12 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 21:28:12 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 21:28:12 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:12 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 21:28:12 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 21:28:12 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 21:28:12 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 21:28:12 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 21:28:12 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 21:28:12 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 21:28:12 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 21:28:12 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:12 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 21:28:12 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:12 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:12 | major-upgrade-logical-replica/5-upgrade | real 0m9.551s logger.go:42: 21:28:12 | major-upgrade-logical-replica/5-upgrade | user 0m0.247s logger.go:42: 21:28:12 | major-upgrade-logical-replica/5-upgrade | sys 0m0.461s logger.go:42: 21:28:12 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:12 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 21:28:14 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 21:28:15 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 21:28:16 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 21:28:16 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:28:16 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:28:16 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 21:28:16 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 21:28:16 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 21:28:16 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 21:28:16 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 21:28:16 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 21:28:16 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 21:28:16 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:28:16 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:28:16 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 21:28:16 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 21:28:16 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:16 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 21:28:16 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:16 | major-upgrade-logical-replica/5-upgrade | real 0m1.940s logger.go:42: 21:28:16 | major-upgrade-logical-replica/5-upgrade | user 0m0.088s logger.go:42: 21:28:16 | major-upgrade-logical-replica/5-upgrade | sys 0m0.098s logger.go:42: 21:28:16 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:16 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 21:28:16 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:16 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 21:28:16 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 21:28:16 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 21:28:16 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 21:28:16 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:28:16 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:28:16 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 21:28:16 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 21:28:16 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 21:28:16 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 21:28:16 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 21:28:16 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 21:28:16 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 21:28:16 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 21:28:16 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 21:28:16 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:28:16 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:28:16 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 21:28:16 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 21:28:16 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:16 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 21:28:16 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 21:28:16 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:16 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 21:28:16 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 21:28:16 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 21:28:16 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 21:28:16 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 21:28:16 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 21:28:16 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 21:28:16 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 21:28:16 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 21:28:16 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 21:28:16 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 21:28:16 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 21:28:16 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 21:28:16 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 21:28:16 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 21:28:16 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 21:28:16 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 21:28:16 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 21:28:16 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 21:28:16 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:16 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 21:28:16 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 21:28:16 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 21:28:16 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 21:28:16 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 21:28:16 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 21:28:16 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 21:28:16 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 21:28:16 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 21:28:16 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 21:28:16 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:16 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 21:28:16 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 21:28:16 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 21:28:16 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 21:28:16 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 21:28:16 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:16 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 21:28:16 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 21:28:16 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 21:28:16 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 21:28:16 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 21:28:16 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 21:28:16 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 21:28:16 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 21:28:16 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:16 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 21:28:16 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:16 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:16 | major-upgrade-logical-replica/5-upgrade | real 0m9.551s logger.go:42: 21:28:16 | major-upgrade-logical-replica/5-upgrade | user 0m0.247s logger.go:42: 21:28:16 | major-upgrade-logical-replica/5-upgrade | sys 0m0.461s logger.go:42: 21:28:16 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:16 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 21:28:18 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 21:28:19 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 21:28:19 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 21:28:19 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:28:19 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:28:19 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 21:28:19 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 21:28:19 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 21:28:19 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 21:28:19 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 21:28:19 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 21:28:19 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 21:28:19 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:28:19 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:28:19 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 21:28:19 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 21:28:19 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:19 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 21:28:19 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:19 | major-upgrade-logical-replica/5-upgrade | real 0m1.940s logger.go:42: 21:28:19 | major-upgrade-logical-replica/5-upgrade | user 0m0.088s logger.go:42: 21:28:19 | major-upgrade-logical-replica/5-upgrade | sys 0m0.098s logger.go:42: 21:28:19 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:19 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 21:28:19 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:19 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 21:28:19 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 21:28:19 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 21:28:19 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 21:28:19 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:28:19 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:28:19 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 21:28:19 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 21:28:19 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 21:28:19 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 21:28:19 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 21:28:19 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 21:28:19 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 21:28:19 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 21:28:19 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 21:28:19 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:28:19 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:28:19 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 21:28:19 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 21:28:19 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:19 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 21:28:19 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 21:28:19 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:19 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 21:28:19 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 21:28:19 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 21:28:19 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 21:28:19 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 21:28:19 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 21:28:19 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 21:28:19 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 21:28:19 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 21:28:19 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 21:28:19 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 21:28:19 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 21:28:19 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 21:28:19 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 21:28:19 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 21:28:19 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 21:28:19 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 21:28:19 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 21:28:19 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 21:28:19 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:19 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 21:28:19 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 21:28:19 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 21:28:19 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 21:28:19 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 21:28:19 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 21:28:19 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 21:28:19 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 21:28:19 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 21:28:19 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 21:28:19 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:19 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 21:28:19 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 21:28:19 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 21:28:19 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 21:28:19 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 21:28:19 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:19 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 21:28:19 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 21:28:19 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 21:28:19 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 21:28:19 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 21:28:19 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 21:28:19 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 21:28:19 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 21:28:19 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:19 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 21:28:19 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:19 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:19 | major-upgrade-logical-replica/5-upgrade | real 0m9.551s logger.go:42: 21:28:19 | major-upgrade-logical-replica/5-upgrade | user 0m0.247s logger.go:42: 21:28:19 | major-upgrade-logical-replica/5-upgrade | sys 0m0.461s logger.go:42: 21:28:19 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:19 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 21:28:22 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 21:28:22 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 21:28:23 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 21:28:23 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:28:23 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:28:23 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 21:28:23 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 21:28:23 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 21:28:23 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 21:28:23 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 21:28:23 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 21:28:23 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 21:28:23 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:28:23 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:28:23 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 21:28:23 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 21:28:23 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:23 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 21:28:23 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:23 | major-upgrade-logical-replica/5-upgrade | real 0m1.940s logger.go:42: 21:28:23 | major-upgrade-logical-replica/5-upgrade | user 0m0.088s logger.go:42: 21:28:23 | major-upgrade-logical-replica/5-upgrade | sys 0m0.098s logger.go:42: 21:28:23 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:23 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 21:28:23 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:23 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 21:28:23 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 21:28:23 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 21:28:23 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 21:28:23 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:28:23 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:28:23 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 21:28:23 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 21:28:23 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 21:28:23 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 21:28:23 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 21:28:23 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 21:28:23 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 21:28:23 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 21:28:23 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 21:28:23 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:28:23 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:28:23 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 21:28:23 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 21:28:23 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:23 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 21:28:23 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 21:28:23 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:23 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 21:28:23 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 21:28:23 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 21:28:23 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 21:28:23 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 21:28:23 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 21:28:23 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 21:28:23 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 21:28:23 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 21:28:23 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 21:28:23 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 21:28:23 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 21:28:23 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 21:28:23 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 21:28:23 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 21:28:23 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 21:28:23 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 21:28:23 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 21:28:23 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 21:28:23 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:23 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 21:28:23 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 21:28:23 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 21:28:23 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 21:28:23 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 21:28:23 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 21:28:23 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 21:28:23 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 21:28:23 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 21:28:23 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 21:28:23 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:23 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 21:28:23 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 21:28:23 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 21:28:23 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 21:28:23 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 21:28:23 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:23 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 21:28:23 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 21:28:23 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 21:28:23 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 21:28:23 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 21:28:23 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 21:28:23 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 21:28:23 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 21:28:23 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:23 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 21:28:23 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:23 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:23 | major-upgrade-logical-replica/5-upgrade | real 0m9.551s logger.go:42: 21:28:23 | major-upgrade-logical-replica/5-upgrade | user 0m0.247s logger.go:42: 21:28:23 | major-upgrade-logical-replica/5-upgrade | sys 0m0.461s logger.go:42: 21:28:23 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:23 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 21:28:25 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 21:28:26 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 21:28:27 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 21:28:27 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:28:27 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:28:27 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 21:28:27 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 21:28:27 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 21:28:27 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 21:28:27 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 21:28:27 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 21:28:27 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 21:28:27 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:28:27 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:28:27 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 21:28:27 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 21:28:27 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:27 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 21:28:27 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:27 | major-upgrade-logical-replica/5-upgrade | real 0m1.940s logger.go:42: 21:28:27 | major-upgrade-logical-replica/5-upgrade | user 0m0.088s logger.go:42: 21:28:27 | major-upgrade-logical-replica/5-upgrade | sys 0m0.098s logger.go:42: 21:28:27 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:27 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 21:28:27 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:27 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 21:28:27 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 21:28:27 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 21:28:27 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 21:28:27 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:28:27 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:28:27 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 21:28:27 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 21:28:27 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 21:28:27 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 21:28:27 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 21:28:27 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 21:28:27 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 21:28:27 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 21:28:27 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 21:28:27 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:28:27 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:28:27 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 21:28:27 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 21:28:27 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:27 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 21:28:27 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 21:28:27 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:27 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 21:28:27 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 21:28:27 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 21:28:27 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 21:28:27 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 21:28:27 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 21:28:27 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 21:28:27 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 21:28:27 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 21:28:27 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 21:28:27 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 21:28:27 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 21:28:27 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 21:28:27 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 21:28:27 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 21:28:27 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 21:28:27 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 21:28:27 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 21:28:27 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 21:28:27 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:27 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 21:28:27 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 21:28:27 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 21:28:27 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 21:28:27 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 21:28:27 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 21:28:27 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 21:28:27 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 21:28:27 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 21:28:27 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 21:28:27 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:27 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 21:28:27 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 21:28:27 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 21:28:27 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 21:28:27 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 21:28:27 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:27 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 21:28:27 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 21:28:27 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 21:28:27 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 21:28:27 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 21:28:27 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 21:28:27 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 21:28:27 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 21:28:27 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:27 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 21:28:27 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:27 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:27 | major-upgrade-logical-replica/5-upgrade | real 0m9.551s logger.go:42: 21:28:27 | major-upgrade-logical-replica/5-upgrade | user 0m0.247s logger.go:42: 21:28:27 | major-upgrade-logical-replica/5-upgrade | sys 0m0.461s logger.go:42: 21:28:27 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:27 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 21:28:29 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 21:28:30 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 21:28:31 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 21:28:31 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:28:31 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:28:31 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 21:28:31 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 21:28:31 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 21:28:31 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 21:28:31 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 21:28:31 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 21:28:31 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 21:28:31 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:28:31 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:28:31 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 21:28:31 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 21:28:31 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:31 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 21:28:31 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:31 | major-upgrade-logical-replica/5-upgrade | real 0m1.940s logger.go:42: 21:28:31 | major-upgrade-logical-replica/5-upgrade | user 0m0.088s logger.go:42: 21:28:31 | major-upgrade-logical-replica/5-upgrade | sys 0m0.098s logger.go:42: 21:28:31 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:31 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 21:28:31 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:31 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 21:28:31 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 21:28:31 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 21:28:31 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 21:28:31 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:28:31 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:28:31 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 21:28:31 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 21:28:31 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 21:28:31 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 21:28:31 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 21:28:31 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 21:28:31 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 21:28:31 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 21:28:31 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 21:28:31 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:28:31 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:28:31 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 21:28:31 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 21:28:31 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:31 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 21:28:31 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 21:28:31 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:31 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 21:28:31 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 21:28:31 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 21:28:31 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 21:28:31 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 21:28:31 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 21:28:31 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 21:28:31 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 21:28:31 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 21:28:31 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 21:28:31 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 21:28:31 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 21:28:31 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 21:28:31 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 21:28:31 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 21:28:31 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 21:28:31 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 21:28:31 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 21:28:31 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 21:28:31 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:31 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 21:28:31 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 21:28:31 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 21:28:31 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 21:28:31 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 21:28:31 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 21:28:31 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 21:28:31 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 21:28:31 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 21:28:31 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 21:28:31 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:31 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 21:28:31 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 21:28:31 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 21:28:31 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 21:28:31 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 21:28:31 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:31 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 21:28:31 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 21:28:31 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 21:28:31 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 21:28:31 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 21:28:31 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 21:28:31 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 21:28:31 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 21:28:31 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:31 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 21:28:31 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:31 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:31 | major-upgrade-logical-replica/5-upgrade | real 0m9.551s logger.go:42: 21:28:31 | major-upgrade-logical-replica/5-upgrade | user 0m0.247s logger.go:42: 21:28:31 | major-upgrade-logical-replica/5-upgrade | sys 0m0.461s logger.go:42: 21:28:31 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:31 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 21:28:33 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 21:28:33 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 21:28:34 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 21:28:34 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:28:34 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:28:34 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 21:28:34 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 21:28:34 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 21:28:34 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 21:28:34 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 21:28:34 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 21:28:34 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 21:28:34 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:28:34 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:28:34 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 21:28:34 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 21:28:34 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:34 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 21:28:34 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:34 | major-upgrade-logical-replica/5-upgrade | real 0m1.940s logger.go:42: 21:28:34 | major-upgrade-logical-replica/5-upgrade | user 0m0.088s logger.go:42: 21:28:34 | major-upgrade-logical-replica/5-upgrade | sys 0m0.098s logger.go:42: 21:28:34 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:34 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 21:28:34 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:34 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 21:28:34 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 21:28:34 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 21:28:34 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 21:28:34 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:28:34 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:28:34 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 21:28:34 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 21:28:34 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 21:28:34 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 21:28:34 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 21:28:34 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 21:28:34 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 21:28:34 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 21:28:34 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 21:28:34 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:28:34 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:28:34 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 21:28:34 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 21:28:34 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:34 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 21:28:34 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 21:28:34 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:34 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 21:28:34 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 21:28:34 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 21:28:34 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 21:28:34 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 21:28:34 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 21:28:34 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 21:28:34 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 21:28:34 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 21:28:34 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 21:28:34 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 21:28:34 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 21:28:34 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 21:28:34 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 21:28:34 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 21:28:34 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 21:28:34 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 21:28:34 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 21:28:34 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 21:28:34 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:34 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 21:28:34 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 21:28:34 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 21:28:34 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 21:28:34 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 21:28:34 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 21:28:34 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 21:28:34 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 21:28:34 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 21:28:34 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 21:28:34 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:34 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 21:28:34 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 21:28:34 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 21:28:34 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 21:28:34 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 21:28:34 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:34 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 21:28:34 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 21:28:34 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 21:28:34 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 21:28:34 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 21:28:34 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 21:28:34 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 21:28:34 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 21:28:34 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:34 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 21:28:34 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:34 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:34 | major-upgrade-logical-replica/5-upgrade | real 0m9.551s logger.go:42: 21:28:34 | major-upgrade-logical-replica/5-upgrade | user 0m0.247s logger.go:42: 21:28:34 | major-upgrade-logical-replica/5-upgrade | sys 0m0.461s logger.go:42: 21:28:34 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:34 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 21:28:36 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 21:28:37 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 21:28:38 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 21:28:38 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:28:38 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:28:38 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 21:28:38 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 21:28:38 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 21:28:38 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 21:28:38 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 21:28:38 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 21:28:38 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 21:28:38 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:28:38 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:28:38 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 21:28:38 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 21:28:38 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:38 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 21:28:38 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:38 | major-upgrade-logical-replica/5-upgrade | real 0m1.940s logger.go:42: 21:28:38 | major-upgrade-logical-replica/5-upgrade | user 0m0.088s logger.go:42: 21:28:38 | major-upgrade-logical-replica/5-upgrade | sys 0m0.098s logger.go:42: 21:28:38 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:38 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 21:28:38 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:38 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 21:28:38 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 21:28:38 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 21:28:38 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 21:28:38 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:28:38 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:28:38 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 21:28:38 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 21:28:38 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 21:28:38 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 21:28:38 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 21:28:38 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 21:28:38 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 21:28:38 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 21:28:38 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 21:28:38 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:28:38 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:28:38 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 21:28:38 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 21:28:38 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:38 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 21:28:38 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 21:28:38 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:38 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 21:28:38 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 21:28:38 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 21:28:38 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 21:28:38 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 21:28:38 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 21:28:38 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 21:28:38 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 21:28:38 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 21:28:38 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 21:28:38 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 21:28:38 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 21:28:38 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 21:28:38 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 21:28:38 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 21:28:38 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 21:28:38 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 21:28:38 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 21:28:38 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 21:28:38 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:38 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 21:28:38 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 21:28:38 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 21:28:38 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 21:28:38 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 21:28:38 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 21:28:38 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 21:28:38 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 21:28:38 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 21:28:38 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 21:28:38 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:38 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 21:28:38 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 21:28:38 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 21:28:38 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 21:28:38 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 21:28:38 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:38 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 21:28:38 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 21:28:38 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 21:28:38 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 21:28:38 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 21:28:38 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 21:28:38 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 21:28:38 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 21:28:38 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:38 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 21:28:38 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:38 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:38 | major-upgrade-logical-replica/5-upgrade | real 0m9.551s logger.go:42: 21:28:38 | major-upgrade-logical-replica/5-upgrade | user 0m0.247s logger.go:42: 21:28:38 | major-upgrade-logical-replica/5-upgrade | sys 0m0.461s logger.go:42: 21:28:38 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:38 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 21:28:40 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 21:28:41 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 21:28:41 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 21:28:41 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:28:41 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:28:41 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 21:28:41 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 21:28:41 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 21:28:41 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 21:28:41 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 21:28:41 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 21:28:41 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 21:28:41 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:28:41 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:28:41 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 21:28:41 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 21:28:41 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:41 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 21:28:41 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:41 | major-upgrade-logical-replica/5-upgrade | real 0m1.940s logger.go:42: 21:28:41 | major-upgrade-logical-replica/5-upgrade | user 0m0.088s logger.go:42: 21:28:41 | major-upgrade-logical-replica/5-upgrade | sys 0m0.098s logger.go:42: 21:28:41 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:41 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 21:28:41 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:41 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 21:28:41 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 21:28:41 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 21:28:41 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 21:28:41 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:28:41 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:28:41 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 21:28:41 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 21:28:41 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 21:28:41 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 21:28:41 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 21:28:41 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 21:28:41 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 21:28:41 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 21:28:41 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 21:28:41 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:28:41 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:28:41 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 21:28:41 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 21:28:41 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:41 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 21:28:41 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 21:28:41 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:41 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 21:28:41 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 21:28:41 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 21:28:41 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 21:28:41 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 21:28:41 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 21:28:41 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 21:28:41 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 21:28:41 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 21:28:41 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 21:28:41 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 21:28:41 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 21:28:41 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 21:28:41 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 21:28:41 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 21:28:41 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 21:28:41 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 21:28:41 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 21:28:41 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 21:28:41 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:41 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 21:28:41 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 21:28:41 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 21:28:41 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 21:28:41 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 21:28:41 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 21:28:41 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 21:28:41 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 21:28:41 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 21:28:41 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 21:28:41 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:41 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 21:28:41 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 21:28:41 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 21:28:41 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 21:28:41 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 21:28:41 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:41 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 21:28:41 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 21:28:41 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 21:28:41 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 21:28:41 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 21:28:41 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 21:28:41 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 21:28:41 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 21:28:41 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:41 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 21:28:41 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:41 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:41 | major-upgrade-logical-replica/5-upgrade | real 0m9.551s logger.go:42: 21:28:41 | major-upgrade-logical-replica/5-upgrade | user 0m0.247s logger.go:42: 21:28:41 | major-upgrade-logical-replica/5-upgrade | sys 0m0.461s logger.go:42: 21:28:41 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:41 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 21:28:44 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 21:28:44 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 21:28:45 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 21:28:45 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:28:45 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:28:45 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 21:28:45 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 21:28:45 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 21:28:45 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 21:28:45 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 21:28:45 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 21:28:45 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 21:28:45 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:28:45 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:28:45 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 21:28:45 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 21:28:45 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:45 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 21:28:45 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:45 | major-upgrade-logical-replica/5-upgrade | real 0m1.940s logger.go:42: 21:28:45 | major-upgrade-logical-replica/5-upgrade | user 0m0.088s logger.go:42: 21:28:45 | major-upgrade-logical-replica/5-upgrade | sys 0m0.098s logger.go:42: 21:28:45 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:45 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 21:28:45 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:45 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 21:28:45 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 21:28:45 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 21:28:45 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 21:28:45 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:28:45 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:28:45 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 21:28:45 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 21:28:45 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 21:28:45 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 21:28:45 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 21:28:45 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 21:28:45 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 21:28:45 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 21:28:45 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 21:28:45 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:28:45 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:28:45 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 21:28:45 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 21:28:45 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:45 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 21:28:45 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 21:28:45 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:45 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 21:28:45 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 21:28:45 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 21:28:45 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 21:28:45 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 21:28:45 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 21:28:45 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 21:28:45 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 21:28:45 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 21:28:45 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 21:28:45 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 21:28:45 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 21:28:45 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 21:28:45 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 21:28:45 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 21:28:45 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 21:28:45 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 21:28:45 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 21:28:45 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 21:28:45 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:45 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 21:28:45 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 21:28:45 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 21:28:45 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 21:28:45 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 21:28:45 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 21:28:45 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 21:28:45 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 21:28:45 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 21:28:45 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 21:28:45 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:45 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 21:28:45 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 21:28:45 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 21:28:45 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 21:28:45 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 21:28:45 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:45 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 21:28:45 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 21:28:45 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 21:28:45 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 21:28:45 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 21:28:45 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 21:28:45 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 21:28:45 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 21:28:45 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:45 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 21:28:45 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:45 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:45 | major-upgrade-logical-replica/5-upgrade | real 0m9.551s logger.go:42: 21:28:45 | major-upgrade-logical-replica/5-upgrade | user 0m0.247s logger.go:42: 21:28:45 | major-upgrade-logical-replica/5-upgrade | sys 0m0.461s logger.go:42: 21:28:45 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:45 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 21:28:47 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 21:28:48 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 21:28:48 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 21:28:48 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:28:48 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:28:48 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 21:28:48 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 21:28:48 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 21:28:48 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 21:28:48 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 21:28:48 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 21:28:48 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 21:28:48 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:28:48 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:28:48 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 21:28:48 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 21:28:48 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:48 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 21:28:48 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:48 | major-upgrade-logical-replica/5-upgrade | real 0m1.940s logger.go:42: 21:28:48 | major-upgrade-logical-replica/5-upgrade | user 0m0.088s logger.go:42: 21:28:48 | major-upgrade-logical-replica/5-upgrade | sys 0m0.098s logger.go:42: 21:28:48 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:48 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 21:28:48 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:48 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 21:28:48 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 21:28:48 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 21:28:48 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 21:28:48 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:28:48 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:28:48 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 21:28:48 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 21:28:48 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 21:28:48 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 21:28:48 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 21:28:48 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 21:28:48 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 21:28:48 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 21:28:48 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 21:28:48 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:28:48 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:28:48 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 21:28:48 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 21:28:48 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:48 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 21:28:48 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 21:28:48 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:48 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 21:28:48 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 21:28:48 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 21:28:48 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 21:28:48 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 21:28:48 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 21:28:48 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 21:28:48 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 21:28:48 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 21:28:48 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 21:28:48 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 21:28:48 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 21:28:48 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 21:28:48 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 21:28:48 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 21:28:48 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 21:28:48 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 21:28:48 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 21:28:48 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 21:28:48 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:48 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 21:28:48 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 21:28:48 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 21:28:48 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 21:28:48 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 21:28:48 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 21:28:48 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 21:28:48 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 21:28:48 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 21:28:48 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 21:28:48 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:48 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 21:28:48 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 21:28:48 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 21:28:48 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 21:28:48 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 21:28:48 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:48 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 21:28:48 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 21:28:48 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 21:28:48 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 21:28:48 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 21:28:48 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 21:28:48 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 21:28:48 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 21:28:48 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:48 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 21:28:48 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:48 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:48 | major-upgrade-logical-replica/5-upgrade | real 0m9.551s logger.go:42: 21:28:48 | major-upgrade-logical-replica/5-upgrade | user 0m0.247s logger.go:42: 21:28:48 | major-upgrade-logical-replica/5-upgrade | sys 0m0.461s logger.go:42: 21:28:48 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:48 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 21:28:51 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 21:28:51 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 21:28:52 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 21:28:52 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:28:52 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:28:52 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 21:28:52 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 21:28:52 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 21:28:52 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 21:28:52 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 21:28:52 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 21:28:52 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 21:28:52 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:28:52 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:28:52 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 21:28:52 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 21:28:52 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:52 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 21:28:52 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:52 | major-upgrade-logical-replica/5-upgrade | real 0m1.940s logger.go:42: 21:28:52 | major-upgrade-logical-replica/5-upgrade | user 0m0.088s logger.go:42: 21:28:52 | major-upgrade-logical-replica/5-upgrade | sys 0m0.098s logger.go:42: 21:28:52 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:52 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 21:28:52 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:52 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 21:28:52 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 21:28:52 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 21:28:52 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 21:28:52 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:28:52 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:28:52 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 21:28:52 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 21:28:52 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 21:28:52 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 21:28:52 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 21:28:52 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 21:28:52 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 21:28:52 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 21:28:52 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 21:28:52 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:28:52 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:28:52 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 21:28:52 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 21:28:52 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:52 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 21:28:52 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 21:28:52 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:52 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 21:28:52 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 21:28:52 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 21:28:52 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 21:28:52 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 21:28:52 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 21:28:52 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 21:28:52 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 21:28:52 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 21:28:52 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 21:28:52 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 21:28:52 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 21:28:52 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 21:28:52 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 21:28:52 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 21:28:52 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 21:28:52 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 21:28:52 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 21:28:52 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 21:28:52 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:52 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 21:28:52 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 21:28:52 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 21:28:52 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 21:28:52 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 21:28:52 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 21:28:52 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 21:28:52 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 21:28:52 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 21:28:52 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 21:28:52 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:52 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 21:28:52 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 21:28:52 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 21:28:52 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 21:28:52 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 21:28:52 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:52 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 21:28:52 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 21:28:52 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 21:28:52 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 21:28:52 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 21:28:52 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 21:28:52 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 21:28:52 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 21:28:52 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:52 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 21:28:52 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:52 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:52 | major-upgrade-logical-replica/5-upgrade | real 0m9.551s logger.go:42: 21:28:52 | major-upgrade-logical-replica/5-upgrade | user 0m0.247s logger.go:42: 21:28:52 | major-upgrade-logical-replica/5-upgrade | sys 0m0.461s logger.go:42: 21:28:52 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:52 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 21:28:54 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 21:28:55 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 21:28:56 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 21:28:56 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:28:56 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:28:56 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 21:28:56 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 21:28:56 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 21:28:56 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 21:28:56 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 21:28:56 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 21:28:56 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 21:28:56 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:28:56 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:28:56 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 21:28:56 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 21:28:56 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:56 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 21:28:56 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:56 | major-upgrade-logical-replica/5-upgrade | real 0m1.940s logger.go:42: 21:28:56 | major-upgrade-logical-replica/5-upgrade | user 0m0.088s logger.go:42: 21:28:56 | major-upgrade-logical-replica/5-upgrade | sys 0m0.098s logger.go:42: 21:28:56 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:56 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 21:28:56 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:56 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 21:28:56 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 21:28:56 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 21:28:56 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 21:28:56 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:28:56 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:28:56 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 21:28:56 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 21:28:56 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 21:28:56 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 21:28:56 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 21:28:56 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 21:28:56 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 21:28:56 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 21:28:56 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 21:28:56 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:28:56 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:28:56 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 21:28:56 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 21:28:56 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:56 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 21:28:56 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 21:28:56 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:56 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 21:28:56 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 21:28:56 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 21:28:56 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 21:28:56 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 21:28:56 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 21:28:56 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 21:28:56 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 21:28:56 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 21:28:56 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 21:28:56 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 21:28:56 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 21:28:56 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 21:28:56 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 21:28:56 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 21:28:56 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 21:28:56 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 21:28:56 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 21:28:56 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 21:28:56 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:56 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 21:28:56 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 21:28:56 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 21:28:56 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 21:28:56 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 21:28:56 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 21:28:56 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 21:28:56 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 21:28:56 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 21:28:56 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 21:28:56 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:56 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 21:28:56 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 21:28:56 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 21:28:56 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 21:28:56 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 21:28:56 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:56 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 21:28:56 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 21:28:56 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 21:28:56 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 21:28:56 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 21:28:56 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 21:28:56 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 21:28:56 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 21:28:56 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:56 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 21:28:56 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:56 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:56 | major-upgrade-logical-replica/5-upgrade | real 0m9.551s logger.go:42: 21:28:56 | major-upgrade-logical-replica/5-upgrade | user 0m0.247s logger.go:42: 21:28:56 | major-upgrade-logical-replica/5-upgrade | sys 0m0.461s logger.go:42: 21:28:56 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:28:56 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 21:28:58 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 21:28:59 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 21:29:00 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 21:29:00 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:29:00 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:29:00 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 21:29:00 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 21:29:00 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 21:29:00 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 21:29:00 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 21:29:00 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 21:29:00 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 21:29:00 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:29:00 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:29:00 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 21:29:00 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 21:29:00 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:00 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 21:29:00 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:00 | major-upgrade-logical-replica/5-upgrade | real 0m1.940s logger.go:42: 21:29:00 | major-upgrade-logical-replica/5-upgrade | user 0m0.088s logger.go:42: 21:29:00 | major-upgrade-logical-replica/5-upgrade | sys 0m0.098s logger.go:42: 21:29:00 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:00 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 21:29:00 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:00 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 21:29:00 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 21:29:00 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 21:29:00 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 21:29:00 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:29:00 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:29:00 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 21:29:00 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 21:29:00 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 21:29:00 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 21:29:00 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 21:29:00 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 21:29:00 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 21:29:00 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 21:29:00 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 21:29:00 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:29:00 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:29:00 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 21:29:00 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 21:29:00 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:00 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 21:29:00 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 21:29:00 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:00 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 21:29:00 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 21:29:00 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 21:29:00 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 21:29:00 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 21:29:00 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 21:29:00 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 21:29:00 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 21:29:00 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 21:29:00 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 21:29:00 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 21:29:00 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 21:29:00 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 21:29:00 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 21:29:00 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 21:29:00 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 21:29:00 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 21:29:00 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 21:29:00 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 21:29:00 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:00 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 21:29:00 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 21:29:00 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 21:29:00 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 21:29:00 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 21:29:00 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 21:29:00 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 21:29:00 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 21:29:00 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 21:29:00 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 21:29:00 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:00 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 21:29:00 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 21:29:00 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 21:29:00 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 21:29:00 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 21:29:00 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:00 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 21:29:00 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 21:29:00 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 21:29:00 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 21:29:00 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 21:29:00 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 21:29:00 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 21:29:00 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 21:29:00 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:00 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 21:29:00 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:00 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:00 | major-upgrade-logical-replica/5-upgrade | real 0m9.551s logger.go:42: 21:29:00 | major-upgrade-logical-replica/5-upgrade | user 0m0.247s logger.go:42: 21:29:00 | major-upgrade-logical-replica/5-upgrade | sys 0m0.461s logger.go:42: 21:29:00 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:00 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 21:29:02 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 21:29:02 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 21:29:03 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 21:29:03 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:29:03 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:29:03 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 21:29:03 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 21:29:03 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 21:29:03 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 21:29:03 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 21:29:03 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 21:29:03 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 21:29:03 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:29:03 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:29:03 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 21:29:03 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 21:29:03 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:03 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 21:29:03 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:03 | major-upgrade-logical-replica/5-upgrade | real 0m1.940s logger.go:42: 21:29:03 | major-upgrade-logical-replica/5-upgrade | user 0m0.088s logger.go:42: 21:29:03 | major-upgrade-logical-replica/5-upgrade | sys 0m0.098s logger.go:42: 21:29:03 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:03 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 21:29:03 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:03 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 21:29:03 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 21:29:03 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 21:29:03 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 21:29:03 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:29:03 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:29:03 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 21:29:03 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 21:29:03 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 21:29:03 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 21:29:03 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 21:29:03 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 21:29:03 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 21:29:03 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 21:29:03 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 21:29:03 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:29:03 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:29:03 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 21:29:03 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 21:29:03 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:03 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 21:29:03 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 21:29:03 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:03 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 21:29:03 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 21:29:03 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 21:29:03 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 21:29:03 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 21:29:03 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 21:29:03 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 21:29:03 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 21:29:03 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 21:29:03 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 21:29:03 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 21:29:03 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 21:29:03 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 21:29:03 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 21:29:03 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 21:29:03 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 21:29:03 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 21:29:03 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 21:29:03 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 21:29:03 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:03 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 21:29:03 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 21:29:03 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 21:29:03 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 21:29:03 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 21:29:03 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 21:29:03 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 21:29:03 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 21:29:03 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 21:29:03 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 21:29:03 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:03 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 21:29:03 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 21:29:03 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 21:29:03 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 21:29:03 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 21:29:03 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:03 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 21:29:03 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 21:29:03 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 21:29:03 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 21:29:03 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 21:29:03 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 21:29:03 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 21:29:03 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 21:29:03 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:03 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 21:29:03 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:03 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:03 | major-upgrade-logical-replica/5-upgrade | real 0m9.551s logger.go:42: 21:29:03 | major-upgrade-logical-replica/5-upgrade | user 0m0.247s logger.go:42: 21:29:03 | major-upgrade-logical-replica/5-upgrade | sys 0m0.461s logger.go:42: 21:29:03 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:03 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 21:29:05 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 21:29:06 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 21:29:07 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 21:29:07 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:29:07 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:29:07 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 21:29:07 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 21:29:07 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 21:29:07 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 21:29:07 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 21:29:07 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 21:29:07 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 21:29:07 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:29:07 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:29:07 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 21:29:07 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 21:29:07 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:07 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 21:29:07 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:07 | major-upgrade-logical-replica/5-upgrade | real 0m1.940s logger.go:42: 21:29:07 | major-upgrade-logical-replica/5-upgrade | user 0m0.088s logger.go:42: 21:29:07 | major-upgrade-logical-replica/5-upgrade | sys 0m0.098s logger.go:42: 21:29:07 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:07 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 21:29:07 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:07 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 21:29:07 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 21:29:07 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 21:29:07 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 21:29:07 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:29:07 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:29:07 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 21:29:07 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 21:29:07 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 21:29:07 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 21:29:07 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 21:29:07 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 21:29:07 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 21:29:07 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 21:29:07 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 21:29:07 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:29:07 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:29:07 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 21:29:07 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 21:29:07 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:07 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 21:29:07 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 21:29:07 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:07 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 21:29:07 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 21:29:07 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 21:29:07 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 21:29:07 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 21:29:07 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 21:29:07 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 21:29:07 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 21:29:07 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 21:29:07 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 21:29:07 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 21:29:07 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 21:29:07 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 21:29:07 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 21:29:07 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 21:29:07 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 21:29:07 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 21:29:07 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 21:29:07 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 21:29:07 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:07 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 21:29:07 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 21:29:07 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 21:29:07 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 21:29:07 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 21:29:07 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 21:29:07 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 21:29:07 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 21:29:07 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 21:29:07 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 21:29:07 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:07 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 21:29:07 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 21:29:07 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 21:29:07 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 21:29:07 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 21:29:07 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:07 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 21:29:07 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 21:29:07 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 21:29:07 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 21:29:07 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 21:29:07 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 21:29:07 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 21:29:07 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 21:29:07 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:07 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 21:29:07 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:07 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:07 | major-upgrade-logical-replica/5-upgrade | real 0m9.551s logger.go:42: 21:29:07 | major-upgrade-logical-replica/5-upgrade | user 0m0.247s logger.go:42: 21:29:07 | major-upgrade-logical-replica/5-upgrade | sys 0m0.461s logger.go:42: 21:29:07 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:07 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 21:29:09 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 21:29:10 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 21:29:11 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 21:29:11 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:29:11 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:29:11 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 21:29:11 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 21:29:11 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 21:29:11 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 21:29:11 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 21:29:11 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 21:29:11 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 21:29:11 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:29:11 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:29:11 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 21:29:11 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 21:29:11 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:11 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 21:29:11 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:11 | major-upgrade-logical-replica/5-upgrade | real 0m1.940s logger.go:42: 21:29:11 | major-upgrade-logical-replica/5-upgrade | user 0m0.088s logger.go:42: 21:29:11 | major-upgrade-logical-replica/5-upgrade | sys 0m0.098s logger.go:42: 21:29:11 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:11 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 21:29:11 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:11 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 21:29:11 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 21:29:11 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 21:29:11 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 21:29:11 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:29:11 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:29:11 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 21:29:11 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 21:29:11 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 21:29:11 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 21:29:11 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 21:29:11 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 21:29:11 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 21:29:11 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 21:29:11 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 21:29:11 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:29:11 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:29:11 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 21:29:11 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 21:29:11 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:11 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 21:29:11 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 21:29:11 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:11 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 21:29:11 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 21:29:11 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 21:29:11 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 21:29:11 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 21:29:11 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 21:29:11 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 21:29:11 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 21:29:11 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 21:29:11 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 21:29:11 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 21:29:11 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 21:29:11 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 21:29:11 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 21:29:11 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 21:29:11 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 21:29:11 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 21:29:11 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 21:29:11 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 21:29:11 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:11 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 21:29:11 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 21:29:11 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 21:29:11 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 21:29:11 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 21:29:11 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 21:29:11 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 21:29:11 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 21:29:11 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 21:29:11 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 21:29:11 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:11 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 21:29:11 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 21:29:11 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 21:29:11 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 21:29:11 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 21:29:11 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:11 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 21:29:11 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 21:29:11 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 21:29:11 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 21:29:11 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 21:29:11 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 21:29:11 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 21:29:11 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 21:29:11 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:11 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 21:29:11 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:11 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:11 | major-upgrade-logical-replica/5-upgrade | real 0m9.551s logger.go:42: 21:29:11 | major-upgrade-logical-replica/5-upgrade | user 0m0.247s logger.go:42: 21:29:11 | major-upgrade-logical-replica/5-upgrade | sys 0m0.461s logger.go:42: 21:29:11 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:11 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 21:29:13 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 21:29:13 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 21:29:14 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 21:29:14 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:29:14 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:29:14 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 21:29:14 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 21:29:14 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 21:29:14 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 21:29:14 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 21:29:14 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 21:29:14 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 21:29:14 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:29:14 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:29:14 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 21:29:14 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 21:29:14 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:14 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 21:29:14 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:14 | major-upgrade-logical-replica/5-upgrade | real 0m1.940s logger.go:42: 21:29:14 | major-upgrade-logical-replica/5-upgrade | user 0m0.088s logger.go:42: 21:29:14 | major-upgrade-logical-replica/5-upgrade | sys 0m0.098s logger.go:42: 21:29:14 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:14 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 21:29:14 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:14 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 21:29:14 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 21:29:14 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 21:29:14 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 21:29:14 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:29:14 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:29:14 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 21:29:14 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 21:29:14 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 21:29:14 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 21:29:14 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 21:29:14 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 21:29:14 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 21:29:14 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 21:29:14 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 21:29:14 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:29:14 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:29:14 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 21:29:14 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 21:29:14 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:14 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 21:29:14 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 21:29:14 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:14 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 21:29:14 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 21:29:14 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 21:29:14 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 21:29:14 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 21:29:14 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 21:29:14 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 21:29:14 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 21:29:14 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 21:29:14 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 21:29:14 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 21:29:14 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 21:29:14 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 21:29:14 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 21:29:14 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 21:29:14 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 21:29:14 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 21:29:14 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 21:29:14 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 21:29:14 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:14 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 21:29:14 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 21:29:14 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 21:29:14 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 21:29:14 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 21:29:14 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 21:29:14 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 21:29:14 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 21:29:14 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 21:29:14 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 21:29:14 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:14 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 21:29:14 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 21:29:14 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 21:29:14 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 21:29:14 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 21:29:14 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:14 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 21:29:14 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 21:29:14 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 21:29:14 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 21:29:14 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 21:29:14 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 21:29:14 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 21:29:14 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 21:29:14 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:14 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 21:29:14 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:14 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:14 | major-upgrade-logical-replica/5-upgrade | real 0m9.551s logger.go:42: 21:29:14 | major-upgrade-logical-replica/5-upgrade | user 0m0.247s logger.go:42: 21:29:14 | major-upgrade-logical-replica/5-upgrade | sys 0m0.461s logger.go:42: 21:29:14 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:14 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 21:29:16 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 21:29:17 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 21:29:18 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 21:29:18 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:29:18 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:29:18 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 21:29:18 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 21:29:18 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 21:29:18 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 21:29:18 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 21:29:18 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 21:29:18 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 21:29:18 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:29:18 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:29:18 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 21:29:18 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 21:29:18 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:18 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 21:29:18 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:18 | major-upgrade-logical-replica/5-upgrade | real 0m1.940s logger.go:42: 21:29:18 | major-upgrade-logical-replica/5-upgrade | user 0m0.088s logger.go:42: 21:29:18 | major-upgrade-logical-replica/5-upgrade | sys 0m0.098s logger.go:42: 21:29:18 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:18 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 21:29:18 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:18 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 21:29:18 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 21:29:18 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 21:29:18 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 21:29:18 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:29:18 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:29:18 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 21:29:18 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 21:29:18 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 21:29:18 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 21:29:18 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 21:29:18 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 21:29:18 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 21:29:18 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 21:29:18 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 21:29:18 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:29:18 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:29:18 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 21:29:18 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 21:29:18 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:18 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 21:29:18 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 21:29:18 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:18 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 21:29:18 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 21:29:18 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 21:29:18 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 21:29:18 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 21:29:18 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 21:29:18 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 21:29:18 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 21:29:18 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 21:29:18 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 21:29:18 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 21:29:18 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 21:29:18 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 21:29:18 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 21:29:18 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 21:29:18 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 21:29:18 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 21:29:18 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 21:29:18 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 21:29:18 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:18 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 21:29:18 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 21:29:18 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 21:29:18 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 21:29:18 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 21:29:18 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 21:29:18 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 21:29:18 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 21:29:18 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 21:29:18 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 21:29:18 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:18 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 21:29:18 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 21:29:18 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 21:29:18 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 21:29:18 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 21:29:18 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:18 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 21:29:18 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 21:29:18 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 21:29:18 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 21:29:18 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 21:29:18 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 21:29:18 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 21:29:18 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 21:29:18 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:18 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 21:29:18 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:18 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:18 | major-upgrade-logical-replica/5-upgrade | real 0m9.551s logger.go:42: 21:29:18 | major-upgrade-logical-replica/5-upgrade | user 0m0.247s logger.go:42: 21:29:18 | major-upgrade-logical-replica/5-upgrade | sys 0m0.461s logger.go:42: 21:29:18 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:18 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 21:29:20 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 21:29:21 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 21:29:22 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 21:29:22 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:29:22 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:29:22 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 21:29:22 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 21:29:22 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 21:29:22 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 21:29:22 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 21:29:22 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 21:29:22 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 21:29:22 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:29:22 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:29:22 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 21:29:22 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 21:29:22 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:22 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 21:29:22 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:22 | major-upgrade-logical-replica/5-upgrade | real 0m1.940s logger.go:42: 21:29:22 | major-upgrade-logical-replica/5-upgrade | user 0m0.088s logger.go:42: 21:29:22 | major-upgrade-logical-replica/5-upgrade | sys 0m0.098s logger.go:42: 21:29:22 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:22 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 21:29:22 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:22 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 21:29:22 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 21:29:22 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 21:29:22 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 21:29:22 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:29:22 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:29:22 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 21:29:22 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 21:29:22 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 21:29:22 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 21:29:22 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 21:29:22 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 21:29:22 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 21:29:22 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 21:29:22 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 21:29:22 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:29:22 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:29:22 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 21:29:22 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 21:29:22 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:22 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 21:29:22 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 21:29:22 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:22 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 21:29:22 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 21:29:22 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 21:29:22 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 21:29:22 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 21:29:22 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 21:29:22 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 21:29:22 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 21:29:22 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 21:29:22 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 21:29:22 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 21:29:22 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 21:29:22 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 21:29:22 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 21:29:22 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 21:29:22 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 21:29:22 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 21:29:22 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 21:29:22 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 21:29:22 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:22 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 21:29:22 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 21:29:22 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 21:29:22 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 21:29:22 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 21:29:22 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 21:29:22 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 21:29:22 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 21:29:22 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 21:29:22 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 21:29:22 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:22 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 21:29:22 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 21:29:22 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 21:29:22 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 21:29:22 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 21:29:22 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:22 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 21:29:22 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 21:29:22 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 21:29:22 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 21:29:22 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 21:29:22 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 21:29:22 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 21:29:22 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 21:29:22 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:22 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 21:29:22 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:22 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:22 | major-upgrade-logical-replica/5-upgrade | real 0m9.551s logger.go:42: 21:29:22 | major-upgrade-logical-replica/5-upgrade | user 0m0.247s logger.go:42: 21:29:22 | major-upgrade-logical-replica/5-upgrade | sys 0m0.461s logger.go:42: 21:29:22 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:22 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 21:29:24 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 21:29:24 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 21:29:25 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 21:29:25 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:29:25 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:29:25 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 21:29:25 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 21:29:25 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 21:29:25 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 21:29:25 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 21:29:25 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 21:29:25 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 21:29:25 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:29:25 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:29:25 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 21:29:25 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 21:29:25 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:25 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 21:29:25 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:25 | major-upgrade-logical-replica/5-upgrade | real 0m1.940s logger.go:42: 21:29:25 | major-upgrade-logical-replica/5-upgrade | user 0m0.088s logger.go:42: 21:29:25 | major-upgrade-logical-replica/5-upgrade | sys 0m0.098s logger.go:42: 21:29:25 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:25 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 21:29:25 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:25 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 21:29:25 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 21:29:25 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 21:29:25 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 21:29:25 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:29:25 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:29:25 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 21:29:25 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 21:29:25 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 21:29:25 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 21:29:25 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 21:29:25 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 21:29:25 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 21:29:25 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 21:29:25 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 21:29:25 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:29:25 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:29:25 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 21:29:25 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 21:29:25 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:25 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 21:29:25 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 21:29:25 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:25 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 21:29:25 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 21:29:25 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 21:29:25 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 21:29:25 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 21:29:25 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 21:29:25 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 21:29:25 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 21:29:25 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 21:29:25 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 21:29:25 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 21:29:25 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 21:29:25 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 21:29:25 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 21:29:25 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 21:29:25 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 21:29:25 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 21:29:25 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 21:29:25 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 21:29:25 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:25 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 21:29:25 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 21:29:25 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 21:29:25 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 21:29:25 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 21:29:25 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 21:29:25 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 21:29:25 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 21:29:25 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 21:29:25 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 21:29:25 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:25 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 21:29:25 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 21:29:25 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 21:29:25 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 21:29:25 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 21:29:25 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:25 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 21:29:25 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 21:29:25 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 21:29:25 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 21:29:25 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 21:29:25 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 21:29:25 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 21:29:25 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 21:29:25 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:25 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 21:29:25 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:25 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:25 | major-upgrade-logical-replica/5-upgrade | real 0m9.551s logger.go:42: 21:29:25 | major-upgrade-logical-replica/5-upgrade | user 0m0.247s logger.go:42: 21:29:25 | major-upgrade-logical-replica/5-upgrade | sys 0m0.461s logger.go:42: 21:29:25 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:25 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 21:29:28 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 21:29:28 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 21:29:29 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 21:29:29 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:29:29 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:29:29 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 21:29:29 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 21:29:29 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 21:29:29 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 21:29:29 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 21:29:29 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 21:29:29 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 21:29:29 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:29:29 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:29:29 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 21:29:29 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 21:29:29 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:29 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 21:29:29 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:29 | major-upgrade-logical-replica/5-upgrade | real 0m1.940s logger.go:42: 21:29:29 | major-upgrade-logical-replica/5-upgrade | user 0m0.088s logger.go:42: 21:29:29 | major-upgrade-logical-replica/5-upgrade | sys 0m0.098s logger.go:42: 21:29:29 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:29 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 21:29:29 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:29 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 21:29:29 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 21:29:29 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 21:29:29 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 21:29:29 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:29:29 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:29:29 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 21:29:29 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 21:29:29 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 21:29:29 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 21:29:29 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 21:29:29 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 21:29:29 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 21:29:29 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 21:29:29 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 21:29:29 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 21:29:29 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 21:29:29 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 21:29:29 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 21:29:29 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:29 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 21:29:29 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 21:29:29 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:29 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 21:29:29 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 21:29:29 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 21:29:29 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 21:29:29 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 21:29:29 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 21:29:29 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 21:29:29 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 21:29:29 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 21:29:29 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 21:29:29 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 21:29:29 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 21:29:29 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 21:29:29 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 21:29:29 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 21:29:29 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 21:29:29 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 21:29:29 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 21:29:29 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 21:29:29 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:29 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 21:29:29 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 21:29:29 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 21:29:29 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 21:29:29 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 21:29:29 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 21:29:29 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 21:29:29 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 21:29:29 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 21:29:29 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 21:29:29 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:29 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 21:29:29 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 21:29:29 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 21:29:29 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 21:29:29 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 21:29:29 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:29 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 21:29:29 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 21:29:29 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 21:29:29 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 21:29:29 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 21:29:29 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 21:29:29 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 21:29:29 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 21:29:29 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:29 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 21:29:29 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:29 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:29 | major-upgrade-logical-replica/5-upgrade | real 0m9.551s logger.go:42: 21:29:29 | major-upgrade-logical-replica/5-upgrade | user 0m0.247s logger.go:42: 21:29:29 | major-upgrade-logical-replica/5-upgrade | sys 0m0.461s logger.go:42: 21:29:29 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 21:29:29 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 21:29:30 | major-upgrade-logical-replica/5-upgrade | test step completed 5-upgrade logger.go:42: 21:29:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | starting test step 6-verify-replica-invalidated logger.go:42: 21:29:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | running command: [sh -c set -o errexit set -o xtrace source ../../functions primary=$(get_pod_by_role major-upgrade-logical-replica primary name) kubectl -n "${NAMESPACE}" exec "${primary}" -- /pgdata/delete_old_cluster.sh # pg_upgrade only rewrites the data directories of the instance sets, so the # replica still holds a PostgreSQL 17 one. It has to be reported as broken # rather than handed the new binary and left to crash. retry 60 10 bash -c ' state=$(kubectl -n "'"${NAMESPACE}"'" get pg major-upgrade-logical-replica \ -o jsonpath="{.status.logicalReplicas[?(@.name==\"analytics\")].state}") reason=$(kubectl -n "'"${NAMESPACE}"'" get pg major-upgrade-logical-replica \ -o jsonpath="{.status.logicalReplicas[?(@.name==\"analytics\")].reason}") echo "logical replica analytics: ${state}/${reason}" [[ ${state} == "broken" && ${reason} == "SourceUpgraded" ]] ' message=$(kubectl -n "${NAMESPACE}" get pg major-upgrade-logical-replica \ -o jsonpath='{.status.logicalReplicas[?(@.name=="analytics")].message}') echo "message: ${message}" echo "${message}" | grep -q "remove it from spec.logicalReplicas" # The StatefulSet is stopped, so nothing is left crash-looping against a data # directory the new binary cannot open. retry 30 5 bash -c ' replicas=$(kubectl -n "'"${NAMESPACE}"'" get statefulset \ major-upgrade-logical-replica-lr-analytics -o jsonpath="{.spec.replicas}") echo "logical replica statefulset replicas: ${replicas}" [[ ${replicas} == "0" ]] ' # pg_upgrade carries the logical slots over to the new cluster. Nothing will # ever read from them again, so they must not be left pinning WAL. PASS=$(get_psql_user_pass major-upgrade-logical-replica-pguser-postgres) HOST=$(get_psql_user_host major-upgrade-logical-replica-pguser-postgres) no_slots_left() { local got got=$(run_psql \ "SELECT count(*) FROM pg_replication_slots WHERE plugin IS NOT NULL;" \ "postgres://postgres:${PASS}@${HOST}" | xargs) echo "logical slots left on the primary: ${got}" [[ ${got} == "0" ]] } retry 30 5 no_slots_left] logger.go:42: 21:29:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | + source ../../functions logger.go:42: 21:29:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ realpath ../../.. logger.go:42: 21:29:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812 logger.go:42: 21:29:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ CERT_MANAGER_VER=1.20.2 logger.go:42: 21:29:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ CERT_MANAGER_TEST_LABEL=e2e.percona.com/managed-by=percona-postgresql-operator-tests logger.go:42: 21:29:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ CHAOS_MESH_VER=2.5.1 logger.go:42: 21:29:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ BUSYBOX_VER=1.36 logger.go:42: 21:29:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ pwd logger.go:42: 21:29:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ basename /mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests/tests/major-upgrade-logical-replica logger.go:42: 21:29:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ test_name=major-upgrade-logical-replica logger.go:42: 21:29:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ source /mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests/vars.sh logger.go:42: 21:29:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812 logger.go:42: 21:29:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812 logger.go:42: 21:29:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/deploy logger.go:42: 21:29:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/deploy logger.go:42: 21:29:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests logger.go:42: 21:29:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests logger.go:42: 21:29:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests/conf logger.go:42: 21:29:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests/conf logger.go:42: 21:29:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export TEST_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests/tests/major-upgrade-logical-replica/conf logger.go:42: 21:29:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ TEST_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests/tests/major-upgrade-logical-replica/conf logger.go:42: 21:29:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export TEMP_DIR=/tmp/kuttl/pg/major-upgrade-logical-replica logger.go:42: 21:29:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ TEMP_DIR=/tmp/kuttl/pg/major-upgrade-logical-replica logger.go:42: 21:29:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ git rev-parse --abbrev-ref HEAD logger.go:42: 21:29:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export GIT_BRANCH=PR-1812 logger.go:42: 21:29:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ GIT_BRANCH=PR-1812 logger.go:42: 21:29:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export VERSION=PR-1812-09e4007fd logger.go:42: 21:29:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ VERSION=PR-1812-09e4007fd logger.go:42: 21:29:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export SKIP_TEST_WARNINGS=true logger.go:42: 21:29:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ SKIP_TEST_WARNINGS=true logger.go:42: 21:29:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export K8S_UPGRADE_PLATFORM=gke logger.go:42: 21:29:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ K8S_UPGRADE_PLATFORM=gke logger.go:42: 21:29:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export K8S_UPGRADE_REGION=us-central1-a logger.go:42: 21:29:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ K8S_UPGRADE_REGION=us-central1-a logger.go:42: 21:29:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export K8S_UPGRADE_INITIAL_VERSION= logger.go:42: 21:29:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ K8S_UPGRADE_INITIAL_VERSION= logger.go:42: 21:29:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export K8S_UPGRADE_FINAL_VERSION= logger.go:42: 21:29:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ K8S_UPGRADE_FINAL_VERSION= logger.go:42: 21:29:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ -z '' ]] logger.go:42: 21:29:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ detect_k8s_platform logger.go:42: 21:29:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ local detected_platform logger.go:42: 21:29:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ local xtrace_enabled=false logger.go:42: 21:29:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ [[ -o xtrace ]] logger.go:42: 21:29:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ xtrace_enabled=true logger.go:42: 21:29:30 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ set +o xtrace logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ PLATFORM=gke logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export PLATFORM logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE=perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE=perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ ! -n '' ]] logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ -n '' ]] logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export PG_VER=18 logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ PG_VER=18 logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export PG_DISTRIBUTION= logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ PG_DISTRIBUTION= logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ '' == community ]] logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export BUCKET=pg-operator-testing logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ BUCKET=pg-operator-testing logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export PMM_SERVER_VERSION=9.9.9 logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ PMM_SERVER_VERSION=9.9.9 logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE_LOGCOLLECTOR=perconalab/fluentbit:main-logcollector logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE_LOGCOLLECTOR=perconalab/fluentbit:main-logcollector logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export PGOV1_TAG=1.4.0 logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ PGOV1_TAG=1.4.0 logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export PGOV1_VER=14 logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ PGOV1_VER=14 logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export CPGO_VERSION=5.8.7 logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ CPGO_VERSION=5.8.7 logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export MINIO_VER=5.4.0 logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ MINIO_VER=5.4.0 logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export VAULT_VER=0.32.0 logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ VAULT_VER=0.32.0 logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export REGISTRY_NAME=docker.io logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ REGISTRY_NAME=docker.io logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export REGISTRY_NAME_FULL=docker.io/ logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ REGISTRY_NAME_FULL=docker.io/ logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ printenv logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ grep -E '^IMAGE' logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ awk -F= '{print $1}' logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ eval 'echo $IMAGE_BACKREST' logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++++ echo perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ var_value=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == docker.io/* ]] logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == percona/* ]] logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == perconalab/* ]] logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ eval 'echo $IMAGE' logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++++ echo perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ var_value=perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator:PR-1812-09e4007fd == docker.io/* ]] logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator:PR-1812-09e4007fd == percona/* ]] logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator:PR-1812-09e4007fd == perconalab/* ]] logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ new_value=docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ eval 'echo $IMAGE_UPGRADE' logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++++ echo perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ var_value=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == docker.io/* ]] logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == percona/* ]] logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == perconalab/* ]] logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ eval 'echo $IMAGE_LOGCOLLECTOR' logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++++ echo perconalab/fluentbit:main-logcollector logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ var_value=perconalab/fluentbit:main-logcollector logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/fluentbit:main-logcollector == docker.io/* ]] logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/fluentbit:main-logcollector == percona/* ]] logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/fluentbit:main-logcollector == perconalab/* ]] logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ new_value=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ eval 'echo $IMAGE_BASE' logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++++ echo perconalab/percona-postgresql-operator logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ var_value=perconalab/percona-postgresql-operator logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator == docker.io/* ]] logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator == percona/* ]] logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator == perconalab/* ]] logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ new_value=docker.io/perconalab/percona-postgresql-operator logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ eval 'echo $IMAGE_PGBOUNCER' logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++++ echo perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ var_value=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == docker.io/* ]] logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == percona/* ]] logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == perconalab/* ]] logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ eval 'echo $IMAGE_POSTGRESQL' logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++++ echo perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ var_value=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == docker.io/* ]] logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == percona/* ]] logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == perconalab/* ]] logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ eval 'echo $IMAGE_PMM_SERVER' logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++++ echo perconalab/pmm-server:3-dev-latest logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ var_value=perconalab/pmm-server:3-dev-latest logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/pmm-server:3-dev-latest == docker.io/* ]] logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/pmm-server:3-dev-latest == percona/* ]] logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/pmm-server:3-dev-latest == perconalab/* ]] logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ new_value=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ eval 'echo $IMAGE_PMM_CLIENT' logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++++ echo perconalab/pmm-client:3-dev-latest logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ var_value=perconalab/pmm-client:3-dev-latest logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/pmm-client:3-dev-latest == docker.io/* ]] logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/pmm-client:3-dev-latest == percona/* ]] logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/pmm-client:3-dev-latest == perconalab/* ]] logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ new_value=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ which gdate logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | which: no gdate in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ which date logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ date=/usr/sbin/date logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ which gsed logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | which: no gsed in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ which sed logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ sed=/usr/sbin/sed logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ get_pod_by_role major-upgrade-logical-replica primary name logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ local cluster=major-upgrade-logical-replica logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ local role=primary logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ local parameter=name logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ case ${parameter} in logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ local 'jsonpath={.items[].metadata.name}' logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ kubectl get pods --namespace kuttl-test-crack-mammoth --selector=postgres-operator.crunchydata.com/role=primary,postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica -o 'jsonpath={.items[].metadata.name}' logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ echo major-upgrade-logical-replica-instance1-8csm-0 logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | + primary=major-upgrade-logical-replica-instance1-8csm-0 logger.go:42: 21:29:32 | major-upgrade-logical-replica/6-verify-replica-invalidated | + kubectl -n kuttl-test-crack-mammoth exec major-upgrade-logical-replica-instance1-8csm-0 -- /pgdata/delete_old_cluster.sh logger.go:42: 21:29:34 | major-upgrade-logical-replica/6-verify-replica-invalidated | + retry 60 10 bash -c $'\n state=$(kubectl -n "kuttl-test-crack-mammoth" get pg major-upgrade-logical-replica \\\n -o jsonpath="{.status.logicalReplicas[?(@.name==\\"analytics\\")].state}")\n reason=$(kubectl -n "kuttl-test-crack-mammoth" get pg major-upgrade-logical-replica \\\n -o jsonpath="{.status.logicalReplicas[?(@.name==\\"analytics\\")].reason}")\n echo "logical replica analytics: ${state}/${reason}"\n [[ ${state} == "broken" && ${reason} == "SourceUpgraded" ]]\n' logger.go:42: 21:29:34 | major-upgrade-logical-replica/6-verify-replica-invalidated | + local max=60 logger.go:42: 21:29:34 | major-upgrade-logical-replica/6-verify-replica-invalidated | + local delay=10 logger.go:42: 21:29:34 | major-upgrade-logical-replica/6-verify-replica-invalidated | + shift 2 logger.go:42: 21:29:34 | major-upgrade-logical-replica/6-verify-replica-invalidated | + local n=1 logger.go:42: 21:29:34 | major-upgrade-logical-replica/6-verify-replica-invalidated | + bash -c $'\n state=$(kubectl -n "kuttl-test-crack-mammoth" get pg major-upgrade-logical-replica \\\n -o jsonpath="{.status.logicalReplicas[?(@.name==\\"analytics\\")].state}")\n reason=$(kubectl -n "kuttl-test-crack-mammoth" get pg major-upgrade-logical-replica \\\n -o jsonpath="{.status.logicalReplicas[?(@.name==\\"analytics\\")].reason}")\n echo "logical replica analytics: ${state}/${reason}"\n [[ ${state} == "broken" && ${reason} == "SourceUpgraded" ]]\n' logger.go:42: 21:29:35 | major-upgrade-logical-replica/6-verify-replica-invalidated | logical replica analytics: broken/SourceUpgraded logger.go:42: 21:29:35 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ kubectl -n kuttl-test-crack-mammoth get pg major-upgrade-logical-replica -o 'jsonpath={.status.logicalReplicas[?(@.name=="analytics")].message}' logger.go:42: 21:29:35 | major-upgrade-logical-replica/6-verify-replica-invalidated | + message='this replica holds a PostgreSQL 17 data directory and the cluster now runs 18; the pg_upgrade job only rewrites the data directories of the instance sets, so this one cannot be started again; remove it from spec.logicalReplicas, wait for its entry to leave status.logicalReplicas, and add it back to seed it again' logger.go:42: 21:29:35 | major-upgrade-logical-replica/6-verify-replica-invalidated | + echo 'message: this replica holds a PostgreSQL 17 data directory and the cluster now runs 18; the pg_upgrade job only rewrites the data directories of the instance sets, so this one cannot be started again; remove it from spec.logicalReplicas, wait for its entry to leave status.logicalReplicas, and add it back to seed it again' logger.go:42: 21:29:35 | major-upgrade-logical-replica/6-verify-replica-invalidated | message: this replica holds a PostgreSQL 17 data directory and the cluster now runs 18; the pg_upgrade job only rewrites the data directories of the instance sets, so this one cannot be started again; remove it from spec.logicalReplicas, wait for its entry to leave status.logicalReplicas, and add it back to seed it again logger.go:42: 21:29:35 | major-upgrade-logical-replica/6-verify-replica-invalidated | + echo 'this replica holds a PostgreSQL 17 data directory and the cluster now runs 18; the pg_upgrade job only rewrites the data directories of the instance sets, so this one cannot be started again; remove it from spec.logicalReplicas, wait for its entry to leave status.logicalReplicas, and add it back to seed it again' logger.go:42: 21:29:35 | major-upgrade-logical-replica/6-verify-replica-invalidated | + grep -q 'remove it from spec.logicalReplicas' logger.go:42: 21:29:35 | major-upgrade-logical-replica/6-verify-replica-invalidated | + retry 30 5 bash -c $'\n replicas=$(kubectl -n "kuttl-test-crack-mammoth" get statefulset \\\n major-upgrade-logical-replica-lr-analytics -o jsonpath="{.spec.replicas}")\n echo "logical replica statefulset replicas: ${replicas}"\n [[ ${replicas} == "0" ]]\n' logger.go:42: 21:29:35 | major-upgrade-logical-replica/6-verify-replica-invalidated | + local max=30 logger.go:42: 21:29:35 | major-upgrade-logical-replica/6-verify-replica-invalidated | + local delay=5 logger.go:42: 21:29:35 | major-upgrade-logical-replica/6-verify-replica-invalidated | + shift 2 logger.go:42: 21:29:35 | major-upgrade-logical-replica/6-verify-replica-invalidated | + local n=1 logger.go:42: 21:29:35 | major-upgrade-logical-replica/6-verify-replica-invalidated | + bash -c $'\n replicas=$(kubectl -n "kuttl-test-crack-mammoth" get statefulset \\\n major-upgrade-logical-replica-lr-analytics -o jsonpath="{.spec.replicas}")\n echo "logical replica statefulset replicas: ${replicas}"\n [[ ${replicas} == "0" ]]\n' logger.go:42: 21:29:36 | major-upgrade-logical-replica/6-verify-replica-invalidated | logical replica statefulset replicas: 0 logger.go:42: 21:29:36 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ get_psql_user_pass major-upgrade-logical-replica-pguser-postgres logger.go:42: 21:29:36 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ local secret_name=major-upgrade-logical-replica-pguser-postgres logger.go:42: 21:29:36 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ kubectl -n kuttl-test-crack-mammoth get secret/major-upgrade-logical-replica-pguser-postgres '--template={{.data.password | base64decode}}' logger.go:42: 21:29:36 | major-upgrade-logical-replica/6-verify-replica-invalidated | + PASS=sDe1VtqrVUd9fUX0dZMg5ebe logger.go:42: 21:29:36 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ get_psql_user_host major-upgrade-logical-replica-pguser-postgres logger.go:42: 21:29:36 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ local secret_name=major-upgrade-logical-replica-pguser-postgres logger.go:42: 21:29:36 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ kubectl -n kuttl-test-crack-mammoth get secret/major-upgrade-logical-replica-pguser-postgres '--template={{.data.host | base64decode }}' logger.go:42: 21:29:37 | major-upgrade-logical-replica/6-verify-replica-invalidated | + HOST=major-upgrade-logical-replica-primary.kuttl-test-crack-mammoth.svc logger.go:42: 21:29:37 | major-upgrade-logical-replica/6-verify-replica-invalidated | + retry 30 5 no_slots_left logger.go:42: 21:29:37 | major-upgrade-logical-replica/6-verify-replica-invalidated | + local max=30 logger.go:42: 21:29:37 | major-upgrade-logical-replica/6-verify-replica-invalidated | + local delay=5 logger.go:42: 21:29:37 | major-upgrade-logical-replica/6-verify-replica-invalidated | + shift 2 logger.go:42: 21:29:37 | major-upgrade-logical-replica/6-verify-replica-invalidated | + local n=1 logger.go:42: 21:29:37 | major-upgrade-logical-replica/6-verify-replica-invalidated | + no_slots_left logger.go:42: 21:29:37 | major-upgrade-logical-replica/6-verify-replica-invalidated | + local got logger.go:42: 21:29:37 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ run_psql 'SELECT count(*) FROM pg_replication_slots WHERE plugin IS NOT NULL;' postgres://postgres:sDe1VtqrVUd9fUX0dZMg5ebe@major-upgrade-logical-replica-primary.kuttl-test-crack-mammoth.svc logger.go:42: 21:29:37 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ local 'command=SELECT count(*) FROM pg_replication_slots WHERE plugin IS NOT NULL;' logger.go:42: 21:29:37 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ local uri=postgres://postgres:sDe1VtqrVUd9fUX0dZMg5ebe@major-upgrade-logical-replica-primary.kuttl-test-crack-mammoth.svc logger.go:42: 21:29:37 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ local password= logger.go:42: 21:29:37 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ xargs logger.go:42: 21:29:37 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ get_client_pod logger.go:42: 21:29:37 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ kubectl -n kuttl-test-crack-mammoth get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' logger.go:42: 21:29:37 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ kubectl -n kuttl-test-crack-mammoth exec pg-client-d7c7bb45b-xgndw -- bash -c 'printf '\''SELECT count(*) FROM pg_replication_slots WHERE plugin IS NOT NULL;\n'\'' | PGPASSWORD='\'''\'' psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:sDe1VtqrVUd9fUX0dZMg5ebe@major-upgrade-logical-replica-primary.kuttl-test-crack-mammoth.svc' logger.go:42: 21:29:38 | major-upgrade-logical-replica/6-verify-replica-invalidated | + got=0 logger.go:42: 21:29:38 | major-upgrade-logical-replica/6-verify-replica-invalidated | + echo 'logical slots left on the primary: 0' logger.go:42: 21:29:38 | major-upgrade-logical-replica/6-verify-replica-invalidated | logical slots left on the primary: 0 logger.go:42: 21:29:38 | major-upgrade-logical-replica/6-verify-replica-invalidated | + [[ 0 == 0 ]] logger.go:42: 21:29:38 | major-upgrade-logical-replica/6-verify-replica-invalidated | test step completed 6-verify-replica-invalidated logger.go:42: 21:29:38 | major-upgrade-logical-replica/7-create-reseed-backup | starting test step 7-create-reseed-backup logger.go:42: 21:29:38 | major-upgrade-logical-replica/7-create-reseed-backup | running command: [sh -c set -o errexit set -o xtrace source ../../functions # pg_upgrade left the repository holding only pre-upgrade backups, so the # bootstrap Job of the reseed below would restore a PostgreSQL 17 data # directory. Take a full backup of the upgraded cluster first. cat </dev/null 2>&1 && return 1 kubectl -n "${NAMESPACE}" get pvc major-upgrade-logical-replica-lr-analytics-pgdata >/dev/null 2>&1 && return 1 return 0 } retry 60 5 status_gone retry 60 5 objects_gone kubectl -n "${NAMESPACE}" patch pg major-upgrade-logical-replica --type=merge --patch '{ "spec": { "logicalReplicas": [ { "name": "analytics", "databases": ["myapp"], "bootstrapMethod": "pgbackrest", "dataVolumeClaimSpec": { "accessModes": ["ReadWriteOnce"], "resources": {"requests": {"storage": "1Gi"}} } } ] } }'] logger.go:42: 21:30:23 | major-upgrade-logical-replica/8-reseed-logical-replica | + source ../../functions logger.go:42: 21:30:23 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ realpath ../../.. logger.go:42: 21:30:23 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812 logger.go:42: 21:30:23 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ CERT_MANAGER_VER=1.20.2 logger.go:42: 21:30:23 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ CERT_MANAGER_TEST_LABEL=e2e.percona.com/managed-by=percona-postgresql-operator-tests logger.go:42: 21:30:23 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ CHAOS_MESH_VER=2.5.1 logger.go:42: 21:30:23 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ BUSYBOX_VER=1.36 logger.go:42: 21:30:23 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ pwd logger.go:42: 21:30:23 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ basename /mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests/tests/major-upgrade-logical-replica logger.go:42: 21:30:23 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ test_name=major-upgrade-logical-replica logger.go:42: 21:30:23 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ source /mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests/vars.sh logger.go:42: 21:30:23 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812 logger.go:42: 21:30:23 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812 logger.go:42: 21:30:23 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/deploy logger.go:42: 21:30:23 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/deploy logger.go:42: 21:30:23 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests logger.go:42: 21:30:23 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests logger.go:42: 21:30:23 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests/conf logger.go:42: 21:30:23 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests/conf logger.go:42: 21:30:23 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export TEST_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests/tests/major-upgrade-logical-replica/conf logger.go:42: 21:30:23 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ TEST_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests/tests/major-upgrade-logical-replica/conf logger.go:42: 21:30:23 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export TEMP_DIR=/tmp/kuttl/pg/major-upgrade-logical-replica logger.go:42: 21:30:23 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ TEMP_DIR=/tmp/kuttl/pg/major-upgrade-logical-replica logger.go:42: 21:30:23 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ git rev-parse --abbrev-ref HEAD logger.go:42: 21:30:23 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export GIT_BRANCH=PR-1812 logger.go:42: 21:30:23 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ GIT_BRANCH=PR-1812 logger.go:42: 21:30:23 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export VERSION=PR-1812-09e4007fd logger.go:42: 21:30:23 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ VERSION=PR-1812-09e4007fd logger.go:42: 21:30:23 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export SKIP_TEST_WARNINGS=true logger.go:42: 21:30:23 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ SKIP_TEST_WARNINGS=true logger.go:42: 21:30:23 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export K8S_UPGRADE_PLATFORM=gke logger.go:42: 21:30:23 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ K8S_UPGRADE_PLATFORM=gke logger.go:42: 21:30:23 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export K8S_UPGRADE_REGION=us-central1-a logger.go:42: 21:30:23 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ K8S_UPGRADE_REGION=us-central1-a logger.go:42: 21:30:23 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export K8S_UPGRADE_INITIAL_VERSION= logger.go:42: 21:30:23 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ K8S_UPGRADE_INITIAL_VERSION= logger.go:42: 21:30:23 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export K8S_UPGRADE_FINAL_VERSION= logger.go:42: 21:30:23 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ K8S_UPGRADE_FINAL_VERSION= logger.go:42: 21:30:23 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ -z '' ]] logger.go:42: 21:30:23 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ detect_k8s_platform logger.go:42: 21:30:23 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ local detected_platform logger.go:42: 21:30:23 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ local xtrace_enabled=false logger.go:42: 21:30:23 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ [[ -o xtrace ]] logger.go:42: 21:30:23 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ xtrace_enabled=true logger.go:42: 21:30:23 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ set +o xtrace logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ PLATFORM=gke logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export PLATFORM logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE=perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE=perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ ! -n '' ]] logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ -n '' ]] logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export PG_VER=18 logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ PG_VER=18 logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export PG_DISTRIBUTION= logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ PG_DISTRIBUTION= logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ '' == community ]] logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export BUCKET=pg-operator-testing logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ BUCKET=pg-operator-testing logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export PMM_SERVER_VERSION=9.9.9 logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ PMM_SERVER_VERSION=9.9.9 logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_LOGCOLLECTOR=perconalab/fluentbit:main-logcollector logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_LOGCOLLECTOR=perconalab/fluentbit:main-logcollector logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export PGOV1_TAG=1.4.0 logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ PGOV1_TAG=1.4.0 logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export PGOV1_VER=14 logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ PGOV1_VER=14 logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export CPGO_VERSION=5.8.7 logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ CPGO_VERSION=5.8.7 logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export MINIO_VER=5.4.0 logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ MINIO_VER=5.4.0 logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export VAULT_VER=0.32.0 logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ VAULT_VER=0.32.0 logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export REGISTRY_NAME=docker.io logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ REGISTRY_NAME=docker.io logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export REGISTRY_NAME_FULL=docker.io/ logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ REGISTRY_NAME_FULL=docker.io/ logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ printenv logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ grep -E '^IMAGE' logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ awk -F= '{print $1}' logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ eval 'echo $IMAGE_BACKREST' logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++++ echo perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ var_value=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == docker.io/* ]] logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == percona/* ]] logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == perconalab/* ]] logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ eval 'echo $IMAGE' logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++++ echo perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ var_value=perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:PR-1812-09e4007fd == docker.io/* ]] logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:PR-1812-09e4007fd == percona/* ]] logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:PR-1812-09e4007fd == perconalab/* ]] logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ new_value=docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ eval 'echo $IMAGE_UPGRADE' logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++++ echo perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ var_value=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == docker.io/* ]] logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == percona/* ]] logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == perconalab/* ]] logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ eval 'echo $IMAGE_LOGCOLLECTOR' logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++++ echo perconalab/fluentbit:main-logcollector logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ var_value=perconalab/fluentbit:main-logcollector logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/fluentbit:main-logcollector == docker.io/* ]] logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/fluentbit:main-logcollector == percona/* ]] logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/fluentbit:main-logcollector == perconalab/* ]] logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ new_value=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ eval 'echo $IMAGE_BASE' logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++++ echo perconalab/percona-postgresql-operator logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ var_value=perconalab/percona-postgresql-operator logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator == docker.io/* ]] logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator == percona/* ]] logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator == perconalab/* ]] logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ new_value=docker.io/perconalab/percona-postgresql-operator logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ eval 'echo $IMAGE_PGBOUNCER' logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++++ echo perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ var_value=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == docker.io/* ]] logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == percona/* ]] logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == perconalab/* ]] logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ eval 'echo $IMAGE_POSTGRESQL' logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++++ echo perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ var_value=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == docker.io/* ]] logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == percona/* ]] logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == perconalab/* ]] logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ eval 'echo $IMAGE_PMM_SERVER' logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++++ echo perconalab/pmm-server:3-dev-latest logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ var_value=perconalab/pmm-server:3-dev-latest logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/pmm-server:3-dev-latest == docker.io/* ]] logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/pmm-server:3-dev-latest == percona/* ]] logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/pmm-server:3-dev-latest == perconalab/* ]] logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ new_value=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ eval 'echo $IMAGE_PMM_CLIENT' logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++++ echo perconalab/pmm-client:3-dev-latest logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ var_value=perconalab/pmm-client:3-dev-latest logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/pmm-client:3-dev-latest == docker.io/* ]] logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/pmm-client:3-dev-latest == percona/* ]] logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/pmm-client:3-dev-latest == perconalab/* ]] logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ new_value=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ which gdate logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | which: no gdate in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ which date logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ date=/usr/sbin/date logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ which gsed logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | which: no gsed in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ which sed logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ sed=/usr/sbin/sed logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | + kubectl -n kuttl-test-crack-mammoth patch pg major-upgrade-logical-replica --type=json --patch '[{"op": "remove", "path": "/spec/logicalReplicas"}]' logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | perconapgcluster.pgv2.percona.com/major-upgrade-logical-replica patched logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | + retry 60 5 status_gone logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | + local max=60 logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | + local delay=5 logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | + shift 2 logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | + local n=1 logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | + status_gone logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | + local got logger.go:42: 21:30:25 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ kubectl -n kuttl-test-crack-mammoth get pg major-upgrade-logical-replica -o 'jsonpath={.status.logicalReplicas}' logger.go:42: 21:30:26 | major-upgrade-logical-replica/8-reseed-logical-replica | + got='[{"invalidatedAt":"2026-09-01T21:29:26Z","message":"this replica holds a PostgreSQL 17 data directory and the cluster now runs 18; the pg_upgrade job only rewrites the data directories of the instance sets, so this one cannot be started again; remove it from spec.logicalReplicas, wait for its entry to leave status.logicalReplicas, and add it back to seed it again","name":"analytics","postgresVersion":17,"reason":"SourceUpgraded","seededAt":"2026-09-01T21:26:08Z","state":"broken"}]' logger.go:42: 21:30:26 | major-upgrade-logical-replica/8-reseed-logical-replica | + echo 'status.logicalReplicas: '\''[{"invalidatedAt":"2026-09-01T21:29:26Z","message":"this replica holds a PostgreSQL 17 data directory and the cluster now runs 18; the pg_upgrade job only rewrites the data directories of the instance sets, so this one cannot be started again; remove it from spec.logicalReplicas, wait for its entry to leave status.logicalReplicas, and add it back to seed it again","name":"analytics","postgresVersion":17,"reason":"SourceUpgraded","seededAt":"2026-09-01T21:26:08Z","state":"broken"}]'\''' logger.go:42: 21:30:26 | major-upgrade-logical-replica/8-reseed-logical-replica | status.logicalReplicas: '[{"invalidatedAt":"2026-09-01T21:29:26Z","message":"this replica holds a PostgreSQL 17 data directory and the cluster now runs 18; the pg_upgrade job only rewrites the data directories of the instance sets, so this one cannot be started again; remove it from spec.logicalReplicas, wait for its entry to leave status.logicalReplicas, and add it back to seed it again","name":"analytics","postgresVersion":17,"reason":"SourceUpgraded","seededAt":"2026-09-01T21:26:08Z","state":"broken"}]' logger.go:42: 21:30:26 | major-upgrade-logical-replica/8-reseed-logical-replica | + [[ -z [{"invalidatedAt":"2026-09-01T21:29:26Z","message":"this replica holds a PostgreSQL 17 data directory and the cluster now runs 18; the pg_upgrade job only rewrites the data directories of the instance sets, so this one cannot be started again; remove it from spec.logicalReplicas, wait for its entry to leave status.logicalReplicas, and add it back to seed it again","name":"analytics","postgresVersion":17,"reason":"SourceUpgraded","seededAt":"2026-09-01T21:26:08Z","state":"broken"}] ]] logger.go:42: 21:30:26 | major-upgrade-logical-replica/8-reseed-logical-replica | + [[ [{"invalidatedAt":"2026-09-01T21:29:26Z","message":"this replica holds a PostgreSQL 17 data directory and the cluster now runs 18; the pg_upgrade job only rewrites the data directories of the instance sets, so this one cannot be started again; remove it from spec.logicalReplicas, wait for its entry to leave status.logicalReplicas, and add it back to seed it again","name":"analytics","postgresVersion":17,"reason":"SourceUpgraded","seededAt":"2026-09-01T21:26:08Z","state":"broken"}] == \[\] ]] logger.go:42: 21:30:26 | major-upgrade-logical-replica/8-reseed-logical-replica | + [[ 1 -ge 60 ]] logger.go:42: 21:30:26 | major-upgrade-logical-replica/8-reseed-logical-replica | + (( n += 1 )) logger.go:42: 21:30:26 | major-upgrade-logical-replica/8-reseed-logical-replica | + sleep 5 logger.go:42: 21:30:31 | major-upgrade-logical-replica/8-reseed-logical-replica | + status_gone logger.go:42: 21:30:31 | major-upgrade-logical-replica/8-reseed-logical-replica | + local got logger.go:42: 21:30:31 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ kubectl -n kuttl-test-crack-mammoth get pg major-upgrade-logical-replica -o 'jsonpath={.status.logicalReplicas}' logger.go:42: 21:30:31 | major-upgrade-logical-replica/8-reseed-logical-replica | + got= logger.go:42: 21:30:31 | major-upgrade-logical-replica/8-reseed-logical-replica | + echo 'status.logicalReplicas: '\'''\''' logger.go:42: 21:30:31 | major-upgrade-logical-replica/8-reseed-logical-replica | status.logicalReplicas: '' logger.go:42: 21:30:31 | major-upgrade-logical-replica/8-reseed-logical-replica | + [[ -z '' ]] logger.go:42: 21:30:31 | major-upgrade-logical-replica/8-reseed-logical-replica | + retry 60 5 objects_gone logger.go:42: 21:30:31 | major-upgrade-logical-replica/8-reseed-logical-replica | + local max=60 logger.go:42: 21:30:31 | major-upgrade-logical-replica/8-reseed-logical-replica | + local delay=5 logger.go:42: 21:30:31 | major-upgrade-logical-replica/8-reseed-logical-replica | + shift 2 logger.go:42: 21:30:31 | major-upgrade-logical-replica/8-reseed-logical-replica | + local n=1 logger.go:42: 21:30:31 | major-upgrade-logical-replica/8-reseed-logical-replica | + objects_gone logger.go:42: 21:30:31 | major-upgrade-logical-replica/8-reseed-logical-replica | + kubectl -n kuttl-test-crack-mammoth get statefulset major-upgrade-logical-replica-lr-analytics logger.go:42: 21:30:32 | major-upgrade-logical-replica/8-reseed-logical-replica | + kubectl -n kuttl-test-crack-mammoth get pvc major-upgrade-logical-replica-lr-analytics-pgdata logger.go:42: 21:30:33 | major-upgrade-logical-replica/8-reseed-logical-replica | + return 0 logger.go:42: 21:30:33 | major-upgrade-logical-replica/8-reseed-logical-replica | + kubectl -n kuttl-test-crack-mammoth patch pg major-upgrade-logical-replica --type=merge --patch $'{\n "spec": {\n "logicalReplicas": [\n {\n "name": "analytics",\n "databases": ["myapp"],\n "bootstrapMethod": "pgbackrest",\n "dataVolumeClaimSpec": {\n "accessModes": ["ReadWriteOnce"],\n "resources": {"requests": {"storage": "1Gi"}}\n }\n }\n ]\n }\n}' logger.go:42: 21:30:33 | major-upgrade-logical-replica/8-reseed-logical-replica | perconapgcluster.pgv2.percona.com/major-upgrade-logical-replica patched logger.go:42: 21:30:33 | major-upgrade-logical-replica/8-reseed-logical-replica | running command: [sh -c set -o errexit set -o xtrace source ../../functions retry 90 10 bash -c ' state=$(kubectl -n "'"${NAMESPACE}"'" get pg major-upgrade-logical-replica \ -o jsonpath="{.status.logicalReplicas[?(@.name==\"analytics\")].state}") echo "logical replica analytics state: ${state}" [[ ${state} == "ready" ]] ' # Seeded from the upgraded cluster, so it is stamped with the new version. seeded=$(kubectl -n "${NAMESPACE}" get pg major-upgrade-logical-replica \ -o jsonpath='{.status.logicalReplicas[?(@.name=="analytics")].postgresVersion}') echo "seeded at PostgreSQL: ${seeded}" [[ ${seeded} == "18" ]] kubectl -n "${NAMESPACE}" wait --for=condition=Ready pod \ -l "pgv2.percona.com/logical-replica=analytics" --timeout=300s PASS=$(get_psql_user_pass major-upgrade-logical-replica-pguser-postgres) HOST=$(get_psql_user_host major-upgrade-logical-replica-pguser-postgres) REPLICA="major-upgrade-logical-replica-lr-analytics.${NAMESPACE}.svc" # Replication actually streams again on the upgraded cluster. run_psql \ '\c myapp \\\ INSERT INTO myApp (id) VALUES (100501);' \ "postgres://postgres:${PASS}@${HOST}" row_streamed() { local got got=$(run_psql \ '\c myapp \\\ SELECT count(*) FROM myApp WHERE id = 100501;' \ "postgres://postgres:${PASS}@${REPLICA}" | xargs) echo "streamed rows on the replica: ${got}" [[ ${got} == "1" ]] } retry 30 5 row_streamed] logger.go:42: 21:30:33 | major-upgrade-logical-replica/8-reseed-logical-replica | + source ../../functions logger.go:42: 21:30:33 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ realpath ../../.. logger.go:42: 21:30:33 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812 logger.go:42: 21:30:33 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ CERT_MANAGER_VER=1.20.2 logger.go:42: 21:30:33 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ CERT_MANAGER_TEST_LABEL=e2e.percona.com/managed-by=percona-postgresql-operator-tests logger.go:42: 21:30:33 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ CHAOS_MESH_VER=2.5.1 logger.go:42: 21:30:33 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ BUSYBOX_VER=1.36 logger.go:42: 21:30:33 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ pwd logger.go:42: 21:30:33 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ basename /mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests/tests/major-upgrade-logical-replica logger.go:42: 21:30:33 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ test_name=major-upgrade-logical-replica logger.go:42: 21:30:33 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ source /mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests/vars.sh logger.go:42: 21:30:33 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812 logger.go:42: 21:30:33 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812 logger.go:42: 21:30:33 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/deploy logger.go:42: 21:30:33 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/deploy logger.go:42: 21:30:33 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests logger.go:42: 21:30:33 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests logger.go:42: 21:30:33 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests/conf logger.go:42: 21:30:33 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests/conf logger.go:42: 21:30:33 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export TEST_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests/tests/major-upgrade-logical-replica/conf logger.go:42: 21:30:33 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ TEST_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests/tests/major-upgrade-logical-replica/conf logger.go:42: 21:30:33 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export TEMP_DIR=/tmp/kuttl/pg/major-upgrade-logical-replica logger.go:42: 21:30:33 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ TEMP_DIR=/tmp/kuttl/pg/major-upgrade-logical-replica logger.go:42: 21:30:33 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ git rev-parse --abbrev-ref HEAD logger.go:42: 21:30:33 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export GIT_BRANCH=PR-1812 logger.go:42: 21:30:33 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ GIT_BRANCH=PR-1812 logger.go:42: 21:30:33 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export VERSION=PR-1812-09e4007fd logger.go:42: 21:30:33 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ VERSION=PR-1812-09e4007fd logger.go:42: 21:30:33 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export SKIP_TEST_WARNINGS=true logger.go:42: 21:30:33 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ SKIP_TEST_WARNINGS=true logger.go:42: 21:30:33 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export K8S_UPGRADE_PLATFORM=gke logger.go:42: 21:30:33 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ K8S_UPGRADE_PLATFORM=gke logger.go:42: 21:30:33 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export K8S_UPGRADE_REGION=us-central1-a logger.go:42: 21:30:33 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ K8S_UPGRADE_REGION=us-central1-a logger.go:42: 21:30:33 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export K8S_UPGRADE_INITIAL_VERSION= logger.go:42: 21:30:33 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ K8S_UPGRADE_INITIAL_VERSION= logger.go:42: 21:30:33 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export K8S_UPGRADE_FINAL_VERSION= logger.go:42: 21:30:33 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ K8S_UPGRADE_FINAL_VERSION= logger.go:42: 21:30:33 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ -z '' ]] logger.go:42: 21:30:33 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ detect_k8s_platform logger.go:42: 21:30:33 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ local detected_platform logger.go:42: 21:30:33 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ local xtrace_enabled=false logger.go:42: 21:30:33 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ [[ -o xtrace ]] logger.go:42: 21:30:33 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ xtrace_enabled=true logger.go:42: 21:30:33 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ set +o xtrace logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ PLATFORM=gke logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export PLATFORM logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE=perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE=perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ ! -n '' ]] logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ -n '' ]] logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export PG_VER=18 logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ PG_VER=18 logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export PG_DISTRIBUTION= logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ PG_DISTRIBUTION= logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ '' == community ]] logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export BUCKET=pg-operator-testing logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ BUCKET=pg-operator-testing logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export PMM_SERVER_VERSION=9.9.9 logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ PMM_SERVER_VERSION=9.9.9 logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_LOGCOLLECTOR=perconalab/fluentbit:main-logcollector logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_LOGCOLLECTOR=perconalab/fluentbit:main-logcollector logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export PGOV1_TAG=1.4.0 logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ PGOV1_TAG=1.4.0 logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export PGOV1_VER=14 logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ PGOV1_VER=14 logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export CPGO_VERSION=5.8.7 logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ CPGO_VERSION=5.8.7 logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export MINIO_VER=5.4.0 logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ MINIO_VER=5.4.0 logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export VAULT_VER=0.32.0 logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ VAULT_VER=0.32.0 logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export REGISTRY_NAME=docker.io logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ REGISTRY_NAME=docker.io logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export REGISTRY_NAME_FULL=docker.io/ logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ REGISTRY_NAME_FULL=docker.io/ logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ printenv logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ grep -E '^IMAGE' logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ awk -F= '{print $1}' logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ eval 'echo $IMAGE_BACKREST' logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++++ echo perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ var_value=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == docker.io/* ]] logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == percona/* ]] logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == perconalab/* ]] logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ eval 'echo $IMAGE' logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++++ echo perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ var_value=perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:PR-1812-09e4007fd == docker.io/* ]] logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:PR-1812-09e4007fd == percona/* ]] logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:PR-1812-09e4007fd == perconalab/* ]] logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ new_value=docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ eval 'echo $IMAGE_UPGRADE' logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++++ echo perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ var_value=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == docker.io/* ]] logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == percona/* ]] logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == perconalab/* ]] logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ eval 'echo $IMAGE_LOGCOLLECTOR' logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++++ echo perconalab/fluentbit:main-logcollector logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ var_value=perconalab/fluentbit:main-logcollector logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/fluentbit:main-logcollector == docker.io/* ]] logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/fluentbit:main-logcollector == percona/* ]] logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/fluentbit:main-logcollector == perconalab/* ]] logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ new_value=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ eval 'echo $IMAGE_BASE' logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++++ echo perconalab/percona-postgresql-operator logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ var_value=perconalab/percona-postgresql-operator logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator == docker.io/* ]] logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator == percona/* ]] logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator == perconalab/* ]] logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ new_value=docker.io/perconalab/percona-postgresql-operator logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ eval 'echo $IMAGE_PGBOUNCER' logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++++ echo perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ var_value=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == docker.io/* ]] logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == percona/* ]] logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == perconalab/* ]] logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ eval 'echo $IMAGE_POSTGRESQL' logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++++ echo perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ var_value=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == docker.io/* ]] logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == percona/* ]] logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == perconalab/* ]] logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ eval 'echo $IMAGE_PMM_SERVER' logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++++ echo perconalab/pmm-server:3-dev-latest logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ var_value=perconalab/pmm-server:3-dev-latest logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/pmm-server:3-dev-latest == docker.io/* ]] logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/pmm-server:3-dev-latest == percona/* ]] logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/pmm-server:3-dev-latest == perconalab/* ]] logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ new_value=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ eval 'echo $IMAGE_PMM_CLIENT' logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++++ echo perconalab/pmm-client:3-dev-latest logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ var_value=perconalab/pmm-client:3-dev-latest logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/pmm-client:3-dev-latest == docker.io/* ]] logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/pmm-client:3-dev-latest == percona/* ]] logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ [[ perconalab/pmm-client:3-dev-latest == perconalab/* ]] logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ new_value=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ export IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ which gdate logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | which: no gdate in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ which date logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ date=/usr/sbin/date logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ which gsed logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | which: no gsed in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | ++++ which sed logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ sed=/usr/sbin/sed logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | + retry 90 10 bash -c $'\n state=$(kubectl -n "kuttl-test-crack-mammoth" get pg major-upgrade-logical-replica \\\n -o jsonpath="{.status.logicalReplicas[?(@.name==\\"analytics\\")].state}")\n echo "logical replica analytics state: ${state}"\n [[ ${state} == "ready" ]]\n' logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | + local max=90 logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | + local delay=10 logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | + shift 2 logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | + local n=1 logger.go:42: 21:30:35 | major-upgrade-logical-replica/8-reseed-logical-replica | + bash -c $'\n state=$(kubectl -n "kuttl-test-crack-mammoth" get pg major-upgrade-logical-replica \\\n -o jsonpath="{.status.logicalReplicas[?(@.name==\\"analytics\\")].state}")\n echo "logical replica analytics state: ${state}"\n [[ ${state} == "ready" ]]\n' logger.go:42: 21:30:36 | major-upgrade-logical-replica/8-reseed-logical-replica | logical replica analytics state: bootstrapping logger.go:42: 21:30:36 | major-upgrade-logical-replica/8-reseed-logical-replica | + [[ 1 -ge 90 ]] logger.go:42: 21:30:36 | major-upgrade-logical-replica/8-reseed-logical-replica | + (( n += 1 )) logger.go:42: 21:30:36 | major-upgrade-logical-replica/8-reseed-logical-replica | + sleep 10 logger.go:42: 21:30:46 | major-upgrade-logical-replica/8-reseed-logical-replica | + bash -c $'\n state=$(kubectl -n "kuttl-test-crack-mammoth" get pg major-upgrade-logical-replica \\\n -o jsonpath="{.status.logicalReplicas[?(@.name==\\"analytics\\")].state}")\n echo "logical replica analytics state: ${state}"\n [[ ${state} == "ready" ]]\n' logger.go:42: 21:30:46 | major-upgrade-logical-replica/8-reseed-logical-replica | logical replica analytics state: bootstrapping logger.go:42: 21:30:46 | major-upgrade-logical-replica/8-reseed-logical-replica | + [[ 2 -ge 90 ]] logger.go:42: 21:30:46 | major-upgrade-logical-replica/8-reseed-logical-replica | + (( n += 1 )) logger.go:42: 21:30:46 | major-upgrade-logical-replica/8-reseed-logical-replica | + sleep 10 logger.go:42: 21:30:56 | major-upgrade-logical-replica/8-reseed-logical-replica | + bash -c $'\n state=$(kubectl -n "kuttl-test-crack-mammoth" get pg major-upgrade-logical-replica \\\n -o jsonpath="{.status.logicalReplicas[?(@.name==\\"analytics\\")].state}")\n echo "logical replica analytics state: ${state}"\n [[ ${state} == "ready" ]]\n' logger.go:42: 21:30:57 | major-upgrade-logical-replica/8-reseed-logical-replica | logical replica analytics state: bootstrapping logger.go:42: 21:30:57 | major-upgrade-logical-replica/8-reseed-logical-replica | + [[ 3 -ge 90 ]] logger.go:42: 21:30:57 | major-upgrade-logical-replica/8-reseed-logical-replica | + (( n += 1 )) logger.go:42: 21:30:57 | major-upgrade-logical-replica/8-reseed-logical-replica | + sleep 10 logger.go:42: 21:31:07 | major-upgrade-logical-replica/8-reseed-logical-replica | + bash -c $'\n state=$(kubectl -n "kuttl-test-crack-mammoth" get pg major-upgrade-logical-replica \\\n -o jsonpath="{.status.logicalReplicas[?(@.name==\\"analytics\\")].state}")\n echo "logical replica analytics state: ${state}"\n [[ ${state} == "ready" ]]\n' logger.go:42: 21:31:08 | major-upgrade-logical-replica/8-reseed-logical-replica | logical replica analytics state: bootstrapping logger.go:42: 21:31:08 | major-upgrade-logical-replica/8-reseed-logical-replica | + [[ 4 -ge 90 ]] logger.go:42: 21:31:08 | major-upgrade-logical-replica/8-reseed-logical-replica | + (( n += 1 )) logger.go:42: 21:31:08 | major-upgrade-logical-replica/8-reseed-logical-replica | + sleep 10 logger.go:42: 21:31:18 | major-upgrade-logical-replica/8-reseed-logical-replica | + bash -c $'\n state=$(kubectl -n "kuttl-test-crack-mammoth" get pg major-upgrade-logical-replica \\\n -o jsonpath="{.status.logicalReplicas[?(@.name==\\"analytics\\")].state}")\n echo "logical replica analytics state: ${state}"\n [[ ${state} == "ready" ]]\n' logger.go:42: 21:31:18 | major-upgrade-logical-replica/8-reseed-logical-replica | logical replica analytics state: bootstrapping logger.go:42: 21:31:18 | major-upgrade-logical-replica/8-reseed-logical-replica | + [[ 5 -ge 90 ]] logger.go:42: 21:31:18 | major-upgrade-logical-replica/8-reseed-logical-replica | + (( n += 1 )) logger.go:42: 21:31:18 | major-upgrade-logical-replica/8-reseed-logical-replica | + sleep 10 logger.go:42: 21:31:28 | major-upgrade-logical-replica/8-reseed-logical-replica | + bash -c $'\n state=$(kubectl -n "kuttl-test-crack-mammoth" get pg major-upgrade-logical-replica \\\n -o jsonpath="{.status.logicalReplicas[?(@.name==\\"analytics\\")].state}")\n echo "logical replica analytics state: ${state}"\n [[ ${state} == "ready" ]]\n' logger.go:42: 21:31:29 | major-upgrade-logical-replica/8-reseed-logical-replica | logical replica analytics state: ready logger.go:42: 21:31:29 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ kubectl -n kuttl-test-crack-mammoth get pg major-upgrade-logical-replica -o 'jsonpath={.status.logicalReplicas[?(@.name=="analytics")].postgresVersion}' logger.go:42: 21:31:29 | major-upgrade-logical-replica/8-reseed-logical-replica | + seeded=18 logger.go:42: 21:31:29 | major-upgrade-logical-replica/8-reseed-logical-replica | + echo 'seeded at PostgreSQL: 18' logger.go:42: 21:31:29 | major-upgrade-logical-replica/8-reseed-logical-replica | seeded at PostgreSQL: 18 logger.go:42: 21:31:29 | major-upgrade-logical-replica/8-reseed-logical-replica | + [[ 18 == 18 ]] logger.go:42: 21:31:29 | major-upgrade-logical-replica/8-reseed-logical-replica | + kubectl -n kuttl-test-crack-mammoth wait --for=condition=Ready pod -l pgv2.percona.com/logical-replica=analytics --timeout=300s logger.go:42: 21:31:30 | major-upgrade-logical-replica/8-reseed-logical-replica | pod/major-upgrade-logical-replica-lr-analytics-0 condition met logger.go:42: 21:31:30 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ get_psql_user_pass major-upgrade-logical-replica-pguser-postgres logger.go:42: 21:31:30 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ local secret_name=major-upgrade-logical-replica-pguser-postgres logger.go:42: 21:31:30 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ kubectl -n kuttl-test-crack-mammoth get secret/major-upgrade-logical-replica-pguser-postgres '--template={{.data.password | base64decode}}' logger.go:42: 21:31:30 | major-upgrade-logical-replica/8-reseed-logical-replica | + PASS=sDe1VtqrVUd9fUX0dZMg5ebe logger.go:42: 21:31:30 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ get_psql_user_host major-upgrade-logical-replica-pguser-postgres logger.go:42: 21:31:30 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ local secret_name=major-upgrade-logical-replica-pguser-postgres logger.go:42: 21:31:30 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ kubectl -n kuttl-test-crack-mammoth get secret/major-upgrade-logical-replica-pguser-postgres '--template={{.data.host | base64decode }}' logger.go:42: 21:31:31 | major-upgrade-logical-replica/8-reseed-logical-replica | + HOST=major-upgrade-logical-replica-primary.kuttl-test-crack-mammoth.svc logger.go:42: 21:31:31 | major-upgrade-logical-replica/8-reseed-logical-replica | + REPLICA=major-upgrade-logical-replica-lr-analytics.kuttl-test-crack-mammoth.svc logger.go:42: 21:31:31 | major-upgrade-logical-replica/8-reseed-logical-replica | + run_psql '\c myapp \\\ INSERT INTO myApp (id) VALUES (100501);' postgres://postgres:sDe1VtqrVUd9fUX0dZMg5ebe@major-upgrade-logical-replica-primary.kuttl-test-crack-mammoth.svc logger.go:42: 21:31:31 | major-upgrade-logical-replica/8-reseed-logical-replica | + local 'command=\c myapp \\\ INSERT INTO myApp (id) VALUES (100501);' logger.go:42: 21:31:31 | major-upgrade-logical-replica/8-reseed-logical-replica | + local uri=postgres://postgres:sDe1VtqrVUd9fUX0dZMg5ebe@major-upgrade-logical-replica-primary.kuttl-test-crack-mammoth.svc logger.go:42: 21:31:31 | major-upgrade-logical-replica/8-reseed-logical-replica | + local password= logger.go:42: 21:31:31 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ get_client_pod logger.go:42: 21:31:31 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ kubectl -n kuttl-test-crack-mammoth get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' logger.go:42: 21:31:31 | major-upgrade-logical-replica/8-reseed-logical-replica | + kubectl -n kuttl-test-crack-mammoth exec pg-client-d7c7bb45b-xgndw -- bash -c 'printf '\''\c myapp \\\ INSERT INTO myApp (id) VALUES (100501);\n'\'' | PGPASSWORD='\'''\'' psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:sDe1VtqrVUd9fUX0dZMg5ebe@major-upgrade-logical-replica-primary.kuttl-test-crack-mammoth.svc' logger.go:42: 21:31:32 | major-upgrade-logical-replica/8-reseed-logical-replica | + retry 30 5 row_streamed logger.go:42: 21:31:32 | major-upgrade-logical-replica/8-reseed-logical-replica | + local max=30 logger.go:42: 21:31:32 | major-upgrade-logical-replica/8-reseed-logical-replica | + local delay=5 logger.go:42: 21:31:32 | major-upgrade-logical-replica/8-reseed-logical-replica | + shift 2 logger.go:42: 21:31:32 | major-upgrade-logical-replica/8-reseed-logical-replica | + local n=1 logger.go:42: 21:31:32 | major-upgrade-logical-replica/8-reseed-logical-replica | + row_streamed logger.go:42: 21:31:32 | major-upgrade-logical-replica/8-reseed-logical-replica | + local got logger.go:42: 21:31:32 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ run_psql '\c myapp \\\ SELECT count(*) FROM myApp WHERE id = 100501;' postgres://postgres:sDe1VtqrVUd9fUX0dZMg5ebe@major-upgrade-logical-replica-lr-analytics.kuttl-test-crack-mammoth.svc logger.go:42: 21:31:32 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ local 'command=\c myapp \\\ SELECT count(*) FROM myApp WHERE id = 100501;' logger.go:42: 21:31:32 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ local uri=postgres://postgres:sDe1VtqrVUd9fUX0dZMg5ebe@major-upgrade-logical-replica-lr-analytics.kuttl-test-crack-mammoth.svc logger.go:42: 21:31:32 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ xargs logger.go:42: 21:31:32 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ local password= logger.go:42: 21:31:32 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ get_client_pod logger.go:42: 21:31:32 | major-upgrade-logical-replica/8-reseed-logical-replica | +++ kubectl -n kuttl-test-crack-mammoth get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' logger.go:42: 21:31:33 | major-upgrade-logical-replica/8-reseed-logical-replica | ++ kubectl -n kuttl-test-crack-mammoth exec pg-client-d7c7bb45b-xgndw -- bash -c 'printf '\''\c myapp \\\ SELECT count(*) FROM myApp WHERE id = 100501;\n'\'' | PGPASSWORD='\'''\'' psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:sDe1VtqrVUd9fUX0dZMg5ebe@major-upgrade-logical-replica-lr-analytics.kuttl-test-crack-mammoth.svc' logger.go:42: 21:31:34 | major-upgrade-logical-replica/8-reseed-logical-replica | + got=1 logger.go:42: 21:31:34 | major-upgrade-logical-replica/8-reseed-logical-replica | + echo 'streamed rows on the replica: 1' logger.go:42: 21:31:34 | major-upgrade-logical-replica/8-reseed-logical-replica | streamed rows on the replica: 1 logger.go:42: 21:31:34 | major-upgrade-logical-replica/8-reseed-logical-replica | + [[ 1 == 1 ]] logger.go:42: 21:31:34 | major-upgrade-logical-replica/8-reseed-logical-replica | test step completed 8-reseed-logical-replica logger.go:42: 21:31:34 | major-upgrade-logical-replica/99-remove-cluster-gracefully | starting test step 99-remove-cluster-gracefully logger.go:42: 21:31:35 | major-upgrade-logical-replica/99-remove-cluster-gracefully | PerconaPGCluster:kuttl-test-crack-mammoth/major-upgrade-logical-replica deleted logger.go:42: 21:31:35 | major-upgrade-logical-replica/99-remove-cluster-gracefully | PostgresCluster:kuttl-test-crack-mammoth/major-upgrade-logical-replica deleted logger.go:42: 21:31:42 | major-upgrade-logical-replica/99-remove-cluster-gracefully | running command: [sh -c set -o errexit set -o xtrace source ../../functions remove_all_finalizers check_operator_panic destroy_operator] logger.go:42: 21:31:42 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + source ../../functions logger.go:42: 21:31:42 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ realpath ../../.. logger.go:42: 21:31:42 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812 logger.go:42: 21:31:42 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++ CERT_MANAGER_VER=1.20.2 logger.go:42: 21:31:42 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++ CERT_MANAGER_TEST_LABEL=e2e.percona.com/managed-by=percona-postgresql-operator-tests logger.go:42: 21:31:42 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++ CHAOS_MESH_VER=2.5.1 logger.go:42: 21:31:42 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++ BUSYBOX_VER=1.36 logger.go:42: 21:31:42 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ pwd logger.go:42: 21:31:42 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ basename /mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests/tests/major-upgrade-logical-replica logger.go:42: 21:31:42 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++ test_name=major-upgrade-logical-replica logger.go:42: 21:31:42 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++ source /mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests/vars.sh logger.go:42: 21:31:42 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812 logger.go:42: 21:31:42 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812 logger.go:42: 21:31:42 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/deploy logger.go:42: 21:31:42 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/deploy logger.go:42: 21:31:42 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests logger.go:42: 21:31:42 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests logger.go:42: 21:31:42 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests/conf logger.go:42: 21:31:42 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests/conf logger.go:42: 21:31:42 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export TEST_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests/tests/major-upgrade-logical-replica/conf logger.go:42: 21:31:42 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ TEST_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/e2e-tests/tests/major-upgrade-logical-replica/conf logger.go:42: 21:31:42 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export TEMP_DIR=/tmp/kuttl/pg/major-upgrade-logical-replica logger.go:42: 21:31:42 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ TEMP_DIR=/tmp/kuttl/pg/major-upgrade-logical-replica logger.go:42: 21:31:42 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ git rev-parse --abbrev-ref HEAD logger.go:42: 21:31:42 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export GIT_BRANCH=PR-1812 logger.go:42: 21:31:42 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ GIT_BRANCH=PR-1812 logger.go:42: 21:31:42 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export VERSION=PR-1812-09e4007fd logger.go:42: 21:31:42 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ VERSION=PR-1812-09e4007fd logger.go:42: 21:31:42 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export SKIP_TEST_WARNINGS=true logger.go:42: 21:31:42 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ SKIP_TEST_WARNINGS=true logger.go:42: 21:31:42 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export K8S_UPGRADE_PLATFORM=gke logger.go:42: 21:31:42 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ K8S_UPGRADE_PLATFORM=gke logger.go:42: 21:31:42 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export K8S_UPGRADE_REGION=us-central1-a logger.go:42: 21:31:42 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ K8S_UPGRADE_REGION=us-central1-a logger.go:42: 21:31:42 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export K8S_UPGRADE_INITIAL_VERSION= logger.go:42: 21:31:42 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ K8S_UPGRADE_INITIAL_VERSION= logger.go:42: 21:31:42 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export K8S_UPGRADE_FINAL_VERSION= logger.go:42: 21:31:42 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ K8S_UPGRADE_FINAL_VERSION= logger.go:42: 21:31:42 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ -z '' ]] logger.go:42: 21:31:42 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ detect_k8s_platform logger.go:42: 21:31:42 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ local detected_platform logger.go:42: 21:31:42 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ local xtrace_enabled=false logger.go:42: 21:31:42 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ [[ -o xtrace ]] logger.go:42: 21:31:42 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ xtrace_enabled=true logger.go:42: 21:31:42 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ set +o xtrace logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ PLATFORM=gke logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export PLATFORM logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export IMAGE=perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ IMAGE=perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ ! -n '' ]] logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ -n '' ]] logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export PG_VER=18 logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ PG_VER=18 logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export PG_DISTRIBUTION= logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ PG_DISTRIBUTION= logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ '' == community ]] logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export BUCKET=pg-operator-testing logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ BUCKET=pg-operator-testing logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export PMM_SERVER_VERSION=9.9.9 logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ PMM_SERVER_VERSION=9.9.9 logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export IMAGE_LOGCOLLECTOR=perconalab/fluentbit:main-logcollector logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ IMAGE_LOGCOLLECTOR=perconalab/fluentbit:main-logcollector logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export PGOV1_TAG=1.4.0 logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ PGOV1_TAG=1.4.0 logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export PGOV1_VER=14 logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ PGOV1_VER=14 logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export CPGO_VERSION=5.8.7 logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ CPGO_VERSION=5.8.7 logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export MINIO_VER=5.4.0 logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ MINIO_VER=5.4.0 logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export VAULT_VER=0.32.0 logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ VAULT_VER=0.32.0 logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export REGISTRY_NAME=docker.io logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ REGISTRY_NAME=docker.io logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export REGISTRY_NAME_FULL=docker.io/ logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ REGISTRY_NAME_FULL=docker.io/ logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ printenv logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ grep -E '^IMAGE' logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ awk -F= '{print $1}' logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ eval 'echo $IMAGE_BACKREST' logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++++ echo perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ var_value=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == docker.io/* ]] logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == percona/* ]] logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == perconalab/* ]] logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ echo IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ echo perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ eval 'echo $IMAGE' logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++++ echo perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ var_value=perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/percona-postgresql-operator:PR-1812-09e4007fd == docker.io/* ]] logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/percona-postgresql-operator:PR-1812-09e4007fd == percona/* ]] logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/percona-postgresql-operator:PR-1812-09e4007fd == perconalab/* ]] logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ new_value=docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ echo IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ eval 'echo $IMAGE_UPGRADE' logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++++ echo perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ var_value=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == docker.io/* ]] logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == percona/* ]] logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == perconalab/* ]] logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ echo IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ eval 'echo $IMAGE_LOGCOLLECTOR' logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++++ echo perconalab/fluentbit:main-logcollector logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ var_value=perconalab/fluentbit:main-logcollector logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/fluentbit:main-logcollector == docker.io/* ]] logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/fluentbit:main-logcollector == percona/* ]] logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/fluentbit:main-logcollector == perconalab/* ]] logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ new_value=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ echo IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ eval 'echo $IMAGE_BASE' logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++++ echo perconalab/percona-postgresql-operator logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ var_value=perconalab/percona-postgresql-operator logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/percona-postgresql-operator == docker.io/* ]] logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/percona-postgresql-operator == percona/* ]] logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/percona-postgresql-operator == perconalab/* ]] logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ new_value=docker.io/perconalab/percona-postgresql-operator logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ echo IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ eval 'echo $IMAGE_PGBOUNCER' logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++++ echo perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ var_value=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == docker.io/* ]] logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == percona/* ]] logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == perconalab/* ]] logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ echo IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ eval 'echo $IMAGE_POSTGRESQL' logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++++ echo perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ var_value=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == docker.io/* ]] logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == percona/* ]] logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == perconalab/* ]] logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ echo IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ eval 'echo $IMAGE_PMM_SERVER' logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++++ echo perconalab/pmm-server:3-dev-latest logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ var_value=perconalab/pmm-server:3-dev-latest logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/pmm-server:3-dev-latest == docker.io/* ]] logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/pmm-server:3-dev-latest == percona/* ]] logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/pmm-server:3-dev-latest == perconalab/* ]] logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ new_value=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ echo IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ eval 'echo $IMAGE_PMM_CLIENT' logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++++ echo perconalab/pmm-client:3-dev-latest logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ var_value=perconalab/pmm-client:3-dev-latest logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/pmm-client:3-dev-latest == docker.io/* ]] logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/pmm-client:3-dev-latest == percona/* ]] logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ [[ perconalab/pmm-client:3-dev-latest == perconalab/* ]] logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ new_value=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ export IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ echo IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ which gdate logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | which: no gdate in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ which date logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ date=/usr/sbin/date logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ which gsed logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | which: no gsed in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1812/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++++ which sed logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ sed=/usr/sbin/sed logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + remove_all_finalizers logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + resource_types=("pg-restore" "pg-backup" "pg") logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + for resource in "${resource_types[@]}" logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + echo 'removing all finalizers for pg-restore resources' logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | removing all finalizers for pg-restore resources logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + kubectl -n kuttl-test-crack-mammoth get pg-restore -o json logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + jq '.items[] | .metadata.name' -r logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + IFS= logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + read -r name logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + for resource in "${resource_types[@]}" logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + echo 'removing all finalizers for pg-backup resources' logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | removing all finalizers for pg-backup resources logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + kubectl -n kuttl-test-crack-mammoth get pg-backup -o json logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + jq '.items[] | .metadata.name' -r logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + IFS= logger.go:42: 21:31:44 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + read -r name logger.go:42: 21:31:45 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++ kubectl -n kuttl-test-crack-mammoth get pg-backup major-upgrade-logical-replica-backup-v892-cwlkw -o yaml logger.go:42: 21:31:45 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++ yq '.metadata.finalizers | length' logger.go:42: 21:31:45 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + [[ 0 == 0 ]] logger.go:42: 21:31:45 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + continue logger.go:42: 21:31:45 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + IFS= logger.go:42: 21:31:45 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + read -r name logger.go:42: 21:31:45 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++ kubectl -n kuttl-test-crack-mammoth get pg-backup major-upgrade-logical-replica-backup-zhhh-br4p6 -o yaml logger.go:42: 21:31:45 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++ yq '.metadata.finalizers | length' logger.go:42: 21:31:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + [[ 0 == 0 ]] logger.go:42: 21:31:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + continue logger.go:42: 21:31:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + IFS= logger.go:42: 21:31:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + read -r name logger.go:42: 21:31:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++ kubectl -n kuttl-test-crack-mammoth get pg-backup major-upgrade-logical-replica-reseed-repo1 -o yaml logger.go:42: 21:31:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++ yq '.metadata.finalizers | length' logger.go:42: 21:31:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + [[ 0 == 0 ]] logger.go:42: 21:31:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + continue logger.go:42: 21:31:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + IFS= logger.go:42: 21:31:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + read -r name logger.go:42: 21:31:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++ kubectl -n kuttl-test-crack-mammoth get pg-backup major-upgrade-logical-replica-seed-repo1 -o yaml logger.go:42: 21:31:46 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++ yq '.metadata.finalizers | length' logger.go:42: 21:31:47 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + [[ 0 == 0 ]] logger.go:42: 21:31:47 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + continue logger.go:42: 21:31:47 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + IFS= logger.go:42: 21:31:47 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + read -r name logger.go:42: 21:31:47 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + for resource in "${resource_types[@]}" logger.go:42: 21:31:47 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + echo 'removing all finalizers for pg resources' logger.go:42: 21:31:47 | major-upgrade-logical-replica/99-remove-cluster-gracefully | removing all finalizers for pg resources logger.go:42: 21:31:47 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + kubectl -n kuttl-test-crack-mammoth get pg -o json logger.go:42: 21:31:47 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + jq '.items[] | .metadata.name' -r logger.go:42: 21:31:47 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + IFS= logger.go:42: 21:31:47 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + read -r name logger.go:42: 21:31:47 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + check_operator_panic logger.go:42: 21:31:47 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++ get_operator_pod logger.go:42: 21:31:47 | major-upgrade-logical-replica/99-remove-cluster-gracefully | +++ kubectl get pods -n pg-operator --selector=app.kubernetes.io/name=percona-postgresql-operator -o 'jsonpath={.items[].metadata.name}' logger.go:42: 21:31:48 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++ echo percona-postgresql-operator-85d86fb9f8-twskn logger.go:42: 21:31:48 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + local operator_pod=percona-postgresql-operator-85d86fb9f8-twskn logger.go:42: 21:31:48 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + local panic_log logger.go:42: 21:31:48 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++ kubectl logs -n pg-operator percona-postgresql-operator-85d86fb9f8-twskn -c operator logger.go:42: 21:31:48 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++ grep -A 100 'Observed a panic' logger.go:42: 21:31:49 | major-upgrade-logical-replica/99-remove-cluster-gracefully | ++ true logger.go:42: 21:31:49 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + panic_log= logger.go:42: 21:31:49 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + '[' -n '' ']' logger.go:42: 21:31:49 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + destroy_operator logger.go:42: 21:31:49 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + kubectl -n pg-operator delete deployment percona-postgresql-operator --force --grace-period=0 logger.go:42: 21:31:49 | major-upgrade-logical-replica/99-remove-cluster-gracefully | Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. logger.go:42: 21:31:49 | major-upgrade-logical-replica/99-remove-cluster-gracefully | deployment.apps "percona-postgresql-operator" force deleted from pg-operator namespace logger.go:42: 21:31:49 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + [[ -n pg-operator ]] logger.go:42: 21:31:49 | major-upgrade-logical-replica/99-remove-cluster-gracefully | + kubectl delete namespace pg-operator --force --grace-period=0 --ignore-not-found logger.go:42: 21:31:50 | major-upgrade-logical-replica/99-remove-cluster-gracefully | Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. logger.go:42: 21:31:50 | major-upgrade-logical-replica/99-remove-cluster-gracefully | namespace "pg-operator" force deleted logger.go:42: 21:32:01 | major-upgrade-logical-replica/99-remove-cluster-gracefully | test step completed 99-remove-cluster-gracefully logger.go:42: 21:32:02 | major-upgrade-logical-replica | major-upgrade-logical-replica events from ns kuttl-test-crack-mammoth: logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:21:40 +0000 UTC Normal Pod pg-client-d7c7bb45b-xgndw Scheduled Successfully assigned kuttl-test-crack-mammoth/pg-client-d7c7bb45b-xgndw to gke-jen-pg-1812-09e4007f-default-pool-bb01df06-zfs6 default-scheduler logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:21:40 +0000 UTC Normal ReplicaSet.apps pg-client-d7c7bb45b SuccessfulCreate Created pod: pg-client-d7c7bb45b-xgndw replicaset-controller logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:21:40 +0000 UTC Normal Deployment.apps pg-client ScalingReplicaSet Scaled up replica set pg-client-d7c7bb45b from 0 to 1 deployment-controller logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:21:42 +0000 UTC Normal Pod pg-client-d7c7bb45b-xgndw.spec.containers{pg-client} Pulling Pulling image "perconalab/percona-distribution-postgresql:16" kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:21:52 +0000 UTC Normal PersistentVolumeClaim major-upgrade-logical-replica-instance1-5fpf-pgdata WaitForFirstConsumer waiting for first consumer to be created before binding persistentvolume-controller logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:21:52 +0000 UTC Normal PersistentVolumeClaim major-upgrade-logical-replica-instance1-8csm-pgdata WaitForFirstConsumer waiting for first consumer to be created before binding persistentvolume-controller logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:21:52 +0000 UTC Normal PersistentVolumeClaim major-upgrade-logical-replica-instance1-8csm-pgdata ExternalProvisioning Waiting for a volume to be created either by the external provisioner 'pd.csi.storage.gke.io' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. persistentvolume-controller logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:21:52 +0000 UTC Normal PersistentVolumeClaim major-upgrade-logical-replica-instance1-8csm-pgdata Provisioning External provisioner is provisioning volume for claim "kuttl-test-crack-mammoth/major-upgrade-logical-replica-instance1-8csm-pgdata" pd.csi.storage.gke.io_gke-c6aaa511db014d729735-cd02-f74b-vm_d17f65ab-3170-4d7d-8dcd-19344a0ccb2e logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:21:52 +0000 UTC Normal StatefulSet.apps major-upgrade-logical-replica-instance1-8csm SuccessfulCreate create Pod major-upgrade-logical-replica-instance1-8csm-0 in StatefulSet major-upgrade-logical-replica-instance1-8csm successful statefulset-controller logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:21:53 +0000 UTC Normal PersistentVolumeClaim major-upgrade-logical-replica-instance1-5fpf-pgdata ExternalProvisioning Waiting for a volume to be created either by the external provisioner 'pd.csi.storage.gke.io' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. persistentvolume-controller logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:21:53 +0000 UTC Normal PersistentVolumeClaim major-upgrade-logical-replica-instance1-5fpf-pgdata Provisioning External provisioner is provisioning volume for claim "kuttl-test-crack-mammoth/major-upgrade-logical-replica-instance1-5fpf-pgdata" pd.csi.storage.gke.io_gke-c6aaa511db014d729735-cd02-f74b-vm_d17f65ab-3170-4d7d-8dcd-19344a0ccb2e logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:21:53 +0000 UTC Normal StatefulSet.apps major-upgrade-logical-replica-instance1-5fpf SuccessfulCreate create Pod major-upgrade-logical-replica-instance1-5fpf-0 in StatefulSet major-upgrade-logical-replica-instance1-5fpf successful statefulset-controller logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:21:54 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-8csm-0 Scheduled Successfully assigned kuttl-test-crack-mammoth/major-upgrade-logical-replica-instance1-8csm-0 to gke-jen-pg-1812-09e4007f-default-pool-bb01df06-w6r9 default-scheduler logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:21:54 +0000 UTC Normal PersistentVolumeClaim major-upgrade-logical-replica-instance1-8csm-pgdata ProvisioningSucceeded Successfully provisioned volume pvc-7fb7bdde-2d9b-4f38-a24a-a551dfcfedc8 pd.csi.storage.gke.io_gke-c6aaa511db014d729735-cd02-f74b-vm_d17f65ab-3170-4d7d-8dcd-19344a0ccb2e logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:21:54 +0000 UTC Warning Pod major-upgrade-logical-replica-repo-host-0 FailedScheduling 0/3 nodes are available: persistentvolumeclaim "major-upgrade-logical-replica-repo1" not found. not found default-scheduler logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:21:54 +0000 UTC Normal StatefulSet.apps major-upgrade-logical-replica-repo-host SuccessfulCreate create Pod major-upgrade-logical-replica-repo-host-0 in StatefulSet major-upgrade-logical-replica-repo-host successful statefulset-controller logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:21:54 +0000 UTC Normal PersistentVolumeClaim major-upgrade-logical-replica-repo1 WaitForFirstConsumer waiting for first consumer to be created before binding persistentvolume-controller logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:21:54 +0000 UTC Normal PersistentVolumeClaim major-upgrade-logical-replica-repo1 Provisioning External provisioner is provisioning volume for claim "kuttl-test-crack-mammoth/major-upgrade-logical-replica-repo1" pd.csi.storage.gke.io_gke-c6aaa511db014d729735-cd02-f74b-vm_d17f65ab-3170-4d7d-8dcd-19344a0ccb2e logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:21:54 +0000 UTC Normal PersistentVolumeClaim major-upgrade-logical-replica-repo1 ExternalProvisioning Waiting for a volume to be created either by the external provisioner 'pd.csi.storage.gke.io' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. persistentvolume-controller logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:21:54 +0000 UTC Normal PostgresCluster.upstream.pgv2.percona.com major-upgrade-logical-replica RepoHostCreated created pgBackRest repository host StatefulSet/major-upgrade-logical-replica-repo-host postgrescluster-controller logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:21:55 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-5fpf-0 Scheduled Successfully assigned kuttl-test-crack-mammoth/major-upgrade-logical-replica-instance1-5fpf-0 to gke-jen-pg-1812-09e4007f-default-pool-bb01df06-zfs6 default-scheduler logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:21:55 +0000 UTC Normal PersistentVolumeClaim major-upgrade-logical-replica-instance1-5fpf-pgdata ProvisioningSucceeded Successfully provisioned volume pvc-6390c87f-6bea-4394-964d-b84f079ca0bd pd.csi.storage.gke.io_gke-c6aaa511db014d729735-cd02-f74b-vm_d17f65ab-3170-4d7d-8dcd-19344a0ccb2e logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:21:55 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-6dcfcbb767-84526 Scheduled Successfully assigned kuttl-test-crack-mammoth/major-upgrade-logical-replica-pgbouncer-6dcfcbb767-84526 to gke-jen-pg-1812-09e4007f-default-pool-bb01df06-7d01 default-scheduler logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:21:55 +0000 UTC Normal ReplicaSet.apps major-upgrade-logical-replica-pgbouncer-6dcfcbb767 SuccessfulCreate Created pod: major-upgrade-logical-replica-pgbouncer-6dcfcbb767-84526 replicaset-controller logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:21:55 +0000 UTC Normal Deployment.apps major-upgrade-logical-replica-pgbouncer ScalingReplicaSet Scaled up replica set major-upgrade-logical-replica-pgbouncer-6dcfcbb767 from 0 to 1 deployment-controller logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:21:56 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-6dcfcbb767-84526.spec.initContainers{pgbouncer-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd" kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:21:56 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0 Scheduled Successfully assigned kuttl-test-crack-mammoth/major-upgrade-logical-replica-repo-host-0 to gke-jen-pg-1812-09e4007f-default-pool-bb01df06-7d01 default-scheduler logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:21:56 +0000 UTC Normal PersistentVolumeClaim major-upgrade-logical-replica-repo1 ProvisioningSucceeded Successfully provisioned volume pvc-12fb5cc4-5e13-46dd-b4e0-264d936fe0c1 pd.csi.storage.gke.io_gke-c6aaa511db014d729735-cd02-f74b-vm_d17f65ab-3170-4d7d-8dcd-19344a0ccb2e logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:22:00 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-6dcfcbb767-84526.spec.initContainers{pgbouncer-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd" in 4.291s (4.291s including waiting). Image size: 87060507 bytes. kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:22:00 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-6dcfcbb767-84526.spec.initContainers{pgbouncer-init} Created Created container: pgbouncer-init kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:22:00 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-6dcfcbb767-84526.spec.initContainers{pgbouncer-init} Started Started container pgbouncer-init kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:22:00 +0000 UTC Normal Pod pg-client-d7c7bb45b-xgndw.spec.containers{pg-client} Pulled Successfully pulled image "perconalab/percona-distribution-postgresql:16" in 18.804s (18.804s including waiting). Image size: 445181537 bytes. kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:22:00 +0000 UTC Normal Pod pg-client-d7c7bb45b-xgndw.spec.containers{pg-client} Created Created container: pg-client kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:22:00 +0000 UTC Normal Pod pg-client-d7c7bb45b-xgndw.spec.containers{pg-client} Started Started container pg-client kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:22:01 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-5fpf-0 SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-6390c87f-6bea-4394-964d-b84f079ca0bd" attachdetach-controller logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:22:01 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-8csm-0 SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-7fb7bdde-2d9b-4f38-a24a-a551dfcfedc8" attachdetach-controller logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:22:01 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0 SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-12fb5cc4-5e13-46dd-b4e0-264d936fe0c1" attachdetach-controller logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:22:02 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-8csm-0.spec.initContainers{database-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd" kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:22:03 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-8csm-0.spec.initContainers{database-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd" in 146ms (146ms including waiting). Image size: 87060507 bytes. kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:22:03 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-8csm-0.spec.initContainers{database-init} Created Created container: database-init kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:22:03 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-8csm-0.spec.initContainers{database-init} Started Started container database-init kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:22:03 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-8csm-0.spec.initContainers{postgres-startup} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:22:03 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-8csm-0.spec.initContainers{postgres-startup} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" in 136ms (136ms including waiting). Image size: 447492625 bytes. kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:22:03 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-8csm-0.spec.initContainers{postgres-startup} Created Created container: postgres-startup kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:22:03 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-6dcfcbb767-84526.spec.containers{pgbouncer} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbouncer17" kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:22:03 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.initContainers{pgbackrest-log-dir} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest17" kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:22:04 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-8csm-0.spec.initContainers{postgres-startup} Started Started container postgres-startup kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:22:04 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-8csm-0.spec.initContainers{nss-wrapper-init-logcollector} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:22:04 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-8csm-0.spec.initContainers{nss-wrapper-init-logcollector} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" in 118ms (118ms including waiting). Image size: 447492625 bytes. kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:22:04 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-8csm-0.spec.initContainers{nss-wrapper-init-logcollector} Created Created container: nss-wrapper-init-logcollector kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:22:05 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-5fpf-0.spec.initContainers{database-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd" kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:22:05 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-5fpf-0.spec.initContainers{database-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd" in 167ms (167ms including waiting). Image size: 87060507 bytes. kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:22:05 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-5fpf-0.spec.initContainers{database-init} Created Created container: database-init kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:22:05 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-5fpf-0.spec.initContainers{database-init} Started Started container database-init kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:22:05 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-8csm-0.spec.initContainers{nss-wrapper-init-logcollector} Started Started container nss-wrapper-init-logcollector kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:22:05 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-8csm-0.spec.initContainers{nss-wrapper-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:22:05 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-8csm-0.spec.initContainers{nss-wrapper-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" in 143ms (143ms including waiting). Image size: 447492625 bytes. kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:22:05 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-8csm-0.spec.initContainers{nss-wrapper-init} Created Created container: nss-wrapper-init kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:22:06 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-8csm-0.spec.initContainers{nss-wrapper-init} Started Started container nss-wrapper-init kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:22:06 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-8csm-0.spec.containers{database} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:22:06 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-8csm-0.spec.containers{database} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" in 118ms (118ms including waiting). Image size: 447492625 bytes. kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:22:06 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-8csm-0.spec.containers{database} Created Created container: database kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:22:07 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-8csm-0.spec.containers{database} Started Started container database kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:22:07 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-8csm-0.spec.containers{replication-cert-copy} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:22:07 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-8csm-0.spec.containers{replication-cert-copy} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" in 174ms (174ms including waiting). Image size: 447492625 bytes. kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:22:07 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-8csm-0.spec.containers{replication-cert-copy} Created Created container: replication-cert-copy kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:22:07 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-8csm-0.spec.containers{replication-cert-copy} Started Started container replication-cert-copy kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:22:07 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-8csm-0.spec.containers{logs} Pulled Container image "docker.io/perconalab/fluentbit:main-logcollector" already present on machine kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:22:08 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-6dcfcbb767-84526.spec.containers{pgbouncer} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbouncer17" in 5.012s (5.012s including waiting). Image size: 84603846 bytes. kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:22:08 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-6dcfcbb767-84526.spec.containers{pgbouncer} Created Created container: pgbouncer kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:22:08 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-6dcfcbb767-84526.spec.containers{pgbouncer} Started Started container pgbouncer kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:22:08 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-6dcfcbb767-84526.spec.containers{pgbouncer-config} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbouncer17" kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:22:08 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-6dcfcbb767-84526.spec.containers{pgbouncer-config} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbouncer17" in 186ms (186ms including waiting). Image size: 84603846 bytes. kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:22:08 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-6dcfcbb767-84526.spec.containers{pgbouncer-config} Created Created container: pgbouncer-config kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:22:08 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-6dcfcbb767-84526.spec.containers{pgbouncer-config} Started Started container pgbouncer-config kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:22:10 +0000 UTC Warning Pod major-upgrade-logical-replica-instance1-8csm-0.spec.containers{database} Unhealthy Readiness probe failed: kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:22:10 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.initContainers{pgbackrest-log-dir} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest17" in 7.089s (7.089s including waiting). Image size: 122679036 bytes. kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:22:10 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.initContainers{pgbackrest-log-dir} Created Created container: pgbackrest-log-dir kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:22:11 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.initContainers{pgbackrest-log-dir} Started Started container pgbackrest-log-dir kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:22:13 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-5fpf-0.spec.initContainers{postgres-startup} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:22:14 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.initContainers{nss-wrapper-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest17" kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:22:14 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.initContainers{nss-wrapper-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest17" in 119ms (119ms including waiting). Image size: 122679036 bytes. kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:22:14 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.initContainers{nss-wrapper-init} Created Created container: nss-wrapper-init kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:22:14 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.initContainers{nss-wrapper-init} Started Started container nss-wrapper-init kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:22:15 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.containers{pgbackrest} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest17" kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:22:15 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.containers{pgbackrest} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest17" in 111ms (111ms including waiting). Image size: 122679036 bytes. kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:22:15 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.containers{pgbackrest} Created Created container: pgbackrest kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:22:15 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.containers{pgbackrest} Started Started container pgbackrest kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:22:15 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.containers{pgbackrest-config} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest17" kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:22:15 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.containers{pgbackrest-config} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest17" in 115ms (115ms including waiting). Image size: 122679036 bytes. kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:22:15 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.containers{pgbackrest-config} Created Created container: pgbackrest-config kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:22:16 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.containers{pgbackrest-config} Started Started container pgbackrest-config kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:22:18 +0000 UTC Normal PostgresCluster.upstream.pgv2.percona.com major-upgrade-logical-replica StanzasCreated pgBackRest stanza creation completed successfully postgrescluster-controller logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:22:19 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-zhhh-ch9hw Scheduled Successfully assigned kuttl-test-crack-mammoth/major-upgrade-logical-replica-backup-zhhh-ch9hw to gke-jen-pg-1812-09e4007f-default-pool-bb01df06-w6r9 default-scheduler logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:22:19 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-zhhh-ch9hw.spec.initContainers{pgbackrest-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd" kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:22:19 +0000 UTC Normal Job.batch major-upgrade-logical-replica-backup-zhhh SuccessfulCreate Created pod: major-upgrade-logical-replica-backup-zhhh-ch9hw job-controller logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:22:20 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-zhhh-ch9hw.spec.initContainers{pgbackrest-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd" in 177ms (177ms including waiting). Image size: 87060507 bytes. kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:22:20 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-zhhh-ch9hw.spec.initContainers{pgbackrest-init} Created Created container: pgbackrest-init kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:22:20 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-zhhh-ch9hw.spec.initContainers{pgbackrest-init} Started Started container pgbackrest-init kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:22:20 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-zhhh-ch9hw.spec.containers{pgbackrest} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest17" kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:22:20 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-zhhh-ch9hw.spec.containers{pgbackrest} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest17" in 122ms (122ms including waiting). Image size: 122679036 bytes. kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:22:21 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-zhhh-ch9hw.spec.containers{pgbackrest} Created Created container: pgbackrest kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:22:21 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-zhhh-ch9hw.spec.containers{pgbackrest} Started Started container pgbackrest kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:22:24 +0000 UTC Warning PostgresCluster.upstream.pgv2.percona.com major-upgrade-logical-replica UnableToCreateStanzas repo1-path = /pgbackrest/repo1 2026-09-01 21:22:24.134 P00 ERROR: [050]: raised from remote-0 tls protocol on 'major-upgrade-logical-replica-repo-host-0.major-upgrade-logical-replica-pods.kuttl-test-crack-mammoth.svc.cluster.local': unable to acquire lock on file '/tmp/pgbackrest/db-backup-1.lock': Resource temporarily unavailable HINT: is another pgBackRest process running? 2026-09-01 21:22:24.377 P00 ERROR: [050]: raised from remote-0 tls protocol on 'major-upgrade-logical-replica-repo-host-0.major-upgrade-logical-replica-pods.kuttl-test-crack-mammoth.svc.cluster.local': unable to acquire lock on file '/tmp/pgbackrest/db-backup-1.lock': Resource temporarily unavailable HINT: is another pgBackRest process running? : command terminated with exit code 50 postgrescluster-controller logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:22:31 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-5fpf-0.spec.initContainers{postgres-startup} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" in 17.773s (17.773s including waiting). Image size: 447492625 bytes. kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:22:31 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-5fpf-0.spec.initContainers{postgres-startup} Created Created container: postgres-startup kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:22:31 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-5fpf-0.spec.initContainers{postgres-startup} Started Started container postgres-startup kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:22:40 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-5fpf-0.spec.initContainers{nss-wrapper-init-logcollector} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:22:40 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-5fpf-0.spec.initContainers{nss-wrapper-init-logcollector} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" in 132ms (132ms including waiting). Image size: 447492625 bytes. kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:22:40 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-5fpf-0.spec.initContainers{nss-wrapper-init-logcollector} Created Created container: nss-wrapper-init-logcollector kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:22:40 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-5fpf-0.spec.initContainers{nss-wrapper-init-logcollector} Started Started container nss-wrapper-init-logcollector kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:22:41 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-5fpf-0.spec.initContainers{nss-wrapper-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:22:41 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-5fpf-0.spec.initContainers{nss-wrapper-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" in 125ms (125ms including waiting). Image size: 447492625 bytes. kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:22:41 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-5fpf-0.spec.initContainers{nss-wrapper-init} Created Created container: nss-wrapper-init kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:22:41 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-5fpf-0.spec.initContainers{nss-wrapper-init} Started Started container nss-wrapper-init kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:22:42 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-5fpf-0.spec.containers{database} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:22:42 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-5fpf-0.spec.containers{database} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" in 135ms (135ms including waiting). Image size: 447492625 bytes. kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:22:42 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-5fpf-0.spec.containers{database} Created Created container: database kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:22:42 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-5fpf-0.spec.containers{database} Started Started container database kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:22:42 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-5fpf-0.spec.containers{replication-cert-copy} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:22:42 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-5fpf-0.spec.containers{replication-cert-copy} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" in 143ms (143ms including waiting). Image size: 447492625 bytes. kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:22:42 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-5fpf-0.spec.containers{replication-cert-copy} Created Created container: replication-cert-copy kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:22:42 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-5fpf-0.spec.containers{replication-cert-copy} Started Started container replication-cert-copy kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:22:42 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-5fpf-0.spec.containers{logs} Pulling Pulling image "docker.io/perconalab/fluentbit:main-logcollector" kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:22:56 +0000 UTC Warning Pod major-upgrade-logical-replica-instance1-5fpf-0.spec.containers{database} Unhealthy Readiness probe failed: kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:23:09 +0000 UTC Normal Job.batch major-upgrade-logical-replica-backup-zhhh Completed Job completed job-controller logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:23:24 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-r99w-6drb8 Scheduled Successfully assigned kuttl-test-crack-mammoth/major-upgrade-logical-replica-backup-r99w-6drb8 to gke-jen-pg-1812-09e4007f-default-pool-bb01df06-zfs6 default-scheduler logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:23:24 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-r99w-6drb8.spec.initContainers{pgbackrest-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd" kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:23:24 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-r99w-6drb8.spec.initContainers{pgbackrest-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd" in 136ms (136ms including waiting). Image size: 87060507 bytes. kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:23:24 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-r99w-6drb8.spec.initContainers{pgbackrest-init} Created Created container: pgbackrest-init kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:23:24 +0000 UTC Normal Job.batch major-upgrade-logical-replica-backup-r99w SuccessfulCreate Created pod: major-upgrade-logical-replica-backup-r99w-6drb8 job-controller logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:23:25 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-r99w-6drb8.spec.initContainers{pgbackrest-init} Started Started container pgbackrest-init kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:23:26 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-r99w-6drb8.spec.containers{pgbackrest} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest17" kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:23:26 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-r99w-6drb8.spec.containers{pgbackrest} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest17" in 150ms (150ms including waiting). Image size: 122679036 bytes. kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:23:26 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-r99w-6drb8.spec.containers{pgbackrest} Created Created container: pgbackrest kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:23:26 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-r99w-6drb8.spec.containers{pgbackrest} Started Started container pgbackrest kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:25:17 +0000 UTC Normal Job.batch major-upgrade-logical-replica-backup-r99w Completed Job completed job-controller logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:25:39 +0000 UTC Normal PersistentVolumeClaim major-upgrade-logical-replica-lr-analytics-pgdata WaitForFirstConsumer waiting for first consumer to be created before binding persistentvolume-controller logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:25:40 +0000 UTC Normal Job.batch major-upgrade-logical-replica-lr-analytics-bootstrap SuccessfulCreate Created pod: major-upgrade-logical-replica-lr-analytics-bootstrap-j9mtk job-controller logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:25:40 +0000 UTC Normal PersistentVolumeClaim major-upgrade-logical-replica-lr-analytics-pgdata ExternalProvisioning Waiting for a volume to be created either by the external provisioner 'pd.csi.storage.gke.io' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. persistentvolume-controller logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:25:40 +0000 UTC Normal PersistentVolumeClaim major-upgrade-logical-replica-lr-analytics-pgdata Provisioning External provisioner is provisioning volume for claim "kuttl-test-crack-mammoth/major-upgrade-logical-replica-lr-analytics-pgdata" pd.csi.storage.gke.io_gke-c6aaa511db014d729735-cd02-f74b-vm_d17f65ab-3170-4d7d-8dcd-19344a0ccb2e logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:25:41 +0000 UTC Normal PersistentVolumeClaim major-upgrade-logical-replica-lr-analytics-pgdata ProvisioningSucceeded Successfully provisioned volume pvc-7a051bd1-dc77-497e-863d-a6b6d5f8c6ff pd.csi.storage.gke.io_gke-c6aaa511db014d729735-cd02-f74b-vm_d17f65ab-3170-4d7d-8dcd-19344a0ccb2e logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:25:42 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-j9mtk Scheduled Successfully assigned kuttl-test-crack-mammoth/major-upgrade-logical-replica-lr-analytics-bootstrap-j9mtk to gke-jen-pg-1812-09e4007f-default-pool-bb01df06-zfs6 default-scheduler logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:25:48 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-j9mtk SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-7a051bd1-dc77-497e-863d-a6b6d5f8c6ff" attachdetach-controller logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:25:49 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-j9mtk.spec.initContainers{database-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd" kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:25:49 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-j9mtk.spec.initContainers{database-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd" in 169ms (169ms including waiting). Image size: 87060507 bytes. kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:25:50 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-j9mtk.spec.initContainers{database-init} Created Created container: database-init kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:25:50 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-j9mtk.spec.initContainers{database-init} Started Started container database-init kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:25:50 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-j9mtk.spec.initContainers{nss-wrapper-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:25:50 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-j9mtk.spec.initContainers{nss-wrapper-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" in 127ms (127ms including waiting). Image size: 447492625 bytes. kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:25:50 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-j9mtk.spec.initContainers{nss-wrapper-init} Created Created container: nss-wrapper-init kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:25:51 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-j9mtk.spec.initContainers{nss-wrapper-init} Started Started container nss-wrapper-init kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:25:51 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-j9mtk.spec.containers{logical-replica-bootstrap} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:25:51 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-j9mtk.spec.containers{logical-replica-bootstrap} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" in 116ms (116ms including waiting). Image size: 447492625 bytes. kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:25:51 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-j9mtk.spec.containers{logical-replica-bootstrap} Created Created container: logical-replica-bootstrap kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:25:52 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-j9mtk.spec.containers{logical-replica-bootstrap} Started Started container logical-replica-bootstrap kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:26:07 +0000 UTC Normal Job.batch major-upgrade-logical-replica-lr-analytics-bootstrap Completed Job completed job-controller logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:26:08 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0 Scheduled Successfully assigned kuttl-test-crack-mammoth/major-upgrade-logical-replica-lr-analytics-0 to gke-jen-pg-1812-09e4007f-default-pool-bb01df06-zfs6 default-scheduler logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:26:08 +0000 UTC Normal StatefulSet.apps major-upgrade-logical-replica-lr-analytics SuccessfulCreate create Pod major-upgrade-logical-replica-lr-analytics-0 in StatefulSet major-upgrade-logical-replica-lr-analytics successful statefulset-controller logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:26:10 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0.spec.initContainers{nss-wrapper-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:26:10 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0.spec.initContainers{nss-wrapper-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" in 115ms (115ms including waiting). Image size: 447492625 bytes. kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:26:10 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0.spec.initContainers{nss-wrapper-init} Created Created container: nss-wrapper-init kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:26:10 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0.spec.initContainers{nss-wrapper-init} Started Started container nss-wrapper-init kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:26:10 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0.spec.containers{database} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:26:11 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0.spec.containers{database} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" in 151ms (151ms including waiting). Image size: 447492625 bytes. kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:26:11 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0.spec.containers{database} Created Created container: database kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:26:11 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0.spec.containers{database} Started Started container database kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:26:35 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0.spec.containers{database} Killing Stopping container database kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:26:35 +0000 UTC Normal StatefulSet.apps major-upgrade-logical-replica-lr-analytics SuccessfulDelete delete Pod major-upgrade-logical-replica-lr-analytics-0 in StatefulSet major-upgrade-logical-replica-lr-analytics successful statefulset-controller logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:26:37 +0000 UTC Normal StatefulSet.apps major-upgrade-logical-replica-instance1-5fpf SuccessfulDelete delete Pod major-upgrade-logical-replica-instance1-5fpf-0 in StatefulSet major-upgrade-logical-replica-instance1-5fpf successful statefulset-controller logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:26:39 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-6dcfcbb767-84526.spec.containers{pgbouncer} Killing Stopping container pgbouncer kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:26:39 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-6dcfcbb767-84526.spec.containers{pgbouncer-config} Killing Stopping container pgbouncer-config kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:26:39 +0000 UTC Normal ReplicaSet.apps major-upgrade-logical-replica-pgbouncer-6dcfcbb767 SuccessfulDelete Deleted pod: major-upgrade-logical-replica-pgbouncer-6dcfcbb767-84526 replicaset-controller logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:26:39 +0000 UTC Normal Deployment.apps major-upgrade-logical-replica-pgbouncer ScalingReplicaSet Scaled down replica set major-upgrade-logical-replica-pgbouncer-6dcfcbb767 from 1 to 0 deployment-controller logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:26:43 +0000 UTC Normal StatefulSet.apps major-upgrade-logical-replica-instance1-8csm SuccessfulDelete delete Pod major-upgrade-logical-replica-instance1-8csm-0 in StatefulSet major-upgrade-logical-replica-instance1-8csm successful statefulset-controller logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:26:45 +0000 UTC Normal PodDisruptionBudget.policy major-upgrade-logical-replica-set-instance1 NoPods No matching pods found controllermanager logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:26:47 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.containers{pgbackrest} Killing Stopping container pgbackrest kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:26:47 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.containers{pgbackrest-config} Killing Stopping container pgbackrest-config kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:26:47 +0000 UTC Normal StatefulSet.apps major-upgrade-logical-replica-repo-host SuccessfulDelete delete Pod major-upgrade-logical-replica-repo-host-0 in StatefulSet major-upgrade-logical-replica-repo-host successful statefulset-controller logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:26:50 +0000 UTC Normal Pod 17-to-18-pgdata-qmrgm Scheduled Successfully assigned kuttl-test-crack-mammoth/17-to-18-pgdata-qmrgm to gke-jen-pg-1812-09e4007f-default-pool-bb01df06-7d01 default-scheduler logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:26:50 +0000 UTC Normal Job.batch 17-to-18-pgdata SuccessfulCreate Created pod: 17-to-18-pgdata-qmrgm job-controller logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:26:57 +0000 UTC Normal Pod 17-to-18-pgdata-qmrgm SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-7fb7bdde-2d9b-4f38-a24a-a551dfcfedc8" attachdetach-controller logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:26:59 +0000 UTC Normal Pod 17-to-18-pgdata-qmrgm.spec.initContainers{database-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd" kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:27:00 +0000 UTC Normal Pod 17-to-18-pgdata-qmrgm.spec.initContainers{database-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd" in 137ms (137ms including waiting). Image size: 87060507 bytes. kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:27:00 +0000 UTC Normal Pod 17-to-18-pgdata-qmrgm.spec.initContainers{database-init} Created Created container: database-init kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:27:00 +0000 UTC Normal Pod 17-to-18-pgdata-qmrgm.spec.initContainers{database-init} Started Started container database-init kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:27:01 +0000 UTC Normal Pod 17-to-18-pgdata-qmrgm.spec.containers{database} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-upgrade" kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:27:36 +0000 UTC Normal Pod 17-to-18-pgdata-qmrgm.spec.containers{database} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-upgrade" in 35.414s (35.414s including waiting). Image size: 952798836 bytes. kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:27:36 +0000 UTC Normal Pod 17-to-18-pgdata-qmrgm.spec.containers{database} Created Created container: database kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:27:37 +0000 UTC Normal Pod 17-to-18-pgdata-qmrgm.spec.containers{database} Started Started container database kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:27:53 +0000 UTC Normal Job.batch 17-to-18-pgdata Completed Job completed job-controller logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:27:54 +0000 UTC Normal Pod 17-to-18-major-upgrade-logical-replica-instance1-5fpf-zr9bb Scheduled Successfully assigned kuttl-test-crack-mammoth/17-to-18-major-upgrade-logical-replica-instance1-5fpf-zr9bb to gke-jen-pg-1812-09e4007f-default-pool-bb01df06-7d01 default-scheduler logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:27:54 +0000 UTC Normal Job.batch 17-to-18-major-upgrade-logical-replica-instance1-5fpf SuccessfulCreate Created pod: 17-to-18-major-upgrade-logical-replica-instance1-5fpf-zr9bb job-controller logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:28:00 +0000 UTC Normal Pod 17-to-18-major-upgrade-logical-replica-instance1-5fpf-zr9bb SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-6390c87f-6bea-4394-964d-b84f079ca0bd" attachdetach-controller logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:28:01 +0000 UTC Normal Pod 17-to-18-major-upgrade-logical-replica-instance1-5fpf-zr9bb.spec.containers{database} Pulled Container image "docker.io/perconalab/percona-postgresql-operator:main-upgrade" already present on machine kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:28:01 +0000 UTC Normal Pod 17-to-18-major-upgrade-logical-replica-instance1-5fpf-zr9bb.spec.containers{database} Created Created container: database kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:28:01 +0000 UTC Normal Pod 17-to-18-major-upgrade-logical-replica-instance1-5fpf-zr9bb.spec.containers{database} Started Started container database kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:28:04 +0000 UTC Normal Job.batch 17-to-18-major-upgrade-logical-replica-instance1-5fpf Completed Job completed job-controller logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:28:08 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-8csm-0 Scheduled Successfully assigned kuttl-test-crack-mammoth/major-upgrade-logical-replica-instance1-8csm-0 to gke-jen-pg-1812-09e4007f-default-pool-bb01df06-w6r9 default-scheduler logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:28:09 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0 Scheduled Successfully assigned kuttl-test-crack-mammoth/major-upgrade-logical-replica-repo-host-0 to gke-jen-pg-1812-09e4007f-default-pool-bb01df06-7d01 default-scheduler logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:28:10 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-cf7fb59f-b9gm9 Scheduled Successfully assigned kuttl-test-crack-mammoth/major-upgrade-logical-replica-pgbouncer-cf7fb59f-b9gm9 to gke-jen-pg-1812-09e4007f-default-pool-bb01df06-zfs6 default-scheduler logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:28:10 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-cf7fb59f-b9gm9.spec.initContainers{pgbouncer-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd" kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:28:10 +0000 UTC Normal ReplicaSet.apps major-upgrade-logical-replica-pgbouncer-cf7fb59f SuccessfulCreate Created pod: major-upgrade-logical-replica-pgbouncer-cf7fb59f-b9gm9 replicaset-controller logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:28:10 +0000 UTC Normal Deployment.apps major-upgrade-logical-replica-pgbouncer ScalingReplicaSet Scaled up replica set major-upgrade-logical-replica-pgbouncer-cf7fb59f from 0 to 1 deployment-controller logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:28:11 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-cf7fb59f-b9gm9.spec.initContainers{pgbouncer-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd" in 142ms (142ms including waiting). Image size: 87060507 bytes. kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:28:11 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-cf7fb59f-b9gm9.spec.initContainers{pgbouncer-init} Created Created container: pgbouncer-init kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:28:11 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-cf7fb59f-b9gm9.spec.initContainers{pgbouncer-init} Started Started container pgbouncer-init kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:28:12 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-cf7fb59f-b9gm9.spec.containers{pgbouncer} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18" kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:28:14 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-8csm-0 SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-7fb7bdde-2d9b-4f38-a24a-a551dfcfedc8" attachdetach-controller logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:28:15 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-8csm-0.spec.initContainers{database-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd" kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:28:15 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-8csm-0.spec.initContainers{database-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd" in 152ms (152ms including waiting). Image size: 87060507 bytes. kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:28:15 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-8csm-0.spec.initContainers{database-init} Created Created container: database-init kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:28:15 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-8csm-0.spec.initContainers{database-init} Started Started container database-init kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:28:15 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-cf7fb59f-b9gm9.spec.containers{pgbouncer} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18" in 2.677s (2.677s including waiting). Image size: 84617369 bytes. kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:28:15 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-cf7fb59f-b9gm9.spec.containers{pgbouncer} Created Created container: pgbouncer kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:28:15 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-cf7fb59f-b9gm9.spec.containers{pgbouncer} Started Started container pgbouncer kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:28:15 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-cf7fb59f-b9gm9.spec.containers{pgbouncer-config} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18" kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:28:15 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-cf7fb59f-b9gm9.spec.containers{pgbouncer-config} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18" in 129ms (129ms including waiting). Image size: 84617369 bytes. kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:28:15 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-cf7fb59f-b9gm9.spec.containers{pgbouncer-config} Created Created container: pgbouncer-config kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:28:15 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-cf7fb59f-b9gm9.spec.containers{pgbouncer-config} Started Started container pgbouncer-config kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:28:16 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-8csm-0.spec.initContainers{postgres-startup} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:28:16 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-8csm-0.spec.initContainers{postgres-startup} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" in 126ms (126ms including waiting). Image size: 449168764 bytes. kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:28:16 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-8csm-0.spec.initContainers{postgres-startup} Created Created container: postgres-startup kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:28:16 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-8csm-0.spec.initContainers{postgres-startup} Started Started container postgres-startup kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:28:17 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-8csm-0.spec.initContainers{nss-wrapper-init-logcollector} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:28:17 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-8csm-0.spec.initContainers{nss-wrapper-init-logcollector} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" in 141ms (141ms including waiting). Image size: 449168764 bytes. kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:28:17 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-8csm-0.spec.initContainers{nss-wrapper-init-logcollector} Created Created container: nss-wrapper-init-logcollector kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:28:17 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-8csm-0.spec.initContainers{nss-wrapper-init-logcollector} Started Started container nss-wrapper-init-logcollector kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:28:18 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-8csm-0.spec.initContainers{nss-wrapper-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:28:18 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-8csm-0.spec.initContainers{nss-wrapper-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" in 106ms (106ms including waiting). Image size: 449168764 bytes. kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:28:18 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-8csm-0.spec.initContainers{nss-wrapper-init} Created Created container: nss-wrapper-init kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:28:18 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-8csm-0.spec.initContainers{nss-wrapper-init} Started Started container nss-wrapper-init kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:28:19 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-8csm-0.spec.containers{database} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:28:19 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-8csm-0.spec.containers{database} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" in 157ms (157ms including waiting). Image size: 449168764 bytes. kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:28:19 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-8csm-0.spec.containers{database} Created Created container: database kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:28:19 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-8csm-0.spec.containers{database} Started Started container database kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:28:19 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-8csm-0.spec.containers{replication-cert-copy} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:28:19 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-8csm-0.spec.containers{replication-cert-copy} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" in 128ms (128ms including waiting). Image size: 449168764 bytes. kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:28:19 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-8csm-0.spec.containers{replication-cert-copy} Created Created container: replication-cert-copy kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:28:19 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-8csm-0.spec.containers{replication-cert-copy} Started Started container replication-cert-copy kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:28:19 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-8csm-0.spec.containers{logs} Pulled Container image "docker.io/perconalab/fluentbit:main-logcollector" already present on machine kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:28:26 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-5fpf-0 Scheduled Successfully assigned kuttl-test-crack-mammoth/major-upgrade-logical-replica-instance1-5fpf-0 to gke-jen-pg-1812-09e4007f-default-pool-bb01df06-zfs6 default-scheduler logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:28:30 +0000 UTC Warning PostgresCluster.upstream.pgv2.percona.com major-upgrade-logical-replica UnableToCreateStanzas repo1-path = /pgbackrest/repo1 2026-09-01 21:28:30.397 P00 ERROR: [049]: unable to get address for 'major-upgrade-logical-replica-repo-host-0.major-upgrade-logical-replica-pods.kuttl-test-crack-mammoth.svc.cluster.local': [-2] Name or service not known 2026-09-01 21:28:30.606 P00 ERROR: [049]: unable to get address for 'major-upgrade-logical-replica-repo-host-0.major-upgrade-logical-replica-pods.kuttl-test-crack-mammoth.svc.cluster.local': [-2] Name or service not known : command terminated with exit code 49 postgrescluster-controller logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:28:33 +0000 UTC Warning PostgresCluster.upstream.pgv2.percona.com major-upgrade-logical-replica UnableToCreateStanzas repo1-path = /pgbackrest/repo1 2026-09-01 21:28:33.472 P00 ERROR: [049]: unable to get address for 'major-upgrade-logical-replica-repo-host-0.major-upgrade-logical-replica-pods.kuttl-test-crack-mammoth.svc.cluster.local': [-2] Name or service not known 2026-09-01 21:28:33.682 P00 ERROR: [049]: unable to get address for 'major-upgrade-logical-replica-repo-host-0.major-upgrade-logical-replica-pods.kuttl-test-crack-mammoth.svc.cluster.local': [-2] Name or service not known : command terminated with exit code 49 postgrescluster-controller logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:28:36 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0 SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-12fb5cc4-5e13-46dd-b4e0-264d936fe0c1" attachdetach-controller logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:28:36 +0000 UTC Warning PostgresCluster.upstream.pgv2.percona.com major-upgrade-logical-replica UnableToCreateStanzas repo1-path = /pgbackrest/repo1 2026-09-01 21:28:36.469 P00 ERROR: [049]: unable to get address for 'major-upgrade-logical-replica-repo-host-0.major-upgrade-logical-replica-pods.kuttl-test-crack-mammoth.svc.cluster.local': [-2] Name or service not known 2026-09-01 21:28:36.678 P00 ERROR: [049]: unable to get address for 'major-upgrade-logical-replica-repo-host-0.major-upgrade-logical-replica-pods.kuttl-test-crack-mammoth.svc.cluster.local': [-2] Name or service not known : command terminated with exit code 49 postgrescluster-controller logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:28:38 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.initContainers{pgbackrest-log-dir} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18" kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:28:42 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.initContainers{pgbackrest-log-dir} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18" in 4.854s (4.854s including waiting). Image size: 122692543 bytes. kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:28:42 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.initContainers{pgbackrest-log-dir} Created Created container: pgbackrest-log-dir kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:28:42 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.initContainers{pgbackrest-log-dir} Started Started container pgbackrest-log-dir kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:28:44 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.initContainers{nss-wrapper-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18" kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:28:44 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.initContainers{nss-wrapper-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18" in 121ms (121ms including waiting). Image size: 122692543 bytes. kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:28:44 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.initContainers{nss-wrapper-init} Created Created container: nss-wrapper-init kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:28:45 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-5fpf-0 SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-6390c87f-6bea-4394-964d-b84f079ca0bd" attachdetach-controller logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:28:45 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.initContainers{nss-wrapper-init} Started Started container nss-wrapper-init kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:28:45 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.containers{pgbackrest} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18" kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:28:45 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.containers{pgbackrest} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18" in 141ms (141ms including waiting). Image size: 122692543 bytes. kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:28:46 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-5fpf-0.spec.initContainers{database-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd" kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:28:46 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-5fpf-0.spec.initContainers{database-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd" in 130ms (130ms including waiting). Image size: 87060507 bytes. kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:28:46 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-5fpf-0.spec.initContainers{database-init} Created Created container: database-init kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:28:46 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-5fpf-0.spec.initContainers{database-init} Started Started container database-init kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:28:46 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.containers{pgbackrest} Created Created container: pgbackrest kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:28:46 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.containers{pgbackrest} Started Started container pgbackrest kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:28:46 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.containers{pgbackrest-config} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18" kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:28:46 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.containers{pgbackrest-config} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18" in 143ms (143ms including waiting). Image size: 122692543 bytes. kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:28:46 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.containers{pgbackrest-config} Created Created container: pgbackrest-config kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:28:46 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.containers{pgbackrest-config} Started Started container pgbackrest-config kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:28:48 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-5fpf-0.spec.initContainers{postgres-startup} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:28:50 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-v892-jfcjs Scheduled Successfully assigned kuttl-test-crack-mammoth/major-upgrade-logical-replica-backup-v892-jfcjs to gke-jen-pg-1812-09e4007f-default-pool-bb01df06-7d01 default-scheduler logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:28:50 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-v892-jfcjs.spec.initContainers{pgbackrest-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd" kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:28:50 +0000 UTC Normal Job.batch major-upgrade-logical-replica-backup-v892 SuccessfulCreate Created pod: major-upgrade-logical-replica-backup-v892-jfcjs job-controller logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:28:51 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-v892-jfcjs.spec.initContainers{pgbackrest-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd" in 150ms (150ms including waiting). Image size: 87060507 bytes. kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:28:51 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-v892-jfcjs.spec.initContainers{pgbackrest-init} Created Created container: pgbackrest-init kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:28:51 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-v892-jfcjs.spec.initContainers{pgbackrest-init} Started Started container pgbackrest-init kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:28:51 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-v892-jfcjs.spec.containers{pgbackrest} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18" kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:28:51 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-v892-jfcjs.spec.containers{pgbackrest} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18" in 132ms (132ms including waiting). Image size: 122692543 bytes. kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:28:52 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-v892-jfcjs.spec.containers{pgbackrest} Created Created container: pgbackrest kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:28:52 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-v892-jfcjs.spec.containers{pgbackrest} Started Started container pgbackrest kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:28:54 +0000 UTC Warning PostgresCluster.upstream.pgv2.percona.com major-upgrade-logical-replica UnableToCreateStanzas repo1-path = /pgbackrest/repo1 2026-09-01 21:28:54.030 P00 ERROR: [050]: raised from remote-0 tls protocol on 'major-upgrade-logical-replica-repo-host-0.major-upgrade-logical-replica-pods.kuttl-test-crack-mammoth.svc.cluster.local': unable to acquire lock on file '/tmp/pgbackrest/db-backup-1.lock': Resource temporarily unavailable HINT: is another pgBackRest process running? 2026-09-01 21:28:54.252 P00 ERROR: [050]: raised from remote-0 tls protocol on 'major-upgrade-logical-replica-repo-host-0.major-upgrade-logical-replica-pods.kuttl-test-crack-mammoth.svc.cluster.local': unable to acquire lock on file '/tmp/pgbackrest/db-backup-1.lock': Resource temporarily unavailable HINT: is another pgBackRest process running? : command terminated with exit code 50 postgrescluster-controller logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:29:05 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-5fpf-0.spec.initContainers{postgres-startup} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" in 17.158s (17.158s including waiting). Image size: 449168764 bytes. kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:29:05 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-5fpf-0.spec.initContainers{postgres-startup} Created Created container: postgres-startup kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:29:05 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-5fpf-0.spec.initContainers{postgres-startup} Started Started container postgres-startup kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:29:13 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-5fpf-0.spec.initContainers{nss-wrapper-init-logcollector} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:29:13 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-5fpf-0.spec.initContainers{nss-wrapper-init-logcollector} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" in 153ms (153ms including waiting). Image size: 449168764 bytes. kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:29:13 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-5fpf-0.spec.initContainers{nss-wrapper-init-logcollector} Created Created container: nss-wrapper-init-logcollector kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:29:13 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-5fpf-0.spec.initContainers{nss-wrapper-init-logcollector} Started Started container nss-wrapper-init-logcollector kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:29:14 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-5fpf-0.spec.initContainers{nss-wrapper-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:29:14 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-5fpf-0.spec.initContainers{nss-wrapper-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" in 130ms (131ms including waiting). Image size: 449168764 bytes. kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:29:14 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-5fpf-0.spec.initContainers{nss-wrapper-init} Created Created container: nss-wrapper-init kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:29:14 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-5fpf-0.spec.initContainers{nss-wrapper-init} Started Started container nss-wrapper-init kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:29:15 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-5fpf-0.spec.containers{database} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:29:15 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-5fpf-0.spec.containers{database} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" in 96ms (96ms including waiting). Image size: 449168764 bytes. kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:29:15 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-5fpf-0.spec.containers{database} Created Created container: database kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:29:15 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-5fpf-0.spec.containers{database} Started Started container database kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:29:15 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-5fpf-0.spec.containers{replication-cert-copy} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:29:15 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-5fpf-0.spec.containers{replication-cert-copy} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" in 112ms (112ms including waiting). Image size: 449168764 bytes. kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:29:16 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-5fpf-0.spec.containers{replication-cert-copy} Created Created container: replication-cert-copy kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:29:16 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-5fpf-0.spec.containers{replication-cert-copy} Started Started container replication-cert-copy kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:29:16 +0000 UTC Normal Pod major-upgrade-logical-replica-instance1-5fpf-0.spec.containers{logs} Pulled Container image "docker.io/perconalab/fluentbit:main-logcollector" already present on machine kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:29:26 +0000 UTC Warning PerconaPGCluster.pgv2.percona.com major-upgrade-logical-replica LogicalReplicaInvalidated Logical replica "analytics" must be seeded again: the cluster was upgraded from PostgreSQL 17 to 18 after it was seeded perconapgcluster-controller logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:29:48 +0000 UTC Normal Job.batch major-upgrade-logical-replica-backup-v892 Completed Job completed job-controller logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:29:57 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-wkkq-kjqv4 Scheduled Successfully assigned kuttl-test-crack-mammoth/major-upgrade-logical-replica-backup-wkkq-kjqv4 to gke-jen-pg-1812-09e4007f-default-pool-bb01df06-7d01 default-scheduler logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:29:57 +0000 UTC Normal Job.batch major-upgrade-logical-replica-backup-wkkq SuccessfulCreate Created pod: major-upgrade-logical-replica-backup-wkkq-kjqv4 job-controller logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:29:58 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-wkkq-kjqv4.spec.initContainers{pgbackrest-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd" kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:29:58 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-wkkq-kjqv4.spec.initContainers{pgbackrest-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd" in 175ms (175ms including waiting). Image size: 87060507 bytes. kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:29:58 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-wkkq-kjqv4.spec.initContainers{pgbackrest-init} Created Created container: pgbackrest-init kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:29:58 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-wkkq-kjqv4.spec.initContainers{pgbackrest-init} Started Started container pgbackrest-init kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:30:00 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-wkkq-kjqv4.spec.containers{pgbackrest} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18" kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:30:00 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-wkkq-kjqv4.spec.containers{pgbackrest} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18" in 121ms (121ms including waiting). Image size: 122692543 bytes. kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:30:00 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-wkkq-kjqv4.spec.containers{pgbackrest} Created Created container: pgbackrest kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:30:00 +0000 UTC Normal Pod major-upgrade-logical-replica-backup-wkkq-kjqv4.spec.containers{pgbackrest} Started Started container pgbackrest kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:30:15 +0000 UTC Normal Job.batch major-upgrade-logical-replica-backup-wkkq Completed Job completed job-controller logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:30:38 +0000 UTC Normal PersistentVolumeClaim major-upgrade-logical-replica-lr-analytics-pgdata WaitForFirstConsumer waiting for first consumer to be created before binding persistentvolume-controller logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:30:39 +0000 UTC Normal Job.batch major-upgrade-logical-replica-lr-analytics-bootstrap SuccessfulCreate Created pod: major-upgrade-logical-replica-lr-analytics-bootstrap-4n2wx job-controller logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:30:39 +0000 UTC Normal PersistentVolumeClaim major-upgrade-logical-replica-lr-analytics-pgdata ExternalProvisioning Waiting for a volume to be created either by the external provisioner 'pd.csi.storage.gke.io' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. persistentvolume-controller logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:30:39 +0000 UTC Normal PersistentVolumeClaim major-upgrade-logical-replica-lr-analytics-pgdata Provisioning External provisioner is provisioning volume for claim "kuttl-test-crack-mammoth/major-upgrade-logical-replica-lr-analytics-pgdata" pd.csi.storage.gke.io_gke-c6aaa511db014d729735-cd02-f74b-vm_d17f65ab-3170-4d7d-8dcd-19344a0ccb2e logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:30:40 +0000 UTC Normal PersistentVolumeClaim major-upgrade-logical-replica-lr-analytics-pgdata ProvisioningSucceeded Successfully provisioned volume pvc-61ff4a5e-b1d7-434f-b4a9-9b5f77b96c81 pd.csi.storage.gke.io_gke-c6aaa511db014d729735-cd02-f74b-vm_d17f65ab-3170-4d7d-8dcd-19344a0ccb2e logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:30:41 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-4n2wx Scheduled Successfully assigned kuttl-test-crack-mammoth/major-upgrade-logical-replica-lr-analytics-bootstrap-4n2wx to gke-jen-pg-1812-09e4007f-default-pool-bb01df06-w6r9 default-scheduler logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:30:47 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-4n2wx SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-61ff4a5e-b1d7-434f-b4a9-9b5f77b96c81" attachdetach-controller logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:30:48 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-4n2wx.spec.initContainers{database-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd" kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:30:48 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-4n2wx.spec.initContainers{database-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:PR-1812-09e4007fd" in 178ms (178ms including waiting). Image size: 87060507 bytes. kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:30:48 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-4n2wx.spec.initContainers{database-init} Created Created container: database-init kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:30:48 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-4n2wx.spec.initContainers{database-init} Started Started container database-init kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:30:49 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-4n2wx.spec.initContainers{nss-wrapper-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:30:49 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-4n2wx.spec.initContainers{nss-wrapper-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" in 166ms (166ms including waiting). Image size: 449168764 bytes. kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:30:49 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-4n2wx.spec.initContainers{nss-wrapper-init} Created Created container: nss-wrapper-init kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:30:50 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-4n2wx.spec.initContainers{nss-wrapper-init} Started Started container nss-wrapper-init kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:30:50 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-4n2wx.spec.containers{logical-replica-bootstrap} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:30:50 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-4n2wx.spec.containers{logical-replica-bootstrap} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" in 140ms (140ms including waiting). Image size: 449168764 bytes. kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:30:50 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-4n2wx.spec.containers{logical-replica-bootstrap} Created Created container: logical-replica-bootstrap kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:30:51 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-bootstrap-4n2wx.spec.containers{logical-replica-bootstrap} Started Started container logical-replica-bootstrap kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:31:04 +0000 UTC Normal Job.batch major-upgrade-logical-replica-lr-analytics-bootstrap Completed Job completed job-controller logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:31:05 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0 Scheduled Successfully assigned kuttl-test-crack-mammoth/major-upgrade-logical-replica-lr-analytics-0 to gke-jen-pg-1812-09e4007f-default-pool-bb01df06-w6r9 default-scheduler logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:31:05 +0000 UTC Normal StatefulSet.apps major-upgrade-logical-replica-lr-analytics SuccessfulCreate create Pod major-upgrade-logical-replica-lr-analytics-0 in StatefulSet major-upgrade-logical-replica-lr-analytics successful statefulset-controller logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:31:14 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0.spec.initContainers{nss-wrapper-init} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:31:14 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0.spec.initContainers{nss-wrapper-init} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" in 205ms (205ms including waiting). Image size: 449168764 bytes. kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:31:14 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0.spec.initContainers{nss-wrapper-init} Created Created container: nss-wrapper-init kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:31:14 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0.spec.initContainers{nss-wrapper-init} Started Started container nss-wrapper-init kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:31:15 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0.spec.containers{database} Pulling Pulling image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:31:16 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0.spec.containers{database} Pulled Successfully pulled image "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" in 151ms (151ms including waiting). Image size: 449168764 bytes. kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:31:16 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0.spec.containers{database} Created Created container: database kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:31:16 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0.spec.containers{database} Started Started container database kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:31:35 +0000 UTC Normal Pod major-upgrade-logical-replica-lr-analytics-0.spec.containers{database} Killing Stopping container database kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:31:43 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-cf7fb59f-b9gm9.spec.containers{pgbouncer} Killing Stopping container pgbouncer kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:31:43 +0000 UTC Normal Pod major-upgrade-logical-replica-pgbouncer-cf7fb59f-b9gm9.spec.containers{pgbouncer-config} Killing Stopping container pgbouncer-config kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:31:43 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.containers{pgbackrest} Killing Stopping container pgbackrest kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | 2026-09-01 21:31:43 +0000 UTC Normal Pod major-upgrade-logical-replica-repo-host-0.spec.containers{pgbackrest-config} Killing Stopping container pgbackrest-config kubelet logger.go:42: 21:32:02 | major-upgrade-logical-replica | Deleting namespace "kuttl-test-crack-mammoth" === NAME kuttl harness.go:403: run tests finished harness.go:520: cleaning up harness.go:577: removing temp folder: "" --- PASS: kuttl (715.18s) --- PASS: kuttl/harness (0.00s) --- PASS: kuttl/harness/major-upgrade-logical-replica (714.42s) PASS