=== RUN kuttl harness.go:460: starting setup harness.go:258: running tests using configured kubeconfig. harness.go:281: Successful connection to cluster at: https://35.253.6.43 harness.go:366: running tests harness.go:77: going to run test suite with timeout of 600 seconds for each step harness.go:378: testsuite: e2e-tests/tests has 59 tests === RUN kuttl/harness === RUN kuttl/harness/major-upgrade-logical-replica === PAUSE kuttl/harness/major-upgrade-logical-replica === CONT kuttl/harness/major-upgrade-logical-replica logger.go:42: 23:56:15 | major-upgrade-logical-replica | Ignoring "conf": does not begin with a number followed by a dash. logger.go:42: 23:56:15 | major-upgrade-logical-replica | Creating namespace "kuttl-test-evident-monkey" logger.go:42: 23:56:16 | major-upgrade-logical-replica/0-deploy-operator | starting test step 0-deploy-operator logger.go:42: 23:56:16 | major-upgrade-logical-replica/0-deploy-operator | running command: [sh -c set -o errexit set -o xtrace source ../../functions init_temp_dir # do this only in the first TestStep deploy_operator deploy_client deploy_s3_secrets] logger.go:42: 23:56:16 | major-upgrade-logical-replica/0-deploy-operator | + source ../../functions logger.go:42: 23:56:16 | major-upgrade-logical-replica/0-deploy-operator | +++ realpath ../../.. logger.go:42: 23:56:16 | major-upgrade-logical-replica/0-deploy-operator | ++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1786 logger.go:42: 23:56:16 | major-upgrade-logical-replica/0-deploy-operator | ++ CERT_MANAGER_VER=1.20.2 logger.go:42: 23:56:16 | major-upgrade-logical-replica/0-deploy-operator | ++ CERT_MANAGER_TEST_LABEL=e2e.percona.com/managed-by=percona-postgresql-operator-tests logger.go:42: 23:56:16 | major-upgrade-logical-replica/0-deploy-operator | ++ CHAOS_MESH_VER=2.5.1 logger.go:42: 23:56:16 | major-upgrade-logical-replica/0-deploy-operator | ++++ pwd logger.go:42: 23:56:16 | major-upgrade-logical-replica/0-deploy-operator | +++ basename /mnt/jenkins/workspace/cloud-pg-operator_PR-1786/e2e-tests/tests/major-upgrade-logical-replica logger.go:42: 23:56:16 | major-upgrade-logical-replica/0-deploy-operator | ++ test_name=major-upgrade-logical-replica logger.go:42: 23:56:16 | major-upgrade-logical-replica/0-deploy-operator | ++ source /mnt/jenkins/workspace/cloud-pg-operator_PR-1786/e2e-tests/vars.sh logger.go:42: 23:56:16 | major-upgrade-logical-replica/0-deploy-operator | +++ export ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1786 logger.go:42: 23:56:16 | major-upgrade-logical-replica/0-deploy-operator | +++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1786 logger.go:42: 23:56:16 | major-upgrade-logical-replica/0-deploy-operator | +++ export DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1786/deploy logger.go:42: 23:56:16 | major-upgrade-logical-replica/0-deploy-operator | +++ DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1786/deploy logger.go:42: 23:56:16 | major-upgrade-logical-replica/0-deploy-operator | +++ export TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1786/e2e-tests logger.go:42: 23:56:16 | major-upgrade-logical-replica/0-deploy-operator | +++ TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1786/e2e-tests logger.go:42: 23:56:16 | major-upgrade-logical-replica/0-deploy-operator | +++ export TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1786/e2e-tests/conf logger.go:42: 23:56:16 | major-upgrade-logical-replica/0-deploy-operator | +++ TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1786/e2e-tests/conf logger.go:42: 23:56:16 | major-upgrade-logical-replica/0-deploy-operator | +++ export TEST_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1786/e2e-tests/tests/major-upgrade-logical-replica/conf logger.go:42: 23:56:16 | major-upgrade-logical-replica/0-deploy-operator | +++ TEST_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1786/e2e-tests/tests/major-upgrade-logical-replica/conf logger.go:42: 23:56:16 | major-upgrade-logical-replica/0-deploy-operator | +++ export TEMP_DIR=/tmp/kuttl/pg/major-upgrade-logical-replica logger.go:42: 23:56:16 | major-upgrade-logical-replica/0-deploy-operator | +++ TEMP_DIR=/tmp/kuttl/pg/major-upgrade-logical-replica logger.go:42: 23:56:16 | major-upgrade-logical-replica/0-deploy-operator | ++++ git rev-parse --abbrev-ref HEAD logger.go:42: 23:56:16 | major-upgrade-logical-replica/0-deploy-operator | +++ export GIT_BRANCH=PR-1786 logger.go:42: 23:56:16 | major-upgrade-logical-replica/0-deploy-operator | +++ GIT_BRANCH=PR-1786 logger.go:42: 23:56:16 | major-upgrade-logical-replica/0-deploy-operator | +++ export VERSION=PR-1786-391a86551 logger.go:42: 23:56:16 | major-upgrade-logical-replica/0-deploy-operator | +++ VERSION=PR-1786-391a86551 logger.go:42: 23:56:16 | major-upgrade-logical-replica/0-deploy-operator | +++ export SKIP_TEST_WARNINGS=true logger.go:42: 23:56:16 | major-upgrade-logical-replica/0-deploy-operator | +++ SKIP_TEST_WARNINGS=true logger.go:42: 23:56:16 | major-upgrade-logical-replica/0-deploy-operator | +++ export K8S_UPGRADE_PLATFORM=gke logger.go:42: 23:56:16 | major-upgrade-logical-replica/0-deploy-operator | +++ K8S_UPGRADE_PLATFORM=gke logger.go:42: 23:56:16 | major-upgrade-logical-replica/0-deploy-operator | +++ export K8S_UPGRADE_REGION=us-central1-a logger.go:42: 23:56:16 | major-upgrade-logical-replica/0-deploy-operator | +++ K8S_UPGRADE_REGION=us-central1-a logger.go:42: 23:56:16 | major-upgrade-logical-replica/0-deploy-operator | +++ export K8S_UPGRADE_INITIAL_VERSION= logger.go:42: 23:56:16 | major-upgrade-logical-replica/0-deploy-operator | +++ K8S_UPGRADE_INITIAL_VERSION= logger.go:42: 23:56:16 | major-upgrade-logical-replica/0-deploy-operator | +++ export K8S_UPGRADE_FINAL_VERSION= logger.go:42: 23:56:16 | major-upgrade-logical-replica/0-deploy-operator | +++ K8S_UPGRADE_FINAL_VERSION= logger.go:42: 23:56:16 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ -z '' ]] logger.go:42: 23:56:16 | major-upgrade-logical-replica/0-deploy-operator | ++++ detect_k8s_platform logger.go:42: 23:56:16 | major-upgrade-logical-replica/0-deploy-operator | ++++ local detected_platform logger.go:42: 23:56:16 | major-upgrade-logical-replica/0-deploy-operator | ++++ local xtrace_enabled=false logger.go:42: 23:56:16 | major-upgrade-logical-replica/0-deploy-operator | ++++ [[ -o xtrace ]] logger.go:42: 23:56:16 | major-upgrade-logical-replica/0-deploy-operator | ++++ xtrace_enabled=true logger.go:42: 23:56:16 | major-upgrade-logical-replica/0-deploy-operator | ++++ set +o xtrace logger.go:42: 23:56:18 | major-upgrade-logical-replica/0-deploy-operator | +++ PLATFORM=gke logger.go:42: 23:56:18 | major-upgrade-logical-replica/0-deploy-operator | +++ export PLATFORM logger.go:42: 23:56:18 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 23:56:18 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 23:56:18 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE=perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 23:56:18 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE=perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 23:56:18 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ ! -n '' ]] logger.go:42: 23:56:18 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ -n '' ]] logger.go:42: 23:56:18 | major-upgrade-logical-replica/0-deploy-operator | +++ export PG_VER=18 logger.go:42: 23:56:18 | major-upgrade-logical-replica/0-deploy-operator | +++ PG_VER=18 logger.go:42: 23:56:18 | major-upgrade-logical-replica/0-deploy-operator | +++ export ARCH= logger.go:42: 23:56:18 | major-upgrade-logical-replica/0-deploy-operator | +++ ARCH= logger.go:42: 23:56:18 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ -z '' ]] logger.go:42: 23:56:18 | major-upgrade-logical-replica/0-deploy-operator | ++++ kubectl get nodes -o 'jsonpath={range .items[*]}{.metadata.labels.kubernetes\.io/arch}{"\n"}{end}' logger.go:42: 23:56:18 | major-upgrade-logical-replica/0-deploy-operator | ++++ sort -u logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ node_arches=amd64 logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ amd64 == arm64 ]] logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ export PG_DISTRIBUTION= logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ PG_DISTRIBUTION= logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ '' == community ]] logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ export BUCKET=pg-operator-testing logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ BUCKET=pg-operator-testing logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ export PMM_SERVER_VERSION=9.9.9 logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ PMM_SERVER_VERSION=9.9.9 logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE_LOGCOLLECTOR=perconalab/fluentbit:main-logcollector logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE_LOGCOLLECTOR=perconalab/fluentbit:main-logcollector logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ export PGOV1_TAG=1.4.0 logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ PGOV1_TAG=1.4.0 logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ export PGOV1_VER=14 logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ PGOV1_VER=14 logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ export CPGO_VERSION=5.8.7 logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ CPGO_VERSION=5.8.7 logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ export MINIO_VER=5.4.0 logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ MINIO_VER=5.4.0 logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ export VAULT_VER=0.32.0 logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ VAULT_VER=0.32.0 logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE_AWS_CLI=docker.io/amazon/aws-cli:2.34.60 logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE_AWS_CLI=docker.io/amazon/aws-cli:2.34.60 logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ export REGISTRY_NAME=docker.io logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ REGISTRY_NAME=docker.io logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ export REGISTRY_NAME_FULL=docker.io/ logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ REGISTRY_NAME_FULL=docker.io/ logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | ++++ printenv logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | ++++ grep -E '^IMAGE' logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | ++++ awk -F= '{print $1}' logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | ++++ eval 'echo $IMAGE_BACKREST' logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++++ echo perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ var_value=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == docker.io/* ]] logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == percona/* ]] logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == perconalab/* ]] logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ echo IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ echo perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | ++++ eval 'echo $IMAGE' logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++++ echo perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ var_value=perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:PR-1786-391a86551 == docker.io/* ]] logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:PR-1786-391a86551 == percona/* ]] logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:PR-1786-391a86551 == perconalab/* ]] logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ new_value=docker.io/perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ echo IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | docker.io/perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | ++++ eval 'echo $IMAGE_UPGRADE' logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++++ echo perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ var_value=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == docker.io/* ]] logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == percona/* ]] logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == perconalab/* ]] logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ echo IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | docker.io/perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | ++++ eval 'echo $IMAGE_LOGCOLLECTOR' logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++++ echo perconalab/fluentbit:main-logcollector logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ var_value=perconalab/fluentbit:main-logcollector logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/fluentbit:main-logcollector == docker.io/* ]] logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/fluentbit:main-logcollector == percona/* ]] logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/fluentbit:main-logcollector == perconalab/* ]] logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ new_value=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ echo IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | docker.io/perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | ++++ eval 'echo $IMAGE_BASE' logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++++ echo perconalab/percona-postgresql-operator logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ var_value=perconalab/percona-postgresql-operator logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator == docker.io/* ]] logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator == percona/* ]] logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator == perconalab/* ]] logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ new_value=docker.io/perconalab/percona-postgresql-operator logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ echo IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | docker.io/perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | ++++ eval 'echo $IMAGE_PGBOUNCER' logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++++ echo perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ var_value=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == docker.io/* ]] logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == percona/* ]] logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == perconalab/* ]] logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ echo IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | docker.io/perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | ++++ eval 'echo $IMAGE_POSTGRESQL' logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++++ echo perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ var_value=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == docker.io/* ]] logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == percona/* ]] logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == perconalab/* ]] logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ echo IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | docker.io/perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | ++++ eval 'echo $IMAGE_PMM_SERVER' logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++++ echo perconalab/pmm-server:3-dev-latest logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ var_value=perconalab/pmm-server:3-dev-latest logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/pmm-server:3-dev-latest == docker.io/* ]] logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/pmm-server:3-dev-latest == percona/* ]] logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/pmm-server:3-dev-latest == perconalab/* ]] logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ new_value=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ echo IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | docker.io/perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | ++++ eval 'echo $IMAGE_PMM_CLIENT' logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++++ echo perconalab/pmm-client:3-dev-latest logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ var_value=perconalab/pmm-client:3-dev-latest logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/pmm-client:3-dev-latest == docker.io/* ]] logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/pmm-client:3-dev-latest == percona/* ]] logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ perconalab/pmm-client:3-dev-latest == perconalab/* ]] logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ new_value=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ export IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ echo IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | docker.io/perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | ++++ eval 'echo $IMAGE_AWS_CLI' logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++++ echo docker.io/amazon/aws-cli:2.34.60 logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ var_value=docker.io/amazon/aws-cli:2.34.60 logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ [[ docker.io/amazon/aws-cli:2.34.60 == docker.io/* ]] logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ continue logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | ++++ which gdate logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | which: no gdate in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1786/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | ++++ which date logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ date=/usr/sbin/date logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | ++++ which gsed logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | which: no gsed in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1786/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | ++++ which sed logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | +++ sed=/usr/sbin/sed logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | ++ arch_scheduling_enabled logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | ++ [[ -n '' ]] logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | + init_temp_dir logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | + rm -rf /tmp/kuttl/pg/major-upgrade-logical-replica logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | + mkdir -p /tmp/kuttl/pg/major-upgrade-logical-replica logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | + deploy_operator logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | + local cw_prefix= logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | + destroy_operator logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | + kubectl -n pg-operator delete deployment percona-postgresql-operator --force --grace-period=0 logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | Error from server (NotFound): deployments.apps "percona-postgresql-operator" not found logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | + true logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | + [[ -n pg-operator ]] logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | + kubectl delete namespace pg-operator --force --grace-period=0 --ignore-not-found logger.go:42: 23:56:19 | major-upgrade-logical-replica/0-deploy-operator | Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. logger.go:42: 23:56:20 | major-upgrade-logical-replica/0-deploy-operator | + [[ -n pg-operator ]] logger.go:42: 23:56:20 | major-upgrade-logical-replica/0-deploy-operator | + create_namespace pg-operator logger.go:42: 23:56:20 | major-upgrade-logical-replica/0-deploy-operator | + local namespace=pg-operator logger.go:42: 23:56:20 | major-upgrade-logical-replica/0-deploy-operator | + [[ gke == openshift ]] logger.go:42: 23:56:20 | major-upgrade-logical-replica/0-deploy-operator | + kubectl delete namespace pg-operator --ignore-not-found logger.go:42: 23:56:21 | major-upgrade-logical-replica/0-deploy-operator | + kubectl wait --for=delete namespace pg-operator logger.go:42: 23:56:21 | major-upgrade-logical-replica/0-deploy-operator | + kubectl create namespace pg-operator logger.go:42: 23:56:22 | major-upgrade-logical-replica/0-deploy-operator | namespace/pg-operator created logger.go:42: 23:56:22 | major-upgrade-logical-replica/0-deploy-operator | + cw_prefix=cw- logger.go:42: 23:56:22 | major-upgrade-logical-replica/0-deploy-operator | + kubectl -n pg-operator apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pg-operator_PR-1786/deploy/crd.yaml logger.go:42: 23:56:23 | major-upgrade-logical-replica/0-deploy-operator | customresourcedefinition.apiextensions.k8s.io/crunchybridgeclusters.upstream.pgv2.percona.com serverside-applied logger.go:42: 23:56:23 | major-upgrade-logical-replica/0-deploy-operator | customresourcedefinition.apiextensions.k8s.io/perconapgbackups.pgv2.percona.com serverside-applied logger.go:42: 23:56:25 | major-upgrade-logical-replica/0-deploy-operator | customresourcedefinition.apiextensions.k8s.io/perconapgclusters.pgv2.percona.com serverside-applied logger.go:42: 23:56:25 | major-upgrade-logical-replica/0-deploy-operator | customresourcedefinition.apiextensions.k8s.io/perconapgrestores.pgv2.percona.com serverside-applied logger.go:42: 23:56:25 | major-upgrade-logical-replica/0-deploy-operator | customresourcedefinition.apiextensions.k8s.io/perconapgupgrades.pgv2.percona.com serverside-applied logger.go:42: 23:56:26 | major-upgrade-logical-replica/0-deploy-operator | customresourcedefinition.apiextensions.k8s.io/pgadmins.upstream.pgv2.percona.com serverside-applied logger.go:42: 23:56:26 | major-upgrade-logical-replica/0-deploy-operator | customresourcedefinition.apiextensions.k8s.io/pgupgrades.upstream.pgv2.percona.com serverside-applied logger.go:42: 23:56:27 | major-upgrade-logical-replica/0-deploy-operator | customresourcedefinition.apiextensions.k8s.io/postgresclusters.upstream.pgv2.percona.com serverside-applied logger.go:42: 23:56:27 | major-upgrade-logical-replica/0-deploy-operator | + kubectl -n pg-operator apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pg-operator_PR-1786/deploy/cw-rbac.yaml logger.go:42: 23:56:28 | major-upgrade-logical-replica/0-deploy-operator | serviceaccount/percona-postgresql-operator serverside-applied logger.go:42: 23:56:28 | major-upgrade-logical-replica/0-deploy-operator | clusterrole.rbac.authorization.k8s.io/percona-postgresql-operator serverside-applied logger.go:42: 23:56:29 | major-upgrade-logical-replica/0-deploy-operator | clusterrolebinding.rbac.authorization.k8s.io/percona-postgresql-operator serverside-applied logger.go:42: 23:56:29 | major-upgrade-logical-replica/0-deploy-operator | + local disable_telemetry=true logger.go:42: 23:56:29 | major-upgrade-logical-replica/0-deploy-operator | + '[' major-upgrade-logical-replica == telemetry-transfer ']' logger.go:42: 23:56:29 | major-upgrade-logical-replica/0-deploy-operator | + yq eval '.spec.template.spec.containers[0].image = "docker.io/perconalab/percona-postgresql-operator:PR-1786-391a86551"' /mnt/jenkins/workspace/cloud-pg-operator_PR-1786/deploy/cw-operator.yaml logger.go:42: 23:56:29 | major-upgrade-logical-replica/0-deploy-operator | + yq eval '(.spec.template.spec.containers[] | select(.name=="operator") | .env[] | select(.name=="DISABLE_TELEMETRY") | .value) = "true"' - logger.go:42: 23:56:29 | major-upgrade-logical-replica/0-deploy-operator | + yq eval '(.spec.template.spec.containers[] | select(.name=="operator") | .env[] | select(.name=="LOG_LEVEL") | .value) = "DEBUG"' - logger.go:42: 23:56:29 | major-upgrade-logical-replica/0-deploy-operator | + yq eval '(.spec.template.spec.containers[] | select(.name=="operator") | .env[] | select(.name=="PGO_FEATURE_GATES") | .value) = ""' - logger.go:42: 23:56:29 | major-upgrade-logical-replica/0-deploy-operator | + add_arch_to_podspecs - logger.go:42: 23:56:29 | major-upgrade-logical-replica/0-deploy-operator | + arch_scheduling_enabled logger.go:42: 23:56:29 | major-upgrade-logical-replica/0-deploy-operator | + [[ -n '' ]] logger.go:42: 23:56:29 | major-upgrade-logical-replica/0-deploy-operator | + cat - logger.go:42: 23:56:29 | major-upgrade-logical-replica/0-deploy-operator | + kubectl -n pg-operator apply -f - logger.go:42: 23:56:29 | major-upgrade-logical-replica/0-deploy-operator | + return logger.go:42: 23:56:30 | major-upgrade-logical-replica/0-deploy-operator | deployment.apps/percona-postgresql-operator created logger.go:42: 23:56:30 | major-upgrade-logical-replica/0-deploy-operator | + deploy_client logger.go:42: 23:56:30 | major-upgrade-logical-replica/0-deploy-operator | + add_arch_to_podspecs /mnt/jenkins/workspace/cloud-pg-operator_PR-1786/e2e-tests/conf/client.yaml logger.go:42: 23:56:30 | major-upgrade-logical-replica/0-deploy-operator | + arch_scheduling_enabled logger.go:42: 23:56:30 | major-upgrade-logical-replica/0-deploy-operator | + [[ -n '' ]] logger.go:42: 23:56:30 | major-upgrade-logical-replica/0-deploy-operator | + cat /mnt/jenkins/workspace/cloud-pg-operator_PR-1786/e2e-tests/conf/client.yaml logger.go:42: 23:56:30 | major-upgrade-logical-replica/0-deploy-operator | + kubectl -n kuttl-test-evident-monkey apply -f - logger.go:42: 23:56:30 | major-upgrade-logical-replica/0-deploy-operator | + return logger.go:42: 23:56:31 | major-upgrade-logical-replica/0-deploy-operator | deployment.apps/pg-client created logger.go:42: 23:56:31 | major-upgrade-logical-replica/0-deploy-operator | + deploy_s3_secrets logger.go:42: 23:56:31 | major-upgrade-logical-replica/0-deploy-operator | + set +o xtrace logger.go:42: 23:56:32 | major-upgrade-logical-replica/0-deploy-operator | secret/major-upgrade-logical-replica-pgbackrest-secrets created logger.go:42: 23:56:33 | major-upgrade-logical-replica/0-deploy-operator | secret/aws-s3-secret created logger.go:42: 23:56:34 | major-upgrade-logical-replica/0-deploy-operator | secret/do-spaces-secret created logger.go:42: 23:56:34 | major-upgrade-logical-replica/0-deploy-operator | secret/gcp-cs-secret created logger.go:42: 23:56:34 | major-upgrade-logical-replica/0-deploy-operator | secret/azure-secret created logger.go:42: 23:56:36 | major-upgrade-logical-replica/0-deploy-operator | secret/minio-secret created logger.go:42: 23:56:36 | major-upgrade-logical-replica/0-deploy-operator | running command: [sh -c kubectl assert exist-enhanced deployment percona-postgresql-operator -n ${OPERATOR_NS:-$NAMESPACE} --field-selector status.readyReplicas=1] logger.go:42: 23:56:36 | major-upgrade-logical-replica/0-deploy-operator | ASSERT deployment percona-postgresql-operator matching field criteria 'status.readyReplicas=1' should exist. logger.go:42: 23:56:37 | major-upgrade-logical-replica/0-deploy-operator | INFO Found 1 resource(s). logger.go:42: 23:56:37 | major-upgrade-logical-replica/0-deploy-operator | NAME NAMESPACE COL0 logger.go:42: 23:56:37 | major-upgrade-logical-replica/0-deploy-operator | percona-postgresql-operator pg-operator 1 logger.go:42: 23:56:37 | major-upgrade-logical-replica/0-deploy-operator | ASSERT PASS logger.go:42: 23:56:37 | major-upgrade-logical-replica/0-deploy-operator | test step completed 0-deploy-operator logger.go:42: 23:56:37 | major-upgrade-logical-replica/1-create-cluster | starting test step 1-create-cluster logger.go:42: 23:56:37 | major-upgrade-logical-replica/1-create-cluster | running command: [sh -c set -o errexit set -o xtrace source ../../functions postgres_image=$(get_container_image "postgres" 17) pgbouncer_image=$(get_container_image "pgbouncer" 17) pgbackrest_image=$(get_container_image "pgbackrest" 17) get_cr \ | yq eval " .spec.postgresVersion = 17 | .spec.image = \"${postgres_image}\" | .spec.proxy.pgBouncer.image = \"${pgbouncer_image}\" | .spec.backups.pgbackrest.image = \"${pgbackrest_image}\" | .spec.patroni.removeDataDirectoryOnDivergedTimelines = true" \ | kubectl -n "${NAMESPACE}" apply -f -] logger.go:42: 23:56:37 | major-upgrade-logical-replica/1-create-cluster | + source ../../functions logger.go:42: 23:56:37 | major-upgrade-logical-replica/1-create-cluster | +++ realpath ../../.. logger.go:42: 23:56:37 | major-upgrade-logical-replica/1-create-cluster | ++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1786 logger.go:42: 23:56:37 | major-upgrade-logical-replica/1-create-cluster | ++ CERT_MANAGER_VER=1.20.2 logger.go:42: 23:56:37 | major-upgrade-logical-replica/1-create-cluster | ++ CERT_MANAGER_TEST_LABEL=e2e.percona.com/managed-by=percona-postgresql-operator-tests logger.go:42: 23:56:37 | major-upgrade-logical-replica/1-create-cluster | ++ CHAOS_MESH_VER=2.5.1 logger.go:42: 23:56:37 | major-upgrade-logical-replica/1-create-cluster | ++++ pwd logger.go:42: 23:56:37 | major-upgrade-logical-replica/1-create-cluster | +++ basename /mnt/jenkins/workspace/cloud-pg-operator_PR-1786/e2e-tests/tests/major-upgrade-logical-replica logger.go:42: 23:56:37 | major-upgrade-logical-replica/1-create-cluster | ++ test_name=major-upgrade-logical-replica logger.go:42: 23:56:37 | major-upgrade-logical-replica/1-create-cluster | ++ source /mnt/jenkins/workspace/cloud-pg-operator_PR-1786/e2e-tests/vars.sh logger.go:42: 23:56:37 | major-upgrade-logical-replica/1-create-cluster | +++ export ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1786 logger.go:42: 23:56:37 | major-upgrade-logical-replica/1-create-cluster | +++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1786 logger.go:42: 23:56:37 | major-upgrade-logical-replica/1-create-cluster | +++ export DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1786/deploy logger.go:42: 23:56:37 | major-upgrade-logical-replica/1-create-cluster | +++ DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1786/deploy logger.go:42: 23:56:37 | major-upgrade-logical-replica/1-create-cluster | +++ export TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1786/e2e-tests logger.go:42: 23:56:37 | major-upgrade-logical-replica/1-create-cluster | +++ TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1786/e2e-tests logger.go:42: 23:56:37 | major-upgrade-logical-replica/1-create-cluster | +++ export TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1786/e2e-tests/conf logger.go:42: 23:56:37 | major-upgrade-logical-replica/1-create-cluster | +++ TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1786/e2e-tests/conf logger.go:42: 23:56:37 | major-upgrade-logical-replica/1-create-cluster | +++ export TEST_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1786/e2e-tests/tests/major-upgrade-logical-replica/conf logger.go:42: 23:56:37 | major-upgrade-logical-replica/1-create-cluster | +++ TEST_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1786/e2e-tests/tests/major-upgrade-logical-replica/conf logger.go:42: 23:56:37 | major-upgrade-logical-replica/1-create-cluster | +++ export TEMP_DIR=/tmp/kuttl/pg/major-upgrade-logical-replica logger.go:42: 23:56:37 | major-upgrade-logical-replica/1-create-cluster | +++ TEMP_DIR=/tmp/kuttl/pg/major-upgrade-logical-replica logger.go:42: 23:56:37 | major-upgrade-logical-replica/1-create-cluster | ++++ git rev-parse --abbrev-ref HEAD logger.go:42: 23:56:37 | major-upgrade-logical-replica/1-create-cluster | +++ export GIT_BRANCH=PR-1786 logger.go:42: 23:56:37 | major-upgrade-logical-replica/1-create-cluster | +++ GIT_BRANCH=PR-1786 logger.go:42: 23:56:37 | major-upgrade-logical-replica/1-create-cluster | +++ export VERSION=PR-1786-391a86551 logger.go:42: 23:56:37 | major-upgrade-logical-replica/1-create-cluster | +++ VERSION=PR-1786-391a86551 logger.go:42: 23:56:37 | major-upgrade-logical-replica/1-create-cluster | +++ export SKIP_TEST_WARNINGS=true logger.go:42: 23:56:37 | major-upgrade-logical-replica/1-create-cluster | +++ SKIP_TEST_WARNINGS=true logger.go:42: 23:56:37 | major-upgrade-logical-replica/1-create-cluster | +++ export K8S_UPGRADE_PLATFORM=gke logger.go:42: 23:56:37 | major-upgrade-logical-replica/1-create-cluster | +++ K8S_UPGRADE_PLATFORM=gke logger.go:42: 23:56:37 | major-upgrade-logical-replica/1-create-cluster | +++ export K8S_UPGRADE_REGION=us-central1-a logger.go:42: 23:56:37 | major-upgrade-logical-replica/1-create-cluster | +++ K8S_UPGRADE_REGION=us-central1-a logger.go:42: 23:56:37 | major-upgrade-logical-replica/1-create-cluster | +++ export K8S_UPGRADE_INITIAL_VERSION= logger.go:42: 23:56:37 | major-upgrade-logical-replica/1-create-cluster | +++ K8S_UPGRADE_INITIAL_VERSION= logger.go:42: 23:56:37 | major-upgrade-logical-replica/1-create-cluster | +++ export K8S_UPGRADE_FINAL_VERSION= logger.go:42: 23:56:37 | major-upgrade-logical-replica/1-create-cluster | +++ K8S_UPGRADE_FINAL_VERSION= logger.go:42: 23:56:37 | major-upgrade-logical-replica/1-create-cluster | +++ [[ -z '' ]] logger.go:42: 23:56:37 | major-upgrade-logical-replica/1-create-cluster | ++++ detect_k8s_platform logger.go:42: 23:56:37 | major-upgrade-logical-replica/1-create-cluster | ++++ local detected_platform logger.go:42: 23:56:37 | major-upgrade-logical-replica/1-create-cluster | ++++ local xtrace_enabled=false logger.go:42: 23:56:37 | major-upgrade-logical-replica/1-create-cluster | ++++ [[ -o xtrace ]] logger.go:42: 23:56:37 | major-upgrade-logical-replica/1-create-cluster | ++++ xtrace_enabled=true logger.go:42: 23:56:37 | major-upgrade-logical-replica/1-create-cluster | ++++ set +o xtrace logger.go:42: 23:56:39 | major-upgrade-logical-replica/1-create-cluster | +++ PLATFORM=gke logger.go:42: 23:56:39 | major-upgrade-logical-replica/1-create-cluster | +++ export PLATFORM logger.go:42: 23:56:39 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 23:56:39 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 23:56:39 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE=perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 23:56:39 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE=perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 23:56:39 | major-upgrade-logical-replica/1-create-cluster | +++ [[ ! -n '' ]] logger.go:42: 23:56:39 | major-upgrade-logical-replica/1-create-cluster | +++ [[ -n '' ]] logger.go:42: 23:56:39 | major-upgrade-logical-replica/1-create-cluster | +++ export PG_VER=18 logger.go:42: 23:56:39 | major-upgrade-logical-replica/1-create-cluster | +++ PG_VER=18 logger.go:42: 23:56:39 | major-upgrade-logical-replica/1-create-cluster | +++ export ARCH= logger.go:42: 23:56:39 | major-upgrade-logical-replica/1-create-cluster | +++ ARCH= logger.go:42: 23:56:39 | major-upgrade-logical-replica/1-create-cluster | +++ [[ -z '' ]] logger.go:42: 23:56:39 | major-upgrade-logical-replica/1-create-cluster | ++++ kubectl get nodes -o 'jsonpath={range .items[*]}{.metadata.labels.kubernetes\.io/arch}{"\n"}{end}' logger.go:42: 23:56:39 | major-upgrade-logical-replica/1-create-cluster | ++++ sort -u logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ node_arches=amd64 logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ [[ amd64 == arm64 ]] logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ export PG_DISTRIBUTION= logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ PG_DISTRIBUTION= logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ [[ '' == community ]] logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ export BUCKET=pg-operator-testing logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ BUCKET=pg-operator-testing logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ export PMM_SERVER_VERSION=9.9.9 logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ PMM_SERVER_VERSION=9.9.9 logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE_LOGCOLLECTOR=perconalab/fluentbit:main-logcollector logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE_LOGCOLLECTOR=perconalab/fluentbit:main-logcollector logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ export PGOV1_TAG=1.4.0 logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ PGOV1_TAG=1.4.0 logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ export PGOV1_VER=14 logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ PGOV1_VER=14 logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ export CPGO_VERSION=5.8.7 logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ CPGO_VERSION=5.8.7 logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ export MINIO_VER=5.4.0 logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ MINIO_VER=5.4.0 logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ export VAULT_VER=0.32.0 logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ VAULT_VER=0.32.0 logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE_AWS_CLI=docker.io/amazon/aws-cli:2.34.60 logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE_AWS_CLI=docker.io/amazon/aws-cli:2.34.60 logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ export REGISTRY_NAME=docker.io logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ REGISTRY_NAME=docker.io logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ export REGISTRY_NAME_FULL=docker.io/ logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ REGISTRY_NAME_FULL=docker.io/ logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | ++++ printenv logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | ++++ grep -E '^IMAGE' logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | ++++ awk -F= '{print $1}' logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | ++++ eval 'echo $IMAGE_BACKREST' logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++++ echo perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ var_value=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == docker.io/* ]] logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == percona/* ]] logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == perconalab/* ]] logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ echo IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ echo perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | ++++ eval 'echo $IMAGE' logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++++ echo perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ var_value=perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:PR-1786-391a86551 == docker.io/* ]] logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:PR-1786-391a86551 == percona/* ]] logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:PR-1786-391a86551 == perconalab/* ]] logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ new_value=docker.io/perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ echo IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | docker.io/perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | ++++ eval 'echo $IMAGE_UPGRADE' logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++++ echo perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ var_value=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == docker.io/* ]] logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == percona/* ]] logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == perconalab/* ]] logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ echo IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | docker.io/perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | ++++ eval 'echo $IMAGE_LOGCOLLECTOR' logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++++ echo perconalab/fluentbit:main-logcollector logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ var_value=perconalab/fluentbit:main-logcollector logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/fluentbit:main-logcollector == docker.io/* ]] logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/fluentbit:main-logcollector == percona/* ]] logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/fluentbit:main-logcollector == perconalab/* ]] logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ new_value=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ echo IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | docker.io/perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | ++++ eval 'echo $IMAGE_BASE' logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++++ echo perconalab/percona-postgresql-operator logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ var_value=perconalab/percona-postgresql-operator logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator == docker.io/* ]] logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator == percona/* ]] logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator == perconalab/* ]] logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ new_value=docker.io/perconalab/percona-postgresql-operator logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ echo IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | docker.io/perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | ++++ eval 'echo $IMAGE_PGBOUNCER' logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++++ echo perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ var_value=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == docker.io/* ]] logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == percona/* ]] logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == perconalab/* ]] logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ echo IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | docker.io/perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | ++++ eval 'echo $IMAGE_POSTGRESQL' logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++++ echo perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ var_value=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == docker.io/* ]] logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == percona/* ]] logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == perconalab/* ]] logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ echo IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | docker.io/perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | ++++ eval 'echo $IMAGE_PMM_SERVER' logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++++ echo perconalab/pmm-server:3-dev-latest logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ var_value=perconalab/pmm-server:3-dev-latest logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/pmm-server:3-dev-latest == docker.io/* ]] logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/pmm-server:3-dev-latest == percona/* ]] logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/pmm-server:3-dev-latest == perconalab/* ]] logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ new_value=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ echo IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | docker.io/perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | ++++ eval 'echo $IMAGE_PMM_CLIENT' logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++++ echo perconalab/pmm-client:3-dev-latest logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ var_value=perconalab/pmm-client:3-dev-latest logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/pmm-client:3-dev-latest == docker.io/* ]] logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/pmm-client:3-dev-latest == percona/* ]] logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ [[ perconalab/pmm-client:3-dev-latest == perconalab/* ]] logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ new_value=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ export IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ echo IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | docker.io/perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | ++++ eval 'echo $IMAGE_AWS_CLI' logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++++ echo docker.io/amazon/aws-cli:2.34.60 logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ var_value=docker.io/amazon/aws-cli:2.34.60 logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ [[ docker.io/amazon/aws-cli:2.34.60 == docker.io/* ]] logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ continue logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | ++++ which gdate logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | which: no gdate in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1786/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | ++++ which date logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ date=/usr/sbin/date logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | ++++ which gsed logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | which: no gsed in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1786/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | ++++ which sed logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | +++ sed=/usr/sbin/sed logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | ++ arch_scheduling_enabled logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | ++ [[ -n '' ]] logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | ++ get_container_image postgres 17 logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | ++ local component=postgres logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | ++ local pgVersion=17 logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | ++ [[ docker.io/perconalab/percona-postgresql-operator:PR-1786-391a86551 == percona/* ]] logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | ++ [[ docker.io/perconalab/percona-postgresql-operator:PR-1786-391a86551 == docker.io/percona/* ]] logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | ++ [[ -n '' ]] logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | ++ local operatorVersion=PR-1786-391a86551 logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | ++ [[ ! PR-1786-391a86551 =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | ++ operatorVersion=main logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | ++ [[ '' == community ]] logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | ++ [[ postgres == pgbouncer ]] logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | ++ [[ postgres == pgbackrest ]] logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | ++ [[ postgres == upgrade ]] logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | ++ [[ postgres == postgis ]] logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | ++ echo docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | + postgres_image=docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | ++ get_container_image pgbouncer 17 logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | ++ local component=pgbouncer logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | ++ local pgVersion=17 logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | ++ [[ docker.io/perconalab/percona-postgresql-operator:PR-1786-391a86551 == percona/* ]] logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | ++ [[ docker.io/perconalab/percona-postgresql-operator:PR-1786-391a86551 == docker.io/percona/* ]] logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | ++ [[ -n '' ]] logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | ++ local operatorVersion=PR-1786-391a86551 logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | ++ [[ ! PR-1786-391a86551 =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | ++ operatorVersion=main logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | ++ [[ '' == community ]] logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | ++ [[ pgbouncer == pgbouncer ]] logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | ++ echo docker.io/perconalab/percona-postgresql-operator:main-pgbouncer17 logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | + pgbouncer_image=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer17 logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | ++ get_container_image pgbackrest 17 logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | ++ local component=pgbackrest logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | ++ local pgVersion=17 logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | ++ [[ docker.io/perconalab/percona-postgresql-operator:PR-1786-391a86551 == percona/* ]] logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | ++ [[ docker.io/perconalab/percona-postgresql-operator:PR-1786-391a86551 == docker.io/percona/* ]] logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | ++ [[ -n '' ]] logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | ++ local operatorVersion=PR-1786-391a86551 logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | ++ [[ ! PR-1786-391a86551 =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | ++ operatorVersion=main logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | ++ [[ '' == community ]] logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | ++ [[ pgbackrest == pgbouncer ]] logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | ++ [[ pgbackrest == pgbackrest ]] logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | ++ echo docker.io/perconalab/percona-postgresql-operator:main-pgbackrest17 logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | + pgbackrest_image=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest17 logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | + get_cr logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | + local cr_name=major-upgrade-logical-replica logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | + local repo_path= logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | + local source_path= logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | + local platform=gke logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | + yq eval $'\n .spec.postgresVersion = 17 |\n .spec.image = "docker.io/perconalab/percona-postgresql-operator:main-ppg17-postgres" |\n .spec.proxy.pgBouncer.image = "docker.io/perconalab/percona-postgresql-operator:main-pgbouncer17" |\n .spec.backups.pgbackrest.image = "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest17" |\n .spec.patroni.removeDataDirectoryOnDivergedTimelines = true' logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | + local cr_file=/mnt/jenkins/workspace/cloud-pg-operator_PR-1786/e2e-tests/tests/major-upgrade-logical-replica/conf/major-upgrade-logical-replica.yaml logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | + local platform_cr_file=/mnt/jenkins/workspace/cloud-pg-operator_PR-1786/e2e-tests/tests/major-upgrade-logical-replica/conf/major-upgrade-logical-replica-gke.yaml logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | + local spaces_cr_file=/mnt/jenkins/workspace/cloud-pg-operator_PR-1786/e2e-tests/tests/major-upgrade-logical-replica/conf/major-upgrade-logical-replica-spaces.yaml logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | + [[ -f /mnt/jenkins/workspace/cloud-pg-operator_PR-1786/e2e-tests/tests/major-upgrade-logical-replica/conf/major-upgrade-logical-replica-gke.yaml ]] logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | + [[ gke == doks ]] logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | + kubectl -n kuttl-test-evident-monkey apply -f - logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | + crs=('/mnt/jenkins/workspace/cloud-pg-operator_PR-1786/deploy/cr.yaml') logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | + local crs logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | + [[ -f /mnt/jenkins/workspace/cloud-pg-operator_PR-1786/e2e-tests/tests/major-upgrade-logical-replica/conf/major-upgrade-logical-replica.yaml ]] logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | + crs+=($cr_file) logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | + yq eval-all $'\n\t\tselect(fileIndex == 0) * (select(fileIndex == 1) // {}) |\n\t\t.metadata.name = "major-upgrade-logical-replica" |\n\t\t.metadata.labels = {"e2e":"major-upgrade-logical-replica"} |\n\t\t.spec.postgresVersion = 18 |\n\t\t.spec.users += [{"name":"postgres","password":{"type":"AlphaNumeric"}}] |\n\t\t.spec.users += [{"name":"major-upgrade-logical-replica","password":{"type":"AlphaNumeric"}}] |\n\t\t.spec.image = "docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres" |\n\t\t.spec.initContainer.image = "docker.io/perconalab/percona-postgresql-operator:PR-1786-391a86551" |\n\t\t.spec.backups.pgbackrest.image = "docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18" |\n\t\t.spec.proxy.pgBouncer.image = "docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18" |\n\t\t.spec.logcollector.image = "docker.io/perconalab/fluentbit:main-logcollector" |\n\t\t.spec.pmm.image = "docker.io/perconalab/pmm-client:3-dev-latest" |\n\t\t.spec.pmm.secret = "major-upgrade-logical-replica-pmm-secret" |\n\t\t.spec.pmm.customClusterName = "major-upgrade-logical-replica-pmm-custom-name" |\n\t\t.spec.pmm.postgresParams = "--environment=dev-postgres"\n\t' /mnt/jenkins/workspace/cloud-pg-operator_PR-1786/deploy/cr.yaml /mnt/jenkins/workspace/cloud-pg-operator_PR-1786/e2e-tests/tests/major-upgrade-logical-replica/conf/major-upgrade-logical-replica.yaml logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | + /usr/sbin/sed -i 's|||g' /tmp/kuttl/pg/major-upgrade-logical-replica/cr.yaml logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | + /usr/sbin/sed -i 's|||g' /tmp/kuttl/pg/major-upgrade-logical-replica/cr.yaml logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | + /usr/sbin/sed -i 's||pg-operator-testing|g' /tmp/kuttl/pg/major-upgrade-logical-replica/cr.yaml logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | + /usr/sbin/sed -i 's||docker.io/perconalab/percona-postgresql-operator:PR-1786-391a86551|g' /tmp/kuttl/pg/major-upgrade-logical-replica/cr.yaml logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | + [[ gke == openshift ]] logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | + arch_scheduling_enabled logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | + [[ -n '' ]] logger.go:42: 23:56:40 | major-upgrade-logical-replica/1-create-cluster | + cat /tmp/kuttl/pg/major-upgrade-logical-replica/cr.yaml logger.go:42: 23:56:41 | major-upgrade-logical-replica/1-create-cluster | perconapgcluster.pgv2.percona.com/major-upgrade-logical-replica created logger.go:42: 23:57:33 | major-upgrade-logical-replica/1-create-cluster | test step completed 1-create-cluster logger.go:42: 23:57:33 | major-upgrade-logical-replica/2-write-data | starting test step 2-write-data logger.go:42: 23:57:33 | major-upgrade-logical-replica/2-write-data | running command: [sh -c set -o errexit set -o xtrace source ../../functions PASS=$(get_psql_user_pass major-upgrade-logical-replica-pguser-postgres) HOST=$(get_psql_user_host major-upgrade-logical-replica-pguser-postgres) run_psql_local \ 'CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);' \ "postgres:${PASS}@${HOST}" run_psql_local \ '\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)' \ "postgres:${PASS}@${HOST}"] logger.go:42: 23:57:33 | major-upgrade-logical-replica/2-write-data | + source ../../functions logger.go:42: 23:57:33 | major-upgrade-logical-replica/2-write-data | +++ realpath ../../.. logger.go:42: 23:57:33 | major-upgrade-logical-replica/2-write-data | ++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1786 logger.go:42: 23:57:33 | major-upgrade-logical-replica/2-write-data | ++ CERT_MANAGER_VER=1.20.2 logger.go:42: 23:57:33 | major-upgrade-logical-replica/2-write-data | ++ CERT_MANAGER_TEST_LABEL=e2e.percona.com/managed-by=percona-postgresql-operator-tests logger.go:42: 23:57:33 | major-upgrade-logical-replica/2-write-data | ++ CHAOS_MESH_VER=2.5.1 logger.go:42: 23:57:33 | major-upgrade-logical-replica/2-write-data | ++++ pwd logger.go:42: 23:57:33 | major-upgrade-logical-replica/2-write-data | +++ basename /mnt/jenkins/workspace/cloud-pg-operator_PR-1786/e2e-tests/tests/major-upgrade-logical-replica logger.go:42: 23:57:33 | major-upgrade-logical-replica/2-write-data | ++ test_name=major-upgrade-logical-replica logger.go:42: 23:57:33 | major-upgrade-logical-replica/2-write-data | ++ source /mnt/jenkins/workspace/cloud-pg-operator_PR-1786/e2e-tests/vars.sh logger.go:42: 23:57:33 | major-upgrade-logical-replica/2-write-data | +++ export ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1786 logger.go:42: 23:57:33 | major-upgrade-logical-replica/2-write-data | +++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1786 logger.go:42: 23:57:33 | major-upgrade-logical-replica/2-write-data | +++ export DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1786/deploy logger.go:42: 23:57:33 | major-upgrade-logical-replica/2-write-data | +++ DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1786/deploy logger.go:42: 23:57:33 | major-upgrade-logical-replica/2-write-data | +++ export TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1786/e2e-tests logger.go:42: 23:57:33 | major-upgrade-logical-replica/2-write-data | +++ TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1786/e2e-tests logger.go:42: 23:57:33 | major-upgrade-logical-replica/2-write-data | +++ export TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1786/e2e-tests/conf logger.go:42: 23:57:33 | major-upgrade-logical-replica/2-write-data | +++ TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1786/e2e-tests/conf logger.go:42: 23:57:33 | major-upgrade-logical-replica/2-write-data | +++ export TEST_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1786/e2e-tests/tests/major-upgrade-logical-replica/conf logger.go:42: 23:57:33 | major-upgrade-logical-replica/2-write-data | +++ TEST_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1786/e2e-tests/tests/major-upgrade-logical-replica/conf logger.go:42: 23:57:33 | major-upgrade-logical-replica/2-write-data | +++ export TEMP_DIR=/tmp/kuttl/pg/major-upgrade-logical-replica logger.go:42: 23:57:33 | major-upgrade-logical-replica/2-write-data | +++ TEMP_DIR=/tmp/kuttl/pg/major-upgrade-logical-replica logger.go:42: 23:57:33 | major-upgrade-logical-replica/2-write-data | ++++ git rev-parse --abbrev-ref HEAD logger.go:42: 23:57:33 | major-upgrade-logical-replica/2-write-data | +++ export GIT_BRANCH=PR-1786 logger.go:42: 23:57:33 | major-upgrade-logical-replica/2-write-data | +++ GIT_BRANCH=PR-1786 logger.go:42: 23:57:33 | major-upgrade-logical-replica/2-write-data | +++ export VERSION=PR-1786-391a86551 logger.go:42: 23:57:33 | major-upgrade-logical-replica/2-write-data | +++ VERSION=PR-1786-391a86551 logger.go:42: 23:57:33 | major-upgrade-logical-replica/2-write-data | +++ export SKIP_TEST_WARNINGS=true logger.go:42: 23:57:33 | major-upgrade-logical-replica/2-write-data | +++ SKIP_TEST_WARNINGS=true logger.go:42: 23:57:33 | major-upgrade-logical-replica/2-write-data | +++ export K8S_UPGRADE_PLATFORM=gke logger.go:42: 23:57:33 | major-upgrade-logical-replica/2-write-data | +++ K8S_UPGRADE_PLATFORM=gke logger.go:42: 23:57:33 | major-upgrade-logical-replica/2-write-data | +++ export K8S_UPGRADE_REGION=us-central1-a logger.go:42: 23:57:33 | major-upgrade-logical-replica/2-write-data | +++ K8S_UPGRADE_REGION=us-central1-a logger.go:42: 23:57:33 | major-upgrade-logical-replica/2-write-data | +++ export K8S_UPGRADE_INITIAL_VERSION= logger.go:42: 23:57:33 | major-upgrade-logical-replica/2-write-data | +++ K8S_UPGRADE_INITIAL_VERSION= logger.go:42: 23:57:33 | major-upgrade-logical-replica/2-write-data | +++ export K8S_UPGRADE_FINAL_VERSION= logger.go:42: 23:57:33 | major-upgrade-logical-replica/2-write-data | +++ K8S_UPGRADE_FINAL_VERSION= logger.go:42: 23:57:33 | major-upgrade-logical-replica/2-write-data | +++ [[ -z '' ]] logger.go:42: 23:57:33 | major-upgrade-logical-replica/2-write-data | ++++ detect_k8s_platform logger.go:42: 23:57:33 | major-upgrade-logical-replica/2-write-data | ++++ local detected_platform logger.go:42: 23:57:33 | major-upgrade-logical-replica/2-write-data | ++++ local xtrace_enabled=false logger.go:42: 23:57:33 | major-upgrade-logical-replica/2-write-data | ++++ [[ -o xtrace ]] logger.go:42: 23:57:33 | major-upgrade-logical-replica/2-write-data | ++++ xtrace_enabled=true logger.go:42: 23:57:33 | major-upgrade-logical-replica/2-write-data | ++++ set +o xtrace logger.go:42: 23:57:35 | major-upgrade-logical-replica/2-write-data | +++ PLATFORM=gke logger.go:42: 23:57:35 | major-upgrade-logical-replica/2-write-data | +++ export PLATFORM logger.go:42: 23:57:35 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 23:57:35 | major-upgrade-logical-replica/2-write-data | +++ IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 23:57:35 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE=perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 23:57:35 | major-upgrade-logical-replica/2-write-data | +++ IMAGE=perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 23:57:35 | major-upgrade-logical-replica/2-write-data | +++ [[ ! -n '' ]] logger.go:42: 23:57:35 | major-upgrade-logical-replica/2-write-data | +++ [[ -n '' ]] logger.go:42: 23:57:35 | major-upgrade-logical-replica/2-write-data | +++ export PG_VER=18 logger.go:42: 23:57:35 | major-upgrade-logical-replica/2-write-data | +++ PG_VER=18 logger.go:42: 23:57:35 | major-upgrade-logical-replica/2-write-data | +++ export ARCH= logger.go:42: 23:57:35 | major-upgrade-logical-replica/2-write-data | +++ ARCH= logger.go:42: 23:57:35 | major-upgrade-logical-replica/2-write-data | +++ [[ -z '' ]] logger.go:42: 23:57:35 | major-upgrade-logical-replica/2-write-data | ++++ sort -u logger.go:42: 23:57:35 | major-upgrade-logical-replica/2-write-data | ++++ kubectl get nodes -o 'jsonpath={range .items[*]}{.metadata.labels.kubernetes\.io/arch}{"\n"}{end}' logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ node_arches=amd64 logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ [[ amd64 == arm64 ]] logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ export PG_DISTRIBUTION= logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ PG_DISTRIBUTION= logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ [[ '' == community ]] logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ export BUCKET=pg-operator-testing logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ BUCKET=pg-operator-testing logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ export PMM_SERVER_VERSION=9.9.9 logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ PMM_SERVER_VERSION=9.9.9 logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE_LOGCOLLECTOR=perconalab/fluentbit:main-logcollector logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ IMAGE_LOGCOLLECTOR=perconalab/fluentbit:main-logcollector logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ export PGOV1_TAG=1.4.0 logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ PGOV1_TAG=1.4.0 logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ export PGOV1_VER=14 logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ PGOV1_VER=14 logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ export CPGO_VERSION=5.8.7 logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ CPGO_VERSION=5.8.7 logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ export MINIO_VER=5.4.0 logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ MINIO_VER=5.4.0 logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ export VAULT_VER=0.32.0 logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ VAULT_VER=0.32.0 logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE_AWS_CLI=docker.io/amazon/aws-cli:2.34.60 logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ IMAGE_AWS_CLI=docker.io/amazon/aws-cli:2.34.60 logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ export REGISTRY_NAME=docker.io logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ REGISTRY_NAME=docker.io logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ export REGISTRY_NAME_FULL=docker.io/ logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ REGISTRY_NAME_FULL=docker.io/ logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | ++++ printenv logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | ++++ grep -E '^IMAGE' logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | ++++ awk -F= '{print $1}' logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | ++++ eval 'echo $IMAGE_BACKREST' logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++++ echo perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ var_value=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == docker.io/* ]] logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == percona/* ]] logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == perconalab/* ]] logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ echo IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ echo perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | ++++ eval 'echo $IMAGE' logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++++ echo perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ var_value=perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator:PR-1786-391a86551 == docker.io/* ]] logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator:PR-1786-391a86551 == percona/* ]] logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator:PR-1786-391a86551 == perconalab/* ]] logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ new_value=docker.io/perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ echo IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | docker.io/perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | ++++ eval 'echo $IMAGE_UPGRADE' logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++++ echo perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ var_value=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == docker.io/* ]] logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == percona/* ]] logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == perconalab/* ]] logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ echo IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | docker.io/perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | ++++ eval 'echo $IMAGE_LOGCOLLECTOR' logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++++ echo perconalab/fluentbit:main-logcollector logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ var_value=perconalab/fluentbit:main-logcollector logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/fluentbit:main-logcollector == docker.io/* ]] logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/fluentbit:main-logcollector == percona/* ]] logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/fluentbit:main-logcollector == perconalab/* ]] logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ new_value=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ echo IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | docker.io/perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | ++++ eval 'echo $IMAGE_BASE' logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++++ echo perconalab/percona-postgresql-operator logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ var_value=perconalab/percona-postgresql-operator logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator == docker.io/* ]] logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator == percona/* ]] logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator == perconalab/* ]] logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ new_value=docker.io/perconalab/percona-postgresql-operator logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ echo IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | docker.io/perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | ++++ eval 'echo $IMAGE_PGBOUNCER' logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++++ echo perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ var_value=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == docker.io/* ]] logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == percona/* ]] logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == perconalab/* ]] logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ echo IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | docker.io/perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | ++++ eval 'echo $IMAGE_POSTGRESQL' logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++++ echo perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ var_value=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == docker.io/* ]] logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == percona/* ]] logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == perconalab/* ]] logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ echo IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | docker.io/perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | ++++ eval 'echo $IMAGE_PMM_SERVER' logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++++ echo perconalab/pmm-server:3-dev-latest logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ var_value=perconalab/pmm-server:3-dev-latest logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/pmm-server:3-dev-latest == docker.io/* ]] logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/pmm-server:3-dev-latest == percona/* ]] logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/pmm-server:3-dev-latest == perconalab/* ]] logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ new_value=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ echo IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | docker.io/perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | ++++ eval 'echo $IMAGE_PMM_CLIENT' logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++++ echo perconalab/pmm-client:3-dev-latest logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ var_value=perconalab/pmm-client:3-dev-latest logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/pmm-client:3-dev-latest == docker.io/* ]] logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/pmm-client:3-dev-latest == percona/* ]] logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ [[ perconalab/pmm-client:3-dev-latest == perconalab/* ]] logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ new_value=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ export IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ echo IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | docker.io/perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | ++++ eval 'echo $IMAGE_AWS_CLI' logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++++ echo docker.io/amazon/aws-cli:2.34.60 logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ var_value=docker.io/amazon/aws-cli:2.34.60 logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ [[ docker.io/amazon/aws-cli:2.34.60 == docker.io/* ]] logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ continue logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | ++++ which gdate logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | which: no gdate in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1786/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | ++++ which date logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ date=/usr/sbin/date logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | ++++ which gsed logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | which: no gsed in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1786/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | ++++ which sed logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | +++ sed=/usr/sbin/sed logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | ++ arch_scheduling_enabled logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | ++ [[ -n '' ]] logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | ++ get_psql_user_pass major-upgrade-logical-replica-pguser-postgres logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | ++ local secret_name=major-upgrade-logical-replica-pguser-postgres logger.go:42: 23:57:36 | major-upgrade-logical-replica/2-write-data | ++ kubectl -n kuttl-test-evident-monkey get secret/major-upgrade-logical-replica-pguser-postgres '--template={{.data.password | base64decode}}' logger.go:42: 23:57:37 | major-upgrade-logical-replica/2-write-data | + PASS=wJQUHcT1WtEk7Gtt4SKRMFCJ logger.go:42: 23:57:37 | major-upgrade-logical-replica/2-write-data | ++ get_psql_user_host major-upgrade-logical-replica-pguser-postgres logger.go:42: 23:57:37 | major-upgrade-logical-replica/2-write-data | ++ local secret_name=major-upgrade-logical-replica-pguser-postgres logger.go:42: 23:57:37 | major-upgrade-logical-replica/2-write-data | ++ kubectl -n kuttl-test-evident-monkey get secret/major-upgrade-logical-replica-pguser-postgres '--template={{.data.host | base64decode }}' logger.go:42: 23:57:37 | major-upgrade-logical-replica/2-write-data | + HOST=major-upgrade-logical-replica-primary.kuttl-test-evident-monkey.svc logger.go:42: 23:57:37 | major-upgrade-logical-replica/2-write-data | + run_psql_local 'CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);' postgres:wJQUHcT1WtEk7Gtt4SKRMFCJ@major-upgrade-logical-replica-primary.kuttl-test-evident-monkey.svc logger.go:42: 23:57:37 | major-upgrade-logical-replica/2-write-data | + local 'command=CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);' logger.go:42: 23:57:37 | major-upgrade-logical-replica/2-write-data | + local uri=postgres:wJQUHcT1WtEk7Gtt4SKRMFCJ@major-upgrade-logical-replica-primary.kuttl-test-evident-monkey.svc logger.go:42: 23:57:37 | major-upgrade-logical-replica/2-write-data | + local driver=postgres logger.go:42: 23:57:37 | major-upgrade-logical-replica/2-write-data | ++ get_client_pod logger.go:42: 23:57:37 | major-upgrade-logical-replica/2-write-data | ++ kubectl -n kuttl-test-evident-monkey get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' logger.go:42: 23:57:38 | major-upgrade-logical-replica/2-write-data | + kubectl -n kuttl-test-evident-monkey exec pg-client-57cfd999d5-t74nf -- bash -c 'printf '\''CREATE DATABASE myapp; \c myapp \\\ CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY);\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://'\''postgres:wJQUHcT1WtEk7Gtt4SKRMFCJ@major-upgrade-logical-replica-primary.kuttl-test-evident-monkey.svc'\''' logger.go:42: 23:57:39 | major-upgrade-logical-replica/2-write-data | + run_psql_local '\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)' postgres:wJQUHcT1WtEk7Gtt4SKRMFCJ@major-upgrade-logical-replica-primary.kuttl-test-evident-monkey.svc logger.go:42: 23:57:39 | major-upgrade-logical-replica/2-write-data | + local 'command=\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)' logger.go:42: 23:57:39 | major-upgrade-logical-replica/2-write-data | + local uri=postgres:wJQUHcT1WtEk7Gtt4SKRMFCJ@major-upgrade-logical-replica-primary.kuttl-test-evident-monkey.svc logger.go:42: 23:57:39 | major-upgrade-logical-replica/2-write-data | + local driver=postgres logger.go:42: 23:57:39 | major-upgrade-logical-replica/2-write-data | ++ get_client_pod logger.go:42: 23:57:39 | major-upgrade-logical-replica/2-write-data | ++ kubectl -n kuttl-test-evident-monkey get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' logger.go:42: 23:57:39 | major-upgrade-logical-replica/2-write-data | + kubectl -n kuttl-test-evident-monkey exec pg-client-57cfd999d5-t74nf -- bash -c 'printf '\''\c myapp \\\ INSERT INTO myApp (id) VALUES (100500)\n'\'' | psql -v ON_ERROR_STOP=1 -t -q postgres://'\''postgres:wJQUHcT1WtEk7Gtt4SKRMFCJ@major-upgrade-logical-replica-primary.kuttl-test-evident-monkey.svc'\''' logger.go:42: 23:57:40 | major-upgrade-logical-replica/2-write-data | test step completed 2-write-data logger.go:42: 23:57:40 | major-upgrade-logical-replica/3-create-seed-backup | starting test step 3-create-seed-backup logger.go:42: 23:57:40 | major-upgrade-logical-replica/3-create-seed-backup | running command: [sh -c set -o errexit set -o xtrace source ../../functions # The pgbackrest bootstrap Job restores the latest full backup, and the # one taken at cluster creation predates the rows written above. Take a # new one so the seeded data is actually in the backup being restored. cat </dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 00:01:25 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 00:01:27 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 00:01:29 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 00:01:31 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 00:01:33 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 00:01:35 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 00:01:36 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 00:01:38 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 00:01:40 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 00:01:42 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 00:01:44 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 00:01:44 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 00:01:45 | major-upgrade-logical-replica/5-upgrade | Error from server (BadRequest): container "database" in pod "17-to-18-pgdata-z958w" is waiting to start: PodInitializing logger.go:42: 00:01:48 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 00:01:48 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 00:01:49 | major-upgrade-logical-replica/5-upgrade | Error from server (BadRequest): container "database" in pod "17-to-18-pgdata-z958w" is waiting to start: PodInitializing logger.go:42: 00:01:51 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 00:01:52 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 00:01:53 | major-upgrade-logical-replica/5-upgrade | Error from server (BadRequest): container "database" in pod "17-to-18-pgdata-z958w" is waiting to start: PodInitializing logger.go:42: 00:01:55 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 00:01:56 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 00:01:57 | major-upgrade-logical-replica/5-upgrade | Error from server (BadRequest): container "database" in pod "17-to-18-pgdata-z958w" is waiting to start: PodInitializing logger.go:42: 00:01:59 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 00:02:00 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 00:02:01 | major-upgrade-logical-replica/5-upgrade | Error from server (BadRequest): container "database" in pod "17-to-18-pgdata-z958w" is waiting to start: PodInitializing logger.go:42: 00:02:03 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 00:02:04 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 00:02:05 | major-upgrade-logical-replica/5-upgrade | Error from server (BadRequest): container "database" in pod "17-to-18-pgdata-z958w" is waiting to start: PodInitializing logger.go:42: 00:02:07 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 00:02:08 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 00:02:09 | major-upgrade-logical-replica/5-upgrade | Error from server (BadRequest): container "database" in pod "17-to-18-pgdata-z958w" is waiting to start: PodInitializing logger.go:42: 00:02:11 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 00:02:12 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 00:02:13 | major-upgrade-logical-replica/5-upgrade | Error from server (BadRequest): container "database" in pod "17-to-18-pgdata-z958w" is waiting to start: PodInitializing logger.go:42: 00:02:15 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 00:02:16 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 00:02:17 | major-upgrade-logical-replica/5-upgrade | Error from server (BadRequest): container "database" in pod "17-to-18-pgdata-z958w" is waiting to start: PodInitializing logger.go:42: 00:02:19 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 00:02:19 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 00:02:20 | major-upgrade-logical-replica/5-upgrade | Error from server (BadRequest): container "database" in pod "17-to-18-pgdata-z958w" is waiting to start: PodInitializing logger.go:42: 00:02:23 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 00:02:23 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 00:02:24 | major-upgrade-logical-replica/5-upgrade | Error from server (BadRequest): container "database" in pod "17-to-18-pgdata-z958w" is waiting to start: PodInitializing logger.go:42: 00:02:27 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 00:02:27 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 00:02:28 | major-upgrade-logical-replica/5-upgrade | Performing PostgreSQL upgrade from version "17" to "18" ... logger.go:42: 00:02:28 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:28 | major-upgrade-logical-replica/5-upgrade | Step 1: Making new pgdata directory... logger.go:42: 00:02:28 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:28 | major-upgrade-logical-replica/5-upgrade | Step 2: Initializing new pgdata directory... logger.go:42: 00:02:28 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:28 | major-upgrade-logical-replica/5-upgrade | The files belonging to this database system will be owned by user "postgres". logger.go:42: 00:02:28 | major-upgrade-logical-replica/5-upgrade | This user must also own the server process. logger.go:42: 00:02:28 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:28 | major-upgrade-logical-replica/5-upgrade | The database cluster will be initialized with locale "en_US.utf-8". logger.go:42: 00:02:28 | major-upgrade-logical-replica/5-upgrade | The default database encoding has accordingly been set to "UTF8". logger.go:42: 00:02:28 | major-upgrade-logical-replica/5-upgrade | The default text search configuration will be set to "english". logger.go:42: 00:02:28 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:28 | major-upgrade-logical-replica/5-upgrade | Data page checksums are enabled. logger.go:42: 00:02:28 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:28 | major-upgrade-logical-replica/5-upgrade | fixing permissions on existing directory /pgdata/pg18 ... ok logger.go:42: 00:02:28 | major-upgrade-logical-replica/5-upgrade | creating subdirectories ... ok logger.go:42: 00:02:28 | major-upgrade-logical-replica/5-upgrade | selecting dynamic shared memory implementation ... posix logger.go:42: 00:02:28 | major-upgrade-logical-replica/5-upgrade | selecting default "max_connections" ... 100 logger.go:42: 00:02:28 | major-upgrade-logical-replica/5-upgrade | selecting default "shared_buffers" ... 128MB logger.go:42: 00:02:28 | major-upgrade-logical-replica/5-upgrade | selecting default time zone ... UTC logger.go:42: 00:02:28 | major-upgrade-logical-replica/5-upgrade | creating configuration files ... ok logger.go:42: 00:02:28 | major-upgrade-logical-replica/5-upgrade | running bootstrap script ... ok logger.go:42: 00:02:28 | major-upgrade-logical-replica/5-upgrade | performing post-bootstrap initialization ... ok logger.go:42: 00:02:28 | major-upgrade-logical-replica/5-upgrade | syncing data to disk ... ok logger.go:42: 00:02:28 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:28 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:28 | major-upgrade-logical-replica/5-upgrade | Success. You can now start the database server using: logger.go:42: 00:02:28 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:28 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/pg_ctl -D /pgdata/pg18 -l logfile start logger.go:42: 00:02:28 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:28 | major-upgrade-logical-replica/5-upgrade | initdb: warning: enabling "trust" authentication for local connections logger.go:42: 00:02:28 | major-upgrade-logical-replica/5-upgrade | initdb: hint: You can change this by editing pg_hba.conf or using the option -A, or --auth-local and --auth-host, the next time you run initdb. logger.go:42: 00:02:28 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:28 | major-upgrade-logical-replica/5-upgrade | Step 3: Setting the expected permissions on the old pgdata directory... logger.go:42: 00:02:28 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:28 | major-upgrade-logical-replica/5-upgrade | Step 4: Copying shared_preload_libraries setting to new postgresql.conf file... logger.go:42: 00:02:28 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:28 | major-upgrade-logical-replica/5-upgrade | Step 4b: Copying logical replication settings to new postgresql.conf file... logger.go:42: 00:02:28 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:28 | major-upgrade-logical-replica/5-upgrade | Step 5: Running pg_upgrade check... logger.go:42: 00:02:28 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:28 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 00:02:28 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 00:02:28 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 00:02:28 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:02:28 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:02:28 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:02:28 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:02:28 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:02:28 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:02:28 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:02:28 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:02:28 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:02:28 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 00:02:28 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:02:28 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:02:28 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 00:02:28 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 00:02:28 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:28 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 00:02:28 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:28 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 00:02:28 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:28 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:28 | major-upgrade-logical-replica/5-upgrade | real 0m1.849s logger.go:42: 00:02:28 | major-upgrade-logical-replica/5-upgrade | user 0m0.090s logger.go:42: 00:02:28 | major-upgrade-logical-replica/5-upgrade | sys 0m0.084s logger.go:42: 00:02:28 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 00:02:28 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 00:02:28 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 00:02:28 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:02:28 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:02:28 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:02:28 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:02:28 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:02:28 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:02:28 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:02:28 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:02:28 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:02:28 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 00:02:31 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 00:02:31 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 00:02:32 | major-upgrade-logical-replica/5-upgrade | selecting default "max_connections" ... 100 logger.go:42: 00:02:32 | major-upgrade-logical-replica/5-upgrade | selecting default "shared_buffers" ... 128MB logger.go:42: 00:02:32 | major-upgrade-logical-replica/5-upgrade | selecting default time zone ... UTC logger.go:42: 00:02:32 | major-upgrade-logical-replica/5-upgrade | creating configuration files ... ok logger.go:42: 00:02:32 | major-upgrade-logical-replica/5-upgrade | running bootstrap script ... ok logger.go:42: 00:02:32 | major-upgrade-logical-replica/5-upgrade | performing post-bootstrap initialization ... ok logger.go:42: 00:02:32 | major-upgrade-logical-replica/5-upgrade | syncing data to disk ... ok logger.go:42: 00:02:32 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:32 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:32 | major-upgrade-logical-replica/5-upgrade | Success. You can now start the database server using: logger.go:42: 00:02:32 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:32 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/pg_ctl -D /pgdata/pg18 -l logfile start logger.go:42: 00:02:32 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:32 | major-upgrade-logical-replica/5-upgrade | initdb: warning: enabling "trust" authentication for local connections logger.go:42: 00:02:32 | major-upgrade-logical-replica/5-upgrade | initdb: hint: You can change this by editing pg_hba.conf or using the option -A, or --auth-local and --auth-host, the next time you run initdb. logger.go:42: 00:02:32 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:32 | major-upgrade-logical-replica/5-upgrade | Step 3: Setting the expected permissions on the old pgdata directory... logger.go:42: 00:02:32 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:32 | major-upgrade-logical-replica/5-upgrade | Step 4: Copying shared_preload_libraries setting to new postgresql.conf file... logger.go:42: 00:02:32 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:32 | major-upgrade-logical-replica/5-upgrade | Step 4b: Copying logical replication settings to new postgresql.conf file... logger.go:42: 00:02:32 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:32 | major-upgrade-logical-replica/5-upgrade | Step 5: Running pg_upgrade check... logger.go:42: 00:02:32 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:32 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 00:02:32 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 00:02:32 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 00:02:32 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:02:32 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:02:32 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:02:32 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:02:32 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:02:32 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:02:32 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:02:32 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:02:32 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:02:32 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 00:02:32 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:02:32 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:02:32 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 00:02:32 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 00:02:32 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:32 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 00:02:32 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:32 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 00:02:32 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:32 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:32 | major-upgrade-logical-replica/5-upgrade | real 0m1.849s logger.go:42: 00:02:32 | major-upgrade-logical-replica/5-upgrade | user 0m0.090s logger.go:42: 00:02:32 | major-upgrade-logical-replica/5-upgrade | sys 0m0.084s logger.go:42: 00:02:32 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 00:02:32 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 00:02:32 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 00:02:32 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:02:32 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:02:32 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:02:32 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:02:32 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:02:32 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:02:32 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:02:32 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:02:32 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:02:32 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 00:02:32 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 00:02:32 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 00:02:32 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:02:32 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:02:32 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 00:02:32 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 00:02:32 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:32 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 00:02:32 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 00:02:32 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:32 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 00:02:32 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 00:02:32 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 00:02:32 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 00:02:32 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 00:02:32 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 00:02:32 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 00:02:32 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 00:02:32 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 00:02:32 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 00:02:32 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 00:02:32 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 00:02:32 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 00:02:32 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 00:02:32 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 00:02:32 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 00:02:32 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 00:02:32 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 00:02:32 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 00:02:32 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:32 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 00:02:32 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 00:02:32 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 00:02:32 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 00:02:32 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 00:02:32 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 00:02:32 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 00:02:35 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 00:02:35 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 00:02:36 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:02:36 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:02:36 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:02:36 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:02:36 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:02:36 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:02:36 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:02:36 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:02:36 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:02:36 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 00:02:36 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:02:36 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:02:36 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 00:02:36 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 00:02:36 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:36 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 00:02:36 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:36 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 00:02:36 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:36 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:36 | major-upgrade-logical-replica/5-upgrade | real 0m1.849s logger.go:42: 00:02:36 | major-upgrade-logical-replica/5-upgrade | user 0m0.090s logger.go:42: 00:02:36 | major-upgrade-logical-replica/5-upgrade | sys 0m0.084s logger.go:42: 00:02:36 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 00:02:36 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 00:02:36 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 00:02:36 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:02:36 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:02:36 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:02:36 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:02:36 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:02:36 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:02:36 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:02:36 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:02:36 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:02:36 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 00:02:36 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 00:02:36 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 00:02:36 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:02:36 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:02:36 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 00:02:36 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 00:02:36 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:36 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 00:02:36 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 00:02:36 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:36 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 00:02:36 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 00:02:36 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 00:02:36 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 00:02:36 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 00:02:36 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 00:02:36 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 00:02:36 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 00:02:36 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 00:02:36 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 00:02:36 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 00:02:36 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 00:02:36 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 00:02:36 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 00:02:36 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 00:02:36 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 00:02:36 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 00:02:36 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 00:02:36 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 00:02:36 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:36 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 00:02:36 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 00:02:36 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 00:02:36 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 00:02:36 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 00:02:36 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 00:02:36 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 00:02:36 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 00:02:36 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 00:02:36 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 00:02:36 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:36 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 00:02:36 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 00:02:36 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 00:02:36 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 00:02:36 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 00:02:36 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:36 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 00:02:36 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 00:02:36 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 00:02:36 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 00:02:36 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 00:02:36 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 00:02:36 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 00:02:36 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 00:02:36 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:36 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 00:02:36 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:36 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:36 | major-upgrade-logical-replica/5-upgrade | real 0m5.830s logger.go:42: 00:02:36 | major-upgrade-logical-replica/5-upgrade | user 0m0.233s logger.go:42: 00:02:36 | major-upgrade-logical-replica/5-upgrade | sys 0m0.376s logger.go:42: 00:02:36 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:36 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 00:02:38 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 00:02:39 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 00:02:40 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:02:40 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:02:40 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:02:40 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:02:40 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:02:40 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:02:40 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:02:40 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:02:40 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:02:40 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 00:02:40 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:02:40 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:02:40 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 00:02:40 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 00:02:40 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:40 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 00:02:40 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:40 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 00:02:40 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:40 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:40 | major-upgrade-logical-replica/5-upgrade | real 0m1.849s logger.go:42: 00:02:40 | major-upgrade-logical-replica/5-upgrade | user 0m0.090s logger.go:42: 00:02:40 | major-upgrade-logical-replica/5-upgrade | sys 0m0.084s logger.go:42: 00:02:40 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 00:02:40 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 00:02:40 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 00:02:40 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:02:40 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:02:40 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:02:40 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:02:40 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:02:40 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:02:40 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:02:40 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:02:40 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:02:40 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 00:02:40 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 00:02:40 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 00:02:40 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:02:40 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:02:40 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 00:02:40 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 00:02:40 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:40 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 00:02:40 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 00:02:40 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:40 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 00:02:40 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 00:02:40 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 00:02:40 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 00:02:40 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 00:02:40 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 00:02:40 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 00:02:40 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 00:02:40 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 00:02:40 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 00:02:40 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 00:02:40 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 00:02:40 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 00:02:40 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 00:02:40 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 00:02:40 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 00:02:40 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 00:02:40 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 00:02:40 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 00:02:40 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:40 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 00:02:40 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 00:02:40 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 00:02:40 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 00:02:40 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 00:02:40 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 00:02:40 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 00:02:40 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 00:02:40 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 00:02:40 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 00:02:40 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:40 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 00:02:40 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 00:02:40 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 00:02:40 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 00:02:40 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 00:02:40 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:40 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 00:02:40 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 00:02:40 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 00:02:40 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 00:02:40 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 00:02:40 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 00:02:40 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 00:02:40 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 00:02:40 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:40 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 00:02:40 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:40 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:40 | major-upgrade-logical-replica/5-upgrade | real 0m5.830s logger.go:42: 00:02:40 | major-upgrade-logical-replica/5-upgrade | user 0m0.233s logger.go:42: 00:02:40 | major-upgrade-logical-replica/5-upgrade | sys 0m0.376s logger.go:42: 00:02:40 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:40 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 00:02:42 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 00:02:43 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 00:02:44 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:02:44 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:02:44 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:02:44 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:02:44 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:02:44 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:02:44 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:02:44 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:02:44 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:02:44 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 00:02:44 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:02:44 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:02:44 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 00:02:44 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 00:02:44 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:44 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 00:02:44 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:44 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 00:02:44 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:44 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:44 | major-upgrade-logical-replica/5-upgrade | real 0m1.849s logger.go:42: 00:02:44 | major-upgrade-logical-replica/5-upgrade | user 0m0.090s logger.go:42: 00:02:44 | major-upgrade-logical-replica/5-upgrade | sys 0m0.084s logger.go:42: 00:02:44 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 00:02:44 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 00:02:44 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 00:02:44 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:02:44 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:02:44 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:02:44 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:02:44 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:02:44 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:02:44 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:02:44 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:02:44 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:02:44 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 00:02:44 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 00:02:44 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 00:02:44 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:02:44 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:02:44 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 00:02:44 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 00:02:44 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:44 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 00:02:44 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 00:02:44 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:44 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 00:02:44 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 00:02:44 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 00:02:44 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 00:02:44 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 00:02:44 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 00:02:44 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 00:02:44 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 00:02:44 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 00:02:44 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 00:02:44 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 00:02:44 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 00:02:44 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 00:02:44 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 00:02:44 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 00:02:44 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 00:02:44 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 00:02:44 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 00:02:44 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 00:02:44 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:44 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 00:02:44 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 00:02:44 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 00:02:44 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 00:02:44 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 00:02:44 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 00:02:44 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 00:02:44 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 00:02:44 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 00:02:44 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 00:02:44 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:44 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 00:02:44 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 00:02:44 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 00:02:44 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 00:02:44 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 00:02:44 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:44 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 00:02:44 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 00:02:44 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 00:02:44 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 00:02:44 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 00:02:44 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 00:02:44 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 00:02:44 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 00:02:44 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:44 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 00:02:44 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:44 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:44 | major-upgrade-logical-replica/5-upgrade | real 0m5.830s logger.go:42: 00:02:44 | major-upgrade-logical-replica/5-upgrade | user 0m0.233s logger.go:42: 00:02:44 | major-upgrade-logical-replica/5-upgrade | sys 0m0.376s logger.go:42: 00:02:44 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:44 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 00:02:46 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 00:02:47 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 00:02:48 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:02:48 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:02:48 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:02:48 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:02:48 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:02:48 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:02:48 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:02:48 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:02:48 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:02:48 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 00:02:48 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:02:48 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:02:48 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 00:02:48 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 00:02:48 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:48 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 00:02:48 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:48 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 00:02:48 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:48 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:48 | major-upgrade-logical-replica/5-upgrade | real 0m1.849s logger.go:42: 00:02:48 | major-upgrade-logical-replica/5-upgrade | user 0m0.090s logger.go:42: 00:02:48 | major-upgrade-logical-replica/5-upgrade | sys 0m0.084s logger.go:42: 00:02:48 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 00:02:48 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 00:02:48 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 00:02:48 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:02:48 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:02:48 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:02:48 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:02:48 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:02:48 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:02:48 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:02:48 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:02:48 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:02:48 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 00:02:48 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 00:02:48 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 00:02:48 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:02:48 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:02:48 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 00:02:48 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 00:02:48 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:48 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 00:02:48 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 00:02:48 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:48 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 00:02:48 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 00:02:48 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 00:02:48 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 00:02:48 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 00:02:48 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 00:02:48 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 00:02:48 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 00:02:48 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 00:02:48 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 00:02:48 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 00:02:48 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 00:02:48 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 00:02:48 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 00:02:48 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 00:02:48 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 00:02:48 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 00:02:48 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 00:02:48 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 00:02:48 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:48 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 00:02:48 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 00:02:48 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 00:02:48 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 00:02:48 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 00:02:48 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 00:02:48 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 00:02:48 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 00:02:48 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 00:02:48 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 00:02:48 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:48 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 00:02:48 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 00:02:48 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 00:02:48 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 00:02:48 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 00:02:48 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:48 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 00:02:48 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 00:02:48 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 00:02:48 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 00:02:48 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 00:02:48 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 00:02:48 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 00:02:48 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 00:02:48 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:48 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 00:02:48 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:48 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:48 | major-upgrade-logical-replica/5-upgrade | real 0m5.830s logger.go:42: 00:02:48 | major-upgrade-logical-replica/5-upgrade | user 0m0.233s logger.go:42: 00:02:48 | major-upgrade-logical-replica/5-upgrade | sys 0m0.376s logger.go:42: 00:02:48 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:48 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 00:02:50 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 00:02:51 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 00:02:52 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:02:52 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:02:52 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:02:52 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:02:52 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:02:52 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:02:52 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:02:52 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:02:52 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:02:52 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 00:02:52 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:02:52 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:02:52 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 00:02:52 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 00:02:52 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:52 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 00:02:52 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:52 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 00:02:52 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:52 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:52 | major-upgrade-logical-replica/5-upgrade | real 0m1.849s logger.go:42: 00:02:52 | major-upgrade-logical-replica/5-upgrade | user 0m0.090s logger.go:42: 00:02:52 | major-upgrade-logical-replica/5-upgrade | sys 0m0.084s logger.go:42: 00:02:52 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 00:02:52 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 00:02:52 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 00:02:52 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:02:52 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:02:52 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:02:52 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:02:52 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:02:52 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:02:52 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:02:52 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:02:52 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:02:52 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 00:02:52 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 00:02:52 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 00:02:52 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:02:52 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:02:52 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 00:02:52 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 00:02:52 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:52 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 00:02:52 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 00:02:52 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:52 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 00:02:52 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 00:02:52 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 00:02:52 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 00:02:52 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 00:02:52 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 00:02:52 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 00:02:52 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 00:02:52 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 00:02:52 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 00:02:52 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 00:02:52 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 00:02:52 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 00:02:52 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 00:02:52 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 00:02:52 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 00:02:52 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 00:02:52 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 00:02:52 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 00:02:52 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:52 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 00:02:52 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 00:02:52 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 00:02:52 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 00:02:52 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 00:02:52 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 00:02:52 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 00:02:52 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 00:02:52 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 00:02:52 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 00:02:52 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:52 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 00:02:52 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 00:02:52 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 00:02:52 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 00:02:52 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 00:02:52 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:52 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 00:02:52 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 00:02:52 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 00:02:52 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 00:02:52 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 00:02:52 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 00:02:52 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 00:02:52 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 00:02:52 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:52 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 00:02:52 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:52 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:52 | major-upgrade-logical-replica/5-upgrade | real 0m5.830s logger.go:42: 00:02:52 | major-upgrade-logical-replica/5-upgrade | user 0m0.233s logger.go:42: 00:02:52 | major-upgrade-logical-replica/5-upgrade | sys 0m0.376s logger.go:42: 00:02:52 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:52 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 00:02:54 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 00:02:55 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 00:02:56 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:02:56 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:02:56 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:02:56 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:02:56 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:02:56 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:02:56 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:02:56 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:02:56 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:02:56 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 00:02:56 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:02:56 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:02:56 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 00:02:56 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 00:02:56 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:56 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 00:02:56 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:56 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 00:02:56 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:56 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:56 | major-upgrade-logical-replica/5-upgrade | real 0m1.849s logger.go:42: 00:02:56 | major-upgrade-logical-replica/5-upgrade | user 0m0.090s logger.go:42: 00:02:56 | major-upgrade-logical-replica/5-upgrade | sys 0m0.084s logger.go:42: 00:02:56 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 00:02:56 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 00:02:56 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 00:02:56 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:02:56 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:02:56 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:02:56 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:02:56 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:02:56 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:02:56 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:02:56 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:02:56 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:02:56 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 00:02:56 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 00:02:56 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 00:02:56 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:02:56 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:02:56 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 00:02:56 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 00:02:56 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:56 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 00:02:56 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 00:02:56 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:56 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 00:02:56 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 00:02:56 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 00:02:56 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 00:02:56 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 00:02:56 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 00:02:56 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 00:02:56 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 00:02:56 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 00:02:56 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 00:02:56 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 00:02:56 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 00:02:56 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 00:02:56 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 00:02:56 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 00:02:56 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 00:02:56 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 00:02:56 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 00:02:56 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 00:02:56 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:56 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 00:02:56 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 00:02:56 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 00:02:56 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 00:02:56 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 00:02:56 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 00:02:56 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 00:02:56 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 00:02:56 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 00:02:56 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 00:02:56 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:56 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 00:02:56 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 00:02:56 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 00:02:56 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 00:02:56 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 00:02:56 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:56 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 00:02:56 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 00:02:56 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 00:02:56 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 00:02:56 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 00:02:56 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 00:02:56 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 00:02:56 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 00:02:56 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:56 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 00:02:56 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:56 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:56 | major-upgrade-logical-replica/5-upgrade | real 0m5.830s logger.go:42: 00:02:56 | major-upgrade-logical-replica/5-upgrade | user 0m0.233s logger.go:42: 00:02:56 | major-upgrade-logical-replica/5-upgrade | sys 0m0.376s logger.go:42: 00:02:56 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:02:56 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 00:02:58 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 00:02:59 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 00:03:00 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:03:00 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:03:00 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:03:00 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:03:00 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:03:00 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:03:00 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:03:00 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:03:00 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:03:00 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 00:03:00 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:03:00 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:03:00 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 00:03:00 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 00:03:00 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:00 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 00:03:00 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:00 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 00:03:00 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:00 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:00 | major-upgrade-logical-replica/5-upgrade | real 0m1.849s logger.go:42: 00:03:00 | major-upgrade-logical-replica/5-upgrade | user 0m0.090s logger.go:42: 00:03:00 | major-upgrade-logical-replica/5-upgrade | sys 0m0.084s logger.go:42: 00:03:00 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 00:03:00 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 00:03:00 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 00:03:00 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:03:00 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:03:00 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:03:00 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:03:00 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:03:00 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:03:00 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:03:00 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:03:00 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:03:00 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 00:03:00 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 00:03:00 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 00:03:00 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:03:00 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:03:00 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 00:03:00 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 00:03:00 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:00 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 00:03:00 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 00:03:00 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:00 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 00:03:00 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 00:03:00 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 00:03:00 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 00:03:00 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 00:03:00 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 00:03:00 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 00:03:00 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 00:03:00 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 00:03:00 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 00:03:00 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 00:03:00 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 00:03:00 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 00:03:00 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 00:03:00 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 00:03:00 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 00:03:00 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 00:03:00 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 00:03:00 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 00:03:00 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:00 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 00:03:00 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 00:03:00 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 00:03:00 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 00:03:00 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 00:03:00 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 00:03:00 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 00:03:00 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 00:03:00 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 00:03:00 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 00:03:00 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:00 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 00:03:00 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 00:03:00 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 00:03:00 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 00:03:00 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 00:03:00 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:00 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 00:03:00 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 00:03:00 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 00:03:00 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 00:03:00 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 00:03:00 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 00:03:00 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 00:03:00 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 00:03:00 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:00 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 00:03:00 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:00 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:00 | major-upgrade-logical-replica/5-upgrade | real 0m5.830s logger.go:42: 00:03:00 | major-upgrade-logical-replica/5-upgrade | user 0m0.233s logger.go:42: 00:03:00 | major-upgrade-logical-replica/5-upgrade | sys 0m0.376s logger.go:42: 00:03:00 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:00 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 00:03:02 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 00:03:03 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 00:03:04 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:03:04 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:03:04 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:03:04 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:03:04 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:03:04 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:03:04 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:03:04 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:03:04 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:03:04 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 00:03:04 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:03:04 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:03:04 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 00:03:04 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 00:03:04 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:04 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 00:03:04 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:04 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 00:03:04 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:04 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:04 | major-upgrade-logical-replica/5-upgrade | real 0m1.849s logger.go:42: 00:03:04 | major-upgrade-logical-replica/5-upgrade | user 0m0.090s logger.go:42: 00:03:04 | major-upgrade-logical-replica/5-upgrade | sys 0m0.084s logger.go:42: 00:03:04 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 00:03:04 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 00:03:04 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 00:03:04 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:03:04 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:03:04 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:03:04 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:03:04 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:03:04 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:03:04 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:03:04 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:03:04 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:03:04 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 00:03:04 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 00:03:04 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 00:03:04 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:03:04 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:03:04 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 00:03:04 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 00:03:04 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:04 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 00:03:04 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 00:03:04 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:04 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 00:03:04 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 00:03:04 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 00:03:04 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 00:03:04 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 00:03:04 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 00:03:04 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 00:03:04 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 00:03:04 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 00:03:04 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 00:03:04 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 00:03:04 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 00:03:04 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 00:03:04 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 00:03:04 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 00:03:04 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 00:03:04 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 00:03:04 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 00:03:04 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 00:03:04 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:04 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 00:03:04 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 00:03:04 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 00:03:04 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 00:03:04 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 00:03:04 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 00:03:04 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 00:03:04 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 00:03:04 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 00:03:04 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 00:03:04 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:04 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 00:03:04 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 00:03:04 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 00:03:04 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 00:03:04 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 00:03:04 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:04 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 00:03:04 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 00:03:04 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 00:03:04 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 00:03:04 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 00:03:04 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 00:03:04 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 00:03:04 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 00:03:04 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:04 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 00:03:04 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:04 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:04 | major-upgrade-logical-replica/5-upgrade | real 0m5.830s logger.go:42: 00:03:04 | major-upgrade-logical-replica/5-upgrade | user 0m0.233s logger.go:42: 00:03:04 | major-upgrade-logical-replica/5-upgrade | sys 0m0.376s logger.go:42: 00:03:04 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:04 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 00:03:06 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 00:03:07 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 00:03:08 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:03:08 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:03:08 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:03:08 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:03:08 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:03:08 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:03:08 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:03:08 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:03:08 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:03:08 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 00:03:08 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:03:08 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:03:08 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 00:03:08 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 00:03:08 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:08 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 00:03:08 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:08 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 00:03:08 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:08 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:08 | major-upgrade-logical-replica/5-upgrade | real 0m1.849s logger.go:42: 00:03:08 | major-upgrade-logical-replica/5-upgrade | user 0m0.090s logger.go:42: 00:03:08 | major-upgrade-logical-replica/5-upgrade | sys 0m0.084s logger.go:42: 00:03:08 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 00:03:08 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 00:03:08 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 00:03:08 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:03:08 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:03:08 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:03:08 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:03:08 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:03:08 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:03:08 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:03:08 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:03:08 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:03:08 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 00:03:08 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 00:03:08 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 00:03:08 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:03:08 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:03:08 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 00:03:08 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 00:03:08 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:08 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 00:03:08 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 00:03:08 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:08 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 00:03:08 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 00:03:08 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 00:03:08 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 00:03:08 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 00:03:08 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 00:03:08 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 00:03:08 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 00:03:08 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 00:03:08 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 00:03:08 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 00:03:08 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 00:03:08 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 00:03:08 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 00:03:08 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 00:03:08 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 00:03:08 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 00:03:08 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 00:03:08 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 00:03:08 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:08 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 00:03:08 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 00:03:08 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 00:03:08 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 00:03:08 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 00:03:08 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 00:03:08 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 00:03:08 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 00:03:08 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 00:03:08 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 00:03:08 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:08 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 00:03:08 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 00:03:08 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 00:03:08 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 00:03:08 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 00:03:08 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:08 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 00:03:08 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 00:03:08 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 00:03:08 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 00:03:08 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 00:03:08 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 00:03:08 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 00:03:08 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 00:03:08 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:08 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 00:03:08 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:08 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:08 | major-upgrade-logical-replica/5-upgrade | real 0m5.830s logger.go:42: 00:03:08 | major-upgrade-logical-replica/5-upgrade | user 0m0.233s logger.go:42: 00:03:08 | major-upgrade-logical-replica/5-upgrade | sys 0m0.376s logger.go:42: 00:03:08 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:08 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 00:03:10 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 00:03:11 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 00:03:12 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:03:12 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:03:12 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:03:12 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:03:12 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:03:12 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:03:12 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:03:12 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:03:12 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:03:12 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 00:03:12 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:03:12 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:03:12 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 00:03:12 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 00:03:12 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:12 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 00:03:12 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:12 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 00:03:12 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:12 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:12 | major-upgrade-logical-replica/5-upgrade | real 0m1.849s logger.go:42: 00:03:12 | major-upgrade-logical-replica/5-upgrade | user 0m0.090s logger.go:42: 00:03:12 | major-upgrade-logical-replica/5-upgrade | sys 0m0.084s logger.go:42: 00:03:12 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 00:03:12 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 00:03:12 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 00:03:12 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:03:12 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:03:12 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:03:12 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:03:12 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:03:12 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:03:12 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:03:12 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:03:12 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:03:12 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 00:03:12 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 00:03:12 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 00:03:12 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:03:12 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:03:12 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 00:03:12 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 00:03:12 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:12 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 00:03:12 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 00:03:12 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:12 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 00:03:12 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 00:03:12 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 00:03:12 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 00:03:12 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 00:03:12 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 00:03:12 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 00:03:12 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 00:03:12 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 00:03:12 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 00:03:12 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 00:03:12 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 00:03:12 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 00:03:12 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 00:03:12 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 00:03:12 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 00:03:12 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 00:03:12 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 00:03:12 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 00:03:12 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:12 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 00:03:12 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 00:03:12 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 00:03:12 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 00:03:12 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 00:03:12 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 00:03:12 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 00:03:12 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 00:03:12 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 00:03:12 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 00:03:12 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:12 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 00:03:12 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 00:03:12 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 00:03:12 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 00:03:12 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 00:03:12 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:12 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 00:03:12 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 00:03:12 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 00:03:12 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 00:03:12 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 00:03:12 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 00:03:12 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 00:03:12 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 00:03:12 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:12 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 00:03:12 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:12 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:12 | major-upgrade-logical-replica/5-upgrade | real 0m5.830s logger.go:42: 00:03:12 | major-upgrade-logical-replica/5-upgrade | user 0m0.233s logger.go:42: 00:03:12 | major-upgrade-logical-replica/5-upgrade | sys 0m0.376s logger.go:42: 00:03:12 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:12 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 00:03:14 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 00:03:15 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 00:03:16 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:03:16 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:03:16 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:03:16 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:03:16 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:03:16 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:03:16 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:03:16 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:03:16 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:03:16 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 00:03:16 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:03:16 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:03:16 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 00:03:16 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 00:03:16 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:16 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 00:03:16 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:16 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 00:03:16 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:16 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:16 | major-upgrade-logical-replica/5-upgrade | real 0m1.849s logger.go:42: 00:03:16 | major-upgrade-logical-replica/5-upgrade | user 0m0.090s logger.go:42: 00:03:16 | major-upgrade-logical-replica/5-upgrade | sys 0m0.084s logger.go:42: 00:03:16 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 00:03:16 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 00:03:16 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 00:03:16 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:03:16 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:03:16 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:03:16 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:03:16 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:03:16 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:03:16 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:03:16 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:03:16 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:03:16 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 00:03:16 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 00:03:16 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 00:03:16 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:03:16 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:03:16 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 00:03:16 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 00:03:16 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:16 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 00:03:16 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 00:03:16 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:16 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 00:03:16 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 00:03:16 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 00:03:16 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 00:03:16 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 00:03:16 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 00:03:16 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 00:03:16 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 00:03:16 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 00:03:16 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 00:03:16 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 00:03:16 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 00:03:16 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 00:03:16 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 00:03:16 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 00:03:16 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 00:03:16 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 00:03:16 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 00:03:16 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 00:03:16 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:16 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 00:03:16 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 00:03:16 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 00:03:16 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 00:03:16 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 00:03:16 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 00:03:16 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 00:03:16 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 00:03:16 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 00:03:16 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 00:03:16 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:16 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 00:03:16 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 00:03:16 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 00:03:16 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 00:03:16 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 00:03:16 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:16 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 00:03:16 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 00:03:16 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 00:03:16 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 00:03:16 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 00:03:16 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 00:03:16 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 00:03:16 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 00:03:16 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:16 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 00:03:16 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:16 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:16 | major-upgrade-logical-replica/5-upgrade | real 0m5.830s logger.go:42: 00:03:16 | major-upgrade-logical-replica/5-upgrade | user 0m0.233s logger.go:42: 00:03:16 | major-upgrade-logical-replica/5-upgrade | sys 0m0.376s logger.go:42: 00:03:16 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:16 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 00:03:18 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 00:03:19 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 00:03:20 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:03:20 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:03:20 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:03:20 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:03:20 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:03:20 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:03:20 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:03:20 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:03:20 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:03:20 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 00:03:20 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:03:20 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:03:20 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 00:03:20 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 00:03:20 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:20 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 00:03:20 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:20 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 00:03:20 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:20 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:20 | major-upgrade-logical-replica/5-upgrade | real 0m1.849s logger.go:42: 00:03:20 | major-upgrade-logical-replica/5-upgrade | user 0m0.090s logger.go:42: 00:03:20 | major-upgrade-logical-replica/5-upgrade | sys 0m0.084s logger.go:42: 00:03:20 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 00:03:20 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 00:03:20 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 00:03:20 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:03:20 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:03:20 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:03:20 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:03:20 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:03:20 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:03:20 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:03:20 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:03:20 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:03:20 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 00:03:20 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 00:03:20 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 00:03:20 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:03:20 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:03:20 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 00:03:20 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 00:03:20 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:20 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 00:03:20 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 00:03:20 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:20 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 00:03:20 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 00:03:20 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 00:03:20 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 00:03:20 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 00:03:20 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 00:03:20 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 00:03:20 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 00:03:20 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 00:03:20 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 00:03:20 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 00:03:20 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 00:03:20 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 00:03:20 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 00:03:20 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 00:03:20 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 00:03:20 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 00:03:20 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 00:03:20 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 00:03:20 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:20 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 00:03:20 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 00:03:20 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 00:03:20 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 00:03:20 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 00:03:20 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 00:03:20 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 00:03:20 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 00:03:20 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 00:03:20 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 00:03:20 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:20 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 00:03:20 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 00:03:20 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 00:03:20 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 00:03:20 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 00:03:20 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:20 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 00:03:20 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 00:03:20 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 00:03:20 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 00:03:20 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 00:03:20 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 00:03:20 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 00:03:20 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 00:03:20 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:20 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 00:03:20 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:20 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:20 | major-upgrade-logical-replica/5-upgrade | real 0m5.830s logger.go:42: 00:03:20 | major-upgrade-logical-replica/5-upgrade | user 0m0.233s logger.go:42: 00:03:20 | major-upgrade-logical-replica/5-upgrade | sys 0m0.376s logger.go:42: 00:03:20 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:20 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 00:03:22 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 00:03:23 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 00:03:24 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:03:24 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:03:24 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:03:24 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:03:24 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:03:24 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:03:24 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:03:24 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:03:24 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:03:24 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 00:03:24 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:03:24 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:03:24 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 00:03:24 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 00:03:24 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:24 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 00:03:24 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:24 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 00:03:24 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:24 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:24 | major-upgrade-logical-replica/5-upgrade | real 0m1.849s logger.go:42: 00:03:24 | major-upgrade-logical-replica/5-upgrade | user 0m0.090s logger.go:42: 00:03:24 | major-upgrade-logical-replica/5-upgrade | sys 0m0.084s logger.go:42: 00:03:24 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 00:03:24 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 00:03:24 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 00:03:24 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:03:24 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:03:24 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:03:24 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:03:24 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:03:24 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:03:24 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:03:24 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:03:24 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:03:24 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 00:03:24 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 00:03:24 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 00:03:24 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:03:24 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:03:24 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 00:03:24 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 00:03:24 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:24 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 00:03:24 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 00:03:24 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:24 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 00:03:24 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 00:03:24 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 00:03:24 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 00:03:24 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 00:03:24 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 00:03:24 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 00:03:24 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 00:03:24 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 00:03:24 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 00:03:24 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 00:03:24 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 00:03:24 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 00:03:24 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 00:03:24 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 00:03:24 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 00:03:24 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 00:03:24 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 00:03:24 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 00:03:24 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:24 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 00:03:24 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 00:03:24 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 00:03:24 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 00:03:24 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 00:03:24 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 00:03:24 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 00:03:24 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 00:03:24 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 00:03:24 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 00:03:24 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:24 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 00:03:24 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 00:03:24 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 00:03:24 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 00:03:24 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 00:03:24 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:24 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 00:03:24 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 00:03:24 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 00:03:24 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 00:03:24 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 00:03:24 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 00:03:24 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 00:03:24 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 00:03:24 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:24 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 00:03:24 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:24 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:24 | major-upgrade-logical-replica/5-upgrade | real 0m5.830s logger.go:42: 00:03:24 | major-upgrade-logical-replica/5-upgrade | user 0m0.233s logger.go:42: 00:03:24 | major-upgrade-logical-replica/5-upgrade | sys 0m0.376s logger.go:42: 00:03:24 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:24 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 00:03:26 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 00:03:27 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 00:03:28 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:03:28 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:03:28 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:03:28 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:03:28 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:03:28 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:03:28 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:03:28 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:03:28 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:03:28 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 00:03:28 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:03:28 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:03:28 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 00:03:28 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 00:03:28 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:28 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 00:03:28 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:28 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 00:03:28 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:28 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:28 | major-upgrade-logical-replica/5-upgrade | real 0m1.849s logger.go:42: 00:03:28 | major-upgrade-logical-replica/5-upgrade | user 0m0.090s logger.go:42: 00:03:28 | major-upgrade-logical-replica/5-upgrade | sys 0m0.084s logger.go:42: 00:03:28 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 00:03:28 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 00:03:28 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 00:03:28 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:03:28 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:03:28 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:03:28 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:03:28 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:03:28 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:03:28 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:03:28 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:03:28 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:03:28 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 00:03:28 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 00:03:28 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 00:03:28 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:03:28 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:03:28 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 00:03:28 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 00:03:28 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:28 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 00:03:28 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 00:03:28 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:28 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 00:03:28 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 00:03:28 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 00:03:28 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 00:03:28 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 00:03:28 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 00:03:28 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 00:03:28 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 00:03:28 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 00:03:28 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 00:03:28 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 00:03:28 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 00:03:28 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 00:03:28 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 00:03:28 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 00:03:28 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 00:03:28 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 00:03:28 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 00:03:28 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 00:03:28 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:28 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 00:03:28 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 00:03:28 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 00:03:28 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 00:03:28 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 00:03:28 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 00:03:28 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 00:03:28 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 00:03:28 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 00:03:28 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 00:03:28 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:28 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 00:03:28 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 00:03:28 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 00:03:28 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 00:03:28 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 00:03:28 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:28 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 00:03:28 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 00:03:28 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 00:03:28 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 00:03:28 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 00:03:28 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 00:03:28 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 00:03:28 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 00:03:28 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:28 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 00:03:28 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:28 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:28 | major-upgrade-logical-replica/5-upgrade | real 0m5.830s logger.go:42: 00:03:28 | major-upgrade-logical-replica/5-upgrade | user 0m0.233s logger.go:42: 00:03:28 | major-upgrade-logical-replica/5-upgrade | sys 0m0.376s logger.go:42: 00:03:28 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:28 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 00:03:30 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 00:03:31 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 00:03:32 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:03:32 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:03:32 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:03:32 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:03:32 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:03:32 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:03:32 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:03:32 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:03:32 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:03:32 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 00:03:32 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:03:32 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:03:32 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 00:03:32 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 00:03:32 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:32 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 00:03:32 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:32 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 00:03:32 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:32 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:32 | major-upgrade-logical-replica/5-upgrade | real 0m1.849s logger.go:42: 00:03:32 | major-upgrade-logical-replica/5-upgrade | user 0m0.090s logger.go:42: 00:03:32 | major-upgrade-logical-replica/5-upgrade | sys 0m0.084s logger.go:42: 00:03:32 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 00:03:32 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 00:03:32 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 00:03:32 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:03:32 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:03:32 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:03:32 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:03:32 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:03:32 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:03:32 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:03:32 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:03:32 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:03:32 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 00:03:32 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 00:03:32 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 00:03:32 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:03:32 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:03:32 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 00:03:32 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 00:03:32 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:32 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 00:03:32 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 00:03:32 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:32 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 00:03:32 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 00:03:32 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 00:03:32 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 00:03:32 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 00:03:32 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 00:03:32 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 00:03:32 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 00:03:32 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 00:03:32 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 00:03:32 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 00:03:32 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 00:03:32 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 00:03:32 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 00:03:32 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 00:03:32 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 00:03:32 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 00:03:32 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 00:03:32 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 00:03:32 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:32 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 00:03:32 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 00:03:32 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 00:03:32 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 00:03:32 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 00:03:32 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 00:03:32 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 00:03:32 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 00:03:32 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 00:03:32 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 00:03:32 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:32 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 00:03:32 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 00:03:32 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 00:03:32 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 00:03:32 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 00:03:32 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:32 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 00:03:32 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 00:03:32 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 00:03:32 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 00:03:32 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 00:03:32 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 00:03:32 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 00:03:32 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 00:03:32 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:32 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 00:03:32 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:32 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:32 | major-upgrade-logical-replica/5-upgrade | real 0m5.830s logger.go:42: 00:03:32 | major-upgrade-logical-replica/5-upgrade | user 0m0.233s logger.go:42: 00:03:32 | major-upgrade-logical-replica/5-upgrade | sys 0m0.376s logger.go:42: 00:03:32 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:32 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 00:03:34 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 00:03:35 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 00:03:36 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:03:36 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:03:36 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:03:36 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:03:36 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:03:36 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:03:36 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:03:36 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:03:36 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:03:36 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 00:03:36 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:03:36 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:03:36 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 00:03:36 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 00:03:36 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:36 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 00:03:36 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:36 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 00:03:36 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:36 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:36 | major-upgrade-logical-replica/5-upgrade | real 0m1.849s logger.go:42: 00:03:36 | major-upgrade-logical-replica/5-upgrade | user 0m0.090s logger.go:42: 00:03:36 | major-upgrade-logical-replica/5-upgrade | sys 0m0.084s logger.go:42: 00:03:36 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 00:03:36 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 00:03:36 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 00:03:36 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:03:36 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:03:36 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:03:36 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:03:36 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:03:36 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:03:36 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:03:36 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:03:36 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:03:36 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 00:03:36 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 00:03:36 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 00:03:36 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:03:36 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:03:36 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 00:03:36 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 00:03:36 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:36 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 00:03:36 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 00:03:36 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:36 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 00:03:36 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 00:03:36 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 00:03:36 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 00:03:36 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 00:03:36 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 00:03:36 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 00:03:36 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 00:03:36 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 00:03:36 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 00:03:36 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 00:03:36 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 00:03:36 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 00:03:36 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 00:03:36 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 00:03:36 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 00:03:36 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 00:03:36 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 00:03:36 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 00:03:36 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:36 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 00:03:36 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 00:03:36 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 00:03:36 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 00:03:36 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 00:03:36 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 00:03:36 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 00:03:36 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 00:03:36 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 00:03:36 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 00:03:36 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:36 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 00:03:36 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 00:03:36 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 00:03:36 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 00:03:36 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 00:03:36 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:36 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 00:03:36 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 00:03:36 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 00:03:36 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 00:03:36 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 00:03:36 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 00:03:36 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 00:03:36 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 00:03:36 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:36 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 00:03:36 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:36 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:36 | major-upgrade-logical-replica/5-upgrade | real 0m5.830s logger.go:42: 00:03:36 | major-upgrade-logical-replica/5-upgrade | user 0m0.233s logger.go:42: 00:03:36 | major-upgrade-logical-replica/5-upgrade | sys 0m0.376s logger.go:42: 00:03:36 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:36 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 00:03:38 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 00:03:38 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 00:03:39 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:03:39 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:03:39 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:03:39 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:03:39 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:03:39 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:03:39 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:03:39 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:03:39 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:03:39 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 00:03:39 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:03:39 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:03:39 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 00:03:39 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 00:03:39 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:39 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 00:03:39 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:39 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 00:03:39 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:39 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:39 | major-upgrade-logical-replica/5-upgrade | real 0m1.849s logger.go:42: 00:03:39 | major-upgrade-logical-replica/5-upgrade | user 0m0.090s logger.go:42: 00:03:39 | major-upgrade-logical-replica/5-upgrade | sys 0m0.084s logger.go:42: 00:03:39 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 00:03:39 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 00:03:39 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 00:03:39 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:03:39 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:03:39 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:03:39 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:03:39 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:03:39 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:03:39 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:03:39 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:03:39 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:03:39 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 00:03:39 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 00:03:39 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 00:03:39 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:03:39 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:03:39 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 00:03:39 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 00:03:39 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:39 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 00:03:39 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 00:03:39 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:39 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 00:03:39 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 00:03:39 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 00:03:39 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 00:03:39 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 00:03:39 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 00:03:39 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 00:03:39 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 00:03:39 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 00:03:39 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 00:03:39 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 00:03:39 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 00:03:39 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 00:03:39 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 00:03:39 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 00:03:39 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 00:03:39 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 00:03:39 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 00:03:39 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 00:03:39 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:39 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 00:03:39 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 00:03:39 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 00:03:39 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 00:03:39 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 00:03:39 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 00:03:39 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 00:03:39 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 00:03:39 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 00:03:39 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 00:03:39 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:39 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 00:03:39 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 00:03:39 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 00:03:39 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 00:03:39 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 00:03:39 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:39 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 00:03:39 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 00:03:39 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 00:03:39 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 00:03:39 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 00:03:39 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 00:03:39 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 00:03:39 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 00:03:39 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:39 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 00:03:39 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:39 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:39 | major-upgrade-logical-replica/5-upgrade | real 0m5.830s logger.go:42: 00:03:39 | major-upgrade-logical-replica/5-upgrade | user 0m0.233s logger.go:42: 00:03:39 | major-upgrade-logical-replica/5-upgrade | sys 0m0.376s logger.go:42: 00:03:39 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:39 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 00:03:42 | major-upgrade-logical-replica/5-upgrade | running command: [sh -c set -o errexit job=$(kubectl -n "${NAMESPACE}" get job \ -l postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica \ -l postgres-operator.crunchydata.com/role=pgupgrade \ -o name | head -1) if [[ -n ${job} ]]; then echo "--- pg_upgrade job log ---" kubectl -n "${NAMESPACE}" logs "${job}" --tail=100 || true if kubectl -n "${NAMESPACE}" logs "${job}" 2>/dev/null | grep -q 'must be "logical"'; then echo 'ERROR: pg_upgrade rejected the new cluster over wal_level' exit 1 fi fi] logger.go:42: 00:03:42 | major-upgrade-logical-replica/5-upgrade | --- pg_upgrade job log --- logger.go:42: 00:03:43 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:03:43 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:03:43 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:03:43 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:03:43 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:03:43 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:03:43 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:03:43 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:03:43 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:03:43 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 00:03:43 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:03:43 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:03:43 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 00:03:43 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 00:03:43 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:43 | major-upgrade-logical-replica/5-upgrade | *Clusters are compatible* logger.go:42: 00:03:43 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:43 | major-upgrade-logical-replica/5-upgrade | Step 6: Running pg_upgrade... logger.go:42: 00:03:43 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:43 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:43 | major-upgrade-logical-replica/5-upgrade | real 0m1.849s logger.go:42: 00:03:43 | major-upgrade-logical-replica/5-upgrade | user 0m0.090s logger.go:42: 00:03:43 | major-upgrade-logical-replica/5-upgrade | sys 0m0.084s logger.go:42: 00:03:43 | major-upgrade-logical-replica/5-upgrade | Performing Consistency Checks logger.go:42: 00:03:43 | major-upgrade-logical-replica/5-upgrade | ----------------------------- logger.go:42: 00:03:43 | major-upgrade-logical-replica/5-upgrade | Checking cluster versions ok logger.go:42: 00:03:43 | major-upgrade-logical-replica/5-upgrade | Checking database connection settings ok logger.go:42: 00:03:43 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:03:43 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:03:43 | major-upgrade-logical-replica/5-upgrade | Checking for contrib/isn with bigint-passing mismatch ok logger.go:42: 00:03:43 | major-upgrade-logical-replica/5-upgrade | Checking for valid logical replication slots ok logger.go:42: 00:03:43 | major-upgrade-logical-replica/5-upgrade | Checking for subscription state ok logger.go:42: 00:03:43 | major-upgrade-logical-replica/5-upgrade | Checking data type usage ok logger.go:42: 00:03:43 | major-upgrade-logical-replica/5-upgrade | Checking for objects affected by Unicode update ok logger.go:42: 00:03:43 | major-upgrade-logical-replica/5-upgrade | Checking for not-null constraint inconsistencies ok logger.go:42: 00:03:43 | major-upgrade-logical-replica/5-upgrade | Creating dump of global objects ok logger.go:42: 00:03:43 | major-upgrade-logical-replica/5-upgrade | Creating dump of database schemas ok logger.go:42: 00:03:43 | major-upgrade-logical-replica/5-upgrade | Checking for presence of required libraries ok logger.go:42: 00:03:43 | major-upgrade-logical-replica/5-upgrade | Checking database user is the install user ok logger.go:42: 00:03:43 | major-upgrade-logical-replica/5-upgrade | Checking for prepared transactions ok logger.go:42: 00:03:43 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster tablespace directories ok logger.go:42: 00:03:43 | major-upgrade-logical-replica/5-upgrade | Checking for new cluster logical replication slots ok logger.go:42: 00:03:43 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:43 | major-upgrade-logical-replica/5-upgrade | If pg_upgrade fails after this point, you must re-initdb the logger.go:42: 00:03:43 | major-upgrade-logical-replica/5-upgrade | new cluster before continuing. logger.go:42: 00:03:43 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:43 | major-upgrade-logical-replica/5-upgrade | Performing Upgrade logger.go:42: 00:03:43 | major-upgrade-logical-replica/5-upgrade | ------------------ logger.go:42: 00:03:43 | major-upgrade-logical-replica/5-upgrade | Setting locale and encoding for new cluster ok logger.go:42: 00:03:43 | major-upgrade-logical-replica/5-upgrade | Analyzing all rows in the new cluster ok logger.go:42: 00:03:43 | major-upgrade-logical-replica/5-upgrade | Freezing all rows in the new cluster ok logger.go:42: 00:03:43 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_xact ok logger.go:42: 00:03:43 | major-upgrade-logical-replica/5-upgrade | Copying old pg_xact to new server ok logger.go:42: 00:03:43 | major-upgrade-logical-replica/5-upgrade | Setting oldest XID for new cluster ok logger.go:42: 00:03:43 | major-upgrade-logical-replica/5-upgrade | Setting next transaction ID and epoch for new cluster ok logger.go:42: 00:03:43 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/offsets ok logger.go:42: 00:03:43 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/offsets to new server ok logger.go:42: 00:03:43 | major-upgrade-logical-replica/5-upgrade | Deleting files from new pg_multixact/members ok logger.go:42: 00:03:43 | major-upgrade-logical-replica/5-upgrade | Copying old pg_multixact/members to new server ok logger.go:42: 00:03:43 | major-upgrade-logical-replica/5-upgrade | Setting next multixact ID and offset for new cluster ok logger.go:42: 00:03:43 | major-upgrade-logical-replica/5-upgrade | Resetting WAL archives ok logger.go:42: 00:03:43 | major-upgrade-logical-replica/5-upgrade | Setting frozenxid and minmxid counters in new cluster ok logger.go:42: 00:03:43 | major-upgrade-logical-replica/5-upgrade | Restoring global objects in the new cluster ok logger.go:42: 00:03:43 | major-upgrade-logical-replica/5-upgrade | Restoring database schemas in the new cluster ok logger.go:42: 00:03:43 | major-upgrade-logical-replica/5-upgrade | Adding ".old" suffix to old "global/pg_control" ok logger.go:42: 00:03:43 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:43 | major-upgrade-logical-replica/5-upgrade | If you want to start the old cluster, you will need to remove logger.go:42: 00:03:43 | major-upgrade-logical-replica/5-upgrade | the ".old" suffix from "/pgdata/pg17/global/pg_control.old". logger.go:42: 00:03:43 | major-upgrade-logical-replica/5-upgrade | Because "link" mode was used, the old cluster cannot be safely logger.go:42: 00:03:43 | major-upgrade-logical-replica/5-upgrade | started once the new cluster has been started. logger.go:42: 00:03:43 | major-upgrade-logical-replica/5-upgrade | Linking user relation files ok logger.go:42: 00:03:43 | major-upgrade-logical-replica/5-upgrade | Setting next OID for new cluster ok logger.go:42: 00:03:43 | major-upgrade-logical-replica/5-upgrade | Restoring logical replication slots in the new cluster ok logger.go:42: 00:03:43 | major-upgrade-logical-replica/5-upgrade | Sync data directory to disk ok logger.go:42: 00:03:43 | major-upgrade-logical-replica/5-upgrade | Creating script to delete old cluster ok logger.go:42: 00:03:43 | major-upgrade-logical-replica/5-upgrade | Checking for extension updates notice logger.go:42: 00:03:43 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:43 | major-upgrade-logical-replica/5-upgrade | Your installation contains extensions that should be updated logger.go:42: 00:03:43 | major-upgrade-logical-replica/5-upgrade | with the ALTER EXTENSION command. The file logger.go:42: 00:03:43 | major-upgrade-logical-replica/5-upgrade | update_extensions.sql logger.go:42: 00:03:43 | major-upgrade-logical-replica/5-upgrade | when executed by psql by the database superuser will update logger.go:42: 00:03:43 | major-upgrade-logical-replica/5-upgrade | these extensions. logger.go:42: 00:03:43 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:43 | major-upgrade-logical-replica/5-upgrade | Upgrade Complete logger.go:42: 00:03:43 | major-upgrade-logical-replica/5-upgrade | ---------------- logger.go:42: 00:03:43 | major-upgrade-logical-replica/5-upgrade | Some statistics are not transferred by pg_upgrade. logger.go:42: 00:03:43 | major-upgrade-logical-replica/5-upgrade | Once you start the new server, consider running these two commands: logger.go:42: 00:03:43 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-in-stages --missing-stats-only logger.go:42: 00:03:43 | major-upgrade-logical-replica/5-upgrade | /usr/pgsql-18/bin/vacuumdb --all --analyze-only logger.go:42: 00:03:43 | major-upgrade-logical-replica/5-upgrade | Running this script will delete the old cluster's data files: logger.go:42: 00:03:43 | major-upgrade-logical-replica/5-upgrade | ./delete_old_cluster.sh logger.go:42: 00:03:43 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:43 | major-upgrade-logical-replica/5-upgrade | Step 7: Copying patroni.dynamic.json... logger.go:42: 00:03:43 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:43 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:43 | major-upgrade-logical-replica/5-upgrade | real 0m5.830s logger.go:42: 00:03:43 | major-upgrade-logical-replica/5-upgrade | user 0m0.233s logger.go:42: 00:03:43 | major-upgrade-logical-replica/5-upgrade | sys 0m0.376s logger.go:42: 00:03:43 | major-upgrade-logical-replica/5-upgrade | logger.go:42: 00:03:43 | major-upgrade-logical-replica/5-upgrade | pg_upgrade Job Complete! logger.go:42: 00:03:44 | major-upgrade-logical-replica/5-upgrade | test step completed 5-upgrade logger.go:42: 00:03:44 | major-upgrade-logical-replica/6-verify-replica-invalidated | starting test step 6-verify-replica-invalidated logger.go:42: 00:03:44 | major-upgrade-logical-replica/6-verify-replica-invalidated | running command: [sh -c set -o errexit set -o xtrace source ../../functions primary=$(get_pod_by_role major-upgrade-logical-replica primary name) kubectl -n "${NAMESPACE}" exec "${primary}" -- /pgdata/delete_old_cluster.sh # pg_upgrade only rewrites the data directories of the instance sets, so the # replica still holds a PostgreSQL 17 one. It has to be reported as broken # rather than handed the new binary and left to crash. retry 60 10 bash -c ' state=$(kubectl -n "'"${NAMESPACE}"'" get pg major-upgrade-logical-replica \ -o jsonpath="{.status.logicalReplicas[?(@.name==\"analytics\")].state}") reason=$(kubectl -n "'"${NAMESPACE}"'" get pg major-upgrade-logical-replica \ -o jsonpath="{.status.logicalReplicas[?(@.name==\"analytics\")].reason}") echo "logical replica analytics: ${state}/${reason}" [[ ${state} == "broken" && ${reason} == "SourceUpgraded" ]] ' message=$(kubectl -n "${NAMESPACE}" get pg major-upgrade-logical-replica \ -o jsonpath='{.status.logicalReplicas[?(@.name=="analytics")].message}') echo "message: ${message}" echo "${message}" | grep -q "remove it from spec.logicalReplicas" # The StatefulSet is stopped, so nothing is left crash-looping against a data # directory the new binary cannot open. retry 30 5 bash -c ' replicas=$(kubectl -n "'"${NAMESPACE}"'" get statefulset \ major-upgrade-logical-replica-lr-analytics -o jsonpath="{.spec.replicas}") echo "logical replica statefulset replicas: ${replicas}" [[ ${replicas} == "0" ]] ' # pg_upgrade carries the logical slots over to the new cluster. Nothing will # ever read from them again, so they must not be left pinning WAL. PASS=$(get_psql_user_pass major-upgrade-logical-replica-pguser-postgres) HOST=$(get_psql_user_host major-upgrade-logical-replica-pguser-postgres) no_slots_left() { local got got=$(run_psql \ "SELECT count(*) FROM pg_replication_slots WHERE plugin IS NOT NULL;" \ "postgres://postgres:${PASS}@${HOST}" | xargs) echo "logical slots left on the primary: ${got}" [[ ${got} == "0" ]] } retry 30 5 no_slots_left] logger.go:42: 00:03:44 | major-upgrade-logical-replica/6-verify-replica-invalidated | + source ../../functions logger.go:42: 00:03:44 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ realpath ../../.. logger.go:42: 00:03:44 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1786 logger.go:42: 00:03:44 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ CERT_MANAGER_VER=1.20.2 logger.go:42: 00:03:44 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ CERT_MANAGER_TEST_LABEL=e2e.percona.com/managed-by=percona-postgresql-operator-tests logger.go:42: 00:03:44 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ CHAOS_MESH_VER=2.5.1 logger.go:42: 00:03:44 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ pwd logger.go:42: 00:03:44 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ basename /mnt/jenkins/workspace/cloud-pg-operator_PR-1786/e2e-tests/tests/major-upgrade-logical-replica logger.go:42: 00:03:44 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ test_name=major-upgrade-logical-replica logger.go:42: 00:03:44 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ source /mnt/jenkins/workspace/cloud-pg-operator_PR-1786/e2e-tests/vars.sh logger.go:42: 00:03:44 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1786 logger.go:42: 00:03:44 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1786 logger.go:42: 00:03:44 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1786/deploy logger.go:42: 00:03:44 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1786/deploy logger.go:42: 00:03:44 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1786/e2e-tests logger.go:42: 00:03:44 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1786/e2e-tests logger.go:42: 00:03:44 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1786/e2e-tests/conf logger.go:42: 00:03:44 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1786/e2e-tests/conf logger.go:42: 00:03:44 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export TEST_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1786/e2e-tests/tests/major-upgrade-logical-replica/conf logger.go:42: 00:03:44 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ TEST_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1786/e2e-tests/tests/major-upgrade-logical-replica/conf logger.go:42: 00:03:44 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export TEMP_DIR=/tmp/kuttl/pg/major-upgrade-logical-replica logger.go:42: 00:03:44 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ TEMP_DIR=/tmp/kuttl/pg/major-upgrade-logical-replica logger.go:42: 00:03:44 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ git rev-parse --abbrev-ref HEAD logger.go:42: 00:03:44 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export GIT_BRANCH=PR-1786 logger.go:42: 00:03:44 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ GIT_BRANCH=PR-1786 logger.go:42: 00:03:44 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export VERSION=PR-1786-391a86551 logger.go:42: 00:03:44 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ VERSION=PR-1786-391a86551 logger.go:42: 00:03:44 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export SKIP_TEST_WARNINGS=true logger.go:42: 00:03:44 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ SKIP_TEST_WARNINGS=true logger.go:42: 00:03:44 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export K8S_UPGRADE_PLATFORM=gke logger.go:42: 00:03:44 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ K8S_UPGRADE_PLATFORM=gke logger.go:42: 00:03:44 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export K8S_UPGRADE_REGION=us-central1-a logger.go:42: 00:03:44 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ K8S_UPGRADE_REGION=us-central1-a logger.go:42: 00:03:44 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export K8S_UPGRADE_INITIAL_VERSION= logger.go:42: 00:03:44 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ K8S_UPGRADE_INITIAL_VERSION= logger.go:42: 00:03:44 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export K8S_UPGRADE_FINAL_VERSION= logger.go:42: 00:03:44 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ K8S_UPGRADE_FINAL_VERSION= logger.go:42: 00:03:44 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ -z '' ]] logger.go:42: 00:03:44 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ detect_k8s_platform logger.go:42: 00:03:44 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ local detected_platform logger.go:42: 00:03:44 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ local xtrace_enabled=false logger.go:42: 00:03:44 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ [[ -o xtrace ]] logger.go:42: 00:03:44 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ xtrace_enabled=true logger.go:42: 00:03:44 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ set +o xtrace logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ PLATFORM=gke logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export PLATFORM logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE=perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE=perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ ! -n '' ]] logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ -n '' ]] logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export PG_VER=18 logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ PG_VER=18 logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export ARCH= logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ ARCH= logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ -z '' ]] logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ kubectl get nodes -o 'jsonpath={range .items[*]}{.metadata.labels.kubernetes\.io/arch}{"\n"}{end}' logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ sort -u logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ node_arches=amd64 logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ amd64 == arm64 ]] logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export PG_DISTRIBUTION= logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ PG_DISTRIBUTION= logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ '' == community ]] logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export BUCKET=pg-operator-testing logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ BUCKET=pg-operator-testing logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export PMM_SERVER_VERSION=9.9.9 logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ PMM_SERVER_VERSION=9.9.9 logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE_LOGCOLLECTOR=perconalab/fluentbit:main-logcollector logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE_LOGCOLLECTOR=perconalab/fluentbit:main-logcollector logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export PGOV1_TAG=1.4.0 logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ PGOV1_TAG=1.4.0 logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export PGOV1_VER=14 logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ PGOV1_VER=14 logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export CPGO_VERSION=5.8.7 logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ CPGO_VERSION=5.8.7 logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export MINIO_VER=5.4.0 logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ MINIO_VER=5.4.0 logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export VAULT_VER=0.32.0 logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ VAULT_VER=0.32.0 logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE_AWS_CLI=docker.io/amazon/aws-cli:2.34.60 logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE_AWS_CLI=docker.io/amazon/aws-cli:2.34.60 logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export REGISTRY_NAME=docker.io logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ REGISTRY_NAME=docker.io logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export REGISTRY_NAME_FULL=docker.io/ logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ REGISTRY_NAME_FULL=docker.io/ logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ printenv logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ grep -E '^IMAGE' logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ awk -F= '{print $1}' logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ eval 'echo $IMAGE_BACKREST' logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++++ echo perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ var_value=perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == docker.io/* ]] logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == percona/* ]] logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator:main-pgbackrest18 == perconalab/* ]] logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | IMAGE_BACKREST=docker.io/perconalab/percona-postgresql-operator:main-pgbackrest18 logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ eval 'echo $IMAGE' logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++++ echo perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ var_value=perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator:PR-1786-391a86551 == docker.io/* ]] logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator:PR-1786-391a86551 == percona/* ]] logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator:PR-1786-391a86551 == perconalab/* ]] logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ new_value=docker.io/perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | IMAGE=docker.io/perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | docker.io/perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ eval 'echo $IMAGE_UPGRADE' logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++++ echo perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ var_value=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == docker.io/* ]] logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == percona/* ]] logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator:main-upgrade == perconalab/* ]] logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | IMAGE_UPGRADE=docker.io/perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | docker.io/perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ eval 'echo $IMAGE_LOGCOLLECTOR' logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++++ echo perconalab/fluentbit:main-logcollector logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ var_value=perconalab/fluentbit:main-logcollector logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/fluentbit:main-logcollector == docker.io/* ]] logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/fluentbit:main-logcollector == percona/* ]] logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/fluentbit:main-logcollector == perconalab/* ]] logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ new_value=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | IMAGE_LOGCOLLECTOR=docker.io/perconalab/fluentbit:main-logcollector logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | docker.io/perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ eval 'echo $IMAGE_BASE' logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++++ echo perconalab/percona-postgresql-operator logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ var_value=perconalab/percona-postgresql-operator logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator == docker.io/* ]] logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator == percona/* ]] logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator == perconalab/* ]] logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ new_value=docker.io/perconalab/percona-postgresql-operator logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | IMAGE_BASE=docker.io/perconalab/percona-postgresql-operator logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | docker.io/perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ eval 'echo $IMAGE_PGBOUNCER' logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++++ echo perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ var_value=perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == docker.io/* ]] logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == percona/* ]] logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator:main-pgbouncer18 == perconalab/* ]] logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | IMAGE_PGBOUNCER=docker.io/perconalab/percona-postgresql-operator:main-pgbouncer18 logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | docker.io/perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ eval 'echo $IMAGE_POSTGRESQL' logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++++ echo perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ var_value=perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == docker.io/* ]] logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == percona/* ]] logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/percona-postgresql-operator:main-ppg18-postgres == perconalab/* ]] logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ new_value=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | IMAGE_POSTGRESQL=docker.io/perconalab/percona-postgresql-operator:main-ppg18-postgres logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | docker.io/perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ eval 'echo $IMAGE_PMM_SERVER' logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++++ echo perconalab/pmm-server:3-dev-latest logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ var_value=perconalab/pmm-server:3-dev-latest logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/pmm-server:3-dev-latest == docker.io/* ]] logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/pmm-server:3-dev-latest == percona/* ]] logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/pmm-server:3-dev-latest == perconalab/* ]] logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ new_value=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | IMAGE_PMM_SERVER=docker.io/perconalab/pmm-server:3-dev-latest logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | docker.io/perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ eval 'echo $IMAGE_PMM_CLIENT' logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++++ echo perconalab/pmm-client:3-dev-latest logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ var_value=perconalab/pmm-client:3-dev-latest logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/pmm-client:3-dev-latest == docker.io/* ]] logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/pmm-client:3-dev-latest == percona/* ]] logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ perconalab/pmm-client:3-dev-latest == perconalab/* ]] logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ new_value=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ export IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | IMAGE_PMM_CLIENT=docker.io/perconalab/pmm-client:3-dev-latest logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ echo docker.io/perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | docker.io/perconalab/percona-postgresql-operator:PR-1786-391a86551 logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ for var in $(printenv | grep -E '^IMAGE' | awk -F'=' '{print $1}') logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ eval 'echo $IMAGE_AWS_CLI' logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++++ echo docker.io/amazon/aws-cli:2.34.60 logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ var_value=docker.io/amazon/aws-cli:2.34.60 logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ [[ docker.io/amazon/aws-cli:2.34.60 == docker.io/* ]] logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ continue logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ which gdate logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | which: no gdate in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1786/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ which date logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ date=/usr/sbin/date logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ which gsed logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | which: no gsed in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1786/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++++ which sed logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ sed=/usr/sbin/sed logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ arch_scheduling_enabled logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ [[ -n '' ]] logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ get_pod_by_role major-upgrade-logical-replica primary name logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ local cluster=major-upgrade-logical-replica logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ local role=primary logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ local parameter=name logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ case ${parameter} in logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ local 'jsonpath={.items[].metadata.name}' logger.go:42: 00:03:47 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ kubectl get pods --namespace kuttl-test-evident-monkey --selector=postgres-operator.crunchydata.com/role=primary,postgres-operator.crunchydata.com/cluster=major-upgrade-logical-replica -o 'jsonpath={.items[].metadata.name}' logger.go:42: 00:03:48 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ echo major-upgrade-logical-replica-instance1-cl6m-0 logger.go:42: 00:03:48 | major-upgrade-logical-replica/6-verify-replica-invalidated | + primary=major-upgrade-logical-replica-instance1-cl6m-0 logger.go:42: 00:03:48 | major-upgrade-logical-replica/6-verify-replica-invalidated | + kubectl -n kuttl-test-evident-monkey exec major-upgrade-logical-replica-instance1-cl6m-0 -- /pgdata/delete_old_cluster.sh logger.go:42: 00:03:50 | major-upgrade-logical-replica/6-verify-replica-invalidated | + retry 60 10 bash -c $'\n state=$(kubectl -n "kuttl-test-evident-monkey" get pg major-upgrade-logical-replica \\\n -o jsonpath="{.status.logicalReplicas[?(@.name==\\"analytics\\")].state}")\n reason=$(kubectl -n "kuttl-test-evident-monkey" get pg major-upgrade-logical-replica \\\n -o jsonpath="{.status.logicalReplicas[?(@.name==\\"analytics\\")].reason}")\n echo "logical replica analytics: ${state}/${reason}"\n [[ ${state} == "broken" && ${reason} == "SourceUpgraded" ]]\n' logger.go:42: 00:03:50 | major-upgrade-logical-replica/6-verify-replica-invalidated | + local max=60 logger.go:42: 00:03:50 | major-upgrade-logical-replica/6-verify-replica-invalidated | + local delay=10 logger.go:42: 00:03:50 | major-upgrade-logical-replica/6-verify-replica-invalidated | + shift 2 logger.go:42: 00:03:50 | major-upgrade-logical-replica/6-verify-replica-invalidated | + local n=1 logger.go:42: 00:03:50 | major-upgrade-logical-replica/6-verify-replica-invalidated | + bash -c $'\n state=$(kubectl -n "kuttl-test-evident-monkey" get pg major-upgrade-logical-replica \\\n -o jsonpath="{.status.logicalReplicas[?(@.name==\\"analytics\\")].state}")\n reason=$(kubectl -n "kuttl-test-evident-monkey" get pg major-upgrade-logical-replica \\\n -o jsonpath="{.status.logicalReplicas[?(@.name==\\"analytics\\")].reason}")\n echo "logical replica analytics: ${state}/${reason}"\n [[ ${state} == "broken" && ${reason} == "SourceUpgraded" ]]\n' logger.go:42: 00:03:51 | major-upgrade-logical-replica/6-verify-replica-invalidated | logical replica analytics: broken/SourceUpgraded logger.go:42: 00:03:51 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ kubectl -n kuttl-test-evident-monkey get pg major-upgrade-logical-replica -o 'jsonpath={.status.logicalReplicas[?(@.name=="analytics")].message}' logger.go:42: 00:03:52 | major-upgrade-logical-replica/6-verify-replica-invalidated | + message='this replica holds a PostgreSQL 17 data directory and the cluster now runs 18; the pg_upgrade job only rewrites the data directories of the instance sets, so this one cannot be started again; remove it from spec.logicalReplicas, wait for its entry to leave status.logicalReplicas, and add it back to seed it again' logger.go:42: 00:03:52 | major-upgrade-logical-replica/6-verify-replica-invalidated | + echo 'message: this replica holds a PostgreSQL 17 data directory and the cluster now runs 18; the pg_upgrade job only rewrites the data directories of the instance sets, so this one cannot be started again; remove it from spec.logicalReplicas, wait for its entry to leave status.logicalReplicas, and add it back to seed it again' logger.go:42: 00:03:52 | major-upgrade-logical-replica/6-verify-replica-invalidated | message: this replica holds a PostgreSQL 17 data directory and the cluster now runs 18; the pg_upgrade job only rewrites the data directories of the instance sets, so this one cannot be started again; remove it from spec.logicalReplicas, wait for its entry to leave status.logicalReplicas, and add it back to seed it again logger.go:42: 00:03:52 | major-upgrade-logical-replica/6-verify-replica-invalidated | + echo 'this replica holds a PostgreSQL 17 data directory and the cluster now runs 18; the pg_upgrade job only rewrites the data directories of the instance sets, so this one cannot be started again; remove it from spec.logicalReplicas, wait for its entry to leave status.logicalReplicas, and add it back to seed it again' logger.go:42: 00:03:52 | major-upgrade-logical-replica/6-verify-replica-invalidated | + grep -q 'remove it from spec.logicalReplicas' logger.go:42: 00:03:52 | major-upgrade-logical-replica/6-verify-replica-invalidated | + retry 30 5 bash -c $'\n replicas=$(kubectl -n "kuttl-test-evident-monkey" get statefulset \\\n major-upgrade-logical-replica-lr-analytics -o jsonpath="{.spec.replicas}")\n echo "logical replica statefulset replicas: ${replicas}"\n [[ ${replicas} == "0" ]]\n' logger.go:42: 00:03:52 | major-upgrade-logical-replica/6-verify-replica-invalidated | + local max=30 logger.go:42: 00:03:52 | major-upgrade-logical-replica/6-verify-replica-invalidated | + local delay=5 logger.go:42: 00:03:52 | major-upgrade-logical-replica/6-verify-replica-invalidated | + shift 2 logger.go:42: 00:03:52 | major-upgrade-logical-replica/6-verify-replica-invalidated | + local n=1 logger.go:42: 00:03:52 | major-upgrade-logical-replica/6-verify-replica-invalidated | + bash -c $'\n replicas=$(kubectl -n "kuttl-test-evident-monkey" get statefulset \\\n major-upgrade-logical-replica-lr-analytics -o jsonpath="{.spec.replicas}")\n echo "logical replica statefulset replicas: ${replicas}"\n [[ ${replicas} == "0" ]]\n' logger.go:42: 00:03:52 | major-upgrade-logical-replica/6-verify-replica-invalidated | logical replica statefulset replicas: 0 logger.go:42: 00:03:52 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ get_psql_user_pass major-upgrade-logical-replica-pguser-postgres logger.go:42: 00:03:52 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ local secret_name=major-upgrade-logical-replica-pguser-postgres logger.go:42: 00:03:52 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ kubectl -n kuttl-test-evident-monkey get secret/major-upgrade-logical-replica-pguser-postgres '--template={{.data.password | base64decode}}' logger.go:42: 00:03:53 | major-upgrade-logical-replica/6-verify-replica-invalidated | + PASS=wJQUHcT1WtEk7Gtt4SKRMFCJ logger.go:42: 00:03:53 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ get_psql_user_host major-upgrade-logical-replica-pguser-postgres logger.go:42: 00:03:53 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ local secret_name=major-upgrade-logical-replica-pguser-postgres logger.go:42: 00:03:53 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ kubectl -n kuttl-test-evident-monkey get secret/major-upgrade-logical-replica-pguser-postgres '--template={{.data.host | base64decode }}' logger.go:42: 00:03:54 | major-upgrade-logical-replica/6-verify-replica-invalidated | + HOST=major-upgrade-logical-replica-primary.kuttl-test-evident-monkey.svc logger.go:42: 00:03:54 | major-upgrade-logical-replica/6-verify-replica-invalidated | + retry 30 5 no_slots_left logger.go:42: 00:03:54 | major-upgrade-logical-replica/6-verify-replica-invalidated | + local max=30 logger.go:42: 00:03:54 | major-upgrade-logical-replica/6-verify-replica-invalidated | + local delay=5 logger.go:42: 00:03:54 | major-upgrade-logical-replica/6-verify-replica-invalidated | + shift 2 logger.go:42: 00:03:54 | major-upgrade-logical-replica/6-verify-replica-invalidated | + local n=1 logger.go:42: 00:03:54 | major-upgrade-logical-replica/6-verify-replica-invalidated | + no_slots_left logger.go:42: 00:03:54 | major-upgrade-logical-replica/6-verify-replica-invalidated | + local got logger.go:42: 00:03:54 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ run_psql 'SELECT count(*) FROM pg_replication_slots WHERE plugin IS NOT NULL;' postgres://postgres:wJQUHcT1WtEk7Gtt4SKRMFCJ@major-upgrade-logical-replica-primary.kuttl-test-evident-monkey.svc logger.go:42: 00:03:54 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ xargs logger.go:42: 00:03:54 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ local 'command=SELECT count(*) FROM pg_replication_slots WHERE plugin IS NOT NULL;' logger.go:42: 00:03:54 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ local uri=postgres://postgres:wJQUHcT1WtEk7Gtt4SKRMFCJ@major-upgrade-logical-replica-primary.kuttl-test-evident-monkey.svc logger.go:42: 00:03:54 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ local password= logger.go:42: 00:03:54 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ get_client_pod logger.go:42: 00:03:54 | major-upgrade-logical-replica/6-verify-replica-invalidated | +++ kubectl -n kuttl-test-evident-monkey get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}' logger.go:42: 00:03:54 | major-upgrade-logical-replica/6-verify-replica-invalidated | ++ kubectl -n kuttl-test-evident-monkey exec pg-client-57cfd999d5-t74nf -- bash -c 'printf '\''SELECT count(*) FROM pg_replication_slots WHERE plugin IS NOT NULL;\n'\'' | PGPASSWORD='\'''\'' psql -v ON_ERROR_STOP=1 -t -q postgres://postgres:wJQUHcT1WtEk7Gtt4SKRMFCJ@major-upgrade-logical-replica-primary.kuttl-test-evident-monkey.svc' logger.go:42: 00:03:55 | major-upgrade-logical-replica/6-verify-replica-invalidated | + got=0 logger.go:42: 00:03:55 | major-upgrade-logical-replica/6-verify-replica-invalidated | + echo 'logical slots left on the primary: 0' logger.go:42: 00:03:55 | major-upgrade-logical-replica/6-verify-replica-invalidated | logical slots left on the primary: 0 logger.go:42: 00:03:55 | major-upgrade-logical-replica/6-verify-replica-invalidated | + [[ 0 == 0 ]] logger.go:42: 00:03:55 | major-upgrade-logical-replica/6-verify-replica-invalidated | test step completed 6-verify-replica-invalidated logger.go:42: 00:03:55 | major-upgrade-logical-replica/7-create-reseed-backup | starting test step 7-create-reseed-backup logger.go:42: 00:03:55 | major-upgrade-logical-replica/7-create-reseed-backup | running command: [sh -c set -o errexit set -o xtrace source ../../functions # pg_upgrade left the repository holding only pre-upgrade backups, so the # bootstrap Job of the reseed below would restore a PostgreSQL 17 data # directory. Take a full backup of the upgraded cluster first. cat </dev/null 2>&1 && return 1 kubectl -n "${NAMESPACE}" get pvc major-upgrade-logical-replica-lr-analytics-pgdata >/dev/null 2>&1 && return 1 return 0 } retry 60 5 status_gone retry 60 5 objects_gone merge_patch_pg major-upgrade-logical-replica <