=== RUN kuttl harness.go:459: starting setup harness.go:254: running tests using configured kubeconfig. harness.go:277: Successful connection to cluster at: https://34.135.243.56 harness.go:362: running tests harness.go:74: going to run test suite with timeout of 180 seconds for each step harness.go:374: testsuite: e2e-tests/tests has 26 tests === RUN kuttl/harness === RUN kuttl/harness/custom-extensions === PAUSE kuttl/harness/custom-extensions === CONT kuttl/harness/custom-extensions logger.go:42: 16:50:18 | custom-extensions | Creating namespace: kuttl-test-picked-cow logger.go:42: 16:50:18 | custom-extensions/0-deploy-operator | starting test step 0-deploy-operator logger.go:42: 16:50:18 | custom-extensions/0-deploy-operator | running command: [sh -c set -o errexit set -o xtrace source ../../functions init_temp_dir # do this only in the first TestStep deploy_operator deploy_client deploy_s3_secrets deploy_minio copy_custom_extensions_form_aws] logger.go:42: 16:50:18 | custom-extensions/0-deploy-operator | + source ../../functions logger.go:42: 16:50:18 | custom-extensions/0-deploy-operator | +++ realpath ../../.. logger.go:42: 16:50:18 | custom-extensions/0-deploy-operator | ++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1214 logger.go:42: 16:50:18 | custom-extensions/0-deploy-operator | ++ CERT_MANAGER_VER=1.17.1 logger.go:42: 16:50:18 | custom-extensions/0-deploy-operator | ++++ pwd logger.go:42: 16:50:18 | custom-extensions/0-deploy-operator | +++ basename /mnt/jenkins/workspace/cloud-pg-operator_PR-1214/e2e-tests/tests/custom-extensions logger.go:42: 16:50:18 | custom-extensions/0-deploy-operator | ++ test_name=custom-extensions logger.go:42: 16:50:18 | custom-extensions/0-deploy-operator | ++ source /mnt/jenkins/workspace/cloud-pg-operator_PR-1214/e2e-tests/vars.sh logger.go:42: 16:50:18 | custom-extensions/0-deploy-operator | +++ export ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1214 logger.go:42: 16:50:18 | custom-extensions/0-deploy-operator | +++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1214 logger.go:42: 16:50:18 | custom-extensions/0-deploy-operator | +++ export DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1214/deploy logger.go:42: 16:50:18 | custom-extensions/0-deploy-operator | +++ DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1214/deploy logger.go:42: 16:50:18 | custom-extensions/0-deploy-operator | +++ export TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1214/e2e-tests logger.go:42: 16:50:18 | custom-extensions/0-deploy-operator | +++ TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1214/e2e-tests logger.go:42: 16:50:18 | custom-extensions/0-deploy-operator | +++ export TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1214/e2e-tests/conf logger.go:42: 16:50:18 | custom-extensions/0-deploy-operator | +++ TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1214/e2e-tests/conf logger.go:42: 16:50:18 | custom-extensions/0-deploy-operator | +++ export TEMP_DIR=/tmp/kuttl/pg/custom-extensions logger.go:42: 16:50:18 | custom-extensions/0-deploy-operator | +++ TEMP_DIR=/tmp/kuttl/pg/custom-extensions logger.go:42: 16:50:18 | custom-extensions/0-deploy-operator | ++++ git rev-parse --abbrev-ref HEAD logger.go:42: 16:50:18 | custom-extensions/0-deploy-operator | +++ export GIT_BRANCH=PR-1214 logger.go:42: 16:50:18 | custom-extensions/0-deploy-operator | +++ GIT_BRANCH=PR-1214 logger.go:42: 16:50:18 | custom-extensions/0-deploy-operator | +++ export VERSION=PR-1214-6ef3702c0 logger.go:42: 16:50:18 | custom-extensions/0-deploy-operator | +++ VERSION=PR-1214-6ef3702c0 logger.go:42: 16:50:18 | custom-extensions/0-deploy-operator | +++ export IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 16:50:18 | custom-extensions/0-deploy-operator | +++ IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 16:50:18 | custom-extensions/0-deploy-operator | +++ export IMAGE=perconalab/percona-postgresql-operator:PR-1214-6ef3702c0 logger.go:42: 16:50:18 | custom-extensions/0-deploy-operator | +++ IMAGE=perconalab/percona-postgresql-operator:PR-1214-6ef3702c0 logger.go:42: 16:50:18 | custom-extensions/0-deploy-operator | +++ export PG_VER=17 logger.go:42: 16:50:18 | custom-extensions/0-deploy-operator | +++ PG_VER=17 logger.go:42: 16:50:18 | custom-extensions/0-deploy-operator | +++ export IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer17 logger.go:42: 16:50:18 | custom-extensions/0-deploy-operator | +++ IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer17 logger.go:42: 16:50:18 | custom-extensions/0-deploy-operator | +++ export IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg17-postgres logger.go:42: 16:50:18 | custom-extensions/0-deploy-operator | +++ IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg17-postgres logger.go:42: 16:50:18 | custom-extensions/0-deploy-operator | +++ export IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest17 logger.go:42: 16:50:18 | custom-extensions/0-deploy-operator | +++ IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest17 logger.go:42: 16:50:18 | custom-extensions/0-deploy-operator | +++ export IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 16:50:18 | custom-extensions/0-deploy-operator | +++ IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 16:50:18 | custom-extensions/0-deploy-operator | +++ export BUCKET=pg-operator-testing logger.go:42: 16:50:18 | custom-extensions/0-deploy-operator | +++ BUCKET=pg-operator-testing logger.go:42: 16:50:18 | custom-extensions/0-deploy-operator | +++ export PMM_SERVER_VERSION=9.9.9 logger.go:42: 16:50:18 | custom-extensions/0-deploy-operator | +++ PMM_SERVER_VERSION=9.9.9 logger.go:42: 16:50:18 | custom-extensions/0-deploy-operator | +++ export IMAGE_PMM_CLIENT=perconalab/pmm-client:dev-latest logger.go:42: 16:50:18 | custom-extensions/0-deploy-operator | +++ IMAGE_PMM_CLIENT=perconalab/pmm-client:dev-latest logger.go:42: 16:50:18 | custom-extensions/0-deploy-operator | +++ export IMAGE_PMM_SERVER=perconalab/pmm-server:dev-latest logger.go:42: 16:50:18 | custom-extensions/0-deploy-operator | +++ IMAGE_PMM_SERVER=perconalab/pmm-server:dev-latest logger.go:42: 16:50:18 | custom-extensions/0-deploy-operator | +++ export IMAGE_PMM3_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 16:50:18 | custom-extensions/0-deploy-operator | +++ IMAGE_PMM3_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 16:50:18 | custom-extensions/0-deploy-operator | +++ export IMAGE_PMM3_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 16:50:18 | custom-extensions/0-deploy-operator | +++ IMAGE_PMM3_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 16:50:18 | custom-extensions/0-deploy-operator | +++ export PGOV1_TAG=1.4.0 logger.go:42: 16:50:18 | custom-extensions/0-deploy-operator | +++ PGOV1_TAG=1.4.0 logger.go:42: 16:50:18 | custom-extensions/0-deploy-operator | +++ export PGOV1_VER=14 logger.go:42: 16:50:18 | custom-extensions/0-deploy-operator | +++ PGOV1_VER=14 logger.go:42: 16:50:18 | custom-extensions/0-deploy-operator | +++ export MINIO_VER=5.4.0 logger.go:42: 16:50:18 | custom-extensions/0-deploy-operator | +++ MINIO_VER=5.4.0 logger.go:42: 16:50:18 | custom-extensions/0-deploy-operator | ++++ which gdate logger.go:42: 16:50:18 | custom-extensions/0-deploy-operator | which: no gdate in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1214/bin/:/home/ec2-user/.krew/bin:/usr/local/bin:/usr/bin) logger.go:42: 16:50:18 | custom-extensions/0-deploy-operator | ++++ which date logger.go:42: 16:50:18 | custom-extensions/0-deploy-operator | +++ date=/usr/bin/date logger.go:42: 16:50:18 | custom-extensions/0-deploy-operator | ++++ which gsed logger.go:42: 16:50:18 | custom-extensions/0-deploy-operator | which: no gsed in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1214/bin/:/home/ec2-user/.krew/bin:/usr/local/bin:/usr/bin) logger.go:42: 16:50:18 | custom-extensions/0-deploy-operator | ++++ which sed logger.go:42: 16:50:18 | custom-extensions/0-deploy-operator | +++ sed=/usr/bin/sed logger.go:42: 16:50:18 | custom-extensions/0-deploy-operator | +++ command -v oc logger.go:42: 16:50:18 | custom-extensions/0-deploy-operator | ++ oc get projects logger.go:42: 16:50:18 | custom-extensions/0-deploy-operator | + init_temp_dir logger.go:42: 16:50:18 | custom-extensions/0-deploy-operator | + rm -rf /tmp/kuttl/pg/custom-extensions logger.go:42: 16:50:18 | custom-extensions/0-deploy-operator | + mkdir -p /tmp/kuttl/pg/custom-extensions logger.go:42: 16:50:18 | custom-extensions/0-deploy-operator | + deploy_operator logger.go:42: 16:50:18 | custom-extensions/0-deploy-operator | + local cw_prefix= logger.go:42: 16:50:18 | custom-extensions/0-deploy-operator | + destroy_operator logger.go:42: 16:50:18 | custom-extensions/0-deploy-operator | + kubectl -n pg-operator delete deployment percona-postgresql-operator --force --grace-period=0 logger.go:42: 16:50:18 | custom-extensions/0-deploy-operator | Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. logger.go:42: 16:50:18 | custom-extensions/0-deploy-operator | deployment.apps "percona-postgresql-operator" force deleted logger.go:42: 16:50:18 | custom-extensions/0-deploy-operator | + [[ -n pg-operator ]] logger.go:42: 16:50:18 | custom-extensions/0-deploy-operator | + kubectl delete namespace pg-operator --force --grace-period=0 logger.go:42: 16:50:18 | custom-extensions/0-deploy-operator | Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. logger.go:42: 16:50:19 | custom-extensions/0-deploy-operator | namespace "pg-operator" force deleted logger.go:42: 16:50:26 | custom-extensions/0-deploy-operator | + [[ -n pg-operator ]] logger.go:42: 16:50:26 | custom-extensions/0-deploy-operator | + create_namespace pg-operator logger.go:42: 16:50:26 | custom-extensions/0-deploy-operator | + local namespace=pg-operator logger.go:42: 16:50:26 | custom-extensions/0-deploy-operator | + [[ -n '' ]] logger.go:42: 16:50:26 | custom-extensions/0-deploy-operator | + kubectl delete namespace pg-operator --ignore-not-found logger.go:42: 16:50:26 | custom-extensions/0-deploy-operator | + kubectl wait --for=delete namespace pg-operator logger.go:42: 16:50:27 | custom-extensions/0-deploy-operator | + kubectl create namespace pg-operator logger.go:42: 16:50:27 | custom-extensions/0-deploy-operator | namespace/pg-operator created logger.go:42: 16:50:27 | custom-extensions/0-deploy-operator | + cw_prefix=cw- logger.go:42: 16:50:27 | custom-extensions/0-deploy-operator | + kubectl -n pg-operator apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pg-operator_PR-1214/deploy/crd.yaml logger.go:42: 16:50:28 | custom-extensions/0-deploy-operator | customresourcedefinition.apiextensions.k8s.io/crunchybridgeclusters.postgres-operator.crunchydata.com serverside-applied logger.go:42: 16:50:28 | custom-extensions/0-deploy-operator | customresourcedefinition.apiextensions.k8s.io/perconapgbackups.pgv2.percona.com serverside-applied logger.go:42: 16:50:30 | custom-extensions/0-deploy-operator | customresourcedefinition.apiextensions.k8s.io/perconapgclusters.pgv2.percona.com serverside-applied logger.go:42: 16:50:30 | custom-extensions/0-deploy-operator | customresourcedefinition.apiextensions.k8s.io/perconapgrestores.pgv2.percona.com serverside-applied logger.go:42: 16:50:31 | custom-extensions/0-deploy-operator | customresourcedefinition.apiextensions.k8s.io/perconapgupgrades.pgv2.percona.com serverside-applied logger.go:42: 16:50:31 | custom-extensions/0-deploy-operator | customresourcedefinition.apiextensions.k8s.io/pgadmins.postgres-operator.crunchydata.com serverside-applied logger.go:42: 16:50:31 | custom-extensions/0-deploy-operator | customresourcedefinition.apiextensions.k8s.io/pgupgrades.postgres-operator.crunchydata.com serverside-applied logger.go:42: 16:50:33 | custom-extensions/0-deploy-operator | customresourcedefinition.apiextensions.k8s.io/postgresclusters.postgres-operator.crunchydata.com serverside-applied logger.go:42: 16:50:33 | custom-extensions/0-deploy-operator | + kubectl -n pg-operator apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pg-operator_PR-1214/deploy/cw-rbac.yaml logger.go:42: 16:50:34 | custom-extensions/0-deploy-operator | serviceaccount/percona-postgresql-operator serverside-applied logger.go:42: 16:50:34 | custom-extensions/0-deploy-operator | clusterrole.rbac.authorization.k8s.io/percona-postgresql-operator serverside-applied logger.go:42: 16:50:34 | custom-extensions/0-deploy-operator | clusterrolebinding.rbac.authorization.k8s.io/percona-postgresql-operator serverside-applied logger.go:42: 16:50:34 | custom-extensions/0-deploy-operator | + local disable_telemetry=true logger.go:42: 16:50:34 | custom-extensions/0-deploy-operator | + '[' custom-extensions == telemetry-transfer ']' logger.go:42: 16:50:34 | custom-extensions/0-deploy-operator | + yq eval '.spec.template.spec.containers[0].image = "perconalab/percona-postgresql-operator:PR-1214-6ef3702c0"' /mnt/jenkins/workspace/cloud-pg-operator_PR-1214/deploy/cw-operator.yaml logger.go:42: 16:50:34 | custom-extensions/0-deploy-operator | + yq eval '(.spec.template.spec.containers[] | select(.name=="operator") | .env[] | select(.name=="DISABLE_TELEMETRY") | .value) = "true"' - logger.go:42: 16:50:34 | custom-extensions/0-deploy-operator | + kubectl -n pg-operator apply -f - logger.go:42: 16:50:35 | custom-extensions/0-deploy-operator | deployment.apps/percona-postgresql-operator created logger.go:42: 16:50:35 | custom-extensions/0-deploy-operator | + deploy_client logger.go:42: 16:50:35 | custom-extensions/0-deploy-operator | + kubectl -n kuttl-test-picked-cow apply -f /mnt/jenkins/workspace/cloud-pg-operator_PR-1214/e2e-tests/conf/client.yaml logger.go:42: 16:50:36 | custom-extensions/0-deploy-operator | deployment.apps/pg-client created logger.go:42: 16:50:36 | custom-extensions/0-deploy-operator | + deploy_s3_secrets logger.go:42: 16:50:36 | custom-extensions/0-deploy-operator | + set +o xtrace logger.go:42: 16:50:37 | custom-extensions/0-deploy-operator | secret/aws-s3-secret created logger.go:42: 16:50:37 | custom-extensions/0-deploy-operator | secret/gcp-cs-secret created logger.go:42: 16:50:37 | custom-extensions/0-deploy-operator | secret/azure-secret created logger.go:42: 16:50:38 | custom-extensions/0-deploy-operator | secret/minio-secret created logger.go:42: 16:50:38 | custom-extensions/0-deploy-operator | + deploy_minio logger.go:42: 16:50:38 | custom-extensions/0-deploy-operator | + local access_key logger.go:42: 16:50:38 | custom-extensions/0-deploy-operator | + local secret_key logger.go:42: 16:50:38 | custom-extensions/0-deploy-operator | ++ kubectl -n kuttl-test-picked-cow get secret minio-secret -o 'jsonpath={.data.AWS_ACCESS_KEY_ID}' logger.go:42: 16:50:38 | custom-extensions/0-deploy-operator | ++ base64 -d logger.go:42: 16:50:39 | custom-extensions/0-deploy-operator | + access_key=some-access-key logger.go:42: 16:50:39 | custom-extensions/0-deploy-operator | ++ kubectl -n kuttl-test-picked-cow get secret minio-secret -o 'jsonpath={.data.AWS_SECRET_ACCESS_KEY}' logger.go:42: 16:50:39 | custom-extensions/0-deploy-operator | ++ base64 -d logger.go:42: 16:50:39 | custom-extensions/0-deploy-operator | + secret_key=some-secret-key logger.go:42: 16:50:39 | custom-extensions/0-deploy-operator | + helm uninstall -n kuttl-test-picked-cow minio-service logger.go:42: 16:50:40 | custom-extensions/0-deploy-operator | Error: uninstall: Release not loaded: minio-service: release: not found logger.go:42: 16:50:40 | custom-extensions/0-deploy-operator | + : logger.go:42: 16:50:40 | custom-extensions/0-deploy-operator | + helm repo remove minio logger.go:42: 16:50:40 | custom-extensions/0-deploy-operator | "minio" has been removed from your repositories logger.go:42: 16:50:40 | custom-extensions/0-deploy-operator | + helm repo add minio https://charts.min.io/ logger.go:42: 16:50:40 | custom-extensions/0-deploy-operator | "minio" has been added to your repositories logger.go:42: 16:50:40 | custom-extensions/0-deploy-operator | +++ printf %q some-access-key logger.go:42: 16:50:40 | custom-extensions/0-deploy-operator | ++ printf %q some-access-key logger.go:42: 16:50:40 | custom-extensions/0-deploy-operator | +++ printf %q some-secret-key logger.go:42: 16:50:40 | custom-extensions/0-deploy-operator | ++ printf %q some-secret-key logger.go:42: 16:50:40 | custom-extensions/0-deploy-operator | + retry 10 60 helm install minio-service -n kuttl-test-picked-cow --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/.minio/ --set persistence.size=2G --set securityContext.enabled=false minio/minio logger.go:42: 16:50:40 | custom-extensions/0-deploy-operator | + local max=10 logger.go:42: 16:50:40 | custom-extensions/0-deploy-operator | + local delay=60 logger.go:42: 16:50:40 | custom-extensions/0-deploy-operator | + shift 2 logger.go:42: 16:50:40 | custom-extensions/0-deploy-operator | + local n=1 logger.go:42: 16:50:40 | custom-extensions/0-deploy-operator | + helm install minio-service -n kuttl-test-picked-cow --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/.minio/ --set persistence.size=2G --set securityContext.enabled=false minio/minio logger.go:42: 16:51:08 | custom-extensions/0-deploy-operator | NAME: minio-service logger.go:42: 16:51:08 | custom-extensions/0-deploy-operator | LAST DEPLOYED: Thu Jul 17 16:50:41 2025 logger.go:42: 16:51:08 | custom-extensions/0-deploy-operator | NAMESPACE: kuttl-test-picked-cow logger.go:42: 16:51:08 | custom-extensions/0-deploy-operator | STATUS: deployed logger.go:42: 16:51:08 | custom-extensions/0-deploy-operator | REVISION: 1 logger.go:42: 16:51:08 | custom-extensions/0-deploy-operator | TEST SUITE: None logger.go:42: 16:51:08 | custom-extensions/0-deploy-operator | NOTES: logger.go:42: 16:51:08 | custom-extensions/0-deploy-operator | MinIO can be accessed via port 9000 on the following DNS name from within your cluster: logger.go:42: 16:51:08 | custom-extensions/0-deploy-operator | minio-service.kuttl-test-picked-cow.cluster.local logger.go:42: 16:51:08 | custom-extensions/0-deploy-operator | logger.go:42: 16:51:08 | custom-extensions/0-deploy-operator | To access MinIO from localhost, run the below commands: logger.go:42: 16:51:08 | custom-extensions/0-deploy-operator | logger.go:42: 16:51:08 | custom-extensions/0-deploy-operator | 1. export POD_NAME=$(kubectl get pods --namespace kuttl-test-picked-cow -l "release=minio-service" -o jsonpath="{.items[0].metadata.name}") logger.go:42: 16:51:08 | custom-extensions/0-deploy-operator | logger.go:42: 16:51:08 | custom-extensions/0-deploy-operator | 2. kubectl port-forward $POD_NAME 9000 --namespace kuttl-test-picked-cow logger.go:42: 16:51:08 | custom-extensions/0-deploy-operator | logger.go:42: 16:51:08 | custom-extensions/0-deploy-operator | Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/ logger.go:42: 16:51:08 | custom-extensions/0-deploy-operator | logger.go:42: 16:51:08 | custom-extensions/0-deploy-operator | You can now access MinIO server on http://localhost:9000. Follow the below steps to connect to MinIO server with mc client: logger.go:42: 16:51:08 | custom-extensions/0-deploy-operator | logger.go:42: 16:51:08 | custom-extensions/0-deploy-operator | 1. Download the MinIO mc client - https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart logger.go:42: 16:51:08 | custom-extensions/0-deploy-operator | logger.go:42: 16:51:08 | custom-extensions/0-deploy-operator | 2. export MC_HOST_minio-service-local=http://$(kubectl get secret --namespace kuttl-test-picked-cow minio-service -o jsonpath="{.data.rootUser}" | base64 --decode):$(kubectl get secret --namespace kuttl-test-picked-cow minio-service -o jsonpath="{.data.rootPassword}" | base64 --decode)@localhost:9000 logger.go:42: 16:51:08 | custom-extensions/0-deploy-operator | logger.go:42: 16:51:08 | custom-extensions/0-deploy-operator | 3. mc ls minio-service-local logger.go:42: 16:51:08 | custom-extensions/0-deploy-operator | ++ kubectl -n kuttl-test-picked-cow get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}' logger.go:42: 16:51:09 | custom-extensions/0-deploy-operator | + MINIO_POD=minio-service-86dfccd949-qfp7b logger.go:42: 16:51:09 | custom-extensions/0-deploy-operator | + wait_pod minio-service-86dfccd949-qfp7b logger.go:42: 16:51:09 | custom-extensions/0-deploy-operator | + local pod=minio-service-86dfccd949-qfp7b logger.go:42: 16:51:09 | custom-extensions/0-deploy-operator | + set +o xtrace logger.go:42: 16:51:09 | custom-extensions/0-deploy-operator | minio-service-86dfccd949-qfp7btrue logger.go:42: 16:51:09 | custom-extensions/0-deploy-operator | + kubectl -n kuttl-test-picked-cow run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- bash -c 'AWS_ACCESS_KEY_ID='\''some-access-key'\'' AWS_SECRET_ACCESS_KEY='\''some-secret-key'\'' AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing' logger.go:42: 16:51:13 | custom-extensions/0-deploy-operator | If you don't see a command prompt, try pressing enter. logger.go:42: 16:51:20 | custom-extensions/0-deploy-operator | pod "aws-cli" deleted logger.go:42: 16:51:20 | custom-extensions/0-deploy-operator | + copy_custom_extensions_form_aws logger.go:42: 16:51:20 | custom-extensions/0-deploy-operator | + set +o xtrace logger.go:42: 16:51:25 | custom-extensions/0-deploy-operator | If you don't see a command prompt, try pressing enter. logger.go:42: 16:51:27 | custom-extensions/0-deploy-operator | Completed 148.6 KiB/30.8 MiB (123.7 KiB/s) with 11 file(s) remaining download: s3://pg-extensions/pg_cron-pg13-1.6.1.tar.gz to tmp/pg_cron-pg13-1.6.1.tar.gz logger.go:42: 16:51:27 | custom-extensions/0-deploy-operator | Completed 148.6 KiB/30.8 MiB (123.7 KiB/s) with 10 file(s) remaining Completed 300.0 KiB/30.8 MiB (245.5 KiB/s) with 10 file(s) remaining download: s3://pg-extensions/pg_cron-pg14-1.6.1.tar.gz to tmp/pg_cron-pg14-1.6.1.tar.gz logger.go:42: 16:51:27 | custom-extensions/0-deploy-operator | Completed 300.0 KiB/30.8 MiB (245.5 KiB/s) with 9 file(s) remaining Completed 444.4 KiB/30.8 MiB (357.0 KiB/s) with 9 file(s) remaining download: s3://pg-extensions/pg_cron-pg12-1.6.1.tar.gz to tmp/pg_cron-pg12-1.6.1.tar.gz logger.go:42: 16:51:27 | custom-extensions/0-deploy-operator | Completed 444.4 KiB/30.8 MiB (357.0 KiB/s) with 8 file(s) remaining Completed 614.4 KiB/30.8 MiB (463.6 KiB/s) with 8 file(s) remaining download: s3://pg-extensions/pg_cron-pg17-1.6.4.tar.gz to tmp/pg_cron-pg17-1.6.4.tar.gz logger.go:42: 16:51:27 | custom-extensions/0-deploy-operator | Completed 614.4 KiB/30.8 MiB (463.6 KiB/s) with 7 file(s) remaining Completed 783.7 KiB/30.8 MiB (589.7 KiB/s) with 7 file(s) remaining download: s3://pg-extensions/pg_cron-pg17-1.6.1.tar.gz to tmp/pg_cron-pg17-1.6.1.tar.gz logger.go:42: 16:51:27 | custom-extensions/0-deploy-operator | Completed 783.7 KiB/30.8 MiB (589.7 KiB/s) with 6 file(s) remaining Completed 949.2 KiB/30.8 MiB (704.3 KiB/s) with 6 file(s) remaining download: s3://pg-extensions/pg_cron-pg16-1.6.1.tar.gz to tmp/pg_cron-pg16-1.6.1.tar.gz logger.go:42: 16:51:27 | custom-extensions/0-deploy-operator | Completed 949.2 KiB/30.8 MiB (704.3 KiB/s) with 5 file(s) remaining Completed 1.2 MiB/30.8 MiB (889.5 KiB/s) with 5 file(s) remaining Completed 1.3 MiB/30.8 MiB (972.8 KiB/s) with 5 file(s) remaining download: s3://pg-extensions/pg_cron-pg15-1.6.1.tar.gz to tmp/pg_cron-pg15-1.6.1.tar.gz logger.go:42: 16:51:27 | custom-extensions/0-deploy-operator | Completed 1.3 MiB/30.8 MiB (972.8 KiB/s) with 4 file(s) remaining Completed 1.6 MiB/30.8 MiB (1.1 MiB/s) with 4 file(s) remaining Completed 1.8 MiB/30.8 MiB (1.2 MiB/s) with 4 file(s) remaining Completed 2.1 MiB/30.8 MiB (1.4 MiB/s) with 4 file(s) remaining Completed 2.3 MiB/30.8 MiB (1.5 MiB/s) with 4 file(s) remaining Completed 2.6 MiB/30.8 MiB (1.7 MiB/s) with 4 file(s) remaining Completed 2.8 MiB/30.8 MiB (1.9 MiB/s) with 4 file(s) remaining Completed 3.1 MiB/30.8 MiB (2.0 MiB/s) with 4 file(s) remaining Completed 3.3 MiB/30.8 MiB (2.1 MiB/s) with 4 file(s) remaining Completed 3.6 MiB/30.8 MiB (2.3 MiB/s) with 4 file(s) remaining Completed 3.8 MiB/30.8 MiB (2.4 MiB/s) with 4 file(s) remaining Completed 4.1 MiB/30.8 MiB (2.5 MiB/s) with 4 file(s) remaining Completed 4.3 MiB/30.8 MiB (2.7 MiB/s) with 4 file(s) remaining Completed 4.6 MiB/30.8 MiB (2.8 MiB/s) with 4 file(s) remaining Completed 4.8 MiB/30.8 MiB (3.0 MiB/s) with 4 file(s) remaining Completed 5.1 MiB/30.8 MiB (3.1 MiB/s) with 4 file(s) remaining Completed 5.3 MiB/30.8 MiB (3.2 MiB/s) with 4 file(s) remaining Completed 5.6 MiB/30.8 MiB (3.4 MiB/s) with 4 file(s) remaining Completed 5.8 MiB/30.8 MiB (3.5 MiB/s) with 4 file(s) remaining Completed 6.1 MiB/30.8 MiB (3.6 MiB/s) with 4 file(s) remaining Completed 6.3 MiB/30.8 MiB (3.8 MiB/s) with 4 file(s) remaining Completed 6.6 MiB/30.8 MiB (3.9 MiB/s) with 4 file(s) remaining Completed 6.8 MiB/30.8 MiB (4.1 MiB/s) with 4 file(s) remaining Completed 7.1 MiB/30.8 MiB (4.2 MiB/s) with 4 file(s) remaining Completed 7.3 MiB/30.8 MiB (4.3 MiB/s) with 4 file(s) remaining Completed 7.6 MiB/30.8 MiB (4.4 MiB/s) with 4 file(s) remaining Completed 7.8 MiB/30.8 MiB (4.5 MiB/s) with 4 file(s) remaining Completed 8.1 MiB/30.8 MiB (4.6 MiB/s) with 4 file(s) remaining Completed 8.3 MiB/30.8 MiB (4.8 MiB/s) with 4 file(s) remaining Completed 8.6 MiB/30.8 MiB (4.9 MiB/s) with 4 file(s) remaining Completed 8.8 MiB/30.8 MiB (5.0 MiB/s) with 4 file(s) remaining Completed 9.1 MiB/30.8 MiB (5.2 MiB/s) with 4 file(s) remaining Completed 9.3 MiB/30.8 MiB (5.3 MiB/s) with 4 file(s) remaining Completed 9.6 MiB/30.8 MiB (5.5 MiB/s) with 4 file(s) remaining Completed 9.8 MiB/30.8 MiB (5.6 MiB/s) with 4 file(s) remaining Completed 10.1 MiB/30.8 MiB (5.7 MiB/s) with 4 file(s) remaining Completed 10.3 MiB/30.8 MiB (5.9 MiB/s) with 4 file(s) remaining Completed 10.6 MiB/30.8 MiB (6.0 MiB/s) with 4 file(s) remaining Completed 10.8 MiB/30.8 MiB (6.1 MiB/s) with 4 file(s) remaining Completed 11.1 MiB/30.8 MiB (6.2 MiB/s) with 4 file(s) remaining Completed 11.3 MiB/30.8 MiB (6.3 MiB/s) with 4 file(s) remaining Completed 11.6 MiB/30.8 MiB (6.5 MiB/s) with 4 file(s) remaining Completed 11.8 MiB/30.8 MiB (6.6 MiB/s) with 4 file(s) remaining Completed 12.1 MiB/30.8 MiB (6.7 MiB/s) with 4 file(s) remaining Completed 12.3 MiB/30.8 MiB (6.9 MiB/s) with 4 file(s) remaining Completed 12.6 MiB/30.8 MiB (7.0 MiB/s) with 4 file(s) remaining Completed 12.8 MiB/30.8 MiB (7.1 MiB/s) with 4 file(s) remaining Completed 13.1 MiB/30.8 MiB (7.3 MiB/s) with 4 file(s) remaining Completed 13.3 MiB/30.8 MiB (7.3 MiB/s) with 4 file(s) remaining Completed 13.6 MiB/30.8 MiB (7.4 MiB/s) with 4 file(s) remaining Completed 13.8 MiB/30.8 MiB (7.6 MiB/s) with 4 file(s) remaining Completed 14.1 MiB/30.8 MiB (7.6 MiB/s) with 4 file(s) remaining Completed 14.3 MiB/30.8 MiB (7.8 MiB/s) with 4 file(s) remaining Completed 14.6 MiB/30.8 MiB (7.9 MiB/s) with 4 file(s) remaining Completed 14.8 MiB/30.8 MiB (8.0 MiB/s) with 4 file(s) remaining Completed 15.1 MiB/30.8 MiB (8.1 MiB/s) with 4 file(s) remaining Completed 15.3 MiB/30.8 MiB (8.3 MiB/s) with 4 file(s) remaining Completed 15.6 MiB/30.8 MiB (8.4 MiB/s) with 4 file(s) remaining Completed 15.8 MiB/30.8 MiB (8.5 MiB/s) with 4 file(s) remaining Completed 16.1 MiB/30.8 MiB (8.6 MiB/s) with 4 file(s) remaining Completed 16.3 MiB/30.8 MiB (8.8 MiB/s) with 4 file(s) remaining Completed 16.6 MiB/30.8 MiB (8.9 MiB/s) with 4 file(s) remaining Completed 16.8 MiB/30.8 MiB (9.0 MiB/s) with 4 file(s) remaining Completed 17.1 MiB/30.8 MiB (9.2 MiB/s) with 4 file(s) remaining Completed 17.3 MiB/30.8 MiB (9.3 MiB/s) with 4 file(s) remaining Completed 17.6 MiB/30.8 MiB (9.4 MiB/s) with 4 file(s) remaining Completed 17.8 MiB/30.8 MiB (9.5 MiB/s) with 4 file(s) remaining Completed 18.1 MiB/30.8 MiB (9.7 MiB/s) with 4 file(s) remaining Completed 18.3 MiB/30.8 MiB (9.8 MiB/s) with 4 file(s) remaining Completed 18.6 MiB/30.8 MiB (9.9 MiB/s) with 4 file(s) remaining Completed 18.8 MiB/30.8 MiB (10.0 MiB/s) with 4 file(s) remaining Completed 19.1 MiB/30.8 MiB (10.1 MiB/s) with 4 file(s) remaining Completed 19.3 MiB/30.8 MiB (10.3 MiB/s) with 4 file(s) remaining Completed 19.6 MiB/30.8 MiB (10.4 MiB/s) with 4 file(s) remaining Completed 19.8 MiB/30.8 MiB (10.5 MiB/s) with 4 file(s) remaining Completed 20.1 MiB/30.8 MiB (10.6 MiB/s) with 4 file(s) remaining Completed 20.3 MiB/30.8 MiB (10.8 MiB/s) with 4 file(s) remaining Completed 20.6 MiB/30.8 MiB (10.9 MiB/s) with 4 file(s) remaining Completed 20.8 MiB/30.8 MiB (11.0 MiB/s) with 4 file(s) remaining Completed 21.1 MiB/30.8 MiB (11.1 MiB/s) with 4 file(s) remaining Completed 21.3 MiB/30.8 MiB (11.3 MiB/s) with 4 file(s) remaining Completed 21.6 MiB/30.8 MiB (11.4 MiB/s) with 4 file(s) remaining Completed 21.8 MiB/30.8 MiB (11.5 MiB/s) with 4 file(s) remaining Completed 22.1 MiB/30.8 MiB (11.6 MiB/s) with 4 file(s) remaining Completed 22.3 MiB/30.8 MiB (11.7 MiB/s) with 4 file(s) remaining Completed 22.4 MiB/30.8 MiB (11.8 MiB/s) with 4 file(s) remaining download: s3://pg-extensions/documentdb-pg16-0.101-0.tar.gz to tmp/documentdb-pg16-0.101-0.tar.gz logger.go:42: 16:51:27 | custom-extensions/0-deploy-operator | Completed 22.4 MiB/30.8 MiB (11.8 MiB/s) with 3 file(s) remaining Completed 22.6 MiB/30.8 MiB (11.6 MiB/s) with 3 file(s) remaining Completed 22.9 MiB/30.8 MiB (11.7 MiB/s) with 3 file(s) remaining Completed 23.1 MiB/30.8 MiB (11.8 MiB/s) with 3 file(s) remaining Completed 23.4 MiB/30.8 MiB (11.9 MiB/s) with 3 file(s) remaining Completed 23.6 MiB/30.8 MiB (12.1 MiB/s) with 3 file(s) remaining Completed 23.9 MiB/30.8 MiB (12.2 MiB/s) with 3 file(s) remaining Completed 24.1 MiB/30.8 MiB (12.3 MiB/s) with 3 file(s) remaining Completed 24.4 MiB/30.8 MiB (12.4 MiB/s) with 3 file(s) remaining Completed 24.6 MiB/30.8 MiB (12.5 MiB/s) with 3 file(s) remaining Completed 24.9 MiB/30.8 MiB (12.6 MiB/s) with 3 file(s) remaining Completed 25.1 MiB/30.8 MiB (12.8 MiB/s) with 3 file(s) remaining Completed 25.4 MiB/30.8 MiB (12.9 MiB/s) with 3 file(s) remaining Completed 25.6 MiB/30.8 MiB (13.0 MiB/s) with 3 file(s) remaining Completed 25.7 MiB/30.8 MiB (13.0 MiB/s) with 3 file(s) remaining download: s3://pg-extensions/documentdb-pg16-0.101-2.tar.gz to tmp/documentdb-pg16-0.101-2.tar.gz logger.go:42: 16:51:27 | custom-extensions/0-deploy-operator | Completed 25.7 MiB/30.8 MiB (13.0 MiB/s) with 2 file(s) remaining Completed 26.0 MiB/30.8 MiB (13.1 MiB/s) with 2 file(s) remaining Completed 26.2 MiB/30.8 MiB (13.2 MiB/s) with 2 file(s) remaining Completed 26.5 MiB/30.8 MiB (13.4 MiB/s) with 2 file(s) remaining Completed 26.7 MiB/30.8 MiB (13.5 MiB/s) with 2 file(s) remaining Completed 27.0 MiB/30.8 MiB (13.6 MiB/s) with 2 file(s) remaining Completed 27.2 MiB/30.8 MiB (13.7 MiB/s) with 2 file(s) remaining Completed 27.5 MiB/30.8 MiB (13.8 MiB/s) with 2 file(s) remaining Completed 27.7 MiB/30.8 MiB (13.9 MiB/s) with 2 file(s) remaining Completed 28.0 MiB/30.8 MiB (14.0 MiB/s) with 2 file(s) remaining Completed 28.2 MiB/30.8 MiB (14.2 MiB/s) with 2 file(s) remaining Completed 28.5 MiB/30.8 MiB (14.3 MiB/s) with 2 file(s) remaining Completed 28.7 MiB/30.8 MiB (14.4 MiB/s) with 2 file(s) remaining Completed 28.7 MiB/30.8 MiB (14.4 MiB/s) with 2 file(s) remaining download: s3://pg-extensions/timescaledb-pg15-2.11.2.tar.gz to tmp/timescaledb-pg15-2.11.2.tar.gz logger.go:42: 16:51:28 | custom-extensions/0-deploy-operator | Completed 28.7 MiB/30.8 MiB (14.4 MiB/s) with 1 file(s) remaining Completed 29.0 MiB/30.8 MiB (14.0 MiB/s) with 1 file(s) remaining Completed 29.2 MiB/30.8 MiB (14.0 MiB/s) with 1 file(s) remaining Completed 29.5 MiB/30.8 MiB (14.1 MiB/s) with 1 file(s) remaining Completed 29.7 MiB/30.8 MiB (14.2 MiB/s) with 1 file(s) remaining Completed 30.0 MiB/30.8 MiB (14.3 MiB/s) with 1 file(s) remaining Completed 30.2 MiB/30.8 MiB (14.5 MiB/s) with 1 file(s) remaining Completed 30.5 MiB/30.8 MiB (14.6 MiB/s) with 1 file(s) remaining Completed 30.7 MiB/30.8 MiB (14.7 MiB/s) with 1 file(s) remaining Completed 30.8 MiB/30.8 MiB (14.7 MiB/s) with 1 file(s) remaining download: s3://pg-extensions/documentdb-pg16-0.101-1.tar.gz to tmp/documentdb-pg16-0.101-1.tar.gz logger.go:42: 16:51:28 | custom-extensions/0-deploy-operator | Completed 144.3 KiB/30.8 MiB (1.4 MiB/s) with 11 file(s) remaining upload: tmp/pg_cron-pg12-1.6.1.tar.gz to s3://operator-testing/pg_cron-pg12-1.6.1.tar.gz logger.go:42: 16:51:28 | custom-extensions/0-deploy-operator | Completed 144.3 KiB/30.8 MiB (1.4 MiB/s) with 10 file(s) remaining Completed 293.0 KiB/30.8 MiB (2.8 MiB/s) with 10 file(s) remaining upload: tmp/pg_cron-pg13-1.6.1.tar.gz to s3://operator-testing/pg_cron-pg13-1.6.1.tar.gz logger.go:42: 16:51:28 | custom-extensions/0-deploy-operator | Completed 293.0 KiB/30.8 MiB (2.8 MiB/s) with 9 file(s) remaining Completed 444.4 KiB/30.8 MiB (4.2 MiB/s) with 9 file(s) remaining upload: tmp/pg_cron-pg14-1.6.1.tar.gz to s3://operator-testing/pg_cron-pg14-1.6.1.tar.gz logger.go:42: 16:51:28 | custom-extensions/0-deploy-operator | Completed 444.4 KiB/30.8 MiB (4.2 MiB/s) with 8 file(s) remaining Completed 614.4 KiB/30.8 MiB (4.4 MiB/s) with 8 file(s) remaining upload: tmp/pg_cron-pg17-1.6.4.tar.gz to s3://operator-testing/pg_cron-pg17-1.6.4.tar.gz logger.go:42: 16:51:28 | custom-extensions/0-deploy-operator | Completed 614.4 KiB/30.8 MiB (4.4 MiB/s) with 7 file(s) remaining Completed 779.8 KiB/30.8 MiB (5.5 MiB/s) with 7 file(s) remaining upload: tmp/pg_cron-pg16-1.6.1.tar.gz to s3://operator-testing/pg_cron-pg16-1.6.1.tar.gz logger.go:42: 16:51:28 | custom-extensions/0-deploy-operator | Completed 779.8 KiB/30.8 MiB (5.5 MiB/s) with 6 file(s) remaining Completed 934.7 KiB/30.8 MiB (6.1 MiB/s) with 6 file(s) remaining upload: tmp/pg_cron-pg15-1.6.1.tar.gz to s3://operator-testing/pg_cron-pg15-1.6.1.tar.gz logger.go:42: 16:51:28 | custom-extensions/0-deploy-operator | Completed 934.7 KiB/30.8 MiB (6.1 MiB/s) with 5 file(s) remaining Completed 1.1 MiB/30.8 MiB (7.0 MiB/s) with 5 file(s) remaining upload: tmp/pg_cron-pg17-1.6.1.tar.gz to s3://operator-testing/pg_cron-pg17-1.6.1.tar.gz logger.go:42: 16:51:28 | custom-extensions/0-deploy-operator | Completed 1.1 MiB/30.8 MiB (7.0 MiB/s) with 4 file(s) remaining Completed 1.3 MiB/30.8 MiB (7.8 MiB/s) with 4 file(s) remaining Completed 1.6 MiB/30.8 MiB (9.1 MiB/s) with 4 file(s) remaining Completed 1.8 MiB/30.8 MiB (10.4 MiB/s) with 4 file(s) remaining Completed 2.1 MiB/30.8 MiB (11.6 MiB/s) with 4 file(s) remaining Completed 2.3 MiB/30.8 MiB (12.7 MiB/s) with 4 file(s) remaining Completed 2.6 MiB/30.8 MiB (13.8 MiB/s) with 4 file(s) remaining Completed 2.8 MiB/30.8 MiB (14.9 MiB/s) with 4 file(s) remaining Completed 3.1 MiB/30.8 MiB (16.0 MiB/s) with 4 file(s) remaining Completed 3.3 MiB/30.8 MiB (17.1 MiB/s) with 4 file(s) remaining Completed 3.6 MiB/30.8 MiB (18.0 MiB/s) with 4 file(s) remaining Completed 3.8 MiB/30.8 MiB (19.3 MiB/s) with 4 file(s) remaining Completed 4.1 MiB/30.8 MiB (20.3 MiB/s) with 4 file(s) remaining Completed 4.3 MiB/30.8 MiB (21.8 MiB/s) with 4 file(s) remaining Completed 4.6 MiB/30.8 MiB (22.5 MiB/s) with 4 file(s) remaining Completed 4.8 MiB/30.8 MiB (23.5 MiB/s) with 4 file(s) remaining Completed 5.1 MiB/30.8 MiB (24.3 MiB/s) with 4 file(s) remaining Completed 5.3 MiB/30.8 MiB (25.3 MiB/s) with 4 file(s) remaining Completed 5.6 MiB/30.8 MiB (26.4 MiB/s) with 4 file(s) remaining Completed 5.8 MiB/30.8 MiB (27.3 MiB/s) with 4 file(s) remaining Completed 6.1 MiB/30.8 MiB (28.3 MiB/s) with 4 file(s) remaining Completed 6.3 MiB/30.8 MiB (29.3 MiB/s) with 4 file(s) remaining Completed 6.6 MiB/30.8 MiB (30.4 MiB/s) with 4 file(s) remaining Completed 6.8 MiB/30.8 MiB (30.6 MiB/s) with 4 file(s) remaining Completed 7.1 MiB/30.8 MiB (31.6 MiB/s) with 4 file(s) remaining Completed 7.3 MiB/30.8 MiB (32.6 MiB/s) with 4 file(s) remaining Completed 7.6 MiB/30.8 MiB (33.2 MiB/s) with 4 file(s) remaining Completed 7.8 MiB/30.8 MiB (34.1 MiB/s) with 4 file(s) remaining Completed 8.1 MiB/30.8 MiB (34.9 MiB/s) with 4 file(s) remaining Completed 8.3 MiB/30.8 MiB (36.1 MiB/s) with 4 file(s) remaining Completed 8.6 MiB/30.8 MiB (36.9 MiB/s) with 4 file(s) remaining Completed 8.8 MiB/30.8 MiB (37.4 MiB/s) with 4 file(s) remaining Completed 9.1 MiB/30.8 MiB (38.2 MiB/s) with 4 file(s) remaining Completed 9.3 MiB/30.8 MiB (39.1 MiB/s) with 4 file(s) remaining Completed 9.6 MiB/30.8 MiB (40.0 MiB/s) with 4 file(s) remaining Completed 9.8 MiB/30.8 MiB (40.5 MiB/s) with 4 file(s) remaining Completed 10.1 MiB/30.8 MiB (41.3 MiB/s) with 4 file(s) remaining Completed 10.3 MiB/30.8 MiB (41.9 MiB/s) with 4 file(s) remaining Completed 10.6 MiB/30.8 MiB (42.9 MiB/s) with 4 file(s) remaining Completed 10.8 MiB/30.8 MiB (43.9 MiB/s) with 4 file(s) remaining Completed 11.1 MiB/30.8 MiB (44.4 MiB/s) with 4 file(s) remaining Completed 11.3 MiB/30.8 MiB (45.0 MiB/s) with 4 file(s) remaining Completed 11.6 MiB/30.8 MiB (45.8 MiB/s) with 4 file(s) remaining Completed 11.8 MiB/30.8 MiB (46.3 MiB/s) with 4 file(s) remaining Completed 12.1 MiB/30.8 MiB (47.1 MiB/s) with 4 file(s) remaining Completed 12.3 MiB/30.8 MiB (47.8 MiB/s) with 4 file(s) remaining Completed 12.6 MiB/30.8 MiB (48.7 MiB/s) with 4 file(s) remaining Completed 12.8 MiB/30.8 MiB (49.0 MiB/s) with 4 file(s) remaining Completed 13.1 MiB/30.8 MiB (49.8 MiB/s) with 4 file(s) remaining Completed 13.3 MiB/30.8 MiB (50.6 MiB/s) with 4 file(s) remaining Completed 13.6 MiB/30.8 MiB (51.3 MiB/s) with 4 file(s) remaining Completed 13.8 MiB/30.8 MiB (52.0 MiB/s) with 4 file(s) remaining Completed 14.1 MiB/30.8 MiB (52.6 MiB/s) with 4 file(s) remaining Completed 14.3 MiB/30.8 MiB (53.2 MiB/s) with 4 file(s) remaining Completed 14.6 MiB/30.8 MiB (53.8 MiB/s) with 4 file(s) remaining Completed 14.8 MiB/30.8 MiB (54.7 MiB/s) with 4 file(s) remaining Completed 15.1 MiB/30.8 MiB (54.9 MiB/s) with 4 file(s) remaining Completed 15.3 MiB/30.8 MiB (55.4 MiB/s) with 4 file(s) remaining Completed 15.6 MiB/30.8 MiB (56.2 MiB/s) with 4 file(s) remaining Completed 15.8 MiB/30.8 MiB (56.8 MiB/s) with 4 file(s) remaining Completed 16.1 MiB/30.8 MiB (57.7 MiB/s) with 4 file(s) remaining Completed 16.3 MiB/30.8 MiB (57.8 MiB/s) with 4 file(s) remaining Completed 16.6 MiB/30.8 MiB (58.4 MiB/s) with 4 file(s) remaining Completed 16.8 MiB/30.8 MiB (58.9 MiB/s) with 4 file(s) remaining Completed 17.1 MiB/30.8 MiB (59.6 MiB/s) with 4 file(s) remaining Completed 17.3 MiB/30.8 MiB (60.0 MiB/s) with 4 file(s) remaining Completed 17.6 MiB/30.8 MiB (60.6 MiB/s) with 4 file(s) remaining Completed 17.8 MiB/30.8 MiB (60.9 MiB/s) with 4 file(s) remaining Completed 18.1 MiB/30.8 MiB (61.4 MiB/s) with 4 file(s) remaining Completed 18.3 MiB/30.8 MiB (62.2 MiB/s) with 4 file(s) remaining Completed 18.6 MiB/30.8 MiB (62.7 MiB/s) with 4 file(s) remaining Completed 18.8 MiB/30.8 MiB (63.5 MiB/s) with 4 file(s) remaining Completed 19.1 MiB/30.8 MiB (63.0 MiB/s) with 4 file(s) remaining Completed 19.3 MiB/30.8 MiB (63.7 MiB/s) with 4 file(s) remaining Completed 19.6 MiB/30.8 MiB (64.5 MiB/s) with 4 file(s) remaining Completed 19.8 MiB/30.8 MiB (65.0 MiB/s) with 4 file(s) remaining Completed 20.1 MiB/30.8 MiB (65.3 MiB/s) with 4 file(s) remaining Completed 20.3 MiB/30.8 MiB (65.7 MiB/s) with 4 file(s) remaining Completed 20.6 MiB/30.8 MiB (66.0 MiB/s) with 4 file(s) remaining Completed 20.8 MiB/30.8 MiB (66.6 MiB/s) with 4 file(s) remaining Completed 21.1 MiB/30.8 MiB (67.1 MiB/s) with 4 file(s) remaining Completed 21.3 MiB/30.8 MiB (67.5 MiB/s) with 4 file(s) remaining Completed 21.6 MiB/30.8 MiB (68.2 MiB/s) with 4 file(s) remaining Completed 21.8 MiB/30.8 MiB (68.7 MiB/s) with 4 file(s) remaining Completed 22.1 MiB/30.8 MiB (69.3 MiB/s) with 4 file(s) remaining Completed 22.3 MiB/30.8 MiB (69.7 MiB/s) with 4 file(s) remaining Completed 22.6 MiB/30.8 MiB (69.8 MiB/s) with 4 file(s) remaining Completed 22.8 MiB/30.8 MiB (70.2 MiB/s) with 4 file(s) remaining Completed 23.1 MiB/30.8 MiB (70.6 MiB/s) with 4 file(s) remaining Completed 23.3 MiB/30.8 MiB (70.9 MiB/s) with 4 file(s) remaining Completed 23.6 MiB/30.8 MiB (71.5 MiB/s) with 4 file(s) remaining Completed 23.8 MiB/30.8 MiB (72.0 MiB/s) with 4 file(s) remaining Completed 24.1 MiB/30.8 MiB (72.5 MiB/s) with 4 file(s) remaining Completed 24.3 MiB/30.8 MiB (72.8 MiB/s) with 4 file(s) remaining Completed 24.6 MiB/30.8 MiB (73.3 MiB/s) with 4 file(s) remaining Completed 24.8 MiB/30.8 MiB (73.8 MiB/s) with 4 file(s) remaining Completed 25.1 MiB/30.8 MiB (74.2 MiB/s) with 4 file(s) remaining Completed 25.3 MiB/30.8 MiB (74.0 MiB/s) with 4 file(s) remaining Completed 25.6 MiB/30.8 MiB (74.3 MiB/s) with 4 file(s) remaining Completed 25.8 MiB/30.8 MiB (74.6 MiB/s) with 4 file(s) remaining Completed 26.1 MiB/30.8 MiB (75.2 MiB/s) with 4 file(s) remaining Completed 26.3 MiB/30.8 MiB (75.4 MiB/s) with 4 file(s) remaining Completed 26.6 MiB/30.8 MiB (75.9 MiB/s) with 4 file(s) remaining Completed 26.8 MiB/30.8 MiB (76.0 MiB/s) with 4 file(s) remaining Completed 27.1 MiB/30.8 MiB (76.6 MiB/s) with 4 file(s) remaining Completed 27.3 MiB/30.8 MiB (77.1 MiB/s) with 4 file(s) remaining Completed 27.6 MiB/30.8 MiB (77.4 MiB/s) with 4 file(s) remaining Completed 27.8 MiB/30.8 MiB (77.7 MiB/s) with 4 file(s) remaining Completed 28.1 MiB/30.8 MiB (78.2 MiB/s) with 4 file(s) remaining Completed 28.3 MiB/30.8 MiB (78.9 MiB/s) with 4 file(s) remaining Completed 28.6 MiB/30.8 MiB (79.0 MiB/s) with 4 file(s) remaining Completed 28.8 MiB/30.8 MiB (79.5 MiB/s) with 4 file(s) remaining Completed 29.1 MiB/30.8 MiB (79.7 MiB/s) with 4 file(s) remaining Completed 29.3 MiB/30.8 MiB (80.1 MiB/s) with 4 file(s) remaining Completed 29.6 MiB/30.8 MiB (80.5 MiB/s) with 4 file(s) remaining Completed 29.8 MiB/30.8 MiB (80.9 MiB/s) with 4 file(s) remaining Completed 30.1 MiB/30.8 MiB (80.7 MiB/s) with 4 file(s) remaining Completed 30.1 MiB/30.8 MiB (80.3 MiB/s) with 4 file(s) remaining upload: tmp/documentdb-pg16-0.101-2.tar.gz to s3://operator-testing/documentdb-pg16-0.101-2.tar.gz logger.go:42: 16:51:28 | custom-extensions/0-deploy-operator | Completed 30.1 MiB/30.8 MiB (80.3 MiB/s) with 3 file(s) remaining Completed 30.4 MiB/30.8 MiB (80.5 MiB/s) with 3 file(s) remaining Completed 30.6 MiB/30.8 MiB (80.9 MiB/s) with 3 file(s) remaining Completed 30.7 MiB/30.8 MiB (73.4 MiB/s) with 3 file(s) remaining upload: tmp/documentdb-pg16-0.101-0.tar.gz to s3://operator-testing/documentdb-pg16-0.101-0.tar.gz logger.go:42: 16:51:28 | custom-extensions/0-deploy-operator | Completed 30.7 MiB/30.8 MiB (73.4 MiB/s) with 2 file(s) remaining Completed 30.7 MiB/30.8 MiB (73.1 MiB/s) with 2 file(s) remaining Completed 30.8 MiB/30.8 MiB (73.0 MiB/s) with 2 file(s) remaining upload: tmp/timescaledb-pg15-2.11.2.tar.gz to s3://operator-testing/timescaledb-pg15-2.11.2.tar.gz logger.go:42: 16:51:28 | custom-extensions/0-deploy-operator | Completed 30.8 MiB/30.8 MiB (73.0 MiB/s) with 1 file(s) remaining upload: tmp/documentdb-pg16-0.101-1.tar.gz to s3://operator-testing/documentdb-pg16-0.101-1.tar.gz logger.go:42: 16:51:30 | custom-extensions/0-deploy-operator | pod "aws-cli" deleted logger.go:42: 16:51:31 | custom-extensions/0-deploy-operator | running command: [sh -c kubectl assert exist-enhanced deployment percona-postgresql-operator -n ${OPERATOR_NS:-$NAMESPACE} --field-selector status.readyReplicas=1] logger.go:42: 16:51:31 | custom-extensions/0-deploy-operator | ASSERT deployment percona-postgresql-operator matching field criteria 'status.readyReplicas=1' should exist. logger.go:42: 16:51:32 | custom-extensions/0-deploy-operator | INFO Found 1 resource(s). logger.go:42: 16:51:32 | custom-extensions/0-deploy-operator | NAME NAMESPACE COL0 logger.go:42: 16:51:32 | custom-extensions/0-deploy-operator | percona-postgresql-operator pg-operator 1 logger.go:42: 16:51:32 | custom-extensions/0-deploy-operator | ASSERT PASS logger.go:42: 16:51:32 | custom-extensions/0-deploy-operator | test step completed 0-deploy-operator logger.go:42: 16:51:32 | custom-extensions/1-create-cluster | starting test step 1-create-cluster logger.go:42: 16:51:32 | custom-extensions/1-create-cluster | running command: [sh -c set -o errexit set -o xtrace source ../../functions get_cr | kubectl -n "${NAMESPACE}" apply -f -] logger.go:42: 16:51:32 | custom-extensions/1-create-cluster | + source ../../functions logger.go:42: 16:51:32 | custom-extensions/1-create-cluster | +++ realpath ../../.. logger.go:42: 16:51:32 | custom-extensions/1-create-cluster | ++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1214 logger.go:42: 16:51:32 | custom-extensions/1-create-cluster | ++ CERT_MANAGER_VER=1.17.1 logger.go:42: 16:51:32 | custom-extensions/1-create-cluster | ++++ pwd logger.go:42: 16:51:32 | custom-extensions/1-create-cluster | +++ basename /mnt/jenkins/workspace/cloud-pg-operator_PR-1214/e2e-tests/tests/custom-extensions logger.go:42: 16:51:32 | custom-extensions/1-create-cluster | ++ test_name=custom-extensions logger.go:42: 16:51:32 | custom-extensions/1-create-cluster | ++ source /mnt/jenkins/workspace/cloud-pg-operator_PR-1214/e2e-tests/vars.sh logger.go:42: 16:51:32 | custom-extensions/1-create-cluster | +++ export ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1214 logger.go:42: 16:51:32 | custom-extensions/1-create-cluster | +++ ROOT_REPO=/mnt/jenkins/workspace/cloud-pg-operator_PR-1214 logger.go:42: 16:51:32 | custom-extensions/1-create-cluster | +++ export DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1214/deploy logger.go:42: 16:51:32 | custom-extensions/1-create-cluster | +++ DEPLOY_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1214/deploy logger.go:42: 16:51:32 | custom-extensions/1-create-cluster | +++ export TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1214/e2e-tests logger.go:42: 16:51:32 | custom-extensions/1-create-cluster | +++ TESTS_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1214/e2e-tests logger.go:42: 16:51:32 | custom-extensions/1-create-cluster | +++ export TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1214/e2e-tests/conf logger.go:42: 16:51:32 | custom-extensions/1-create-cluster | +++ TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-pg-operator_PR-1214/e2e-tests/conf logger.go:42: 16:51:32 | custom-extensions/1-create-cluster | +++ export TEMP_DIR=/tmp/kuttl/pg/custom-extensions logger.go:42: 16:51:32 | custom-extensions/1-create-cluster | +++ TEMP_DIR=/tmp/kuttl/pg/custom-extensions logger.go:42: 16:51:32 | custom-extensions/1-create-cluster | ++++ git rev-parse --abbrev-ref HEAD logger.go:42: 16:51:32 | custom-extensions/1-create-cluster | +++ export GIT_BRANCH=PR-1214 logger.go:42: 16:51:32 | custom-extensions/1-create-cluster | +++ GIT_BRANCH=PR-1214 logger.go:42: 16:51:32 | custom-extensions/1-create-cluster | +++ export VERSION=PR-1214-6ef3702c0 logger.go:42: 16:51:32 | custom-extensions/1-create-cluster | +++ VERSION=PR-1214-6ef3702c0 logger.go:42: 16:51:32 | custom-extensions/1-create-cluster | +++ export IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 16:51:32 | custom-extensions/1-create-cluster | +++ IMAGE_BASE=perconalab/percona-postgresql-operator logger.go:42: 16:51:32 | custom-extensions/1-create-cluster | +++ export IMAGE=perconalab/percona-postgresql-operator:PR-1214-6ef3702c0 logger.go:42: 16:51:32 | custom-extensions/1-create-cluster | +++ IMAGE=perconalab/percona-postgresql-operator:PR-1214-6ef3702c0 logger.go:42: 16:51:32 | custom-extensions/1-create-cluster | +++ export PG_VER=17 logger.go:42: 16:51:32 | custom-extensions/1-create-cluster | +++ PG_VER=17 logger.go:42: 16:51:32 | custom-extensions/1-create-cluster | +++ export IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer17 logger.go:42: 16:51:32 | custom-extensions/1-create-cluster | +++ IMAGE_PGBOUNCER=perconalab/percona-postgresql-operator:main-pgbouncer17 logger.go:42: 16:51:32 | custom-extensions/1-create-cluster | +++ export IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg17-postgres logger.go:42: 16:51:32 | custom-extensions/1-create-cluster | +++ IMAGE_POSTGRESQL=perconalab/percona-postgresql-operator:main-ppg17-postgres logger.go:42: 16:51:32 | custom-extensions/1-create-cluster | +++ export IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest17 logger.go:42: 16:51:32 | custom-extensions/1-create-cluster | +++ IMAGE_BACKREST=perconalab/percona-postgresql-operator:main-pgbackrest17 logger.go:42: 16:51:32 | custom-extensions/1-create-cluster | +++ export IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 16:51:32 | custom-extensions/1-create-cluster | +++ IMAGE_UPGRADE=perconalab/percona-postgresql-operator:main-upgrade logger.go:42: 16:51:32 | custom-extensions/1-create-cluster | +++ export BUCKET=pg-operator-testing logger.go:42: 16:51:32 | custom-extensions/1-create-cluster | +++ BUCKET=pg-operator-testing logger.go:42: 16:51:32 | custom-extensions/1-create-cluster | +++ export PMM_SERVER_VERSION=9.9.9 logger.go:42: 16:51:32 | custom-extensions/1-create-cluster | +++ PMM_SERVER_VERSION=9.9.9 logger.go:42: 16:51:32 | custom-extensions/1-create-cluster | +++ export IMAGE_PMM_CLIENT=perconalab/pmm-client:dev-latest logger.go:42: 16:51:32 | custom-extensions/1-create-cluster | +++ IMAGE_PMM_CLIENT=perconalab/pmm-client:dev-latest logger.go:42: 16:51:32 | custom-extensions/1-create-cluster | +++ export IMAGE_PMM_SERVER=perconalab/pmm-server:dev-latest logger.go:42: 16:51:32 | custom-extensions/1-create-cluster | +++ IMAGE_PMM_SERVER=perconalab/pmm-server:dev-latest logger.go:42: 16:51:32 | custom-extensions/1-create-cluster | +++ export IMAGE_PMM3_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 16:51:32 | custom-extensions/1-create-cluster | +++ IMAGE_PMM3_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 16:51:32 | custom-extensions/1-create-cluster | +++ export IMAGE_PMM3_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 16:51:32 | custom-extensions/1-create-cluster | +++ IMAGE_PMM3_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 16:51:32 | custom-extensions/1-create-cluster | +++ export PGOV1_TAG=1.4.0 logger.go:42: 16:51:32 | custom-extensions/1-create-cluster | +++ PGOV1_TAG=1.4.0 logger.go:42: 16:51:32 | custom-extensions/1-create-cluster | +++ export PGOV1_VER=14 logger.go:42: 16:51:32 | custom-extensions/1-create-cluster | +++ PGOV1_VER=14 logger.go:42: 16:51:32 | custom-extensions/1-create-cluster | +++ export MINIO_VER=5.4.0 logger.go:42: 16:51:32 | custom-extensions/1-create-cluster | +++ MINIO_VER=5.4.0 logger.go:42: 16:51:32 | custom-extensions/1-create-cluster | ++++ which gdate logger.go:42: 16:51:32 | custom-extensions/1-create-cluster | which: no gdate in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1214/bin/:/home/ec2-user/.krew/bin:/usr/local/bin:/usr/bin) logger.go:42: 16:51:32 | custom-extensions/1-create-cluster | ++++ which date logger.go:42: 16:51:32 | custom-extensions/1-create-cluster | +++ date=/usr/bin/date logger.go:42: 16:51:32 | custom-extensions/1-create-cluster | ++++ which gsed logger.go:42: 16:51:32 | custom-extensions/1-create-cluster | which: no gsed in (/mnt/jenkins/workspace/cloud-pg-operator_PR-1214/bin/:/home/ec2-user/.krew/bin:/usr/local/bin:/usr/bin) logger.go:42: 16:51:32 | custom-extensions/1-create-cluster | ++++ which sed logger.go:42: 16:51:32 | custom-extensions/1-create-cluster | +++ sed=/usr/bin/sed logger.go:42: 16:51:32 | custom-extensions/1-create-cluster | +++ command -v oc logger.go:42: 16:51:32 | custom-extensions/1-create-cluster | ++ oc get projects logger.go:42: 16:51:32 | custom-extensions/1-create-cluster | + get_cr logger.go:42: 16:51:32 | custom-extensions/1-create-cluster | + kubectl -n kuttl-test-picked-cow apply -f - logger.go:42: 16:51:32 | custom-extensions/1-create-cluster | + local cr_name= logger.go:42: 16:51:32 | custom-extensions/1-create-cluster | + '[' -z ']' logger.go:42: 16:51:32 | custom-extensions/1-create-cluster | + cr_name=custom-extensions logger.go:42: 16:51:32 | custom-extensions/1-create-cluster | + local repo_path= logger.go:42: 16:51:32 | custom-extensions/1-create-cluster | + local source_path= logger.go:42: 16:51:32 | custom-extensions/1-create-cluster | + yq eval ' logger.go:42: 16:51:32 | custom-extensions/1-create-cluster | .metadata.name = "custom-extensions" | logger.go:42: 16:51:32 | custom-extensions/1-create-cluster | .metadata.labels = {"e2e":"custom-extensions"} | logger.go:42: 16:51:32 | custom-extensions/1-create-cluster | .spec.postgresVersion = 17 | logger.go:42: 16:51:32 | custom-extensions/1-create-cluster | .spec.users += [{"name":"postgres","password":{"type":"AlphaNumeric"}}] | logger.go:42: 16:51:32 | custom-extensions/1-create-cluster | .spec.users += [{"name":"custom-extensions","password":{"type":"AlphaNumeric"}}] | logger.go:42: 16:51:32 | custom-extensions/1-create-cluster | .spec.image = "perconalab/percona-postgresql-operator:main-ppg17-postgres" | logger.go:42: 16:51:32 | custom-extensions/1-create-cluster | .spec.initContainer.image = "perconalab/percona-postgresql-operator:PR-1214-6ef3702c0" | logger.go:42: 16:51:32 | custom-extensions/1-create-cluster | .spec.backups.pgbackrest.image = "perconalab/percona-postgresql-operator:main-pgbackrest17" | logger.go:42: 16:51:32 | custom-extensions/1-create-cluster | .spec.proxy.pgBouncer.image = "perconalab/percona-postgresql-operator:main-pgbouncer17" | logger.go:42: 16:51:32 | custom-extensions/1-create-cluster | .spec.pmm.image = "perconalab/pmm-client:dev-latest" | logger.go:42: 16:51:32 | custom-extensions/1-create-cluster | .spec.pmm.secret = "custom-extensions-pmm-secret" | logger.go:42: 16:51:32 | custom-extensions/1-create-cluster | .spec.pmm.customClusterName = "custom-extensions-pmm-custom-name" | logger.go:42: 16:51:32 | custom-extensions/1-create-cluster | .spec.pmm.postgresParams = "--environment=dev-postgres" logger.go:42: 16:51:32 | custom-extensions/1-create-cluster | ' /mnt/jenkins/workspace/cloud-pg-operator_PR-1214/deploy/cr.yaml logger.go:42: 16:51:32 | custom-extensions/1-create-cluster | + [[ -n '' ]] logger.go:42: 16:51:32 | custom-extensions/1-create-cluster | + case $test_name in logger.go:42: 16:51:32 | custom-extensions/1-create-cluster | + yq eval -i ' logger.go:42: 16:51:32 | custom-extensions/1-create-cluster | .spec.extensions.image = "perconalab/percona-postgresql-operator:PR-1214-6ef3702c0" | logger.go:42: 16:51:32 | custom-extensions/1-create-cluster | .spec.extensions.imagePullPolicy = "Always" | logger.go:42: 16:51:32 | custom-extensions/1-create-cluster | .spec.extensions.storage = {"type": "s3", "bucket": "operator-testing", "region": "us-east-1", "endpoint": "http://minio-service:9000", "forcePathStyle": "true", "disableSSL": "true", "secret": {"name": "minio-secret"}} logger.go:42: 16:51:32 | custom-extensions/1-create-cluster | ' /tmp/kuttl/pg/custom-extensions/cr.yaml logger.go:42: 16:51:32 | custom-extensions/1-create-cluster | + cat /tmp/kuttl/pg/custom-extensions/cr.yaml logger.go:42: 16:51:33 | custom-extensions/1-create-cluster | perconapgcluster.pgv2.percona.com/custom-extensions created logger.go:42: 16:52:44 | custom-extensions/1-create-cluster | test step completed 1-create-cluster logger.go:42: 16:52:44 | custom-extensions/2-install-pg_cron | starting test step 2-install-pg_cron logger.go:42: 16:52:45 | custom-extensions/2-install-pg_cron | PerconaPGCluster:kuttl-test-picked-cow/custom-extensions updated logger.go:42: 16:54:45 | custom-extensions/2-install-pg_cron | test step failed 2-install-pg_cron case.go:396: failed in step 2-install-pg_cron case.go:398: --- StatefulSet:kuttl-test-picked-cow/ +++ StatefulSet:kuttl-test-picked-cow/custom-extensions-instance1-26c9 @@ -2,9 +2,18 @@ kind: StatefulSet metadata: labels: + app.kubernetes.io/component: pg + app.kubernetes.io/instance: custom-extensions + app.kubernetes.io/managed-by: percona-postgresql-operator + app.kubernetes.io/name: percona-postgresql + app.kubernetes.io/part-of: percona-postgresql + pgv2.percona.com/version: 2.7.0 postgres-operator.crunchydata.com/cluster: custom-extensions postgres-operator.crunchydata.com/data: postgres + postgres-operator.crunchydata.com/instance: custom-extensions-instance1-26c9 postgres-operator.crunchydata.com/instance-set: instance1 + managedFields: '[... elided field over 10 lines long ...]' + name: custom-extensions-instance1-26c9 namespace: kuttl-test-picked-cow ownerReferences: - apiVersion: postgres-operator.crunchydata.com/v1beta1 @@ -12,14 +21,144 @@ controller: true kind: PostgresCluster name: custom-extensions + uid: 21f03dae-4598-4757-bee8-7bb6124f7b59 spec: + persistentVolumeClaimRetentionPolicy: + whenDeleted: Retain + whenScaled: Retain + podManagementPolicy: OrderedReady + replicas: 1 + revisionHistoryLimit: 0 + selector: + matchLabels: + postgres-operator.crunchydata.com/cluster: custom-extensions + postgres-operator.crunchydata.com/instance: custom-extensions-instance1-26c9 + postgres-operator.crunchydata.com/instance-set: instance1 + serviceName: custom-extensions-pods template: + metadata: '[... elided field over 10 lines long ...]' spec: + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchLabels: + postgres-operator.crunchydata.com/data: postgres + topologyKey: kubernetes.io/hostname + weight: 1 + containers: '[... elided field over 10 lines long ...]' + dnsPolicy: ClusterFirst + enableServiceLinks: false initContainers: - - name: postgres-startup + - command: + - bash + - -ceu + - -- + - |- + declare -r expected_major_version="$1" pgwal_directory="$2" pgbrLog_directory="$3" + permissions() { while [[ -n "$1" ]]; do set "${1%/*}" "$@"; done; shift; stat -Lc '%A %4u %4g %n' "$@"; } + halt() { local rc=$?; >&2 echo "$@"; exit "${rc/#0/1}"; } + results() { printf '::postgres-operator: %s::%s\n' "$@"; } + recreate() ( + local tmp; tmp=$(mktemp -d -p "${1%/*}"); GLOBIGNORE='.:..'; set -x + chmod "$2" "${tmp}"; mv "$1"/* "${tmp}"; rmdir "$1"; mv "${tmp}" "$1" + ) + safelink() ( + local desired="$1" name="$2" current + current=$(realpath "${name}") + if [[ "${current}" == "${desired}" ]]; then return; fi + set -x; mv --no-target-directory "${current}" "${desired}" + ln --no-dereference --force --symbolic "${desired}" "${name}" + ) + echo Initializing ... + results 'uid' "$(id -u ||:)" 'gid' "$(id -G ||:)" + if [[ "${pgwal_directory}" == *"pgwal/"* ]] && [[ ! -d "/pgwal/pgbackrest-spool" ]];then rm -rf "/pgdata/pgbackrest-spool" && mkdir -p "/pgwal/pgbackrest-spool" && ln --force --symbolic "/pgwal/pgbackrest-spool" "/pgdata/pgbackrest-spool";fi + if [[ ! -e "/pgdata/pgbackrest-spool" ]];then rm -rf /pgdata/pgbackrest-spool;fi + results 'postgres path' "$(command -v postgres ||:)" + results 'postgres version' "${postgres_version:=$(postgres --version ||:)}" + [[ "${postgres_version}" =~ ") ${expected_major_version}"($|[^0-9]) ]] || + halt Expected PostgreSQL version "${expected_major_version}" + results 'config directory' "${PGDATA:?}" + postgres_data_directory=$([[ -d "${PGDATA}" ]] && postgres -C data_directory || echo "${PGDATA}") + results 'data directory' "${postgres_data_directory}" + [[ "${postgres_data_directory}" == "${PGDATA}" ]] || + halt Expected matching config and data directories + bootstrap_dir="${postgres_data_directory}_bootstrap" + [[ -d "${bootstrap_dir}" ]] && results 'bootstrap directory' "${bootstrap_dir}" + [[ -d "${bootstrap_dir}" ]] && postgres_data_directory="${bootstrap_dir}" + if [[ ! -e "${postgres_data_directory}" || -O "${postgres_data_directory}" ]]; then + install --directory --mode=0700 "${postgres_data_directory}" + elif [[ -w "${postgres_data_directory}" && -g "${postgres_data_directory}" ]]; then + recreate "${postgres_data_directory}" '0700' + else (halt Permissions!); fi || + halt "$(permissions "${postgres_data_directory}" ||:)" + results 'pgBackRest log directory' "${pgbrLog_directory}" + install --directory --mode=0775 "${pgbrLog_directory}" || + halt "$(permissions "${pgbrLog_directory}" ||:)" + install -D --mode=0600 -t "/tmp/replication" "/pgconf/tls/replication"/{tls.crt,tls.key,ca.crt} + + [[ -f "${postgres_data_directory}/PG_VERSION" ]] || exit 0 + results 'data version' "${postgres_data_version:=$(< "${postgres_data_directory}/PG_VERSION")}" + [[ "${postgres_data_version}" == "${expected_major_version}" ]] || + halt Expected PostgreSQL data version "${expected_major_version}" + [[ ! -f "${postgres_data_directory}/postgresql.conf" ]] && + touch "${postgres_data_directory}/postgresql.conf" + safelink "${pgwal_directory}" "${postgres_data_directory}/pg_wal" + results 'wal directory' "$(realpath "${postgres_data_directory}/pg_wal" ||:)" + rm -f "${postgres_data_directory}/recovery.signal" + - startup + - "17" + - /pgdata/pg17_wal + - /pgdata/pgbackrest/log + env: + - name: PGDATA + value: /pgdata/pg17 + - name: PGHOST + value: /tmp/postgres + - name: PGPORT + value: "5432" + - name: KRB5_CONFIG + value: /etc/postgres/krb5.conf + - name: KRB5RCACHEDIR + value: /tmp + - name: LDAPTLS_CACERT + value: /etc/postgres/ldap/ca.crt + image: perconalab/percona-postgresql-operator:main-ppg17-postgres + imagePullPolicy: Always + name: postgres-startup + resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + privileged: false + readOnlyRootFilesystem: true + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /pgconf/tls + name: cert-volume + readOnly: true + - mountPath: /pgdata + name: postgres-data + - mountPath: /tmp + name: tmp - command: - /usr/local/bin/relocate-extensions.sh - resources: {} + env: + - name: PG_VERSION + value: "17" + image: perconalab/percona-postgresql-operator:main-ppg17-postgres + imagePullPolicy: Always + name: extension-relocator-17 + resources: {} + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File volumeMounts: - mountPath: /pgdata name: postgres-data @@ -31,35 +170,135 @@ - name: STORAGE_TYPE value: s3 - name: STORAGE_ENDPOINT + value: http://minio-service:9000 - name: STORAGE_REGION value: us-east-1 - name: STORAGE_BUCKET value: operator-testing - name: INSTALL_EXTENSIONS + value: pg_cron-pg17-1.6.1 - name: PG_VERSION + value: "17" - name: PGDATA_EXTENSIONS - - name: STORAGE_DISABLE_SSL - value: "true" - - name: STORAGE_FORCE_PATH_STYLE - value: "true" + value: /pgdata/extension/17 envFrom: - secretRef: name: minio-secret - resources: {} + image: perconalab/percona-postgresql-operator:PR-1214-6ef3702c0 + imagePullPolicy: Always + name: extension-installer-17 + resources: {} + securityContext: + runAsUser: 26 + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File volumeMounts: - mountPath: /pgdata name: postgres-data - - name: postgres-data - - name: postgres-data + - mountPath: /usr/pgsql-17/share/extension + name: postgres-data + subPath: extension/17/usr/pgsql-17/share/extension + - mountPath: /usr/pgsql-17/lib + name: postgres-data + subPath: extension/17/usr/pgsql-17/lib - mountPath: /tmp name: tmp - command: - /usr/local/bin/init-entrypoint.sh - - name: nss-wrapper-init + image: perconalab/percona-postgresql-operator:PR-1214-6ef3702c0 + imagePullPolicy: Always + name: database-init + resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + privileged: false + readOnlyRootFilesystem: true + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /opt/crunchy + name: crunchy-bin + - mountPath: /tmp + name: tmp + - command: + - bash + - -c + - "export NSS_WRAPPER_SUBDIR=postgres CRUNCHY_NSS_USERNAME=postgres CRUNCHY_NSS_USER_DESC=\"postgres\" + \n# Define nss_wrapper directory and passwd & group files that will be utilized + by nss_wrapper. The\n# nss_wrapper_env.sh script (which also sets these + vars) isn't sourced here since the nss_wrapper\n# has not yet been setup, + and we therefore don't yet want the nss_wrapper vars in the environment.\nmkdir + -p /tmp/nss_wrapper\nchmod g+rwx /tmp/nss_wrapper\n\nNSS_WRAPPER_DIR=\"/tmp/nss_wrapper/${NSS_WRAPPER_SUBDIR}\"\nNSS_WRAPPER_PASSWD=\"${NSS_WRAPPER_DIR}/passwd\"\nNSS_WRAPPER_GROUP=\"${NSS_WRAPPER_DIR}/group\"\n\n# + create the nss_wrapper directory\nmkdir -p \"${NSS_WRAPPER_DIR}\"\n\n# grab + the current user ID and group ID\nUSER_ID=$(id -u)\nexport USER_ID\nGROUP_ID=$(id + -g)\nexport GROUP_ID\n\n# get copies of the passwd and group files\n[[ -f + \"${NSS_WRAPPER_PASSWD}\" ]] || cp \"/etc/passwd\" \"${NSS_WRAPPER_PASSWD}\"\n[[ + -f \"${NSS_WRAPPER_GROUP}\" ]] || cp \"/etc/group\" \"${NSS_WRAPPER_GROUP}\"\n\n# + if the username is missing from the passwd file, then add it\nif [[ ! $(cat + \"${NSS_WRAPPER_PASSWD}\") =~ ${CRUNCHY_NSS_USERNAME}:x:${USER_ID} ]]; then\n + \ echo \"nss_wrapper: adding user\"\n passwd_tmp=\"${NSS_WRAPPER_DIR}/passwd_tmp\"\n + \ cp \"${NSS_WRAPPER_PASSWD}\" \"${passwd_tmp}\"\n sed -i \"/${CRUNCHY_NSS_USERNAME}:x:/d\" + \"${passwd_tmp}\"\n # needed for OCP 4.x because crio updates /etc/passwd + with an entry for USER_ID\n sed -i \"/${USER_ID}:x:/d\" \"${passwd_tmp}\"\n + \ printf '${CRUNCHY_NSS_USERNAME}:x:${USER_ID}:${GROUP_ID}:${CRUNCHY_NSS_USER_DESC}:${HOME}:/bin/bash\\n' + >> \"${passwd_tmp}\"\n envsubst < \"${passwd_tmp}\" > \"${NSS_WRAPPER_PASSWD}\"\n + \ rm \"${passwd_tmp}\"\nelse\n echo \"nss_wrapper: user exists\"\nfi\n\n# + if the username (which will be the same as the group name) is missing from + group file, then add it\nif [[ ! $(cat \"${NSS_WRAPPER_GROUP}\") =~ ${CRUNCHY_NSS_USERNAME}:x:${USER_ID} + ]]; then\n echo \"nss_wrapper: adding group\"\n group_tmp=\"${NSS_WRAPPER_DIR}/group_tmp\"\n + \ cp \"${NSS_WRAPPER_GROUP}\" \"${group_tmp}\"\n sed -i \"/${CRUNCHY_NSS_USERNAME}:x:/d\" + \"${group_tmp}\"\n printf '${CRUNCHY_NSS_USERNAME}:x:${USER_ID}:${CRUNCHY_NSS_USERNAME}\\n' + >> \"${group_tmp}\"\n envsubst < \"${group_tmp}\" > \"${NSS_WRAPPER_GROUP}\"\n + \ rm \"${group_tmp}\"\nelse\n echo \"nss_wrapper: group exists\"\nfi\n\n# + export the nss_wrapper env vars\n# define nss_wrapper directory and passwd + & group files that will be utilized by nss_wrapper\nNSS_WRAPPER_DIR=\"/tmp/nss_wrapper/${NSS_WRAPPER_SUBDIR}\"\nNSS_WRAPPER_PASSWD=\"${NSS_WRAPPER_DIR}/passwd\"\nNSS_WRAPPER_GROUP=\"${NSS_WRAPPER_DIR}/group\"\n\nexport + LD_PRELOAD=/usr/lib64/libnss_wrapper.so\nexport NSS_WRAPPER_PASSWD=\"${NSS_WRAPPER_PASSWD}\"\nexport + NSS_WRAPPER_GROUP=\"${NSS_WRAPPER_GROUP}\"\n\necho \"nss_wrapper: environment + configured\"\n" + image: perconalab/percona-postgresql-operator:main-ppg17-postgres + imagePullPolicy: Always + name: nss-wrapper-init + resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + privileged: false + readOnlyRootFilesystem: true + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /tmp + name: tmp + restartPolicy: Always + schedulerName: default-scheduler + securityContext: + fsGroup: 26 + fsGroupChangePolicy: OnRootMismatch + serviceAccount: custom-extensions-instance + serviceAccountName: custom-extensions-instance + shareProcessNamespace: true + terminationGracePeriodSeconds: 30 + topologySpreadConstraints: '[... elided field over 10 lines long ...]' + volumes: '[... elided field over 10 lines long ...]' + updateStrategy: + type: OnDelete status: - availableReplicas: 1 + availableReplicas: 0 + collisionCount: 0 + currentRevision: custom-extensions-instance1-26c9-668dcdcc97 observedGeneration: 2 - readyReplicas: 1 replicas: 1 + updateRevision: custom-extensions-instance1-26c9-68d65559c9 updatedReplicas: 1 case.go:398: resource StatefulSet:kuttl-test-picked-cow/: .spec.template.spec.initContainers.env: slice length mismatch: 9 != 7 case.go:398: --- StatefulSet:kuttl-test-picked-cow/ +++ StatefulSet:kuttl-test-picked-cow/custom-extensions-instance1-46tm @@ -2,9 +2,18 @@ kind: StatefulSet metadata: labels: + app.kubernetes.io/component: pg + app.kubernetes.io/instance: custom-extensions + app.kubernetes.io/managed-by: percona-postgresql-operator + app.kubernetes.io/name: percona-postgresql + app.kubernetes.io/part-of: percona-postgresql + pgv2.percona.com/version: 2.7.0 postgres-operator.crunchydata.com/cluster: custom-extensions postgres-operator.crunchydata.com/data: postgres + postgres-operator.crunchydata.com/instance: custom-extensions-instance1-46tm postgres-operator.crunchydata.com/instance-set: instance1 + managedFields: '[... elided field over 10 lines long ...]' + name: custom-extensions-instance1-46tm namespace: kuttl-test-picked-cow ownerReferences: - apiVersion: postgres-operator.crunchydata.com/v1beta1 @@ -12,14 +21,144 @@ controller: true kind: PostgresCluster name: custom-extensions + uid: 21f03dae-4598-4757-bee8-7bb6124f7b59 spec: + persistentVolumeClaimRetentionPolicy: + whenDeleted: Retain + whenScaled: Retain + podManagementPolicy: OrderedReady + replicas: 1 + revisionHistoryLimit: 0 + selector: + matchLabels: + postgres-operator.crunchydata.com/cluster: custom-extensions + postgres-operator.crunchydata.com/instance: custom-extensions-instance1-46tm + postgres-operator.crunchydata.com/instance-set: instance1 + serviceName: custom-extensions-pods template: + metadata: '[... elided field over 10 lines long ...]' spec: + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchLabels: + postgres-operator.crunchydata.com/data: postgres + topologyKey: kubernetes.io/hostname + weight: 1 + containers: '[... elided field over 10 lines long ...]' + dnsPolicy: ClusterFirst + enableServiceLinks: false initContainers: - - name: postgres-startup + - command: + - bash + - -ceu + - -- + - |- + declare -r expected_major_version="$1" pgwal_directory="$2" pgbrLog_directory="$3" + permissions() { while [[ -n "$1" ]]; do set "${1%/*}" "$@"; done; shift; stat -Lc '%A %4u %4g %n' "$@"; } + halt() { local rc=$?; >&2 echo "$@"; exit "${rc/#0/1}"; } + results() { printf '::postgres-operator: %s::%s\n' "$@"; } + recreate() ( + local tmp; tmp=$(mktemp -d -p "${1%/*}"); GLOBIGNORE='.:..'; set -x + chmod "$2" "${tmp}"; mv "$1"/* "${tmp}"; rmdir "$1"; mv "${tmp}" "$1" + ) + safelink() ( + local desired="$1" name="$2" current + current=$(realpath "${name}") + if [[ "${current}" == "${desired}" ]]; then return; fi + set -x; mv --no-target-directory "${current}" "${desired}" + ln --no-dereference --force --symbolic "${desired}" "${name}" + ) + echo Initializing ... + results 'uid' "$(id -u ||:)" 'gid' "$(id -G ||:)" + if [[ "${pgwal_directory}" == *"pgwal/"* ]] && [[ ! -d "/pgwal/pgbackrest-spool" ]];then rm -rf "/pgdata/pgbackrest-spool" && mkdir -p "/pgwal/pgbackrest-spool" && ln --force --symbolic "/pgwal/pgbackrest-spool" "/pgdata/pgbackrest-spool";fi + if [[ ! -e "/pgdata/pgbackrest-spool" ]];then rm -rf /pgdata/pgbackrest-spool;fi + results 'postgres path' "$(command -v postgres ||:)" + results 'postgres version' "${postgres_version:=$(postgres --version ||:)}" + [[ "${postgres_version}" =~ ") ${expected_major_version}"($|[^0-9]) ]] || + halt Expected PostgreSQL version "${expected_major_version}" + results 'config directory' "${PGDATA:?}" + postgres_data_directory=$([[ -d "${PGDATA}" ]] && postgres -C data_directory || echo "${PGDATA}") + results 'data directory' "${postgres_data_directory}" + [[ "${postgres_data_directory}" == "${PGDATA}" ]] || + halt Expected matching config and data directories + bootstrap_dir="${postgres_data_directory}_bootstrap" + [[ -d "${bootstrap_dir}" ]] && results 'bootstrap directory' "${bootstrap_dir}" + [[ -d "${bootstrap_dir}" ]] && postgres_data_directory="${bootstrap_dir}" + if [[ ! -e "${postgres_data_directory}" || -O "${postgres_data_directory}" ]]; then + install --directory --mode=0700 "${postgres_data_directory}" + elif [[ -w "${postgres_data_directory}" && -g "${postgres_data_directory}" ]]; then + recreate "${postgres_data_directory}" '0700' + else (halt Permissions!); fi || + halt "$(permissions "${postgres_data_directory}" ||:)" + results 'pgBackRest log directory' "${pgbrLog_directory}" + install --directory --mode=0775 "${pgbrLog_directory}" || + halt "$(permissions "${pgbrLog_directory}" ||:)" + install -D --mode=0600 -t "/tmp/replication" "/pgconf/tls/replication"/{tls.crt,tls.key,ca.crt} + + [[ -f "${postgres_data_directory}/PG_VERSION" ]] || exit 0 + results 'data version' "${postgres_data_version:=$(< "${postgres_data_directory}/PG_VERSION")}" + [[ "${postgres_data_version}" == "${expected_major_version}" ]] || + halt Expected PostgreSQL data version "${expected_major_version}" + [[ ! -f "${postgres_data_directory}/postgresql.conf" ]] && + touch "${postgres_data_directory}/postgresql.conf" + safelink "${pgwal_directory}" "${postgres_data_directory}/pg_wal" + results 'wal directory' "$(realpath "${postgres_data_directory}/pg_wal" ||:)" + rm -f "${postgres_data_directory}/recovery.signal" + - startup + - "17" + - /pgdata/pg17_wal + - /pgdata/pgbackrest/log + env: + - name: PGDATA + value: /pgdata/pg17 + - name: PGHOST + value: /tmp/postgres + - name: PGPORT + value: "5432" + - name: KRB5_CONFIG + value: /etc/postgres/krb5.conf + - name: KRB5RCACHEDIR + value: /tmp + - name: LDAPTLS_CACERT + value: /etc/postgres/ldap/ca.crt + image: perconalab/percona-postgresql-operator:main-ppg17-postgres + imagePullPolicy: Always + name: postgres-startup + resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + privileged: false + readOnlyRootFilesystem: true + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /pgconf/tls + name: cert-volume + readOnly: true + - mountPath: /pgdata + name: postgres-data + - mountPath: /tmp + name: tmp - command: - /usr/local/bin/relocate-extensions.sh - resources: {} + env: + - name: PG_VERSION + value: "17" + image: perconalab/percona-postgresql-operator:main-ppg17-postgres + imagePullPolicy: Always + name: extension-relocator-17 + resources: {} + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File volumeMounts: - mountPath: /pgdata name: postgres-data @@ -31,35 +170,135 @@ - name: STORAGE_TYPE value: s3 - name: STORAGE_ENDPOINT + value: http://minio-service:9000 - name: STORAGE_REGION value: us-east-1 - name: STORAGE_BUCKET value: operator-testing - name: INSTALL_EXTENSIONS + value: pg_cron-pg17-1.6.1 - name: PG_VERSION + value: "17" - name: PGDATA_EXTENSIONS - - name: STORAGE_DISABLE_SSL - value: "true" - - name: STORAGE_FORCE_PATH_STYLE - value: "true" + value: /pgdata/extension/17 envFrom: - secretRef: name: minio-secret - resources: {} + image: perconalab/percona-postgresql-operator:PR-1214-6ef3702c0 + imagePullPolicy: Always + name: extension-installer-17 + resources: {} + securityContext: + runAsUser: 26 + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File volumeMounts: - mountPath: /pgdata name: postgres-data - - name: postgres-data - - name: postgres-data + - mountPath: /usr/pgsql-17/share/extension + name: postgres-data + subPath: extension/17/usr/pgsql-17/share/extension + - mountPath: /usr/pgsql-17/lib + name: postgres-data + subPath: extension/17/usr/pgsql-17/lib - mountPath: /tmp name: tmp - command: - /usr/local/bin/init-entrypoint.sh - - name: nss-wrapper-init + image: perconalab/percona-postgresql-operator:PR-1214-6ef3702c0 + imagePullPolicy: Always + name: database-init + resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + privileged: false + readOnlyRootFilesystem: true + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /opt/crunchy + name: crunchy-bin + - mountPath: /tmp + name: tmp + - command: + - bash + - -c + - "export NSS_WRAPPER_SUBDIR=postgres CRUNCHY_NSS_USERNAME=postgres CRUNCHY_NSS_USER_DESC=\"postgres\" + \n# Define nss_wrapper directory and passwd & group files that will be utilized + by nss_wrapper. The\n# nss_wrapper_env.sh script (which also sets these + vars) isn't sourced here since the nss_wrapper\n# has not yet been setup, + and we therefore don't yet want the nss_wrapper vars in the environment.\nmkdir + -p /tmp/nss_wrapper\nchmod g+rwx /tmp/nss_wrapper\n\nNSS_WRAPPER_DIR=\"/tmp/nss_wrapper/${NSS_WRAPPER_SUBDIR}\"\nNSS_WRAPPER_PASSWD=\"${NSS_WRAPPER_DIR}/passwd\"\nNSS_WRAPPER_GROUP=\"${NSS_WRAPPER_DIR}/group\"\n\n# + create the nss_wrapper directory\nmkdir -p \"${NSS_WRAPPER_DIR}\"\n\n# grab + the current user ID and group ID\nUSER_ID=$(id -u)\nexport USER_ID\nGROUP_ID=$(id + -g)\nexport GROUP_ID\n\n# get copies of the passwd and group files\n[[ -f + \"${NSS_WRAPPER_PASSWD}\" ]] || cp \"/etc/passwd\" \"${NSS_WRAPPER_PASSWD}\"\n[[ + -f \"${NSS_WRAPPER_GROUP}\" ]] || cp \"/etc/group\" \"${NSS_WRAPPER_GROUP}\"\n\n# + if the username is missing from the passwd file, then add it\nif [[ ! $(cat + \"${NSS_WRAPPER_PASSWD}\") =~ ${CRUNCHY_NSS_USERNAME}:x:${USER_ID} ]]; then\n + \ echo \"nss_wrapper: adding user\"\n passwd_tmp=\"${NSS_WRAPPER_DIR}/passwd_tmp\"\n + \ cp \"${NSS_WRAPPER_PASSWD}\" \"${passwd_tmp}\"\n sed -i \"/${CRUNCHY_NSS_USERNAME}:x:/d\" + \"${passwd_tmp}\"\n # needed for OCP 4.x because crio updates /etc/passwd + with an entry for USER_ID\n sed -i \"/${USER_ID}:x:/d\" \"${passwd_tmp}\"\n + \ printf '${CRUNCHY_NSS_USERNAME}:x:${USER_ID}:${GROUP_ID}:${CRUNCHY_NSS_USER_DESC}:${HOME}:/bin/bash\\n' + >> \"${passwd_tmp}\"\n envsubst < \"${passwd_tmp}\" > \"${NSS_WRAPPER_PASSWD}\"\n + \ rm \"${passwd_tmp}\"\nelse\n echo \"nss_wrapper: user exists\"\nfi\n\n# + if the username (which will be the same as the group name) is missing from + group file, then add it\nif [[ ! $(cat \"${NSS_WRAPPER_GROUP}\") =~ ${CRUNCHY_NSS_USERNAME}:x:${USER_ID} + ]]; then\n echo \"nss_wrapper: adding group\"\n group_tmp=\"${NSS_WRAPPER_DIR}/group_tmp\"\n + \ cp \"${NSS_WRAPPER_GROUP}\" \"${group_tmp}\"\n sed -i \"/${CRUNCHY_NSS_USERNAME}:x:/d\" + \"${group_tmp}\"\n printf '${CRUNCHY_NSS_USERNAME}:x:${USER_ID}:${CRUNCHY_NSS_USERNAME}\\n' + >> \"${group_tmp}\"\n envsubst < \"${group_tmp}\" > \"${NSS_WRAPPER_GROUP}\"\n + \ rm \"${group_tmp}\"\nelse\n echo \"nss_wrapper: group exists\"\nfi\n\n# + export the nss_wrapper env vars\n# define nss_wrapper directory and passwd + & group files that will be utilized by nss_wrapper\nNSS_WRAPPER_DIR=\"/tmp/nss_wrapper/${NSS_WRAPPER_SUBDIR}\"\nNSS_WRAPPER_PASSWD=\"${NSS_WRAPPER_DIR}/passwd\"\nNSS_WRAPPER_GROUP=\"${NSS_WRAPPER_DIR}/group\"\n\nexport + LD_PRELOAD=/usr/lib64/libnss_wrapper.so\nexport NSS_WRAPPER_PASSWD=\"${NSS_WRAPPER_PASSWD}\"\nexport + NSS_WRAPPER_GROUP=\"${NSS_WRAPPER_GROUP}\"\n\necho \"nss_wrapper: environment + configured\"\n" + image: perconalab/percona-postgresql-operator:main-ppg17-postgres + imagePullPolicy: Always + name: nss-wrapper-init + resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + privileged: false + readOnlyRootFilesystem: true + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /tmp + name: tmp + restartPolicy: Always + schedulerName: default-scheduler + securityContext: + fsGroup: 26 + fsGroupChangePolicy: OnRootMismatch + serviceAccount: custom-extensions-instance + serviceAccountName: custom-extensions-instance + shareProcessNamespace: true + terminationGracePeriodSeconds: 30 + topologySpreadConstraints: '[... elided field over 10 lines long ...]' + volumes: '[... elided field over 10 lines long ...]' + updateStrategy: + type: OnDelete status: - availableReplicas: 1 + availableReplicas: 0 + collisionCount: 0 + currentRevision: custom-extensions-instance1-46tm-7c7d7996d observedGeneration: 2 - readyReplicas: 1 replicas: 1 + updateRevision: custom-extensions-instance1-46tm-f4968d64d updatedReplicas: 1 case.go:398: resource StatefulSet:kuttl-test-picked-cow/: .spec.template.spec.initContainers.env: slice length mismatch: 9 != 7 case.go:398: --- StatefulSet:kuttl-test-picked-cow/ +++ StatefulSet:kuttl-test-picked-cow/custom-extensions-instance1-f95b @@ -2,9 +2,18 @@ kind: StatefulSet metadata: labels: + app.kubernetes.io/component: pg + app.kubernetes.io/instance: custom-extensions + app.kubernetes.io/managed-by: percona-postgresql-operator + app.kubernetes.io/name: percona-postgresql + app.kubernetes.io/part-of: percona-postgresql + pgv2.percona.com/version: 2.7.0 postgres-operator.crunchydata.com/cluster: custom-extensions postgres-operator.crunchydata.com/data: postgres + postgres-operator.crunchydata.com/instance: custom-extensions-instance1-f95b postgres-operator.crunchydata.com/instance-set: instance1 + managedFields: '[... elided field over 10 lines long ...]' + name: custom-extensions-instance1-f95b namespace: kuttl-test-picked-cow ownerReferences: - apiVersion: postgres-operator.crunchydata.com/v1beta1 @@ -12,14 +21,144 @@ controller: true kind: PostgresCluster name: custom-extensions + uid: 21f03dae-4598-4757-bee8-7bb6124f7b59 spec: + persistentVolumeClaimRetentionPolicy: + whenDeleted: Retain + whenScaled: Retain + podManagementPolicy: OrderedReady + replicas: 1 + revisionHistoryLimit: 0 + selector: + matchLabels: + postgres-operator.crunchydata.com/cluster: custom-extensions + postgres-operator.crunchydata.com/instance: custom-extensions-instance1-f95b + postgres-operator.crunchydata.com/instance-set: instance1 + serviceName: custom-extensions-pods template: + metadata: '[... elided field over 10 lines long ...]' spec: + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchLabels: + postgres-operator.crunchydata.com/data: postgres + topologyKey: kubernetes.io/hostname + weight: 1 + containers: '[... elided field over 10 lines long ...]' + dnsPolicy: ClusterFirst + enableServiceLinks: false initContainers: - - name: postgres-startup + - command: + - bash + - -ceu + - -- + - |- + declare -r expected_major_version="$1" pgwal_directory="$2" pgbrLog_directory="$3" + permissions() { while [[ -n "$1" ]]; do set "${1%/*}" "$@"; done; shift; stat -Lc '%A %4u %4g %n' "$@"; } + halt() { local rc=$?; >&2 echo "$@"; exit "${rc/#0/1}"; } + results() { printf '::postgres-operator: %s::%s\n' "$@"; } + recreate() ( + local tmp; tmp=$(mktemp -d -p "${1%/*}"); GLOBIGNORE='.:..'; set -x + chmod "$2" "${tmp}"; mv "$1"/* "${tmp}"; rmdir "$1"; mv "${tmp}" "$1" + ) + safelink() ( + local desired="$1" name="$2" current + current=$(realpath "${name}") + if [[ "${current}" == "${desired}" ]]; then return; fi + set -x; mv --no-target-directory "${current}" "${desired}" + ln --no-dereference --force --symbolic "${desired}" "${name}" + ) + echo Initializing ... + results 'uid' "$(id -u ||:)" 'gid' "$(id -G ||:)" + if [[ "${pgwal_directory}" == *"pgwal/"* ]] && [[ ! -d "/pgwal/pgbackrest-spool" ]];then rm -rf "/pgdata/pgbackrest-spool" && mkdir -p "/pgwal/pgbackrest-spool" && ln --force --symbolic "/pgwal/pgbackrest-spool" "/pgdata/pgbackrest-spool";fi + if [[ ! -e "/pgdata/pgbackrest-spool" ]];then rm -rf /pgdata/pgbackrest-spool;fi + results 'postgres path' "$(command -v postgres ||:)" + results 'postgres version' "${postgres_version:=$(postgres --version ||:)}" + [[ "${postgres_version}" =~ ") ${expected_major_version}"($|[^0-9]) ]] || + halt Expected PostgreSQL version "${expected_major_version}" + results 'config directory' "${PGDATA:?}" + postgres_data_directory=$([[ -d "${PGDATA}" ]] && postgres -C data_directory || echo "${PGDATA}") + results 'data directory' "${postgres_data_directory}" + [[ "${postgres_data_directory}" == "${PGDATA}" ]] || + halt Expected matching config and data directories + bootstrap_dir="${postgres_data_directory}_bootstrap" + [[ -d "${bootstrap_dir}" ]] && results 'bootstrap directory' "${bootstrap_dir}" + [[ -d "${bootstrap_dir}" ]] && postgres_data_directory="${bootstrap_dir}" + if [[ ! -e "${postgres_data_directory}" || -O "${postgres_data_directory}" ]]; then + install --directory --mode=0700 "${postgres_data_directory}" + elif [[ -w "${postgres_data_directory}" && -g "${postgres_data_directory}" ]]; then + recreate "${postgres_data_directory}" '0700' + else (halt Permissions!); fi || + halt "$(permissions "${postgres_data_directory}" ||:)" + results 'pgBackRest log directory' "${pgbrLog_directory}" + install --directory --mode=0775 "${pgbrLog_directory}" || + halt "$(permissions "${pgbrLog_directory}" ||:)" + install -D --mode=0600 -t "/tmp/replication" "/pgconf/tls/replication"/{tls.crt,tls.key,ca.crt} + + [[ -f "${postgres_data_directory}/PG_VERSION" ]] || exit 0 + results 'data version' "${postgres_data_version:=$(< "${postgres_data_directory}/PG_VERSION")}" + [[ "${postgres_data_version}" == "${expected_major_version}" ]] || + halt Expected PostgreSQL data version "${expected_major_version}" + [[ ! -f "${postgres_data_directory}/postgresql.conf" ]] && + touch "${postgres_data_directory}/postgresql.conf" + safelink "${pgwal_directory}" "${postgres_data_directory}/pg_wal" + results 'wal directory' "$(realpath "${postgres_data_directory}/pg_wal" ||:)" + rm -f "${postgres_data_directory}/recovery.signal" + - startup + - "17" + - /pgdata/pg17_wal + - /pgdata/pgbackrest/log + env: + - name: PGDATA + value: /pgdata/pg17 + - name: PGHOST + value: /tmp/postgres + - name: PGPORT + value: "5432" + - name: KRB5_CONFIG + value: /etc/postgres/krb5.conf + - name: KRB5RCACHEDIR + value: /tmp + - name: LDAPTLS_CACERT + value: /etc/postgres/ldap/ca.crt + image: perconalab/percona-postgresql-operator:main-ppg17-postgres + imagePullPolicy: Always + name: postgres-startup + resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + privileged: false + readOnlyRootFilesystem: true + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /pgconf/tls + name: cert-volume + readOnly: true + - mountPath: /pgdata + name: postgres-data + - mountPath: /tmp + name: tmp - command: - /usr/local/bin/relocate-extensions.sh - resources: {} + env: + - name: PG_VERSION + value: "17" + image: perconalab/percona-postgresql-operator:main-ppg17-postgres + imagePullPolicy: Always + name: extension-relocator-17 + resources: {} + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File volumeMounts: - mountPath: /pgdata name: postgres-data @@ -31,35 +170,135 @@ - name: STORAGE_TYPE value: s3 - name: STORAGE_ENDPOINT + value: http://minio-service:9000 - name: STORAGE_REGION value: us-east-1 - name: STORAGE_BUCKET value: operator-testing - name: INSTALL_EXTENSIONS + value: pg_cron-pg17-1.6.1 - name: PG_VERSION + value: "17" - name: PGDATA_EXTENSIONS - - name: STORAGE_DISABLE_SSL - value: "true" - - name: STORAGE_FORCE_PATH_STYLE - value: "true" + value: /pgdata/extension/17 envFrom: - secretRef: name: minio-secret - resources: {} + image: perconalab/percona-postgresql-operator:PR-1214-6ef3702c0 + imagePullPolicy: Always + name: extension-installer-17 + resources: {} + securityContext: + runAsUser: 26 + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File volumeMounts: - mountPath: /pgdata name: postgres-data - - name: postgres-data - - name: postgres-data + - mountPath: /usr/pgsql-17/share/extension + name: postgres-data + subPath: extension/17/usr/pgsql-17/share/extension + - mountPath: /usr/pgsql-17/lib + name: postgres-data + subPath: extension/17/usr/pgsql-17/lib - mountPath: /tmp name: tmp - command: - /usr/local/bin/init-entrypoint.sh - - name: nss-wrapper-init + image: perconalab/percona-postgresql-operator:PR-1214-6ef3702c0 + imagePullPolicy: Always + name: database-init + resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + privileged: false + readOnlyRootFilesystem: true + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /opt/crunchy + name: crunchy-bin + - mountPath: /tmp + name: tmp + - command: + - bash + - -c + - "export NSS_WRAPPER_SUBDIR=postgres CRUNCHY_NSS_USERNAME=postgres CRUNCHY_NSS_USER_DESC=\"postgres\" + \n# Define nss_wrapper directory and passwd & group files that will be utilized + by nss_wrapper. The\n# nss_wrapper_env.sh script (which also sets these + vars) isn't sourced here since the nss_wrapper\n# has not yet been setup, + and we therefore don't yet want the nss_wrapper vars in the environment.\nmkdir + -p /tmp/nss_wrapper\nchmod g+rwx /tmp/nss_wrapper\n\nNSS_WRAPPER_DIR=\"/tmp/nss_wrapper/${NSS_WRAPPER_SUBDIR}\"\nNSS_WRAPPER_PASSWD=\"${NSS_WRAPPER_DIR}/passwd\"\nNSS_WRAPPER_GROUP=\"${NSS_WRAPPER_DIR}/group\"\n\n# + create the nss_wrapper directory\nmkdir -p \"${NSS_WRAPPER_DIR}\"\n\n# grab + the current user ID and group ID\nUSER_ID=$(id -u)\nexport USER_ID\nGROUP_ID=$(id + -g)\nexport GROUP_ID\n\n# get copies of the passwd and group files\n[[ -f + \"${NSS_WRAPPER_PASSWD}\" ]] || cp \"/etc/passwd\" \"${NSS_WRAPPER_PASSWD}\"\n[[ + -f \"${NSS_WRAPPER_GROUP}\" ]] || cp \"/etc/group\" \"${NSS_WRAPPER_GROUP}\"\n\n# + if the username is missing from the passwd file, then add it\nif [[ ! $(cat + \"${NSS_WRAPPER_PASSWD}\") =~ ${CRUNCHY_NSS_USERNAME}:x:${USER_ID} ]]; then\n + \ echo \"nss_wrapper: adding user\"\n passwd_tmp=\"${NSS_WRAPPER_DIR}/passwd_tmp\"\n + \ cp \"${NSS_WRAPPER_PASSWD}\" \"${passwd_tmp}\"\n sed -i \"/${CRUNCHY_NSS_USERNAME}:x:/d\" + \"${passwd_tmp}\"\n # needed for OCP 4.x because crio updates /etc/passwd + with an entry for USER_ID\n sed -i \"/${USER_ID}:x:/d\" \"${passwd_tmp}\"\n + \ printf '${CRUNCHY_NSS_USERNAME}:x:${USER_ID}:${GROUP_ID}:${CRUNCHY_NSS_USER_DESC}:${HOME}:/bin/bash\\n' + >> \"${passwd_tmp}\"\n envsubst < \"${passwd_tmp}\" > \"${NSS_WRAPPER_PASSWD}\"\n + \ rm \"${passwd_tmp}\"\nelse\n echo \"nss_wrapper: user exists\"\nfi\n\n# + if the username (which will be the same as the group name) is missing from + group file, then add it\nif [[ ! $(cat \"${NSS_WRAPPER_GROUP}\") =~ ${CRUNCHY_NSS_USERNAME}:x:${USER_ID} + ]]; then\n echo \"nss_wrapper: adding group\"\n group_tmp=\"${NSS_WRAPPER_DIR}/group_tmp\"\n + \ cp \"${NSS_WRAPPER_GROUP}\" \"${group_tmp}\"\n sed -i \"/${CRUNCHY_NSS_USERNAME}:x:/d\" + \"${group_tmp}\"\n printf '${CRUNCHY_NSS_USERNAME}:x:${USER_ID}:${CRUNCHY_NSS_USERNAME}\\n' + >> \"${group_tmp}\"\n envsubst < \"${group_tmp}\" > \"${NSS_WRAPPER_GROUP}\"\n + \ rm \"${group_tmp}\"\nelse\n echo \"nss_wrapper: group exists\"\nfi\n\n# + export the nss_wrapper env vars\n# define nss_wrapper directory and passwd + & group files that will be utilized by nss_wrapper\nNSS_WRAPPER_DIR=\"/tmp/nss_wrapper/${NSS_WRAPPER_SUBDIR}\"\nNSS_WRAPPER_PASSWD=\"${NSS_WRAPPER_DIR}/passwd\"\nNSS_WRAPPER_GROUP=\"${NSS_WRAPPER_DIR}/group\"\n\nexport + LD_PRELOAD=/usr/lib64/libnss_wrapper.so\nexport NSS_WRAPPER_PASSWD=\"${NSS_WRAPPER_PASSWD}\"\nexport + NSS_WRAPPER_GROUP=\"${NSS_WRAPPER_GROUP}\"\n\necho \"nss_wrapper: environment + configured\"\n" + image: perconalab/percona-postgresql-operator:main-ppg17-postgres + imagePullPolicy: Always + name: nss-wrapper-init + resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + privileged: false + readOnlyRootFilesystem: true + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /tmp + name: tmp + restartPolicy: Always + schedulerName: default-scheduler + securityContext: + fsGroup: 26 + fsGroupChangePolicy: OnRootMismatch + serviceAccount: custom-extensions-instance + serviceAccountName: custom-extensions-instance + shareProcessNamespace: true + terminationGracePeriodSeconds: 30 + topologySpreadConstraints: '[... elided field over 10 lines long ...]' + volumes: '[... elided field over 10 lines long ...]' + updateStrategy: + type: OnDelete status: - availableReplicas: 1 + availableReplicas: 0 + collisionCount: 0 + currentRevision: custom-extensions-instance1-f95b-7f94567c8c observedGeneration: 2 - readyReplicas: 1 replicas: 1 + updateRevision: custom-extensions-instance1-f95b-5bcc6f86fc updatedReplicas: 1 case.go:398: resource StatefulSet:kuttl-test-picked-cow/: .spec.template.spec.initContainers.env: slice length mismatch: 9 != 7 case.go:398: --- PerconaPGCluster:kuttl-test-picked-cow/custom-extensions +++ PerconaPGCluster:kuttl-test-picked-cow/custom-extensions @@ -1,18 +1,40 @@ apiVersion: pgv2.percona.com/v2 kind: PerconaPGCluster metadata: + annotations: + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"pgv2.percona.com/v2","kind":"PerconaPGCluster","metadata":{"annotations":{},"labels":{"e2e":"custom-extensions"},"name":"custom-extensions","namespace":"kuttl-test-picked-cow"},"spec":{"backups":{"pgbackrest":{"image":"perconalab/percona-postgresql-operator:main-pgbackrest17","manual":{"options":["--type=full"],"repoName":"repo1"},"repoHost":{"affinity":{"podAntiAffinity":{"preferredDuringSchedulingIgnoredDuringExecution":[{"podAffinityTerm":{"labelSelector":{"matchLabels":{"postgres-operator.crunchydata.com/data":"pgbackrest"}},"topologyKey":"kubernetes.io/hostname"},"weight":1}]}}},"repos":[{"name":"repo1","schedules":{"full":"0 0 * * 6"},"volume":{"volumeClaimSpec":{"accessModes":["ReadWriteOnce"],"resources":{"requests":{"storage":"1Gi"}}}}}]}},"crVersion":"2.7.0","extensions":{"image":"perconalab/percona-postgresql-operator:PR-1214-6ef3702c0","imagePullPolicy":"Always","storage":{"bucket":"operator-testing","disableSSL":"true","endpoint":"http://minio-service:9000","forcePathStyle":"true","region":"us-east-1","secret":{"name":"minio-secret"},"type":"s3"}},"image":"perconalab/percona-postgresql-operator:main-ppg17-postgres","imagePullPolicy":"Always","initContainer":{"image":"perconalab/percona-postgresql-operator:PR-1214-6ef3702c0"},"instances":[{"affinity":{"podAntiAffinity":{"preferredDuringSchedulingIgnoredDuringExecution":[{"podAffinityTerm":{"labelSelector":{"matchLabels":{"postgres-operator.crunchydata.com/data":"postgres"}},"topologyKey":"kubernetes.io/hostname"},"weight":1}]}},"dataVolumeClaimSpec":{"accessModes":["ReadWriteOnce"],"resources":{"requests":{"storage":"1Gi"}}},"name":"instance1","replicas":3}],"pmm":{"customClusterName":"custom-extensions-pmm-custom-name","enabled":false,"image":"perconalab/pmm-client:dev-latest","postgresParams":"--environment=dev-postgres","secret":"custom-extensions-pmm-secret","serverHost":"monitoring-service"},"postgresVersion":17,"proxy":{"pgBouncer":{"affinity":{"podAntiAffinity":{"preferredDuringSchedulingIgnoredDuringExecution":[{"podAffinityTerm":{"labelSelector":{"matchLabels":{"postgres-operator.crunchydata.com/role":"pgbouncer"}},"topologyKey":"kubernetes.io/hostname"},"weight":1}]}},"image":"perconalab/percona-postgresql-operator:main-pgbouncer17","replicas":3}},"users":[{"name":"postgres","password":{"type":"AlphaNumeric"}},{"name":"custom-extensions","password":{"type":"AlphaNumeric"}}]}} + finalizers: + - internal.percona.com/stop-watchers + labels: + e2e: custom-extensions + managedFields: '[... elided field over 10 lines long ...]' name: custom-extensions namespace: kuttl-test-picked-cow +spec: '[... elided field over 10 lines long ...]' status: + conditions: + - lastTransitionTime: "2025-07-17T16:52:44Z" + message: "" + reason: AllConditionsAreTrue + status: "True" + type: ReadyForBackup + host: custom-extensions-pgbouncer.kuttl-test-picked-cow.svc + installedCustomExtensions: + - pg_cron + observedGeneration: 2 + patroniVersion: 4.0.5 pgbouncer: ready: 3 size: 3 postgres: + imageID: docker.io/perconalab/percona-postgresql-operator@sha256:9bc95df4db2d5c130be5e0a14065f034ba56efbb6d2650f38a63a57a0d9738f1 instances: - name: instance1 - ready: 3 + ready: 0 size: 3 - ready: 3 + ready: 0 size: 3 - state: ready + version: 17 + state: initializing case.go:398: resource PerconaPGCluster:kuttl-test-picked-cow/custom-extensions: .status.postgres.ready: value mismatch, expected: 3 != actual: 0 logger.go:42: 16:54:46 | custom-extensions | custom-extensions events from ns kuttl-test-picked-cow: logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:50:36 +0000 UTC Normal Pod pg-client-84d6c45668-dsqf4 Binding Scheduled Successfully assigned kuttl-test-picked-cow/pg-client-84d6c45668-dsqf4 to gke-jen-pg-1214-6ef3702c-default-pool-1d807c88-3ll9 default-scheduler logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:50:36 +0000 UTC Normal Pod pg-client-84d6c45668-dsqf4.spec.containers{pg-client} Pulling Pulling image "perconalab/percona-distribution-postgresql:16" kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:50:36 +0000 UTC Normal ReplicaSet.apps pg-client-84d6c45668 SuccessfulCreate Created pod: pg-client-84d6c45668-dsqf4 replicaset-controller logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:50:36 +0000 UTC Normal Deployment.apps pg-client ScalingReplicaSet Scaled up replica set pg-client-84d6c45668 to 1 deployment-controller logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:50:43 +0000 UTC Normal ReplicaSet.apps minio-service-86dfccd949 SuccessfulCreate Created pod: minio-service-86dfccd949-qfp7b replicaset-controller logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:50:43 +0000 UTC Normal PersistentVolumeClaim minio-service WaitForFirstConsumer waiting for first consumer to be created before binding persistentvolume-controller logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:50:43 +0000 UTC Normal Deployment.apps minio-service ScalingReplicaSet Scaled up replica set minio-service-86dfccd949 to 1 deployment-controller logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:50:43 +0000 UTC Normal PersistentVolumeClaim minio-service ExternalProvisioning Waiting for a volume to be created either by the external provisioner 'pd.csi.storage.gke.io' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. persistentvolume-controller logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:50:43 +0000 UTC Normal PersistentVolumeClaim minio-service Provisioning External provisioner is provisioning volume for claim "kuttl-test-picked-cow/minio-service" pd.csi.storage.gke.io_gke-99b7d2d58ef44b779c18-14b9-df8c-vm_147ee2fa-5d1c-4851-becc-83c77599508a logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:50:44 +0000 UTC Normal Pod minio-service-post-job-ft5qn Binding Scheduled Successfully assigned kuttl-test-picked-cow/minio-service-post-job-ft5qn to gke-jen-pg-1214-6ef3702c-default-pool-1d807c88-dk51 default-scheduler logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:50:44 +0000 UTC Normal Job.batch minio-service-post-job SuccessfulCreate Created pod: minio-service-post-job-ft5qn job-controller logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:50:45 +0000 UTC Normal Pod minio-service-post-job-ft5qn.spec.containers{minio-make-user} Pulling Pulling image "quay.io/minio/mc:RELEASE.2024-11-21T17-21-54Z" kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:50:47 +0000 UTC Normal Pod minio-service-86dfccd949-qfp7b Binding Scheduled Successfully assigned kuttl-test-picked-cow/minio-service-86dfccd949-qfp7b to gke-jen-pg-1214-6ef3702c-default-pool-1d807c88-3ll9 default-scheduler logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:50:47 +0000 UTC Normal Pod minio-service-post-job-ft5qn.spec.containers{minio-make-user} Pulled Successfully pulled image "quay.io/minio/mc:RELEASE.2024-11-21T17-21-54Z" in 2.756s (2.756s including waiting). Image size: 28122288 bytes. kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:50:47 +0000 UTC Normal Pod minio-service-post-job-ft5qn.spec.containers{minio-make-user} Created Created container: minio-make-user kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:50:47 +0000 UTC Normal Pod minio-service-post-job-ft5qn.spec.containers{minio-make-user} Started Started container minio-make-user kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:50:47 +0000 UTC Normal PersistentVolumeClaim minio-service ProvisioningSucceeded Successfully provisioned volume pvc-c113afff-0db9-4030-a0a9-7bef4b2262d2 pd.csi.storage.gke.io_gke-99b7d2d58ef44b779c18-14b9-df8c-vm_147ee2fa-5d1c-4851-becc-83c77599508a logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:50:55 +0000 UTC Normal Pod minio-service-86dfccd949-qfp7b SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-c113afff-0db9-4030-a0a9-7bef4b2262d2" attachdetach-controller logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:50:57 +0000 UTC Normal Pod minio-service-86dfccd949-qfp7b.spec.containers{minio} Pulled Container image "quay.io/minio/minio:RELEASE.2024-12-18T13-15-44Z" already present on machine kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:50:57 +0000 UTC Normal Pod minio-service-86dfccd949-qfp7b.spec.containers{minio} Created Created container: minio kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:50:57 +0000 UTC Normal Pod minio-service-86dfccd949-qfp7b.spec.containers{minio} Started Started container minio kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:50:58 +0000 UTC Normal Pod pg-client-84d6c45668-dsqf4.spec.containers{pg-client} Pulled Successfully pulled image "perconalab/percona-distribution-postgresql:16" in 21.88s (21.88s including waiting). Image size: 324679112 bytes. kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:50:58 +0000 UTC Normal Pod pg-client-84d6c45668-dsqf4.spec.containers{pg-client} Created Created container: pg-client kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:50:58 +0000 UTC Normal Pod pg-client-84d6c45668-dsqf4.spec.containers{pg-client} Started Started container pg-client kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:08 +0000 UTC Normal Job.batch minio-service-post-job Completed Job completed job-controller logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:10 +0000 UTC Normal Pod aws-cli Binding Scheduled Successfully assigned kuttl-test-picked-cow/aws-cli to gke-jen-pg-1214-6ef3702c-default-pool-1d807c88-3ll9 default-scheduler logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:11 +0000 UTC Normal Pod aws-cli.spec.containers{aws-cli} Pulling Pulling image "perconalab/awscli" kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:12 +0000 UTC Normal Pod aws-cli.spec.containers{aws-cli} Pulled Successfully pulled image "perconalab/awscli" in 1.24s (1.241s including waiting). Image size: 30314917 bytes. kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:12 +0000 UTC Normal Pod aws-cli.spec.containers{aws-cli} Created Created container: aws-cli kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:12 +0000 UTC Normal Pod aws-cli.spec.containers{aws-cli} Started Started container aws-cli kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:22 +0000 UTC Normal Pod aws-cli Binding Scheduled Successfully assigned kuttl-test-picked-cow/aws-cli to gke-jen-pg-1214-6ef3702c-default-pool-1d807c88-3ll9 default-scheduler logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:23 +0000 UTC Normal Pod aws-cli.spec.containers{aws-cli} Pulling Pulling image "perconalab/awscli" kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:24 +0000 UTC Normal Pod aws-cli.spec.containers{aws-cli} Pulled Successfully pulled image "perconalab/awscli" in 1.239s (1.239s including waiting). Image size: 30314917 bytes. kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:24 +0000 UTC Normal Pod aws-cli.spec.containers{aws-cli} Created Created container: aws-cli kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:24 +0000 UTC Normal Pod aws-cli.spec.containers{aws-cli} Started Started container aws-cli kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:33 +0000 UTC Normal Pod custom-extensions-patroni-version-check Binding Scheduled Successfully assigned kuttl-test-picked-cow/custom-extensions-patroni-version-check to gke-jen-pg-1214-6ef3702c-default-pool-1d807c88-3ll9 default-scheduler logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:33 +0000 UTC Normal Pod custom-extensions-patroni-version-check.spec.containers{patroni-version-check} Pulled Container image "perconalab/percona-postgresql-operator:main-ppg17-postgres" already present on machine kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:33 +0000 UTC Normal Pod custom-extensions-patroni-version-check.spec.containers{patroni-version-check} Created Created container: patroni-version-check kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:33 +0000 UTC Normal Pod custom-extensions-patroni-version-check.spec.containers{patroni-version-check} Started Started container patroni-version-check kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:38 +0000 UTC Normal Pod custom-extensions-patroni-version-check.spec.containers{patroni-version-check} Killing Stopping container patroni-version-check kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:39 +0000 UTC Normal PersistentVolumeClaim custom-extensions-instance1-26c9-pgdata WaitForFirstConsumer waiting for first consumer to be created before binding persistentvolume-controller logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:40 +0000 UTC Normal PersistentVolumeClaim custom-extensions-instance1-26c9-pgdata ExternalProvisioning Waiting for a volume to be created either by the external provisioner 'pd.csi.storage.gke.io' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. persistentvolume-controller logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:40 +0000 UTC Normal PersistentVolumeClaim custom-extensions-instance1-26c9-pgdata Provisioning External provisioner is provisioning volume for claim "kuttl-test-picked-cow/custom-extensions-instance1-26c9-pgdata" pd.csi.storage.gke.io_gke-99b7d2d58ef44b779c18-14b9-df8c-vm_147ee2fa-5d1c-4851-becc-83c77599508a logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:40 +0000 UTC Normal StatefulSet.apps custom-extensions-instance1-26c9 SuccessfulCreate create Pod custom-extensions-instance1-26c9-0 in StatefulSet custom-extensions-instance1-26c9 successful statefulset-controller logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:40 +0000 UTC Normal PersistentVolumeClaim custom-extensions-instance1-f95b-pgdata WaitForFirstConsumer waiting for first consumer to be created before binding persistentvolume-controller logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:41 +0000 UTC Normal PersistentVolumeClaim custom-extensions-instance1-46tm-pgdata WaitForFirstConsumer waiting for first consumer to be created before binding persistentvolume-controller logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:41 +0000 UTC Normal StatefulSet.apps custom-extensions-instance1-46tm SuccessfulCreate create Pod custom-extensions-instance1-46tm-0 in StatefulSet custom-extensions-instance1-46tm successful statefulset-controller logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:41 +0000 UTC Normal PersistentVolumeClaim custom-extensions-instance1-f95b-pgdata ExternalProvisioning Waiting for a volume to be created either by the external provisioner 'pd.csi.storage.gke.io' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. persistentvolume-controller logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:41 +0000 UTC Normal PersistentVolumeClaim custom-extensions-instance1-f95b-pgdata Provisioning External provisioner is provisioning volume for claim "kuttl-test-picked-cow/custom-extensions-instance1-f95b-pgdata" pd.csi.storage.gke.io_gke-99b7d2d58ef44b779c18-14b9-df8c-vm_147ee2fa-5d1c-4851-becc-83c77599508a logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:41 +0000 UTC Normal StatefulSet.apps custom-extensions-instance1-f95b SuccessfulCreate create Pod custom-extensions-instance1-f95b-0 in StatefulSet custom-extensions-instance1-f95b successful statefulset-controller logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:42 +0000 UTC Normal PersistentVolumeClaim custom-extensions-instance1-46tm-pgdata Provisioning External provisioner is provisioning volume for claim "kuttl-test-picked-cow/custom-extensions-instance1-46tm-pgdata" pd.csi.storage.gke.io_gke-99b7d2d58ef44b779c18-14b9-df8c-vm_147ee2fa-5d1c-4851-becc-83c77599508a logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:42 +0000 UTC Normal PersistentVolumeClaim custom-extensions-instance1-46tm-pgdata ExternalProvisioning Waiting for a volume to be created either by the external provisioner 'pd.csi.storage.gke.io' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. persistentvolume-controller logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:42 +0000 UTC Normal StatefulSet.apps custom-extensions-repo-host SuccessfulCreate create Pod custom-extensions-repo-host-0 in StatefulSet custom-extensions-repo-host successful statefulset-controller logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:42 +0000 UTC Normal PersistentVolumeClaim custom-extensions-repo1 WaitForFirstConsumer waiting for first consumer to be created before binding persistentvolume-controller logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:42 +0000 UTC Normal PersistentVolumeClaim custom-extensions-repo1 ExternalProvisioning Waiting for a volume to be created either by the external provisioner 'pd.csi.storage.gke.io' or manually by the system administrator. If volume creation is delayed, please verify that the provisioner is running and correctly registered. persistentvolume-controller logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:42 +0000 UTC Normal PersistentVolumeClaim custom-extensions-repo1 Provisioning External provisioner is provisioning volume for claim "kuttl-test-picked-cow/custom-extensions-repo1" pd.csi.storage.gke.io_gke-99b7d2d58ef44b779c18-14b9-df8c-vm_147ee2fa-5d1c-4851-becc-83c77599508a logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:42 +0000 UTC Normal PostgresCluster.postgres-operator.crunchydata.com custom-extensions RepoHostCreated created pgBackRest repository host StatefulSet/custom-extensions-repo-host postgrescluster-controller logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:43 +0000 UTC Normal Pod custom-extensions-pgbouncer-7dff54776b-lw9b2 Binding Scheduled Successfully assigned kuttl-test-picked-cow/custom-extensions-pgbouncer-7dff54776b-lw9b2 to gke-jen-pg-1214-6ef3702c-default-pool-1d807c88-3ll9 default-scheduler logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:43 +0000 UTC Normal Pod custom-extensions-pgbouncer-7dff54776b-lw9b2.spec.containers{pgbouncer} Pulling Pulling image "perconalab/percona-postgresql-operator:main-pgbouncer17" kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:43 +0000 UTC Normal Pod custom-extensions-pgbouncer-7dff54776b-rlfl8 Binding Scheduled Successfully assigned kuttl-test-picked-cow/custom-extensions-pgbouncer-7dff54776b-rlfl8 to gke-jen-pg-1214-6ef3702c-default-pool-1d807c88-s568 default-scheduler logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:43 +0000 UTC Normal Pod custom-extensions-pgbouncer-7dff54776b-rlfl8.spec.containers{pgbouncer} Pulling Pulling image "perconalab/percona-postgresql-operator:main-pgbouncer17" kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:43 +0000 UTC Normal Pod custom-extensions-pgbouncer-7dff54776b-sd8lw Binding Scheduled Successfully assigned kuttl-test-picked-cow/custom-extensions-pgbouncer-7dff54776b-sd8lw to gke-jen-pg-1214-6ef3702c-default-pool-1d807c88-dk51 default-scheduler logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:43 +0000 UTC Normal Pod custom-extensions-pgbouncer-7dff54776b-sd8lw.spec.containers{pgbouncer} Pulling Pulling image "perconalab/percona-postgresql-operator:main-pgbouncer17" kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:43 +0000 UTC Normal ReplicaSet.apps custom-extensions-pgbouncer-7dff54776b SuccessfulCreate Created pod: custom-extensions-pgbouncer-7dff54776b-lw9b2 replicaset-controller logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:43 +0000 UTC Normal ReplicaSet.apps custom-extensions-pgbouncer-7dff54776b SuccessfulCreate Created pod: custom-extensions-pgbouncer-7dff54776b-rlfl8 replicaset-controller logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:43 +0000 UTC Normal ReplicaSet.apps custom-extensions-pgbouncer-7dff54776b SuccessfulCreate Created pod: custom-extensions-pgbouncer-7dff54776b-sd8lw replicaset-controller logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:43 +0000 UTC Normal Deployment.apps custom-extensions-pgbouncer ScalingReplicaSet Scaled up replica set custom-extensions-pgbouncer-7dff54776b to 3 deployment-controller logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:43 +0000 UTC Normal PodDisruptionBudget.policy custom-extensions-pgbouncer NoPods No matching pods found controllermanager logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:44 +0000 UTC Normal PersistentVolumeClaim custom-extensions-instance1-26c9-pgdata ProvisioningSucceeded Successfully provisioned volume pvc-1200ce91-5eb8-4094-8b06-b27c9c205443 pd.csi.storage.gke.io_gke-99b7d2d58ef44b779c18-14b9-df8c-vm_147ee2fa-5d1c-4851-becc-83c77599508a logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:44 +0000 UTC Normal Pod custom-extensions-pgbouncer-7dff54776b-lw9b2.spec.containers{pgbouncer} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:main-pgbouncer17" in 1.204s (1.204s including waiting). Image size: 82623388 bytes. kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:44 +0000 UTC Normal Pod custom-extensions-pgbouncer-7dff54776b-lw9b2.spec.containers{pgbouncer} Created Created container: pgbouncer kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:44 +0000 UTC Normal Pod custom-extensions-pgbouncer-7dff54776b-lw9b2.spec.containers{pgbouncer} Started Started container pgbouncer kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:44 +0000 UTC Normal Pod custom-extensions-pgbouncer-7dff54776b-lw9b2.spec.containers{pgbouncer-config} Pulling Pulling image "perconalab/percona-postgresql-operator:main-pgbouncer17" kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:44 +0000 UTC Normal Pod custom-extensions-pgbouncer-7dff54776b-sd8lw.spec.containers{pgbouncer} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:main-pgbouncer17" in 1.214s (1.214s including waiting). Image size: 82623388 bytes. kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:45 +0000 UTC Normal Pod custom-extensions-instance1-26c9-0 Binding Scheduled Successfully assigned kuttl-test-picked-cow/custom-extensions-instance1-26c9-0 to gke-jen-pg-1214-6ef3702c-default-pool-1d807c88-3ll9 default-scheduler logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:45 +0000 UTC Normal PersistentVolumeClaim custom-extensions-instance1-46tm-pgdata ProvisioningSucceeded Successfully provisioned volume pvc-15568f87-981d-4995-aa3a-9802eb771934 pd.csi.storage.gke.io_gke-99b7d2d58ef44b779c18-14b9-df8c-vm_147ee2fa-5d1c-4851-becc-83c77599508a logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:45 +0000 UTC Normal PersistentVolumeClaim custom-extensions-instance1-f95b-pgdata ProvisioningSucceeded Successfully provisioned volume pvc-f056e290-758d-4c88-b964-a634062adea8 pd.csi.storage.gke.io_gke-99b7d2d58ef44b779c18-14b9-df8c-vm_147ee2fa-5d1c-4851-becc-83c77599508a logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:45 +0000 UTC Normal Pod custom-extensions-pgbouncer-7dff54776b-rlfl8.spec.containers{pgbouncer} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:main-pgbouncer17" in 1.217s (1.217s including waiting). Image size: 82623388 bytes. kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:45 +0000 UTC Normal Pod custom-extensions-pgbouncer-7dff54776b-rlfl8.spec.containers{pgbouncer} Created Created container: pgbouncer kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:45 +0000 UTC Normal Pod custom-extensions-pgbouncer-7dff54776b-rlfl8.spec.containers{pgbouncer} Started Started container pgbouncer kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:45 +0000 UTC Normal Pod custom-extensions-pgbouncer-7dff54776b-rlfl8.spec.containers{pgbouncer-config} Pulling Pulling image "perconalab/percona-postgresql-operator:main-pgbouncer17" kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:45 +0000 UTC Normal Pod custom-extensions-pgbouncer-7dff54776b-sd8lw.spec.containers{pgbouncer} Created Created container: pgbouncer kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:45 +0000 UTC Normal Pod custom-extensions-pgbouncer-7dff54776b-sd8lw.spec.containers{pgbouncer} Started Started container pgbouncer kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:45 +0000 UTC Normal Pod custom-extensions-pgbouncer-7dff54776b-sd8lw.spec.containers{pgbouncer-config} Pulling Pulling image "perconalab/percona-postgresql-operator:main-pgbouncer17" kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:46 +0000 UTC Normal Pod custom-extensions-instance1-46tm-0 Binding Scheduled Successfully assigned kuttl-test-picked-cow/custom-extensions-instance1-46tm-0 to gke-jen-pg-1214-6ef3702c-default-pool-1d807c88-s568 default-scheduler logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:46 +0000 UTC Normal Pod custom-extensions-instance1-f95b-0 Binding Scheduled Successfully assigned kuttl-test-picked-cow/custom-extensions-instance1-f95b-0 to gke-jen-pg-1214-6ef3702c-default-pool-1d807c88-dk51 default-scheduler logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:46 +0000 UTC Normal Pod custom-extensions-pgbouncer-7dff54776b-lw9b2.spec.containers{pgbouncer-config} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:main-pgbouncer17" in 1.182s (1.182s including waiting). Image size: 82623388 bytes. kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:46 +0000 UTC Normal Pod custom-extensions-pgbouncer-7dff54776b-lw9b2.spec.containers{pgbouncer-config} Created Created container: pgbouncer-config kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:46 +0000 UTC Normal Pod custom-extensions-pgbouncer-7dff54776b-lw9b2.spec.containers{pgbouncer-config} Started Started container pgbouncer-config kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:46 +0000 UTC Normal Pod custom-extensions-pgbouncer-7dff54776b-rlfl8.spec.containers{pgbouncer-config} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:main-pgbouncer17" in 1.21s (1.21s including waiting). Image size: 82623388 bytes. kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:46 +0000 UTC Normal Pod custom-extensions-pgbouncer-7dff54776b-rlfl8.spec.containers{pgbouncer-config} Created Created container: pgbouncer-config kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:46 +0000 UTC Normal Pod custom-extensions-pgbouncer-7dff54776b-rlfl8.spec.containers{pgbouncer-config} Started Started container pgbouncer-config kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:46 +0000 UTC Normal Pod custom-extensions-pgbouncer-7dff54776b-sd8lw.spec.containers{pgbouncer-config} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:main-pgbouncer17" in 1.188s (1.188s including waiting). Image size: 82623388 bytes. kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:46 +0000 UTC Normal Pod custom-extensions-pgbouncer-7dff54776b-sd8lw.spec.containers{pgbouncer-config} Created Created container: pgbouncer-config kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:46 +0000 UTC Normal Pod custom-extensions-pgbouncer-7dff54776b-sd8lw.spec.containers{pgbouncer-config} Started Started container pgbouncer-config kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:46 +0000 UTC Normal Pod custom-extensions-repo-host-0 Binding Scheduled Successfully assigned kuttl-test-picked-cow/custom-extensions-repo-host-0 to gke-jen-pg-1214-6ef3702c-default-pool-1d807c88-dk51 default-scheduler logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:46 +0000 UTC Normal PersistentVolumeClaim custom-extensions-repo1 ProvisioningSucceeded Successfully provisioned volume pvc-dccf6b9e-5775-4917-ad0e-d3dff04f909c pd.csi.storage.gke.io_gke-99b7d2d58ef44b779c18-14b9-df8c-vm_147ee2fa-5d1c-4851-becc-83c77599508a logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:53 +0000 UTC Normal Pod custom-extensions-instance1-26c9-0 SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-1200ce91-5eb8-4094-8b06-b27c9c205443" attachdetach-controller logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:53 +0000 UTC Normal Pod custom-extensions-instance1-46tm-0 SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-15568f87-981d-4995-aa3a-9802eb771934" attachdetach-controller logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:54 +0000 UTC Normal Pod custom-extensions-instance1-26c9-0.spec.initContainers{postgres-startup} Pulling Pulling image "perconalab/percona-postgresql-operator:main-ppg17-postgres" kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:54 +0000 UTC Normal Pod custom-extensions-instance1-46tm-0.spec.initContainers{postgres-startup} Pulling Pulling image "perconalab/percona-postgresql-operator:main-ppg17-postgres" kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:54 +0000 UTC Normal Pod custom-extensions-instance1-f95b-0 SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-f056e290-758d-4c88-b964-a634062adea8" attachdetach-controller logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:55 +0000 UTC Normal Pod custom-extensions-instance1-26c9-0.spec.initContainers{postgres-startup} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:main-ppg17-postgres" in 1.253s (1.253s including waiting). Image size: 529428770 bytes. kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:55 +0000 UTC Normal Pod custom-extensions-instance1-26c9-0.spec.initContainers{postgres-startup} Created Created container: postgres-startup kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:55 +0000 UTC Normal Pod custom-extensions-instance1-26c9-0.spec.initContainers{postgres-startup} Started Started container postgres-startup kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:55 +0000 UTC Normal Pod custom-extensions-instance1-46tm-0.spec.initContainers{postgres-startup} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:main-ppg17-postgres" in 1.258s (1.258s including waiting). Image size: 529428770 bytes. kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:55 +0000 UTC Normal Pod custom-extensions-instance1-46tm-0.spec.initContainers{postgres-startup} Created Created container: postgres-startup kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:56 +0000 UTC Normal Pod custom-extensions-instance1-26c9-0.spec.initContainers{extension-relocator-17} Pulling Pulling image "perconalab/percona-postgresql-operator:main-ppg17-postgres" kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:56 +0000 UTC Normal Pod custom-extensions-instance1-46tm-0.spec.initContainers{postgres-startup} Started Started container postgres-startup kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:56 +0000 UTC Normal Pod custom-extensions-instance1-46tm-0.spec.initContainers{extension-relocator-17} Pulling Pulling image "perconalab/percona-postgresql-operator:main-ppg17-postgres" kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:56 +0000 UTC Normal Pod custom-extensions-instance1-f95b-0.spec.initContainers{postgres-startup} Pulling Pulling image "perconalab/percona-postgresql-operator:main-ppg17-postgres" kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:57 +0000 UTC Normal Pod custom-extensions-instance1-26c9-0.spec.initContainers{extension-relocator-17} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:main-ppg17-postgres" in 1.248s (1.248s including waiting). Image size: 529428770 bytes. kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:57 +0000 UTC Normal Pod custom-extensions-instance1-26c9-0.spec.initContainers{extension-relocator-17} Created Created container: extension-relocator-17 kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:57 +0000 UTC Normal Pod custom-extensions-instance1-26c9-0.spec.initContainers{extension-relocator-17} Started Started container extension-relocator-17 kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:57 +0000 UTC Normal Pod custom-extensions-instance1-f95b-0.spec.initContainers{postgres-startup} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:main-ppg17-postgres" in 1.23s (1.23s including waiting). Image size: 529428770 bytes. kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:57 +0000 UTC Normal Pod custom-extensions-instance1-f95b-0.spec.initContainers{postgres-startup} Created Created container: postgres-startup kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:57 +0000 UTC Normal Pod custom-extensions-instance1-f95b-0.spec.initContainers{postgres-startup} Started Started container postgres-startup kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:57 +0000 UTC Normal Pod custom-extensions-repo-host-0 SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-dccf6b9e-5775-4917-ad0e-d3dff04f909c" attachdetach-controller logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:58 +0000 UTC Normal Pod custom-extensions-instance1-26c9-0.spec.initContainers{extension-installer-17} Pulling Pulling image "perconalab/percona-postgresql-operator:PR-1214-6ef3702c0" kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:58 +0000 UTC Normal Pod custom-extensions-instance1-46tm-0.spec.initContainers{extension-relocator-17} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:main-ppg17-postgres" in 1.223s (1.223s including waiting). Image size: 529428770 bytes. kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:58 +0000 UTC Normal Pod custom-extensions-instance1-46tm-0.spec.initContainers{extension-relocator-17} Created Created container: extension-relocator-17 kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:58 +0000 UTC Normal Pod custom-extensions-instance1-46tm-0.spec.initContainers{extension-relocator-17} Started Started container extension-relocator-17 kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:58 +0000 UTC Normal Pod custom-extensions-instance1-46tm-0.spec.initContainers{extension-installer-17} Pulling Pulling image "perconalab/percona-postgresql-operator:PR-1214-6ef3702c0" kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:58 +0000 UTC Normal Pod custom-extensions-instance1-f95b-0.spec.initContainers{extension-relocator-17} Pulling Pulling image "perconalab/percona-postgresql-operator:main-ppg17-postgres" kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:58 +0000 UTC Normal Pod custom-extensions-repo-host-0.spec.initContainers{pgbackrest-log-dir} Pulling Pulling image "perconalab/percona-postgresql-operator:main-pgbackrest17" kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:59 +0000 UTC Normal Pod custom-extensions-instance1-26c9-0.spec.initContainers{extension-installer-17} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:PR-1214-6ef3702c0" in 1.276s (1.276s including waiting). Image size: 95771923 bytes. kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:59 +0000 UTC Normal Pod custom-extensions-instance1-26c9-0.spec.initContainers{extension-installer-17} Created Created container: extension-installer-17 kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:59 +0000 UTC Normal Pod custom-extensions-instance1-26c9-0.spec.initContainers{extension-installer-17} Started Started container extension-installer-17 kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:59 +0000 UTC Normal Pod custom-extensions-instance1-f95b-0.spec.initContainers{extension-relocator-17} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:main-ppg17-postgres" in 1.32s (1.32s including waiting). Image size: 529428770 bytes. kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:59 +0000 UTC Normal Pod custom-extensions-instance1-f95b-0.spec.initContainers{extension-relocator-17} Created Created container: extension-relocator-17 kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:59 +0000 UTC Normal Pod custom-extensions-instance1-f95b-0.spec.initContainers{extension-relocator-17} Started Started container extension-relocator-17 kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:59 +0000 UTC Normal Pod custom-extensions-repo-host-0.spec.initContainers{pgbackrest-log-dir} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:main-pgbackrest17" in 1.277s (1.277s including waiting). Image size: 153232642 bytes. kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:51:59 +0000 UTC Normal Pod custom-extensions-repo-host-0.spec.initContainers{pgbackrest-log-dir} Created Created container: pgbackrest-log-dir kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:52:00 +0000 UTC Normal Pod custom-extensions-instance1-26c9-0.spec.initContainers{database-init} Pulling Pulling image "perconalab/percona-postgresql-operator:PR-1214-6ef3702c0" kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:52:00 +0000 UTC Normal Pod custom-extensions-instance1-46tm-0.spec.initContainers{extension-installer-17} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:PR-1214-6ef3702c0" in 1.15s (1.15s including waiting). Image size: 95771923 bytes. kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:52:00 +0000 UTC Normal Pod custom-extensions-instance1-46tm-0.spec.initContainers{extension-installer-17} Created Created container: extension-installer-17 kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:52:00 +0000 UTC Normal Pod custom-extensions-instance1-46tm-0.spec.initContainers{extension-installer-17} Started Started container extension-installer-17 kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:52:00 +0000 UTC Normal Pod custom-extensions-instance1-46tm-0.spec.initContainers{database-init} Pulling Pulling image "perconalab/percona-postgresql-operator:PR-1214-6ef3702c0" kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:52:00 +0000 UTC Normal Pod custom-extensions-instance1-f95b-0.spec.initContainers{extension-installer-17} Pulling Pulling image "perconalab/percona-postgresql-operator:PR-1214-6ef3702c0" kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:52:00 +0000 UTC Normal Pod custom-extensions-repo-host-0.spec.initContainers{pgbackrest-log-dir} Started Started container pgbackrest-log-dir kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:52:00 +0000 UTC Normal Pod custom-extensions-repo-host-0.spec.initContainers{nss-wrapper-init} Pulling Pulling image "perconalab/percona-postgresql-operator:main-pgbackrest17" kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:52:01 +0000 UTC Normal Pod custom-extensions-instance1-26c9-0.spec.initContainers{database-init} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:PR-1214-6ef3702c0" in 1.242s (1.242s including waiting). Image size: 95771923 bytes. kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:52:01 +0000 UTC Normal Pod custom-extensions-instance1-26c9-0.spec.initContainers{database-init} Created Created container: database-init kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:52:01 +0000 UTC Normal Pod custom-extensions-instance1-26c9-0.spec.initContainers{database-init} Started Started container database-init kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:52:01 +0000 UTC Normal Pod custom-extensions-instance1-f95b-0.spec.initContainers{extension-installer-17} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:PR-1214-6ef3702c0" in 1.273s (1.273s including waiting). Image size: 95771923 bytes. kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:52:01 +0000 UTC Normal Pod custom-extensions-instance1-f95b-0.spec.initContainers{extension-installer-17} Created Created container: extension-installer-17 kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:52:01 +0000 UTC Normal Pod custom-extensions-instance1-f95b-0.spec.initContainers{extension-installer-17} Started Started container extension-installer-17 kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:52:01 +0000 UTC Normal Pod custom-extensions-repo-host-0.spec.initContainers{nss-wrapper-init} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:main-pgbackrest17" in 1.19s (1.19s including waiting). Image size: 153232642 bytes. kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:52:01 +0000 UTC Normal Pod custom-extensions-repo-host-0.spec.initContainers{nss-wrapper-init} Created Created container: nss-wrapper-init kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:52:01 +0000 UTC Normal Pod custom-extensions-repo-host-0.spec.initContainers{nss-wrapper-init} Started Started container nss-wrapper-init kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:52:02 +0000 UTC Normal Pod custom-extensions-instance1-26c9-0.spec.initContainers{nss-wrapper-init} Pulling Pulling image "perconalab/percona-postgresql-operator:main-ppg17-postgres" kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:52:02 +0000 UTC Normal Pod custom-extensions-instance1-46tm-0.spec.initContainers{database-init} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:PR-1214-6ef3702c0" in 1.236s (1.236s including waiting). Image size: 95771923 bytes. kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:52:02 +0000 UTC Normal Pod custom-extensions-instance1-46tm-0.spec.initContainers{database-init} Created Created container: database-init kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:52:02 +0000 UTC Normal Pod custom-extensions-instance1-46tm-0.spec.initContainers{database-init} Started Started container database-init kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:52:02 +0000 UTC Normal Pod custom-extensions-instance1-46tm-0.spec.initContainers{nss-wrapper-init} Pulling Pulling image "perconalab/percona-postgresql-operator:main-ppg17-postgres" kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:52:02 +0000 UTC Normal Pod custom-extensions-instance1-f95b-0.spec.initContainers{database-init} Pulling Pulling image "perconalab/percona-postgresql-operator:PR-1214-6ef3702c0" kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:52:02 +0000 UTC Normal Pod custom-extensions-repo-host-0.spec.containers{pgbackrest} Pulling Pulling image "perconalab/percona-postgresql-operator:main-pgbackrest17" kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:52:03 +0000 UTC Normal Pod custom-extensions-instance1-26c9-0.spec.initContainers{nss-wrapper-init} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:main-ppg17-postgres" in 1.266s (1.266s including waiting). Image size: 529428770 bytes. kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:52:03 +0000 UTC Normal Pod custom-extensions-instance1-26c9-0.spec.initContainers{nss-wrapper-init} Created Created container: nss-wrapper-init kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:52:03 +0000 UTC Normal Pod custom-extensions-instance1-f95b-0.spec.initContainers{database-init} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:PR-1214-6ef3702c0" in 1.253s (1.253s including waiting). Image size: 95771923 bytes. kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:52:03 +0000 UTC Normal Pod custom-extensions-instance1-f95b-0.spec.initContainers{database-init} Created Created container: database-init kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:52:03 +0000 UTC Normal Pod custom-extensions-instance1-f95b-0.spec.initContainers{database-init} Started Started container database-init kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:52:03 +0000 UTC Normal Pod custom-extensions-repo-host-0.spec.containers{pgbackrest} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:main-pgbackrest17" in 1.224s (1.224s including waiting). Image size: 153232642 bytes. kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:52:03 +0000 UTC Normal Pod custom-extensions-repo-host-0.spec.containers{pgbackrest} Created Created container: pgbackrest kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:52:03 +0000 UTC Normal Pod custom-extensions-repo-host-0.spec.containers{pgbackrest} Started Started container pgbackrest kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:52:03 +0000 UTC Normal Pod custom-extensions-repo-host-0.spec.containers{pgbackrest-config} Pulling Pulling image "perconalab/percona-postgresql-operator:main-pgbackrest17" kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:52:04 +0000 UTC Normal Pod custom-extensions-instance1-26c9-0.spec.initContainers{nss-wrapper-init} Started Started container nss-wrapper-init kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:52:04 +0000 UTC Normal Pod custom-extensions-instance1-26c9-0.spec.containers{database} Pulling Pulling image "perconalab/percona-postgresql-operator:main-ppg17-postgres" kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:52:04 +0000 UTC Normal Pod custom-extensions-instance1-46tm-0.spec.initContainers{nss-wrapper-init} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:main-ppg17-postgres" in 1.233s (1.233s including waiting). Image size: 529428770 bytes. kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:52:04 +0000 UTC Normal Pod custom-extensions-instance1-46tm-0.spec.initContainers{nss-wrapper-init} Created Created container: nss-wrapper-init kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:52:04 +0000 UTC Normal Pod custom-extensions-instance1-46tm-0.spec.initContainers{nss-wrapper-init} Started Started container nss-wrapper-init kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:52:04 +0000 UTC Normal Pod custom-extensions-instance1-46tm-0.spec.containers{database} Pulling Pulling image "perconalab/percona-postgresql-operator:main-ppg17-postgres" kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:52:04 +0000 UTC Normal Pod custom-extensions-instance1-f95b-0.spec.initContainers{nss-wrapper-init} Pulling Pulling image "perconalab/percona-postgresql-operator:main-ppg17-postgres" kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:52:04 +0000 UTC Normal Pod custom-extensions-repo-host-0.spec.containers{pgbackrest-config} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:main-pgbackrest17" in 1.303s (1.303s including waiting). Image size: 153232642 bytes. kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:52:04 +0000 UTC Normal Pod custom-extensions-repo-host-0.spec.containers{pgbackrest-config} Created Created container: pgbackrest-config kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:52:05 +0000 UTC Normal Pod custom-extensions-instance1-26c9-0.spec.containers{database} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:main-ppg17-postgres" in 1.25s (1.25s including waiting). Image size: 529428770 bytes. kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:52:05 +0000 UTC Normal Pod custom-extensions-instance1-26c9-0.spec.containers{database} Created Created container: database kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:52:05 +0000 UTC Normal Pod custom-extensions-instance1-f95b-0.spec.initContainers{nss-wrapper-init} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:main-ppg17-postgres" in 1.268s (1.268s including waiting). Image size: 529428770 bytes. kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:52:05 +0000 UTC Normal Pod custom-extensions-instance1-f95b-0.spec.initContainers{nss-wrapper-init} Created Created container: nss-wrapper-init kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:52:05 +0000 UTC Normal Pod custom-extensions-instance1-f95b-0.spec.initContainers{nss-wrapper-init} Started Started container nss-wrapper-init kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:52:05 +0000 UTC Normal Pod custom-extensions-repo-host-0.spec.containers{pgbackrest-config} Started Started container pgbackrest-config kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:52:06 +0000 UTC Normal Pod custom-extensions-instance1-26c9-0.spec.containers{database} Started Started container database kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:52:06 +0000 UTC Normal Pod custom-extensions-instance1-26c9-0.spec.containers{replication-cert-copy} Pulling Pulling image "perconalab/percona-postgresql-operator:main-ppg17-postgres" kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:52:06 +0000 UTC Normal Pod custom-extensions-instance1-46tm-0.spec.containers{database} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:main-ppg17-postgres" in 1.225s (1.225s including waiting). Image size: 529428770 bytes. kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:52:06 +0000 UTC Normal Pod custom-extensions-instance1-46tm-0.spec.containers{database} Created Created container: database kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:52:06 +0000 UTC Normal Pod custom-extensions-instance1-46tm-0.spec.containers{database} Started Started container database kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:52:06 +0000 UTC Normal Pod custom-extensions-instance1-46tm-0.spec.containers{replication-cert-copy} Pulling Pulling image "perconalab/percona-postgresql-operator:main-ppg17-postgres" kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:52:06 +0000 UTC Normal Pod custom-extensions-instance1-f95b-0.spec.containers{database} Pulling Pulling image "perconalab/percona-postgresql-operator:main-ppg17-postgres" kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:52:07 +0000 UTC Normal Pod custom-extensions-instance1-f95b-0.spec.containers{database} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:main-ppg17-postgres" in 871ms (871ms including waiting). Image size: 529428770 bytes. kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:52:07 +0000 UTC Normal Pod custom-extensions-instance1-f95b-0.spec.containers{database} Created Created container: database kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:52:07 +0000 UTC Normal Pod custom-extensions-instance1-f95b-0.spec.containers{database} Started Started container database kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:52:07 +0000 UTC Normal Pod custom-extensions-instance1-f95b-0.spec.containers{replication-cert-copy} Pulling Pulling image "perconalab/percona-postgresql-operator:main-ppg17-postgres" kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:52:12 +0000 UTC Normal Pod custom-extensions-backup-xh2t-qlpnj Binding Scheduled Successfully assigned kuttl-test-picked-cow/custom-extensions-backup-xh2t-qlpnj to gke-jen-pg-1214-6ef3702c-default-pool-1d807c88-3ll9 default-scheduler logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:52:12 +0000 UTC Normal Job.batch custom-extensions-backup-xh2t SuccessfulCreate Created pod: custom-extensions-backup-xh2t-qlpnj job-controller logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:52:12 +0000 UTC Normal PostgresCluster.postgres-operator.crunchydata.com custom-extensions StanzasCreated pgBackRest stanza creation completed successfully postgrescluster-controller logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:52:13 +0000 UTC Normal Pod custom-extensions-backup-xh2t-qlpnj.spec.initContainers{pgbackrest-init} Pulling Pulling image "perconalab/percona-postgresql-operator:PR-1214-6ef3702c0" kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:52:14 +0000 UTC Normal Pod custom-extensions-backup-xh2t-qlpnj.spec.initContainers{pgbackrest-init} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:PR-1214-6ef3702c0" in 835ms (835ms including waiting). Image size: 95771923 bytes. kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:52:14 +0000 UTC Normal Pod custom-extensions-backup-xh2t-qlpnj.spec.initContainers{pgbackrest-init} Created Created container: pgbackrest-init kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:52:14 +0000 UTC Normal Pod custom-extensions-backup-xh2t-qlpnj.spec.initContainers{pgbackrest-init} Started Started container pgbackrest-init kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:52:14 +0000 UTC Normal Pod custom-extensions-backup-xh2t-qlpnj.spec.containers{pgbackrest} Pulling Pulling image "perconalab/percona-postgresql-operator:main-pgbackrest17" kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:52:15 +0000 UTC Normal Pod custom-extensions-backup-xh2t-qlpnj.spec.containers{pgbackrest} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:main-pgbackrest17" in 1.207s (1.207s including waiting). Image size: 153232642 bytes. kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:52:15 +0000 UTC Normal Pod custom-extensions-backup-xh2t-qlpnj.spec.containers{pgbackrest} Created Created container: pgbackrest kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:52:16 +0000 UTC Normal Pod custom-extensions-backup-xh2t-qlpnj.spec.containers{pgbackrest} Started Started container pgbackrest kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:52:17 +0000 UTC Warning PostgresCluster.postgres-operator.crunchydata.com custom-extensions UnableToCreateStanzas command terminated with exit code 50: repo1-path = /pgbackrest/repo1 2025-07-17 16:52:17.620 P00 ERROR: [050]: unable to acquire lock on file '/tmp/pgbackrest/db-backup-1.lock': Resource temporarily unavailable HINT: is another pgBackRest process running? 2025-07-17 16:52:17.698 P00 ERROR: [050]: unable to acquire lock on file '/tmp/pgbackrest/db-backup-1.lock': Resource temporarily unavailable HINT: is another pgBackRest process running? postgrescluster-controller logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:52:42 +0000 UTC Normal Job.batch custom-extensions-backup-xh2t Completed Job completed job-controller logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:52:51 +0000 UTC Normal Pod custom-extensions-instance1-f95b-0 Binding Scheduled Successfully assigned kuttl-test-picked-cow/custom-extensions-instance1-f95b-0 to gke-jen-pg-1214-6ef3702c-default-pool-1d807c88-dk51 default-scheduler logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:52:52 +0000 UTC Normal Pod custom-extensions-instance1-f95b-0.spec.initContainers{postgres-startup} Pulling Pulling image "perconalab/percona-postgresql-operator:main-ppg17-postgres" kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:52:53 +0000 UTC Normal Pod custom-extensions-instance1-f95b-0.spec.initContainers{postgres-startup} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:main-ppg17-postgres" in 852ms (852ms including waiting). Image size: 529428770 bytes. kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:52:53 +0000 UTC Normal Pod custom-extensions-instance1-f95b-0.spec.initContainers{postgres-startup} Created Created container: postgres-startup kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:52:53 +0000 UTC Normal Pod custom-extensions-instance1-f95b-0.spec.initContainers{postgres-startup} Started Started container postgres-startup kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:52:54 +0000 UTC Normal Pod custom-extensions-instance1-f95b-0.spec.initContainers{extension-relocator-17} Pulling Pulling image "perconalab/percona-postgresql-operator:main-ppg17-postgres" kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:52:55 +0000 UTC Normal Pod custom-extensions-instance1-f95b-0.spec.initContainers{extension-relocator-17} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:main-ppg17-postgres" in 845ms (845ms including waiting). Image size: 529428770 bytes. kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:52:55 +0000 UTC Normal Pod custom-extensions-instance1-f95b-0.spec.initContainers{extension-relocator-17} Created Created container: extension-relocator-17 kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:52:55 +0000 UTC Normal Pod custom-extensions-instance1-f95b-0.spec.initContainers{extension-relocator-17} Started Started container extension-relocator-17 kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:52:56 +0000 UTC Normal Pod custom-extensions-instance1-f95b-0.spec.initContainers{extension-installer-17} Pulling Pulling image "perconalab/percona-postgresql-operator:PR-1214-6ef3702c0" kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:52:57 +0000 UTC Normal Pod custom-extensions-instance1-f95b-0.spec.initContainers{extension-installer-17} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:PR-1214-6ef3702c0" in 850ms (850ms including waiting). Image size: 95771923 bytes. kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:52:57 +0000 UTC Normal Pod custom-extensions-instance1-f95b-0.spec.initContainers{extension-installer-17} Created Created container: extension-installer-17 kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:52:57 +0000 UTC Normal Pod custom-extensions-instance1-f95b-0.spec.initContainers{extension-installer-17} Started Started container extension-installer-17 kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:53:00 +0000 UTC Normal Pod custom-extensions-instance1-f95b-0.spec.initContainers{extension-installer-17} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:PR-1214-6ef3702c0" in 876ms (876ms including waiting). Image size: 95771923 bytes. kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:53:01 +0000 UTC Warning Pod custom-extensions-instance1-f95b-0.spec.initContainers{extension-installer-17} BackOff Back-off restarting failed container extension-installer-17 in pod custom-extensions-instance1-f95b-0_kuttl-test-picked-cow(f2774fb4-f7a0-4837-8620-42d781e79d38) kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:53:15 +0000 UTC Normal Pod custom-extensions-instance1-f95b-0.spec.initContainers{extension-installer-17} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:PR-1214-6ef3702c0" in 820ms (820ms including waiting). Image size: 95771923 bytes. kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:53:17 +0000 UTC Normal Pod custom-extensions-instance1-46tm-0 Binding Scheduled Successfully assigned kuttl-test-picked-cow/custom-extensions-instance1-46tm-0 to gke-jen-pg-1214-6ef3702c-default-pool-1d807c88-s568 default-scheduler logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:53:19 +0000 UTC Warning Pod custom-extensions-instance1-46tm-0 FailedMount MountVolume.SetUp failed for volume "pgbackrest-server" : failed to sync secret cache: timed out waiting for the condition kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:53:19 +0000 UTC Warning Pod custom-extensions-instance1-46tm-0 FailedMount MountVolume.SetUp failed for volume "kube-api-access-446db" : failed to sync configmap cache: timed out waiting for the condition kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:53:19 +0000 UTC Warning Pod custom-extensions-instance1-46tm-0 FailedMount MountVolume.SetUp failed for volume "pgbackrest-config" : failed to sync configmap cache: timed out waiting for the condition kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:53:19 +0000 UTC Warning Pod custom-extensions-instance1-46tm-0 FailedMount MountVolume.SetUp failed for volume "patroni-config" : failed to sync configmap cache: timed out waiting for the condition kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:53:20 +0000 UTC Normal Pod custom-extensions-instance1-46tm-0.spec.initContainers{postgres-startup} Pulling Pulling image "perconalab/percona-postgresql-operator:main-ppg17-postgres" kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:53:21 +0000 UTC Normal Pod custom-extensions-instance1-46tm-0.spec.initContainers{postgres-startup} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:main-ppg17-postgres" in 830ms (830ms including waiting). Image size: 529428770 bytes. kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:53:21 +0000 UTC Normal Pod custom-extensions-instance1-46tm-0.spec.initContainers{postgres-startup} Created Created container: postgres-startup kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:53:21 +0000 UTC Normal Pod custom-extensions-instance1-46tm-0.spec.initContainers{postgres-startup} Started Started container postgres-startup kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:53:22 +0000 UTC Normal Pod custom-extensions-instance1-46tm-0.spec.initContainers{extension-relocator-17} Pulling Pulling image "perconalab/percona-postgresql-operator:main-ppg17-postgres" kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:53:22 +0000 UTC Normal Pod custom-extensions-instance1-46tm-0.spec.initContainers{extension-relocator-17} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:main-ppg17-postgres" in 840ms (840ms including waiting). Image size: 529428770 bytes. kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:53:23 +0000 UTC Normal Pod custom-extensions-instance1-46tm-0.spec.initContainers{extension-relocator-17} Created Created container: extension-relocator-17 kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:53:23 +0000 UTC Normal Pod custom-extensions-instance1-46tm-0.spec.initContainers{extension-relocator-17} Started Started container extension-relocator-17 kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:53:24 +0000 UTC Normal Pod custom-extensions-instance1-46tm-0.spec.initContainers{extension-installer-17} Pulling Pulling image "perconalab/percona-postgresql-operator:PR-1214-6ef3702c0" kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:53:24 +0000 UTC Normal Pod custom-extensions-instance1-46tm-0.spec.initContainers{extension-installer-17} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:PR-1214-6ef3702c0" in 820ms (820ms including waiting). Image size: 95771923 bytes. kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:53:25 +0000 UTC Normal Pod custom-extensions-instance1-46tm-0.spec.initContainers{extension-installer-17} Created Created container: extension-installer-17 kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:53:25 +0000 UTC Normal Pod custom-extensions-instance1-46tm-0.spec.initContainers{extension-installer-17} Started Started container extension-installer-17 kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:53:27 +0000 UTC Normal Pod custom-extensions-instance1-46tm-0.spec.initContainers{extension-installer-17} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:PR-1214-6ef3702c0" in 842ms (842ms including waiting). Image size: 95771923 bytes. kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:53:28 +0000 UTC Normal Pod custom-extensions-instance1-26c9-0 Binding Scheduled Successfully assigned kuttl-test-picked-cow/custom-extensions-instance1-26c9-0 to gke-jen-pg-1214-6ef3702c-default-pool-1d807c88-3ll9 default-scheduler logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:53:29 +0000 UTC Warning Pod custom-extensions-instance1-46tm-0.spec.initContainers{extension-installer-17} BackOff Back-off restarting failed container extension-installer-17 in pod custom-extensions-instance1-46tm-0_kuttl-test-picked-cow(a9088a1a-9032-4efb-bf5c-847a93789c21) kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:53:32 +0000 UTC Normal Pod custom-extensions-instance1-26c9-0.spec.initContainers{postgres-startup} Pulling Pulling image "perconalab/percona-postgresql-operator:main-ppg17-postgres" kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:53:33 +0000 UTC Normal Pod custom-extensions-instance1-26c9-0.spec.initContainers{postgres-startup} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:main-ppg17-postgres" in 832ms (832ms including waiting). Image size: 529428770 bytes. kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:53:33 +0000 UTC Normal Pod custom-extensions-instance1-26c9-0.spec.initContainers{postgres-startup} Created Created container: postgres-startup kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:53:33 +0000 UTC Normal Pod custom-extensions-instance1-26c9-0.spec.initContainers{postgres-startup} Started Started container postgres-startup kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:53:33 +0000 UTC Normal Pod custom-extensions-instance1-26c9-0.spec.initContainers{extension-relocator-17} Pulling Pulling image "perconalab/percona-postgresql-operator:main-ppg17-postgres" kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:53:34 +0000 UTC Normal Pod custom-extensions-instance1-26c9-0.spec.initContainers{extension-relocator-17} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:main-ppg17-postgres" in 853ms (853ms including waiting). Image size: 529428770 bytes. kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:53:34 +0000 UTC Normal Pod custom-extensions-instance1-26c9-0.spec.initContainers{extension-relocator-17} Created Created container: extension-relocator-17 kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:53:34 +0000 UTC Normal Pod custom-extensions-instance1-26c9-0.spec.initContainers{extension-relocator-17} Started Started container extension-relocator-17 kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:53:35 +0000 UTC Normal Pod custom-extensions-instance1-26c9-0.spec.initContainers{extension-installer-17} Pulling Pulling image "perconalab/percona-postgresql-operator:PR-1214-6ef3702c0" kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:53:36 +0000 UTC Normal Pod custom-extensions-instance1-26c9-0.spec.initContainers{extension-installer-17} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:PR-1214-6ef3702c0" in 260ms (260ms including waiting). Image size: 95771923 bytes. kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:53:36 +0000 UTC Normal Pod custom-extensions-instance1-26c9-0.spec.initContainers{extension-installer-17} Created Created container: extension-installer-17 kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:53:36 +0000 UTC Normal Pod custom-extensions-instance1-26c9-0.spec.initContainers{extension-installer-17} Started Started container extension-installer-17 kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:53:38 +0000 UTC Normal Pod custom-extensions-instance1-26c9-0.spec.initContainers{extension-installer-17} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:PR-1214-6ef3702c0" in 789ms (789ms including waiting). Image size: 95771923 bytes. kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:53:39 +0000 UTC Warning Pod custom-extensions-instance1-26c9-0.spec.initContainers{extension-installer-17} BackOff Back-off restarting failed container extension-installer-17 in pod custom-extensions-instance1-26c9-0_kuttl-test-picked-cow(af93fd0c-e28e-407a-9c28-20f2d3cbf9b6) kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:53:40 +0000 UTC Normal Pod custom-extensions-instance1-46tm-0.spec.initContainers{extension-installer-17} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:PR-1214-6ef3702c0" in 834ms (834ms including waiting). Image size: 95771923 bytes. kubelet logger.go:42: 16:54:46 | custom-extensions | 2025-07-17 16:53:57 +0000 UTC Normal Pod custom-extensions-instance1-26c9-0.spec.initContainers{extension-installer-17} Pulled Successfully pulled image "perconalab/percona-postgresql-operator:PR-1214-6ef3702c0" in 1.201s (1.202s including waiting). Image size: 95771923 bytes. kubelet logger.go:42: 16:54:46 | custom-extensions | Deleting namespace: kuttl-test-picked-cow === NAME kuttl harness.go:403: run tests finished harness.go:510: cleaning up harness.go:567: removing temp folder: "" --- FAIL: kuttl (312.87s) --- FAIL: kuttl/harness (0.00s) --- FAIL: kuttl/harness/custom-extensions (312.21s) FAIL