=== RUN kuttl harness.go:459: starting setup harness.go:254: running tests using configured kubeconfig. harness.go:277: Successful connection to cluster at: https://34.29.28.142 harness.go:362: running tests harness.go:74: going to run test suite with timeout of 180 seconds for each step harness.go:374: testsuite: e2e-tests/tests has 46 tests === RUN kuttl/harness === RUN kuttl/harness/storage === PAUSE kuttl/harness/storage === CONT kuttl/harness/storage logger.go:42: 02:39:23 | storage | Creating namespace: kuttl-test-becoming-piranha logger.go:42: 02:39:23 | storage/0-deploy-operator | starting test step 0-deploy-operator logger.go:42: 02:39:23 | storage/0-deploy-operator | running command: [sh -c set -o errexit set -o xtrace source ../../functions init_temp_dir # do this only in the first TestStep deploy_operator deploy_tls_cluster_secrets deploy_client] logger.go:42: 02:39:23 | storage/0-deploy-operator | + source ../../functions logger.go:42: 02:39:23 | storage/0-deploy-operator | +++ realpath ../../.. logger.go:42: 02:39:23 | storage/0-deploy-operator | ++ ROOT_REPO=/mnt/jenkins/workspace/cloud-ps-operator_PR-1161 logger.go:42: 02:39:23 | storage/0-deploy-operator | ++++ pwd logger.go:42: 02:39:23 | storage/0-deploy-operator | +++ basename /mnt/jenkins/workspace/cloud-ps-operator_PR-1161/e2e-tests/tests/storage logger.go:42: 02:39:23 | storage/0-deploy-operator | ++ test_name=storage logger.go:42: 02:39:23 | storage/0-deploy-operator | ++ source /mnt/jenkins/workspace/cloud-ps-operator_PR-1161/e2e-tests/vars.sh logger.go:42: 02:39:23 | storage/0-deploy-operator | +++ export ROOT_REPO=/mnt/jenkins/workspace/cloud-ps-operator_PR-1161 logger.go:42: 02:39:23 | storage/0-deploy-operator | +++ ROOT_REPO=/mnt/jenkins/workspace/cloud-ps-operator_PR-1161 logger.go:42: 02:39:23 | storage/0-deploy-operator | +++ export DEPLOY_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-1161/deploy logger.go:42: 02:39:23 | storage/0-deploy-operator | +++ DEPLOY_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-1161/deploy logger.go:42: 02:39:23 | storage/0-deploy-operator | +++ export TESTS_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-1161/e2e-tests logger.go:42: 02:39:23 | storage/0-deploy-operator | +++ TESTS_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-1161/e2e-tests logger.go:42: 02:39:23 | storage/0-deploy-operator | +++ export TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-1161/e2e-tests/conf logger.go:42: 02:39:23 | storage/0-deploy-operator | +++ TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-1161/e2e-tests/conf logger.go:42: 02:39:23 | storage/0-deploy-operator | +++ export TEMP_DIR=/tmp/kuttl/ps/storage logger.go:42: 02:39:23 | storage/0-deploy-operator | +++ TEMP_DIR=/tmp/kuttl/ps/storage logger.go:42: 02:39:23 | storage/0-deploy-operator | ++++ git rev-parse --abbrev-ref HEAD logger.go:42: 02:39:23 | storage/0-deploy-operator | +++ export GIT_BRANCH=PR-1161 logger.go:42: 02:39:23 | storage/0-deploy-operator | +++ GIT_BRANCH=PR-1161 logger.go:42: 02:39:23 | storage/0-deploy-operator | +++ export VERSION=PR-1161-87502e4f logger.go:42: 02:39:23 | storage/0-deploy-operator | +++ VERSION=PR-1161-87502e4f logger.go:42: 02:39:23 | storage/0-deploy-operator | +++ export IMAGE=perconalab/percona-server-mysql-operator:PR-1161-87502e4f logger.go:42: 02:39:23 | storage/0-deploy-operator | +++ IMAGE=perconalab/percona-server-mysql-operator:PR-1161-87502e4f logger.go:42: 02:39:23 | storage/0-deploy-operator | +++ [[ -z 8.4 ]] logger.go:42: 02:39:23 | storage/0-deploy-operator | +++ export MYSQL_VERSION=8.4 logger.go:42: 02:39:23 | storage/0-deploy-operator | +++ MYSQL_VERSION=8.4 logger.go:42: 02:39:23 | storage/0-deploy-operator | +++ export IMAGE_MYSQL=perconalab/percona-server-mysql-operator:main-psmysql8.4 logger.go:42: 02:39:23 | storage/0-deploy-operator | +++ IMAGE_MYSQL=perconalab/percona-server-mysql-operator:main-psmysql8.4 logger.go:42: 02:39:23 | storage/0-deploy-operator | +++ export IMAGE_BACKUP=perconalab/percona-server-mysql-operator:main-backup8.4 logger.go:42: 02:39:23 | storage/0-deploy-operator | +++ IMAGE_BACKUP=perconalab/percona-server-mysql-operator:main-backup8.4 logger.go:42: 02:39:23 | storage/0-deploy-operator | +++ export IMAGE_ORCHESTRATOR=perconalab/percona-server-mysql-operator:main-orchestrator logger.go:42: 02:39:23 | storage/0-deploy-operator | +++ IMAGE_ORCHESTRATOR=perconalab/percona-server-mysql-operator:main-orchestrator logger.go:42: 02:39:23 | storage/0-deploy-operator | +++ export IMAGE_ROUTER=perconalab/percona-server-mysql-operator:main-router8.4 logger.go:42: 02:39:23 | storage/0-deploy-operator | +++ IMAGE_ROUTER=perconalab/percona-server-mysql-operator:main-router8.4 logger.go:42: 02:39:23 | storage/0-deploy-operator | +++ export IMAGE_TOOLKIT=perconalab/percona-server-mysql-operator:main-toolkit logger.go:42: 02:39:23 | storage/0-deploy-operator | +++ IMAGE_TOOLKIT=perconalab/percona-server-mysql-operator:main-toolkit logger.go:42: 02:39:23 | storage/0-deploy-operator | +++ export IMAGE_HAPROXY=perconalab/percona-server-mysql-operator:main-haproxy logger.go:42: 02:39:23 | storage/0-deploy-operator | +++ IMAGE_HAPROXY=perconalab/percona-server-mysql-operator:main-haproxy logger.go:42: 02:39:23 | storage/0-deploy-operator | +++ export PMM_SERVER_VERSION=1.4.3 logger.go:42: 02:39:23 | storage/0-deploy-operator | +++ PMM_SERVER_VERSION=1.4.3 logger.go:42: 02:39:23 | storage/0-deploy-operator | +++ export IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 02:39:23 | storage/0-deploy-operator | +++ IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 02:39:23 | storage/0-deploy-operator | +++ export IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 02:39:23 | storage/0-deploy-operator | +++ IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 02:39:23 | storage/0-deploy-operator | +++ export CERT_MANAGER_VER=1.18.2 logger.go:42: 02:39:23 | storage/0-deploy-operator | +++ CERT_MANAGER_VER=1.18.2 logger.go:42: 02:39:23 | storage/0-deploy-operator | +++ export MINIO_VER=5.4.0 logger.go:42: 02:39:23 | storage/0-deploy-operator | +++ MINIO_VER=5.4.0 logger.go:42: 02:39:23 | storage/0-deploy-operator | +++ export CHAOS_MESH_VER=2.7.2 logger.go:42: 02:39:23 | storage/0-deploy-operator | +++ CHAOS_MESH_VER=2.7.2 logger.go:42: 02:39:23 | storage/0-deploy-operator | +++ export VAULT_VER=0.16.1 logger.go:42: 02:39:23 | storage/0-deploy-operator | +++ VAULT_VER=0.16.1 logger.go:42: 02:39:23 | storage/0-deploy-operator | ++++ which gdate logger.go:42: 02:39:23 | storage/0-deploy-operator | which: no gdate in (/mnt/jenkins/workspace/cloud-ps-operator_PR-1161/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 02:39:23 | storage/0-deploy-operator | ++++ which date logger.go:42: 02:39:23 | storage/0-deploy-operator | +++ date=/usr/sbin/date logger.go:42: 02:39:23 | storage/0-deploy-operator | +++ oc get projects logger.go:42: 02:39:23 | storage/0-deploy-operator | +++ : logger.go:42: 02:39:23 | storage/0-deploy-operator | +++ kubectl get nodes logger.go:42: 02:39:23 | storage/0-deploy-operator | +++ grep '^minikube' logger.go:42: 02:39:24 | storage/0-deploy-operator | +++ which gsed logger.go:42: 02:39:24 | storage/0-deploy-operator | which: no gsed in (/mnt/jenkins/workspace/cloud-ps-operator_PR-1161/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 02:39:24 | storage/0-deploy-operator | +++ which sed logger.go:42: 02:39:24 | storage/0-deploy-operator | ++ sed=/usr/sbin/sed logger.go:42: 02:39:24 | storage/0-deploy-operator | ++ oc get projects logger.go:42: 02:39:24 | storage/0-deploy-operator | +++ kubectl version -o json logger.go:42: 02:39:24 | storage/0-deploy-operator | +++ grep '\-eks\-' logger.go:42: 02:39:24 | storage/0-deploy-operator | +++ jq -r .serverVersion.gitVersion logger.go:42: 02:39:24 | storage/0-deploy-operator | grep: warning: stray \ before - logger.go:42: 02:39:24 | storage/0-deploy-operator | Warning: version difference between client (1.34) and server (1.31) exceeds the supported minor version skew of +/-1 logger.go:42: 02:39:24 | storage/0-deploy-operator | ++ '[' ']' logger.go:42: 02:39:24 | storage/0-deploy-operator | ++ EKS=0 logger.go:42: 02:39:24 | storage/0-deploy-operator | + init_temp_dir logger.go:42: 02:39:24 | storage/0-deploy-operator | + rm -rf /tmp/kuttl/ps/storage logger.go:42: 02:39:24 | storage/0-deploy-operator | + mkdir -p /tmp/kuttl/ps/storage logger.go:42: 02:39:24 | storage/0-deploy-operator | + deploy_operator logger.go:42: 02:39:24 | storage/0-deploy-operator | + destroy_operator logger.go:42: 02:39:24 | storage/0-deploy-operator | + kubectl -n ps-operator delete deployment percona-server-mysql-operator --force --grace-period=0 logger.go:42: 02:39:24 | storage/0-deploy-operator | Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. logger.go:42: 02:39:25 | storage/0-deploy-operator | deployment.apps "percona-server-mysql-operator" force deleted from ps-operator namespace logger.go:42: 02:39:25 | storage/0-deploy-operator | + [[ -n ps-operator ]] logger.go:42: 02:39:25 | storage/0-deploy-operator | + kubectl delete namespace ps-operator --force --grace-period=0 logger.go:42: 02:39:25 | storage/0-deploy-operator | Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. logger.go:42: 02:39:25 | storage/0-deploy-operator | namespace "ps-operator" force deleted logger.go:42: 02:39:35 | storage/0-deploy-operator | + [[ -n ps-operator ]] logger.go:42: 02:39:35 | storage/0-deploy-operator | + create_namespace ps-operator logger.go:42: 02:39:35 | storage/0-deploy-operator | + local namespace=ps-operator logger.go:42: 02:39:35 | storage/0-deploy-operator | + [[ -n '' ]] logger.go:42: 02:39:35 | storage/0-deploy-operator | + kubectl delete namespace ps-operator --ignore-not-found logger.go:42: 02:39:35 | storage/0-deploy-operator | + kubectl wait --for=delete namespace ps-operator logger.go:42: 02:39:36 | storage/0-deploy-operator | + kubectl create namespace ps-operator logger.go:42: 02:39:36 | storage/0-deploy-operator | namespace/ps-operator created logger.go:42: 02:39:36 | storage/0-deploy-operator | + apply_crd logger.go:42: 02:39:36 | storage/0-deploy-operator | + kubectl -n ps-operator apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-ps-operator_PR-1161/deploy/crd.yaml logger.go:42: 02:39:37 | storage/0-deploy-operator | customresourcedefinition.apiextensions.k8s.io/perconaservermysqlbackups.ps.percona.com serverside-applied logger.go:42: 02:39:37 | storage/0-deploy-operator | customresourcedefinition.apiextensions.k8s.io/perconaservermysqlrestores.ps.percona.com serverside-applied logger.go:42: 02:39:39 | storage/0-deploy-operator | customresourcedefinition.apiextensions.k8s.io/perconaservermysqls.ps.percona.com serverside-applied logger.go:42: 02:39:39 | storage/0-deploy-operator | + apply_rbac logger.go:42: 02:39:39 | storage/0-deploy-operator | + local rbac_file logger.go:42: 02:39:39 | storage/0-deploy-operator | + '[' -n ps-operator ']' logger.go:42: 02:39:39 | storage/0-deploy-operator | + rbac_file=/mnt/jenkins/workspace/cloud-ps-operator_PR-1161/deploy/cw-rbac.yaml logger.go:42: 02:39:39 | storage/0-deploy-operator | + kubectl -n ps-operator apply -f /mnt/jenkins/workspace/cloud-ps-operator_PR-1161/deploy/cw-rbac.yaml logger.go:42: 02:39:40 | storage/0-deploy-operator | serviceaccount/percona-server-mysql-operator created logger.go:42: 02:39:40 | storage/0-deploy-operator | role.rbac.authorization.k8s.io/percona-server-mysql-operator-leaderelection created logger.go:42: 02:39:40 | storage/0-deploy-operator | clusterrole.rbac.authorization.k8s.io/percona-server-mysql-operator unchanged logger.go:42: 02:39:41 | storage/0-deploy-operator | rolebinding.rbac.authorization.k8s.io/percona-server-mysql-operator-leaderelection created logger.go:42: 02:39:41 | storage/0-deploy-operator | clusterrolebinding.rbac.authorization.k8s.io/percona-server-mysql-operator unchanged logger.go:42: 02:39:41 | storage/0-deploy-operator | + local operator_file logger.go:42: 02:39:41 | storage/0-deploy-operator | + '[' -n ps-operator ']' logger.go:42: 02:39:41 | storage/0-deploy-operator | + operator_file=/mnt/jenkins/workspace/cloud-ps-operator_PR-1161/deploy/cw-operator.yaml logger.go:42: 02:39:41 | storage/0-deploy-operator | + yq eval '(select(documentIndex==1).spec.template.spec.containers[] | select(.name=="manager").env[] | select(.name=="LOG_LEVEL").value) = "VERBOSE"' logger.go:42: 02:39:41 | storage/0-deploy-operator | + yq eval '(select(documentIndex==1).spec.template.spec.containers[] | select(.name=="manager").env[] | select(.name=="DISABLE_TELEMETRY").value) = "true"' logger.go:42: 02:39:41 | storage/0-deploy-operator | + kubectl -n ps-operator apply -f - logger.go:42: 02:39:41 | storage/0-deploy-operator | ++ printf 'select(documentIndex==1).spec.template.spec.containers[0].image="%s"' perconalab/percona-server-mysql-operator:PR-1161-87502e4f logger.go:42: 02:39:41 | storage/0-deploy-operator | + yq eval 'select(documentIndex==1).spec.template.spec.containers[0].image="perconalab/percona-server-mysql-operator:PR-1161-87502e4f"' /mnt/jenkins/workspace/cloud-ps-operator_PR-1161/deploy/cw-operator.yaml logger.go:42: 02:39:42 | storage/0-deploy-operator | configmap/percona-server-mysql-operator-config created logger.go:42: 02:39:43 | storage/0-deploy-operator | deployment.apps/percona-server-mysql-operator created logger.go:42: 02:39:43 | storage/0-deploy-operator | + deploy_tls_cluster_secrets logger.go:42: 02:39:43 | storage/0-deploy-operator | + kubectl -n kuttl-test-becoming-piranha apply -f /mnt/jenkins/workspace/cloud-ps-operator_PR-1161/e2e-tests/conf/ssl-secret.yaml logger.go:42: 02:39:44 | storage/0-deploy-operator | secret/test-ssl created logger.go:42: 02:39:44 | storage/0-deploy-operator | + deploy_client logger.go:42: 02:39:44 | storage/0-deploy-operator | + kubectl -n kuttl-test-becoming-piranha apply -f - logger.go:42: 02:39:44 | storage/0-deploy-operator | ++ printf '.spec.containers[0].image="%s"' perconalab/percona-server-mysql-operator:main-psmysql8.4 logger.go:42: 02:39:44 | storage/0-deploy-operator | + yq eval '.spec.containers[0].image="perconalab/percona-server-mysql-operator:main-psmysql8.4"' /mnt/jenkins/workspace/cloud-ps-operator_PR-1161/e2e-tests/conf/client.yaml logger.go:42: 02:39:45 | storage/0-deploy-operator | pod/mysql-client created logger.go:42: 02:39:46 | storage/0-deploy-operator | running command: [sh -c kubectl assert exist-enhanced deployment percona-server-mysql-operator -n ${OPERATOR_NS:-$NAMESPACE} --field-selector status.readyReplicas=1] logger.go:42: 02:39:46 | storage/0-deploy-operator | ASSERT deployment percona-server-mysql-operator matching field criteria 'status.readyReplicas=1' should exist. logger.go:42: 02:39:46 | storage/0-deploy-operator | ASSERT FAIL Resource(s) not found. logger.go:42: 02:39:47 | storage/0-deploy-operator | running command: [sh -c kubectl assert exist-enhanced deployment percona-server-mysql-operator -n ${OPERATOR_NS:-$NAMESPACE} --field-selector status.readyReplicas=1] logger.go:42: 02:39:47 | storage/0-deploy-operator | ASSERT deployment percona-server-mysql-operator matching field criteria 'status.readyReplicas=1' should exist. logger.go:42: 02:39:48 | storage/0-deploy-operator | ASSERT FAIL Resource(s) not found. logger.go:42: 02:39:49 | storage/0-deploy-operator | running command: [sh -c kubectl assert exist-enhanced deployment percona-server-mysql-operator -n ${OPERATOR_NS:-$NAMESPACE} --field-selector status.readyReplicas=1] logger.go:42: 02:39:49 | storage/0-deploy-operator | ASSERT deployment percona-server-mysql-operator matching field criteria 'status.readyReplicas=1' should exist. logger.go:42: 02:39:50 | storage/0-deploy-operator | ASSERT FAIL Resource(s) not found. logger.go:42: 02:39:51 | storage/0-deploy-operator | running command: [sh -c kubectl assert exist-enhanced deployment percona-server-mysql-operator -n ${OPERATOR_NS:-$NAMESPACE} --field-selector status.readyReplicas=1] logger.go:42: 02:39:51 | storage/0-deploy-operator | ASSERT deployment percona-server-mysql-operator matching field criteria 'status.readyReplicas=1' should exist. logger.go:42: 02:39:52 | storage/0-deploy-operator | ASSERT FAIL Resource(s) not found. logger.go:42: 02:39:53 | storage/0-deploy-operator | running command: [sh -c kubectl assert exist-enhanced deployment percona-server-mysql-operator -n ${OPERATOR_NS:-$NAMESPACE} --field-selector status.readyReplicas=1] logger.go:42: 02:39:53 | storage/0-deploy-operator | ASSERT deployment percona-server-mysql-operator matching field criteria 'status.readyReplicas=1' should exist. logger.go:42: 02:39:53 | storage/0-deploy-operator | ASSERT FAIL Resource(s) not found. logger.go:42: 02:39:55 | storage/0-deploy-operator | running command: [sh -c kubectl assert exist-enhanced deployment percona-server-mysql-operator -n ${OPERATOR_NS:-$NAMESPACE} --field-selector status.readyReplicas=1] logger.go:42: 02:39:55 | storage/0-deploy-operator | ASSERT deployment percona-server-mysql-operator matching field criteria 'status.readyReplicas=1' should exist. logger.go:42: 02:39:55 | storage/0-deploy-operator | INFO Found 1 resource(s). logger.go:42: 02:39:55 | storage/0-deploy-operator | NAME NAMESPACE COL0 logger.go:42: 02:39:55 | storage/0-deploy-operator | percona-server-mysql-operator ps-operator 1 logger.go:42: 02:39:55 | storage/0-deploy-operator | ASSERT PASS logger.go:42: 02:39:55 | storage/0-deploy-operator | test step completed 0-deploy-operator logger.go:42: 02:39:55 | storage/1-create-cluster-volume-emptypath | starting test step 1-create-cluster-volume-emptypath logger.go:42: 02:39:55 | storage/1-create-cluster-volume-emptypath | running command: [sh -c set -o errexit set -o xtrace source ../../functions get_cr \ | yq eval '.spec.mysql.volumeSpec = {"emptyDir": {} }' - \ | kubectl -n "${NAMESPACE}" apply -f -] logger.go:42: 02:39:55 | storage/1-create-cluster-volume-emptypath | + source ../../functions logger.go:42: 02:39:55 | storage/1-create-cluster-volume-emptypath | +++ realpath ../../.. logger.go:42: 02:39:55 | storage/1-create-cluster-volume-emptypath | ++ ROOT_REPO=/mnt/jenkins/workspace/cloud-ps-operator_PR-1161 logger.go:42: 02:39:55 | storage/1-create-cluster-volume-emptypath | ++++ pwd logger.go:42: 02:39:55 | storage/1-create-cluster-volume-emptypath | +++ basename /mnt/jenkins/workspace/cloud-ps-operator_PR-1161/e2e-tests/tests/storage logger.go:42: 02:39:55 | storage/1-create-cluster-volume-emptypath | ++ test_name=storage logger.go:42: 02:39:55 | storage/1-create-cluster-volume-emptypath | ++ source /mnt/jenkins/workspace/cloud-ps-operator_PR-1161/e2e-tests/vars.sh logger.go:42: 02:39:55 | storage/1-create-cluster-volume-emptypath | +++ export ROOT_REPO=/mnt/jenkins/workspace/cloud-ps-operator_PR-1161 logger.go:42: 02:39:55 | storage/1-create-cluster-volume-emptypath | +++ ROOT_REPO=/mnt/jenkins/workspace/cloud-ps-operator_PR-1161 logger.go:42: 02:39:55 | storage/1-create-cluster-volume-emptypath | +++ export DEPLOY_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-1161/deploy logger.go:42: 02:39:55 | storage/1-create-cluster-volume-emptypath | +++ DEPLOY_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-1161/deploy logger.go:42: 02:39:55 | storage/1-create-cluster-volume-emptypath | +++ export TESTS_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-1161/e2e-tests logger.go:42: 02:39:55 | storage/1-create-cluster-volume-emptypath | +++ TESTS_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-1161/e2e-tests logger.go:42: 02:39:55 | storage/1-create-cluster-volume-emptypath | +++ export TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-1161/e2e-tests/conf logger.go:42: 02:39:55 | storage/1-create-cluster-volume-emptypath | +++ TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-1161/e2e-tests/conf logger.go:42: 02:39:55 | storage/1-create-cluster-volume-emptypath | +++ export TEMP_DIR=/tmp/kuttl/ps/storage logger.go:42: 02:39:55 | storage/1-create-cluster-volume-emptypath | +++ TEMP_DIR=/tmp/kuttl/ps/storage logger.go:42: 02:39:55 | storage/1-create-cluster-volume-emptypath | ++++ git rev-parse --abbrev-ref HEAD logger.go:42: 02:39:55 | storage/1-create-cluster-volume-emptypath | +++ export GIT_BRANCH=PR-1161 logger.go:42: 02:39:55 | storage/1-create-cluster-volume-emptypath | +++ GIT_BRANCH=PR-1161 logger.go:42: 02:39:55 | storage/1-create-cluster-volume-emptypath | +++ export VERSION=PR-1161-87502e4f logger.go:42: 02:39:55 | storage/1-create-cluster-volume-emptypath | +++ VERSION=PR-1161-87502e4f logger.go:42: 02:39:55 | storage/1-create-cluster-volume-emptypath | +++ export IMAGE=perconalab/percona-server-mysql-operator:PR-1161-87502e4f logger.go:42: 02:39:55 | storage/1-create-cluster-volume-emptypath | +++ IMAGE=perconalab/percona-server-mysql-operator:PR-1161-87502e4f logger.go:42: 02:39:55 | storage/1-create-cluster-volume-emptypath | +++ [[ -z 8.4 ]] logger.go:42: 02:39:55 | storage/1-create-cluster-volume-emptypath | +++ export MYSQL_VERSION=8.4 logger.go:42: 02:39:55 | storage/1-create-cluster-volume-emptypath | +++ MYSQL_VERSION=8.4 logger.go:42: 02:39:55 | storage/1-create-cluster-volume-emptypath | +++ export IMAGE_MYSQL=perconalab/percona-server-mysql-operator:main-psmysql8.4 logger.go:42: 02:39:55 | storage/1-create-cluster-volume-emptypath | +++ IMAGE_MYSQL=perconalab/percona-server-mysql-operator:main-psmysql8.4 logger.go:42: 02:39:55 | storage/1-create-cluster-volume-emptypath | +++ export IMAGE_BACKUP=perconalab/percona-server-mysql-operator:main-backup8.4 logger.go:42: 02:39:55 | storage/1-create-cluster-volume-emptypath | +++ IMAGE_BACKUP=perconalab/percona-server-mysql-operator:main-backup8.4 logger.go:42: 02:39:55 | storage/1-create-cluster-volume-emptypath | +++ export IMAGE_ORCHESTRATOR=perconalab/percona-server-mysql-operator:main-orchestrator logger.go:42: 02:39:55 | storage/1-create-cluster-volume-emptypath | +++ IMAGE_ORCHESTRATOR=perconalab/percona-server-mysql-operator:main-orchestrator logger.go:42: 02:39:55 | storage/1-create-cluster-volume-emptypath | +++ export IMAGE_ROUTER=perconalab/percona-server-mysql-operator:main-router8.4 logger.go:42: 02:39:55 | storage/1-create-cluster-volume-emptypath | +++ IMAGE_ROUTER=perconalab/percona-server-mysql-operator:main-router8.4 logger.go:42: 02:39:55 | storage/1-create-cluster-volume-emptypath | +++ export IMAGE_TOOLKIT=perconalab/percona-server-mysql-operator:main-toolkit logger.go:42: 02:39:55 | storage/1-create-cluster-volume-emptypath | +++ IMAGE_TOOLKIT=perconalab/percona-server-mysql-operator:main-toolkit logger.go:42: 02:39:55 | storage/1-create-cluster-volume-emptypath | +++ export IMAGE_HAPROXY=perconalab/percona-server-mysql-operator:main-haproxy logger.go:42: 02:39:55 | storage/1-create-cluster-volume-emptypath | +++ IMAGE_HAPROXY=perconalab/percona-server-mysql-operator:main-haproxy logger.go:42: 02:39:55 | storage/1-create-cluster-volume-emptypath | +++ export PMM_SERVER_VERSION=1.4.3 logger.go:42: 02:39:55 | storage/1-create-cluster-volume-emptypath | +++ PMM_SERVER_VERSION=1.4.3 logger.go:42: 02:39:55 | storage/1-create-cluster-volume-emptypath | +++ export IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 02:39:55 | storage/1-create-cluster-volume-emptypath | +++ IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 02:39:55 | storage/1-create-cluster-volume-emptypath | +++ export IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 02:39:55 | storage/1-create-cluster-volume-emptypath | +++ IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 02:39:55 | storage/1-create-cluster-volume-emptypath | +++ export CERT_MANAGER_VER=1.18.2 logger.go:42: 02:39:55 | storage/1-create-cluster-volume-emptypath | +++ CERT_MANAGER_VER=1.18.2 logger.go:42: 02:39:55 | storage/1-create-cluster-volume-emptypath | +++ export MINIO_VER=5.4.0 logger.go:42: 02:39:55 | storage/1-create-cluster-volume-emptypath | +++ MINIO_VER=5.4.0 logger.go:42: 02:39:55 | storage/1-create-cluster-volume-emptypath | +++ export CHAOS_MESH_VER=2.7.2 logger.go:42: 02:39:55 | storage/1-create-cluster-volume-emptypath | +++ CHAOS_MESH_VER=2.7.2 logger.go:42: 02:39:55 | storage/1-create-cluster-volume-emptypath | +++ export VAULT_VER=0.16.1 logger.go:42: 02:39:55 | storage/1-create-cluster-volume-emptypath | +++ VAULT_VER=0.16.1 logger.go:42: 02:39:55 | storage/1-create-cluster-volume-emptypath | ++++ which gdate logger.go:42: 02:39:55 | storage/1-create-cluster-volume-emptypath | which: no gdate in (/mnt/jenkins/workspace/cloud-ps-operator_PR-1161/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 02:39:55 | storage/1-create-cluster-volume-emptypath | ++++ which date logger.go:42: 02:39:55 | storage/1-create-cluster-volume-emptypath | +++ date=/usr/sbin/date logger.go:42: 02:39:55 | storage/1-create-cluster-volume-emptypath | +++ oc get projects logger.go:42: 02:39:55 | storage/1-create-cluster-volume-emptypath | +++ : logger.go:42: 02:39:55 | storage/1-create-cluster-volume-emptypath | +++ kubectl get nodes logger.go:42: 02:39:55 | storage/1-create-cluster-volume-emptypath | +++ grep '^minikube' logger.go:42: 02:39:56 | storage/1-create-cluster-volume-emptypath | +++ which gsed logger.go:42: 02:39:56 | storage/1-create-cluster-volume-emptypath | which: no gsed in (/mnt/jenkins/workspace/cloud-ps-operator_PR-1161/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 02:39:56 | storage/1-create-cluster-volume-emptypath | +++ which sed logger.go:42: 02:39:56 | storage/1-create-cluster-volume-emptypath | ++ sed=/usr/sbin/sed logger.go:42: 02:39:56 | storage/1-create-cluster-volume-emptypath | ++ oc get projects logger.go:42: 02:39:56 | storage/1-create-cluster-volume-emptypath | +++ kubectl version -o json logger.go:42: 02:39:56 | storage/1-create-cluster-volume-emptypath | +++ jq -r .serverVersion.gitVersion logger.go:42: 02:39:56 | storage/1-create-cluster-volume-emptypath | +++ grep '\-eks\-' logger.go:42: 02:39:56 | storage/1-create-cluster-volume-emptypath | grep: warning: stray \ before - logger.go:42: 02:39:56 | storage/1-create-cluster-volume-emptypath | Warning: version difference between client (1.34) and server (1.31) exceeds the supported minor version skew of +/-1 logger.go:42: 02:39:56 | storage/1-create-cluster-volume-emptypath | ++ '[' ']' logger.go:42: 02:39:56 | storage/1-create-cluster-volume-emptypath | ++ EKS=0 logger.go:42: 02:39:56 | storage/1-create-cluster-volume-emptypath | + get_cr logger.go:42: 02:39:56 | storage/1-create-cluster-volume-emptypath | + yq eval '.spec.mysql.volumeSpec = {"emptyDir": {} }' - logger.go:42: 02:39:56 | storage/1-create-cluster-volume-emptypath | + local name_suffix= logger.go:42: 02:39:56 | storage/1-create-cluster-volume-emptypath | + local image_mysql=perconalab/percona-server-mysql-operator:main-psmysql8.4 logger.go:42: 02:39:56 | storage/1-create-cluster-volume-emptypath | + local image_backup=perconalab/percona-server-mysql-operator:main-backup8.4 logger.go:42: 02:39:56 | storage/1-create-cluster-volume-emptypath | + local image_orchestrator=perconalab/percona-server-mysql-operator:main-orchestrator logger.go:42: 02:39:56 | storage/1-create-cluster-volume-emptypath | + local image_router=perconalab/percona-server-mysql-operator:main-router8.4 logger.go:42: 02:39:56 | storage/1-create-cluster-volume-emptypath | + kubectl -n kuttl-test-becoming-piranha apply -f - logger.go:42: 02:39:56 | storage/1-create-cluster-volume-emptypath | + local image_toolkit=perconalab/percona-server-mysql-operator:main-toolkit logger.go:42: 02:39:56 | storage/1-create-cluster-volume-emptypath | + local image_haproxy=perconalab/percona-server-mysql-operator:main-haproxy logger.go:42: 02:39:56 | storage/1-create-cluster-volume-emptypath | + local image_pmm_client=perconalab/pmm-client:3-dev-latest logger.go:42: 02:39:56 | storage/1-create-cluster-volume-emptypath | + local cr_file=/mnt/jenkins/workspace/cloud-ps-operator_PR-1161/deploy/cr.yaml logger.go:42: 02:39:56 | storage/1-create-cluster-volume-emptypath | + yq eval 'del(.spec.secretsName)' - logger.go:42: 02:39:56 | storage/1-create-cluster-volume-emptypath | + yq eval '.spec.sslSecretName="test-ssl"' - logger.go:42: 02:39:56 | storage/1-create-cluster-volume-emptypath | ++ printf '.metadata.name="%s"' storage logger.go:42: 02:39:56 | storage/1-create-cluster-volume-emptypath | ++ printf '.spec.initContainer.image="%s"' perconalab/percona-server-mysql-operator:PR-1161-87502e4f logger.go:42: 02:39:56 | storage/1-create-cluster-volume-emptypath | + yq eval '.metadata.name="storage"' /mnt/jenkins/workspace/cloud-ps-operator_PR-1161/deploy/cr.yaml logger.go:42: 02:39:56 | storage/1-create-cluster-volume-emptypath | + yq eval '.spec.upgradeOptions.apply="disabled"' - logger.go:42: 02:39:56 | storage/1-create-cluster-volume-emptypath | + yq eval '.spec.initContainer.image="perconalab/percona-server-mysql-operator:PR-1161-87502e4f"' - logger.go:42: 02:39:56 | storage/1-create-cluster-volume-emptypath | + yq eval '.spec.mysql.clusterType="async"' - logger.go:42: 02:39:56 | storage/1-create-cluster-volume-emptypath | + yq eval .spec.mysql.gracePeriod=30 - logger.go:42: 02:39:56 | storage/1-create-cluster-volume-emptypath | + yq eval .spec.orchestrator.enabled=true - logger.go:42: 02:39:56 | storage/1-create-cluster-volume-emptypath | ++ printf '.spec.mysql.image="%s"' perconalab/percona-server-mysql-operator:main-psmysql8.4 logger.go:42: 02:39:56 | storage/1-create-cluster-volume-emptypath | ++ printf '.spec.orchestrator.image="%s"' perconalab/percona-server-mysql-operator:main-orchestrator logger.go:42: 02:39:56 | storage/1-create-cluster-volume-emptypath | + yq eval '.spec.mysql.image="perconalab/percona-server-mysql-operator:main-psmysql8.4"' - logger.go:42: 02:39:56 | storage/1-create-cluster-volume-emptypath | ++ printf '.spec.toolkit.image="%s"' perconalab/percona-server-mysql-operator:main-toolkit logger.go:42: 02:39:56 | storage/1-create-cluster-volume-emptypath | ++ printf '.spec.pmm.image="%s"' perconalab/pmm-client:3-dev-latest logger.go:42: 02:39:56 | storage/1-create-cluster-volume-emptypath | ++ printf '.spec.proxy.router.image="%s"' perconalab/percona-server-mysql-operator:main-router8.4 logger.go:42: 02:39:56 | storage/1-create-cluster-volume-emptypath | + '[' -n '' ']' logger.go:42: 02:39:56 | storage/1-create-cluster-volume-emptypath | + yq eval - logger.go:42: 02:39:56 | storage/1-create-cluster-volume-emptypath | + yq eval '.spec.orchestrator.image="perconalab/percona-server-mysql-operator:main-orchestrator"' - logger.go:42: 02:39:56 | storage/1-create-cluster-volume-emptypath | ++ printf '.spec.backup.image="%s"' perconalab/percona-server-mysql-operator:main-backup8.4 logger.go:42: 02:39:56 | storage/1-create-cluster-volume-emptypath | + yq eval '.spec.backup.image="perconalab/percona-server-mysql-operator:main-backup8.4"' - logger.go:42: 02:39:56 | storage/1-create-cluster-volume-emptypath | + yq eval '.spec.toolkit.image="perconalab/percona-server-mysql-operator:main-toolkit"' - logger.go:42: 02:39:56 | storage/1-create-cluster-volume-emptypath | + yq eval '.spec.pmm.image="perconalab/pmm-client:3-dev-latest"' - logger.go:42: 02:39:56 | storage/1-create-cluster-volume-emptypath | ++ printf '.spec.proxy.haproxy.image="%s"' perconalab/percona-server-mysql-operator:main-haproxy logger.go:42: 02:39:56 | storage/1-create-cluster-volume-emptypath | + yq eval '.spec.proxy.haproxy.image="perconalab/percona-server-mysql-operator:main-haproxy"' - logger.go:42: 02:39:56 | storage/1-create-cluster-volume-emptypath | + yq eval '.spec.proxy.router.image="perconalab/percona-server-mysql-operator:main-router8.4"' - logger.go:42: 02:39:57 | storage/1-create-cluster-volume-emptypath | perconaservermysql.ps.percona.com/storage created logger.go:42: 02:46:58 | storage/1-create-cluster-volume-emptypath | test step failed 1-create-cluster-volume-emptypath case.go:396: failed in step 1-create-cluster-volume-emptypath case.go:398: --- StatefulSet:kuttl-test-becoming-piranha/storage-mysql +++ StatefulSet:kuttl-test-becoming-piranha/storage-mysql @@ -1,13 +1,32 @@ apiVersion: apps/v1 kind: StatefulSet metadata: + annotations: + percona.com/last-config-hash: 5329803c51cf72912acd45eefb1709ce + labels: + app.kubernetes.io/component: database + app.kubernetes.io/instance: storage + app.kubernetes.io/managed-by: percona-server-mysql-operator + app.kubernetes.io/name: mysql + app.kubernetes.io/part-of: percona-server + managedFields: '[... elided field over 10 lines long ...]' name: storage-mysql namespace: kuttl-test-becoming-piranha + ownerReferences: + - apiVersion: ps.percona.com/v1 + blockOwnerDeletion: true + controller: true + kind: PerconaServerMySQL + name: storage + uid: e1661e51-a3a4-4cc4-a763-a460cb3ef381 +spec: '[... elided field over 10 lines long ...]' status: + availableReplicas: 0 collisionCount: 0 - currentReplicas: 3 + currentReplicas: 1 + currentRevision: storage-mysql-6f77cf47df observedGeneration: 1 - readyReplicas: 3 - replicas: 3 - updatedReplicas: 3 + replicas: 1 + updateRevision: storage-mysql-6f77cf47df + updatedReplicas: 1 case.go:398: resource StatefulSet:kuttl-test-becoming-piranha/storage-mysql: .status.currentReplicas: value mismatch, expected: 3 != actual: 1 case.go:398: statefulsets.apps "storage-haproxy" not found case.go:398: --- PerconaServerMySQL:kuttl-test-becoming-piranha/storage +++ PerconaServerMySQL:kuttl-test-becoming-piranha/storage @@ -1,22 +1,28 @@ apiVersion: ps.percona.com/v1 kind: PerconaServerMySQL metadata: + annotations: + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"ps.percona.com/v1","kind":"PerconaServerMySQL","metadata":{"annotations":{},"finalizers":["percona.com/delete-mysql-pods-in-order"],"name":"storage","namespace":"kuttl-test-becoming-piranha"},"spec":{"backup":{"enabled":true,"image":"perconalab/percona-server-mysql-operator:main-backup8.4","imagePullPolicy":"Always","pitr":{"enabled":false},"storages":{"s3-us-west":{"s3":{"bucket":"S3-BACKUP-BUCKET-NAME-HERE","credentialsSecret":"ps-cluster1-s3-credentials","region":"us-west-2"},"type":"s3","verifyTLS":true}}},"crVersion":"1.0.0","initContainer":{"image":"perconalab/percona-server-mysql-operator:PR-1161-87502e4f"},"mysql":{"affinity":{"antiAffinityTopologyKey":"kubernetes.io/hostname"},"autoRecovery":true,"clusterType":"async","exposePrimary":{"enabled":true},"gracePeriod":30,"image":"perconalab/percona-server-mysql-operator:main-psmysql8.4","imagePullPolicy":"Always","podDisruptionBudget":{"maxUnavailable":1},"resources":{"limits":{"memory":"4Gi"},"requests":{"memory":"2Gi"}},"size":3,"volumeSpec":{"emptyDir":{}}},"orchestrator":{"affinity":{"antiAffinityTopologyKey":"kubernetes.io/hostname"},"configuration":"","enabled":true,"gracePeriod":30,"image":"perconalab/percona-server-mysql-operator:main-orchestrator","imagePullPolicy":"Always","podDisruptionBudget":{"maxUnavailable":1},"resources":{"limits":{"memory":"256M"},"requests":{"memory":"128M"}},"size":3},"pmm":{"enabled":false,"image":"perconalab/pmm-client:3-dev-latest","imagePullPolicy":"Always","livenessProbe":{"failureThreshold":3,"periodSeconds":5,"successThreshold":1,"timeoutSeconds":3},"readinessProbe":{"failureThreshold":3,"periodSeconds":5,"successThreshold":1,"timeoutSeconds":3},"resources":{"requests":{"cpu":"300m","memory":"150M"}},"serverHost":"monitoring-service"},"proxy":{"haproxy":{"affinity":{"antiAffinityTopologyKey":"kubernetes.io/hostname"},"enabled":true,"gracePeriod":30,"image":"perconalab/percona-server-mysql-operator:main-haproxy","imagePullPolicy":"Always","podDisruptionBudget":{"maxUnavailable":1},"resources":{"requests":{"cpu":"600m","memory":"1Gi"}},"size":3},"router":{"affinity":{"antiAffinityTopologyKey":"kubernetes.io/hostname"},"enabled":false,"gracePeriod":30,"image":"perconalab/percona-server-mysql-operator:main-router8.4","imagePullPolicy":"Always","podDisruptionBudget":{"maxUnavailable":1},"resources":{"limits":{"memory":"256M"},"requests":{"memory":"256M"}},"size":3}},"sslSecretName":"test-ssl","toolkit":{"image":"perconalab/percona-server-mysql-operator:main-toolkit","imagePullPolicy":"Always"},"updateStrategy":"SmartUpdate","upgradeOptions":{"apply":"disabled","versionServiceEndpoint":"https://check.percona.com"}}} finalizers: - percona.com/delete-mysql-pods-in-order + managedFields: '[... elided field over 10 lines long ...]' name: storage namespace: kuttl-test-becoming-piranha +spec: '[... elided field over 10 lines long ...]' status: + conditions: '[... elided field over 10 lines long ...]' haproxy: - ready: 3 size: 3 - state: ready + state: initializing + host: storage-haproxy.kuttl-test-becoming-piranha mysql: - ready: 3 size: 3 - state: ready + state: initializing orchestrator: ready: 3 size: 3 state: ready - state: ready + router: {} + state: error case.go:398: resource PerconaServerMySQL:kuttl-test-becoming-piranha/storage: .status.haproxy.ready: key is missing from map logger.go:42: 02:46:58 | storage | storage events from ns kuttl-test-becoming-piranha: logger.go:42: 02:46:58 | storage | 2025-11-12 02:39:45 +0000 UTC Normal Pod mysql-client Binding Scheduled Successfully assigned kuttl-test-becoming-piranha/mysql-client to gke-jen-ps-1161-87502e4f-default-pool-914a03bb-swh9 default-scheduler logger.go:42: 02:46:58 | storage | 2025-11-12 02:39:45 +0000 UTC Normal Pod mysql-client.spec.containers{mysql-client} Pulled Container image "perconalab/percona-server-mysql-operator:main-psmysql8.4" already present on machine kubelet logger.go:42: 02:46:58 | storage | 2025-11-12 02:39:45 +0000 UTC Normal Pod mysql-client.spec.containers{mysql-client} Created Created container: mysql-client kubelet logger.go:42: 02:46:58 | storage | 2025-11-12 02:39:45 +0000 UTC Normal Pod mysql-client.spec.containers{mysql-client} Started Started container mysql-client kubelet logger.go:42: 02:46:58 | storage | 2025-11-12 02:39:58 +0000 UTC Normal Pod storage-mysql-0 Binding Scheduled Successfully assigned kuttl-test-becoming-piranha/storage-mysql-0 to gke-jen-ps-1161-87502e4f-default-pool-914a03bb-sgz1 default-scheduler logger.go:42: 02:46:58 | storage | 2025-11-12 02:39:58 +0000 UTC Normal StatefulSet.apps storage-mysql SuccessfulCreate create Pod storage-mysql-0 in StatefulSet storage-mysql successful statefulset-controller logger.go:42: 02:46:58 | storage | 2025-11-12 02:39:59 +0000 UTC Warning Pod storage-mysql-0 FailedMount MountVolume.SetUp failed for volume "users" : secret "internal-storage" not found kubelet logger.go:42: 02:46:58 | storage | 2025-11-12 02:39:59 +0000 UTC Normal Pod storage-orc-0 Binding Scheduled Successfully assigned kuttl-test-becoming-piranha/storage-orc-0 to gke-jen-ps-1161-87502e4f-default-pool-914a03bb-swh9 default-scheduler logger.go:42: 02:46:58 | storage | 2025-11-12 02:39:59 +0000 UTC Normal StatefulSet.apps storage-orc SuccessfulCreate create Pod storage-orc-0 in StatefulSet storage-orc successful statefulset-controller logger.go:42: 02:46:58 | storage | 2025-11-12 02:39:59 +0000 UTC Normal PodDisruptionBudget.policy storage-orchestrator NoPods No matching pods found controllermanager logger.go:42: 02:46:58 | storage | 2025-11-12 02:39:59 +0000 UTC Warning PerconaServerMySQL.ps.percona.com storage ClusterStateChanged -> Initializing ps-controller logger.go:42: 02:46:58 | storage | 2025-11-12 02:40:00 +0000 UTC Normal Pod storage-mysql-0.spec.initContainers{mysql-init} Pulling Pulling image "perconalab/percona-server-mysql-operator:PR-1161-87502e4f" kubelet logger.go:42: 02:46:58 | storage | 2025-11-12 02:40:00 +0000 UTC Normal Pod storage-orc-0.spec.initContainers{orchestrator-init} Pulling Pulling image "perconalab/percona-server-mysql-operator:PR-1161-87502e4f" kubelet logger.go:42: 02:46:58 | storage | 2025-11-12 02:40:00 +0000 UTC Normal Pod storage-orc-0.spec.initContainers{orchestrator-init} Pulled Successfully pulled image "perconalab/percona-server-mysql-operator:PR-1161-87502e4f" in 123ms (123ms including waiting). Image size: 113063860 bytes. kubelet logger.go:42: 02:46:58 | storage | 2025-11-12 02:40:00 +0000 UTC Normal Pod storage-orc-0.spec.initContainers{orchestrator-init} Created Created container: orchestrator-init kubelet logger.go:42: 02:46:58 | storage | 2025-11-12 02:40:00 +0000 UTC Normal Pod storage-orc-0.spec.initContainers{orchestrator-init} Started Started container orchestrator-init kubelet logger.go:42: 02:46:58 | storage | 2025-11-12 02:40:01 +0000 UTC Normal Pod storage-mysql-0.spec.initContainers{mysql-init} Pulled Successfully pulled image "perconalab/percona-server-mysql-operator:PR-1161-87502e4f" in 224ms (224ms including waiting). Image size: 113063860 bytes. kubelet logger.go:42: 02:46:58 | storage | 2025-11-12 02:40:01 +0000 UTC Normal Pod storage-mysql-0.spec.initContainers{mysql-init} Created Created container: mysql-init kubelet logger.go:42: 02:46:58 | storage | 2025-11-12 02:40:01 +0000 UTC Normal Pod storage-mysql-0.spec.initContainers{mysql-init} Started Started container mysql-init kubelet logger.go:42: 02:46:58 | storage | 2025-11-12 02:40:02 +0000 UTC Normal Pod storage-mysql-0.spec.containers{mysql} Pulling Pulling image "perconalab/percona-server-mysql-operator:main-psmysql8.4" kubelet logger.go:42: 02:46:58 | storage | 2025-11-12 02:40:02 +0000 UTC Normal Pod storage-mysql-0.spec.containers{mysql} Pulled Successfully pulled image "perconalab/percona-server-mysql-operator:main-psmysql8.4" in 99ms (99ms including waiting). Image size: 450511378 bytes. kubelet logger.go:42: 02:46:58 | storage | 2025-11-12 02:40:02 +0000 UTC Normal Pod storage-mysql-0.spec.containers{mysql} Created Created container: mysql kubelet logger.go:42: 02:46:58 | storage | 2025-11-12 02:40:02 +0000 UTC Normal Pod storage-mysql-0.spec.containers{mysql} Started Started container mysql kubelet logger.go:42: 02:46:58 | storage | 2025-11-12 02:40:02 +0000 UTC Normal Pod storage-mysql-0.spec.containers{xtrabackup} Pulling Pulling image "perconalab/percona-server-mysql-operator:main-backup8.4" kubelet logger.go:42: 02:46:58 | storage | 2025-11-12 02:40:02 +0000 UTC Normal Pod storage-mysql-0.spec.containers{xtrabackup} Pulled Successfully pulled image "perconalab/percona-server-mysql-operator:main-backup8.4" in 100ms (100ms including waiting). Image size: 549939332 bytes. kubelet logger.go:42: 02:46:58 | storage | 2025-11-12 02:40:02 +0000 UTC Normal Pod storage-orc-0.spec.containers{orchestrator} Pulling Pulling image "perconalab/percona-server-mysql-operator:main-orchestrator" kubelet logger.go:42: 02:46:58 | storage | 2025-11-12 02:40:02 +0000 UTC Normal Pod storage-orc-0.spec.containers{orchestrator} Pulled Successfully pulled image "perconalab/percona-server-mysql-operator:main-orchestrator" in 80ms (80ms including waiting). Image size: 77414761 bytes. kubelet logger.go:42: 02:46:58 | storage | 2025-11-12 02:40:02 +0000 UTC Normal Pod storage-orc-0.spec.containers{orchestrator} Created Created container: orchestrator kubelet logger.go:42: 02:46:58 | storage | 2025-11-12 02:40:02 +0000 UTC Normal Pod storage-orc-0.spec.containers{orchestrator} Started Started container orchestrator kubelet logger.go:42: 02:46:58 | storage | 2025-11-12 02:40:02 +0000 UTC Normal Pod storage-orc-0.spec.containers{mysql-monit} Pulling Pulling image "perconalab/percona-server-mysql-operator:main-orchestrator" kubelet logger.go:42: 02:46:58 | storage | 2025-11-12 02:40:02 +0000 UTC Normal Pod storage-orc-0.spec.containers{mysql-monit} Pulled Successfully pulled image "perconalab/percona-server-mysql-operator:main-orchestrator" in 83ms (83ms including waiting). Image size: 77414761 bytes. kubelet logger.go:42: 02:46:58 | storage | 2025-11-12 02:40:02 +0000 UTC Normal Pod storage-orc-0.spec.containers{mysql-monit} Created Created container: mysql-monit kubelet logger.go:42: 02:46:58 | storage | 2025-11-12 02:40:02 +0000 UTC Normal Pod storage-orc-0.spec.containers{mysql-monit} Started Started container mysql-monit kubelet logger.go:42: 02:46:58 | storage | 2025-11-12 02:40:03 +0000 UTC Normal Pod storage-mysql-0.spec.containers{xtrabackup} Created Created container: xtrabackup kubelet logger.go:42: 02:46:58 | storage | 2025-11-12 02:40:03 +0000 UTC Normal Pod storage-mysql-0.spec.containers{xtrabackup} Started Started container xtrabackup kubelet logger.go:42: 02:46:58 | storage | 2025-11-12 02:40:03 +0000 UTC Normal Pod storage-mysql-0.spec.containers{pt-heartbeat} Pulling Pulling image "perconalab/percona-server-mysql-operator:main-toolkit" kubelet logger.go:42: 02:46:58 | storage | 2025-11-12 02:40:03 +0000 UTC Normal Pod storage-mysql-0.spec.containers{pt-heartbeat} Pulled Successfully pulled image "perconalab/percona-server-mysql-operator:main-toolkit" in 78ms (78ms including waiting). Image size: 139956959 bytes. kubelet logger.go:42: 02:46:58 | storage | 2025-11-12 02:40:03 +0000 UTC Normal Pod storage-mysql-0.spec.containers{pt-heartbeat} Created Created container: pt-heartbeat kubelet logger.go:42: 02:46:58 | storage | 2025-11-12 02:40:03 +0000 UTC Normal Pod storage-mysql-0.spec.containers{pt-heartbeat} Started Started container pt-heartbeat kubelet logger.go:42: 02:46:58 | storage | 2025-11-12 02:40:35 +0000 UTC Normal Pod storage-orc-1 Binding Scheduled Successfully assigned kuttl-test-becoming-piranha/storage-orc-1 to gke-jen-ps-1161-87502e4f-default-pool-914a03bb-5v5n default-scheduler logger.go:42: 02:46:58 | storage | 2025-11-12 02:40:35 +0000 UTC Normal Pod storage-orc-1.spec.initContainers{orchestrator-init} Pulling Pulling image "perconalab/percona-server-mysql-operator:PR-1161-87502e4f" kubelet logger.go:42: 02:46:58 | storage | 2025-11-12 02:40:35 +0000 UTC Normal Pod storage-orc-1.spec.initContainers{orchestrator-init} Pulled Successfully pulled image "perconalab/percona-server-mysql-operator:PR-1161-87502e4f" in 117ms (117ms including waiting). Image size: 113063860 bytes. kubelet logger.go:42: 02:46:58 | storage | 2025-11-12 02:40:35 +0000 UTC Normal Pod storage-orc-1.spec.initContainers{orchestrator-init} Created Created container: orchestrator-init kubelet logger.go:42: 02:46:58 | storage | 2025-11-12 02:40:35 +0000 UTC Normal Pod storage-orc-1.spec.initContainers{orchestrator-init} Started Started container orchestrator-init kubelet logger.go:42: 02:46:58 | storage | 2025-11-12 02:40:35 +0000 UTC Normal StatefulSet.apps storage-orc SuccessfulCreate create Pod storage-orc-1 in StatefulSet storage-orc successful statefulset-controller logger.go:42: 02:46:58 | storage | 2025-11-12 02:40:37 +0000 UTC Normal Pod storage-orc-1.spec.containers{orchestrator} Pulling Pulling image "perconalab/percona-server-mysql-operator:main-orchestrator" kubelet logger.go:42: 02:46:58 | storage | 2025-11-12 02:40:37 +0000 UTC Normal Pod storage-orc-1.spec.containers{orchestrator} Pulled Successfully pulled image "perconalab/percona-server-mysql-operator:main-orchestrator" in 85ms (85ms including waiting). Image size: 77414761 bytes. kubelet logger.go:42: 02:46:58 | storage | 2025-11-12 02:40:37 +0000 UTC Normal Pod storage-orc-1.spec.containers{orchestrator} Created Created container: orchestrator kubelet logger.go:42: 02:46:58 | storage | 2025-11-12 02:40:37 +0000 UTC Normal Pod storage-orc-1.spec.containers{orchestrator} Started Started container orchestrator kubelet logger.go:42: 02:46:58 | storage | 2025-11-12 02:40:37 +0000 UTC Normal Pod storage-orc-1.spec.containers{mysql-monit} Pulling Pulling image "perconalab/percona-server-mysql-operator:main-orchestrator" kubelet logger.go:42: 02:46:58 | storage | 2025-11-12 02:40:37 +0000 UTC Normal Pod storage-orc-1.spec.containers{mysql-monit} Pulled Successfully pulled image "perconalab/percona-server-mysql-operator:main-orchestrator" in 120ms (120ms including waiting). Image size: 77414761 bytes. kubelet logger.go:42: 02:46:58 | storage | 2025-11-12 02:40:37 +0000 UTC Normal Pod storage-orc-1.spec.containers{mysql-monit} Created Created container: mysql-monit kubelet logger.go:42: 02:46:58 | storage | 2025-11-12 02:40:37 +0000 UTC Normal Pod storage-orc-1.spec.containers{mysql-monit} Started Started container mysql-monit kubelet logger.go:42: 02:46:58 | storage | 2025-11-12 02:40:37 +0000 UTC Warning PerconaServerMySQL.ps.percona.com storage ReconcileError Failed to reconcile cluster ps-controller logger.go:42: 02:46:58 | storage | 2025-11-12 02:41:10 +0000 UTC Normal Pod storage-orc-2 Binding Scheduled Successfully assigned kuttl-test-becoming-piranha/storage-orc-2 to gke-jen-ps-1161-87502e4f-default-pool-914a03bb-sgz1 default-scheduler logger.go:42: 02:46:58 | storage | 2025-11-12 02:41:10 +0000 UTC Normal StatefulSet.apps storage-orc SuccessfulCreate create Pod storage-orc-2 in StatefulSet storage-orc successful statefulset-controller logger.go:42: 02:46:58 | storage | 2025-11-12 02:41:11 +0000 UTC Normal Pod storage-orc-2.spec.initContainers{orchestrator-init} Pulling Pulling image "perconalab/percona-server-mysql-operator:PR-1161-87502e4f" kubelet logger.go:42: 02:46:58 | storage | 2025-11-12 02:41:11 +0000 UTC Normal Pod storage-orc-2.spec.initContainers{orchestrator-init} Pulled Successfully pulled image "perconalab/percona-server-mysql-operator:PR-1161-87502e4f" in 123ms (123ms including waiting). Image size: 113063860 bytes. kubelet logger.go:42: 02:46:58 | storage | 2025-11-12 02:41:11 +0000 UTC Normal Pod storage-orc-2.spec.initContainers{orchestrator-init} Created Created container: orchestrator-init kubelet logger.go:42: 02:46:58 | storage | 2025-11-12 02:41:11 +0000 UTC Normal Pod storage-orc-2.spec.initContainers{orchestrator-init} Started Started container orchestrator-init kubelet logger.go:42: 02:46:58 | storage | 2025-11-12 02:41:13 +0000 UTC Normal Pod storage-orc-2.spec.containers{orchestrator} Pulling Pulling image "perconalab/percona-server-mysql-operator:main-orchestrator" kubelet logger.go:42: 02:46:58 | storage | 2025-11-12 02:41:13 +0000 UTC Normal Pod storage-orc-2.spec.containers{orchestrator} Pulled Successfully pulled image "perconalab/percona-server-mysql-operator:main-orchestrator" in 92ms (92ms including waiting). Image size: 77414761 bytes. kubelet logger.go:42: 02:46:58 | storage | 2025-11-12 02:41:13 +0000 UTC Normal Pod storage-orc-2.spec.containers{orchestrator} Created Created container: orchestrator kubelet logger.go:42: 02:46:58 | storage | 2025-11-12 02:41:13 +0000 UTC Normal Pod storage-orc-2.spec.containers{orchestrator} Started Started container orchestrator kubelet logger.go:42: 02:46:58 | storage | 2025-11-12 02:41:14 +0000 UTC Normal Pod storage-orc-2.spec.containers{mysql-monit} Pulling Pulling image "perconalab/percona-server-mysql-operator:main-orchestrator" kubelet logger.go:42: 02:46:58 | storage | 2025-11-12 02:41:14 +0000 UTC Normal Pod storage-orc-2.spec.containers{mysql-monit} Pulled Successfully pulled image "perconalab/percona-server-mysql-operator:main-orchestrator" in 91ms (91ms including waiting). Image size: 77414761 bytes. kubelet logger.go:42: 02:46:58 | storage | 2025-11-12 02:41:14 +0000 UTC Normal Pod storage-orc-2.spec.containers{mysql-monit} Created Created container: mysql-monit kubelet logger.go:42: 02:46:58 | storage | 2025-11-12 02:41:14 +0000 UTC Normal Pod storage-orc-2.spec.containers{mysql-monit} Started Started container mysql-monit kubelet logger.go:42: 02:46:58 | storage | 2025-11-12 02:41:15 +0000 UTC Warning Pod storage-mysql-0.spec.containers{mysql} Unhealthy Startup probe failed: 2025/11/12 02:40:20 Waiting for MySQL ready state 2025/11/12 02:40:20 MySQL is ready 2025/11/12 02:40:20 Peers: [6463613739373365.storage-mysql-unready.kuttl-test-becoming-piranha] 2025/11/12 02:40:20 FQDN: storage-mysql-0.storage-mysql.kuttl-test-becoming-piranha 2025/11/12 02:40:25 Primary: 6463613739373365.storage-mysql-unready.kuttl-test-becoming-piranha Replicas: [storage-mysql-0.storage-mysql.kuttl-test-becoming-piranha] 2025/11/12 02:40:25 lookup storage-mysql-0 [10.50.10.15] 2025/11/12 02:40:25 PodIP: 10.50.10.15 2025/11/12 02:41:15 bootstrap finished in 55.056717 seconds 2025/11/12 02:41:15 bootstrap failed: get primary IP: lookup 6463613739373365.storage-mysql-unready.kuttl-test-becoming-piranha: lookup 6463613739373365.storage-mysql-unready.kuttl-test-becoming-piranha on 34.118.224.10:53: read udp 10.50.10.15:33761->34.118.224.10:53: i/o timeout kubelet logger.go:42: 02:46:58 | storage | 2025-11-12 02:41:15 +0000 UTC Normal Pod storage-mysql-0.spec.containers{mysql} Killing Container mysql failed startup probe, will be restarted kubelet logger.go:42: 02:46:58 | storage | 2025-11-12 02:41:19 +0000 UTC Normal Pod storage-mysql-0.spec.containers{mysql} Pulled Successfully pulled image "perconalab/percona-server-mysql-operator:main-psmysql8.4" in 92ms (92ms including waiting). Image size: 450511378 bytes. kubelet logger.go:42: 02:46:58 | storage | 2025-11-12 02:45:20 +0000 UTC Warning Pod storage-mysql-0.spec.containers{mysql} Unhealthy Startup probe failed: 2025/11/12 02:45:10 Waiting for MySQL ready state 2025/11/12 02:45:10 MySQL is ready 2025/11/12 02:45:10 Peers: [6463613739373365.storage-mysql-unready.kuttl-test-becoming-piranha] 2025/11/12 02:45:10 FQDN: storage-mysql-0.storage-mysql.kuttl-test-becoming-piranha 2025/11/12 02:45:20 bootstrap finished in 10.005086 seconds 2025/11/12 02:45:20 bootstrap failed: select donor: connect to 6463613739373365.storage-mysql-unready.kuttl-test-becoming-piranha: ping DB: dial tcp: lookup 6463613739373365.storage-mysql-unready.kuttl-test-becoming-piranha: i/o timeout kubelet logger.go:42: 02:46:58 | storage | 2025-11-12 02:46:36 +0000 UTC Warning PerconaServerMySQL.ps.percona.com storage ClusterStateChanged Error -> Initializing ps-controller logger.go:42: 02:46:58 | storage | Deleting namespace: kuttl-test-becoming-piranha === NAME kuttl harness.go:403: run tests finished harness.go:510: cleaning up harness.go:567: removing temp folder: "" --- FAIL: kuttl (476.98s) --- FAIL: kuttl/harness (0.00s) --- FAIL: kuttl/harness/storage (476.26s) FAIL