=== RUN kuttl harness.go:459: starting setup harness.go:254: running tests using configured kubeconfig. harness.go:277: Successful connection to cluster at: https://34.170.171.4 harness.go:362: running tests harness.go:74: going to run test suite with timeout of 180 seconds for each step harness.go:374: testsuite: e2e-tests/tests has 39 tests === RUN kuttl/harness === RUN kuttl/harness/storage === PAUSE kuttl/harness/storage === CONT kuttl/harness/storage logger.go:42: 20:38:07 | storage | Creating namespace: kuttl-test-careful-iguana logger.go:42: 20:38:08 | storage/0-deploy-operator | starting test step 0-deploy-operator logger.go:42: 20:38:08 | storage/0-deploy-operator | running command: [sh -c set -o errexit set -o xtrace source ../../functions init_temp_dir # do this only in the first TestStep deploy_operator deploy_non_tls_cluster_secrets deploy_tls_cluster_secrets deploy_client] logger.go:42: 20:38:08 | storage/0-deploy-operator | + source ../../functions logger.go:42: 20:38:08 | storage/0-deploy-operator | +++ realpath ../../.. logger.go:42: 20:38:08 | storage/0-deploy-operator | ++ ROOT_REPO=/mnt/jenkins/workspace/cloud-ps-operator_PR-1012 logger.go:42: 20:38:08 | storage/0-deploy-operator | ++++ pwd logger.go:42: 20:38:08 | storage/0-deploy-operator | +++ basename /mnt/jenkins/workspace/cloud-ps-operator_PR-1012/e2e-tests/tests/storage logger.go:42: 20:38:08 | storage/0-deploy-operator | ++ test_name=storage logger.go:42: 20:38:08 | storage/0-deploy-operator | ++ source /mnt/jenkins/workspace/cloud-ps-operator_PR-1012/e2e-tests/vars.sh logger.go:42: 20:38:08 | storage/0-deploy-operator | +++ export ROOT_REPO=/mnt/jenkins/workspace/cloud-ps-operator_PR-1012 logger.go:42: 20:38:08 | storage/0-deploy-operator | +++ ROOT_REPO=/mnt/jenkins/workspace/cloud-ps-operator_PR-1012 logger.go:42: 20:38:08 | storage/0-deploy-operator | +++ export DEPLOY_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-1012/deploy logger.go:42: 20:38:08 | storage/0-deploy-operator | +++ DEPLOY_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-1012/deploy logger.go:42: 20:38:08 | storage/0-deploy-operator | +++ export TESTS_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-1012/e2e-tests logger.go:42: 20:38:08 | storage/0-deploy-operator | +++ TESTS_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-1012/e2e-tests logger.go:42: 20:38:08 | storage/0-deploy-operator | +++ export TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-1012/e2e-tests/conf logger.go:42: 20:38:08 | storage/0-deploy-operator | +++ TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-1012/e2e-tests/conf logger.go:42: 20:38:08 | storage/0-deploy-operator | +++ export TEMP_DIR=/tmp/kuttl/ps/storage logger.go:42: 20:38:08 | storage/0-deploy-operator | +++ TEMP_DIR=/tmp/kuttl/ps/storage logger.go:42: 20:38:08 | storage/0-deploy-operator | ++++ git rev-parse --abbrev-ref HEAD logger.go:42: 20:38:08 | storage/0-deploy-operator | +++ export GIT_BRANCH=PR-1012 logger.go:42: 20:38:08 | storage/0-deploy-operator | +++ GIT_BRANCH=PR-1012 logger.go:42: 20:38:08 | storage/0-deploy-operator | +++ export VERSION=PR-1012-344c570a logger.go:42: 20:38:08 | storage/0-deploy-operator | +++ VERSION=PR-1012-344c570a logger.go:42: 20:38:08 | storage/0-deploy-operator | +++ export IMAGE=perconalab/percona-server-mysql-operator:PR-1012-344c570a logger.go:42: 20:38:08 | storage/0-deploy-operator | +++ IMAGE=perconalab/percona-server-mysql-operator:PR-1012-344c570a logger.go:42: 20:38:08 | storage/0-deploy-operator | +++ export MYSQL_VERSION=8.0 logger.go:42: 20:38:08 | storage/0-deploy-operator | +++ MYSQL_VERSION=8.0 logger.go:42: 20:38:08 | storage/0-deploy-operator | +++ export IMAGE_MYSQL=perconalab/percona-server-mysql-operator:main-psmysql8.0 logger.go:42: 20:38:08 | storage/0-deploy-operator | +++ IMAGE_MYSQL=perconalab/percona-server-mysql-operator:main-psmysql8.0 logger.go:42: 20:38:08 | storage/0-deploy-operator | +++ export IMAGE_BACKUP=perconalab/percona-server-mysql-operator:main-backup8.0 logger.go:42: 20:38:08 | storage/0-deploy-operator | +++ IMAGE_BACKUP=perconalab/percona-server-mysql-operator:main-backup8.0 logger.go:42: 20:38:08 | storage/0-deploy-operator | +++ export IMAGE_ORCHESTRATOR=perconalab/percona-server-mysql-operator:main-orchestrator logger.go:42: 20:38:08 | storage/0-deploy-operator | +++ IMAGE_ORCHESTRATOR=perconalab/percona-server-mysql-operator:main-orchestrator logger.go:42: 20:38:08 | storage/0-deploy-operator | +++ export IMAGE_ROUTER=perconalab/percona-server-mysql-operator:main-router8.0 logger.go:42: 20:38:08 | storage/0-deploy-operator | +++ IMAGE_ROUTER=perconalab/percona-server-mysql-operator:main-router8.0 logger.go:42: 20:38:08 | storage/0-deploy-operator | +++ export IMAGE_TOOLKIT=perconalab/percona-server-mysql-operator:main-toolkit logger.go:42: 20:38:08 | storage/0-deploy-operator | +++ IMAGE_TOOLKIT=perconalab/percona-server-mysql-operator:main-toolkit logger.go:42: 20:38:08 | storage/0-deploy-operator | +++ export IMAGE_HAPROXY=perconalab/percona-server-mysql-operator:main-haproxy logger.go:42: 20:38:08 | storage/0-deploy-operator | +++ IMAGE_HAPROXY=perconalab/percona-server-mysql-operator:main-haproxy logger.go:42: 20:38:08 | storage/0-deploy-operator | +++ export PMM_SERVER_VERSION=1.4.3 logger.go:42: 20:38:08 | storage/0-deploy-operator | +++ PMM_SERVER_VERSION=1.4.3 logger.go:42: 20:38:08 | storage/0-deploy-operator | +++ export IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 20:38:08 | storage/0-deploy-operator | +++ IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 20:38:08 | storage/0-deploy-operator | +++ export IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 20:38:08 | storage/0-deploy-operator | +++ IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 20:38:08 | storage/0-deploy-operator | +++ export CERT_MANAGER_VER=1.18.2 logger.go:42: 20:38:08 | storage/0-deploy-operator | +++ CERT_MANAGER_VER=1.18.2 logger.go:42: 20:38:08 | storage/0-deploy-operator | +++ export MINIO_VER=5.4.0 logger.go:42: 20:38:08 | storage/0-deploy-operator | +++ MINIO_VER=5.4.0 logger.go:42: 20:38:08 | storage/0-deploy-operator | +++ export CHAOS_MESH_VER=2.7.2 logger.go:42: 20:38:08 | storage/0-deploy-operator | +++ CHAOS_MESH_VER=2.7.2 logger.go:42: 20:38:08 | storage/0-deploy-operator | +++ export VAULT_VER=0.16.1 logger.go:42: 20:38:08 | storage/0-deploy-operator | +++ VAULT_VER=0.16.1 logger.go:42: 20:38:08 | storage/0-deploy-operator | ++++ which gdate logger.go:42: 20:38:08 | storage/0-deploy-operator | which: no gdate in (/mnt/jenkins/workspace/cloud-ps-operator_PR-1012/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 20:38:08 | storage/0-deploy-operator | ++++ which date logger.go:42: 20:38:08 | storage/0-deploy-operator | +++ date=/usr/sbin/date logger.go:42: 20:38:08 | storage/0-deploy-operator | +++ oc get projects logger.go:42: 20:38:08 | storage/0-deploy-operator | +++ : logger.go:42: 20:38:08 | storage/0-deploy-operator | +++ kubectl get nodes logger.go:42: 20:38:08 | storage/0-deploy-operator | +++ grep '^minikube' logger.go:42: 20:38:08 | storage/0-deploy-operator | ++ oc get projects logger.go:42: 20:38:08 | storage/0-deploy-operator | +++ kubectl version -o json logger.go:42: 20:38:08 | storage/0-deploy-operator | +++ jq -r .serverVersion.gitVersion logger.go:42: 20:38:08 | storage/0-deploy-operator | +++ grep '\-eks\-' logger.go:42: 20:38:08 | storage/0-deploy-operator | grep: warning: stray \ before - logger.go:42: 20:38:09 | storage/0-deploy-operator | Warning: version difference between client (1.34) and server (1.30) exceeds the supported minor version skew of +/-1 logger.go:42: 20:38:09 | storage/0-deploy-operator | ++ '[' ']' logger.go:42: 20:38:09 | storage/0-deploy-operator | ++ EKS=0 logger.go:42: 20:38:09 | storage/0-deploy-operator | + init_temp_dir logger.go:42: 20:38:09 | storage/0-deploy-operator | + rm -rf /tmp/kuttl/ps/storage logger.go:42: 20:38:09 | storage/0-deploy-operator | + mkdir -p /tmp/kuttl/ps/storage logger.go:42: 20:38:09 | storage/0-deploy-operator | + deploy_operator logger.go:42: 20:38:09 | storage/0-deploy-operator | + destroy_operator logger.go:42: 20:38:09 | storage/0-deploy-operator | + kubectl -n ps-operator delete deployment percona-server-mysql-operator --force --grace-period=0 logger.go:42: 20:38:09 | storage/0-deploy-operator | Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. logger.go:42: 20:38:09 | storage/0-deploy-operator | Error from server (NotFound): deployments.apps "percona-server-mysql-operator" not found logger.go:42: 20:38:09 | storage/0-deploy-operator | + true logger.go:42: 20:38:09 | storage/0-deploy-operator | + [[ -n ps-operator ]] logger.go:42: 20:38:09 | storage/0-deploy-operator | + kubectl delete namespace ps-operator --force --grace-period=0 logger.go:42: 20:38:10 | storage/0-deploy-operator | Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. logger.go:42: 20:38:10 | storage/0-deploy-operator | Error from server (NotFound): namespaces "ps-operator" not found logger.go:42: 20:38:10 | storage/0-deploy-operator | + true logger.go:42: 20:38:10 | storage/0-deploy-operator | + [[ -n ps-operator ]] logger.go:42: 20:38:10 | storage/0-deploy-operator | + create_namespace ps-operator logger.go:42: 20:38:10 | storage/0-deploy-operator | + local namespace=ps-operator logger.go:42: 20:38:10 | storage/0-deploy-operator | + [[ -n '' ]] logger.go:42: 20:38:10 | storage/0-deploy-operator | + kubectl delete namespace ps-operator --ignore-not-found logger.go:42: 20:38:11 | storage/0-deploy-operator | + kubectl wait --for=delete namespace ps-operator logger.go:42: 20:38:11 | storage/0-deploy-operator | + kubectl create namespace ps-operator logger.go:42: 20:38:12 | storage/0-deploy-operator | namespace/ps-operator created logger.go:42: 20:38:12 | storage/0-deploy-operator | + kubectl -n ps-operator apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-ps-operator_PR-1012/deploy/crd.yaml logger.go:42: 20:38:13 | storage/0-deploy-operator | customresourcedefinition.apiextensions.k8s.io/perconaservermysqlbackups.ps.percona.com serverside-applied logger.go:42: 20:38:13 | storage/0-deploy-operator | customresourcedefinition.apiextensions.k8s.io/perconaservermysqlrestores.ps.percona.com serverside-applied logger.go:42: 20:38:14 | storage/0-deploy-operator | customresourcedefinition.apiextensions.k8s.io/perconaservermysqls.ps.percona.com serverside-applied logger.go:42: 20:38:14 | storage/0-deploy-operator | + '[' -n ps-operator ']' logger.go:42: 20:38:14 | storage/0-deploy-operator | + kubectl -n ps-operator apply -f /mnt/jenkins/workspace/cloud-ps-operator_PR-1012/deploy/cw-rbac.yaml logger.go:42: 20:38:16 | storage/0-deploy-operator | serviceaccount/percona-server-mysql-operator created logger.go:42: 20:38:16 | storage/0-deploy-operator | role.rbac.authorization.k8s.io/percona-server-mysql-operator-leaderelection created logger.go:42: 20:38:16 | storage/0-deploy-operator | clusterrole.rbac.authorization.k8s.io/percona-server-mysql-operator unchanged logger.go:42: 20:38:17 | storage/0-deploy-operator | rolebinding.rbac.authorization.k8s.io/percona-server-mysql-operator-leaderelection created logger.go:42: 20:38:17 | storage/0-deploy-operator | clusterrolebinding.rbac.authorization.k8s.io/percona-server-mysql-operator unchanged logger.go:42: 20:38:17 | storage/0-deploy-operator | + yq eval '(select(documentIndex==1).spec.template.spec.containers[] | select(.name=="manager").env[] | select(.name=="DISABLE_TELEMETRY").value) = "true"' logger.go:42: 20:38:17 | storage/0-deploy-operator | + yq eval '(select(documentIndex==1).spec.template.spec.containers[] | select(.name=="manager").env[] | select(.name=="LOG_LEVEL").value) = "DEBUG"' logger.go:42: 20:38:17 | storage/0-deploy-operator | ++ printf 'select(documentIndex==1).spec.template.spec.containers[0].image="%s"' perconalab/percona-server-mysql-operator:PR-1012-344c570a logger.go:42: 20:38:17 | storage/0-deploy-operator | + kubectl -n ps-operator apply -f - logger.go:42: 20:38:17 | storage/0-deploy-operator | + yq eval 'select(documentIndex==1).spec.template.spec.containers[0].image="perconalab/percona-server-mysql-operator:PR-1012-344c570a"' /mnt/jenkins/workspace/cloud-ps-operator_PR-1012/deploy/cw-operator.yaml logger.go:42: 20:38:18 | storage/0-deploy-operator | configmap/percona-server-mysql-operator-config created logger.go:42: 20:38:19 | storage/0-deploy-operator | deployment.apps/percona-server-mysql-operator created logger.go:42: 20:38:19 | storage/0-deploy-operator | + deploy_non_tls_cluster_secrets logger.go:42: 20:38:19 | storage/0-deploy-operator | + kubectl -n kuttl-test-careful-iguana apply -f /mnt/jenkins/workspace/cloud-ps-operator_PR-1012/e2e-tests/conf/secrets.yaml logger.go:42: 20:38:20 | storage/0-deploy-operator | secret/test-secrets created logger.go:42: 20:38:20 | storage/0-deploy-operator | + deploy_tls_cluster_secrets logger.go:42: 20:38:20 | storage/0-deploy-operator | + kubectl -n kuttl-test-careful-iguana apply -f /mnt/jenkins/workspace/cloud-ps-operator_PR-1012/e2e-tests/conf/ssl-secret.yaml logger.go:42: 20:38:22 | storage/0-deploy-operator | secret/test-ssl created logger.go:42: 20:38:22 | storage/0-deploy-operator | + deploy_client logger.go:42: 20:38:22 | storage/0-deploy-operator | + kubectl -n kuttl-test-careful-iguana apply -f - logger.go:42: 20:38:22 | storage/0-deploy-operator | ++ printf '.spec.containers[0].image="%s"' perconalab/percona-server-mysql-operator:main-psmysql8.0 logger.go:42: 20:38:22 | storage/0-deploy-operator | + yq eval '.spec.containers[0].image="perconalab/percona-server-mysql-operator:main-psmysql8.0"' /mnt/jenkins/workspace/cloud-ps-operator_PR-1012/e2e-tests/conf/client.yaml logger.go:42: 20:38:23 | storage/0-deploy-operator | pod/mysql-client created logger.go:42: 20:38:24 | storage/0-deploy-operator | running command: [sh -c kubectl assert exist-enhanced deployment percona-server-mysql-operator -n ${OPERATOR_NS:-$NAMESPACE} --field-selector status.readyReplicas=1] logger.go:42: 20:38:24 | storage/0-deploy-operator | ASSERT deployment percona-server-mysql-operator matching field criteria 'status.readyReplicas=1' should exist. logger.go:42: 20:38:24 | storage/0-deploy-operator | ASSERT FAIL Resource(s) not found. logger.go:42: 20:38:26 | storage/0-deploy-operator | running command: [sh -c kubectl assert exist-enhanced deployment percona-server-mysql-operator -n ${OPERATOR_NS:-$NAMESPACE} --field-selector status.readyReplicas=1] logger.go:42: 20:38:26 | storage/0-deploy-operator | ASSERT deployment percona-server-mysql-operator matching field criteria 'status.readyReplicas=1' should exist. logger.go:42: 20:38:26 | storage/0-deploy-operator | ASSERT FAIL Resource(s) not found. logger.go:42: 20:38:28 | storage/0-deploy-operator | running command: [sh -c kubectl assert exist-enhanced deployment percona-server-mysql-operator -n ${OPERATOR_NS:-$NAMESPACE} --field-selector status.readyReplicas=1] logger.go:42: 20:38:28 | storage/0-deploy-operator | ASSERT deployment percona-server-mysql-operator matching field criteria 'status.readyReplicas=1' should exist. logger.go:42: 20:38:28 | storage/0-deploy-operator | ASSERT FAIL Resource(s) not found. logger.go:42: 20:38:30 | storage/0-deploy-operator | running command: [sh -c kubectl assert exist-enhanced deployment percona-server-mysql-operator -n ${OPERATOR_NS:-$NAMESPACE} --field-selector status.readyReplicas=1] logger.go:42: 20:38:30 | storage/0-deploy-operator | ASSERT deployment percona-server-mysql-operator matching field criteria 'status.readyReplicas=1' should exist. logger.go:42: 20:38:30 | storage/0-deploy-operator | INFO Found 1 resource(s). logger.go:42: 20:38:30 | storage/0-deploy-operator | NAME NAMESPACE COL0 logger.go:42: 20:38:30 | storage/0-deploy-operator | percona-server-mysql-operator ps-operator 1 logger.go:42: 20:38:30 | storage/0-deploy-operator | ASSERT PASS logger.go:42: 20:38:30 | storage/0-deploy-operator | test step completed 0-deploy-operator logger.go:42: 20:38:30 | storage/1-create-cluster-volume-emptypath | starting test step 1-create-cluster-volume-emptypath logger.go:42: 20:38:30 | storage/1-create-cluster-volume-emptypath | running command: [sh -c set -o errexit set -o xtrace source ../../functions get_cr \ | yq eval '.spec.mysql.volumeSpec = {"emptyDir": {} }' - \ | kubectl -n "${NAMESPACE}" apply -f -] logger.go:42: 20:38:30 | storage/1-create-cluster-volume-emptypath | + source ../../functions logger.go:42: 20:38:30 | storage/1-create-cluster-volume-emptypath | +++ realpath ../../.. logger.go:42: 20:38:30 | storage/1-create-cluster-volume-emptypath | ++ ROOT_REPO=/mnt/jenkins/workspace/cloud-ps-operator_PR-1012 logger.go:42: 20:38:30 | storage/1-create-cluster-volume-emptypath | ++++ pwd logger.go:42: 20:38:30 | storage/1-create-cluster-volume-emptypath | +++ basename /mnt/jenkins/workspace/cloud-ps-operator_PR-1012/e2e-tests/tests/storage logger.go:42: 20:38:30 | storage/1-create-cluster-volume-emptypath | ++ test_name=storage logger.go:42: 20:38:30 | storage/1-create-cluster-volume-emptypath | ++ source /mnt/jenkins/workspace/cloud-ps-operator_PR-1012/e2e-tests/vars.sh logger.go:42: 20:38:30 | storage/1-create-cluster-volume-emptypath | +++ export ROOT_REPO=/mnt/jenkins/workspace/cloud-ps-operator_PR-1012 logger.go:42: 20:38:30 | storage/1-create-cluster-volume-emptypath | +++ ROOT_REPO=/mnt/jenkins/workspace/cloud-ps-operator_PR-1012 logger.go:42: 20:38:30 | storage/1-create-cluster-volume-emptypath | +++ export DEPLOY_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-1012/deploy logger.go:42: 20:38:30 | storage/1-create-cluster-volume-emptypath | +++ DEPLOY_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-1012/deploy logger.go:42: 20:38:30 | storage/1-create-cluster-volume-emptypath | +++ export TESTS_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-1012/e2e-tests logger.go:42: 20:38:30 | storage/1-create-cluster-volume-emptypath | +++ TESTS_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-1012/e2e-tests logger.go:42: 20:38:30 | storage/1-create-cluster-volume-emptypath | +++ export TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-1012/e2e-tests/conf logger.go:42: 20:38:30 | storage/1-create-cluster-volume-emptypath | +++ TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-1012/e2e-tests/conf logger.go:42: 20:38:30 | storage/1-create-cluster-volume-emptypath | +++ export TEMP_DIR=/tmp/kuttl/ps/storage logger.go:42: 20:38:30 | storage/1-create-cluster-volume-emptypath | +++ TEMP_DIR=/tmp/kuttl/ps/storage logger.go:42: 20:38:30 | storage/1-create-cluster-volume-emptypath | ++++ git rev-parse --abbrev-ref HEAD logger.go:42: 20:38:30 | storage/1-create-cluster-volume-emptypath | +++ export GIT_BRANCH=PR-1012 logger.go:42: 20:38:30 | storage/1-create-cluster-volume-emptypath | +++ GIT_BRANCH=PR-1012 logger.go:42: 20:38:30 | storage/1-create-cluster-volume-emptypath | +++ export VERSION=PR-1012-344c570a logger.go:42: 20:38:30 | storage/1-create-cluster-volume-emptypath | +++ VERSION=PR-1012-344c570a logger.go:42: 20:38:30 | storage/1-create-cluster-volume-emptypath | +++ export IMAGE=perconalab/percona-server-mysql-operator:PR-1012-344c570a logger.go:42: 20:38:30 | storage/1-create-cluster-volume-emptypath | +++ IMAGE=perconalab/percona-server-mysql-operator:PR-1012-344c570a logger.go:42: 20:38:30 | storage/1-create-cluster-volume-emptypath | +++ export MYSQL_VERSION=8.0 logger.go:42: 20:38:30 | storage/1-create-cluster-volume-emptypath | +++ MYSQL_VERSION=8.0 logger.go:42: 20:38:30 | storage/1-create-cluster-volume-emptypath | +++ export IMAGE_MYSQL=perconalab/percona-server-mysql-operator:main-psmysql8.0 logger.go:42: 20:38:30 | storage/1-create-cluster-volume-emptypath | +++ IMAGE_MYSQL=perconalab/percona-server-mysql-operator:main-psmysql8.0 logger.go:42: 20:38:30 | storage/1-create-cluster-volume-emptypath | +++ export IMAGE_BACKUP=perconalab/percona-server-mysql-operator:main-backup8.0 logger.go:42: 20:38:30 | storage/1-create-cluster-volume-emptypath | +++ IMAGE_BACKUP=perconalab/percona-server-mysql-operator:main-backup8.0 logger.go:42: 20:38:30 | storage/1-create-cluster-volume-emptypath | +++ export IMAGE_ORCHESTRATOR=perconalab/percona-server-mysql-operator:main-orchestrator logger.go:42: 20:38:30 | storage/1-create-cluster-volume-emptypath | +++ IMAGE_ORCHESTRATOR=perconalab/percona-server-mysql-operator:main-orchestrator logger.go:42: 20:38:30 | storage/1-create-cluster-volume-emptypath | +++ export IMAGE_ROUTER=perconalab/percona-server-mysql-operator:main-router8.0 logger.go:42: 20:38:30 | storage/1-create-cluster-volume-emptypath | +++ IMAGE_ROUTER=perconalab/percona-server-mysql-operator:main-router8.0 logger.go:42: 20:38:30 | storage/1-create-cluster-volume-emptypath | +++ export IMAGE_TOOLKIT=perconalab/percona-server-mysql-operator:main-toolkit logger.go:42: 20:38:30 | storage/1-create-cluster-volume-emptypath | +++ IMAGE_TOOLKIT=perconalab/percona-server-mysql-operator:main-toolkit logger.go:42: 20:38:30 | storage/1-create-cluster-volume-emptypath | +++ export IMAGE_HAPROXY=perconalab/percona-server-mysql-operator:main-haproxy logger.go:42: 20:38:30 | storage/1-create-cluster-volume-emptypath | +++ IMAGE_HAPROXY=perconalab/percona-server-mysql-operator:main-haproxy logger.go:42: 20:38:30 | storage/1-create-cluster-volume-emptypath | +++ export PMM_SERVER_VERSION=1.4.3 logger.go:42: 20:38:30 | storage/1-create-cluster-volume-emptypath | +++ PMM_SERVER_VERSION=1.4.3 logger.go:42: 20:38:30 | storage/1-create-cluster-volume-emptypath | +++ export IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 20:38:30 | storage/1-create-cluster-volume-emptypath | +++ IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 20:38:30 | storage/1-create-cluster-volume-emptypath | +++ export IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 20:38:30 | storage/1-create-cluster-volume-emptypath | +++ IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 20:38:30 | storage/1-create-cluster-volume-emptypath | +++ export CERT_MANAGER_VER=1.18.2 logger.go:42: 20:38:30 | storage/1-create-cluster-volume-emptypath | +++ CERT_MANAGER_VER=1.18.2 logger.go:42: 20:38:30 | storage/1-create-cluster-volume-emptypath | +++ export MINIO_VER=5.4.0 logger.go:42: 20:38:30 | storage/1-create-cluster-volume-emptypath | +++ MINIO_VER=5.4.0 logger.go:42: 20:38:30 | storage/1-create-cluster-volume-emptypath | +++ export CHAOS_MESH_VER=2.7.2 logger.go:42: 20:38:30 | storage/1-create-cluster-volume-emptypath | +++ CHAOS_MESH_VER=2.7.2 logger.go:42: 20:38:30 | storage/1-create-cluster-volume-emptypath | +++ export VAULT_VER=0.16.1 logger.go:42: 20:38:30 | storage/1-create-cluster-volume-emptypath | +++ VAULT_VER=0.16.1 logger.go:42: 20:38:30 | storage/1-create-cluster-volume-emptypath | ++++ which gdate logger.go:42: 20:38:30 | storage/1-create-cluster-volume-emptypath | which: no gdate in (/mnt/jenkins/workspace/cloud-ps-operator_PR-1012/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 20:38:30 | storage/1-create-cluster-volume-emptypath | ++++ which date logger.go:42: 20:38:30 | storage/1-create-cluster-volume-emptypath | +++ date=/usr/sbin/date logger.go:42: 20:38:30 | storage/1-create-cluster-volume-emptypath | +++ oc get projects logger.go:42: 20:38:30 | storage/1-create-cluster-volume-emptypath | +++ : logger.go:42: 20:38:30 | storage/1-create-cluster-volume-emptypath | +++ kubectl get nodes logger.go:42: 20:38:30 | storage/1-create-cluster-volume-emptypath | +++ grep '^minikube' logger.go:42: 20:38:31 | storage/1-create-cluster-volume-emptypath | ++ oc get projects logger.go:42: 20:38:31 | storage/1-create-cluster-volume-emptypath | +++ kubectl version -o json logger.go:42: 20:38:31 | storage/1-create-cluster-volume-emptypath | +++ jq -r .serverVersion.gitVersion logger.go:42: 20:38:31 | storage/1-create-cluster-volume-emptypath | +++ grep '\-eks\-' logger.go:42: 20:38:31 | storage/1-create-cluster-volume-emptypath | grep: warning: stray \ before - logger.go:42: 20:38:31 | storage/1-create-cluster-volume-emptypath | Warning: version difference between client (1.34) and server (1.30) exceeds the supported minor version skew of +/-1 logger.go:42: 20:38:31 | storage/1-create-cluster-volume-emptypath | ++ '[' ']' logger.go:42: 20:38:31 | storage/1-create-cluster-volume-emptypath | ++ EKS=0 logger.go:42: 20:38:31 | storage/1-create-cluster-volume-emptypath | + get_cr logger.go:42: 20:38:31 | storage/1-create-cluster-volume-emptypath | + local name_suffix= logger.go:42: 20:38:31 | storage/1-create-cluster-volume-emptypath | + yq eval '.spec.mysql.volumeSpec = {"emptyDir": {} }' - logger.go:42: 20:38:31 | storage/1-create-cluster-volume-emptypath | + kubectl -n kuttl-test-careful-iguana apply -f - logger.go:42: 20:38:31 | storage/1-create-cluster-volume-emptypath | + yq eval '.spec.secretsName="test-secrets"' - logger.go:42: 20:38:31 | storage/1-create-cluster-volume-emptypath | ++ printf '.metadata.name="%s"' storage logger.go:42: 20:38:31 | storage/1-create-cluster-volume-emptypath | ++ printf '.spec.initContainer.image="%s"' perconalab/percona-server-mysql-operator:PR-1012-344c570a logger.go:42: 20:38:31 | storage/1-create-cluster-volume-emptypath | + yq eval '.spec.sslSecretName="test-ssl"' - logger.go:42: 20:38:31 | storage/1-create-cluster-volume-emptypath | + yq eval '.spec.upgradeOptions.apply="disabled"' - logger.go:42: 20:38:31 | storage/1-create-cluster-volume-emptypath | + yq eval '.metadata.name="storage"' /mnt/jenkins/workspace/cloud-ps-operator_PR-1012/deploy/cr.yaml logger.go:42: 20:38:31 | storage/1-create-cluster-volume-emptypath | + yq eval '.spec.initContainer.image="perconalab/percona-server-mysql-operator:PR-1012-344c570a"' - logger.go:42: 20:38:31 | storage/1-create-cluster-volume-emptypath | + yq eval '.spec.mysql.clusterType="async"' - logger.go:42: 20:38:31 | storage/1-create-cluster-volume-emptypath | + yq eval .spec.mysql.gracePeriod=30 - logger.go:42: 20:38:31 | storage/1-create-cluster-volume-emptypath | + yq eval .spec.orchestrator.enabled=true - logger.go:42: 20:38:31 | storage/1-create-cluster-volume-emptypath | ++ printf '.spec.backup.image="%s"' perconalab/percona-server-mysql-operator:main-backup8.0 logger.go:42: 20:38:31 | storage/1-create-cluster-volume-emptypath | ++ printf '.spec.orchestrator.image="%s"' perconalab/percona-server-mysql-operator:main-orchestrator logger.go:42: 20:38:31 | storage/1-create-cluster-volume-emptypath | ++ printf '.spec.toolkit.image="%s"' perconalab/percona-server-mysql-operator:main-toolkit logger.go:42: 20:38:31 | storage/1-create-cluster-volume-emptypath | + yq eval '.spec.backup.image="perconalab/percona-server-mysql-operator:main-backup8.0"' - logger.go:42: 20:38:31 | storage/1-create-cluster-volume-emptypath | ++ printf '.spec.proxy.router.image="%s"' perconalab/percona-server-mysql-operator:main-router8.0 logger.go:42: 20:38:31 | storage/1-create-cluster-volume-emptypath | ++ printf '.spec.mysql.image="%s"' perconalab/percona-server-mysql-operator:main-psmysql8.0 logger.go:42: 20:38:31 | storage/1-create-cluster-volume-emptypath | + yq eval '.spec.proxy.router.image="perconalab/percona-server-mysql-operator:main-router8.0"' - logger.go:42: 20:38:31 | storage/1-create-cluster-volume-emptypath | + yq eval '.spec.mysql.image="perconalab/percona-server-mysql-operator:main-psmysql8.0"' - logger.go:42: 20:38:31 | storage/1-create-cluster-volume-emptypath | ++ printf '.spec.pmm.image="%s"' perconalab/pmm-client:3-dev-latest logger.go:42: 20:38:31 | storage/1-create-cluster-volume-emptypath | + yq eval '.spec.orchestrator.image="perconalab/percona-server-mysql-operator:main-orchestrator"' - logger.go:42: 20:38:31 | storage/1-create-cluster-volume-emptypath | + '[' -n '' ']' logger.go:42: 20:38:31 | storage/1-create-cluster-volume-emptypath | + yq eval - logger.go:42: 20:38:31 | storage/1-create-cluster-volume-emptypath | ++ printf '.spec.proxy.haproxy.image="%s"' perconalab/percona-server-mysql-operator:main-haproxy logger.go:42: 20:38:31 | storage/1-create-cluster-volume-emptypath | + yq eval '.spec.toolkit.image="perconalab/percona-server-mysql-operator:main-toolkit"' - logger.go:42: 20:38:31 | storage/1-create-cluster-volume-emptypath | + yq eval '.spec.proxy.haproxy.image="perconalab/percona-server-mysql-operator:main-haproxy"' - logger.go:42: 20:38:31 | storage/1-create-cluster-volume-emptypath | + yq eval '.spec.pmm.image="perconalab/pmm-client:3-dev-latest"' - logger.go:42: 20:38:33 | storage/1-create-cluster-volume-emptypath | perconaservermysql.ps.percona.com/storage created logger.go:42: 20:45:29 | storage/1-create-cluster-volume-emptypath | test step completed 1-create-cluster-volume-emptypath logger.go:42: 20:45:29 | storage/2-write-data | starting test step 2-write-data logger.go:42: 20:45:29 | storage/2-write-data | running command: [sh -c set -o errexit set -o xtrace source ../../functions sleep 5 run_mysql \ "CREATE DATABASE IF NOT EXISTS myDB; CREATE TABLE IF NOT EXISTS myDB.myTable (id int PRIMARY KEY)" \ "-h $(get_haproxy_svc $(get_cluster_name)) -uroot -proot_password" run_mysql \ "INSERT myDB.myTable (id) VALUES (100500)" \ "-h $(get_haproxy_svc $(get_cluster_name)) -uroot -proot_password" mysql_data=$(kubectl exec -n "${NAMESPACE}" "$test_name"-mysql-0 -- \ find /var/lib/mysql/myDB -type f -size +0) if [[ -z $mysql_data ]]; then echo "No data files found — test failed" exit 1 fi] logger.go:42: 20:45:29 | storage/2-write-data | + source ../../functions logger.go:42: 20:45:29 | storage/2-write-data | +++ realpath ../../.. logger.go:42: 20:45:29 | storage/2-write-data | ++ ROOT_REPO=/mnt/jenkins/workspace/cloud-ps-operator_PR-1012 logger.go:42: 20:45:29 | storage/2-write-data | ++++ pwd logger.go:42: 20:45:29 | storage/2-write-data | +++ basename /mnt/jenkins/workspace/cloud-ps-operator_PR-1012/e2e-tests/tests/storage logger.go:42: 20:45:29 | storage/2-write-data | ++ test_name=storage logger.go:42: 20:45:29 | storage/2-write-data | ++ source /mnt/jenkins/workspace/cloud-ps-operator_PR-1012/e2e-tests/vars.sh logger.go:42: 20:45:29 | storage/2-write-data | +++ export ROOT_REPO=/mnt/jenkins/workspace/cloud-ps-operator_PR-1012 logger.go:42: 20:45:29 | storage/2-write-data | +++ ROOT_REPO=/mnt/jenkins/workspace/cloud-ps-operator_PR-1012 logger.go:42: 20:45:29 | storage/2-write-data | +++ export DEPLOY_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-1012/deploy logger.go:42: 20:45:29 | storage/2-write-data | +++ DEPLOY_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-1012/deploy logger.go:42: 20:45:29 | storage/2-write-data | +++ export TESTS_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-1012/e2e-tests logger.go:42: 20:45:29 | storage/2-write-data | +++ TESTS_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-1012/e2e-tests logger.go:42: 20:45:29 | storage/2-write-data | +++ export TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-1012/e2e-tests/conf logger.go:42: 20:45:29 | storage/2-write-data | +++ TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-1012/e2e-tests/conf logger.go:42: 20:45:29 | storage/2-write-data | +++ export TEMP_DIR=/tmp/kuttl/ps/storage logger.go:42: 20:45:29 | storage/2-write-data | +++ TEMP_DIR=/tmp/kuttl/ps/storage logger.go:42: 20:45:29 | storage/2-write-data | ++++ git rev-parse --abbrev-ref HEAD logger.go:42: 20:45:29 | storage/2-write-data | +++ export GIT_BRANCH=PR-1012 logger.go:42: 20:45:29 | storage/2-write-data | +++ GIT_BRANCH=PR-1012 logger.go:42: 20:45:29 | storage/2-write-data | +++ export VERSION=PR-1012-344c570a logger.go:42: 20:45:29 | storage/2-write-data | +++ VERSION=PR-1012-344c570a logger.go:42: 20:45:29 | storage/2-write-data | +++ export IMAGE=perconalab/percona-server-mysql-operator:PR-1012-344c570a logger.go:42: 20:45:29 | storage/2-write-data | +++ IMAGE=perconalab/percona-server-mysql-operator:PR-1012-344c570a logger.go:42: 20:45:29 | storage/2-write-data | +++ export MYSQL_VERSION=8.0 logger.go:42: 20:45:29 | storage/2-write-data | +++ MYSQL_VERSION=8.0 logger.go:42: 20:45:29 | storage/2-write-data | +++ export IMAGE_MYSQL=perconalab/percona-server-mysql-operator:main-psmysql8.0 logger.go:42: 20:45:29 | storage/2-write-data | +++ IMAGE_MYSQL=perconalab/percona-server-mysql-operator:main-psmysql8.0 logger.go:42: 20:45:29 | storage/2-write-data | +++ export IMAGE_BACKUP=perconalab/percona-server-mysql-operator:main-backup8.0 logger.go:42: 20:45:29 | storage/2-write-data | +++ IMAGE_BACKUP=perconalab/percona-server-mysql-operator:main-backup8.0 logger.go:42: 20:45:29 | storage/2-write-data | +++ export IMAGE_ORCHESTRATOR=perconalab/percona-server-mysql-operator:main-orchestrator logger.go:42: 20:45:29 | storage/2-write-data | +++ IMAGE_ORCHESTRATOR=perconalab/percona-server-mysql-operator:main-orchestrator logger.go:42: 20:45:29 | storage/2-write-data | +++ export IMAGE_ROUTER=perconalab/percona-server-mysql-operator:main-router8.0 logger.go:42: 20:45:29 | storage/2-write-data | +++ IMAGE_ROUTER=perconalab/percona-server-mysql-operator:main-router8.0 logger.go:42: 20:45:29 | storage/2-write-data | +++ export IMAGE_TOOLKIT=perconalab/percona-server-mysql-operator:main-toolkit logger.go:42: 20:45:29 | storage/2-write-data | +++ IMAGE_TOOLKIT=perconalab/percona-server-mysql-operator:main-toolkit logger.go:42: 20:45:29 | storage/2-write-data | +++ export IMAGE_HAPROXY=perconalab/percona-server-mysql-operator:main-haproxy logger.go:42: 20:45:29 | storage/2-write-data | +++ IMAGE_HAPROXY=perconalab/percona-server-mysql-operator:main-haproxy logger.go:42: 20:45:29 | storage/2-write-data | +++ export PMM_SERVER_VERSION=1.4.3 logger.go:42: 20:45:29 | storage/2-write-data | +++ PMM_SERVER_VERSION=1.4.3 logger.go:42: 20:45:29 | storage/2-write-data | +++ export IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 20:45:29 | storage/2-write-data | +++ IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 20:45:29 | storage/2-write-data | +++ export IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 20:45:29 | storage/2-write-data | +++ IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 20:45:29 | storage/2-write-data | +++ export CERT_MANAGER_VER=1.18.2 logger.go:42: 20:45:29 | storage/2-write-data | +++ CERT_MANAGER_VER=1.18.2 logger.go:42: 20:45:29 | storage/2-write-data | +++ export MINIO_VER=5.4.0 logger.go:42: 20:45:29 | storage/2-write-data | +++ MINIO_VER=5.4.0 logger.go:42: 20:45:29 | storage/2-write-data | +++ export CHAOS_MESH_VER=2.7.2 logger.go:42: 20:45:29 | storage/2-write-data | +++ CHAOS_MESH_VER=2.7.2 logger.go:42: 20:45:29 | storage/2-write-data | +++ export VAULT_VER=0.16.1 logger.go:42: 20:45:29 | storage/2-write-data | +++ VAULT_VER=0.16.1 logger.go:42: 20:45:29 | storage/2-write-data | ++++ which gdate logger.go:42: 20:45:29 | storage/2-write-data | which: no gdate in (/mnt/jenkins/workspace/cloud-ps-operator_PR-1012/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 20:45:29 | storage/2-write-data | ++++ which date logger.go:42: 20:45:29 | storage/2-write-data | +++ date=/usr/sbin/date logger.go:42: 20:45:29 | storage/2-write-data | +++ oc get projects logger.go:42: 20:45:29 | storage/2-write-data | +++ : logger.go:42: 20:45:29 | storage/2-write-data | +++ kubectl get nodes logger.go:42: 20:45:29 | storage/2-write-data | +++ grep '^minikube' logger.go:42: 20:45:30 | storage/2-write-data | ++ oc get projects logger.go:42: 20:45:30 | storage/2-write-data | +++ kubectl version -o json logger.go:42: 20:45:30 | storage/2-write-data | +++ jq -r .serverVersion.gitVersion logger.go:42: 20:45:30 | storage/2-write-data | +++ grep '\-eks\-' logger.go:42: 20:45:30 | storage/2-write-data | grep: warning: stray \ before - logger.go:42: 20:45:31 | storage/2-write-data | Warning: version difference between client (1.34) and server (1.30) exceeds the supported minor version skew of +/-1 logger.go:42: 20:45:31 | storage/2-write-data | ++ '[' ']' logger.go:42: 20:45:31 | storage/2-write-data | ++ EKS=0 logger.go:42: 20:45:31 | storage/2-write-data | + sleep 5 logger.go:42: 20:45:36 | storage/2-write-data | +++ get_cluster_name logger.go:42: 20:45:36 | storage/2-write-data | +++ kubectl -n kuttl-test-careful-iguana get ps -o 'jsonpath={.items[0].metadata.name}' logger.go:42: 20:45:36 | storage/2-write-data | ++ get_haproxy_svc storage logger.go:42: 20:45:36 | storage/2-write-data | ++ local cluster=storage logger.go:42: 20:45:36 | storage/2-write-data | ++ echo storage-haproxy logger.go:42: 20:45:36 | storage/2-write-data | + run_mysql 'CREATE DATABASE IF NOT EXISTS myDB; CREATE TABLE IF NOT EXISTS myDB.myTable (id int PRIMARY KEY)' '-h storage-haproxy -uroot -proot_password' logger.go:42: 20:45:36 | storage/2-write-data | + local 'command=CREATE DATABASE IF NOT EXISTS myDB; CREATE TABLE IF NOT EXISTS myDB.myTable (id int PRIMARY KEY)' logger.go:42: 20:45:36 | storage/2-write-data | + local 'uri=-h storage-haproxy -uroot -proot_password' logger.go:42: 20:45:36 | storage/2-write-data | + local pod= logger.go:42: 20:45:36 | storage/2-write-data | ++ get_client_pod logger.go:42: 20:45:36 | storage/2-write-data | ++ kubectl -n kuttl-test-careful-iguana get pods --selector=name=mysql-client -o 'jsonpath={.items[].metadata.name}' logger.go:42: 20:45:37 | storage/2-write-data | + client_pod=mysql-client logger.go:42: 20:45:37 | storage/2-write-data | + wait_pod mysql-client logger.go:42: 20:45:37 | storage/2-write-data | + local pod=mysql-client logger.go:42: 20:45:37 | storage/2-write-data | + set +o xtrace logger.go:42: 20:45:37 | storage/2-write-data | mysql-clienttrue logger.go:42: 20:45:37 | storage/2-write-data | + kubectl -n kuttl-test-careful-iguana exec mysql-client -- bash -c 'printf '\''%s\n'\'' "CREATE DATABASE IF NOT EXISTS myDB; CREATE TABLE IF NOT EXISTS myDB.myTable (id int PRIMARY KEY)" | mysql -sN -h storage-haproxy -uroot -proot_password' logger.go:42: 20:45:37 | storage/2-write-data | + sed -e 's/mysql: //' logger.go:42: 20:45:37 | storage/2-write-data | + grep -v 'Using a password on the command line interface can be insecure.' logger.go:42: 20:45:38 | storage/2-write-data | + : logger.go:42: 20:45:38 | storage/2-write-data | +++ get_cluster_name logger.go:42: 20:45:38 | storage/2-write-data | +++ kubectl -n kuttl-test-careful-iguana get ps -o 'jsonpath={.items[0].metadata.name}' logger.go:42: 20:45:39 | storage/2-write-data | ++ get_haproxy_svc storage logger.go:42: 20:45:39 | storage/2-write-data | ++ local cluster=storage logger.go:42: 20:45:39 | storage/2-write-data | ++ echo storage-haproxy logger.go:42: 20:45:39 | storage/2-write-data | + run_mysql 'INSERT myDB.myTable (id) VALUES (100500)' '-h storage-haproxy -uroot -proot_password' logger.go:42: 20:45:39 | storage/2-write-data | + local 'command=INSERT myDB.myTable (id) VALUES (100500)' logger.go:42: 20:45:39 | storage/2-write-data | + local 'uri=-h storage-haproxy -uroot -proot_password' logger.go:42: 20:45:39 | storage/2-write-data | + local pod= logger.go:42: 20:45:39 | storage/2-write-data | ++ get_client_pod logger.go:42: 20:45:39 | storage/2-write-data | ++ kubectl -n kuttl-test-careful-iguana get pods --selector=name=mysql-client -o 'jsonpath={.items[].metadata.name}' logger.go:42: 20:45:39 | storage/2-write-data | + client_pod=mysql-client logger.go:42: 20:45:39 | storage/2-write-data | + wait_pod mysql-client logger.go:42: 20:45:39 | storage/2-write-data | + local pod=mysql-client logger.go:42: 20:45:39 | storage/2-write-data | + set +o xtrace logger.go:42: 20:45:40 | storage/2-write-data | mysql-clienttrue logger.go:42: 20:45:40 | storage/2-write-data | + kubectl -n kuttl-test-careful-iguana exec mysql-client -- bash -c 'printf '\''%s\n'\'' "INSERT myDB.myTable (id) VALUES (100500)" | mysql -sN -h storage-haproxy -uroot -proot_password' logger.go:42: 20:45:40 | storage/2-write-data | + sed -e 's/mysql: //' logger.go:42: 20:45:40 | storage/2-write-data | + grep -v 'Using a password on the command line interface can be insecure.' logger.go:42: 20:45:41 | storage/2-write-data | + : logger.go:42: 20:45:41 | storage/2-write-data | ++ kubectl exec -n kuttl-test-careful-iguana storage-mysql-0 -- find /var/lib/mysql/myDB -type f -size +0 logger.go:42: 20:45:41 | storage/2-write-data | Defaulted container "mysql" out of: mysql, xtrabackup, pt-heartbeat, mysql-init (init) logger.go:42: 20:45:42 | storage/2-write-data | + mysql_data=/var/lib/mysql/myDB/myTable.ibd logger.go:42: 20:45:42 | storage/2-write-data | + [[ -z /var/lib/mysql/myDB/myTable.ibd ]] logger.go:42: 20:45:42 | storage/2-write-data | test step completed 2-write-data logger.go:42: 20:45:42 | storage/98-drop-finalizer | starting test step 98-drop-finalizer logger.go:42: 20:45:43 | storage/98-drop-finalizer | PerconaServerMySQL:kuttl-test-careful-iguana/storage updated logger.go:42: 20:45:43 | storage/98-drop-finalizer | test step completed 98-drop-finalizer logger.go:42: 20:45:43 | storage/99-remove-cluster-gracefully | starting test step 99-remove-cluster-gracefully logger.go:42: 20:45:43 | storage/99-remove-cluster-gracefully | running command: [sh -c set -o errexit set -o xtrace source ../../functions destroy_operator] logger.go:42: 20:45:43 | storage/99-remove-cluster-gracefully | + source ../../functions logger.go:42: 20:45:43 | storage/99-remove-cluster-gracefully | +++ realpath ../../.. logger.go:42: 20:45:43 | storage/99-remove-cluster-gracefully | ++ ROOT_REPO=/mnt/jenkins/workspace/cloud-ps-operator_PR-1012 logger.go:42: 20:45:43 | storage/99-remove-cluster-gracefully | ++++ pwd logger.go:42: 20:45:43 | storage/99-remove-cluster-gracefully | +++ basename /mnt/jenkins/workspace/cloud-ps-operator_PR-1012/e2e-tests/tests/storage logger.go:42: 20:45:43 | storage/99-remove-cluster-gracefully | ++ test_name=storage logger.go:42: 20:45:43 | storage/99-remove-cluster-gracefully | ++ source /mnt/jenkins/workspace/cloud-ps-operator_PR-1012/e2e-tests/vars.sh logger.go:42: 20:45:43 | storage/99-remove-cluster-gracefully | +++ export ROOT_REPO=/mnt/jenkins/workspace/cloud-ps-operator_PR-1012 logger.go:42: 20:45:43 | storage/99-remove-cluster-gracefully | +++ ROOT_REPO=/mnt/jenkins/workspace/cloud-ps-operator_PR-1012 logger.go:42: 20:45:43 | storage/99-remove-cluster-gracefully | +++ export DEPLOY_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-1012/deploy logger.go:42: 20:45:43 | storage/99-remove-cluster-gracefully | +++ DEPLOY_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-1012/deploy logger.go:42: 20:45:43 | storage/99-remove-cluster-gracefully | +++ export TESTS_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-1012/e2e-tests logger.go:42: 20:45:43 | storage/99-remove-cluster-gracefully | +++ TESTS_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-1012/e2e-tests logger.go:42: 20:45:43 | storage/99-remove-cluster-gracefully | +++ export TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-1012/e2e-tests/conf logger.go:42: 20:45:43 | storage/99-remove-cluster-gracefully | +++ TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-1012/e2e-tests/conf logger.go:42: 20:45:43 | storage/99-remove-cluster-gracefully | +++ export TEMP_DIR=/tmp/kuttl/ps/storage logger.go:42: 20:45:43 | storage/99-remove-cluster-gracefully | +++ TEMP_DIR=/tmp/kuttl/ps/storage logger.go:42: 20:45:43 | storage/99-remove-cluster-gracefully | ++++ git rev-parse --abbrev-ref HEAD logger.go:42: 20:45:43 | storage/99-remove-cluster-gracefully | +++ export GIT_BRANCH=PR-1012 logger.go:42: 20:45:43 | storage/99-remove-cluster-gracefully | +++ GIT_BRANCH=PR-1012 logger.go:42: 20:45:43 | storage/99-remove-cluster-gracefully | +++ export VERSION=PR-1012-344c570a logger.go:42: 20:45:43 | storage/99-remove-cluster-gracefully | +++ VERSION=PR-1012-344c570a logger.go:42: 20:45:43 | storage/99-remove-cluster-gracefully | +++ export IMAGE=perconalab/percona-server-mysql-operator:PR-1012-344c570a logger.go:42: 20:45:43 | storage/99-remove-cluster-gracefully | +++ IMAGE=perconalab/percona-server-mysql-operator:PR-1012-344c570a logger.go:42: 20:45:43 | storage/99-remove-cluster-gracefully | +++ export MYSQL_VERSION=8.0 logger.go:42: 20:45:43 | storage/99-remove-cluster-gracefully | +++ MYSQL_VERSION=8.0 logger.go:42: 20:45:43 | storage/99-remove-cluster-gracefully | +++ export IMAGE_MYSQL=perconalab/percona-server-mysql-operator:main-psmysql8.0 logger.go:42: 20:45:43 | storage/99-remove-cluster-gracefully | +++ IMAGE_MYSQL=perconalab/percona-server-mysql-operator:main-psmysql8.0 logger.go:42: 20:45:43 | storage/99-remove-cluster-gracefully | +++ export IMAGE_BACKUP=perconalab/percona-server-mysql-operator:main-backup8.0 logger.go:42: 20:45:43 | storage/99-remove-cluster-gracefully | +++ IMAGE_BACKUP=perconalab/percona-server-mysql-operator:main-backup8.0 logger.go:42: 20:45:43 | storage/99-remove-cluster-gracefully | +++ export IMAGE_ORCHESTRATOR=perconalab/percona-server-mysql-operator:main-orchestrator logger.go:42: 20:45:43 | storage/99-remove-cluster-gracefully | +++ IMAGE_ORCHESTRATOR=perconalab/percona-server-mysql-operator:main-orchestrator logger.go:42: 20:45:43 | storage/99-remove-cluster-gracefully | +++ export IMAGE_ROUTER=perconalab/percona-server-mysql-operator:main-router8.0 logger.go:42: 20:45:43 | storage/99-remove-cluster-gracefully | +++ IMAGE_ROUTER=perconalab/percona-server-mysql-operator:main-router8.0 logger.go:42: 20:45:43 | storage/99-remove-cluster-gracefully | +++ export IMAGE_TOOLKIT=perconalab/percona-server-mysql-operator:main-toolkit logger.go:42: 20:45:43 | storage/99-remove-cluster-gracefully | +++ IMAGE_TOOLKIT=perconalab/percona-server-mysql-operator:main-toolkit logger.go:42: 20:45:43 | storage/99-remove-cluster-gracefully | +++ export IMAGE_HAPROXY=perconalab/percona-server-mysql-operator:main-haproxy logger.go:42: 20:45:43 | storage/99-remove-cluster-gracefully | +++ IMAGE_HAPROXY=perconalab/percona-server-mysql-operator:main-haproxy logger.go:42: 20:45:43 | storage/99-remove-cluster-gracefully | +++ export PMM_SERVER_VERSION=1.4.3 logger.go:42: 20:45:43 | storage/99-remove-cluster-gracefully | +++ PMM_SERVER_VERSION=1.4.3 logger.go:42: 20:45:43 | storage/99-remove-cluster-gracefully | +++ export IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 20:45:43 | storage/99-remove-cluster-gracefully | +++ IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 20:45:43 | storage/99-remove-cluster-gracefully | +++ export IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 20:45:43 | storage/99-remove-cluster-gracefully | +++ IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 20:45:43 | storage/99-remove-cluster-gracefully | +++ export CERT_MANAGER_VER=1.18.2 logger.go:42: 20:45:43 | storage/99-remove-cluster-gracefully | +++ CERT_MANAGER_VER=1.18.2 logger.go:42: 20:45:43 | storage/99-remove-cluster-gracefully | +++ export MINIO_VER=5.4.0 logger.go:42: 20:45:43 | storage/99-remove-cluster-gracefully | +++ MINIO_VER=5.4.0 logger.go:42: 20:45:43 | storage/99-remove-cluster-gracefully | +++ export CHAOS_MESH_VER=2.7.2 logger.go:42: 20:45:43 | storage/99-remove-cluster-gracefully | +++ CHAOS_MESH_VER=2.7.2 logger.go:42: 20:45:43 | storage/99-remove-cluster-gracefully | +++ export VAULT_VER=0.16.1 logger.go:42: 20:45:43 | storage/99-remove-cluster-gracefully | +++ VAULT_VER=0.16.1 logger.go:42: 20:45:43 | storage/99-remove-cluster-gracefully | ++++ which gdate logger.go:42: 20:45:43 | storage/99-remove-cluster-gracefully | which: no gdate in (/mnt/jenkins/workspace/cloud-ps-operator_PR-1012/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 20:45:43 | storage/99-remove-cluster-gracefully | ++++ which date logger.go:42: 20:45:43 | storage/99-remove-cluster-gracefully | +++ date=/usr/sbin/date logger.go:42: 20:45:43 | storage/99-remove-cluster-gracefully | +++ oc get projects logger.go:42: 20:45:43 | storage/99-remove-cluster-gracefully | +++ : logger.go:42: 20:45:43 | storage/99-remove-cluster-gracefully | +++ kubectl get nodes logger.go:42: 20:45:43 | storage/99-remove-cluster-gracefully | +++ grep '^minikube' logger.go:42: 20:45:44 | storage/99-remove-cluster-gracefully | ++ oc get projects logger.go:42: 20:45:44 | storage/99-remove-cluster-gracefully | +++ kubectl version -o json logger.go:42: 20:45:44 | storage/99-remove-cluster-gracefully | +++ jq -r .serverVersion.gitVersion logger.go:42: 20:45:44 | storage/99-remove-cluster-gracefully | +++ grep '\-eks\-' logger.go:42: 20:45:44 | storage/99-remove-cluster-gracefully | grep: warning: stray \ before - logger.go:42: 20:45:44 | storage/99-remove-cluster-gracefully | Warning: version difference between client (1.34) and server (1.30) exceeds the supported minor version skew of +/-1 logger.go:42: 20:45:44 | storage/99-remove-cluster-gracefully | ++ '[' ']' logger.go:42: 20:45:44 | storage/99-remove-cluster-gracefully | ++ EKS=0 logger.go:42: 20:45:44 | storage/99-remove-cluster-gracefully | + destroy_operator logger.go:42: 20:45:44 | storage/99-remove-cluster-gracefully | + kubectl -n ps-operator delete deployment percona-server-mysql-operator --force --grace-period=0 logger.go:42: 20:45:45 | storage/99-remove-cluster-gracefully | Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. logger.go:42: 20:45:45 | storage/99-remove-cluster-gracefully | deployment.apps "percona-server-mysql-operator" force deleted from ps-operator namespace logger.go:42: 20:45:45 | storage/99-remove-cluster-gracefully | + [[ -n ps-operator ]] logger.go:42: 20:45:45 | storage/99-remove-cluster-gracefully | + kubectl delete namespace ps-operator --force --grace-period=0 logger.go:42: 20:45:45 | storage/99-remove-cluster-gracefully | Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. logger.go:42: 20:45:46 | storage/99-remove-cluster-gracefully | namespace "ps-operator" force deleted logger.go:42: 20:45:53 | storage/99-remove-cluster-gracefully | test step completed 99-remove-cluster-gracefully logger.go:42: 20:45:53 | storage | storage events from ns kuttl-test-careful-iguana: logger.go:42: 20:45:53 | storage | 2025-09-02 20:38:23 +0000 UTC Normal Pod mysql-client Binding Scheduled Successfully assigned kuttl-test-careful-iguana/mysql-client to gke-jen-ps-1012-344c570a-default-pool-7b175ef2-xffk default-scheduler logger.go:42: 20:45:53 | storage | 2025-09-02 20:38:24 +0000 UTC Normal Pod mysql-client.spec.containers{mysql-client} Pulled Container image "perconalab/percona-server-mysql-operator:main-psmysql8.0" already present on machine kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:38:24 +0000 UTC Normal Pod mysql-client.spec.containers{mysql-client} Created Created container: mysql-client kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:38:24 +0000 UTC Normal Pod mysql-client.spec.containers{mysql-client} Started Started container mysql-client kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:38:34 +0000 UTC Normal Pod storage-mysql-0 Binding Scheduled Successfully assigned kuttl-test-careful-iguana/storage-mysql-0 to gke-jen-ps-1012-344c570a-default-pool-7b175ef2-jv2t default-scheduler logger.go:42: 20:45:53 | storage | 2025-09-02 20:38:34 +0000 UTC Normal StatefulSet.apps storage-mysql SuccessfulCreate create Pod storage-mysql-0 in StatefulSet storage-mysql successful statefulset-controller logger.go:42: 20:45:53 | storage | 2025-09-02 20:38:35 +0000 UTC Normal Pod storage-mysql-0.spec.initContainers{mysql-init} Pulling Pulling image "perconalab/percona-server-mysql-operator:PR-1012-344c570a" kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:38:35 +0000 UTC Normal Pod storage-mysql-0.spec.initContainers{mysql-init} Pulled Successfully pulled image "perconalab/percona-server-mysql-operator:PR-1012-344c570a" in 122ms (122ms including waiting). Image size: 109076562 bytes. kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:38:35 +0000 UTC Normal Pod storage-mysql-0.spec.initContainers{mysql-init} Created Created container: mysql-init kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:38:35 +0000 UTC Normal Pod storage-mysql-0.spec.initContainers{mysql-init} Started Started container mysql-init kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:38:35 +0000 UTC Normal Pod storage-orc-0 Binding Scheduled Successfully assigned kuttl-test-careful-iguana/storage-orc-0 to gke-jen-ps-1012-344c570a-default-pool-7b175ef2-xffk default-scheduler logger.go:42: 20:45:53 | storage | 2025-09-02 20:38:35 +0000 UTC Normal StatefulSet.apps storage-orc SuccessfulCreate create Pod storage-orc-0 in StatefulSet storage-orc successful statefulset-controller logger.go:42: 20:45:53 | storage | 2025-09-02 20:38:35 +0000 UTC Normal PodDisruptionBudget.policy storage-orchestrator NoPods No matching pods found controllermanager logger.go:42: 20:45:53 | storage | 2025-09-02 20:38:35 +0000 UTC Warning PerconaServerMySQL.ps.percona.com storage ClusterStateChanged Error -> Initializing ps-controller logger.go:42: 20:45:53 | storage | 2025-09-02 20:38:36 +0000 UTC Normal Pod storage-mysql-0.spec.containers{mysql} Pulling Pulling image "perconalab/percona-server-mysql-operator:main-psmysql8.0" kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:38:36 +0000 UTC Normal Pod storage-orc-0.spec.initContainers{orchestrator-init} Pulling Pulling image "perconalab/percona-server-mysql-operator:PR-1012-344c570a" kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:38:36 +0000 UTC Normal Pod storage-orc-0.spec.initContainers{orchestrator-init} Pulled Successfully pulled image "perconalab/percona-server-mysql-operator:PR-1012-344c570a" in 104ms (104ms including waiting). Image size: 109076562 bytes. kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:38:36 +0000 UTC Normal Pod storage-orc-0.spec.initContainers{orchestrator-init} Created Created container: orchestrator-init kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:38:36 +0000 UTC Normal Pod storage-orc-0.spec.initContainers{orchestrator-init} Started Started container orchestrator-init kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:38:37 +0000 UTC Normal Pod storage-mysql-0.spec.containers{mysql} Pulled Successfully pulled image "perconalab/percona-server-mysql-operator:main-psmysql8.0" in 114ms (114ms including waiting). Image size: 417064456 bytes. kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:38:37 +0000 UTC Normal Pod storage-mysql-0.spec.containers{mysql} Created Created container: mysql kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:38:37 +0000 UTC Normal Pod storage-mysql-0.spec.containers{mysql} Started Started container mysql kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:38:37 +0000 UTC Normal Pod storage-mysql-0.spec.containers{xtrabackup} Pulling Pulling image "perconalab/percona-server-mysql-operator:main-backup8.0" kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:38:37 +0000 UTC Normal Pod storage-mysql-0.spec.containers{xtrabackup} Pulled Successfully pulled image "perconalab/percona-server-mysql-operator:main-backup8.0" in 100ms (100ms including waiting). Image size: 425405087 bytes. kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:38:37 +0000 UTC Normal Pod storage-mysql-0.spec.containers{xtrabackup} Created Created container: xtrabackup kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:38:37 +0000 UTC Normal Pod storage-mysql-0.spec.containers{xtrabackup} Started Started container xtrabackup kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:38:37 +0000 UTC Normal Pod storage-mysql-0.spec.containers{pt-heartbeat} Pulling Pulling image "perconalab/percona-server-mysql-operator:main-toolkit" kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:38:37 +0000 UTC Normal Pod storage-mysql-0.spec.containers{pt-heartbeat} Pulled Successfully pulled image "perconalab/percona-server-mysql-operator:main-toolkit" in 95ms (95ms including waiting). Image size: 133947579 bytes. kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:38:37 +0000 UTC Normal Pod storage-mysql-0.spec.containers{pt-heartbeat} Created Created container: pt-heartbeat kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:38:37 +0000 UTC Normal Pod storage-mysql-0.spec.containers{pt-heartbeat} Started Started container pt-heartbeat kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:38:38 +0000 UTC Normal Pod storage-orc-0.spec.containers{orchestrator} Pulling Pulling image "perconalab/percona-server-mysql-operator:main-orchestrator" kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:38:38 +0000 UTC Normal Pod storage-orc-0.spec.containers{orchestrator} Pulled Successfully pulled image "perconalab/percona-server-mysql-operator:main-orchestrator" in 90ms (90ms including waiting). Image size: 72482925 bytes. kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:38:38 +0000 UTC Normal Pod storage-orc-0.spec.containers{orchestrator} Created Created container: orchestrator kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:38:38 +0000 UTC Normal Pod storage-orc-0.spec.containers{orchestrator} Started Started container orchestrator kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:38:38 +0000 UTC Normal Pod storage-orc-0.spec.containers{mysql-monit} Pulling Pulling image "perconalab/percona-server-mysql-operator:main-orchestrator" kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:38:38 +0000 UTC Normal Pod storage-orc-0.spec.containers{mysql-monit} Pulled Successfully pulled image "perconalab/percona-server-mysql-operator:main-orchestrator" in 93ms (93ms including waiting). Image size: 72482925 bytes. kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:38:38 +0000 UTC Normal Pod storage-orc-0.spec.containers{mysql-monit} Created Created container: mysql-monit kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:38:38 +0000 UTC Normal Pod storage-orc-0.spec.containers{mysql-monit} Started Started container mysql-monit kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:39:10 +0000 UTC Normal Pod storage-mysql-1 Binding Scheduled Successfully assigned kuttl-test-careful-iguana/storage-mysql-1 to gke-jen-ps-1012-344c570a-default-pool-7b175ef2-tzhr default-scheduler logger.go:42: 20:45:53 | storage | 2025-09-02 20:39:10 +0000 UTC Normal Pod storage-mysql-1.spec.initContainers{mysql-init} Pulling Pulling image "perconalab/percona-server-mysql-operator:PR-1012-344c570a" kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:39:10 +0000 UTC Normal StatefulSet.apps storage-mysql SuccessfulCreate create Pod storage-mysql-1 in StatefulSet storage-mysql successful statefulset-controller logger.go:42: 20:45:53 | storage | 2025-09-02 20:39:11 +0000 UTC Normal Pod storage-mysql-1.spec.initContainers{mysql-init} Pulled Successfully pulled image "perconalab/percona-server-mysql-operator:PR-1012-344c570a" in 138ms (138ms including waiting). Image size: 109076562 bytes. kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:39:11 +0000 UTC Normal Pod storage-mysql-1.spec.initContainers{mysql-init} Created Created container: mysql-init kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:39:11 +0000 UTC Normal Pod storage-mysql-1.spec.initContainers{mysql-init} Started Started container mysql-init kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:39:11 +0000 UTC Normal Pod storage-orc-1 Binding Scheduled Successfully assigned kuttl-test-careful-iguana/storage-orc-1 to gke-jen-ps-1012-344c570a-default-pool-7b175ef2-jv2t default-scheduler logger.go:42: 20:45:53 | storage | 2025-09-02 20:39:11 +0000 UTC Normal Pod storage-orc-1.spec.initContainers{orchestrator-init} Pulling Pulling image "perconalab/percona-server-mysql-operator:PR-1012-344c570a" kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:39:11 +0000 UTC Normal StatefulSet.apps storage-orc SuccessfulCreate create Pod storage-orc-1 in StatefulSet storage-orc successful statefulset-controller logger.go:42: 20:45:53 | storage | 2025-09-02 20:39:12 +0000 UTC Normal Pod storage-mysql-1.spec.containers{mysql} Pulling Pulling image "perconalab/percona-server-mysql-operator:main-psmysql8.0" kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:39:12 +0000 UTC Normal Pod storage-mysql-1.spec.containers{mysql} Pulled Successfully pulled image "perconalab/percona-server-mysql-operator:main-psmysql8.0" in 99ms (99ms including waiting). Image size: 417064456 bytes. kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:39:12 +0000 UTC Normal Pod storage-mysql-1.spec.containers{mysql} Created Created container: mysql kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:39:12 +0000 UTC Normal Pod storage-orc-1.spec.initContainers{orchestrator-init} Pulled Successfully pulled image "perconalab/percona-server-mysql-operator:PR-1012-344c570a" in 106ms (106ms including waiting). Image size: 109076562 bytes. kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:39:12 +0000 UTC Normal Pod storage-orc-1.spec.initContainers{orchestrator-init} Created Created container: orchestrator-init kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:39:12 +0000 UTC Normal Pod storage-orc-1.spec.initContainers{orchestrator-init} Started Started container orchestrator-init kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:39:13 +0000 UTC Normal Pod storage-mysql-1.spec.containers{mysql} Started Started container mysql kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:39:13 +0000 UTC Normal Pod storage-mysql-1.spec.containers{xtrabackup} Pulling Pulling image "perconalab/percona-server-mysql-operator:main-backup8.0" kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:39:13 +0000 UTC Normal Pod storage-mysql-1.spec.containers{xtrabackup} Pulled Successfully pulled image "perconalab/percona-server-mysql-operator:main-backup8.0" in 110ms (110ms including waiting). Image size: 425405087 bytes. kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:39:13 +0000 UTC Normal Pod storage-mysql-1.spec.containers{xtrabackup} Created Created container: xtrabackup kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:39:13 +0000 UTC Normal Pod storage-mysql-1.spec.containers{xtrabackup} Started Started container xtrabackup kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:39:13 +0000 UTC Normal Pod storage-mysql-1.spec.containers{pt-heartbeat} Pulling Pulling image "perconalab/percona-server-mysql-operator:main-toolkit" kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:39:14 +0000 UTC Normal Pod storage-orc-1.spec.containers{orchestrator} Pulling Pulling image "perconalab/percona-server-mysql-operator:main-orchestrator" kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:39:15 +0000 UTC Normal Pod storage-mysql-1.spec.containers{pt-heartbeat} Pulled Successfully pulled image "perconalab/percona-server-mysql-operator:main-toolkit" in 2.713s (2.713s including waiting). Image size: 133947579 bytes. kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:39:15 +0000 UTC Normal Pod storage-mysql-1.spec.containers{pt-heartbeat} Created Created container: pt-heartbeat kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:39:15 +0000 UTC Normal Pod storage-orc-1.spec.containers{orchestrator} Pulled Successfully pulled image "perconalab/percona-server-mysql-operator:main-orchestrator" in 1.145s (1.145s including waiting). Image size: 72482925 bytes. kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:39:15 +0000 UTC Normal Pod storage-orc-1.spec.containers{orchestrator} Created Created container: orchestrator kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:39:15 +0000 UTC Normal Pod storage-orc-1.spec.containers{orchestrator} Started Started container orchestrator kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:39:15 +0000 UTC Normal Pod storage-orc-1.spec.containers{mysql-monit} Pulling Pulling image "perconalab/percona-server-mysql-operator:main-orchestrator" kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:39:15 +0000 UTC Normal Pod storage-orc-1.spec.containers{mysql-monit} Pulled Successfully pulled image "perconalab/percona-server-mysql-operator:main-orchestrator" in 120ms (120ms including waiting). Image size: 72482925 bytes. kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:39:15 +0000 UTC Normal Pod storage-orc-1.spec.containers{mysql-monit} Created Created container: mysql-monit kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:39:15 +0000 UTC Normal Pod storage-orc-1.spec.containers{mysql-monit} Started Started container mysql-monit kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:39:16 +0000 UTC Normal Pod storage-mysql-1.spec.containers{pt-heartbeat} Started Started container pt-heartbeat kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:39:17 +0000 UTC Normal Pod storage-haproxy-0 Binding Scheduled Successfully assigned kuttl-test-careful-iguana/storage-haproxy-0 to gke-jen-ps-1012-344c570a-default-pool-7b175ef2-xffk default-scheduler logger.go:42: 20:45:53 | storage | 2025-09-02 20:39:17 +0000 UTC Normal Pod storage-haproxy-0.spec.initContainers{haproxy-init} Pulling Pulling image "perconalab/percona-server-mysql-operator:PR-1012-344c570a" kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:39:17 +0000 UTC Normal Pod storage-haproxy-0.spec.initContainers{haproxy-init} Pulled Successfully pulled image "perconalab/percona-server-mysql-operator:PR-1012-344c570a" in 150ms (150ms including waiting). Image size: 109076562 bytes. kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:39:17 +0000 UTC Normal StatefulSet.apps storage-haproxy SuccessfulCreate create Pod storage-haproxy-0 in StatefulSet storage-haproxy successful statefulset-controller logger.go:42: 20:45:53 | storage | 2025-09-02 20:39:18 +0000 UTC Normal Pod storage-haproxy-0.spec.initContainers{haproxy-init} Created Created container: haproxy-init kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:39:18 +0000 UTC Normal Pod storage-haproxy-0.spec.initContainers{haproxy-init} Started Started container haproxy-init kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:39:20 +0000 UTC Normal Pod storage-haproxy-0.spec.containers{haproxy} Pulling Pulling image "perconalab/percona-server-mysql-operator:main-haproxy" kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:39:20 +0000 UTC Normal Pod storage-haproxy-0.spec.containers{haproxy} Pulled Successfully pulled image "perconalab/percona-server-mysql-operator:main-haproxy" in 82ms (82ms including waiting). Image size: 105414514 bytes. kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:39:20 +0000 UTC Normal Pod storage-haproxy-0.spec.containers{haproxy} Created Created container: haproxy kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:39:20 +0000 UTC Normal Pod storage-haproxy-0.spec.containers{haproxy} Started Started container haproxy kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:39:20 +0000 UTC Normal Pod storage-haproxy-0.spec.containers{mysql-monit} Pulling Pulling image "perconalab/percona-server-mysql-operator:main-haproxy" kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:39:20 +0000 UTC Normal Pod storage-haproxy-0.spec.containers{mysql-monit} Pulled Successfully pulled image "perconalab/percona-server-mysql-operator:main-haproxy" in 126ms (126ms including waiting). Image size: 105414514 bytes. kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:39:20 +0000 UTC Normal Pod storage-haproxy-0.spec.containers{mysql-monit} Created Created container: mysql-monit kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:39:21 +0000 UTC Normal Pod storage-haproxy-0.spec.containers{mysql-monit} Started Started container mysql-monit kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:39:31 +0000 UTC Warning Pod storage-mysql-1.spec.containers{mysql} Unhealthy Startup probe failed: 2025/09/02 20:39:30 Waiting for MySQL ready state 2025/09/02 20:39:30 MySQL is ready 2025/09/02 20:39:30 Peers: [3634346432393966.storage-mysql-unready.kuttl-test-careful-iguana 6662626339646530.storage-mysql-unready.kuttl-test-careful-iguana] 2025/09/02 20:39:30 FQDN: storage-mysql-1.storage-mysql.kuttl-test-careful-iguana 2025/09/02 20:39:30 Primary: storage-mysql-0.storage-mysql.kuttl-test-careful-iguana Replicas: [storage-mysql-1.storage-mysql.kuttl-test-careful-iguana] 2025/09/02 20:39:30 lookup storage-mysql-1 [10.66.160.24] 2025/09/02 20:39:30 PodIP: 10.66.160.24 2025/09/02 20:39:30 lookup storage-mysql-0.storage-mysql.kuttl-test-careful-iguana [10.66.162.25] 2025/09/02 20:39:30 PrimaryIP: 10.66.162.25 2025/09/02 20:39:30 Donor: storage-mysql-0.storage-mysql.kuttl-test-careful-iguana 2025/09/02 20:39:30 Opening connection to 10.66.160.24 2025/09/02 20:39:30 Clone required: true 2025/09/02 20:39:30 Checking if a clone in progress 2025/09/02 20:39:30 Clone in progress: false 2025/09/02 20:39:30 Cloning from storage-mysql-0.storage-mysql.kuttl-test-careful-iguana 2025/09/02 20:39:31 Clone finished. Restarting container... kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:39:31 +0000 UTC Normal Pod storage-mysql-1.spec.containers{mysql} Killing Container mysql failed startup probe, will be restarted kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:39:34 +0000 UTC Normal Pod storage-mysql-1.spec.containers{mysql} Pulled Successfully pulled image "perconalab/percona-server-mysql-operator:main-psmysql8.0" in 97ms (97ms including waiting). Image size: 417064456 bytes. kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:39:37 +0000 UTC Warning Pod storage-haproxy-0.spec.containers{haproxy} Unhealthy Readiness probe failed: ERROR 2013 (HY000): Lost connection to MySQL server at 'reading initial communication packet', system error: 2 kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:39:47 +0000 UTC Normal Pod storage-orc-2 Binding Scheduled Successfully assigned kuttl-test-careful-iguana/storage-orc-2 to gke-jen-ps-1012-344c570a-default-pool-7b175ef2-tzhr default-scheduler logger.go:42: 20:45:53 | storage | 2025-09-02 20:39:47 +0000 UTC Normal Pod storage-orc-2.spec.initContainers{orchestrator-init} Pulling Pulling image "perconalab/percona-server-mysql-operator:PR-1012-344c570a" kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:39:47 +0000 UTC Normal Pod storage-orc-2.spec.initContainers{orchestrator-init} Pulled Successfully pulled image "perconalab/percona-server-mysql-operator:PR-1012-344c570a" in 138ms (138ms including waiting). Image size: 109076562 bytes. kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:39:47 +0000 UTC Normal Pod storage-orc-2.spec.initContainers{orchestrator-init} Created Created container: orchestrator-init kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:39:47 +0000 UTC Normal Pod storage-orc-2.spec.initContainers{orchestrator-init} Started Started container orchestrator-init kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:39:47 +0000 UTC Normal StatefulSet.apps storage-orc SuccessfulCreate create Pod storage-orc-2 in StatefulSet storage-orc successful statefulset-controller logger.go:42: 20:45:53 | storage | 2025-09-02 20:39:49 +0000 UTC Normal Pod storage-orc-2.spec.containers{orchestrator} Pulling Pulling image "perconalab/percona-server-mysql-operator:main-orchestrator" kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:39:50 +0000 UTC Normal Pod storage-orc-2.spec.containers{orchestrator} Pulled Successfully pulled image "perconalab/percona-server-mysql-operator:main-orchestrator" in 96ms (96ms including waiting). Image size: 72482925 bytes. kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:39:50 +0000 UTC Normal Pod storage-orc-2.spec.containers{orchestrator} Created Created container: orchestrator kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:39:50 +0000 UTC Normal Pod storage-orc-2.spec.containers{orchestrator} Started Started container orchestrator kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:39:50 +0000 UTC Normal Pod storage-orc-2.spec.containers{mysql-monit} Pulling Pulling image "perconalab/percona-server-mysql-operator:main-orchestrator" kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:39:50 +0000 UTC Normal Pod storage-orc-2.spec.containers{mysql-monit} Pulled Successfully pulled image "perconalab/percona-server-mysql-operator:main-orchestrator" in 89ms (89ms including waiting). Image size: 72482925 bytes. kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:39:50 +0000 UTC Normal Pod storage-orc-2.spec.containers{mysql-monit} Created Created container: mysql-monit kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:39:50 +0000 UTC Normal Pod storage-orc-2.spec.containers{mysql-monit} Started Started container mysql-monit kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:40:06 +0000 UTC Normal Pod storage-mysql-2 Binding Scheduled Successfully assigned kuttl-test-careful-iguana/storage-mysql-2 to gke-jen-ps-1012-344c570a-default-pool-7b175ef2-xffk default-scheduler logger.go:42: 20:45:53 | storage | 2025-09-02 20:40:06 +0000 UTC Normal Pod storage-mysql-2.spec.initContainers{mysql-init} Pulling Pulling image "perconalab/percona-server-mysql-operator:PR-1012-344c570a" kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:40:06 +0000 UTC Normal Pod storage-mysql-2.spec.initContainers{mysql-init} Pulled Successfully pulled image "perconalab/percona-server-mysql-operator:PR-1012-344c570a" in 105ms (105ms including waiting). Image size: 109076562 bytes. kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:40:06 +0000 UTC Normal Pod storage-mysql-2.spec.initContainers{mysql-init} Created Created container: mysql-init kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:40:06 +0000 UTC Normal Pod storage-mysql-2.spec.initContainers{mysql-init} Started Started container mysql-init kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:40:06 +0000 UTC Normal StatefulSet.apps storage-mysql SuccessfulCreate create Pod storage-mysql-2 in StatefulSet storage-mysql successful statefulset-controller logger.go:42: 20:45:53 | storage | 2025-09-02 20:40:08 +0000 UTC Normal Pod storage-mysql-2.spec.containers{mysql} Pulling Pulling image "perconalab/percona-server-mysql-operator:main-psmysql8.0" kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:40:08 +0000 UTC Normal Pod storage-mysql-2.spec.containers{mysql} Pulled Successfully pulled image "perconalab/percona-server-mysql-operator:main-psmysql8.0" in 110ms (110ms including waiting). Image size: 417064456 bytes. kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:40:08 +0000 UTC Normal Pod storage-mysql-2.spec.containers{mysql} Created Created container: mysql kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:40:08 +0000 UTC Normal Pod storage-mysql-2.spec.containers{mysql} Started Started container mysql kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:40:08 +0000 UTC Normal Pod storage-mysql-2.spec.containers{xtrabackup} Pulling Pulling image "perconalab/percona-server-mysql-operator:main-backup8.0" kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:40:08 +0000 UTC Normal Pod storage-mysql-2.spec.containers{xtrabackup} Pulled Successfully pulled image "perconalab/percona-server-mysql-operator:main-backup8.0" in 122ms (122ms including waiting). Image size: 425405087 bytes. kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:40:08 +0000 UTC Normal Pod storage-mysql-2.spec.containers{xtrabackup} Created Created container: xtrabackup kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:40:08 +0000 UTC Normal Pod storage-mysql-2.spec.containers{xtrabackup} Started Started container xtrabackup kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:40:08 +0000 UTC Normal Pod storage-mysql-2.spec.containers{pt-heartbeat} Pulling Pulling image "perconalab/percona-server-mysql-operator:main-toolkit" kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:40:11 +0000 UTC Normal Pod storage-mysql-2.spec.containers{pt-heartbeat} Pulled Successfully pulled image "perconalab/percona-server-mysql-operator:main-toolkit" in 2.738s (2.738s including waiting). Image size: 133947579 bytes. kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:40:11 +0000 UTC Normal Pod storage-mysql-2.spec.containers{pt-heartbeat} Created Created container: pt-heartbeat kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:40:11 +0000 UTC Normal Pod storage-mysql-2.spec.containers{pt-heartbeat} Started Started container pt-heartbeat kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:40:12 +0000 UTC Normal Pod storage-haproxy-1 Binding Scheduled Successfully assigned kuttl-test-careful-iguana/storage-haproxy-1 to gke-jen-ps-1012-344c570a-default-pool-7b175ef2-jv2t default-scheduler logger.go:42: 20:45:53 | storage | 2025-09-02 20:40:12 +0000 UTC Normal StatefulSet.apps storage-haproxy SuccessfulCreate create Pod storage-haproxy-1 in StatefulSet storage-haproxy successful statefulset-controller logger.go:42: 20:45:53 | storage | 2025-09-02 20:40:13 +0000 UTC Normal Pod storage-haproxy-1.spec.initContainers{haproxy-init} Pulling Pulling image "perconalab/percona-server-mysql-operator:PR-1012-344c570a" kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:40:13 +0000 UTC Normal Pod storage-haproxy-1.spec.initContainers{haproxy-init} Pulled Successfully pulled image "perconalab/percona-server-mysql-operator:PR-1012-344c570a" in 110ms (110ms including waiting). Image size: 109076562 bytes. kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:40:13 +0000 UTC Normal Pod storage-haproxy-1.spec.initContainers{haproxy-init} Created Created container: haproxy-init kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:40:13 +0000 UTC Normal Pod storage-haproxy-1.spec.initContainers{haproxy-init} Started Started container haproxy-init kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:40:15 +0000 UTC Normal Pod storage-haproxy-1.spec.containers{haproxy} Pulling Pulling image "perconalab/percona-server-mysql-operator:main-haproxy" kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:40:15 +0000 UTC Normal Pod storage-haproxy-1.spec.containers{haproxy} Pulled Successfully pulled image "perconalab/percona-server-mysql-operator:main-haproxy" in 95ms (95ms including waiting). Image size: 105414514 bytes. kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:40:15 +0000 UTC Normal Pod storage-haproxy-1.spec.containers{haproxy} Created Created container: haproxy kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:40:15 +0000 UTC Normal Pod storage-haproxy-1.spec.containers{haproxy} Started Started container haproxy kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:40:15 +0000 UTC Normal Pod storage-haproxy-1.spec.containers{mysql-monit} Pulling Pulling image "perconalab/percona-server-mysql-operator:main-haproxy" kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:40:15 +0000 UTC Normal Pod storage-haproxy-1.spec.containers{mysql-monit} Pulled Successfully pulled image "perconalab/percona-server-mysql-operator:main-haproxy" in 90ms (90ms including waiting). Image size: 105414514 bytes. kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:40:15 +0000 UTC Normal Pod storage-haproxy-1.spec.containers{mysql-monit} Created Created container: mysql-monit kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:40:15 +0000 UTC Normal Pod storage-haproxy-1.spec.containers{mysql-monit} Started Started container mysql-monit kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:40:27 +0000 UTC Warning Pod storage-mysql-2.spec.containers{mysql} Unhealthy Startup probe failed: 2025/09/02 20:40:26 Waiting for MySQL ready state 2025/09/02 20:40:26 MySQL is ready 2025/09/02 20:40:26 Peers: [3634346432393966.storage-mysql-unready.kuttl-test-careful-iguana 6431343938653865.storage-mysql-unready.kuttl-test-careful-iguana 6662626339646530.storage-mysql-unready.kuttl-test-careful-iguana] 2025/09/02 20:40:26 FQDN: storage-mysql-2.storage-mysql.kuttl-test-careful-iguana 2025/09/02 20:40:26 Primary: storage-mysql-0.storage-mysql.kuttl-test-careful-iguana Replicas: [storage-mysql-1.storage-mysql.kuttl-test-careful-iguana storage-mysql-2.storage-mysql.kuttl-test-careful-iguana] 2025/09/02 20:40:26 lookup storage-mysql-2 [10.66.161.31] 2025/09/02 20:40:26 PodIP: 10.66.161.31 2025/09/02 20:40:26 lookup storage-mysql-0.storage-mysql.kuttl-test-careful-iguana [10.66.162.25] 2025/09/02 20:40:26 PrimaryIP: 10.66.162.25 2025/09/02 20:40:26 Donor: storage-mysql-1.storage-mysql.kuttl-test-careful-iguana 2025/09/02 20:40:26 Opening connection to 10.66.161.31 2025/09/02 20:40:26 Clone required: true 2025/09/02 20:40:26 Checking if a clone in progress 2025/09/02 20:40:26 Clone in progress: false 2025/09/02 20:40:26 Cloning from storage-mysql-1.storage-mysql.kuttl-test-careful-iguana 2025/09/02 20:40:27 Clone finished. Restarting container... kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:40:27 +0000 UTC Normal Pod storage-mysql-2.spec.containers{mysql} Killing Container mysql failed startup probe, will be restarted kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:40:30 +0000 UTC Normal Pod storage-mysql-2.spec.containers{mysql} Pulled Successfully pulled image "perconalab/percona-server-mysql-operator:main-psmysql8.0" in 105ms (105ms including waiting). Image size: 417064456 bytes. kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:40:33 +0000 UTC Normal Pod storage-haproxy-2 Binding Scheduled Successfully assigned kuttl-test-careful-iguana/storage-haproxy-2 to gke-jen-ps-1012-344c570a-default-pool-7b175ef2-tzhr default-scheduler logger.go:42: 20:45:53 | storage | 2025-09-02 20:40:33 +0000 UTC Normal StatefulSet.apps storage-haproxy SuccessfulCreate create Pod storage-haproxy-2 in StatefulSet storage-haproxy successful statefulset-controller logger.go:42: 20:45:53 | storage | 2025-09-02 20:40:34 +0000 UTC Normal Pod storage-haproxy-2.spec.initContainers{haproxy-init} Pulling Pulling image "perconalab/percona-server-mysql-operator:PR-1012-344c570a" kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:40:34 +0000 UTC Normal Pod storage-haproxy-2.spec.initContainers{haproxy-init} Pulled Successfully pulled image "perconalab/percona-server-mysql-operator:PR-1012-344c570a" in 160ms (160ms including waiting). Image size: 109076562 bytes. kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:40:34 +0000 UTC Normal Pod storage-haproxy-2.spec.initContainers{haproxy-init} Created Created container: haproxy-init kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:40:34 +0000 UTC Normal Pod storage-haproxy-2.spec.initContainers{haproxy-init} Started Started container haproxy-init kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:40:36 +0000 UTC Normal Pod storage-haproxy-2.spec.containers{haproxy} Pulling Pulling image "perconalab/percona-server-mysql-operator:main-haproxy" kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:40:36 +0000 UTC Normal Pod storage-haproxy-2.spec.containers{haproxy} Pulled Successfully pulled image "perconalab/percona-server-mysql-operator:main-haproxy" in 130ms (130ms including waiting). Image size: 105414514 bytes. kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:40:36 +0000 UTC Normal Pod storage-haproxy-2.spec.containers{haproxy} Created Created container: haproxy kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:40:36 +0000 UTC Normal Pod storage-haproxy-2.spec.containers{haproxy} Started Started container haproxy kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:40:36 +0000 UTC Normal Pod storage-haproxy-2.spec.containers{mysql-monit} Pulling Pulling image "perconalab/percona-server-mysql-operator:main-haproxy" kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:40:36 +0000 UTC Normal Pod storage-haproxy-2.spec.containers{mysql-monit} Pulled Successfully pulled image "perconalab/percona-server-mysql-operator:main-haproxy" in 127ms (127ms including waiting). Image size: 105414514 bytes. kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:40:36 +0000 UTC Normal Pod storage-haproxy-2.spec.containers{mysql-monit} Created Created container: mysql-monit kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:40:36 +0000 UTC Normal Pod storage-haproxy-2.spec.containers{mysql-monit} Started Started container mysql-monit kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:45:28 +0000 UTC Warning PerconaServerMySQL.ps.percona.com storage ClusterStateChanged Initializing -> Ready ps-controller logger.go:42: 20:45:53 | storage | 2025-09-02 20:45:44 +0000 UTC Normal Pod storage-haproxy-0.spec.containers{haproxy} Killing Stopping container haproxy kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:45:44 +0000 UTC Normal Pod storage-haproxy-0.spec.containers{mysql-monit} Killing Stopping container mysql-monit kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:45:44 +0000 UTC Normal Pod storage-haproxy-1.spec.containers{haproxy} Killing Stopping container haproxy kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:45:44 +0000 UTC Normal Pod storage-haproxy-1.spec.containers{mysql-monit} Killing Stopping container mysql-monit kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:45:44 +0000 UTC Warning Pod storage-haproxy-2.spec.containers{haproxy} Unhealthy Readiness probe failed: ERROR 2013 (HY000) at line 1: Lost connection to MySQL server during query kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:45:44 +0000 UTC Normal Pod storage-haproxy-2.spec.containers{haproxy} Killing Stopping container haproxy kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:45:44 +0000 UTC Normal Pod storage-haproxy-2.spec.containers{mysql-monit} Killing Stopping container mysql-monit kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:45:44 +0000 UTC Warning PodDisruptionBudget.policy storage-haproxy CalculateExpectedPodCountFailed Failed to calculate the number of expected pods: found no controllers for pod "storage-haproxy-1" controllermanager logger.go:42: 20:45:53 | storage | 2025-09-02 20:45:44 +0000 UTC Normal Pod storage-mysql-0.spec.containers{mysql} Killing Stopping container mysql kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:45:44 +0000 UTC Normal Pod storage-mysql-0.spec.containers{xtrabackup} Killing Stopping container xtrabackup kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:45:44 +0000 UTC Normal Pod storage-mysql-0.spec.containers{pt-heartbeat} Killing Stopping container pt-heartbeat kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:45:44 +0000 UTC Normal Pod storage-mysql-1.spec.containers{xtrabackup} Killing Stopping container xtrabackup kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:45:44 +0000 UTC Normal Pod storage-mysql-1.spec.containers{mysql} Killing Stopping container mysql kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:45:44 +0000 UTC Normal Pod storage-mysql-1.spec.containers{pt-heartbeat} Killing Stopping container pt-heartbeat kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:45:44 +0000 UTC Normal Pod storage-mysql-2.spec.containers{xtrabackup} Killing Stopping container xtrabackup kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:45:44 +0000 UTC Normal Pod storage-mysql-2.spec.containers{mysql} Killing Stopping container mysql kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:45:44 +0000 UTC Normal Pod storage-mysql-2.spec.containers{pt-heartbeat} Killing Stopping container pt-heartbeat kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:45:44 +0000 UTC Warning PodDisruptionBudget.policy storage-mysql CalculateExpectedPodCountFailed Failed to calculate the number of expected pods: found no controllers for pod "storage-mysql-0" controllermanager logger.go:42: 20:45:53 | storage | 2025-09-02 20:45:44 +0000 UTC Normal Pod storage-orc-0.spec.containers{orchestrator} Killing Stopping container orchestrator kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:45:44 +0000 UTC Normal Pod storage-orc-0.spec.containers{mysql-monit} Killing Stopping container mysql-monit kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:45:44 +0000 UTC Normal Pod storage-orc-1.spec.containers{orchestrator} Killing Stopping container orchestrator kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:45:44 +0000 UTC Normal Pod storage-orc-1.spec.containers{mysql-monit} Killing Stopping container mysql-monit kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:45:44 +0000 UTC Normal Pod storage-orc-2.spec.containers{orchestrator} Killing Stopping container orchestrator kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:45:44 +0000 UTC Normal Pod storage-orc-2.spec.containers{mysql-monit} Killing Stopping container mysql-monit kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:45:44 +0000 UTC Warning PodDisruptionBudget.policy storage-orchestrator CalculateExpectedPodCountFailed Failed to calculate the number of expected pods: found no controllers for pod "storage-orc-0" controllermanager logger.go:42: 20:45:53 | storage | 2025-09-02 20:45:44 +0000 UTC Warning PerconaServerMySQL.ps.percona.com storage ClusterStateChanged Ready -> Initializing ps-controller logger.go:42: 20:45:53 | storage | 2025-09-02 20:45:45 +0000 UTC Warning Pod storage-mysql-0.spec.containers{mysql} Unhealthy Readiness probe failed: 2025/09/02 20:45:45 MySQL state is not ready... kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:45:45 +0000 UTC Warning Pod storage-mysql-1.spec.containers{mysql} Unhealthy Readiness probe failed: 2025/09/02 20:45:45 MySQL state is not ready... kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:45:46 +0000 UTC Warning Pod storage-mysql-2.spec.containers{mysql} Unhealthy Readiness probe failed: 2025/09/02 20:45:46 MySQL state is not ready... kubelet logger.go:42: 20:45:53 | storage | 2025-09-02 20:45:50 +0000 UTC Warning Pod storage-mysql-0.spec.containers{mysql} Unhealthy Readiness probe failed: 2025/09/02 20:45:50 MySQL state is not ready... kubelet logger.go:42: 20:45:53 | storage | Deleting namespace: kuttl-test-careful-iguana === NAME kuttl harness.go:403: run tests finished harness.go:510: cleaning up harness.go:567: removing temp folder: "" --- PASS: kuttl (484.13s) --- PASS: kuttl/harness (0.00s) --- PASS: kuttl/harness/storage (483.33s) PASS