=== RUN kuttl harness.go:462: starting setup harness.go:252: running tests using configured kubeconfig. harness.go:275: Successful connection to cluster at: https://35.184.75.41 harness.go:360: running tests harness.go:73: going to run test suite with timeout of 180 seconds for each step harness.go:372: testsuite: e2e-tests/tests has 28 tests === RUN kuttl/harness === RUN kuttl/harness/gr-self-healing === PAUSE kuttl/harness/gr-self-healing === CONT kuttl/harness/gr-self-healing logger.go:42: 14:40:16 | gr-self-healing | Creating namespace: kuttl-test-secure-shiner logger.go:42: 14:40:16 | gr-self-healing/0-deploy-operator | starting test step 0-deploy-operator logger.go:42: 14:40:16 | gr-self-healing/0-deploy-operator | running command: [sh -c set -o errexit set -o xtrace source ../../functions deploy_operator deploy_non_tls_cluster_secrets deploy_tls_cluster_secrets deploy_client] logger.go:42: 14:40:16 | gr-self-healing/0-deploy-operator | + source ../../functions logger.go:42: 14:40:16 | gr-self-healing/0-deploy-operator | +++ realpath ../../.. logger.go:42: 14:40:16 | gr-self-healing/0-deploy-operator | ++ ROOT_REPO=/mnt/jenkins/workspace/cloud-ps-operator_PR-424 logger.go:42: 14:40:16 | gr-self-healing/0-deploy-operator | ++ source /mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests/vars.sh logger.go:42: 14:40:16 | gr-self-healing/0-deploy-operator | +++ export ROOT_REPO=/mnt/jenkins/workspace/cloud-ps-operator_PR-424 logger.go:42: 14:40:16 | gr-self-healing/0-deploy-operator | +++ ROOT_REPO=/mnt/jenkins/workspace/cloud-ps-operator_PR-424 logger.go:42: 14:40:16 | gr-self-healing/0-deploy-operator | +++ export DEPLOY_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/deploy logger.go:42: 14:40:16 | gr-self-healing/0-deploy-operator | +++ DEPLOY_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/deploy logger.go:42: 14:40:16 | gr-self-healing/0-deploy-operator | +++ export TESTS_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests logger.go:42: 14:40:16 | gr-self-healing/0-deploy-operator | +++ TESTS_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests logger.go:42: 14:40:16 | gr-self-healing/0-deploy-operator | +++ export TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests/conf logger.go:42: 14:40:16 | gr-self-healing/0-deploy-operator | +++ TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests/conf logger.go:42: 14:40:16 | gr-self-healing/0-deploy-operator | ++++ mktemp -d logger.go:42: 14:40:16 | gr-self-healing/0-deploy-operator | +++ export TEMP_DIR=/tmp/tmp.12tChqCeQs logger.go:42: 14:40:16 | gr-self-healing/0-deploy-operator | +++ TEMP_DIR=/tmp/tmp.12tChqCeQs logger.go:42: 14:40:16 | gr-self-healing/0-deploy-operator | ++++ git rev-parse --abbrev-ref HEAD logger.go:42: 14:40:16 | gr-self-healing/0-deploy-operator | +++ export GIT_BRANCH=PR-424 logger.go:42: 14:40:16 | gr-self-healing/0-deploy-operator | +++ GIT_BRANCH=PR-424 logger.go:42: 14:40:16 | gr-self-healing/0-deploy-operator | +++ export VERSION=PR-424-70568ae logger.go:42: 14:40:16 | gr-self-healing/0-deploy-operator | +++ VERSION=PR-424-70568ae logger.go:42: 14:40:16 | gr-self-healing/0-deploy-operator | +++ export IMAGE=perconalab/percona-server-mysql-operator:PR-424-70568ae logger.go:42: 14:40:16 | gr-self-healing/0-deploy-operator | +++ IMAGE=perconalab/percona-server-mysql-operator:PR-424-70568ae logger.go:42: 14:40:16 | gr-self-healing/0-deploy-operator | +++ export IMAGE_MYSQL=perconalab/percona-server-mysql-operator:main-psmysql logger.go:42: 14:40:16 | gr-self-healing/0-deploy-operator | +++ IMAGE_MYSQL=perconalab/percona-server-mysql-operator:main-psmysql logger.go:42: 14:40:16 | gr-self-healing/0-deploy-operator | +++ export IMAGE_BACKUP=perconalab/percona-server-mysql-operator:main-backup logger.go:42: 14:40:16 | gr-self-healing/0-deploy-operator | +++ IMAGE_BACKUP=perconalab/percona-server-mysql-operator:main-backup logger.go:42: 14:40:16 | gr-self-healing/0-deploy-operator | +++ export IMAGE_ORCHESTRATOR=perconalab/percona-server-mysql-operator:main-orchestrator logger.go:42: 14:40:16 | gr-self-healing/0-deploy-operator | +++ IMAGE_ORCHESTRATOR=perconalab/percona-server-mysql-operator:main-orchestrator logger.go:42: 14:40:16 | gr-self-healing/0-deploy-operator | +++ export IMAGE_ROUTER=perconalab/percona-server-mysql-operator:main-router logger.go:42: 14:40:16 | gr-self-healing/0-deploy-operator | +++ IMAGE_ROUTER=perconalab/percona-server-mysql-operator:main-router logger.go:42: 14:40:16 | gr-self-healing/0-deploy-operator | +++ export IMAGE_TOOLKIT=perconalab/percona-server-mysql-operator:main-toolkit logger.go:42: 14:40:16 | gr-self-healing/0-deploy-operator | +++ IMAGE_TOOLKIT=perconalab/percona-server-mysql-operator:main-toolkit logger.go:42: 14:40:16 | gr-self-healing/0-deploy-operator | +++ export IMAGE_HAPROXY=perconalab/percona-server-mysql-operator:main-haproxy logger.go:42: 14:40:16 | gr-self-healing/0-deploy-operator | +++ IMAGE_HAPROXY=perconalab/percona-server-mysql-operator:main-haproxy logger.go:42: 14:40:16 | gr-self-healing/0-deploy-operator | +++ export IMAGE_PMM=perconalab/pmm-client:dev-latest logger.go:42: 14:40:16 | gr-self-healing/0-deploy-operator | +++ IMAGE_PMM=perconalab/pmm-client:dev-latest logger.go:42: 14:40:16 | gr-self-healing/0-deploy-operator | +++ export PMM_SERVER_VERSION=9.9.9 logger.go:42: 14:40:16 | gr-self-healing/0-deploy-operator | +++ PMM_SERVER_VERSION=9.9.9 logger.go:42: 14:40:16 | gr-self-healing/0-deploy-operator | +++ export IMAGE_PMM_SERVER_REPO=perconalab/pmm-server logger.go:42: 14:40:16 | gr-self-healing/0-deploy-operator | +++ IMAGE_PMM_SERVER_REPO=perconalab/pmm-server logger.go:42: 14:40:16 | gr-self-healing/0-deploy-operator | +++ export IMAGE_PMM_SERVER_TAG=dev-latest logger.go:42: 14:40:16 | gr-self-healing/0-deploy-operator | +++ IMAGE_PMM_SERVER_TAG=dev-latest logger.go:42: 14:40:16 | gr-self-healing/0-deploy-operator | ++++ which gdate logger.go:42: 14:40:16 | gr-self-healing/0-deploy-operator | which: no gdate in (/mnt/jenkins/workspace/cloud-ps-operator_PR-424/bin/:/home/ec2-user/google-cloud-sdk/bin:/home/ec2-user/.krew/bin:/usr/local/bin:/usr/bin) logger.go:42: 14:40:16 | gr-self-healing/0-deploy-operator | ++++ which date logger.go:42: 14:40:16 | gr-self-healing/0-deploy-operator | +++ date=/usr/bin/date logger.go:42: 14:40:16 | gr-self-healing/0-deploy-operator | +++ command -v oc logger.go:42: 14:40:16 | gr-self-healing/0-deploy-operator | +++ oc get projects logger.go:42: 14:40:22 | gr-self-healing/0-deploy-operator | error: the server doesn't have a resource type "projects" logger.go:42: 14:40:22 | gr-self-healing/0-deploy-operator | +++ grep '^minikube' logger.go:42: 14:40:22 | gr-self-healing/0-deploy-operator | +++ kubectl get nodes logger.go:42: 14:40:23 | gr-self-healing/0-deploy-operator | ++++ pwd logger.go:42: 14:40:23 | gr-self-healing/0-deploy-operator | +++ basename /mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests/tests/gr-self-healing logger.go:42: 14:40:23 | gr-self-healing/0-deploy-operator | ++ test_name=gr-self-healing logger.go:42: 14:40:23 | gr-self-healing/0-deploy-operator | + deploy_operator logger.go:42: 14:40:23 | gr-self-healing/0-deploy-operator | + kubectl -n kuttl-test-secure-shiner apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-ps-operator_PR-424/deploy/crd.yaml logger.go:42: 14:40:24 | gr-self-healing/0-deploy-operator | customresourcedefinition.apiextensions.k8s.io/perconaservermysqlbackups.ps.percona.com serverside-applied logger.go:42: 14:40:24 | gr-self-healing/0-deploy-operator | customresourcedefinition.apiextensions.k8s.io/perconaservermysqlrestores.ps.percona.com serverside-applied logger.go:42: 14:40:25 | gr-self-healing/0-deploy-operator | customresourcedefinition.apiextensions.k8s.io/perconaservermysqls.ps.percona.com serverside-applied logger.go:42: 14:40:25 | gr-self-healing/0-deploy-operator | + kubectl -n kuttl-test-secure-shiner apply -f /mnt/jenkins/workspace/cloud-ps-operator_PR-424/deploy/rbac.yaml logger.go:42: 14:40:26 | gr-self-healing/0-deploy-operator | serviceaccount/percona-server-mysql-operator created logger.go:42: 14:40:27 | gr-self-healing/0-deploy-operator | serviceaccount/percona-server-mysql-operator-orchestrator created logger.go:42: 14:40:27 | gr-self-healing/0-deploy-operator | role.rbac.authorization.k8s.io/percona-server-mysql-operator-leaderelection created logger.go:42: 14:40:27 | gr-self-healing/0-deploy-operator | role.rbac.authorization.k8s.io/percona-server-mysql-operator created logger.go:42: 14:40:28 | gr-self-healing/0-deploy-operator | role.rbac.authorization.k8s.io/percona-server-mysql-operator-orchestrator created logger.go:42: 14:40:28 | gr-self-healing/0-deploy-operator | rolebinding.rbac.authorization.k8s.io/percona-server-mysql-operator-leaderelection created logger.go:42: 14:40:28 | gr-self-healing/0-deploy-operator | rolebinding.rbac.authorization.k8s.io/percona-server-mysql-operator created logger.go:42: 14:40:29 | gr-self-healing/0-deploy-operator | rolebinding.rbac.authorization.k8s.io/percona-server-mysql-operator-orchestrator created logger.go:42: 14:40:29 | gr-self-healing/0-deploy-operator | + yq eval '(select(documentIndex==1).spec.template.spec.containers[] | select(.name=="manager").env[] | select(.name=="DISABLE_TELEMETRY").value) = "true"' logger.go:42: 14:40:29 | gr-self-healing/0-deploy-operator | + kubectl -n kuttl-test-secure-shiner apply -f - logger.go:42: 14:40:29 | gr-self-healing/0-deploy-operator | ++ printf 'select(documentIndex==1).spec.template.spec.containers[0].image="%s"' perconalab/percona-server-mysql-operator:PR-424-70568ae logger.go:42: 14:40:29 | gr-self-healing/0-deploy-operator | + yq eval 'select(documentIndex==1).spec.template.spec.containers[0].image="perconalab/percona-server-mysql-operator:PR-424-70568ae"' /mnt/jenkins/workspace/cloud-ps-operator_PR-424/deploy/operator.yaml logger.go:42: 14:40:30 | gr-self-healing/0-deploy-operator | configmap/percona-server-mysql-operator-config created logger.go:42: 14:40:30 | gr-self-healing/0-deploy-operator | deployment.apps/percona-server-mysql-operator created logger.go:42: 14:40:30 | gr-self-healing/0-deploy-operator | + deploy_non_tls_cluster_secrets logger.go:42: 14:40:30 | gr-self-healing/0-deploy-operator | + kubectl -n kuttl-test-secure-shiner apply -f /mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests/conf/secrets.yaml logger.go:42: 14:40:31 | gr-self-healing/0-deploy-operator | secret/test-secrets created logger.go:42: 14:40:31 | gr-self-healing/0-deploy-operator | + deploy_tls_cluster_secrets logger.go:42: 14:40:31 | gr-self-healing/0-deploy-operator | + kubectl -n kuttl-test-secure-shiner apply -f /mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests/conf/ssl-secret.yaml logger.go:42: 14:40:33 | gr-self-healing/0-deploy-operator | secret/test-ssl created logger.go:42: 14:40:33 | gr-self-healing/0-deploy-operator | + deploy_client logger.go:42: 14:40:33 | gr-self-healing/0-deploy-operator | + kubectl -n kuttl-test-secure-shiner apply -f /mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests/conf/client.yaml logger.go:42: 14:40:34 | gr-self-healing/0-deploy-operator | pod/mysql-client created logger.go:42: 14:40:35 | gr-self-healing/0-deploy-operator | running command: [sh -c kubectl assert exist-enhanced deployment percona-server-mysql-operator -n ${OPERATOR_NS:-$NAMESPACE} --field-selector status.readyReplicas=1] logger.go:42: 14:40:35 | gr-self-healing/0-deploy-operator | ASSERT deployment percona-server-mysql-operator matching field criteria 'status.readyReplicas=1' should exist. logger.go:42: 14:40:36 | gr-self-healing/0-deploy-operator | ASSERT FAIL Resource(s) not found. logger.go:42: 14:40:37 | gr-self-healing/0-deploy-operator | running command: [sh -c kubectl assert exist-enhanced deployment percona-server-mysql-operator -n ${OPERATOR_NS:-$NAMESPACE} --field-selector status.readyReplicas=1] logger.go:42: 14:40:37 | gr-self-healing/0-deploy-operator | ASSERT deployment percona-server-mysql-operator matching field criteria 'status.readyReplicas=1' should exist. logger.go:42: 14:40:38 | gr-self-healing/0-deploy-operator | ASSERT FAIL Resource(s) not found. logger.go:42: 14:40:39 | gr-self-healing/0-deploy-operator | running command: [sh -c kubectl assert exist-enhanced deployment percona-server-mysql-operator -n ${OPERATOR_NS:-$NAMESPACE} --field-selector status.readyReplicas=1] logger.go:42: 14:40:39 | gr-self-healing/0-deploy-operator | ASSERT deployment percona-server-mysql-operator matching field criteria 'status.readyReplicas=1' should exist. logger.go:42: 14:40:40 | gr-self-healing/0-deploy-operator | ASSERT FAIL Resource(s) not found. logger.go:42: 14:40:41 | gr-self-healing/0-deploy-operator | running command: [sh -c kubectl assert exist-enhanced deployment percona-server-mysql-operator -n ${OPERATOR_NS:-$NAMESPACE} --field-selector status.readyReplicas=1] logger.go:42: 14:40:41 | gr-self-healing/0-deploy-operator | ASSERT deployment percona-server-mysql-operator matching field criteria 'status.readyReplicas=1' should exist. logger.go:42: 14:40:42 | gr-self-healing/0-deploy-operator | INFO Found 1 resource(s). logger.go:42: 14:40:42 | gr-self-healing/0-deploy-operator | NAME NAMESPACE COL0 logger.go:42: 14:40:42 | gr-self-healing/0-deploy-operator | percona-server-mysql-operator kuttl-test-secure-shiner 1 logger.go:42: 14:40:42 | gr-self-healing/0-deploy-operator | ASSERT PASS logger.go:42: 14:40:42 | gr-self-healing/0-deploy-operator | test step completed 0-deploy-operator logger.go:42: 14:40:42 | gr-self-healing/1-deploy-chaos-mesh | starting test step 1-deploy-chaos-mesh logger.go:42: 14:40:42 | gr-self-healing/1-deploy-chaos-mesh | running command: [sh -c set -o errexit set -o xtrace source ../../functions deploy_chaos_mesh] logger.go:42: 14:40:42 | gr-self-healing/1-deploy-chaos-mesh | + source ../../functions logger.go:42: 14:40:42 | gr-self-healing/1-deploy-chaos-mesh | +++ realpath ../../.. logger.go:42: 14:40:42 | gr-self-healing/1-deploy-chaos-mesh | ++ ROOT_REPO=/mnt/jenkins/workspace/cloud-ps-operator_PR-424 logger.go:42: 14:40:42 | gr-self-healing/1-deploy-chaos-mesh | ++ source /mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests/vars.sh logger.go:42: 14:40:42 | gr-self-healing/1-deploy-chaos-mesh | +++ export ROOT_REPO=/mnt/jenkins/workspace/cloud-ps-operator_PR-424 logger.go:42: 14:40:42 | gr-self-healing/1-deploy-chaos-mesh | +++ ROOT_REPO=/mnt/jenkins/workspace/cloud-ps-operator_PR-424 logger.go:42: 14:40:42 | gr-self-healing/1-deploy-chaos-mesh | +++ export DEPLOY_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/deploy logger.go:42: 14:40:42 | gr-self-healing/1-deploy-chaos-mesh | +++ DEPLOY_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/deploy logger.go:42: 14:40:42 | gr-self-healing/1-deploy-chaos-mesh | +++ export TESTS_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests logger.go:42: 14:40:42 | gr-self-healing/1-deploy-chaos-mesh | +++ TESTS_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests logger.go:42: 14:40:42 | gr-self-healing/1-deploy-chaos-mesh | +++ export TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests/conf logger.go:42: 14:40:42 | gr-self-healing/1-deploy-chaos-mesh | +++ TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests/conf logger.go:42: 14:40:42 | gr-self-healing/1-deploy-chaos-mesh | ++++ mktemp -d logger.go:42: 14:40:42 | gr-self-healing/1-deploy-chaos-mesh | +++ export TEMP_DIR=/tmp/tmp.ZK3V9j3EOe logger.go:42: 14:40:42 | gr-self-healing/1-deploy-chaos-mesh | +++ TEMP_DIR=/tmp/tmp.ZK3V9j3EOe logger.go:42: 14:40:42 | gr-self-healing/1-deploy-chaos-mesh | ++++ git rev-parse --abbrev-ref HEAD logger.go:42: 14:40:42 | gr-self-healing/1-deploy-chaos-mesh | +++ export GIT_BRANCH=PR-424 logger.go:42: 14:40:42 | gr-self-healing/1-deploy-chaos-mesh | +++ GIT_BRANCH=PR-424 logger.go:42: 14:40:42 | gr-self-healing/1-deploy-chaos-mesh | +++ export VERSION=PR-424-70568ae logger.go:42: 14:40:42 | gr-self-healing/1-deploy-chaos-mesh | +++ VERSION=PR-424-70568ae logger.go:42: 14:40:42 | gr-self-healing/1-deploy-chaos-mesh | +++ export IMAGE=perconalab/percona-server-mysql-operator:PR-424-70568ae logger.go:42: 14:40:42 | gr-self-healing/1-deploy-chaos-mesh | +++ IMAGE=perconalab/percona-server-mysql-operator:PR-424-70568ae logger.go:42: 14:40:42 | gr-self-healing/1-deploy-chaos-mesh | +++ export IMAGE_MYSQL=perconalab/percona-server-mysql-operator:main-psmysql logger.go:42: 14:40:42 | gr-self-healing/1-deploy-chaos-mesh | +++ IMAGE_MYSQL=perconalab/percona-server-mysql-operator:main-psmysql logger.go:42: 14:40:42 | gr-self-healing/1-deploy-chaos-mesh | +++ export IMAGE_BACKUP=perconalab/percona-server-mysql-operator:main-backup logger.go:42: 14:40:42 | gr-self-healing/1-deploy-chaos-mesh | +++ IMAGE_BACKUP=perconalab/percona-server-mysql-operator:main-backup logger.go:42: 14:40:42 | gr-self-healing/1-deploy-chaos-mesh | +++ export IMAGE_ORCHESTRATOR=perconalab/percona-server-mysql-operator:main-orchestrator logger.go:42: 14:40:42 | gr-self-healing/1-deploy-chaos-mesh | +++ IMAGE_ORCHESTRATOR=perconalab/percona-server-mysql-operator:main-orchestrator logger.go:42: 14:40:42 | gr-self-healing/1-deploy-chaos-mesh | +++ export IMAGE_ROUTER=perconalab/percona-server-mysql-operator:main-router logger.go:42: 14:40:42 | gr-self-healing/1-deploy-chaos-mesh | +++ IMAGE_ROUTER=perconalab/percona-server-mysql-operator:main-router logger.go:42: 14:40:42 | gr-self-healing/1-deploy-chaos-mesh | +++ export IMAGE_TOOLKIT=perconalab/percona-server-mysql-operator:main-toolkit logger.go:42: 14:40:42 | gr-self-healing/1-deploy-chaos-mesh | +++ IMAGE_TOOLKIT=perconalab/percona-server-mysql-operator:main-toolkit logger.go:42: 14:40:42 | gr-self-healing/1-deploy-chaos-mesh | +++ export IMAGE_HAPROXY=perconalab/percona-server-mysql-operator:main-haproxy logger.go:42: 14:40:42 | gr-self-healing/1-deploy-chaos-mesh | +++ IMAGE_HAPROXY=perconalab/percona-server-mysql-operator:main-haproxy logger.go:42: 14:40:42 | gr-self-healing/1-deploy-chaos-mesh | +++ export IMAGE_PMM=perconalab/pmm-client:dev-latest logger.go:42: 14:40:42 | gr-self-healing/1-deploy-chaos-mesh | +++ IMAGE_PMM=perconalab/pmm-client:dev-latest logger.go:42: 14:40:42 | gr-self-healing/1-deploy-chaos-mesh | +++ export PMM_SERVER_VERSION=9.9.9 logger.go:42: 14:40:42 | gr-self-healing/1-deploy-chaos-mesh | +++ PMM_SERVER_VERSION=9.9.9 logger.go:42: 14:40:42 | gr-self-healing/1-deploy-chaos-mesh | +++ export IMAGE_PMM_SERVER_REPO=perconalab/pmm-server logger.go:42: 14:40:42 | gr-self-healing/1-deploy-chaos-mesh | +++ IMAGE_PMM_SERVER_REPO=perconalab/pmm-server logger.go:42: 14:40:42 | gr-self-healing/1-deploy-chaos-mesh | +++ export IMAGE_PMM_SERVER_TAG=dev-latest logger.go:42: 14:40:42 | gr-self-healing/1-deploy-chaos-mesh | +++ IMAGE_PMM_SERVER_TAG=dev-latest logger.go:42: 14:40:42 | gr-self-healing/1-deploy-chaos-mesh | ++++ which gdate logger.go:42: 14:40:42 | gr-self-healing/1-deploy-chaos-mesh | which: no gdate in (/mnt/jenkins/workspace/cloud-ps-operator_PR-424/bin/:/home/ec2-user/google-cloud-sdk/bin:/home/ec2-user/.krew/bin:/usr/local/bin:/usr/bin) logger.go:42: 14:40:42 | gr-self-healing/1-deploy-chaos-mesh | ++++ which date logger.go:42: 14:40:42 | gr-self-healing/1-deploy-chaos-mesh | +++ date=/usr/bin/date logger.go:42: 14:40:42 | gr-self-healing/1-deploy-chaos-mesh | +++ command -v oc logger.go:42: 14:40:42 | gr-self-healing/1-deploy-chaos-mesh | +++ oc get projects logger.go:42: 14:40:48 | gr-self-healing/1-deploy-chaos-mesh | error: the server doesn't have a resource type "projects" logger.go:42: 14:40:48 | gr-self-healing/1-deploy-chaos-mesh | +++ kubectl get nodes logger.go:42: 14:40:48 | gr-self-healing/1-deploy-chaos-mesh | +++ grep '^minikube' logger.go:42: 14:40:49 | gr-self-healing/1-deploy-chaos-mesh | ++++ pwd logger.go:42: 14:40:49 | gr-self-healing/1-deploy-chaos-mesh | +++ basename /mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests/tests/gr-self-healing logger.go:42: 14:40:49 | gr-self-healing/1-deploy-chaos-mesh | ++ test_name=gr-self-healing logger.go:42: 14:40:49 | gr-self-healing/1-deploy-chaos-mesh | + deploy_chaos_mesh logger.go:42: 14:40:49 | gr-self-healing/1-deploy-chaos-mesh | + destroy_chaos_mesh logger.go:42: 14:40:49 | gr-self-healing/1-deploy-chaos-mesh | ++ helm list --all-namespaces --filter chaos-mesh logger.go:42: 14:40:49 | gr-self-healing/1-deploy-chaos-mesh | ++ tail -n1 logger.go:42: 14:40:49 | gr-self-healing/1-deploy-chaos-mesh | ++ sed s/NAMESPACE// logger.go:42: 14:40:49 | gr-self-healing/1-deploy-chaos-mesh | ++ awk '-F ' '{print $2}' logger.go:42: 14:40:49 | gr-self-healing/1-deploy-chaos-mesh | WARNING: Kubernetes configuration file is group-readable. This is insecure. Location: /mnt/jenkins/workspace/cloud-ps-operator_PR-424/kubeconfig logger.go:42: 14:40:49 | gr-self-healing/1-deploy-chaos-mesh | WARNING: Kubernetes configuration file is world-readable. This is insecure. Location: /mnt/jenkins/workspace/cloud-ps-operator_PR-424/kubeconfig logger.go:42: 14:40:49 | gr-self-healing/1-deploy-chaos-mesh | + local chaos_mesh_ns= logger.go:42: 14:40:49 | gr-self-healing/1-deploy-chaos-mesh | ++ kubectl api-resources logger.go:42: 14:40:49 | gr-self-healing/1-deploy-chaos-mesh | ++ grep chaos-mesh logger.go:42: 14:40:49 | gr-self-healing/1-deploy-chaos-mesh | ++ awk '{print $1}' logger.go:42: 14:40:50 | gr-self-healing/1-deploy-chaos-mesh | + '[' -n '' ']' logger.go:42: 14:40:50 | gr-self-healing/1-deploy-chaos-mesh | ++ kubectl get crd logger.go:42: 14:40:50 | gr-self-healing/1-deploy-chaos-mesh | ++ grep chaos-mesh.org logger.go:42: 14:40:50 | gr-self-healing/1-deploy-chaos-mesh | ++ awk '{print $1}' logger.go:42: 14:40:51 | gr-self-healing/1-deploy-chaos-mesh | + timeout 30 kubectl delete crd logger.go:42: 14:40:51 | gr-self-healing/1-deploy-chaos-mesh | error: resource(s) were provided, but no name was specified logger.go:42: 14:40:51 | gr-self-healing/1-deploy-chaos-mesh | + : logger.go:42: 14:40:51 | gr-self-healing/1-deploy-chaos-mesh | ++ kubectl get clusterrolebinding logger.go:42: 14:40:51 | gr-self-healing/1-deploy-chaos-mesh | ++ grep chaos-mesh logger.go:42: 14:40:51 | gr-self-healing/1-deploy-chaos-mesh | ++ awk '{print $1}' logger.go:42: 14:40:52 | gr-self-healing/1-deploy-chaos-mesh | + timeout 30 kubectl delete clusterrolebinding logger.go:42: 14:40:53 | gr-self-healing/1-deploy-chaos-mesh | error: resource(s) were provided, but no name was specified logger.go:42: 14:40:53 | gr-self-healing/1-deploy-chaos-mesh | + : logger.go:42: 14:40:53 | gr-self-healing/1-deploy-chaos-mesh | ++ kubectl get clusterrole logger.go:42: 14:40:53 | gr-self-healing/1-deploy-chaos-mesh | ++ grep chaos-mesh logger.go:42: 14:40:53 | gr-self-healing/1-deploy-chaos-mesh | ++ awk '{print $1}' logger.go:42: 14:40:54 | gr-self-healing/1-deploy-chaos-mesh | + timeout 30 kubectl delete clusterrole logger.go:42: 14:40:54 | gr-self-healing/1-deploy-chaos-mesh | error: resource(s) were provided, but no name was specified logger.go:42: 14:40:54 | gr-self-healing/1-deploy-chaos-mesh | + : logger.go:42: 14:40:54 | gr-self-healing/1-deploy-chaos-mesh | ++ kubectl get MutatingWebhookConfiguration logger.go:42: 14:40:54 | gr-self-healing/1-deploy-chaos-mesh | ++ grep chaos-mesh logger.go:42: 14:40:54 | gr-self-healing/1-deploy-chaos-mesh | ++ awk '{print $1}' logger.go:42: 14:40:55 | gr-self-healing/1-deploy-chaos-mesh | + timeout 30 kubectl delete MutatingWebhookConfiguration logger.go:42: 14:40:55 | gr-self-healing/1-deploy-chaos-mesh | error: resource(s) were provided, but no name was specified logger.go:42: 14:40:55 | gr-self-healing/1-deploy-chaos-mesh | + : logger.go:42: 14:40:55 | gr-self-healing/1-deploy-chaos-mesh | ++ kubectl get ValidatingWebhookConfiguration logger.go:42: 14:40:55 | gr-self-healing/1-deploy-chaos-mesh | ++ grep chaos-mesh logger.go:42: 14:40:55 | gr-self-healing/1-deploy-chaos-mesh | ++ awk '{print $1}' logger.go:42: 14:40:56 | gr-self-healing/1-deploy-chaos-mesh | + timeout 30 kubectl delete ValidatingWebhookConfiguration logger.go:42: 14:40:56 | gr-self-healing/1-deploy-chaos-mesh | error: resource(s) were provided, but no name was specified logger.go:42: 14:40:56 | gr-self-healing/1-deploy-chaos-mesh | + : logger.go:42: 14:40:56 | gr-self-healing/1-deploy-chaos-mesh | ++ kubectl get ValidatingWebhookConfiguration logger.go:42: 14:40:56 | gr-self-healing/1-deploy-chaos-mesh | ++ grep validate-auth logger.go:42: 14:40:56 | gr-self-healing/1-deploy-chaos-mesh | ++ awk '{print $1}' logger.go:42: 14:40:57 | gr-self-healing/1-deploy-chaos-mesh | + timeout 30 kubectl delete ValidatingWebhookConfiguration logger.go:42: 14:40:58 | gr-self-healing/1-deploy-chaos-mesh | error: resource(s) were provided, but no name was specified logger.go:42: 14:40:58 | gr-self-healing/1-deploy-chaos-mesh | + : logger.go:42: 14:40:58 | gr-self-healing/1-deploy-chaos-mesh | + helm repo add chaos-mesh https://charts.chaos-mesh.org logger.go:42: 14:40:58 | gr-self-healing/1-deploy-chaos-mesh | WARNING: Kubernetes configuration file is group-readable. This is insecure. Location: /mnt/jenkins/workspace/cloud-ps-operator_PR-424/kubeconfig logger.go:42: 14:40:58 | gr-self-healing/1-deploy-chaos-mesh | WARNING: Kubernetes configuration file is world-readable. This is insecure. Location: /mnt/jenkins/workspace/cloud-ps-operator_PR-424/kubeconfig logger.go:42: 14:40:58 | gr-self-healing/1-deploy-chaos-mesh | "chaos-mesh" has been added to your repositories logger.go:42: 14:40:58 | gr-self-healing/1-deploy-chaos-mesh | + helm install chaos-mesh chaos-mesh/chaos-mesh --namespace=kuttl-test-secure-shiner --set chaosDaemon.runtime=containerd --set chaosDaemon.socketPath=/run/containerd/containerd.sock --set dashboard.create=false --version 2.5.1 logger.go:42: 14:40:58 | gr-self-healing/1-deploy-chaos-mesh | WARNING: Kubernetes configuration file is group-readable. This is insecure. Location: /mnt/jenkins/workspace/cloud-ps-operator_PR-424/kubeconfig logger.go:42: 14:40:58 | gr-self-healing/1-deploy-chaos-mesh | WARNING: Kubernetes configuration file is world-readable. This is insecure. Location: /mnt/jenkins/workspace/cloud-ps-operator_PR-424/kubeconfig logger.go:42: 14:41:24 | gr-self-healing/1-deploy-chaos-mesh | NAME: chaos-mesh logger.go:42: 14:41:24 | gr-self-healing/1-deploy-chaos-mesh | LAST DEPLOYED: Thu Aug 17 14:41:11 2023 logger.go:42: 14:41:24 | gr-self-healing/1-deploy-chaos-mesh | NAMESPACE: kuttl-test-secure-shiner logger.go:42: 14:41:24 | gr-self-healing/1-deploy-chaos-mesh | STATUS: deployed logger.go:42: 14:41:24 | gr-self-healing/1-deploy-chaos-mesh | REVISION: 1 logger.go:42: 14:41:24 | gr-self-healing/1-deploy-chaos-mesh | TEST SUITE: None logger.go:42: 14:41:24 | gr-self-healing/1-deploy-chaos-mesh | NOTES: logger.go:42: 14:41:24 | gr-self-healing/1-deploy-chaos-mesh | 1. Make sure chaos-mesh components are running logger.go:42: 14:41:24 | gr-self-healing/1-deploy-chaos-mesh | kubectl get pods --namespace kuttl-test-secure-shiner -l app.kubernetes.io/instance=chaos-mesh logger.go:42: 14:41:24 | gr-self-healing/1-deploy-chaos-mesh | + sleep 10 logger.go:42: 14:41:38 | gr-self-healing/1-deploy-chaos-mesh | test step completed 1-deploy-chaos-mesh logger.go:42: 14:41:38 | gr-self-healing/2-create-cluster | starting test step 2-create-cluster logger.go:42: 14:41:38 | gr-self-healing/2-create-cluster | running command: [sh -c set -o errexit set -o xtrace source ../../functions get_cr \ | yq eval '.spec.mysql.clusterType="group-replication"' - \ | yq eval '.spec.mysql.size=3' - \ | yq eval '.spec.mysql.affinity.antiAffinityTopologyKey="none"' - \ | yq eval '.spec.proxy.haproxy.enabled=false' - \ | yq eval '.spec.proxy.router.enabled=true' - \ | yq eval '.spec.proxy.router.size=3' - \ | yq eval '.spec.proxy.router.affinity.antiAffinityTopologyKey="none"' - \ | yq eval '.spec.orchestrator.enabled=false' - \ | kubectl -n "${NAMESPACE}" apply -f -] logger.go:42: 14:41:38 | gr-self-healing/2-create-cluster | + source ../../functions logger.go:42: 14:41:38 | gr-self-healing/2-create-cluster | +++ realpath ../../.. logger.go:42: 14:41:38 | gr-self-healing/2-create-cluster | ++ ROOT_REPO=/mnt/jenkins/workspace/cloud-ps-operator_PR-424 logger.go:42: 14:41:38 | gr-self-healing/2-create-cluster | ++ source /mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests/vars.sh logger.go:42: 14:41:38 | gr-self-healing/2-create-cluster | +++ export ROOT_REPO=/mnt/jenkins/workspace/cloud-ps-operator_PR-424 logger.go:42: 14:41:38 | gr-self-healing/2-create-cluster | +++ ROOT_REPO=/mnt/jenkins/workspace/cloud-ps-operator_PR-424 logger.go:42: 14:41:38 | gr-self-healing/2-create-cluster | +++ export DEPLOY_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/deploy logger.go:42: 14:41:38 | gr-self-healing/2-create-cluster | +++ DEPLOY_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/deploy logger.go:42: 14:41:38 | gr-self-healing/2-create-cluster | +++ export TESTS_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests logger.go:42: 14:41:38 | gr-self-healing/2-create-cluster | +++ TESTS_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests logger.go:42: 14:41:38 | gr-self-healing/2-create-cluster | +++ export TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests/conf logger.go:42: 14:41:38 | gr-self-healing/2-create-cluster | +++ TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests/conf logger.go:42: 14:41:38 | gr-self-healing/2-create-cluster | ++++ mktemp -d logger.go:42: 14:41:38 | gr-self-healing/2-create-cluster | +++ export TEMP_DIR=/tmp/tmp.fMglsA0U3d logger.go:42: 14:41:38 | gr-self-healing/2-create-cluster | +++ TEMP_DIR=/tmp/tmp.fMglsA0U3d logger.go:42: 14:41:38 | gr-self-healing/2-create-cluster | ++++ git rev-parse --abbrev-ref HEAD logger.go:42: 14:41:38 | gr-self-healing/2-create-cluster | +++ export GIT_BRANCH=PR-424 logger.go:42: 14:41:38 | gr-self-healing/2-create-cluster | +++ GIT_BRANCH=PR-424 logger.go:42: 14:41:38 | gr-self-healing/2-create-cluster | +++ export VERSION=PR-424-70568ae logger.go:42: 14:41:38 | gr-self-healing/2-create-cluster | +++ VERSION=PR-424-70568ae logger.go:42: 14:41:38 | gr-self-healing/2-create-cluster | +++ export IMAGE=perconalab/percona-server-mysql-operator:PR-424-70568ae logger.go:42: 14:41:38 | gr-self-healing/2-create-cluster | +++ IMAGE=perconalab/percona-server-mysql-operator:PR-424-70568ae logger.go:42: 14:41:38 | gr-self-healing/2-create-cluster | +++ export IMAGE_MYSQL=perconalab/percona-server-mysql-operator:main-psmysql logger.go:42: 14:41:38 | gr-self-healing/2-create-cluster | +++ IMAGE_MYSQL=perconalab/percona-server-mysql-operator:main-psmysql logger.go:42: 14:41:38 | gr-self-healing/2-create-cluster | +++ export IMAGE_BACKUP=perconalab/percona-server-mysql-operator:main-backup logger.go:42: 14:41:38 | gr-self-healing/2-create-cluster | +++ IMAGE_BACKUP=perconalab/percona-server-mysql-operator:main-backup logger.go:42: 14:41:38 | gr-self-healing/2-create-cluster | +++ export IMAGE_ORCHESTRATOR=perconalab/percona-server-mysql-operator:main-orchestrator logger.go:42: 14:41:38 | gr-self-healing/2-create-cluster | +++ IMAGE_ORCHESTRATOR=perconalab/percona-server-mysql-operator:main-orchestrator logger.go:42: 14:41:38 | gr-self-healing/2-create-cluster | +++ export IMAGE_ROUTER=perconalab/percona-server-mysql-operator:main-router logger.go:42: 14:41:38 | gr-self-healing/2-create-cluster | +++ IMAGE_ROUTER=perconalab/percona-server-mysql-operator:main-router logger.go:42: 14:41:38 | gr-self-healing/2-create-cluster | +++ export IMAGE_TOOLKIT=perconalab/percona-server-mysql-operator:main-toolkit logger.go:42: 14:41:38 | gr-self-healing/2-create-cluster | +++ IMAGE_TOOLKIT=perconalab/percona-server-mysql-operator:main-toolkit logger.go:42: 14:41:38 | gr-self-healing/2-create-cluster | +++ export IMAGE_HAPROXY=perconalab/percona-server-mysql-operator:main-haproxy logger.go:42: 14:41:38 | gr-self-healing/2-create-cluster | +++ IMAGE_HAPROXY=perconalab/percona-server-mysql-operator:main-haproxy logger.go:42: 14:41:38 | gr-self-healing/2-create-cluster | +++ export IMAGE_PMM=perconalab/pmm-client:dev-latest logger.go:42: 14:41:38 | gr-self-healing/2-create-cluster | +++ IMAGE_PMM=perconalab/pmm-client:dev-latest logger.go:42: 14:41:38 | gr-self-healing/2-create-cluster | +++ export PMM_SERVER_VERSION=9.9.9 logger.go:42: 14:41:38 | gr-self-healing/2-create-cluster | +++ PMM_SERVER_VERSION=9.9.9 logger.go:42: 14:41:38 | gr-self-healing/2-create-cluster | +++ export IMAGE_PMM_SERVER_REPO=perconalab/pmm-server logger.go:42: 14:41:38 | gr-self-healing/2-create-cluster | +++ IMAGE_PMM_SERVER_REPO=perconalab/pmm-server logger.go:42: 14:41:38 | gr-self-healing/2-create-cluster | +++ export IMAGE_PMM_SERVER_TAG=dev-latest logger.go:42: 14:41:38 | gr-self-healing/2-create-cluster | +++ IMAGE_PMM_SERVER_TAG=dev-latest logger.go:42: 14:41:38 | gr-self-healing/2-create-cluster | ++++ which gdate logger.go:42: 14:41:38 | gr-self-healing/2-create-cluster | which: no gdate in (/mnt/jenkins/workspace/cloud-ps-operator_PR-424/bin/:/home/ec2-user/google-cloud-sdk/bin:/home/ec2-user/.krew/bin:/usr/local/bin:/usr/bin) logger.go:42: 14:41:38 | gr-self-healing/2-create-cluster | ++++ which date logger.go:42: 14:41:38 | gr-self-healing/2-create-cluster | +++ date=/usr/bin/date logger.go:42: 14:41:38 | gr-self-healing/2-create-cluster | +++ command -v oc logger.go:42: 14:41:38 | gr-self-healing/2-create-cluster | +++ oc get projects logger.go:42: 14:41:43 | gr-self-healing/2-create-cluster | error: the server doesn't have a resource type "projects" logger.go:42: 14:41:43 | gr-self-healing/2-create-cluster | +++ grep '^minikube' logger.go:42: 14:41:43 | gr-self-healing/2-create-cluster | +++ kubectl get nodes logger.go:42: 14:41:44 | gr-self-healing/2-create-cluster | ++++ pwd logger.go:42: 14:41:44 | gr-self-healing/2-create-cluster | +++ basename /mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests/tests/gr-self-healing logger.go:42: 14:41:44 | gr-self-healing/2-create-cluster | ++ test_name=gr-self-healing logger.go:42: 14:41:44 | gr-self-healing/2-create-cluster | + get_cr logger.go:42: 14:41:44 | gr-self-healing/2-create-cluster | + local name_suffix= logger.go:42: 14:41:44 | gr-self-healing/2-create-cluster | + yq eval '.spec.mysql.clusterType="group-replication"' - logger.go:42: 14:41:44 | gr-self-healing/2-create-cluster | + yq eval .spec.mysql.size=3 - logger.go:42: 14:41:44 | gr-self-healing/2-create-cluster | + yq eval '.spec.mysql.affinity.antiAffinityTopologyKey="none"' - logger.go:42: 14:41:44 | gr-self-healing/2-create-cluster | ++ printf '.spec.proxy.router.image="%s"' perconalab/percona-server-mysql-operator:main-router logger.go:42: 14:41:44 | gr-self-healing/2-create-cluster | + yq eval '.spec.proxy.router.image="perconalab/percona-server-mysql-operator:main-router"' - logger.go:42: 14:41:44 | gr-self-healing/2-create-cluster | + '[' -n '' ']' logger.go:42: 14:41:44 | gr-self-healing/2-create-cluster | + yq eval - logger.go:42: 14:41:44 | gr-self-healing/2-create-cluster | + yq eval .spec.proxy.haproxy.enabled=false - logger.go:42: 14:41:44 | gr-self-healing/2-create-cluster | + yq eval .spec.proxy.router.enabled=true - logger.go:42: 14:41:44 | gr-self-healing/2-create-cluster | + yq eval '.spec.proxy.router.affinity.antiAffinityTopologyKey="none"' - logger.go:42: 14:41:44 | gr-self-healing/2-create-cluster | + yq eval .spec.proxy.router.size=3 - logger.go:42: 14:41:44 | gr-self-healing/2-create-cluster | + kubectl -n kuttl-test-secure-shiner apply -f - logger.go:42: 14:41:44 | gr-self-healing/2-create-cluster | + yq eval .spec.orchestrator.enabled=false - logger.go:42: 14:41:44 | gr-self-healing/2-create-cluster | + yq eval '.spec.sslSecretName="test-ssl"' - logger.go:42: 14:41:44 | gr-self-healing/2-create-cluster | + yq eval '.spec.upgradeOptions.apply="disabled"' - logger.go:42: 14:41:44 | gr-self-healing/2-create-cluster | ++ printf '.spec.backup.image="%s"' perconalab/percona-server-mysql-operator:main-backup logger.go:42: 14:41:44 | gr-self-healing/2-create-cluster | ++ printf '.spec.toolkit.image="%s"' perconalab/percona-server-mysql-operator:main-toolkit logger.go:42: 14:41:44 | gr-self-healing/2-create-cluster | + yq eval '.spec.backup.image="perconalab/percona-server-mysql-operator:main-backup"' - logger.go:42: 14:41:44 | gr-self-healing/2-create-cluster | + yq eval '.spec.toolkit.image="perconalab/percona-server-mysql-operator:main-toolkit"' - logger.go:42: 14:41:44 | gr-self-healing/2-create-cluster | ++ printf '.spec.orchestrator.image="%s"' perconalab/percona-server-mysql-operator:main-orchestrator logger.go:42: 14:41:44 | gr-self-healing/2-create-cluster | + yq eval '.spec.orchestrator.image="perconalab/percona-server-mysql-operator:main-orchestrator"' - logger.go:42: 14:41:44 | gr-self-healing/2-create-cluster | ++ printf '.spec.pmm.image="%s"' perconalab/pmm-client:dev-latest logger.go:42: 14:41:44 | gr-self-healing/2-create-cluster | + yq eval '.spec.pmm.image="perconalab/pmm-client:dev-latest"' - logger.go:42: 14:41:44 | gr-self-healing/2-create-cluster | ++ printf '.spec.proxy.haproxy.image="%s"' perconalab/percona-server-mysql-operator:main-haproxy logger.go:42: 14:41:44 | gr-self-healing/2-create-cluster | + yq eval '.spec.proxy.haproxy.image="perconalab/percona-server-mysql-operator:main-haproxy"' - logger.go:42: 14:41:44 | gr-self-healing/2-create-cluster | + yq eval '.spec.secretsName="test-secrets"' - logger.go:42: 14:41:44 | gr-self-healing/2-create-cluster | + yq eval '.spec.mysql.clusterType="async"' - logger.go:42: 14:41:44 | gr-self-healing/2-create-cluster | ++ printf '.metadata.name="%s"' gr-self-healing logger.go:42: 14:41:44 | gr-self-healing/2-create-cluster | + yq eval '.metadata.name="gr-self-healing"' /mnt/jenkins/workspace/cloud-ps-operator_PR-424/deploy/cr.yaml logger.go:42: 14:41:44 | gr-self-healing/2-create-cluster | ++ printf '.spec.initImage="%s"' perconalab/percona-server-mysql-operator:PR-424-70568ae logger.go:42: 14:41:44 | gr-self-healing/2-create-cluster | + yq eval '.spec.initImage="perconalab/percona-server-mysql-operator:PR-424-70568ae"' - logger.go:42: 14:41:44 | gr-self-healing/2-create-cluster | ++ printf '.spec.mysql.image="%s"' perconalab/percona-server-mysql-operator:main-psmysql logger.go:42: 14:41:44 | gr-self-healing/2-create-cluster | + yq eval '.spec.mysql.image="perconalab/percona-server-mysql-operator:main-psmysql"' - logger.go:42: 14:41:46 | gr-self-healing/2-create-cluster | perconaservermysql.ps.percona.com/gr-self-healing created logger.go:42: 14:45:49 | gr-self-healing/2-create-cluster | test step completed 2-create-cluster logger.go:42: 14:45:49 | gr-self-healing/3-write-data | starting test step 3-write-data logger.go:42: 14:45:49 | gr-self-healing/3-write-data | running command: [sh -c set -o errexit set -o xtrace source ../../functions run_mysql \ "CREATE DATABASE IF NOT EXISTS myDB; CREATE TABLE IF NOT EXISTS myDB.myTable (id int PRIMARY KEY)" \ "-h $(get_mysql_router_service $(get_cluster_name)) -P 6446 -uroot -proot_password" run_mysql \ "INSERT myDB.myTable (id) VALUES (100500)" \ "-h $(get_mysql_router_service $(get_cluster_name)) -P 6446 -uroot -proot_password"] logger.go:42: 14:45:49 | gr-self-healing/3-write-data | + source ../../functions logger.go:42: 14:45:49 | gr-self-healing/3-write-data | +++ realpath ../../.. logger.go:42: 14:45:49 | gr-self-healing/3-write-data | ++ ROOT_REPO=/mnt/jenkins/workspace/cloud-ps-operator_PR-424 logger.go:42: 14:45:49 | gr-self-healing/3-write-data | ++ source /mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests/vars.sh logger.go:42: 14:45:49 | gr-self-healing/3-write-data | +++ export ROOT_REPO=/mnt/jenkins/workspace/cloud-ps-operator_PR-424 logger.go:42: 14:45:49 | gr-self-healing/3-write-data | +++ ROOT_REPO=/mnt/jenkins/workspace/cloud-ps-operator_PR-424 logger.go:42: 14:45:49 | gr-self-healing/3-write-data | +++ export DEPLOY_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/deploy logger.go:42: 14:45:49 | gr-self-healing/3-write-data | +++ DEPLOY_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/deploy logger.go:42: 14:45:49 | gr-self-healing/3-write-data | +++ export TESTS_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests logger.go:42: 14:45:49 | gr-self-healing/3-write-data | +++ TESTS_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests logger.go:42: 14:45:49 | gr-self-healing/3-write-data | +++ export TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests/conf logger.go:42: 14:45:49 | gr-self-healing/3-write-data | +++ TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests/conf logger.go:42: 14:45:49 | gr-self-healing/3-write-data | ++++ mktemp -d logger.go:42: 14:45:49 | gr-self-healing/3-write-data | +++ export TEMP_DIR=/tmp/tmp.BUQjXqDOcT logger.go:42: 14:45:49 | gr-self-healing/3-write-data | +++ TEMP_DIR=/tmp/tmp.BUQjXqDOcT logger.go:42: 14:45:49 | gr-self-healing/3-write-data | ++++ git rev-parse --abbrev-ref HEAD logger.go:42: 14:45:49 | gr-self-healing/3-write-data | +++ export GIT_BRANCH=PR-424 logger.go:42: 14:45:49 | gr-self-healing/3-write-data | +++ GIT_BRANCH=PR-424 logger.go:42: 14:45:49 | gr-self-healing/3-write-data | +++ export VERSION=PR-424-70568ae logger.go:42: 14:45:49 | gr-self-healing/3-write-data | +++ VERSION=PR-424-70568ae logger.go:42: 14:45:49 | gr-self-healing/3-write-data | +++ export IMAGE=perconalab/percona-server-mysql-operator:PR-424-70568ae logger.go:42: 14:45:49 | gr-self-healing/3-write-data | +++ IMAGE=perconalab/percona-server-mysql-operator:PR-424-70568ae logger.go:42: 14:45:49 | gr-self-healing/3-write-data | +++ export IMAGE_MYSQL=perconalab/percona-server-mysql-operator:main-psmysql logger.go:42: 14:45:49 | gr-self-healing/3-write-data | +++ IMAGE_MYSQL=perconalab/percona-server-mysql-operator:main-psmysql logger.go:42: 14:45:49 | gr-self-healing/3-write-data | +++ export IMAGE_BACKUP=perconalab/percona-server-mysql-operator:main-backup logger.go:42: 14:45:49 | gr-self-healing/3-write-data | +++ IMAGE_BACKUP=perconalab/percona-server-mysql-operator:main-backup logger.go:42: 14:45:49 | gr-self-healing/3-write-data | +++ export IMAGE_ORCHESTRATOR=perconalab/percona-server-mysql-operator:main-orchestrator logger.go:42: 14:45:49 | gr-self-healing/3-write-data | +++ IMAGE_ORCHESTRATOR=perconalab/percona-server-mysql-operator:main-orchestrator logger.go:42: 14:45:49 | gr-self-healing/3-write-data | +++ export IMAGE_ROUTER=perconalab/percona-server-mysql-operator:main-router logger.go:42: 14:45:49 | gr-self-healing/3-write-data | +++ IMAGE_ROUTER=perconalab/percona-server-mysql-operator:main-router logger.go:42: 14:45:49 | gr-self-healing/3-write-data | +++ export IMAGE_TOOLKIT=perconalab/percona-server-mysql-operator:main-toolkit logger.go:42: 14:45:49 | gr-self-healing/3-write-data | +++ IMAGE_TOOLKIT=perconalab/percona-server-mysql-operator:main-toolkit logger.go:42: 14:45:49 | gr-self-healing/3-write-data | +++ export IMAGE_HAPROXY=perconalab/percona-server-mysql-operator:main-haproxy logger.go:42: 14:45:49 | gr-self-healing/3-write-data | +++ IMAGE_HAPROXY=perconalab/percona-server-mysql-operator:main-haproxy logger.go:42: 14:45:49 | gr-self-healing/3-write-data | +++ export IMAGE_PMM=perconalab/pmm-client:dev-latest logger.go:42: 14:45:49 | gr-self-healing/3-write-data | +++ IMAGE_PMM=perconalab/pmm-client:dev-latest logger.go:42: 14:45:49 | gr-self-healing/3-write-data | +++ export PMM_SERVER_VERSION=9.9.9 logger.go:42: 14:45:49 | gr-self-healing/3-write-data | +++ PMM_SERVER_VERSION=9.9.9 logger.go:42: 14:45:49 | gr-self-healing/3-write-data | +++ export IMAGE_PMM_SERVER_REPO=perconalab/pmm-server logger.go:42: 14:45:49 | gr-self-healing/3-write-data | +++ IMAGE_PMM_SERVER_REPO=perconalab/pmm-server logger.go:42: 14:45:49 | gr-self-healing/3-write-data | +++ export IMAGE_PMM_SERVER_TAG=dev-latest logger.go:42: 14:45:49 | gr-self-healing/3-write-data | +++ IMAGE_PMM_SERVER_TAG=dev-latest logger.go:42: 14:45:49 | gr-self-healing/3-write-data | ++++ which gdate logger.go:42: 14:45:49 | gr-self-healing/3-write-data | which: no gdate in (/mnt/jenkins/workspace/cloud-ps-operator_PR-424/bin/:/home/ec2-user/google-cloud-sdk/bin:/home/ec2-user/.krew/bin:/usr/local/bin:/usr/bin) logger.go:42: 14:45:49 | gr-self-healing/3-write-data | ++++ which date logger.go:42: 14:45:49 | gr-self-healing/3-write-data | +++ date=/usr/bin/date logger.go:42: 14:45:49 | gr-self-healing/3-write-data | +++ command -v oc logger.go:42: 14:45:49 | gr-self-healing/3-write-data | +++ oc get projects logger.go:42: 14:45:55 | gr-self-healing/3-write-data | error: the server doesn't have a resource type "projects" logger.go:42: 14:45:55 | gr-self-healing/3-write-data | +++ kubectl get nodes logger.go:42: 14:45:55 | gr-self-healing/3-write-data | +++ grep '^minikube' logger.go:42: 14:45:55 | gr-self-healing/3-write-data | ++++ pwd logger.go:42: 14:45:55 | gr-self-healing/3-write-data | +++ basename /mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests/tests/gr-self-healing logger.go:42: 14:45:55 | gr-self-healing/3-write-data | ++ test_name=gr-self-healing logger.go:42: 14:45:55 | gr-self-healing/3-write-data | +++ get_cluster_name logger.go:42: 14:45:55 | gr-self-healing/3-write-data | +++ kubectl -n kuttl-test-secure-shiner get ps -o 'jsonpath={.items[0].metadata.name}' logger.go:42: 14:45:56 | gr-self-healing/3-write-data | ++ get_mysql_router_service gr-self-healing logger.go:42: 14:45:56 | gr-self-healing/3-write-data | ++ local cluster=gr-self-healing logger.go:42: 14:45:56 | gr-self-healing/3-write-data | ++ echo gr-self-healing-router logger.go:42: 14:45:56 | gr-self-healing/3-write-data | + run_mysql 'CREATE DATABASE IF NOT EXISTS myDB; CREATE TABLE IF NOT EXISTS myDB.myTable (id int PRIMARY KEY)' '-h gr-self-healing-router -P 6446 -uroot -proot_password' logger.go:42: 14:45:56 | gr-self-healing/3-write-data | + local 'command=CREATE DATABASE IF NOT EXISTS myDB; CREATE TABLE IF NOT EXISTS myDB.myTable (id int PRIMARY KEY)' logger.go:42: 14:45:56 | gr-self-healing/3-write-data | + local 'uri=-h gr-self-healing-router -P 6446 -uroot -proot_password' logger.go:42: 14:45:56 | gr-self-healing/3-write-data | + local pod= logger.go:42: 14:45:56 | gr-self-healing/3-write-data | ++ get_client_pod logger.go:42: 14:45:56 | gr-self-healing/3-write-data | ++ kubectl -n kuttl-test-secure-shiner get pods --selector=name=mysql-client -o 'jsonpath={.items[].metadata.name}' logger.go:42: 14:45:57 | gr-self-healing/3-write-data | + client_pod=mysql-client logger.go:42: 14:45:57 | gr-self-healing/3-write-data | + wait_pod mysql-client logger.go:42: 14:45:57 | gr-self-healing/3-write-data | + local pod=mysql-client logger.go:42: 14:45:57 | gr-self-healing/3-write-data | + set +o xtrace logger.go:42: 14:45:58 | gr-self-healing/3-write-data | mysql-clienttrue logger.go:42: 14:45:58 | gr-self-healing/3-write-data | + kubectl -n kuttl-test-secure-shiner exec mysql-client -- bash -c 'printf '\''%s\n'\'' "CREATE DATABASE IF NOT EXISTS myDB; CREATE TABLE IF NOT EXISTS myDB.myTable (id int PRIMARY KEY)" | mysql -sN -h gr-self-healing-router -P 6446 -uroot -proot_password' logger.go:42: 14:45:58 | gr-self-healing/3-write-data | + sed -e 's/mysql: //' logger.go:42: 14:45:58 | gr-self-healing/3-write-data | + grep -v 'Using a password on the command line interface can be insecure.' logger.go:42: 14:45:59 | gr-self-healing/3-write-data | + : logger.go:42: 14:45:59 | gr-self-healing/3-write-data | +++ get_cluster_name logger.go:42: 14:45:59 | gr-self-healing/3-write-data | +++ kubectl -n kuttl-test-secure-shiner get ps -o 'jsonpath={.items[0].metadata.name}' logger.go:42: 14:46:00 | gr-self-healing/3-write-data | ++ get_mysql_router_service gr-self-healing logger.go:42: 14:46:00 | gr-self-healing/3-write-data | ++ local cluster=gr-self-healing logger.go:42: 14:46:00 | gr-self-healing/3-write-data | ++ echo gr-self-healing-router logger.go:42: 14:46:00 | gr-self-healing/3-write-data | + run_mysql 'INSERT myDB.myTable (id) VALUES (100500)' '-h gr-self-healing-router -P 6446 -uroot -proot_password' logger.go:42: 14:46:00 | gr-self-healing/3-write-data | + local 'command=INSERT myDB.myTable (id) VALUES (100500)' logger.go:42: 14:46:00 | gr-self-healing/3-write-data | + local 'uri=-h gr-self-healing-router -P 6446 -uroot -proot_password' logger.go:42: 14:46:00 | gr-self-healing/3-write-data | + local pod= logger.go:42: 14:46:00 | gr-self-healing/3-write-data | ++ get_client_pod logger.go:42: 14:46:00 | gr-self-healing/3-write-data | ++ kubectl -n kuttl-test-secure-shiner get pods --selector=name=mysql-client -o 'jsonpath={.items[].metadata.name}' logger.go:42: 14:46:01 | gr-self-healing/3-write-data | + client_pod=mysql-client logger.go:42: 14:46:01 | gr-self-healing/3-write-data | + wait_pod mysql-client logger.go:42: 14:46:01 | gr-self-healing/3-write-data | + local pod=mysql-client logger.go:42: 14:46:01 | gr-self-healing/3-write-data | + set +o xtrace logger.go:42: 14:46:01 | gr-self-healing/3-write-data | mysql-clienttrue logger.go:42: 14:46:01 | gr-self-healing/3-write-data | + sed -e 's/mysql: //' logger.go:42: 14:46:01 | gr-self-healing/3-write-data | + grep -v 'Using a password on the command line interface can be insecure.' logger.go:42: 14:46:01 | gr-self-healing/3-write-data | + kubectl -n kuttl-test-secure-shiner exec mysql-client -- bash -c 'printf '\''%s\n'\'' "INSERT myDB.myTable (id) VALUES (100500)" | mysql -sN -h gr-self-healing-router -P 6446 -uroot -proot_password' logger.go:42: 14:46:03 | gr-self-healing/3-write-data | + : logger.go:42: 14:46:04 | gr-self-healing/3-write-data | test step completed 3-write-data logger.go:42: 14:46:04 | gr-self-healing/4-read-from-primary | starting test step 4-read-from-primary logger.go:42: 14:46:04 | gr-self-healing/4-read-from-primary | running command: [sh -c set -o errexit set -o xtrace source ../../functions data=$(run_mysql "SELECT * FROM myDB.myTable" "-h $(get_mysql_router_service $(get_cluster_name)) -P 6446 -uroot -proot_password") kubectl create configmap -n "${NAMESPACE}" 04-read-from-primary --from-literal=data="${data}"] logger.go:42: 14:46:04 | gr-self-healing/4-read-from-primary | + source ../../functions logger.go:42: 14:46:04 | gr-self-healing/4-read-from-primary | +++ realpath ../../.. logger.go:42: 14:46:04 | gr-self-healing/4-read-from-primary | ++ ROOT_REPO=/mnt/jenkins/workspace/cloud-ps-operator_PR-424 logger.go:42: 14:46:04 | gr-self-healing/4-read-from-primary | ++ source /mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests/vars.sh logger.go:42: 14:46:04 | gr-self-healing/4-read-from-primary | +++ export ROOT_REPO=/mnt/jenkins/workspace/cloud-ps-operator_PR-424 logger.go:42: 14:46:04 | gr-self-healing/4-read-from-primary | +++ ROOT_REPO=/mnt/jenkins/workspace/cloud-ps-operator_PR-424 logger.go:42: 14:46:04 | gr-self-healing/4-read-from-primary | +++ export DEPLOY_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/deploy logger.go:42: 14:46:04 | gr-self-healing/4-read-from-primary | +++ DEPLOY_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/deploy logger.go:42: 14:46:04 | gr-self-healing/4-read-from-primary | +++ export TESTS_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests logger.go:42: 14:46:04 | gr-self-healing/4-read-from-primary | +++ TESTS_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests logger.go:42: 14:46:04 | gr-self-healing/4-read-from-primary | +++ export TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests/conf logger.go:42: 14:46:04 | gr-self-healing/4-read-from-primary | +++ TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests/conf logger.go:42: 14:46:04 | gr-self-healing/4-read-from-primary | ++++ mktemp -d logger.go:42: 14:46:04 | gr-self-healing/4-read-from-primary | +++ export TEMP_DIR=/tmp/tmp.PduXeI22I9 logger.go:42: 14:46:04 | gr-self-healing/4-read-from-primary | +++ TEMP_DIR=/tmp/tmp.PduXeI22I9 logger.go:42: 14:46:04 | gr-self-healing/4-read-from-primary | ++++ git rev-parse --abbrev-ref HEAD logger.go:42: 14:46:04 | gr-self-healing/4-read-from-primary | +++ export GIT_BRANCH=PR-424 logger.go:42: 14:46:04 | gr-self-healing/4-read-from-primary | +++ GIT_BRANCH=PR-424 logger.go:42: 14:46:04 | gr-self-healing/4-read-from-primary | +++ export VERSION=PR-424-70568ae logger.go:42: 14:46:04 | gr-self-healing/4-read-from-primary | +++ VERSION=PR-424-70568ae logger.go:42: 14:46:04 | gr-self-healing/4-read-from-primary | +++ export IMAGE=perconalab/percona-server-mysql-operator:PR-424-70568ae logger.go:42: 14:46:04 | gr-self-healing/4-read-from-primary | +++ IMAGE=perconalab/percona-server-mysql-operator:PR-424-70568ae logger.go:42: 14:46:04 | gr-self-healing/4-read-from-primary | +++ export IMAGE_MYSQL=perconalab/percona-server-mysql-operator:main-psmysql logger.go:42: 14:46:04 | gr-self-healing/4-read-from-primary | +++ IMAGE_MYSQL=perconalab/percona-server-mysql-operator:main-psmysql logger.go:42: 14:46:04 | gr-self-healing/4-read-from-primary | +++ export IMAGE_BACKUP=perconalab/percona-server-mysql-operator:main-backup logger.go:42: 14:46:04 | gr-self-healing/4-read-from-primary | +++ IMAGE_BACKUP=perconalab/percona-server-mysql-operator:main-backup logger.go:42: 14:46:04 | gr-self-healing/4-read-from-primary | +++ export IMAGE_ORCHESTRATOR=perconalab/percona-server-mysql-operator:main-orchestrator logger.go:42: 14:46:04 | gr-self-healing/4-read-from-primary | +++ IMAGE_ORCHESTRATOR=perconalab/percona-server-mysql-operator:main-orchestrator logger.go:42: 14:46:04 | gr-self-healing/4-read-from-primary | +++ export IMAGE_ROUTER=perconalab/percona-server-mysql-operator:main-router logger.go:42: 14:46:04 | gr-self-healing/4-read-from-primary | +++ IMAGE_ROUTER=perconalab/percona-server-mysql-operator:main-router logger.go:42: 14:46:04 | gr-self-healing/4-read-from-primary | +++ export IMAGE_TOOLKIT=perconalab/percona-server-mysql-operator:main-toolkit logger.go:42: 14:46:04 | gr-self-healing/4-read-from-primary | +++ IMAGE_TOOLKIT=perconalab/percona-server-mysql-operator:main-toolkit logger.go:42: 14:46:04 | gr-self-healing/4-read-from-primary | +++ export IMAGE_HAPROXY=perconalab/percona-server-mysql-operator:main-haproxy logger.go:42: 14:46:04 | gr-self-healing/4-read-from-primary | +++ IMAGE_HAPROXY=perconalab/percona-server-mysql-operator:main-haproxy logger.go:42: 14:46:04 | gr-self-healing/4-read-from-primary | +++ export IMAGE_PMM=perconalab/pmm-client:dev-latest logger.go:42: 14:46:04 | gr-self-healing/4-read-from-primary | +++ IMAGE_PMM=perconalab/pmm-client:dev-latest logger.go:42: 14:46:04 | gr-self-healing/4-read-from-primary | +++ export PMM_SERVER_VERSION=9.9.9 logger.go:42: 14:46:04 | gr-self-healing/4-read-from-primary | +++ PMM_SERVER_VERSION=9.9.9 logger.go:42: 14:46:04 | gr-self-healing/4-read-from-primary | +++ export IMAGE_PMM_SERVER_REPO=perconalab/pmm-server logger.go:42: 14:46:04 | gr-self-healing/4-read-from-primary | +++ IMAGE_PMM_SERVER_REPO=perconalab/pmm-server logger.go:42: 14:46:04 | gr-self-healing/4-read-from-primary | +++ export IMAGE_PMM_SERVER_TAG=dev-latest logger.go:42: 14:46:04 | gr-self-healing/4-read-from-primary | +++ IMAGE_PMM_SERVER_TAG=dev-latest logger.go:42: 14:46:04 | gr-self-healing/4-read-from-primary | ++++ which gdate logger.go:42: 14:46:04 | gr-self-healing/4-read-from-primary | which: no gdate in (/mnt/jenkins/workspace/cloud-ps-operator_PR-424/bin/:/home/ec2-user/google-cloud-sdk/bin:/home/ec2-user/.krew/bin:/usr/local/bin:/usr/bin) logger.go:42: 14:46:04 | gr-self-healing/4-read-from-primary | ++++ which date logger.go:42: 14:46:04 | gr-self-healing/4-read-from-primary | +++ date=/usr/bin/date logger.go:42: 14:46:04 | gr-self-healing/4-read-from-primary | +++ command -v oc logger.go:42: 14:46:04 | gr-self-healing/4-read-from-primary | +++ oc get projects logger.go:42: 14:46:09 | gr-self-healing/4-read-from-primary | error: the server doesn't have a resource type "projects" logger.go:42: 14:46:09 | gr-self-healing/4-read-from-primary | +++ kubectl get nodes logger.go:42: 14:46:09 | gr-self-healing/4-read-from-primary | +++ grep '^minikube' logger.go:42: 14:46:10 | gr-self-healing/4-read-from-primary | ++++ pwd logger.go:42: 14:46:10 | gr-self-healing/4-read-from-primary | +++ basename /mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests/tests/gr-self-healing logger.go:42: 14:46:10 | gr-self-healing/4-read-from-primary | ++ test_name=gr-self-healing logger.go:42: 14:46:10 | gr-self-healing/4-read-from-primary | ++++ get_cluster_name logger.go:42: 14:46:10 | gr-self-healing/4-read-from-primary | ++++ kubectl -n kuttl-test-secure-shiner get ps -o 'jsonpath={.items[0].metadata.name}' logger.go:42: 14:46:11 | gr-self-healing/4-read-from-primary | +++ get_mysql_router_service gr-self-healing logger.go:42: 14:46:11 | gr-self-healing/4-read-from-primary | +++ local cluster=gr-self-healing logger.go:42: 14:46:11 | gr-self-healing/4-read-from-primary | +++ echo gr-self-healing-router logger.go:42: 14:46:11 | gr-self-healing/4-read-from-primary | ++ run_mysql 'SELECT * FROM myDB.myTable' '-h gr-self-healing-router -P 6446 -uroot -proot_password' logger.go:42: 14:46:11 | gr-self-healing/4-read-from-primary | ++ local 'command=SELECT * FROM myDB.myTable' logger.go:42: 14:46:11 | gr-self-healing/4-read-from-primary | ++ local 'uri=-h gr-self-healing-router -P 6446 -uroot -proot_password' logger.go:42: 14:46:11 | gr-self-healing/4-read-from-primary | ++ local pod= logger.go:42: 14:46:11 | gr-self-healing/4-read-from-primary | +++ get_client_pod logger.go:42: 14:46:11 | gr-self-healing/4-read-from-primary | +++ kubectl -n kuttl-test-secure-shiner get pods --selector=name=mysql-client -o 'jsonpath={.items[].metadata.name}' logger.go:42: 14:46:12 | gr-self-healing/4-read-from-primary | ++ client_pod=mysql-client logger.go:42: 14:46:12 | gr-self-healing/4-read-from-primary | ++ wait_pod mysql-client logger.go:42: 14:46:12 | gr-self-healing/4-read-from-primary | ++ local pod=mysql-client logger.go:42: 14:46:12 | gr-self-healing/4-read-from-primary | ++ set +o xtrace logger.go:42: 14:46:13 | gr-self-healing/4-read-from-primary | mysql-clienttrue logger.go:42: 14:46:13 | gr-self-healing/4-read-from-primary | ++ kubectl -n kuttl-test-secure-shiner exec mysql-client -- bash -c 'printf '\''%s\n'\'' "SELECT * FROM myDB.myTable" | mysql -sN -h gr-self-healing-router -P 6446 -uroot -proot_password' logger.go:42: 14:46:13 | gr-self-healing/4-read-from-primary | ++ sed -e 's/mysql: //' logger.go:42: 14:46:13 | gr-self-healing/4-read-from-primary | ++ grep -v 'Using a password on the command line interface can be insecure.' logger.go:42: 14:46:14 | gr-self-healing/4-read-from-primary | + data=100500 logger.go:42: 14:46:14 | gr-self-healing/4-read-from-primary | + kubectl create configmap -n kuttl-test-secure-shiner 04-read-from-primary --from-literal=data=100500 logger.go:42: 14:46:15 | gr-self-healing/4-read-from-primary | configmap/04-read-from-primary created logger.go:42: 14:46:16 | gr-self-healing/4-read-from-primary | test step completed 4-read-from-primary logger.go:42: 14:46:16 | gr-self-healing/5-kill-primary | starting test step 5-kill-primary logger.go:42: 14:46:16 | gr-self-healing/5-kill-primary | running command: [sh -c set -o errexit set -o xtrace source ../../functions init_pod="$(get_primary_from_group_replication)" kill_pods "${NAMESPACE}" "pod" "$init_pod" "" "primary" sleep 10 # wait a bit for pod to be killed if [ "$init_pod" == "$(get_primary_from_group_replication)" ]; then echo "primary pod was not killed! something went wrong." exit 1 fi] logger.go:42: 14:46:16 | gr-self-healing/5-kill-primary | + source ../../functions logger.go:42: 14:46:16 | gr-self-healing/5-kill-primary | +++ realpath ../../.. logger.go:42: 14:46:16 | gr-self-healing/5-kill-primary | ++ ROOT_REPO=/mnt/jenkins/workspace/cloud-ps-operator_PR-424 logger.go:42: 14:46:16 | gr-self-healing/5-kill-primary | ++ source /mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests/vars.sh logger.go:42: 14:46:16 | gr-self-healing/5-kill-primary | +++ export ROOT_REPO=/mnt/jenkins/workspace/cloud-ps-operator_PR-424 logger.go:42: 14:46:16 | gr-self-healing/5-kill-primary | +++ ROOT_REPO=/mnt/jenkins/workspace/cloud-ps-operator_PR-424 logger.go:42: 14:46:16 | gr-self-healing/5-kill-primary | +++ export DEPLOY_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/deploy logger.go:42: 14:46:16 | gr-self-healing/5-kill-primary | +++ DEPLOY_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/deploy logger.go:42: 14:46:16 | gr-self-healing/5-kill-primary | +++ export TESTS_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests logger.go:42: 14:46:16 | gr-self-healing/5-kill-primary | +++ TESTS_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests logger.go:42: 14:46:16 | gr-self-healing/5-kill-primary | +++ export TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests/conf logger.go:42: 14:46:16 | gr-self-healing/5-kill-primary | +++ TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests/conf logger.go:42: 14:46:16 | gr-self-healing/5-kill-primary | ++++ mktemp -d logger.go:42: 14:46:16 | gr-self-healing/5-kill-primary | +++ export TEMP_DIR=/tmp/tmp.T8jtCfddEw logger.go:42: 14:46:16 | gr-self-healing/5-kill-primary | +++ TEMP_DIR=/tmp/tmp.T8jtCfddEw logger.go:42: 14:46:16 | gr-self-healing/5-kill-primary | ++++ git rev-parse --abbrev-ref HEAD logger.go:42: 14:46:16 | gr-self-healing/5-kill-primary | +++ export GIT_BRANCH=PR-424 logger.go:42: 14:46:16 | gr-self-healing/5-kill-primary | +++ GIT_BRANCH=PR-424 logger.go:42: 14:46:16 | gr-self-healing/5-kill-primary | +++ export VERSION=PR-424-70568ae logger.go:42: 14:46:16 | gr-self-healing/5-kill-primary | +++ VERSION=PR-424-70568ae logger.go:42: 14:46:16 | gr-self-healing/5-kill-primary | +++ export IMAGE=perconalab/percona-server-mysql-operator:PR-424-70568ae logger.go:42: 14:46:16 | gr-self-healing/5-kill-primary | +++ IMAGE=perconalab/percona-server-mysql-operator:PR-424-70568ae logger.go:42: 14:46:16 | gr-self-healing/5-kill-primary | +++ export IMAGE_MYSQL=perconalab/percona-server-mysql-operator:main-psmysql logger.go:42: 14:46:16 | gr-self-healing/5-kill-primary | +++ IMAGE_MYSQL=perconalab/percona-server-mysql-operator:main-psmysql logger.go:42: 14:46:16 | gr-self-healing/5-kill-primary | +++ export IMAGE_BACKUP=perconalab/percona-server-mysql-operator:main-backup logger.go:42: 14:46:16 | gr-self-healing/5-kill-primary | +++ IMAGE_BACKUP=perconalab/percona-server-mysql-operator:main-backup logger.go:42: 14:46:16 | gr-self-healing/5-kill-primary | +++ export IMAGE_ORCHESTRATOR=perconalab/percona-server-mysql-operator:main-orchestrator logger.go:42: 14:46:16 | gr-self-healing/5-kill-primary | +++ IMAGE_ORCHESTRATOR=perconalab/percona-server-mysql-operator:main-orchestrator logger.go:42: 14:46:16 | gr-self-healing/5-kill-primary | +++ export IMAGE_ROUTER=perconalab/percona-server-mysql-operator:main-router logger.go:42: 14:46:16 | gr-self-healing/5-kill-primary | +++ IMAGE_ROUTER=perconalab/percona-server-mysql-operator:main-router logger.go:42: 14:46:16 | gr-self-healing/5-kill-primary | +++ export IMAGE_TOOLKIT=perconalab/percona-server-mysql-operator:main-toolkit logger.go:42: 14:46:16 | gr-self-healing/5-kill-primary | +++ IMAGE_TOOLKIT=perconalab/percona-server-mysql-operator:main-toolkit logger.go:42: 14:46:16 | gr-self-healing/5-kill-primary | +++ export IMAGE_HAPROXY=perconalab/percona-server-mysql-operator:main-haproxy logger.go:42: 14:46:16 | gr-self-healing/5-kill-primary | +++ IMAGE_HAPROXY=perconalab/percona-server-mysql-operator:main-haproxy logger.go:42: 14:46:16 | gr-self-healing/5-kill-primary | +++ export IMAGE_PMM=perconalab/pmm-client:dev-latest logger.go:42: 14:46:16 | gr-self-healing/5-kill-primary | +++ IMAGE_PMM=perconalab/pmm-client:dev-latest logger.go:42: 14:46:16 | gr-self-healing/5-kill-primary | +++ export PMM_SERVER_VERSION=9.9.9 logger.go:42: 14:46:16 | gr-self-healing/5-kill-primary | +++ PMM_SERVER_VERSION=9.9.9 logger.go:42: 14:46:16 | gr-self-healing/5-kill-primary | +++ export IMAGE_PMM_SERVER_REPO=perconalab/pmm-server logger.go:42: 14:46:16 | gr-self-healing/5-kill-primary | +++ IMAGE_PMM_SERVER_REPO=perconalab/pmm-server logger.go:42: 14:46:16 | gr-self-healing/5-kill-primary | +++ export IMAGE_PMM_SERVER_TAG=dev-latest logger.go:42: 14:46:16 | gr-self-healing/5-kill-primary | +++ IMAGE_PMM_SERVER_TAG=dev-latest logger.go:42: 14:46:16 | gr-self-healing/5-kill-primary | ++++ which gdate logger.go:42: 14:46:16 | gr-self-healing/5-kill-primary | which: no gdate in (/mnt/jenkins/workspace/cloud-ps-operator_PR-424/bin/:/home/ec2-user/google-cloud-sdk/bin:/home/ec2-user/.krew/bin:/usr/local/bin:/usr/bin) logger.go:42: 14:46:16 | gr-self-healing/5-kill-primary | ++++ which date logger.go:42: 14:46:16 | gr-self-healing/5-kill-primary | +++ date=/usr/bin/date logger.go:42: 14:46:16 | gr-self-healing/5-kill-primary | +++ command -v oc logger.go:42: 14:46:16 | gr-self-healing/5-kill-primary | +++ oc get projects logger.go:42: 14:46:22 | gr-self-healing/5-kill-primary | error: the server doesn't have a resource type "projects" logger.go:42: 14:46:22 | gr-self-healing/5-kill-primary | +++ kubectl get nodes logger.go:42: 14:46:22 | gr-self-healing/5-kill-primary | +++ grep '^minikube' logger.go:42: 14:46:22 | gr-self-healing/5-kill-primary | ++++ pwd logger.go:42: 14:46:22 | gr-self-healing/5-kill-primary | +++ basename /mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests/tests/gr-self-healing logger.go:42: 14:46:22 | gr-self-healing/5-kill-primary | ++ test_name=gr-self-healing logger.go:42: 14:46:22 | gr-self-healing/5-kill-primary | ++ get_primary_from_group_replication logger.go:42: 14:46:22 | gr-self-healing/5-kill-primary | ++ cut -d. -f1 logger.go:42: 14:46:22 | gr-self-healing/5-kill-primary | ++++ get_cluster_name logger.go:42: 14:46:22 | gr-self-healing/5-kill-primary | ++++ kubectl -n kuttl-test-secure-shiner get ps -o 'jsonpath={.items[0].metadata.name}' logger.go:42: 14:46:23 | gr-self-healing/5-kill-primary | +++ get_mysql_router_service gr-self-healing logger.go:42: 14:46:23 | gr-self-healing/5-kill-primary | +++ local cluster=gr-self-healing logger.go:42: 14:46:23 | gr-self-healing/5-kill-primary | +++ echo gr-self-healing-router logger.go:42: 14:46:23 | gr-self-healing/5-kill-primary | ++ run_mysql 'SELECT MEMBER_HOST FROM performance_schema.replication_group_members where MEMBER_ROLE='\''PRIMARY'\'';' '-h gr-self-healing-router -P 6446 -uroot -proot_password' logger.go:42: 14:46:23 | gr-self-healing/5-kill-primary | ++ local 'command=SELECT MEMBER_HOST FROM performance_schema.replication_group_members where MEMBER_ROLE='\''PRIMARY'\'';' logger.go:42: 14:46:23 | gr-self-healing/5-kill-primary | ++ local 'uri=-h gr-self-healing-router -P 6446 -uroot -proot_password' logger.go:42: 14:46:23 | gr-self-healing/5-kill-primary | ++ local pod= logger.go:42: 14:46:23 | gr-self-healing/5-kill-primary | +++ get_client_pod logger.go:42: 14:46:23 | gr-self-healing/5-kill-primary | +++ kubectl -n kuttl-test-secure-shiner get pods --selector=name=mysql-client -o 'jsonpath={.items[].metadata.name}' logger.go:42: 14:46:24 | gr-self-healing/5-kill-primary | ++ client_pod=mysql-client logger.go:42: 14:46:24 | gr-self-healing/5-kill-primary | ++ wait_pod mysql-client logger.go:42: 14:46:24 | gr-self-healing/5-kill-primary | ++ local pod=mysql-client logger.go:42: 14:46:24 | gr-self-healing/5-kill-primary | ++ set +o xtrace logger.go:42: 14:46:25 | gr-self-healing/5-kill-primary | mysql-clienttrue logger.go:42: 14:46:25 | gr-self-healing/5-kill-primary | ++ kubectl -n kuttl-test-secure-shiner exec mysql-client -- bash -c 'printf '\''%s\n'\'' "SELECT MEMBER_HOST FROM performance_schema.replication_group_members where MEMBER_ROLE='\''PRIMARY'\'';" | mysql -sN -h gr-self-healing-router -P 6446 -uroot -proot_password' logger.go:42: 14:46:25 | gr-self-healing/5-kill-primary | ++ sed -e 's/mysql: //' logger.go:42: 14:46:25 | gr-self-healing/5-kill-primary | ++ grep -v 'Using a password on the command line interface can be insecure.' logger.go:42: 14:46:26 | gr-self-healing/5-kill-primary | + init_pod=gr-self-healing-mysql-0 logger.go:42: 14:46:26 | gr-self-healing/5-kill-primary | + kill_pods kuttl-test-secure-shiner pod gr-self-healing-mysql-0 '' primary logger.go:42: 14:46:26 | gr-self-healing/5-kill-primary | + local ns=kuttl-test-secure-shiner logger.go:42: 14:46:26 | gr-self-healing/5-kill-primary | + local selector=pod logger.go:42: 14:46:26 | gr-self-healing/5-kill-primary | + local pod_label=gr-self-healing-mysql-0 logger.go:42: 14:46:26 | gr-self-healing/5-kill-primary | + local label_value= logger.go:42: 14:46:26 | gr-self-healing/5-kill-primary | + local chaos_suffix=primary logger.go:42: 14:46:26 | gr-self-healing/5-kill-primary | + '[' pod == pod ']' logger.go:42: 14:46:26 | gr-self-healing/5-kill-primary | + yq eval ' logger.go:42: 14:46:26 | gr-self-healing/5-kill-primary | .metadata.name = "chaos-pod-kill-primary" | logger.go:42: 14:46:26 | gr-self-healing/5-kill-primary | del(.spec.selector.pods.test-namespace) | logger.go:42: 14:46:26 | gr-self-healing/5-kill-primary | .spec.selector.pods.kuttl-test-secure-shiner[0] = "gr-self-healing-mysql-0"' /mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests/conf/chaos-pod-kill.yml logger.go:42: 14:46:26 | gr-self-healing/5-kill-primary | + kubectl apply --namespace kuttl-test-secure-shiner -f - logger.go:42: 14:46:28 | gr-self-healing/5-kill-primary | podchaos.chaos-mesh.org/chaos-pod-kill-primary created logger.go:42: 14:46:28 | gr-self-healing/5-kill-primary | + sleep 5 logger.go:42: 14:46:33 | gr-self-healing/5-kill-primary | + sleep 10 logger.go:42: 14:46:43 | gr-self-healing/5-kill-primary | ++ get_primary_from_group_replication logger.go:42: 14:46:43 | gr-self-healing/5-kill-primary | ++ cut -d. -f1 logger.go:42: 14:46:43 | gr-self-healing/5-kill-primary | ++++ get_cluster_name logger.go:42: 14:46:43 | gr-self-healing/5-kill-primary | ++++ kubectl -n kuttl-test-secure-shiner get ps -o 'jsonpath={.items[0].metadata.name}' logger.go:42: 14:46:43 | gr-self-healing/5-kill-primary | +++ get_mysql_router_service gr-self-healing logger.go:42: 14:46:43 | gr-self-healing/5-kill-primary | +++ local cluster=gr-self-healing logger.go:42: 14:46:43 | gr-self-healing/5-kill-primary | +++ echo gr-self-healing-router logger.go:42: 14:46:43 | gr-self-healing/5-kill-primary | ++ run_mysql 'SELECT MEMBER_HOST FROM performance_schema.replication_group_members where MEMBER_ROLE='\''PRIMARY'\'';' '-h gr-self-healing-router -P 6446 -uroot -proot_password' logger.go:42: 14:46:43 | gr-self-healing/5-kill-primary | ++ local 'command=SELECT MEMBER_HOST FROM performance_schema.replication_group_members where MEMBER_ROLE='\''PRIMARY'\'';' logger.go:42: 14:46:43 | gr-self-healing/5-kill-primary | ++ local 'uri=-h gr-self-healing-router -P 6446 -uroot -proot_password' logger.go:42: 14:46:43 | gr-self-healing/5-kill-primary | ++ local pod= logger.go:42: 14:46:43 | gr-self-healing/5-kill-primary | +++ get_client_pod logger.go:42: 14:46:43 | gr-self-healing/5-kill-primary | +++ kubectl -n kuttl-test-secure-shiner get pods --selector=name=mysql-client -o 'jsonpath={.items[].metadata.name}' logger.go:42: 14:46:44 | gr-self-healing/5-kill-primary | ++ client_pod=mysql-client logger.go:42: 14:46:44 | gr-self-healing/5-kill-primary | ++ wait_pod mysql-client logger.go:42: 14:46:44 | gr-self-healing/5-kill-primary | ++ local pod=mysql-client logger.go:42: 14:46:44 | gr-self-healing/5-kill-primary | ++ set +o xtrace logger.go:42: 14:46:45 | gr-self-healing/5-kill-primary | mysql-clienttrue logger.go:42: 14:46:45 | gr-self-healing/5-kill-primary | ++ sed -e 's/mysql: //' logger.go:42: 14:46:45 | gr-self-healing/5-kill-primary | ++ grep -v 'Using a password on the command line interface can be insecure.' logger.go:42: 14:46:45 | gr-self-healing/5-kill-primary | ++ kubectl -n kuttl-test-secure-shiner exec mysql-client -- bash -c 'printf '\''%s\n'\'' "SELECT MEMBER_HOST FROM performance_schema.replication_group_members where MEMBER_ROLE='\''PRIMARY'\'';" | mysql -sN -h gr-self-healing-router -P 6446 -uroot -proot_password' logger.go:42: 14:46:46 | gr-self-healing/5-kill-primary | + '[' gr-self-healing-mysql-0 == gr-self-healing-mysql-1 ']' logger.go:42: 14:47:12 | gr-self-healing/5-kill-primary | test step completed 5-kill-primary logger.go:42: 14:47:12 | gr-self-healing/6-write-data | starting test step 6-write-data logger.go:42: 14:47:12 | gr-self-healing/6-write-data | running command: [sh -c set -o errexit set -o xtrace source ../../functions run_mysql \ "INSERT myDB.myTable (id) VALUES (100501)" \ "-h $(get_mysql_router_service $(get_cluster_name)) -P 6446 -uroot -proot_password"] logger.go:42: 14:47:12 | gr-self-healing/6-write-data | + source ../../functions logger.go:42: 14:47:12 | gr-self-healing/6-write-data | +++ realpath ../../.. logger.go:42: 14:47:12 | gr-self-healing/6-write-data | ++ ROOT_REPO=/mnt/jenkins/workspace/cloud-ps-operator_PR-424 logger.go:42: 14:47:12 | gr-self-healing/6-write-data | ++ source /mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests/vars.sh logger.go:42: 14:47:12 | gr-self-healing/6-write-data | +++ export ROOT_REPO=/mnt/jenkins/workspace/cloud-ps-operator_PR-424 logger.go:42: 14:47:12 | gr-self-healing/6-write-data | +++ ROOT_REPO=/mnt/jenkins/workspace/cloud-ps-operator_PR-424 logger.go:42: 14:47:12 | gr-self-healing/6-write-data | +++ export DEPLOY_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/deploy logger.go:42: 14:47:12 | gr-self-healing/6-write-data | +++ DEPLOY_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/deploy logger.go:42: 14:47:12 | gr-self-healing/6-write-data | +++ export TESTS_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests logger.go:42: 14:47:12 | gr-self-healing/6-write-data | +++ TESTS_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests logger.go:42: 14:47:12 | gr-self-healing/6-write-data | +++ export TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests/conf logger.go:42: 14:47:12 | gr-self-healing/6-write-data | +++ TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests/conf logger.go:42: 14:47:12 | gr-self-healing/6-write-data | ++++ mktemp -d logger.go:42: 14:47:12 | gr-self-healing/6-write-data | +++ export TEMP_DIR=/tmp/tmp.UE96O548JC logger.go:42: 14:47:12 | gr-self-healing/6-write-data | +++ TEMP_DIR=/tmp/tmp.UE96O548JC logger.go:42: 14:47:12 | gr-self-healing/6-write-data | ++++ git rev-parse --abbrev-ref HEAD logger.go:42: 14:47:12 | gr-self-healing/6-write-data | +++ export GIT_BRANCH=PR-424 logger.go:42: 14:47:12 | gr-self-healing/6-write-data | +++ GIT_BRANCH=PR-424 logger.go:42: 14:47:12 | gr-self-healing/6-write-data | +++ export VERSION=PR-424-70568ae logger.go:42: 14:47:12 | gr-self-healing/6-write-data | +++ VERSION=PR-424-70568ae logger.go:42: 14:47:12 | gr-self-healing/6-write-data | +++ export IMAGE=perconalab/percona-server-mysql-operator:PR-424-70568ae logger.go:42: 14:47:12 | gr-self-healing/6-write-data | +++ IMAGE=perconalab/percona-server-mysql-operator:PR-424-70568ae logger.go:42: 14:47:12 | gr-self-healing/6-write-data | +++ export IMAGE_MYSQL=perconalab/percona-server-mysql-operator:main-psmysql logger.go:42: 14:47:12 | gr-self-healing/6-write-data | +++ IMAGE_MYSQL=perconalab/percona-server-mysql-operator:main-psmysql logger.go:42: 14:47:12 | gr-self-healing/6-write-data | +++ export IMAGE_BACKUP=perconalab/percona-server-mysql-operator:main-backup logger.go:42: 14:47:12 | gr-self-healing/6-write-data | +++ IMAGE_BACKUP=perconalab/percona-server-mysql-operator:main-backup logger.go:42: 14:47:12 | gr-self-healing/6-write-data | +++ export IMAGE_ORCHESTRATOR=perconalab/percona-server-mysql-operator:main-orchestrator logger.go:42: 14:47:12 | gr-self-healing/6-write-data | +++ IMAGE_ORCHESTRATOR=perconalab/percona-server-mysql-operator:main-orchestrator logger.go:42: 14:47:12 | gr-self-healing/6-write-data | +++ export IMAGE_ROUTER=perconalab/percona-server-mysql-operator:main-router logger.go:42: 14:47:12 | gr-self-healing/6-write-data | +++ IMAGE_ROUTER=perconalab/percona-server-mysql-operator:main-router logger.go:42: 14:47:12 | gr-self-healing/6-write-data | +++ export IMAGE_TOOLKIT=perconalab/percona-server-mysql-operator:main-toolkit logger.go:42: 14:47:12 | gr-self-healing/6-write-data | +++ IMAGE_TOOLKIT=perconalab/percona-server-mysql-operator:main-toolkit logger.go:42: 14:47:12 | gr-self-healing/6-write-data | +++ export IMAGE_HAPROXY=perconalab/percona-server-mysql-operator:main-haproxy logger.go:42: 14:47:12 | gr-self-healing/6-write-data | +++ IMAGE_HAPROXY=perconalab/percona-server-mysql-operator:main-haproxy logger.go:42: 14:47:12 | gr-self-healing/6-write-data | +++ export IMAGE_PMM=perconalab/pmm-client:dev-latest logger.go:42: 14:47:12 | gr-self-healing/6-write-data | +++ IMAGE_PMM=perconalab/pmm-client:dev-latest logger.go:42: 14:47:12 | gr-self-healing/6-write-data | +++ export PMM_SERVER_VERSION=9.9.9 logger.go:42: 14:47:12 | gr-self-healing/6-write-data | +++ PMM_SERVER_VERSION=9.9.9 logger.go:42: 14:47:12 | gr-self-healing/6-write-data | +++ export IMAGE_PMM_SERVER_REPO=perconalab/pmm-server logger.go:42: 14:47:12 | gr-self-healing/6-write-data | +++ IMAGE_PMM_SERVER_REPO=perconalab/pmm-server logger.go:42: 14:47:12 | gr-self-healing/6-write-data | +++ export IMAGE_PMM_SERVER_TAG=dev-latest logger.go:42: 14:47:12 | gr-self-healing/6-write-data | +++ IMAGE_PMM_SERVER_TAG=dev-latest logger.go:42: 14:47:12 | gr-self-healing/6-write-data | ++++ which gdate logger.go:42: 14:47:12 | gr-self-healing/6-write-data | which: no gdate in (/mnt/jenkins/workspace/cloud-ps-operator_PR-424/bin/:/home/ec2-user/google-cloud-sdk/bin:/home/ec2-user/.krew/bin:/usr/local/bin:/usr/bin) logger.go:42: 14:47:12 | gr-self-healing/6-write-data | ++++ which date logger.go:42: 14:47:12 | gr-self-healing/6-write-data | +++ date=/usr/bin/date logger.go:42: 14:47:12 | gr-self-healing/6-write-data | +++ command -v oc logger.go:42: 14:47:12 | gr-self-healing/6-write-data | +++ oc get projects logger.go:42: 14:47:17 | gr-self-healing/6-write-data | error: the server doesn't have a resource type "projects" logger.go:42: 14:47:17 | gr-self-healing/6-write-data | +++ grep '^minikube' logger.go:42: 14:47:17 | gr-self-healing/6-write-data | +++ kubectl get nodes logger.go:42: 14:47:18 | gr-self-healing/6-write-data | ++++ pwd logger.go:42: 14:47:18 | gr-self-healing/6-write-data | +++ basename /mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests/tests/gr-self-healing logger.go:42: 14:47:18 | gr-self-healing/6-write-data | ++ test_name=gr-self-healing logger.go:42: 14:47:18 | gr-self-healing/6-write-data | +++ get_cluster_name logger.go:42: 14:47:18 | gr-self-healing/6-write-data | +++ kubectl -n kuttl-test-secure-shiner get ps -o 'jsonpath={.items[0].metadata.name}' logger.go:42: 14:47:19 | gr-self-healing/6-write-data | ++ get_mysql_router_service gr-self-healing logger.go:42: 14:47:19 | gr-self-healing/6-write-data | ++ local cluster=gr-self-healing logger.go:42: 14:47:19 | gr-self-healing/6-write-data | ++ echo gr-self-healing-router logger.go:42: 14:47:19 | gr-self-healing/6-write-data | + run_mysql 'INSERT myDB.myTable (id) VALUES (100501)' '-h gr-self-healing-router -P 6446 -uroot -proot_password' logger.go:42: 14:47:19 | gr-self-healing/6-write-data | + local 'command=INSERT myDB.myTable (id) VALUES (100501)' logger.go:42: 14:47:19 | gr-self-healing/6-write-data | + local 'uri=-h gr-self-healing-router -P 6446 -uroot -proot_password' logger.go:42: 14:47:19 | gr-self-healing/6-write-data | + local pod= logger.go:42: 14:47:19 | gr-self-healing/6-write-data | ++ get_client_pod logger.go:42: 14:47:19 | gr-self-healing/6-write-data | ++ kubectl -n kuttl-test-secure-shiner get pods --selector=name=mysql-client -o 'jsonpath={.items[].metadata.name}' logger.go:42: 14:47:20 | gr-self-healing/6-write-data | + client_pod=mysql-client logger.go:42: 14:47:20 | gr-self-healing/6-write-data | + wait_pod mysql-client logger.go:42: 14:47:20 | gr-self-healing/6-write-data | + local pod=mysql-client logger.go:42: 14:47:20 | gr-self-healing/6-write-data | + set +o xtrace logger.go:42: 14:47:21 | gr-self-healing/6-write-data | mysql-clienttrue logger.go:42: 14:47:21 | gr-self-healing/6-write-data | + kubectl -n kuttl-test-secure-shiner exec mysql-client -- bash -c 'printf '\''%s\n'\'' "INSERT myDB.myTable (id) VALUES (100501)" | mysql -sN -h gr-self-healing-router -P 6446 -uroot -proot_password' logger.go:42: 14:47:21 | gr-self-healing/6-write-data | + sed -e 's/mysql: //' logger.go:42: 14:47:21 | gr-self-healing/6-write-data | + grep -v 'Using a password on the command line interface can be insecure.' logger.go:42: 14:47:22 | gr-self-healing/6-write-data | + : logger.go:42: 14:47:23 | gr-self-healing/6-write-data | test step completed 6-write-data logger.go:42: 14:47:23 | gr-self-healing/7-read-from-replicas | starting test step 7-read-from-replicas logger.go:42: 14:47:23 | gr-self-healing/7-read-from-replicas | running command: [sh -c set -o errexit set -o xtrace source ../../functions for i in 0 1 2; do host=$(get_mysql_headless_fqdn $(get_cluster_name) $i) data=$(run_mysql "SELECT * FROM myDB.myTable" "-h ${host} -uroot -proot_password") kubectl create configmap -n "${NAMESPACE}" 07-read-from-replicas-${i} --from-literal=data="${data}" done] logger.go:42: 14:47:23 | gr-self-healing/7-read-from-replicas | + source ../../functions logger.go:42: 14:47:23 | gr-self-healing/7-read-from-replicas | +++ realpath ../../.. logger.go:42: 14:47:23 | gr-self-healing/7-read-from-replicas | ++ ROOT_REPO=/mnt/jenkins/workspace/cloud-ps-operator_PR-424 logger.go:42: 14:47:23 | gr-self-healing/7-read-from-replicas | ++ source /mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests/vars.sh logger.go:42: 14:47:23 | gr-self-healing/7-read-from-replicas | +++ export ROOT_REPO=/mnt/jenkins/workspace/cloud-ps-operator_PR-424 logger.go:42: 14:47:23 | gr-self-healing/7-read-from-replicas | +++ ROOT_REPO=/mnt/jenkins/workspace/cloud-ps-operator_PR-424 logger.go:42: 14:47:23 | gr-self-healing/7-read-from-replicas | +++ export DEPLOY_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/deploy logger.go:42: 14:47:23 | gr-self-healing/7-read-from-replicas | +++ DEPLOY_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/deploy logger.go:42: 14:47:23 | gr-self-healing/7-read-from-replicas | +++ export TESTS_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests logger.go:42: 14:47:23 | gr-self-healing/7-read-from-replicas | +++ TESTS_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests logger.go:42: 14:47:23 | gr-self-healing/7-read-from-replicas | +++ export TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests/conf logger.go:42: 14:47:23 | gr-self-healing/7-read-from-replicas | +++ TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests/conf logger.go:42: 14:47:23 | gr-self-healing/7-read-from-replicas | ++++ mktemp -d logger.go:42: 14:47:23 | gr-self-healing/7-read-from-replicas | +++ export TEMP_DIR=/tmp/tmp.CFDlOzkrKC logger.go:42: 14:47:23 | gr-self-healing/7-read-from-replicas | +++ TEMP_DIR=/tmp/tmp.CFDlOzkrKC logger.go:42: 14:47:23 | gr-self-healing/7-read-from-replicas | ++++ git rev-parse --abbrev-ref HEAD logger.go:42: 14:47:23 | gr-self-healing/7-read-from-replicas | +++ export GIT_BRANCH=PR-424 logger.go:42: 14:47:23 | gr-self-healing/7-read-from-replicas | +++ GIT_BRANCH=PR-424 logger.go:42: 14:47:23 | gr-self-healing/7-read-from-replicas | +++ export VERSION=PR-424-70568ae logger.go:42: 14:47:23 | gr-self-healing/7-read-from-replicas | +++ VERSION=PR-424-70568ae logger.go:42: 14:47:23 | gr-self-healing/7-read-from-replicas | +++ export IMAGE=perconalab/percona-server-mysql-operator:PR-424-70568ae logger.go:42: 14:47:23 | gr-self-healing/7-read-from-replicas | +++ IMAGE=perconalab/percona-server-mysql-operator:PR-424-70568ae logger.go:42: 14:47:23 | gr-self-healing/7-read-from-replicas | +++ export IMAGE_MYSQL=perconalab/percona-server-mysql-operator:main-psmysql logger.go:42: 14:47:23 | gr-self-healing/7-read-from-replicas | +++ IMAGE_MYSQL=perconalab/percona-server-mysql-operator:main-psmysql logger.go:42: 14:47:23 | gr-self-healing/7-read-from-replicas | +++ export IMAGE_BACKUP=perconalab/percona-server-mysql-operator:main-backup logger.go:42: 14:47:23 | gr-self-healing/7-read-from-replicas | +++ IMAGE_BACKUP=perconalab/percona-server-mysql-operator:main-backup logger.go:42: 14:47:23 | gr-self-healing/7-read-from-replicas | +++ export IMAGE_ORCHESTRATOR=perconalab/percona-server-mysql-operator:main-orchestrator logger.go:42: 14:47:23 | gr-self-healing/7-read-from-replicas | +++ IMAGE_ORCHESTRATOR=perconalab/percona-server-mysql-operator:main-orchestrator logger.go:42: 14:47:23 | gr-self-healing/7-read-from-replicas | +++ export IMAGE_ROUTER=perconalab/percona-server-mysql-operator:main-router logger.go:42: 14:47:23 | gr-self-healing/7-read-from-replicas | +++ IMAGE_ROUTER=perconalab/percona-server-mysql-operator:main-router logger.go:42: 14:47:23 | gr-self-healing/7-read-from-replicas | +++ export IMAGE_TOOLKIT=perconalab/percona-server-mysql-operator:main-toolkit logger.go:42: 14:47:23 | gr-self-healing/7-read-from-replicas | +++ IMAGE_TOOLKIT=perconalab/percona-server-mysql-operator:main-toolkit logger.go:42: 14:47:23 | gr-self-healing/7-read-from-replicas | +++ export IMAGE_HAPROXY=perconalab/percona-server-mysql-operator:main-haproxy logger.go:42: 14:47:23 | gr-self-healing/7-read-from-replicas | +++ IMAGE_HAPROXY=perconalab/percona-server-mysql-operator:main-haproxy logger.go:42: 14:47:23 | gr-self-healing/7-read-from-replicas | +++ export IMAGE_PMM=perconalab/pmm-client:dev-latest logger.go:42: 14:47:23 | gr-self-healing/7-read-from-replicas | +++ IMAGE_PMM=perconalab/pmm-client:dev-latest logger.go:42: 14:47:23 | gr-self-healing/7-read-from-replicas | +++ export PMM_SERVER_VERSION=9.9.9 logger.go:42: 14:47:23 | gr-self-healing/7-read-from-replicas | +++ PMM_SERVER_VERSION=9.9.9 logger.go:42: 14:47:23 | gr-self-healing/7-read-from-replicas | +++ export IMAGE_PMM_SERVER_REPO=perconalab/pmm-server logger.go:42: 14:47:23 | gr-self-healing/7-read-from-replicas | +++ IMAGE_PMM_SERVER_REPO=perconalab/pmm-server logger.go:42: 14:47:23 | gr-self-healing/7-read-from-replicas | +++ export IMAGE_PMM_SERVER_TAG=dev-latest logger.go:42: 14:47:23 | gr-self-healing/7-read-from-replicas | +++ IMAGE_PMM_SERVER_TAG=dev-latest logger.go:42: 14:47:23 | gr-self-healing/7-read-from-replicas | ++++ which gdate logger.go:42: 14:47:23 | gr-self-healing/7-read-from-replicas | which: no gdate in (/mnt/jenkins/workspace/cloud-ps-operator_PR-424/bin/:/home/ec2-user/google-cloud-sdk/bin:/home/ec2-user/.krew/bin:/usr/local/bin:/usr/bin) logger.go:42: 14:47:23 | gr-self-healing/7-read-from-replicas | ++++ which date logger.go:42: 14:47:23 | gr-self-healing/7-read-from-replicas | +++ date=/usr/bin/date logger.go:42: 14:47:23 | gr-self-healing/7-read-from-replicas | +++ command -v oc logger.go:42: 14:47:23 | gr-self-healing/7-read-from-replicas | +++ oc get projects logger.go:42: 14:47:29 | gr-self-healing/7-read-from-replicas | error: the server doesn't have a resource type "projects" logger.go:42: 14:47:29 | gr-self-healing/7-read-from-replicas | +++ kubectl get nodes logger.go:42: 14:47:29 | gr-self-healing/7-read-from-replicas | +++ grep '^minikube' logger.go:42: 14:47:29 | gr-self-healing/7-read-from-replicas | ++++ pwd logger.go:42: 14:47:29 | gr-self-healing/7-read-from-replicas | +++ basename /mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests/tests/gr-self-healing logger.go:42: 14:47:29 | gr-self-healing/7-read-from-replicas | ++ test_name=gr-self-healing logger.go:42: 14:47:29 | gr-self-healing/7-read-from-replicas | + for i in 0 1 2 logger.go:42: 14:47:29 | gr-self-healing/7-read-from-replicas | +++ get_cluster_name logger.go:42: 14:47:29 | gr-self-healing/7-read-from-replicas | +++ kubectl -n kuttl-test-secure-shiner get ps -o 'jsonpath={.items[0].metadata.name}' logger.go:42: 14:47:30 | gr-self-healing/7-read-from-replicas | ++ get_mysql_headless_fqdn gr-self-healing 0 logger.go:42: 14:47:30 | gr-self-healing/7-read-from-replicas | ++ local cluster=gr-self-healing logger.go:42: 14:47:30 | gr-self-healing/7-read-from-replicas | ++ local index=0 logger.go:42: 14:47:30 | gr-self-healing/7-read-from-replicas | ++ echo gr-self-healing-mysql-0.gr-self-healing-mysql logger.go:42: 14:47:30 | gr-self-healing/7-read-from-replicas | + host=gr-self-healing-mysql-0.gr-self-healing-mysql logger.go:42: 14:47:30 | gr-self-healing/7-read-from-replicas | ++ run_mysql 'SELECT * FROM myDB.myTable' '-h gr-self-healing-mysql-0.gr-self-healing-mysql -uroot -proot_password' logger.go:42: 14:47:30 | gr-self-healing/7-read-from-replicas | ++ local 'command=SELECT * FROM myDB.myTable' logger.go:42: 14:47:30 | gr-self-healing/7-read-from-replicas | ++ local 'uri=-h gr-self-healing-mysql-0.gr-self-healing-mysql -uroot -proot_password' logger.go:42: 14:47:30 | gr-self-healing/7-read-from-replicas | ++ local pod= logger.go:42: 14:47:30 | gr-self-healing/7-read-from-replicas | +++ get_client_pod logger.go:42: 14:47:30 | gr-self-healing/7-read-from-replicas | +++ kubectl -n kuttl-test-secure-shiner get pods --selector=name=mysql-client -o 'jsonpath={.items[].metadata.name}' logger.go:42: 14:47:31 | gr-self-healing/7-read-from-replicas | ++ client_pod=mysql-client logger.go:42: 14:47:31 | gr-self-healing/7-read-from-replicas | ++ wait_pod mysql-client logger.go:42: 14:47:31 | gr-self-healing/7-read-from-replicas | ++ local pod=mysql-client logger.go:42: 14:47:31 | gr-self-healing/7-read-from-replicas | ++ set +o xtrace logger.go:42: 14:47:32 | gr-self-healing/7-read-from-replicas | mysql-clienttrue logger.go:42: 14:47:32 | gr-self-healing/7-read-from-replicas | ++ kubectl -n kuttl-test-secure-shiner exec mysql-client -- bash -c 'printf '\''%s\n'\'' "SELECT * FROM myDB.myTable" | mysql -sN -h gr-self-healing-mysql-0.gr-self-healing-mysql -uroot -proot_password' logger.go:42: 14:47:32 | gr-self-healing/7-read-from-replicas | ++ sed -e 's/mysql: //' logger.go:42: 14:47:32 | gr-self-healing/7-read-from-replicas | ++ grep -v 'Using a password on the command line interface can be insecure.' logger.go:42: 14:47:33 | gr-self-healing/7-read-from-replicas | + data='100500 logger.go:42: 14:47:33 | gr-self-healing/7-read-from-replicas | 100501' logger.go:42: 14:47:33 | gr-self-healing/7-read-from-replicas | + kubectl create configmap -n kuttl-test-secure-shiner 07-read-from-replicas-0 '--from-literal=data=100500 logger.go:42: 14:47:33 | gr-self-healing/7-read-from-replicas | 100501' logger.go:42: 14:47:34 | gr-self-healing/7-read-from-replicas | configmap/07-read-from-replicas-0 created logger.go:42: 14:47:34 | gr-self-healing/7-read-from-replicas | + for i in 0 1 2 logger.go:42: 14:47:34 | gr-self-healing/7-read-from-replicas | +++ get_cluster_name logger.go:42: 14:47:34 | gr-self-healing/7-read-from-replicas | +++ kubectl -n kuttl-test-secure-shiner get ps -o 'jsonpath={.items[0].metadata.name}' logger.go:42: 14:47:35 | gr-self-healing/7-read-from-replicas | ++ get_mysql_headless_fqdn gr-self-healing 1 logger.go:42: 14:47:35 | gr-self-healing/7-read-from-replicas | ++ local cluster=gr-self-healing logger.go:42: 14:47:35 | gr-self-healing/7-read-from-replicas | ++ local index=1 logger.go:42: 14:47:35 | gr-self-healing/7-read-from-replicas | ++ echo gr-self-healing-mysql-1.gr-self-healing-mysql logger.go:42: 14:47:35 | gr-self-healing/7-read-from-replicas | + host=gr-self-healing-mysql-1.gr-self-healing-mysql logger.go:42: 14:47:35 | gr-self-healing/7-read-from-replicas | ++ run_mysql 'SELECT * FROM myDB.myTable' '-h gr-self-healing-mysql-1.gr-self-healing-mysql -uroot -proot_password' logger.go:42: 14:47:35 | gr-self-healing/7-read-from-replicas | ++ local 'command=SELECT * FROM myDB.myTable' logger.go:42: 14:47:35 | gr-self-healing/7-read-from-replicas | ++ local 'uri=-h gr-self-healing-mysql-1.gr-self-healing-mysql -uroot -proot_password' logger.go:42: 14:47:35 | gr-self-healing/7-read-from-replicas | ++ local pod= logger.go:42: 14:47:35 | gr-self-healing/7-read-from-replicas | +++ get_client_pod logger.go:42: 14:47:35 | gr-self-healing/7-read-from-replicas | +++ kubectl -n kuttl-test-secure-shiner get pods --selector=name=mysql-client -o 'jsonpath={.items[].metadata.name}' logger.go:42: 14:47:35 | gr-self-healing/7-read-from-replicas | ++ client_pod=mysql-client logger.go:42: 14:47:35 | gr-self-healing/7-read-from-replicas | ++ wait_pod mysql-client logger.go:42: 14:47:35 | gr-self-healing/7-read-from-replicas | ++ local pod=mysql-client logger.go:42: 14:47:35 | gr-self-healing/7-read-from-replicas | ++ set +o xtrace logger.go:42: 14:47:36 | gr-self-healing/7-read-from-replicas | mysql-clienttrue logger.go:42: 14:47:36 | gr-self-healing/7-read-from-replicas | ++ kubectl -n kuttl-test-secure-shiner exec mysql-client -- bash -c 'printf '\''%s\n'\'' "SELECT * FROM myDB.myTable" | mysql -sN -h gr-self-healing-mysql-1.gr-self-healing-mysql -uroot -proot_password' logger.go:42: 14:47:36 | gr-self-healing/7-read-from-replicas | ++ sed -e 's/mysql: //' logger.go:42: 14:47:36 | gr-self-healing/7-read-from-replicas | ++ grep -v 'Using a password on the command line interface can be insecure.' logger.go:42: 14:47:37 | gr-self-healing/7-read-from-replicas | + data='100500 logger.go:42: 14:47:37 | gr-self-healing/7-read-from-replicas | 100501' logger.go:42: 14:47:37 | gr-self-healing/7-read-from-replicas | + kubectl create configmap -n kuttl-test-secure-shiner 07-read-from-replicas-1 '--from-literal=data=100500 logger.go:42: 14:47:37 | gr-self-healing/7-read-from-replicas | 100501' logger.go:42: 14:47:38 | gr-self-healing/7-read-from-replicas | configmap/07-read-from-replicas-1 created logger.go:42: 14:47:38 | gr-self-healing/7-read-from-replicas | + for i in 0 1 2 logger.go:42: 14:47:38 | gr-self-healing/7-read-from-replicas | +++ get_cluster_name logger.go:42: 14:47:38 | gr-self-healing/7-read-from-replicas | +++ kubectl -n kuttl-test-secure-shiner get ps -o 'jsonpath={.items[0].metadata.name}' logger.go:42: 14:47:39 | gr-self-healing/7-read-from-replicas | ++ get_mysql_headless_fqdn gr-self-healing 2 logger.go:42: 14:47:39 | gr-self-healing/7-read-from-replicas | ++ local cluster=gr-self-healing logger.go:42: 14:47:39 | gr-self-healing/7-read-from-replicas | ++ local index=2 logger.go:42: 14:47:39 | gr-self-healing/7-read-from-replicas | ++ echo gr-self-healing-mysql-2.gr-self-healing-mysql logger.go:42: 14:47:39 | gr-self-healing/7-read-from-replicas | + host=gr-self-healing-mysql-2.gr-self-healing-mysql logger.go:42: 14:47:39 | gr-self-healing/7-read-from-replicas | ++ run_mysql 'SELECT * FROM myDB.myTable' '-h gr-self-healing-mysql-2.gr-self-healing-mysql -uroot -proot_password' logger.go:42: 14:47:39 | gr-self-healing/7-read-from-replicas | ++ local 'command=SELECT * FROM myDB.myTable' logger.go:42: 14:47:39 | gr-self-healing/7-read-from-replicas | ++ local 'uri=-h gr-self-healing-mysql-2.gr-self-healing-mysql -uroot -proot_password' logger.go:42: 14:47:39 | gr-self-healing/7-read-from-replicas | ++ local pod= logger.go:42: 14:47:39 | gr-self-healing/7-read-from-replicas | +++ get_client_pod logger.go:42: 14:47:39 | gr-self-healing/7-read-from-replicas | +++ kubectl -n kuttl-test-secure-shiner get pods --selector=name=mysql-client -o 'jsonpath={.items[].metadata.name}' logger.go:42: 14:47:40 | gr-self-healing/7-read-from-replicas | ++ client_pod=mysql-client logger.go:42: 14:47:40 | gr-self-healing/7-read-from-replicas | ++ wait_pod mysql-client logger.go:42: 14:47:40 | gr-self-healing/7-read-from-replicas | ++ local pod=mysql-client logger.go:42: 14:47:40 | gr-self-healing/7-read-from-replicas | ++ set +o xtrace logger.go:42: 14:47:40 | gr-self-healing/7-read-from-replicas | mysql-clienttrue logger.go:42: 14:47:40 | gr-self-healing/7-read-from-replicas | ++ sed -e 's/mysql: //' logger.go:42: 14:47:40 | gr-self-healing/7-read-from-replicas | ++ grep -v 'Using a password on the command line interface can be insecure.' logger.go:42: 14:47:40 | gr-self-healing/7-read-from-replicas | ++ kubectl -n kuttl-test-secure-shiner exec mysql-client -- bash -c 'printf '\''%s\n'\'' "SELECT * FROM myDB.myTable" | mysql -sN -h gr-self-healing-mysql-2.gr-self-healing-mysql -uroot -proot_password' logger.go:42: 14:47:42 | gr-self-healing/7-read-from-replicas | + data='100500 logger.go:42: 14:47:42 | gr-self-healing/7-read-from-replicas | 100501' logger.go:42: 14:47:42 | gr-self-healing/7-read-from-replicas | + kubectl create configmap -n kuttl-test-secure-shiner 07-read-from-replicas-2 '--from-literal=data=100500 logger.go:42: 14:47:42 | gr-self-healing/7-read-from-replicas | 100501' logger.go:42: 14:47:43 | gr-self-healing/7-read-from-replicas | configmap/07-read-from-replicas-2 created logger.go:42: 14:47:44 | gr-self-healing/7-read-from-replicas | test step completed 7-read-from-replicas logger.go:42: 14:47:44 | gr-self-healing/8-failure-primary | starting test step 8-failure-primary logger.go:42: 14:47:44 | gr-self-healing/8-failure-primary | running command: [sh -c set -o errexit set -o xtrace source ../../functions failure_pod "${NAMESPACE}" "$(get_primary_from_group_replication)" "primary" sleep 10 # wait a bit for pod to be killed] logger.go:42: 14:47:44 | gr-self-healing/8-failure-primary | + source ../../functions logger.go:42: 14:47:44 | gr-self-healing/8-failure-primary | +++ realpath ../../.. logger.go:42: 14:47:44 | gr-self-healing/8-failure-primary | ++ ROOT_REPO=/mnt/jenkins/workspace/cloud-ps-operator_PR-424 logger.go:42: 14:47:44 | gr-self-healing/8-failure-primary | ++ source /mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests/vars.sh logger.go:42: 14:47:44 | gr-self-healing/8-failure-primary | +++ export ROOT_REPO=/mnt/jenkins/workspace/cloud-ps-operator_PR-424 logger.go:42: 14:47:44 | gr-self-healing/8-failure-primary | +++ ROOT_REPO=/mnt/jenkins/workspace/cloud-ps-operator_PR-424 logger.go:42: 14:47:44 | gr-self-healing/8-failure-primary | +++ export DEPLOY_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/deploy logger.go:42: 14:47:44 | gr-self-healing/8-failure-primary | +++ DEPLOY_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/deploy logger.go:42: 14:47:44 | gr-self-healing/8-failure-primary | +++ export TESTS_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests logger.go:42: 14:47:44 | gr-self-healing/8-failure-primary | +++ TESTS_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests logger.go:42: 14:47:44 | gr-self-healing/8-failure-primary | +++ export TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests/conf logger.go:42: 14:47:44 | gr-self-healing/8-failure-primary | +++ TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests/conf logger.go:42: 14:47:44 | gr-self-healing/8-failure-primary | ++++ mktemp -d logger.go:42: 14:47:44 | gr-self-healing/8-failure-primary | +++ export TEMP_DIR=/tmp/tmp.C9hBXuBMWP logger.go:42: 14:47:44 | gr-self-healing/8-failure-primary | +++ TEMP_DIR=/tmp/tmp.C9hBXuBMWP logger.go:42: 14:47:44 | gr-self-healing/8-failure-primary | ++++ git rev-parse --abbrev-ref HEAD logger.go:42: 14:47:44 | gr-self-healing/8-failure-primary | +++ export GIT_BRANCH=PR-424 logger.go:42: 14:47:44 | gr-self-healing/8-failure-primary | +++ GIT_BRANCH=PR-424 logger.go:42: 14:47:44 | gr-self-healing/8-failure-primary | +++ export VERSION=PR-424-70568ae logger.go:42: 14:47:44 | gr-self-healing/8-failure-primary | +++ VERSION=PR-424-70568ae logger.go:42: 14:47:44 | gr-self-healing/8-failure-primary | +++ export IMAGE=perconalab/percona-server-mysql-operator:PR-424-70568ae logger.go:42: 14:47:44 | gr-self-healing/8-failure-primary | +++ IMAGE=perconalab/percona-server-mysql-operator:PR-424-70568ae logger.go:42: 14:47:44 | gr-self-healing/8-failure-primary | +++ export IMAGE_MYSQL=perconalab/percona-server-mysql-operator:main-psmysql logger.go:42: 14:47:44 | gr-self-healing/8-failure-primary | +++ IMAGE_MYSQL=perconalab/percona-server-mysql-operator:main-psmysql logger.go:42: 14:47:44 | gr-self-healing/8-failure-primary | +++ export IMAGE_BACKUP=perconalab/percona-server-mysql-operator:main-backup logger.go:42: 14:47:44 | gr-self-healing/8-failure-primary | +++ IMAGE_BACKUP=perconalab/percona-server-mysql-operator:main-backup logger.go:42: 14:47:44 | gr-self-healing/8-failure-primary | +++ export IMAGE_ORCHESTRATOR=perconalab/percona-server-mysql-operator:main-orchestrator logger.go:42: 14:47:44 | gr-self-healing/8-failure-primary | +++ IMAGE_ORCHESTRATOR=perconalab/percona-server-mysql-operator:main-orchestrator logger.go:42: 14:47:44 | gr-self-healing/8-failure-primary | +++ export IMAGE_ROUTER=perconalab/percona-server-mysql-operator:main-router logger.go:42: 14:47:44 | gr-self-healing/8-failure-primary | +++ IMAGE_ROUTER=perconalab/percona-server-mysql-operator:main-router logger.go:42: 14:47:44 | gr-self-healing/8-failure-primary | +++ export IMAGE_TOOLKIT=perconalab/percona-server-mysql-operator:main-toolkit logger.go:42: 14:47:44 | gr-self-healing/8-failure-primary | +++ IMAGE_TOOLKIT=perconalab/percona-server-mysql-operator:main-toolkit logger.go:42: 14:47:44 | gr-self-healing/8-failure-primary | +++ export IMAGE_HAPROXY=perconalab/percona-server-mysql-operator:main-haproxy logger.go:42: 14:47:44 | gr-self-healing/8-failure-primary | +++ IMAGE_HAPROXY=perconalab/percona-server-mysql-operator:main-haproxy logger.go:42: 14:47:44 | gr-self-healing/8-failure-primary | +++ export IMAGE_PMM=perconalab/pmm-client:dev-latest logger.go:42: 14:47:44 | gr-self-healing/8-failure-primary | +++ IMAGE_PMM=perconalab/pmm-client:dev-latest logger.go:42: 14:47:44 | gr-self-healing/8-failure-primary | +++ export PMM_SERVER_VERSION=9.9.9 logger.go:42: 14:47:44 | gr-self-healing/8-failure-primary | +++ PMM_SERVER_VERSION=9.9.9 logger.go:42: 14:47:44 | gr-self-healing/8-failure-primary | +++ export IMAGE_PMM_SERVER_REPO=perconalab/pmm-server logger.go:42: 14:47:44 | gr-self-healing/8-failure-primary | +++ IMAGE_PMM_SERVER_REPO=perconalab/pmm-server logger.go:42: 14:47:44 | gr-self-healing/8-failure-primary | +++ export IMAGE_PMM_SERVER_TAG=dev-latest logger.go:42: 14:47:44 | gr-self-healing/8-failure-primary | +++ IMAGE_PMM_SERVER_TAG=dev-latest logger.go:42: 14:47:44 | gr-self-healing/8-failure-primary | ++++ which gdate logger.go:42: 14:47:44 | gr-self-healing/8-failure-primary | which: no gdate in (/mnt/jenkins/workspace/cloud-ps-operator_PR-424/bin/:/home/ec2-user/google-cloud-sdk/bin:/home/ec2-user/.krew/bin:/usr/local/bin:/usr/bin) logger.go:42: 14:47:44 | gr-self-healing/8-failure-primary | ++++ which date logger.go:42: 14:47:44 | gr-self-healing/8-failure-primary | +++ date=/usr/bin/date logger.go:42: 14:47:44 | gr-self-healing/8-failure-primary | +++ command -v oc logger.go:42: 14:47:44 | gr-self-healing/8-failure-primary | +++ oc get projects logger.go:42: 14:47:50 | gr-self-healing/8-failure-primary | error: the server doesn't have a resource type "projects" logger.go:42: 14:47:50 | gr-self-healing/8-failure-primary | +++ grep '^minikube' logger.go:42: 14:47:50 | gr-self-healing/8-failure-primary | +++ kubectl get nodes logger.go:42: 14:47:51 | gr-self-healing/8-failure-primary | ++++ pwd logger.go:42: 14:47:51 | gr-self-healing/8-failure-primary | +++ basename /mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests/tests/gr-self-healing logger.go:42: 14:47:51 | gr-self-healing/8-failure-primary | ++ test_name=gr-self-healing logger.go:42: 14:47:51 | gr-self-healing/8-failure-primary | ++ get_primary_from_group_replication logger.go:42: 14:47:51 | gr-self-healing/8-failure-primary | ++ cut -d. -f1 logger.go:42: 14:47:51 | gr-self-healing/8-failure-primary | ++++ get_cluster_name logger.go:42: 14:47:51 | gr-self-healing/8-failure-primary | ++++ kubectl -n kuttl-test-secure-shiner get ps -o 'jsonpath={.items[0].metadata.name}' logger.go:42: 14:47:52 | gr-self-healing/8-failure-primary | +++ get_mysql_router_service gr-self-healing logger.go:42: 14:47:52 | gr-self-healing/8-failure-primary | +++ local cluster=gr-self-healing logger.go:42: 14:47:52 | gr-self-healing/8-failure-primary | +++ echo gr-self-healing-router logger.go:42: 14:47:52 | gr-self-healing/8-failure-primary | ++ run_mysql 'SELECT MEMBER_HOST FROM performance_schema.replication_group_members where MEMBER_ROLE='\''PRIMARY'\'';' '-h gr-self-healing-router -P 6446 -uroot -proot_password' logger.go:42: 14:47:52 | gr-self-healing/8-failure-primary | ++ local 'command=SELECT MEMBER_HOST FROM performance_schema.replication_group_members where MEMBER_ROLE='\''PRIMARY'\'';' logger.go:42: 14:47:52 | gr-self-healing/8-failure-primary | ++ local 'uri=-h gr-self-healing-router -P 6446 -uroot -proot_password' logger.go:42: 14:47:52 | gr-self-healing/8-failure-primary | ++ local pod= logger.go:42: 14:47:52 | gr-self-healing/8-failure-primary | +++ get_client_pod logger.go:42: 14:47:52 | gr-self-healing/8-failure-primary | +++ kubectl -n kuttl-test-secure-shiner get pods --selector=name=mysql-client -o 'jsonpath={.items[].metadata.name}' logger.go:42: 14:47:52 | gr-self-healing/8-failure-primary | ++ client_pod=mysql-client logger.go:42: 14:47:52 | gr-self-healing/8-failure-primary | ++ wait_pod mysql-client logger.go:42: 14:47:52 | gr-self-healing/8-failure-primary | ++ local pod=mysql-client logger.go:42: 14:47:52 | gr-self-healing/8-failure-primary | ++ set +o xtrace logger.go:42: 14:47:53 | gr-self-healing/8-failure-primary | mysql-clienttrue logger.go:42: 14:47:53 | gr-self-healing/8-failure-primary | ++ sed -e 's/mysql: //' logger.go:42: 14:47:53 | gr-self-healing/8-failure-primary | ++ grep -v 'Using a password on the command line interface can be insecure.' logger.go:42: 14:47:53 | gr-self-healing/8-failure-primary | ++ kubectl -n kuttl-test-secure-shiner exec mysql-client -- bash -c 'printf '\''%s\n'\'' "SELECT MEMBER_HOST FROM performance_schema.replication_group_members where MEMBER_ROLE='\''PRIMARY'\'';" | mysql -sN -h gr-self-healing-router -P 6446 -uroot -proot_password' logger.go:42: 14:47:55 | gr-self-healing/8-failure-primary | + failure_pod kuttl-test-secure-shiner gr-self-healing-mysql-1 primary logger.go:42: 14:47:55 | gr-self-healing/8-failure-primary | + local ns=kuttl-test-secure-shiner logger.go:42: 14:47:55 | gr-self-healing/8-failure-primary | + local pod=gr-self-healing-mysql-1 logger.go:42: 14:47:55 | gr-self-healing/8-failure-primary | + local chaos_suffix=primary logger.go:42: 14:47:55 | gr-self-healing/8-failure-primary | + yq eval ' logger.go:42: 14:47:55 | gr-self-healing/8-failure-primary | .metadata.name = "chaos-pod-failure-primary" | logger.go:42: 14:47:55 | gr-self-healing/8-failure-primary | del(.spec.selector.pods.test-namespace) | logger.go:42: 14:47:55 | gr-self-healing/8-failure-primary | .spec.selector.pods.kuttl-test-secure-shiner[0] = "gr-self-healing-mysql-1"' /mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests/conf/chaos-pod-failure.yml logger.go:42: 14:47:55 | gr-self-healing/8-failure-primary | + kubectl apply --namespace kuttl-test-secure-shiner -f - logger.go:42: 14:47:56 | gr-self-healing/8-failure-primary | podchaos.chaos-mesh.org/chaos-pod-failure-primary created logger.go:42: 14:47:56 | gr-self-healing/8-failure-primary | + sleep 5 logger.go:42: 14:48:01 | gr-self-healing/8-failure-primary | + sleep 10 logger.go:42: 14:49:40 | gr-self-healing/8-failure-primary | test step completed 8-failure-primary logger.go:42: 14:49:40 | gr-self-healing/9-write-data | starting test step 9-write-data logger.go:42: 14:49:40 | gr-self-healing/9-write-data | running command: [sh -c set -o errexit set -o xtrace source ../../functions run_mysql \ "INSERT myDB.myTable (id) VALUES (100502)" \ "-h $(get_mysql_router_service $(get_cluster_name)) -P 6446 -uroot -proot_password"] logger.go:42: 14:49:40 | gr-self-healing/9-write-data | + source ../../functions logger.go:42: 14:49:40 | gr-self-healing/9-write-data | +++ realpath ../../.. logger.go:42: 14:49:40 | gr-self-healing/9-write-data | ++ ROOT_REPO=/mnt/jenkins/workspace/cloud-ps-operator_PR-424 logger.go:42: 14:49:40 | gr-self-healing/9-write-data | ++ source /mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests/vars.sh logger.go:42: 14:49:40 | gr-self-healing/9-write-data | +++ export ROOT_REPO=/mnt/jenkins/workspace/cloud-ps-operator_PR-424 logger.go:42: 14:49:40 | gr-self-healing/9-write-data | +++ ROOT_REPO=/mnt/jenkins/workspace/cloud-ps-operator_PR-424 logger.go:42: 14:49:40 | gr-self-healing/9-write-data | +++ export DEPLOY_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/deploy logger.go:42: 14:49:40 | gr-self-healing/9-write-data | +++ DEPLOY_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/deploy logger.go:42: 14:49:40 | gr-self-healing/9-write-data | +++ export TESTS_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests logger.go:42: 14:49:40 | gr-self-healing/9-write-data | +++ TESTS_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests logger.go:42: 14:49:40 | gr-self-healing/9-write-data | +++ export TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests/conf logger.go:42: 14:49:40 | gr-self-healing/9-write-data | +++ TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests/conf logger.go:42: 14:49:40 | gr-self-healing/9-write-data | ++++ mktemp -d logger.go:42: 14:49:40 | gr-self-healing/9-write-data | +++ export TEMP_DIR=/tmp/tmp.hMb6pFhS1p logger.go:42: 14:49:40 | gr-self-healing/9-write-data | +++ TEMP_DIR=/tmp/tmp.hMb6pFhS1p logger.go:42: 14:49:40 | gr-self-healing/9-write-data | ++++ git rev-parse --abbrev-ref HEAD logger.go:42: 14:49:40 | gr-self-healing/9-write-data | +++ export GIT_BRANCH=PR-424 logger.go:42: 14:49:40 | gr-self-healing/9-write-data | +++ GIT_BRANCH=PR-424 logger.go:42: 14:49:40 | gr-self-healing/9-write-data | +++ export VERSION=PR-424-70568ae logger.go:42: 14:49:40 | gr-self-healing/9-write-data | +++ VERSION=PR-424-70568ae logger.go:42: 14:49:40 | gr-self-healing/9-write-data | +++ export IMAGE=perconalab/percona-server-mysql-operator:PR-424-70568ae logger.go:42: 14:49:40 | gr-self-healing/9-write-data | +++ IMAGE=perconalab/percona-server-mysql-operator:PR-424-70568ae logger.go:42: 14:49:40 | gr-self-healing/9-write-data | +++ export IMAGE_MYSQL=perconalab/percona-server-mysql-operator:main-psmysql logger.go:42: 14:49:40 | gr-self-healing/9-write-data | +++ IMAGE_MYSQL=perconalab/percona-server-mysql-operator:main-psmysql logger.go:42: 14:49:40 | gr-self-healing/9-write-data | +++ export IMAGE_BACKUP=perconalab/percona-server-mysql-operator:main-backup logger.go:42: 14:49:40 | gr-self-healing/9-write-data | +++ IMAGE_BACKUP=perconalab/percona-server-mysql-operator:main-backup logger.go:42: 14:49:40 | gr-self-healing/9-write-data | +++ export IMAGE_ORCHESTRATOR=perconalab/percona-server-mysql-operator:main-orchestrator logger.go:42: 14:49:40 | gr-self-healing/9-write-data | +++ IMAGE_ORCHESTRATOR=perconalab/percona-server-mysql-operator:main-orchestrator logger.go:42: 14:49:40 | gr-self-healing/9-write-data | +++ export IMAGE_ROUTER=perconalab/percona-server-mysql-operator:main-router logger.go:42: 14:49:40 | gr-self-healing/9-write-data | +++ IMAGE_ROUTER=perconalab/percona-server-mysql-operator:main-router logger.go:42: 14:49:40 | gr-self-healing/9-write-data | +++ export IMAGE_TOOLKIT=perconalab/percona-server-mysql-operator:main-toolkit logger.go:42: 14:49:40 | gr-self-healing/9-write-data | +++ IMAGE_TOOLKIT=perconalab/percona-server-mysql-operator:main-toolkit logger.go:42: 14:49:40 | gr-self-healing/9-write-data | +++ export IMAGE_HAPROXY=perconalab/percona-server-mysql-operator:main-haproxy logger.go:42: 14:49:40 | gr-self-healing/9-write-data | +++ IMAGE_HAPROXY=perconalab/percona-server-mysql-operator:main-haproxy logger.go:42: 14:49:40 | gr-self-healing/9-write-data | +++ export IMAGE_PMM=perconalab/pmm-client:dev-latest logger.go:42: 14:49:40 | gr-self-healing/9-write-data | +++ IMAGE_PMM=perconalab/pmm-client:dev-latest logger.go:42: 14:49:40 | gr-self-healing/9-write-data | +++ export PMM_SERVER_VERSION=9.9.9 logger.go:42: 14:49:40 | gr-self-healing/9-write-data | +++ PMM_SERVER_VERSION=9.9.9 logger.go:42: 14:49:40 | gr-self-healing/9-write-data | +++ export IMAGE_PMM_SERVER_REPO=perconalab/pmm-server logger.go:42: 14:49:40 | gr-self-healing/9-write-data | +++ IMAGE_PMM_SERVER_REPO=perconalab/pmm-server logger.go:42: 14:49:40 | gr-self-healing/9-write-data | +++ export IMAGE_PMM_SERVER_TAG=dev-latest logger.go:42: 14:49:40 | gr-self-healing/9-write-data | +++ IMAGE_PMM_SERVER_TAG=dev-latest logger.go:42: 14:49:40 | gr-self-healing/9-write-data | ++++ which gdate logger.go:42: 14:49:40 | gr-self-healing/9-write-data | which: no gdate in (/mnt/jenkins/workspace/cloud-ps-operator_PR-424/bin/:/home/ec2-user/google-cloud-sdk/bin:/home/ec2-user/.krew/bin:/usr/local/bin:/usr/bin) logger.go:42: 14:49:40 | gr-self-healing/9-write-data | ++++ which date logger.go:42: 14:49:40 | gr-self-healing/9-write-data | +++ date=/usr/bin/date logger.go:42: 14:49:40 | gr-self-healing/9-write-data | +++ command -v oc logger.go:42: 14:49:40 | gr-self-healing/9-write-data | +++ oc get projects logger.go:42: 14:49:46 | gr-self-healing/9-write-data | error: the server doesn't have a resource type "projects" logger.go:42: 14:49:46 | gr-self-healing/9-write-data | +++ grep '^minikube' logger.go:42: 14:49:46 | gr-self-healing/9-write-data | +++ kubectl get nodes logger.go:42: 14:49:47 | gr-self-healing/9-write-data | ++++ pwd logger.go:42: 14:49:47 | gr-self-healing/9-write-data | +++ basename /mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests/tests/gr-self-healing logger.go:42: 14:49:47 | gr-self-healing/9-write-data | ++ test_name=gr-self-healing logger.go:42: 14:49:47 | gr-self-healing/9-write-data | +++ get_cluster_name logger.go:42: 14:49:47 | gr-self-healing/9-write-data | +++ kubectl -n kuttl-test-secure-shiner get ps -o 'jsonpath={.items[0].metadata.name}' logger.go:42: 14:49:47 | gr-self-healing/9-write-data | ++ get_mysql_router_service gr-self-healing logger.go:42: 14:49:47 | gr-self-healing/9-write-data | ++ local cluster=gr-self-healing logger.go:42: 14:49:47 | gr-self-healing/9-write-data | ++ echo gr-self-healing-router logger.go:42: 14:49:47 | gr-self-healing/9-write-data | + run_mysql 'INSERT myDB.myTable (id) VALUES (100502)' '-h gr-self-healing-router -P 6446 -uroot -proot_password' logger.go:42: 14:49:47 | gr-self-healing/9-write-data | + local 'command=INSERT myDB.myTable (id) VALUES (100502)' logger.go:42: 14:49:47 | gr-self-healing/9-write-data | + local 'uri=-h gr-self-healing-router -P 6446 -uroot -proot_password' logger.go:42: 14:49:47 | gr-self-healing/9-write-data | + local pod= logger.go:42: 14:49:47 | gr-self-healing/9-write-data | ++ get_client_pod logger.go:42: 14:49:47 | gr-self-healing/9-write-data | ++ kubectl -n kuttl-test-secure-shiner get pods --selector=name=mysql-client -o 'jsonpath={.items[].metadata.name}' logger.go:42: 14:49:48 | gr-self-healing/9-write-data | + client_pod=mysql-client logger.go:42: 14:49:48 | gr-self-healing/9-write-data | + wait_pod mysql-client logger.go:42: 14:49:48 | gr-self-healing/9-write-data | + local pod=mysql-client logger.go:42: 14:49:48 | gr-self-healing/9-write-data | + set +o xtrace logger.go:42: 14:49:49 | gr-self-healing/9-write-data | mysql-clienttrue logger.go:42: 14:49:49 | gr-self-healing/9-write-data | + kubectl -n kuttl-test-secure-shiner exec mysql-client -- bash -c 'printf '\''%s\n'\'' "INSERT myDB.myTable (id) VALUES (100502)" | mysql -sN -h gr-self-healing-router -P 6446 -uroot -proot_password' logger.go:42: 14:49:49 | gr-self-healing/9-write-data | + sed -e 's/mysql: //' logger.go:42: 14:49:49 | gr-self-healing/9-write-data | + grep -v 'Using a password on the command line interface can be insecure.' logger.go:42: 14:49:50 | gr-self-healing/9-write-data | + : logger.go:42: 14:49:51 | gr-self-healing/9-write-data | test step completed 9-write-data logger.go:42: 14:49:51 | gr-self-healing/10-read-from-replicas | starting test step 10-read-from-replicas logger.go:42: 14:49:51 | gr-self-healing/10-read-from-replicas | running command: [sh -c set -o errexit set -o xtrace source ../../functions for i in 0 1 2; do host=$(get_mysql_headless_fqdn $(get_cluster_name) $i) data=$(run_mysql "SELECT * FROM myDB.myTable" "-h ${host} -uroot -proot_password") kubectl create configmap -n "${NAMESPACE}" 10-read-from-replicas-${i} --from-literal=data="${data}" done] logger.go:42: 14:49:51 | gr-self-healing/10-read-from-replicas | + source ../../functions logger.go:42: 14:49:51 | gr-self-healing/10-read-from-replicas | +++ realpath ../../.. logger.go:42: 14:49:51 | gr-self-healing/10-read-from-replicas | ++ ROOT_REPO=/mnt/jenkins/workspace/cloud-ps-operator_PR-424 logger.go:42: 14:49:51 | gr-self-healing/10-read-from-replicas | ++ source /mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests/vars.sh logger.go:42: 14:49:51 | gr-self-healing/10-read-from-replicas | +++ export ROOT_REPO=/mnt/jenkins/workspace/cloud-ps-operator_PR-424 logger.go:42: 14:49:51 | gr-self-healing/10-read-from-replicas | +++ ROOT_REPO=/mnt/jenkins/workspace/cloud-ps-operator_PR-424 logger.go:42: 14:49:51 | gr-self-healing/10-read-from-replicas | +++ export DEPLOY_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/deploy logger.go:42: 14:49:51 | gr-self-healing/10-read-from-replicas | +++ DEPLOY_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/deploy logger.go:42: 14:49:51 | gr-self-healing/10-read-from-replicas | +++ export TESTS_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests logger.go:42: 14:49:51 | gr-self-healing/10-read-from-replicas | +++ TESTS_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests logger.go:42: 14:49:51 | gr-self-healing/10-read-from-replicas | +++ export TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests/conf logger.go:42: 14:49:51 | gr-self-healing/10-read-from-replicas | +++ TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests/conf logger.go:42: 14:49:51 | gr-self-healing/10-read-from-replicas | ++++ mktemp -d logger.go:42: 14:49:51 | gr-self-healing/10-read-from-replicas | +++ export TEMP_DIR=/tmp/tmp.Zmz05rDtnk logger.go:42: 14:49:51 | gr-self-healing/10-read-from-replicas | +++ TEMP_DIR=/tmp/tmp.Zmz05rDtnk logger.go:42: 14:49:51 | gr-self-healing/10-read-from-replicas | ++++ git rev-parse --abbrev-ref HEAD logger.go:42: 14:49:51 | gr-self-healing/10-read-from-replicas | +++ export GIT_BRANCH=PR-424 logger.go:42: 14:49:51 | gr-self-healing/10-read-from-replicas | +++ GIT_BRANCH=PR-424 logger.go:42: 14:49:51 | gr-self-healing/10-read-from-replicas | +++ export VERSION=PR-424-70568ae logger.go:42: 14:49:51 | gr-self-healing/10-read-from-replicas | +++ VERSION=PR-424-70568ae logger.go:42: 14:49:51 | gr-self-healing/10-read-from-replicas | +++ export IMAGE=perconalab/percona-server-mysql-operator:PR-424-70568ae logger.go:42: 14:49:51 | gr-self-healing/10-read-from-replicas | +++ IMAGE=perconalab/percona-server-mysql-operator:PR-424-70568ae logger.go:42: 14:49:51 | gr-self-healing/10-read-from-replicas | +++ export IMAGE_MYSQL=perconalab/percona-server-mysql-operator:main-psmysql logger.go:42: 14:49:51 | gr-self-healing/10-read-from-replicas | +++ IMAGE_MYSQL=perconalab/percona-server-mysql-operator:main-psmysql logger.go:42: 14:49:51 | gr-self-healing/10-read-from-replicas | +++ export IMAGE_BACKUP=perconalab/percona-server-mysql-operator:main-backup logger.go:42: 14:49:51 | gr-self-healing/10-read-from-replicas | +++ IMAGE_BACKUP=perconalab/percona-server-mysql-operator:main-backup logger.go:42: 14:49:51 | gr-self-healing/10-read-from-replicas | +++ export IMAGE_ORCHESTRATOR=perconalab/percona-server-mysql-operator:main-orchestrator logger.go:42: 14:49:51 | gr-self-healing/10-read-from-replicas | +++ IMAGE_ORCHESTRATOR=perconalab/percona-server-mysql-operator:main-orchestrator logger.go:42: 14:49:51 | gr-self-healing/10-read-from-replicas | +++ export IMAGE_ROUTER=perconalab/percona-server-mysql-operator:main-router logger.go:42: 14:49:51 | gr-self-healing/10-read-from-replicas | +++ IMAGE_ROUTER=perconalab/percona-server-mysql-operator:main-router logger.go:42: 14:49:51 | gr-self-healing/10-read-from-replicas | +++ export IMAGE_TOOLKIT=perconalab/percona-server-mysql-operator:main-toolkit logger.go:42: 14:49:51 | gr-self-healing/10-read-from-replicas | +++ IMAGE_TOOLKIT=perconalab/percona-server-mysql-operator:main-toolkit logger.go:42: 14:49:51 | gr-self-healing/10-read-from-replicas | +++ export IMAGE_HAPROXY=perconalab/percona-server-mysql-operator:main-haproxy logger.go:42: 14:49:51 | gr-self-healing/10-read-from-replicas | +++ IMAGE_HAPROXY=perconalab/percona-server-mysql-operator:main-haproxy logger.go:42: 14:49:51 | gr-self-healing/10-read-from-replicas | +++ export IMAGE_PMM=perconalab/pmm-client:dev-latest logger.go:42: 14:49:51 | gr-self-healing/10-read-from-replicas | +++ IMAGE_PMM=perconalab/pmm-client:dev-latest logger.go:42: 14:49:51 | gr-self-healing/10-read-from-replicas | +++ export PMM_SERVER_VERSION=9.9.9 logger.go:42: 14:49:51 | gr-self-healing/10-read-from-replicas | +++ PMM_SERVER_VERSION=9.9.9 logger.go:42: 14:49:51 | gr-self-healing/10-read-from-replicas | +++ export IMAGE_PMM_SERVER_REPO=perconalab/pmm-server logger.go:42: 14:49:51 | gr-self-healing/10-read-from-replicas | +++ IMAGE_PMM_SERVER_REPO=perconalab/pmm-server logger.go:42: 14:49:51 | gr-self-healing/10-read-from-replicas | +++ export IMAGE_PMM_SERVER_TAG=dev-latest logger.go:42: 14:49:51 | gr-self-healing/10-read-from-replicas | +++ IMAGE_PMM_SERVER_TAG=dev-latest logger.go:42: 14:49:51 | gr-self-healing/10-read-from-replicas | ++++ which gdate logger.go:42: 14:49:51 | gr-self-healing/10-read-from-replicas | which: no gdate in (/mnt/jenkins/workspace/cloud-ps-operator_PR-424/bin/:/home/ec2-user/google-cloud-sdk/bin:/home/ec2-user/.krew/bin:/usr/local/bin:/usr/bin) logger.go:42: 14:49:51 | gr-self-healing/10-read-from-replicas | ++++ which date logger.go:42: 14:49:51 | gr-self-healing/10-read-from-replicas | +++ date=/usr/bin/date logger.go:42: 14:49:51 | gr-self-healing/10-read-from-replicas | +++ command -v oc logger.go:42: 14:49:51 | gr-self-healing/10-read-from-replicas | +++ oc get projects logger.go:42: 14:49:57 | gr-self-healing/10-read-from-replicas | error: the server doesn't have a resource type "projects" logger.go:42: 14:49:57 | gr-self-healing/10-read-from-replicas | +++ kubectl get nodes logger.go:42: 14:49:57 | gr-self-healing/10-read-from-replicas | +++ grep '^minikube' logger.go:42: 14:49:58 | gr-self-healing/10-read-from-replicas | ++++ pwd logger.go:42: 14:49:58 | gr-self-healing/10-read-from-replicas | +++ basename /mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests/tests/gr-self-healing logger.go:42: 14:49:58 | gr-self-healing/10-read-from-replicas | ++ test_name=gr-self-healing logger.go:42: 14:49:58 | gr-self-healing/10-read-from-replicas | + for i in 0 1 2 logger.go:42: 14:49:58 | gr-self-healing/10-read-from-replicas | +++ get_cluster_name logger.go:42: 14:49:58 | gr-self-healing/10-read-from-replicas | +++ kubectl -n kuttl-test-secure-shiner get ps -o 'jsonpath={.items[0].metadata.name}' logger.go:42: 14:49:59 | gr-self-healing/10-read-from-replicas | ++ get_mysql_headless_fqdn gr-self-healing 0 logger.go:42: 14:49:59 | gr-self-healing/10-read-from-replicas | ++ local cluster=gr-self-healing logger.go:42: 14:49:59 | gr-self-healing/10-read-from-replicas | ++ local index=0 logger.go:42: 14:49:59 | gr-self-healing/10-read-from-replicas | ++ echo gr-self-healing-mysql-0.gr-self-healing-mysql logger.go:42: 14:49:59 | gr-self-healing/10-read-from-replicas | + host=gr-self-healing-mysql-0.gr-self-healing-mysql logger.go:42: 14:49:59 | gr-self-healing/10-read-from-replicas | ++ run_mysql 'SELECT * FROM myDB.myTable' '-h gr-self-healing-mysql-0.gr-self-healing-mysql -uroot -proot_password' logger.go:42: 14:49:59 | gr-self-healing/10-read-from-replicas | ++ local 'command=SELECT * FROM myDB.myTable' logger.go:42: 14:49:59 | gr-self-healing/10-read-from-replicas | ++ local 'uri=-h gr-self-healing-mysql-0.gr-self-healing-mysql -uroot -proot_password' logger.go:42: 14:49:59 | gr-self-healing/10-read-from-replicas | ++ local pod= logger.go:42: 14:49:59 | gr-self-healing/10-read-from-replicas | +++ get_client_pod logger.go:42: 14:49:59 | gr-self-healing/10-read-from-replicas | +++ kubectl -n kuttl-test-secure-shiner get pods --selector=name=mysql-client -o 'jsonpath={.items[].metadata.name}' logger.go:42: 14:49:59 | gr-self-healing/10-read-from-replicas | ++ client_pod=mysql-client logger.go:42: 14:49:59 | gr-self-healing/10-read-from-replicas | ++ wait_pod mysql-client logger.go:42: 14:49:59 | gr-self-healing/10-read-from-replicas | ++ local pod=mysql-client logger.go:42: 14:49:59 | gr-self-healing/10-read-from-replicas | ++ set +o xtrace logger.go:42: 14:50:00 | gr-self-healing/10-read-from-replicas | mysql-clienttrue logger.go:42: 14:50:00 | gr-self-healing/10-read-from-replicas | ++ kubectl -n kuttl-test-secure-shiner exec mysql-client -- bash -c 'printf '\''%s\n'\'' "SELECT * FROM myDB.myTable" | mysql -sN -h gr-self-healing-mysql-0.gr-self-healing-mysql -uroot -proot_password' logger.go:42: 14:50:00 | gr-self-healing/10-read-from-replicas | ++ sed -e 's/mysql: //' logger.go:42: 14:50:00 | gr-self-healing/10-read-from-replicas | ++ grep -v 'Using a password on the command line interface can be insecure.' logger.go:42: 14:50:02 | gr-self-healing/10-read-from-replicas | + data='100500 logger.go:42: 14:50:02 | gr-self-healing/10-read-from-replicas | 100501 logger.go:42: 14:50:02 | gr-self-healing/10-read-from-replicas | 100502' logger.go:42: 14:50:02 | gr-self-healing/10-read-from-replicas | + kubectl create configmap -n kuttl-test-secure-shiner 10-read-from-replicas-0 '--from-literal=data=100500 logger.go:42: 14:50:02 | gr-self-healing/10-read-from-replicas | 100501 logger.go:42: 14:50:02 | gr-self-healing/10-read-from-replicas | 100502' logger.go:42: 14:50:02 | gr-self-healing/10-read-from-replicas | configmap/10-read-from-replicas-0 created logger.go:42: 14:50:02 | gr-self-healing/10-read-from-replicas | + for i in 0 1 2 logger.go:42: 14:50:02 | gr-self-healing/10-read-from-replicas | +++ get_cluster_name logger.go:42: 14:50:02 | gr-self-healing/10-read-from-replicas | +++ kubectl -n kuttl-test-secure-shiner get ps -o 'jsonpath={.items[0].metadata.name}' logger.go:42: 14:50:03 | gr-self-healing/10-read-from-replicas | ++ get_mysql_headless_fqdn gr-self-healing 1 logger.go:42: 14:50:03 | gr-self-healing/10-read-from-replicas | ++ local cluster=gr-self-healing logger.go:42: 14:50:03 | gr-self-healing/10-read-from-replicas | ++ local index=1 logger.go:42: 14:50:03 | gr-self-healing/10-read-from-replicas | ++ echo gr-self-healing-mysql-1.gr-self-healing-mysql logger.go:42: 14:50:03 | gr-self-healing/10-read-from-replicas | + host=gr-self-healing-mysql-1.gr-self-healing-mysql logger.go:42: 14:50:03 | gr-self-healing/10-read-from-replicas | ++ run_mysql 'SELECT * FROM myDB.myTable' '-h gr-self-healing-mysql-1.gr-self-healing-mysql -uroot -proot_password' logger.go:42: 14:50:03 | gr-self-healing/10-read-from-replicas | ++ local 'command=SELECT * FROM myDB.myTable' logger.go:42: 14:50:03 | gr-self-healing/10-read-from-replicas | ++ local 'uri=-h gr-self-healing-mysql-1.gr-self-healing-mysql -uroot -proot_password' logger.go:42: 14:50:03 | gr-self-healing/10-read-from-replicas | ++ local pod= logger.go:42: 14:50:03 | gr-self-healing/10-read-from-replicas | +++ get_client_pod logger.go:42: 14:50:03 | gr-self-healing/10-read-from-replicas | +++ kubectl -n kuttl-test-secure-shiner get pods --selector=name=mysql-client -o 'jsonpath={.items[].metadata.name}' logger.go:42: 14:50:04 | gr-self-healing/10-read-from-replicas | ++ client_pod=mysql-client logger.go:42: 14:50:04 | gr-self-healing/10-read-from-replicas | ++ wait_pod mysql-client logger.go:42: 14:50:04 | gr-self-healing/10-read-from-replicas | ++ local pod=mysql-client logger.go:42: 14:50:04 | gr-self-healing/10-read-from-replicas | ++ set +o xtrace logger.go:42: 14:50:04 | gr-self-healing/10-read-from-replicas | mysql-clienttrue logger.go:42: 14:50:04 | gr-self-healing/10-read-from-replicas | ++ kubectl -n kuttl-test-secure-shiner exec mysql-client -- bash -c 'printf '\''%s\n'\'' "SELECT * FROM myDB.myTable" | mysql -sN -h gr-self-healing-mysql-1.gr-self-healing-mysql -uroot -proot_password' logger.go:42: 14:50:04 | gr-self-healing/10-read-from-replicas | ++ sed -e 's/mysql: //' logger.go:42: 14:50:04 | gr-self-healing/10-read-from-replicas | ++ grep -v 'Using a password on the command line interface can be insecure.' logger.go:42: 14:50:06 | gr-self-healing/10-read-from-replicas | + data='100500 logger.go:42: 14:50:06 | gr-self-healing/10-read-from-replicas | 100501 logger.go:42: 14:50:06 | gr-self-healing/10-read-from-replicas | 100502' logger.go:42: 14:50:06 | gr-self-healing/10-read-from-replicas | + kubectl create configmap -n kuttl-test-secure-shiner 10-read-from-replicas-1 '--from-literal=data=100500 logger.go:42: 14:50:06 | gr-self-healing/10-read-from-replicas | 100501 logger.go:42: 14:50:06 | gr-self-healing/10-read-from-replicas | 100502' logger.go:42: 14:50:07 | gr-self-healing/10-read-from-replicas | configmap/10-read-from-replicas-1 created logger.go:42: 14:50:07 | gr-self-healing/10-read-from-replicas | + for i in 0 1 2 logger.go:42: 14:50:07 | gr-self-healing/10-read-from-replicas | +++ get_cluster_name logger.go:42: 14:50:07 | gr-self-healing/10-read-from-replicas | +++ kubectl -n kuttl-test-secure-shiner get ps -o 'jsonpath={.items[0].metadata.name}' logger.go:42: 14:50:07 | gr-self-healing/10-read-from-replicas | ++ get_mysql_headless_fqdn gr-self-healing 2 logger.go:42: 14:50:07 | gr-self-healing/10-read-from-replicas | ++ local cluster=gr-self-healing logger.go:42: 14:50:07 | gr-self-healing/10-read-from-replicas | ++ local index=2 logger.go:42: 14:50:07 | gr-self-healing/10-read-from-replicas | ++ echo gr-self-healing-mysql-2.gr-self-healing-mysql logger.go:42: 14:50:07 | gr-self-healing/10-read-from-replicas | + host=gr-self-healing-mysql-2.gr-self-healing-mysql logger.go:42: 14:50:07 | gr-self-healing/10-read-from-replicas | ++ run_mysql 'SELECT * FROM myDB.myTable' '-h gr-self-healing-mysql-2.gr-self-healing-mysql -uroot -proot_password' logger.go:42: 14:50:07 | gr-self-healing/10-read-from-replicas | ++ local 'command=SELECT * FROM myDB.myTable' logger.go:42: 14:50:07 | gr-self-healing/10-read-from-replicas | ++ local 'uri=-h gr-self-healing-mysql-2.gr-self-healing-mysql -uroot -proot_password' logger.go:42: 14:50:07 | gr-self-healing/10-read-from-replicas | ++ local pod= logger.go:42: 14:50:07 | gr-self-healing/10-read-from-replicas | +++ get_client_pod logger.go:42: 14:50:07 | gr-self-healing/10-read-from-replicas | +++ kubectl -n kuttl-test-secure-shiner get pods --selector=name=mysql-client -o 'jsonpath={.items[].metadata.name}' logger.go:42: 14:50:08 | gr-self-healing/10-read-from-replicas | ++ client_pod=mysql-client logger.go:42: 14:50:08 | gr-self-healing/10-read-from-replicas | ++ wait_pod mysql-client logger.go:42: 14:50:08 | gr-self-healing/10-read-from-replicas | ++ local pod=mysql-client logger.go:42: 14:50:08 | gr-self-healing/10-read-from-replicas | ++ set +o xtrace logger.go:42: 14:50:09 | gr-self-healing/10-read-from-replicas | mysql-clienttrue logger.go:42: 14:50:09 | gr-self-healing/10-read-from-replicas | ++ kubectl -n kuttl-test-secure-shiner exec mysql-client -- bash -c 'printf '\''%s\n'\'' "SELECT * FROM myDB.myTable" | mysql -sN -h gr-self-healing-mysql-2.gr-self-healing-mysql -uroot -proot_password' logger.go:42: 14:50:09 | gr-self-healing/10-read-from-replicas | ++ sed -e 's/mysql: //' logger.go:42: 14:50:09 | gr-self-healing/10-read-from-replicas | ++ grep -v 'Using a password on the command line interface can be insecure.' logger.go:42: 14:50:10 | gr-self-healing/10-read-from-replicas | + data='100500 logger.go:42: 14:50:10 | gr-self-healing/10-read-from-replicas | 100501 logger.go:42: 14:50:10 | gr-self-healing/10-read-from-replicas | 100502' logger.go:42: 14:50:10 | gr-self-healing/10-read-from-replicas | + kubectl create configmap -n kuttl-test-secure-shiner 10-read-from-replicas-2 '--from-literal=data=100500 logger.go:42: 14:50:10 | gr-self-healing/10-read-from-replicas | 100501 logger.go:42: 14:50:10 | gr-self-healing/10-read-from-replicas | 100502' logger.go:42: 14:50:11 | gr-self-healing/10-read-from-replicas | configmap/10-read-from-replicas-2 created logger.go:42: 14:50:13 | gr-self-healing/10-read-from-replicas | test step completed 10-read-from-replicas logger.go:42: 14:50:13 | gr-self-healing/11-network-loss-primary | starting test step 11-network-loss-primary logger.go:42: 14:50:13 | gr-self-healing/11-network-loss-primary | running command: [sh -c set -o errexit set -o xtrace source ../../functions network_loss "${NAMESPACE}" "$(get_primary_from_group_replication)" "primary" sleep 30 # wait for new master to get elected] logger.go:42: 14:50:13 | gr-self-healing/11-network-loss-primary | + source ../../functions logger.go:42: 14:50:13 | gr-self-healing/11-network-loss-primary | +++ realpath ../../.. logger.go:42: 14:50:13 | gr-self-healing/11-network-loss-primary | ++ ROOT_REPO=/mnt/jenkins/workspace/cloud-ps-operator_PR-424 logger.go:42: 14:50:13 | gr-self-healing/11-network-loss-primary | ++ source /mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests/vars.sh logger.go:42: 14:50:13 | gr-self-healing/11-network-loss-primary | +++ export ROOT_REPO=/mnt/jenkins/workspace/cloud-ps-operator_PR-424 logger.go:42: 14:50:13 | gr-self-healing/11-network-loss-primary | +++ ROOT_REPO=/mnt/jenkins/workspace/cloud-ps-operator_PR-424 logger.go:42: 14:50:13 | gr-self-healing/11-network-loss-primary | +++ export DEPLOY_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/deploy logger.go:42: 14:50:13 | gr-self-healing/11-network-loss-primary | +++ DEPLOY_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/deploy logger.go:42: 14:50:13 | gr-self-healing/11-network-loss-primary | +++ export TESTS_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests logger.go:42: 14:50:13 | gr-self-healing/11-network-loss-primary | +++ TESTS_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests logger.go:42: 14:50:13 | gr-self-healing/11-network-loss-primary | +++ export TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests/conf logger.go:42: 14:50:13 | gr-self-healing/11-network-loss-primary | +++ TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests/conf logger.go:42: 14:50:13 | gr-self-healing/11-network-loss-primary | ++++ mktemp -d logger.go:42: 14:50:13 | gr-self-healing/11-network-loss-primary | +++ export TEMP_DIR=/tmp/tmp.wKPRGTzLoP logger.go:42: 14:50:13 | gr-self-healing/11-network-loss-primary | +++ TEMP_DIR=/tmp/tmp.wKPRGTzLoP logger.go:42: 14:50:13 | gr-self-healing/11-network-loss-primary | ++++ git rev-parse --abbrev-ref HEAD logger.go:42: 14:50:13 | gr-self-healing/11-network-loss-primary | +++ export GIT_BRANCH=PR-424 logger.go:42: 14:50:13 | gr-self-healing/11-network-loss-primary | +++ GIT_BRANCH=PR-424 logger.go:42: 14:50:13 | gr-self-healing/11-network-loss-primary | +++ export VERSION=PR-424-70568ae logger.go:42: 14:50:13 | gr-self-healing/11-network-loss-primary | +++ VERSION=PR-424-70568ae logger.go:42: 14:50:13 | gr-self-healing/11-network-loss-primary | +++ export IMAGE=perconalab/percona-server-mysql-operator:PR-424-70568ae logger.go:42: 14:50:13 | gr-self-healing/11-network-loss-primary | +++ IMAGE=perconalab/percona-server-mysql-operator:PR-424-70568ae logger.go:42: 14:50:13 | gr-self-healing/11-network-loss-primary | +++ export IMAGE_MYSQL=perconalab/percona-server-mysql-operator:main-psmysql logger.go:42: 14:50:13 | gr-self-healing/11-network-loss-primary | +++ IMAGE_MYSQL=perconalab/percona-server-mysql-operator:main-psmysql logger.go:42: 14:50:13 | gr-self-healing/11-network-loss-primary | +++ export IMAGE_BACKUP=perconalab/percona-server-mysql-operator:main-backup logger.go:42: 14:50:13 | gr-self-healing/11-network-loss-primary | +++ IMAGE_BACKUP=perconalab/percona-server-mysql-operator:main-backup logger.go:42: 14:50:13 | gr-self-healing/11-network-loss-primary | +++ export IMAGE_ORCHESTRATOR=perconalab/percona-server-mysql-operator:main-orchestrator logger.go:42: 14:50:13 | gr-self-healing/11-network-loss-primary | +++ IMAGE_ORCHESTRATOR=perconalab/percona-server-mysql-operator:main-orchestrator logger.go:42: 14:50:13 | gr-self-healing/11-network-loss-primary | +++ export IMAGE_ROUTER=perconalab/percona-server-mysql-operator:main-router logger.go:42: 14:50:13 | gr-self-healing/11-network-loss-primary | +++ IMAGE_ROUTER=perconalab/percona-server-mysql-operator:main-router logger.go:42: 14:50:13 | gr-self-healing/11-network-loss-primary | +++ export IMAGE_TOOLKIT=perconalab/percona-server-mysql-operator:main-toolkit logger.go:42: 14:50:13 | gr-self-healing/11-network-loss-primary | +++ IMAGE_TOOLKIT=perconalab/percona-server-mysql-operator:main-toolkit logger.go:42: 14:50:13 | gr-self-healing/11-network-loss-primary | +++ export IMAGE_HAPROXY=perconalab/percona-server-mysql-operator:main-haproxy logger.go:42: 14:50:13 | gr-self-healing/11-network-loss-primary | +++ IMAGE_HAPROXY=perconalab/percona-server-mysql-operator:main-haproxy logger.go:42: 14:50:13 | gr-self-healing/11-network-loss-primary | +++ export IMAGE_PMM=perconalab/pmm-client:dev-latest logger.go:42: 14:50:13 | gr-self-healing/11-network-loss-primary | +++ IMAGE_PMM=perconalab/pmm-client:dev-latest logger.go:42: 14:50:13 | gr-self-healing/11-network-loss-primary | +++ export PMM_SERVER_VERSION=9.9.9 logger.go:42: 14:50:13 | gr-self-healing/11-network-loss-primary | +++ PMM_SERVER_VERSION=9.9.9 logger.go:42: 14:50:13 | gr-self-healing/11-network-loss-primary | +++ export IMAGE_PMM_SERVER_REPO=perconalab/pmm-server logger.go:42: 14:50:13 | gr-self-healing/11-network-loss-primary | +++ IMAGE_PMM_SERVER_REPO=perconalab/pmm-server logger.go:42: 14:50:13 | gr-self-healing/11-network-loss-primary | +++ export IMAGE_PMM_SERVER_TAG=dev-latest logger.go:42: 14:50:13 | gr-self-healing/11-network-loss-primary | +++ IMAGE_PMM_SERVER_TAG=dev-latest logger.go:42: 14:50:13 | gr-self-healing/11-network-loss-primary | ++++ which gdate logger.go:42: 14:50:13 | gr-self-healing/11-network-loss-primary | which: no gdate in (/mnt/jenkins/workspace/cloud-ps-operator_PR-424/bin/:/home/ec2-user/google-cloud-sdk/bin:/home/ec2-user/.krew/bin:/usr/local/bin:/usr/bin) logger.go:42: 14:50:13 | gr-self-healing/11-network-loss-primary | ++++ which date logger.go:42: 14:50:13 | gr-self-healing/11-network-loss-primary | +++ date=/usr/bin/date logger.go:42: 14:50:13 | gr-self-healing/11-network-loss-primary | +++ command -v oc logger.go:42: 14:50:13 | gr-self-healing/11-network-loss-primary | +++ oc get projects logger.go:42: 14:50:18 | gr-self-healing/11-network-loss-primary | error: the server doesn't have a resource type "projects" logger.go:42: 14:50:18 | gr-self-healing/11-network-loss-primary | +++ kubectl get nodes logger.go:42: 14:50:18 | gr-self-healing/11-network-loss-primary | +++ grep '^minikube' logger.go:42: 14:50:19 | gr-self-healing/11-network-loss-primary | ++++ pwd logger.go:42: 14:50:19 | gr-self-healing/11-network-loss-primary | +++ basename /mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests/tests/gr-self-healing logger.go:42: 14:50:19 | gr-self-healing/11-network-loss-primary | ++ test_name=gr-self-healing logger.go:42: 14:50:19 | gr-self-healing/11-network-loss-primary | ++ get_primary_from_group_replication logger.go:42: 14:50:19 | gr-self-healing/11-network-loss-primary | ++ cut -d. -f1 logger.go:42: 14:50:19 | gr-self-healing/11-network-loss-primary | ++++ get_cluster_name logger.go:42: 14:50:19 | gr-self-healing/11-network-loss-primary | ++++ kubectl -n kuttl-test-secure-shiner get ps -o 'jsonpath={.items[0].metadata.name}' logger.go:42: 14:50:20 | gr-self-healing/11-network-loss-primary | +++ get_mysql_router_service gr-self-healing logger.go:42: 14:50:20 | gr-self-healing/11-network-loss-primary | +++ local cluster=gr-self-healing logger.go:42: 14:50:20 | gr-self-healing/11-network-loss-primary | +++ echo gr-self-healing-router logger.go:42: 14:50:20 | gr-self-healing/11-network-loss-primary | ++ run_mysql 'SELECT MEMBER_HOST FROM performance_schema.replication_group_members where MEMBER_ROLE='\''PRIMARY'\'';' '-h gr-self-healing-router -P 6446 -uroot -proot_password' logger.go:42: 14:50:20 | gr-self-healing/11-network-loss-primary | ++ local 'command=SELECT MEMBER_HOST FROM performance_schema.replication_group_members where MEMBER_ROLE='\''PRIMARY'\'';' logger.go:42: 14:50:20 | gr-self-healing/11-network-loss-primary | ++ local 'uri=-h gr-self-healing-router -P 6446 -uroot -proot_password' logger.go:42: 14:50:20 | gr-self-healing/11-network-loss-primary | ++ local pod= logger.go:42: 14:50:20 | gr-self-healing/11-network-loss-primary | +++ get_client_pod logger.go:42: 14:50:20 | gr-self-healing/11-network-loss-primary | +++ kubectl -n kuttl-test-secure-shiner get pods --selector=name=mysql-client -o 'jsonpath={.items[].metadata.name}' logger.go:42: 14:50:21 | gr-self-healing/11-network-loss-primary | ++ client_pod=mysql-client logger.go:42: 14:50:21 | gr-self-healing/11-network-loss-primary | ++ wait_pod mysql-client logger.go:42: 14:50:21 | gr-self-healing/11-network-loss-primary | ++ local pod=mysql-client logger.go:42: 14:50:21 | gr-self-healing/11-network-loss-primary | ++ set +o xtrace logger.go:42: 14:50:21 | gr-self-healing/11-network-loss-primary | mysql-clienttrue logger.go:42: 14:50:21 | gr-self-healing/11-network-loss-primary | ++ sed -e 's/mysql: //' logger.go:42: 14:50:21 | gr-self-healing/11-network-loss-primary | ++ kubectl -n kuttl-test-secure-shiner exec mysql-client -- bash -c 'printf '\''%s\n'\'' "SELECT MEMBER_HOST FROM performance_schema.replication_group_members where MEMBER_ROLE='\''PRIMARY'\'';" | mysql -sN -h gr-self-healing-router -P 6446 -uroot -proot_password' logger.go:42: 14:50:21 | gr-self-healing/11-network-loss-primary | ++ grep -v 'Using a password on the command line interface can be insecure.' logger.go:42: 14:50:23 | gr-self-healing/11-network-loss-primary | + network_loss kuttl-test-secure-shiner gr-self-healing-mysql-0 primary logger.go:42: 14:50:23 | gr-self-healing/11-network-loss-primary | + local ns=kuttl-test-secure-shiner logger.go:42: 14:50:23 | gr-self-healing/11-network-loss-primary | + local pod=gr-self-healing-mysql-0 logger.go:42: 14:50:23 | gr-self-healing/11-network-loss-primary | + local chaos_suffix=primary logger.go:42: 14:50:23 | gr-self-healing/11-network-loss-primary | + yq eval ' logger.go:42: 14:50:23 | gr-self-healing/11-network-loss-primary | .metadata.name = "chaos-pod-network-loss-primary" | logger.go:42: 14:50:23 | gr-self-healing/11-network-loss-primary | del(.spec.selector.pods.test-namespace) | logger.go:42: 14:50:23 | gr-self-healing/11-network-loss-primary | .spec.selector.pods.kuttl-test-secure-shiner[0] = "gr-self-healing-mysql-0"' /mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests/conf/chaos-network-loss.yml logger.go:42: 14:50:23 | gr-self-healing/11-network-loss-primary | + kubectl apply --namespace kuttl-test-secure-shiner -f - logger.go:42: 14:50:24 | gr-self-healing/11-network-loss-primary | networkchaos.chaos-mesh.org/chaos-pod-network-loss-primary created logger.go:42: 14:50:24 | gr-self-healing/11-network-loss-primary | + sleep 5 logger.go:42: 14:50:29 | gr-self-healing/11-network-loss-primary | + sleep 30 logger.go:42: 14:52:39 | gr-self-healing/11-network-loss-primary | test step completed 11-network-loss-primary logger.go:42: 14:52:39 | gr-self-healing/12-write-data | starting test step 12-write-data logger.go:42: 14:52:39 | gr-self-healing/12-write-data | running command: [sh -c set -o errexit set -o xtrace source ../../functions run_mysql \ "INSERT myDB.myTable (id) VALUES (100503)" \ "-h $(get_mysql_router_service $(get_cluster_name)) -P 6446 -uroot -proot_password"] logger.go:42: 14:52:39 | gr-self-healing/12-write-data | + source ../../functions logger.go:42: 14:52:39 | gr-self-healing/12-write-data | +++ realpath ../../.. logger.go:42: 14:52:39 | gr-self-healing/12-write-data | ++ ROOT_REPO=/mnt/jenkins/workspace/cloud-ps-operator_PR-424 logger.go:42: 14:52:39 | gr-self-healing/12-write-data | ++ source /mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests/vars.sh logger.go:42: 14:52:39 | gr-self-healing/12-write-data | +++ export ROOT_REPO=/mnt/jenkins/workspace/cloud-ps-operator_PR-424 logger.go:42: 14:52:39 | gr-self-healing/12-write-data | +++ ROOT_REPO=/mnt/jenkins/workspace/cloud-ps-operator_PR-424 logger.go:42: 14:52:39 | gr-self-healing/12-write-data | +++ export DEPLOY_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/deploy logger.go:42: 14:52:39 | gr-self-healing/12-write-data | +++ DEPLOY_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/deploy logger.go:42: 14:52:39 | gr-self-healing/12-write-data | +++ export TESTS_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests logger.go:42: 14:52:39 | gr-self-healing/12-write-data | +++ TESTS_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests logger.go:42: 14:52:39 | gr-self-healing/12-write-data | +++ export TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests/conf logger.go:42: 14:52:39 | gr-self-healing/12-write-data | +++ TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests/conf logger.go:42: 14:52:39 | gr-self-healing/12-write-data | ++++ mktemp -d logger.go:42: 14:52:39 | gr-self-healing/12-write-data | +++ export TEMP_DIR=/tmp/tmp.pQzo1696r2 logger.go:42: 14:52:39 | gr-self-healing/12-write-data | +++ TEMP_DIR=/tmp/tmp.pQzo1696r2 logger.go:42: 14:52:39 | gr-self-healing/12-write-data | ++++ git rev-parse --abbrev-ref HEAD logger.go:42: 14:52:39 | gr-self-healing/12-write-data | +++ export GIT_BRANCH=PR-424 logger.go:42: 14:52:39 | gr-self-healing/12-write-data | +++ GIT_BRANCH=PR-424 logger.go:42: 14:52:39 | gr-self-healing/12-write-data | +++ export VERSION=PR-424-70568ae logger.go:42: 14:52:39 | gr-self-healing/12-write-data | +++ VERSION=PR-424-70568ae logger.go:42: 14:52:39 | gr-self-healing/12-write-data | +++ export IMAGE=perconalab/percona-server-mysql-operator:PR-424-70568ae logger.go:42: 14:52:39 | gr-self-healing/12-write-data | +++ IMAGE=perconalab/percona-server-mysql-operator:PR-424-70568ae logger.go:42: 14:52:39 | gr-self-healing/12-write-data | +++ export IMAGE_MYSQL=perconalab/percona-server-mysql-operator:main-psmysql logger.go:42: 14:52:39 | gr-self-healing/12-write-data | +++ IMAGE_MYSQL=perconalab/percona-server-mysql-operator:main-psmysql logger.go:42: 14:52:39 | gr-self-healing/12-write-data | +++ export IMAGE_BACKUP=perconalab/percona-server-mysql-operator:main-backup logger.go:42: 14:52:39 | gr-self-healing/12-write-data | +++ IMAGE_BACKUP=perconalab/percona-server-mysql-operator:main-backup logger.go:42: 14:52:39 | gr-self-healing/12-write-data | +++ export IMAGE_ORCHESTRATOR=perconalab/percona-server-mysql-operator:main-orchestrator logger.go:42: 14:52:39 | gr-self-healing/12-write-data | +++ IMAGE_ORCHESTRATOR=perconalab/percona-server-mysql-operator:main-orchestrator logger.go:42: 14:52:39 | gr-self-healing/12-write-data | +++ export IMAGE_ROUTER=perconalab/percona-server-mysql-operator:main-router logger.go:42: 14:52:39 | gr-self-healing/12-write-data | +++ IMAGE_ROUTER=perconalab/percona-server-mysql-operator:main-router logger.go:42: 14:52:39 | gr-self-healing/12-write-data | +++ export IMAGE_TOOLKIT=perconalab/percona-server-mysql-operator:main-toolkit logger.go:42: 14:52:39 | gr-self-healing/12-write-data | +++ IMAGE_TOOLKIT=perconalab/percona-server-mysql-operator:main-toolkit logger.go:42: 14:52:39 | gr-self-healing/12-write-data | +++ export IMAGE_HAPROXY=perconalab/percona-server-mysql-operator:main-haproxy logger.go:42: 14:52:39 | gr-self-healing/12-write-data | +++ IMAGE_HAPROXY=perconalab/percona-server-mysql-operator:main-haproxy logger.go:42: 14:52:39 | gr-self-healing/12-write-data | +++ export IMAGE_PMM=perconalab/pmm-client:dev-latest logger.go:42: 14:52:39 | gr-self-healing/12-write-data | +++ IMAGE_PMM=perconalab/pmm-client:dev-latest logger.go:42: 14:52:39 | gr-self-healing/12-write-data | +++ export PMM_SERVER_VERSION=9.9.9 logger.go:42: 14:52:39 | gr-self-healing/12-write-data | +++ PMM_SERVER_VERSION=9.9.9 logger.go:42: 14:52:39 | gr-self-healing/12-write-data | +++ export IMAGE_PMM_SERVER_REPO=perconalab/pmm-server logger.go:42: 14:52:39 | gr-self-healing/12-write-data | +++ IMAGE_PMM_SERVER_REPO=perconalab/pmm-server logger.go:42: 14:52:39 | gr-self-healing/12-write-data | +++ export IMAGE_PMM_SERVER_TAG=dev-latest logger.go:42: 14:52:39 | gr-self-healing/12-write-data | +++ IMAGE_PMM_SERVER_TAG=dev-latest logger.go:42: 14:52:39 | gr-self-healing/12-write-data | ++++ which gdate logger.go:42: 14:52:39 | gr-self-healing/12-write-data | which: no gdate in (/mnt/jenkins/workspace/cloud-ps-operator_PR-424/bin/:/home/ec2-user/google-cloud-sdk/bin:/home/ec2-user/.krew/bin:/usr/local/bin:/usr/bin) logger.go:42: 14:52:39 | gr-self-healing/12-write-data | ++++ which date logger.go:42: 14:52:39 | gr-self-healing/12-write-data | +++ date=/usr/bin/date logger.go:42: 14:52:39 | gr-self-healing/12-write-data | +++ command -v oc logger.go:42: 14:52:39 | gr-self-healing/12-write-data | +++ oc get projects logger.go:42: 14:52:45 | gr-self-healing/12-write-data | error: the server doesn't have a resource type "projects" logger.go:42: 14:52:45 | gr-self-healing/12-write-data | +++ grep '^minikube' logger.go:42: 14:52:45 | gr-self-healing/12-write-data | +++ kubectl get nodes logger.go:42: 14:52:45 | gr-self-healing/12-write-data | ++++ pwd logger.go:42: 14:52:45 | gr-self-healing/12-write-data | +++ basename /mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests/tests/gr-self-healing logger.go:42: 14:52:45 | gr-self-healing/12-write-data | ++ test_name=gr-self-healing logger.go:42: 14:52:45 | gr-self-healing/12-write-data | +++ get_cluster_name logger.go:42: 14:52:45 | gr-self-healing/12-write-data | +++ kubectl -n kuttl-test-secure-shiner get ps -o 'jsonpath={.items[0].metadata.name}' logger.go:42: 14:52:46 | gr-self-healing/12-write-data | ++ get_mysql_router_service gr-self-healing logger.go:42: 14:52:46 | gr-self-healing/12-write-data | ++ local cluster=gr-self-healing logger.go:42: 14:52:46 | gr-self-healing/12-write-data | ++ echo gr-self-healing-router logger.go:42: 14:52:46 | gr-self-healing/12-write-data | + run_mysql 'INSERT myDB.myTable (id) VALUES (100503)' '-h gr-self-healing-router -P 6446 -uroot -proot_password' logger.go:42: 14:52:46 | gr-self-healing/12-write-data | + local 'command=INSERT myDB.myTable (id) VALUES (100503)' logger.go:42: 14:52:46 | gr-self-healing/12-write-data | + local 'uri=-h gr-self-healing-router -P 6446 -uroot -proot_password' logger.go:42: 14:52:46 | gr-self-healing/12-write-data | + local pod= logger.go:42: 14:52:46 | gr-self-healing/12-write-data | ++ get_client_pod logger.go:42: 14:52:46 | gr-self-healing/12-write-data | ++ kubectl -n kuttl-test-secure-shiner get pods --selector=name=mysql-client -o 'jsonpath={.items[].metadata.name}' logger.go:42: 14:52:47 | gr-self-healing/12-write-data | + client_pod=mysql-client logger.go:42: 14:52:47 | gr-self-healing/12-write-data | + wait_pod mysql-client logger.go:42: 14:52:47 | gr-self-healing/12-write-data | + local pod=mysql-client logger.go:42: 14:52:47 | gr-self-healing/12-write-data | + set +o xtrace logger.go:42: 14:52:48 | gr-self-healing/12-write-data | mysql-clienttrue logger.go:42: 14:52:48 | gr-self-healing/12-write-data | + kubectl -n kuttl-test-secure-shiner exec mysql-client -- bash -c 'printf '\''%s\n'\'' "INSERT myDB.myTable (id) VALUES (100503)" | mysql -sN -h gr-self-healing-router -P 6446 -uroot -proot_password' logger.go:42: 14:52:48 | gr-self-healing/12-write-data | + sed -e 's/mysql: //' logger.go:42: 14:52:48 | gr-self-healing/12-write-data | + grep -v 'Using a password on the command line interface can be insecure.' logger.go:42: 14:52:49 | gr-self-healing/12-write-data | + : logger.go:42: 14:52:50 | gr-self-healing/12-write-data | test step completed 12-write-data logger.go:42: 14:52:50 | gr-self-healing/13-read-from-replicas | starting test step 13-read-from-replicas logger.go:42: 14:52:50 | gr-self-healing/13-read-from-replicas | running command: [sh -c set -o errexit set -o xtrace source ../../functions for i in 0 1 2; do host=$(get_mysql_headless_fqdn $(get_cluster_name) $i) data=$(run_mysql "SELECT * FROM myDB.myTable" "-h ${host} -uroot -proot_password") kubectl create configmap -n "${NAMESPACE}" 13-read-from-replicas-${i} --from-literal=data="${data}" done] logger.go:42: 14:52:50 | gr-self-healing/13-read-from-replicas | + source ../../functions logger.go:42: 14:52:50 | gr-self-healing/13-read-from-replicas | +++ realpath ../../.. logger.go:42: 14:52:50 | gr-self-healing/13-read-from-replicas | ++ ROOT_REPO=/mnt/jenkins/workspace/cloud-ps-operator_PR-424 logger.go:42: 14:52:50 | gr-self-healing/13-read-from-replicas | ++ source /mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests/vars.sh logger.go:42: 14:52:50 | gr-self-healing/13-read-from-replicas | +++ export ROOT_REPO=/mnt/jenkins/workspace/cloud-ps-operator_PR-424 logger.go:42: 14:52:50 | gr-self-healing/13-read-from-replicas | +++ ROOT_REPO=/mnt/jenkins/workspace/cloud-ps-operator_PR-424 logger.go:42: 14:52:50 | gr-self-healing/13-read-from-replicas | +++ export DEPLOY_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/deploy logger.go:42: 14:52:50 | gr-self-healing/13-read-from-replicas | +++ DEPLOY_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/deploy logger.go:42: 14:52:50 | gr-self-healing/13-read-from-replicas | +++ export TESTS_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests logger.go:42: 14:52:50 | gr-self-healing/13-read-from-replicas | +++ TESTS_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests logger.go:42: 14:52:50 | gr-self-healing/13-read-from-replicas | +++ export TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests/conf logger.go:42: 14:52:50 | gr-self-healing/13-read-from-replicas | +++ TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests/conf logger.go:42: 14:52:50 | gr-self-healing/13-read-from-replicas | ++++ mktemp -d logger.go:42: 14:52:50 | gr-self-healing/13-read-from-replicas | +++ export TEMP_DIR=/tmp/tmp.4tjFY14iLl logger.go:42: 14:52:50 | gr-self-healing/13-read-from-replicas | +++ TEMP_DIR=/tmp/tmp.4tjFY14iLl logger.go:42: 14:52:50 | gr-self-healing/13-read-from-replicas | ++++ git rev-parse --abbrev-ref HEAD logger.go:42: 14:52:50 | gr-self-healing/13-read-from-replicas | +++ export GIT_BRANCH=PR-424 logger.go:42: 14:52:50 | gr-self-healing/13-read-from-replicas | +++ GIT_BRANCH=PR-424 logger.go:42: 14:52:50 | gr-self-healing/13-read-from-replicas | +++ export VERSION=PR-424-70568ae logger.go:42: 14:52:50 | gr-self-healing/13-read-from-replicas | +++ VERSION=PR-424-70568ae logger.go:42: 14:52:50 | gr-self-healing/13-read-from-replicas | +++ export IMAGE=perconalab/percona-server-mysql-operator:PR-424-70568ae logger.go:42: 14:52:50 | gr-self-healing/13-read-from-replicas | +++ IMAGE=perconalab/percona-server-mysql-operator:PR-424-70568ae logger.go:42: 14:52:50 | gr-self-healing/13-read-from-replicas | +++ export IMAGE_MYSQL=perconalab/percona-server-mysql-operator:main-psmysql logger.go:42: 14:52:50 | gr-self-healing/13-read-from-replicas | +++ IMAGE_MYSQL=perconalab/percona-server-mysql-operator:main-psmysql logger.go:42: 14:52:50 | gr-self-healing/13-read-from-replicas | +++ export IMAGE_BACKUP=perconalab/percona-server-mysql-operator:main-backup logger.go:42: 14:52:50 | gr-self-healing/13-read-from-replicas | +++ IMAGE_BACKUP=perconalab/percona-server-mysql-operator:main-backup logger.go:42: 14:52:50 | gr-self-healing/13-read-from-replicas | +++ export IMAGE_ORCHESTRATOR=perconalab/percona-server-mysql-operator:main-orchestrator logger.go:42: 14:52:50 | gr-self-healing/13-read-from-replicas | +++ IMAGE_ORCHESTRATOR=perconalab/percona-server-mysql-operator:main-orchestrator logger.go:42: 14:52:50 | gr-self-healing/13-read-from-replicas | +++ export IMAGE_ROUTER=perconalab/percona-server-mysql-operator:main-router logger.go:42: 14:52:50 | gr-self-healing/13-read-from-replicas | +++ IMAGE_ROUTER=perconalab/percona-server-mysql-operator:main-router logger.go:42: 14:52:50 | gr-self-healing/13-read-from-replicas | +++ export IMAGE_TOOLKIT=perconalab/percona-server-mysql-operator:main-toolkit logger.go:42: 14:52:50 | gr-self-healing/13-read-from-replicas | +++ IMAGE_TOOLKIT=perconalab/percona-server-mysql-operator:main-toolkit logger.go:42: 14:52:50 | gr-self-healing/13-read-from-replicas | +++ export IMAGE_HAPROXY=perconalab/percona-server-mysql-operator:main-haproxy logger.go:42: 14:52:50 | gr-self-healing/13-read-from-replicas | +++ IMAGE_HAPROXY=perconalab/percona-server-mysql-operator:main-haproxy logger.go:42: 14:52:50 | gr-self-healing/13-read-from-replicas | +++ export IMAGE_PMM=perconalab/pmm-client:dev-latest logger.go:42: 14:52:50 | gr-self-healing/13-read-from-replicas | +++ IMAGE_PMM=perconalab/pmm-client:dev-latest logger.go:42: 14:52:50 | gr-self-healing/13-read-from-replicas | +++ export PMM_SERVER_VERSION=9.9.9 logger.go:42: 14:52:50 | gr-self-healing/13-read-from-replicas | +++ PMM_SERVER_VERSION=9.9.9 logger.go:42: 14:52:50 | gr-self-healing/13-read-from-replicas | +++ export IMAGE_PMM_SERVER_REPO=perconalab/pmm-server logger.go:42: 14:52:50 | gr-self-healing/13-read-from-replicas | +++ IMAGE_PMM_SERVER_REPO=perconalab/pmm-server logger.go:42: 14:52:50 | gr-self-healing/13-read-from-replicas | +++ export IMAGE_PMM_SERVER_TAG=dev-latest logger.go:42: 14:52:50 | gr-self-healing/13-read-from-replicas | +++ IMAGE_PMM_SERVER_TAG=dev-latest logger.go:42: 14:52:50 | gr-self-healing/13-read-from-replicas | ++++ which gdate logger.go:42: 14:52:50 | gr-self-healing/13-read-from-replicas | which: no gdate in (/mnt/jenkins/workspace/cloud-ps-operator_PR-424/bin/:/home/ec2-user/google-cloud-sdk/bin:/home/ec2-user/.krew/bin:/usr/local/bin:/usr/bin) logger.go:42: 14:52:50 | gr-self-healing/13-read-from-replicas | ++++ which date logger.go:42: 14:52:50 | gr-self-healing/13-read-from-replicas | +++ date=/usr/bin/date logger.go:42: 14:52:50 | gr-self-healing/13-read-from-replicas | +++ command -v oc logger.go:42: 14:52:50 | gr-self-healing/13-read-from-replicas | +++ oc get projects logger.go:42: 14:52:56 | gr-self-healing/13-read-from-replicas | error: the server doesn't have a resource type "projects" logger.go:42: 14:52:56 | gr-self-healing/13-read-from-replicas | +++ kubectl get nodes logger.go:42: 14:52:56 | gr-self-healing/13-read-from-replicas | +++ grep '^minikube' logger.go:42: 14:52:57 | gr-self-healing/13-read-from-replicas | ++++ pwd logger.go:42: 14:52:57 | gr-self-healing/13-read-from-replicas | +++ basename /mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests/tests/gr-self-healing logger.go:42: 14:52:57 | gr-self-healing/13-read-from-replicas | ++ test_name=gr-self-healing logger.go:42: 14:52:57 | gr-self-healing/13-read-from-replicas | + for i in 0 1 2 logger.go:42: 14:52:57 | gr-self-healing/13-read-from-replicas | +++ get_cluster_name logger.go:42: 14:52:57 | gr-self-healing/13-read-from-replicas | +++ kubectl -n kuttl-test-secure-shiner get ps -o 'jsonpath={.items[0].metadata.name}' logger.go:42: 14:52:58 | gr-self-healing/13-read-from-replicas | ++ get_mysql_headless_fqdn gr-self-healing 0 logger.go:42: 14:52:58 | gr-self-healing/13-read-from-replicas | ++ local cluster=gr-self-healing logger.go:42: 14:52:58 | gr-self-healing/13-read-from-replicas | ++ local index=0 logger.go:42: 14:52:58 | gr-self-healing/13-read-from-replicas | ++ echo gr-self-healing-mysql-0.gr-self-healing-mysql logger.go:42: 14:52:58 | gr-self-healing/13-read-from-replicas | + host=gr-self-healing-mysql-0.gr-self-healing-mysql logger.go:42: 14:52:58 | gr-self-healing/13-read-from-replicas | ++ run_mysql 'SELECT * FROM myDB.myTable' '-h gr-self-healing-mysql-0.gr-self-healing-mysql -uroot -proot_password' logger.go:42: 14:52:58 | gr-self-healing/13-read-from-replicas | ++ local 'command=SELECT * FROM myDB.myTable' logger.go:42: 14:52:58 | gr-self-healing/13-read-from-replicas | ++ local 'uri=-h gr-self-healing-mysql-0.gr-self-healing-mysql -uroot -proot_password' logger.go:42: 14:52:58 | gr-self-healing/13-read-from-replicas | ++ local pod= logger.go:42: 14:52:58 | gr-self-healing/13-read-from-replicas | +++ get_client_pod logger.go:42: 14:52:58 | gr-self-healing/13-read-from-replicas | +++ kubectl -n kuttl-test-secure-shiner get pods --selector=name=mysql-client -o 'jsonpath={.items[].metadata.name}' logger.go:42: 14:52:58 | gr-self-healing/13-read-from-replicas | ++ client_pod=mysql-client logger.go:42: 14:52:58 | gr-self-healing/13-read-from-replicas | ++ wait_pod mysql-client logger.go:42: 14:52:58 | gr-self-healing/13-read-from-replicas | ++ local pod=mysql-client logger.go:42: 14:52:58 | gr-self-healing/13-read-from-replicas | ++ set +o xtrace logger.go:42: 14:52:59 | gr-self-healing/13-read-from-replicas | mysql-clienttrue logger.go:42: 14:52:59 | gr-self-healing/13-read-from-replicas | ++ kubectl -n kuttl-test-secure-shiner exec mysql-client -- bash -c 'printf '\''%s\n'\'' "SELECT * FROM myDB.myTable" | mysql -sN -h gr-self-healing-mysql-0.gr-self-healing-mysql -uroot -proot_password' logger.go:42: 14:52:59 | gr-self-healing/13-read-from-replicas | ++ sed -e 's/mysql: //' logger.go:42: 14:52:59 | gr-self-healing/13-read-from-replicas | ++ grep -v 'Using a password on the command line interface can be insecure.' logger.go:42: 14:53:00 | gr-self-healing/13-read-from-replicas | + data='100500 logger.go:42: 14:53:00 | gr-self-healing/13-read-from-replicas | 100501 logger.go:42: 14:53:00 | gr-self-healing/13-read-from-replicas | 100502 logger.go:42: 14:53:00 | gr-self-healing/13-read-from-replicas | 100503' logger.go:42: 14:53:00 | gr-self-healing/13-read-from-replicas | + kubectl create configmap -n kuttl-test-secure-shiner 13-read-from-replicas-0 '--from-literal=data=100500 logger.go:42: 14:53:00 | gr-self-healing/13-read-from-replicas | 100501 logger.go:42: 14:53:00 | gr-self-healing/13-read-from-replicas | 100502 logger.go:42: 14:53:00 | gr-self-healing/13-read-from-replicas | 100503' logger.go:42: 14:53:01 | gr-self-healing/13-read-from-replicas | configmap/13-read-from-replicas-0 created logger.go:42: 14:53:01 | gr-self-healing/13-read-from-replicas | + for i in 0 1 2 logger.go:42: 14:53:01 | gr-self-healing/13-read-from-replicas | +++ get_cluster_name logger.go:42: 14:53:01 | gr-self-healing/13-read-from-replicas | +++ kubectl -n kuttl-test-secure-shiner get ps -o 'jsonpath={.items[0].metadata.name}' logger.go:42: 14:53:02 | gr-self-healing/13-read-from-replicas | ++ get_mysql_headless_fqdn gr-self-healing 1 logger.go:42: 14:53:02 | gr-self-healing/13-read-from-replicas | ++ local cluster=gr-self-healing logger.go:42: 14:53:02 | gr-self-healing/13-read-from-replicas | ++ local index=1 logger.go:42: 14:53:02 | gr-self-healing/13-read-from-replicas | ++ echo gr-self-healing-mysql-1.gr-self-healing-mysql logger.go:42: 14:53:02 | gr-self-healing/13-read-from-replicas | + host=gr-self-healing-mysql-1.gr-self-healing-mysql logger.go:42: 14:53:02 | gr-self-healing/13-read-from-replicas | ++ run_mysql 'SELECT * FROM myDB.myTable' '-h gr-self-healing-mysql-1.gr-self-healing-mysql -uroot -proot_password' logger.go:42: 14:53:02 | gr-self-healing/13-read-from-replicas | ++ local 'command=SELECT * FROM myDB.myTable' logger.go:42: 14:53:02 | gr-self-healing/13-read-from-replicas | ++ local 'uri=-h gr-self-healing-mysql-1.gr-self-healing-mysql -uroot -proot_password' logger.go:42: 14:53:02 | gr-self-healing/13-read-from-replicas | ++ local pod= logger.go:42: 14:53:02 | gr-self-healing/13-read-from-replicas | +++ get_client_pod logger.go:42: 14:53:02 | gr-self-healing/13-read-from-replicas | +++ kubectl -n kuttl-test-secure-shiner get pods --selector=name=mysql-client -o 'jsonpath={.items[].metadata.name}' logger.go:42: 14:53:03 | gr-self-healing/13-read-from-replicas | ++ client_pod=mysql-client logger.go:42: 14:53:03 | gr-self-healing/13-read-from-replicas | ++ wait_pod mysql-client logger.go:42: 14:53:03 | gr-self-healing/13-read-from-replicas | ++ local pod=mysql-client logger.go:42: 14:53:03 | gr-self-healing/13-read-from-replicas | ++ set +o xtrace logger.go:42: 14:53:03 | gr-self-healing/13-read-from-replicas | mysql-clienttrue logger.go:42: 14:53:03 | gr-self-healing/13-read-from-replicas | ++ kubectl -n kuttl-test-secure-shiner exec mysql-client -- bash -c 'printf '\''%s\n'\'' "SELECT * FROM myDB.myTable" | mysql -sN -h gr-self-healing-mysql-1.gr-self-healing-mysql -uroot -proot_password' logger.go:42: 14:53:03 | gr-self-healing/13-read-from-replicas | ++ sed -e 's/mysql: //' logger.go:42: 14:53:03 | gr-self-healing/13-read-from-replicas | ++ grep -v 'Using a password on the command line interface can be insecure.' logger.go:42: 14:53:05 | gr-self-healing/13-read-from-replicas | + data='100500 logger.go:42: 14:53:05 | gr-self-healing/13-read-from-replicas | 100501 logger.go:42: 14:53:05 | gr-self-healing/13-read-from-replicas | 100502 logger.go:42: 14:53:05 | gr-self-healing/13-read-from-replicas | 100503' logger.go:42: 14:53:05 | gr-self-healing/13-read-from-replicas | + kubectl create configmap -n kuttl-test-secure-shiner 13-read-from-replicas-1 '--from-literal=data=100500 logger.go:42: 14:53:05 | gr-self-healing/13-read-from-replicas | 100501 logger.go:42: 14:53:05 | gr-self-healing/13-read-from-replicas | 100502 logger.go:42: 14:53:05 | gr-self-healing/13-read-from-replicas | 100503' logger.go:42: 14:53:06 | gr-self-healing/13-read-from-replicas | configmap/13-read-from-replicas-1 created logger.go:42: 14:53:06 | gr-self-healing/13-read-from-replicas | + for i in 0 1 2 logger.go:42: 14:53:06 | gr-self-healing/13-read-from-replicas | +++ get_cluster_name logger.go:42: 14:53:06 | gr-self-healing/13-read-from-replicas | +++ kubectl -n kuttl-test-secure-shiner get ps -o 'jsonpath={.items[0].metadata.name}' logger.go:42: 14:53:06 | gr-self-healing/13-read-from-replicas | ++ get_mysql_headless_fqdn gr-self-healing 2 logger.go:42: 14:53:06 | gr-self-healing/13-read-from-replicas | ++ local cluster=gr-self-healing logger.go:42: 14:53:06 | gr-self-healing/13-read-from-replicas | ++ local index=2 logger.go:42: 14:53:06 | gr-self-healing/13-read-from-replicas | ++ echo gr-self-healing-mysql-2.gr-self-healing-mysql logger.go:42: 14:53:06 | gr-self-healing/13-read-from-replicas | + host=gr-self-healing-mysql-2.gr-self-healing-mysql logger.go:42: 14:53:06 | gr-self-healing/13-read-from-replicas | ++ run_mysql 'SELECT * FROM myDB.myTable' '-h gr-self-healing-mysql-2.gr-self-healing-mysql -uroot -proot_password' logger.go:42: 14:53:06 | gr-self-healing/13-read-from-replicas | ++ local 'command=SELECT * FROM myDB.myTable' logger.go:42: 14:53:06 | gr-self-healing/13-read-from-replicas | ++ local 'uri=-h gr-self-healing-mysql-2.gr-self-healing-mysql -uroot -proot_password' logger.go:42: 14:53:06 | gr-self-healing/13-read-from-replicas | ++ local pod= logger.go:42: 14:53:06 | gr-self-healing/13-read-from-replicas | +++ get_client_pod logger.go:42: 14:53:06 | gr-self-healing/13-read-from-replicas | +++ kubectl -n kuttl-test-secure-shiner get pods --selector=name=mysql-client -o 'jsonpath={.items[].metadata.name}' logger.go:42: 14:53:07 | gr-self-healing/13-read-from-replicas | ++ client_pod=mysql-client logger.go:42: 14:53:07 | gr-self-healing/13-read-from-replicas | ++ wait_pod mysql-client logger.go:42: 14:53:07 | gr-self-healing/13-read-from-replicas | ++ local pod=mysql-client logger.go:42: 14:53:07 | gr-self-healing/13-read-from-replicas | ++ set +o xtrace logger.go:42: 14:53:08 | gr-self-healing/13-read-from-replicas | mysql-clienttrue logger.go:42: 14:53:08 | gr-self-healing/13-read-from-replicas | ++ kubectl -n kuttl-test-secure-shiner exec mysql-client -- bash -c 'printf '\''%s\n'\'' "SELECT * FROM myDB.myTable" | mysql -sN -h gr-self-healing-mysql-2.gr-self-healing-mysql -uroot -proot_password' logger.go:42: 14:53:08 | gr-self-healing/13-read-from-replicas | ++ sed -e 's/mysql: //' logger.go:42: 14:53:08 | gr-self-healing/13-read-from-replicas | ++ grep -v 'Using a password on the command line interface can be insecure.' logger.go:42: 14:53:09 | gr-self-healing/13-read-from-replicas | + data='100500 logger.go:42: 14:53:09 | gr-self-healing/13-read-from-replicas | 100501 logger.go:42: 14:53:09 | gr-self-healing/13-read-from-replicas | 100502 logger.go:42: 14:53:09 | gr-self-healing/13-read-from-replicas | 100503' logger.go:42: 14:53:09 | gr-self-healing/13-read-from-replicas | + kubectl create configmap -n kuttl-test-secure-shiner 13-read-from-replicas-2 '--from-literal=data=100500 logger.go:42: 14:53:09 | gr-self-healing/13-read-from-replicas | 100501 logger.go:42: 14:53:09 | gr-self-healing/13-read-from-replicas | 100502 logger.go:42: 14:53:09 | gr-self-healing/13-read-from-replicas | 100503' logger.go:42: 14:53:10 | gr-self-healing/13-read-from-replicas | configmap/13-read-from-replicas-2 created logger.go:42: 14:53:12 | gr-self-healing/13-read-from-replicas | test step completed 13-read-from-replicas logger.go:42: 14:53:12 | gr-self-healing/14-cluster-crash | starting test step 14-cluster-crash logger.go:42: 14:53:12 | gr-self-healing/14-cluster-crash | running command: [sh -c set -o errexit set -o xtrace source ../../functions kill_pods "${NAMESPACE}" "label" "app.kubernetes.io/instance" "gr-self-healing" "cluster-crash" sleep 30 # wait for crash] logger.go:42: 14:53:12 | gr-self-healing/14-cluster-crash | + source ../../functions logger.go:42: 14:53:12 | gr-self-healing/14-cluster-crash | +++ realpath ../../.. logger.go:42: 14:53:12 | gr-self-healing/14-cluster-crash | ++ ROOT_REPO=/mnt/jenkins/workspace/cloud-ps-operator_PR-424 logger.go:42: 14:53:12 | gr-self-healing/14-cluster-crash | ++ source /mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests/vars.sh logger.go:42: 14:53:12 | gr-self-healing/14-cluster-crash | +++ export ROOT_REPO=/mnt/jenkins/workspace/cloud-ps-operator_PR-424 logger.go:42: 14:53:12 | gr-self-healing/14-cluster-crash | +++ ROOT_REPO=/mnt/jenkins/workspace/cloud-ps-operator_PR-424 logger.go:42: 14:53:12 | gr-self-healing/14-cluster-crash | +++ export DEPLOY_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/deploy logger.go:42: 14:53:12 | gr-self-healing/14-cluster-crash | +++ DEPLOY_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/deploy logger.go:42: 14:53:12 | gr-self-healing/14-cluster-crash | +++ export TESTS_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests logger.go:42: 14:53:12 | gr-self-healing/14-cluster-crash | +++ TESTS_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests logger.go:42: 14:53:12 | gr-self-healing/14-cluster-crash | +++ export TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests/conf logger.go:42: 14:53:12 | gr-self-healing/14-cluster-crash | +++ TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests/conf logger.go:42: 14:53:12 | gr-self-healing/14-cluster-crash | ++++ mktemp -d logger.go:42: 14:53:12 | gr-self-healing/14-cluster-crash | +++ export TEMP_DIR=/tmp/tmp.koRe6YZ48O logger.go:42: 14:53:12 | gr-self-healing/14-cluster-crash | +++ TEMP_DIR=/tmp/tmp.koRe6YZ48O logger.go:42: 14:53:12 | gr-self-healing/14-cluster-crash | ++++ git rev-parse --abbrev-ref HEAD logger.go:42: 14:53:12 | gr-self-healing/14-cluster-crash | +++ export GIT_BRANCH=PR-424 logger.go:42: 14:53:12 | gr-self-healing/14-cluster-crash | +++ GIT_BRANCH=PR-424 logger.go:42: 14:53:12 | gr-self-healing/14-cluster-crash | +++ export VERSION=PR-424-70568ae logger.go:42: 14:53:12 | gr-self-healing/14-cluster-crash | +++ VERSION=PR-424-70568ae logger.go:42: 14:53:12 | gr-self-healing/14-cluster-crash | +++ export IMAGE=perconalab/percona-server-mysql-operator:PR-424-70568ae logger.go:42: 14:53:12 | gr-self-healing/14-cluster-crash | +++ IMAGE=perconalab/percona-server-mysql-operator:PR-424-70568ae logger.go:42: 14:53:12 | gr-self-healing/14-cluster-crash | +++ export IMAGE_MYSQL=perconalab/percona-server-mysql-operator:main-psmysql logger.go:42: 14:53:12 | gr-self-healing/14-cluster-crash | +++ IMAGE_MYSQL=perconalab/percona-server-mysql-operator:main-psmysql logger.go:42: 14:53:12 | gr-self-healing/14-cluster-crash | +++ export IMAGE_BACKUP=perconalab/percona-server-mysql-operator:main-backup logger.go:42: 14:53:12 | gr-self-healing/14-cluster-crash | +++ IMAGE_BACKUP=perconalab/percona-server-mysql-operator:main-backup logger.go:42: 14:53:12 | gr-self-healing/14-cluster-crash | +++ export IMAGE_ORCHESTRATOR=perconalab/percona-server-mysql-operator:main-orchestrator logger.go:42: 14:53:12 | gr-self-healing/14-cluster-crash | +++ IMAGE_ORCHESTRATOR=perconalab/percona-server-mysql-operator:main-orchestrator logger.go:42: 14:53:12 | gr-self-healing/14-cluster-crash | +++ export IMAGE_ROUTER=perconalab/percona-server-mysql-operator:main-router logger.go:42: 14:53:12 | gr-self-healing/14-cluster-crash | +++ IMAGE_ROUTER=perconalab/percona-server-mysql-operator:main-router logger.go:42: 14:53:12 | gr-self-healing/14-cluster-crash | +++ export IMAGE_TOOLKIT=perconalab/percona-server-mysql-operator:main-toolkit logger.go:42: 14:53:12 | gr-self-healing/14-cluster-crash | +++ IMAGE_TOOLKIT=perconalab/percona-server-mysql-operator:main-toolkit logger.go:42: 14:53:12 | gr-self-healing/14-cluster-crash | +++ export IMAGE_HAPROXY=perconalab/percona-server-mysql-operator:main-haproxy logger.go:42: 14:53:12 | gr-self-healing/14-cluster-crash | +++ IMAGE_HAPROXY=perconalab/percona-server-mysql-operator:main-haproxy logger.go:42: 14:53:12 | gr-self-healing/14-cluster-crash | +++ export IMAGE_PMM=perconalab/pmm-client:dev-latest logger.go:42: 14:53:12 | gr-self-healing/14-cluster-crash | +++ IMAGE_PMM=perconalab/pmm-client:dev-latest logger.go:42: 14:53:12 | gr-self-healing/14-cluster-crash | +++ export PMM_SERVER_VERSION=9.9.9 logger.go:42: 14:53:12 | gr-self-healing/14-cluster-crash | +++ PMM_SERVER_VERSION=9.9.9 logger.go:42: 14:53:12 | gr-self-healing/14-cluster-crash | +++ export IMAGE_PMM_SERVER_REPO=perconalab/pmm-server logger.go:42: 14:53:12 | gr-self-healing/14-cluster-crash | +++ IMAGE_PMM_SERVER_REPO=perconalab/pmm-server logger.go:42: 14:53:12 | gr-self-healing/14-cluster-crash | +++ export IMAGE_PMM_SERVER_TAG=dev-latest logger.go:42: 14:53:12 | gr-self-healing/14-cluster-crash | +++ IMAGE_PMM_SERVER_TAG=dev-latest logger.go:42: 14:53:12 | gr-self-healing/14-cluster-crash | ++++ which gdate logger.go:42: 14:53:12 | gr-self-healing/14-cluster-crash | which: no gdate in (/mnt/jenkins/workspace/cloud-ps-operator_PR-424/bin/:/home/ec2-user/google-cloud-sdk/bin:/home/ec2-user/.krew/bin:/usr/local/bin:/usr/bin) logger.go:42: 14:53:12 | gr-self-healing/14-cluster-crash | ++++ which date logger.go:42: 14:53:12 | gr-self-healing/14-cluster-crash | +++ date=/usr/bin/date logger.go:42: 14:53:12 | gr-self-healing/14-cluster-crash | +++ command -v oc logger.go:42: 14:53:12 | gr-self-healing/14-cluster-crash | +++ oc get projects logger.go:42: 14:53:17 | gr-self-healing/14-cluster-crash | error: the server doesn't have a resource type "projects" logger.go:42: 14:53:17 | gr-self-healing/14-cluster-crash | +++ grep '^minikube' logger.go:42: 14:53:17 | gr-self-healing/14-cluster-crash | +++ kubectl get nodes logger.go:42: 14:53:18 | gr-self-healing/14-cluster-crash | ++++ pwd logger.go:42: 14:53:18 | gr-self-healing/14-cluster-crash | +++ basename /mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests/tests/gr-self-healing logger.go:42: 14:53:18 | gr-self-healing/14-cluster-crash | ++ test_name=gr-self-healing logger.go:42: 14:53:18 | gr-self-healing/14-cluster-crash | + kill_pods kuttl-test-secure-shiner label app.kubernetes.io/instance gr-self-healing cluster-crash logger.go:42: 14:53:18 | gr-self-healing/14-cluster-crash | + local ns=kuttl-test-secure-shiner logger.go:42: 14:53:18 | gr-self-healing/14-cluster-crash | + local selector=label logger.go:42: 14:53:18 | gr-self-healing/14-cluster-crash | + local pod_label=app.kubernetes.io/instance logger.go:42: 14:53:18 | gr-self-healing/14-cluster-crash | + local label_value=gr-self-healing logger.go:42: 14:53:18 | gr-self-healing/14-cluster-crash | + local chaos_suffix=cluster-crash logger.go:42: 14:53:18 | gr-self-healing/14-cluster-crash | + '[' label == pod ']' logger.go:42: 14:53:18 | gr-self-healing/14-cluster-crash | + '[' label == label ']' logger.go:42: 14:53:18 | gr-self-healing/14-cluster-crash | + yq eval ' logger.go:42: 14:53:18 | gr-self-healing/14-cluster-crash | .metadata.name = "chaos-kill-label-cluster-crash" | logger.go:42: 14:53:18 | gr-self-healing/14-cluster-crash | .spec.mode = "all" | logger.go:42: 14:53:18 | gr-self-healing/14-cluster-crash | del(.spec.selector.pods) | logger.go:42: 14:53:18 | gr-self-healing/14-cluster-crash | .spec.selector.labelSelectors."app.kubernetes.io/instance" = "gr-self-healing"' /mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests/conf/chaos-pod-kill.yml logger.go:42: 14:53:18 | gr-self-healing/14-cluster-crash | + kubectl apply --namespace kuttl-test-secure-shiner -f - logger.go:42: 14:53:19 | gr-self-healing/14-cluster-crash | podchaos.chaos-mesh.org/chaos-kill-label-cluster-crash created logger.go:42: 14:53:20 | gr-self-healing/14-cluster-crash | + sleep 5 logger.go:42: 14:53:25 | gr-self-healing/14-cluster-crash | + sleep 30 logger.go:42: 14:59:42 | gr-self-healing/14-cluster-crash | test step completed 14-cluster-crash logger.go:42: 14:59:42 | gr-self-healing/15-write-data | starting test step 15-write-data logger.go:42: 14:59:42 | gr-self-healing/15-write-data | running command: [sh -c set -o errexit set -o xtrace source ../../functions run_mysql \ "INSERT myDB.myTable (id) VALUES (100504)" \ "-h $(get_mysql_router_service $(get_cluster_name)) -P 6446 -uroot -proot_password"] logger.go:42: 14:59:42 | gr-self-healing/15-write-data | + source ../../functions logger.go:42: 14:59:42 | gr-self-healing/15-write-data | +++ realpath ../../.. logger.go:42: 14:59:42 | gr-self-healing/15-write-data | ++ ROOT_REPO=/mnt/jenkins/workspace/cloud-ps-operator_PR-424 logger.go:42: 14:59:42 | gr-self-healing/15-write-data | ++ source /mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests/vars.sh logger.go:42: 14:59:42 | gr-self-healing/15-write-data | +++ export ROOT_REPO=/mnt/jenkins/workspace/cloud-ps-operator_PR-424 logger.go:42: 14:59:42 | gr-self-healing/15-write-data | +++ ROOT_REPO=/mnt/jenkins/workspace/cloud-ps-operator_PR-424 logger.go:42: 14:59:42 | gr-self-healing/15-write-data | +++ export DEPLOY_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/deploy logger.go:42: 14:59:42 | gr-self-healing/15-write-data | +++ DEPLOY_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/deploy logger.go:42: 14:59:42 | gr-self-healing/15-write-data | +++ export TESTS_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests logger.go:42: 14:59:42 | gr-self-healing/15-write-data | +++ TESTS_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests logger.go:42: 14:59:42 | gr-self-healing/15-write-data | +++ export TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests/conf logger.go:42: 14:59:42 | gr-self-healing/15-write-data | +++ TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests/conf logger.go:42: 14:59:42 | gr-self-healing/15-write-data | ++++ mktemp -d logger.go:42: 14:59:42 | gr-self-healing/15-write-data | +++ export TEMP_DIR=/tmp/tmp.0VhlCWeyda logger.go:42: 14:59:42 | gr-self-healing/15-write-data | +++ TEMP_DIR=/tmp/tmp.0VhlCWeyda logger.go:42: 14:59:42 | gr-self-healing/15-write-data | ++++ git rev-parse --abbrev-ref HEAD logger.go:42: 14:59:42 | gr-self-healing/15-write-data | +++ export GIT_BRANCH=PR-424 logger.go:42: 14:59:42 | gr-self-healing/15-write-data | +++ GIT_BRANCH=PR-424 logger.go:42: 14:59:42 | gr-self-healing/15-write-data | +++ export VERSION=PR-424-70568ae logger.go:42: 14:59:42 | gr-self-healing/15-write-data | +++ VERSION=PR-424-70568ae logger.go:42: 14:59:42 | gr-self-healing/15-write-data | +++ export IMAGE=perconalab/percona-server-mysql-operator:PR-424-70568ae logger.go:42: 14:59:42 | gr-self-healing/15-write-data | +++ IMAGE=perconalab/percona-server-mysql-operator:PR-424-70568ae logger.go:42: 14:59:42 | gr-self-healing/15-write-data | +++ export IMAGE_MYSQL=perconalab/percona-server-mysql-operator:main-psmysql logger.go:42: 14:59:42 | gr-self-healing/15-write-data | +++ IMAGE_MYSQL=perconalab/percona-server-mysql-operator:main-psmysql logger.go:42: 14:59:42 | gr-self-healing/15-write-data | +++ export IMAGE_BACKUP=perconalab/percona-server-mysql-operator:main-backup logger.go:42: 14:59:42 | gr-self-healing/15-write-data | +++ IMAGE_BACKUP=perconalab/percona-server-mysql-operator:main-backup logger.go:42: 14:59:42 | gr-self-healing/15-write-data | +++ export IMAGE_ORCHESTRATOR=perconalab/percona-server-mysql-operator:main-orchestrator logger.go:42: 14:59:42 | gr-self-healing/15-write-data | +++ IMAGE_ORCHESTRATOR=perconalab/percona-server-mysql-operator:main-orchestrator logger.go:42: 14:59:42 | gr-self-healing/15-write-data | +++ export IMAGE_ROUTER=perconalab/percona-server-mysql-operator:main-router logger.go:42: 14:59:42 | gr-self-healing/15-write-data | +++ IMAGE_ROUTER=perconalab/percona-server-mysql-operator:main-router logger.go:42: 14:59:42 | gr-self-healing/15-write-data | +++ export IMAGE_TOOLKIT=perconalab/percona-server-mysql-operator:main-toolkit logger.go:42: 14:59:42 | gr-self-healing/15-write-data | +++ IMAGE_TOOLKIT=perconalab/percona-server-mysql-operator:main-toolkit logger.go:42: 14:59:42 | gr-self-healing/15-write-data | +++ export IMAGE_HAPROXY=perconalab/percona-server-mysql-operator:main-haproxy logger.go:42: 14:59:42 | gr-self-healing/15-write-data | +++ IMAGE_HAPROXY=perconalab/percona-server-mysql-operator:main-haproxy logger.go:42: 14:59:42 | gr-self-healing/15-write-data | +++ export IMAGE_PMM=perconalab/pmm-client:dev-latest logger.go:42: 14:59:42 | gr-self-healing/15-write-data | +++ IMAGE_PMM=perconalab/pmm-client:dev-latest logger.go:42: 14:59:42 | gr-self-healing/15-write-data | +++ export PMM_SERVER_VERSION=9.9.9 logger.go:42: 14:59:42 | gr-self-healing/15-write-data | +++ PMM_SERVER_VERSION=9.9.9 logger.go:42: 14:59:42 | gr-self-healing/15-write-data | +++ export IMAGE_PMM_SERVER_REPO=perconalab/pmm-server logger.go:42: 14:59:42 | gr-self-healing/15-write-data | +++ IMAGE_PMM_SERVER_REPO=perconalab/pmm-server logger.go:42: 14:59:42 | gr-self-healing/15-write-data | +++ export IMAGE_PMM_SERVER_TAG=dev-latest logger.go:42: 14:59:42 | gr-self-healing/15-write-data | +++ IMAGE_PMM_SERVER_TAG=dev-latest logger.go:42: 14:59:42 | gr-self-healing/15-write-data | ++++ which gdate logger.go:42: 14:59:42 | gr-self-healing/15-write-data | which: no gdate in (/mnt/jenkins/workspace/cloud-ps-operator_PR-424/bin/:/home/ec2-user/google-cloud-sdk/bin:/home/ec2-user/.krew/bin:/usr/local/bin:/usr/bin) logger.go:42: 14:59:42 | gr-self-healing/15-write-data | ++++ which date logger.go:42: 14:59:42 | gr-self-healing/15-write-data | +++ date=/usr/bin/date logger.go:42: 14:59:42 | gr-self-healing/15-write-data | +++ command -v oc logger.go:42: 14:59:42 | gr-self-healing/15-write-data | +++ oc get projects logger.go:42: 14:59:48 | gr-self-healing/15-write-data | error: the server doesn't have a resource type "projects" logger.go:42: 14:59:48 | gr-self-healing/15-write-data | +++ grep '^minikube' logger.go:42: 14:59:48 | gr-self-healing/15-write-data | +++ kubectl get nodes logger.go:42: 14:59:49 | gr-self-healing/15-write-data | ++++ pwd logger.go:42: 14:59:49 | gr-self-healing/15-write-data | +++ basename /mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests/tests/gr-self-healing logger.go:42: 14:59:49 | gr-self-healing/15-write-data | ++ test_name=gr-self-healing logger.go:42: 14:59:49 | gr-self-healing/15-write-data | +++ get_cluster_name logger.go:42: 14:59:49 | gr-self-healing/15-write-data | +++ kubectl -n kuttl-test-secure-shiner get ps -o 'jsonpath={.items[0].metadata.name}' logger.go:42: 14:59:50 | gr-self-healing/15-write-data | ++ get_mysql_router_service gr-self-healing logger.go:42: 14:59:50 | gr-self-healing/15-write-data | ++ local cluster=gr-self-healing logger.go:42: 14:59:50 | gr-self-healing/15-write-data | ++ echo gr-self-healing-router logger.go:42: 14:59:50 | gr-self-healing/15-write-data | + run_mysql 'INSERT myDB.myTable (id) VALUES (100504)' '-h gr-self-healing-router -P 6446 -uroot -proot_password' logger.go:42: 14:59:50 | gr-self-healing/15-write-data | + local 'command=INSERT myDB.myTable (id) VALUES (100504)' logger.go:42: 14:59:50 | gr-self-healing/15-write-data | + local 'uri=-h gr-self-healing-router -P 6446 -uroot -proot_password' logger.go:42: 14:59:50 | gr-self-healing/15-write-data | + local pod= logger.go:42: 14:59:50 | gr-self-healing/15-write-data | ++ get_client_pod logger.go:42: 14:59:50 | gr-self-healing/15-write-data | ++ kubectl -n kuttl-test-secure-shiner get pods --selector=name=mysql-client -o 'jsonpath={.items[].metadata.name}' logger.go:42: 14:59:51 | gr-self-healing/15-write-data | + client_pod=mysql-client logger.go:42: 14:59:51 | gr-self-healing/15-write-data | + wait_pod mysql-client logger.go:42: 14:59:51 | gr-self-healing/15-write-data | + local pod=mysql-client logger.go:42: 14:59:51 | gr-self-healing/15-write-data | + set +o xtrace logger.go:42: 14:59:51 | gr-self-healing/15-write-data | mysql-clienttrue logger.go:42: 14:59:51 | gr-self-healing/15-write-data | + kubectl -n kuttl-test-secure-shiner exec mysql-client -- bash -c 'printf '\''%s\n'\'' "INSERT myDB.myTable (id) VALUES (100504)" | mysql -sN -h gr-self-healing-router -P 6446 -uroot -proot_password' logger.go:42: 14:59:51 | gr-self-healing/15-write-data | + sed -e 's/mysql: //' logger.go:42: 14:59:51 | gr-self-healing/15-write-data | + grep -v 'Using a password on the command line interface can be insecure.' logger.go:42: 14:59:53 | gr-self-healing/15-write-data | + : logger.go:42: 14:59:54 | gr-self-healing/15-write-data | test step completed 15-write-data logger.go:42: 14:59:54 | gr-self-healing/16-read-from-replicas | starting test step 16-read-from-replicas logger.go:42: 14:59:54 | gr-self-healing/16-read-from-replicas | running command: [sh -c set -o errexit set -o xtrace source ../../functions for i in 0 1 2; do host=$(get_mysql_headless_fqdn $(get_cluster_name) $i) data=$(run_mysql "SELECT * FROM myDB.myTable" "-h ${host} -uroot -proot_password") kubectl create configmap -n "${NAMESPACE}" 16-read-from-replicas-${i} --from-literal=data="${data}" done] logger.go:42: 14:59:54 | gr-self-healing/16-read-from-replicas | + source ../../functions logger.go:42: 14:59:54 | gr-self-healing/16-read-from-replicas | +++ realpath ../../.. logger.go:42: 14:59:54 | gr-self-healing/16-read-from-replicas | ++ ROOT_REPO=/mnt/jenkins/workspace/cloud-ps-operator_PR-424 logger.go:42: 14:59:54 | gr-self-healing/16-read-from-replicas | ++ source /mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests/vars.sh logger.go:42: 14:59:54 | gr-self-healing/16-read-from-replicas | +++ export ROOT_REPO=/mnt/jenkins/workspace/cloud-ps-operator_PR-424 logger.go:42: 14:59:54 | gr-self-healing/16-read-from-replicas | +++ ROOT_REPO=/mnt/jenkins/workspace/cloud-ps-operator_PR-424 logger.go:42: 14:59:54 | gr-self-healing/16-read-from-replicas | +++ export DEPLOY_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/deploy logger.go:42: 14:59:54 | gr-self-healing/16-read-from-replicas | +++ DEPLOY_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/deploy logger.go:42: 14:59:54 | gr-self-healing/16-read-from-replicas | +++ export TESTS_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests logger.go:42: 14:59:54 | gr-self-healing/16-read-from-replicas | +++ TESTS_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests logger.go:42: 14:59:54 | gr-self-healing/16-read-from-replicas | +++ export TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests/conf logger.go:42: 14:59:54 | gr-self-healing/16-read-from-replicas | +++ TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests/conf logger.go:42: 14:59:54 | gr-self-healing/16-read-from-replicas | ++++ mktemp -d logger.go:42: 14:59:54 | gr-self-healing/16-read-from-replicas | +++ export TEMP_DIR=/tmp/tmp.MGac3FNqw2 logger.go:42: 14:59:54 | gr-self-healing/16-read-from-replicas | +++ TEMP_DIR=/tmp/tmp.MGac3FNqw2 logger.go:42: 14:59:54 | gr-self-healing/16-read-from-replicas | ++++ git rev-parse --abbrev-ref HEAD logger.go:42: 14:59:54 | gr-self-healing/16-read-from-replicas | +++ export GIT_BRANCH=PR-424 logger.go:42: 14:59:54 | gr-self-healing/16-read-from-replicas | +++ GIT_BRANCH=PR-424 logger.go:42: 14:59:54 | gr-self-healing/16-read-from-replicas | +++ export VERSION=PR-424-70568ae logger.go:42: 14:59:54 | gr-self-healing/16-read-from-replicas | +++ VERSION=PR-424-70568ae logger.go:42: 14:59:54 | gr-self-healing/16-read-from-replicas | +++ export IMAGE=perconalab/percona-server-mysql-operator:PR-424-70568ae logger.go:42: 14:59:54 | gr-self-healing/16-read-from-replicas | +++ IMAGE=perconalab/percona-server-mysql-operator:PR-424-70568ae logger.go:42: 14:59:54 | gr-self-healing/16-read-from-replicas | +++ export IMAGE_MYSQL=perconalab/percona-server-mysql-operator:main-psmysql logger.go:42: 14:59:54 | gr-self-healing/16-read-from-replicas | +++ IMAGE_MYSQL=perconalab/percona-server-mysql-operator:main-psmysql logger.go:42: 14:59:54 | gr-self-healing/16-read-from-replicas | +++ export IMAGE_BACKUP=perconalab/percona-server-mysql-operator:main-backup logger.go:42: 14:59:54 | gr-self-healing/16-read-from-replicas | +++ IMAGE_BACKUP=perconalab/percona-server-mysql-operator:main-backup logger.go:42: 14:59:54 | gr-self-healing/16-read-from-replicas | +++ export IMAGE_ORCHESTRATOR=perconalab/percona-server-mysql-operator:main-orchestrator logger.go:42: 14:59:54 | gr-self-healing/16-read-from-replicas | +++ IMAGE_ORCHESTRATOR=perconalab/percona-server-mysql-operator:main-orchestrator logger.go:42: 14:59:54 | gr-self-healing/16-read-from-replicas | +++ export IMAGE_ROUTER=perconalab/percona-server-mysql-operator:main-router logger.go:42: 14:59:54 | gr-self-healing/16-read-from-replicas | +++ IMAGE_ROUTER=perconalab/percona-server-mysql-operator:main-router logger.go:42: 14:59:54 | gr-self-healing/16-read-from-replicas | +++ export IMAGE_TOOLKIT=perconalab/percona-server-mysql-operator:main-toolkit logger.go:42: 14:59:54 | gr-self-healing/16-read-from-replicas | +++ IMAGE_TOOLKIT=perconalab/percona-server-mysql-operator:main-toolkit logger.go:42: 14:59:54 | gr-self-healing/16-read-from-replicas | +++ export IMAGE_HAPROXY=perconalab/percona-server-mysql-operator:main-haproxy logger.go:42: 14:59:54 | gr-self-healing/16-read-from-replicas | +++ IMAGE_HAPROXY=perconalab/percona-server-mysql-operator:main-haproxy logger.go:42: 14:59:54 | gr-self-healing/16-read-from-replicas | +++ export IMAGE_PMM=perconalab/pmm-client:dev-latest logger.go:42: 14:59:54 | gr-self-healing/16-read-from-replicas | +++ IMAGE_PMM=perconalab/pmm-client:dev-latest logger.go:42: 14:59:54 | gr-self-healing/16-read-from-replicas | +++ export PMM_SERVER_VERSION=9.9.9 logger.go:42: 14:59:54 | gr-self-healing/16-read-from-replicas | +++ PMM_SERVER_VERSION=9.9.9 logger.go:42: 14:59:54 | gr-self-healing/16-read-from-replicas | +++ export IMAGE_PMM_SERVER_REPO=perconalab/pmm-server logger.go:42: 14:59:54 | gr-self-healing/16-read-from-replicas | +++ IMAGE_PMM_SERVER_REPO=perconalab/pmm-server logger.go:42: 14:59:54 | gr-self-healing/16-read-from-replicas | +++ export IMAGE_PMM_SERVER_TAG=dev-latest logger.go:42: 14:59:54 | gr-self-healing/16-read-from-replicas | +++ IMAGE_PMM_SERVER_TAG=dev-latest logger.go:42: 14:59:54 | gr-self-healing/16-read-from-replicas | ++++ which gdate logger.go:42: 14:59:54 | gr-self-healing/16-read-from-replicas | which: no gdate in (/mnt/jenkins/workspace/cloud-ps-operator_PR-424/bin/:/home/ec2-user/google-cloud-sdk/bin:/home/ec2-user/.krew/bin:/usr/local/bin:/usr/bin) logger.go:42: 14:59:54 | gr-self-healing/16-read-from-replicas | ++++ which date logger.go:42: 14:59:54 | gr-self-healing/16-read-from-replicas | +++ date=/usr/bin/date logger.go:42: 14:59:54 | gr-self-healing/16-read-from-replicas | +++ command -v oc logger.go:42: 14:59:54 | gr-self-healing/16-read-from-replicas | +++ oc get projects logger.go:42: 15:00:00 | gr-self-healing/16-read-from-replicas | error: the server doesn't have a resource type "projects" logger.go:42: 15:00:00 | gr-self-healing/16-read-from-replicas | +++ grep '^minikube' logger.go:42: 15:00:00 | gr-self-healing/16-read-from-replicas | +++ kubectl get nodes logger.go:42: 15:00:00 | gr-self-healing/16-read-from-replicas | ++++ pwd logger.go:42: 15:00:00 | gr-self-healing/16-read-from-replicas | +++ basename /mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests/tests/gr-self-healing logger.go:42: 15:00:00 | gr-self-healing/16-read-from-replicas | ++ test_name=gr-self-healing logger.go:42: 15:00:00 | gr-self-healing/16-read-from-replicas | + for i in 0 1 2 logger.go:42: 15:00:00 | gr-self-healing/16-read-from-replicas | +++ get_cluster_name logger.go:42: 15:00:00 | gr-self-healing/16-read-from-replicas | +++ kubectl -n kuttl-test-secure-shiner get ps -o 'jsonpath={.items[0].metadata.name}' logger.go:42: 15:00:01 | gr-self-healing/16-read-from-replicas | ++ get_mysql_headless_fqdn gr-self-healing 0 logger.go:42: 15:00:01 | gr-self-healing/16-read-from-replicas | ++ local cluster=gr-self-healing logger.go:42: 15:00:01 | gr-self-healing/16-read-from-replicas | ++ local index=0 logger.go:42: 15:00:01 | gr-self-healing/16-read-from-replicas | ++ echo gr-self-healing-mysql-0.gr-self-healing-mysql logger.go:42: 15:00:01 | gr-self-healing/16-read-from-replicas | + host=gr-self-healing-mysql-0.gr-self-healing-mysql logger.go:42: 15:00:01 | gr-self-healing/16-read-from-replicas | ++ run_mysql 'SELECT * FROM myDB.myTable' '-h gr-self-healing-mysql-0.gr-self-healing-mysql -uroot -proot_password' logger.go:42: 15:00:01 | gr-self-healing/16-read-from-replicas | ++ local 'command=SELECT * FROM myDB.myTable' logger.go:42: 15:00:01 | gr-self-healing/16-read-from-replicas | ++ local 'uri=-h gr-self-healing-mysql-0.gr-self-healing-mysql -uroot -proot_password' logger.go:42: 15:00:01 | gr-self-healing/16-read-from-replicas | ++ local pod= logger.go:42: 15:00:01 | gr-self-healing/16-read-from-replicas | +++ get_client_pod logger.go:42: 15:00:01 | gr-self-healing/16-read-from-replicas | +++ kubectl -n kuttl-test-secure-shiner get pods --selector=name=mysql-client -o 'jsonpath={.items[].metadata.name}' logger.go:42: 15:00:02 | gr-self-healing/16-read-from-replicas | ++ client_pod=mysql-client logger.go:42: 15:00:02 | gr-self-healing/16-read-from-replicas | ++ wait_pod mysql-client logger.go:42: 15:00:02 | gr-self-healing/16-read-from-replicas | ++ local pod=mysql-client logger.go:42: 15:00:02 | gr-self-healing/16-read-from-replicas | ++ set +o xtrace logger.go:42: 15:00:03 | gr-self-healing/16-read-from-replicas | mysql-clienttrue logger.go:42: 15:00:03 | gr-self-healing/16-read-from-replicas | ++ kubectl -n kuttl-test-secure-shiner exec mysql-client -- bash -c 'printf '\''%s\n'\'' "SELECT * FROM myDB.myTable" | mysql -sN -h gr-self-healing-mysql-0.gr-self-healing-mysql -uroot -proot_password' logger.go:42: 15:00:03 | gr-self-healing/16-read-from-replicas | ++ grep -v 'Using a password on the command line interface can be insecure.' logger.go:42: 15:00:03 | gr-self-healing/16-read-from-replicas | ++ sed -e 's/mysql: //' logger.go:42: 15:00:04 | gr-self-healing/16-read-from-replicas | + data='100500 logger.go:42: 15:00:04 | gr-self-healing/16-read-from-replicas | 100501 logger.go:42: 15:00:04 | gr-self-healing/16-read-from-replicas | 100502 logger.go:42: 15:00:04 | gr-self-healing/16-read-from-replicas | 100503 logger.go:42: 15:00:04 | gr-self-healing/16-read-from-replicas | 100504' logger.go:42: 15:00:04 | gr-self-healing/16-read-from-replicas | + kubectl create configmap -n kuttl-test-secure-shiner 16-read-from-replicas-0 '--from-literal=data=100500 logger.go:42: 15:00:04 | gr-self-healing/16-read-from-replicas | 100501 logger.go:42: 15:00:04 | gr-self-healing/16-read-from-replicas | 100502 logger.go:42: 15:00:04 | gr-self-healing/16-read-from-replicas | 100503 logger.go:42: 15:00:04 | gr-self-healing/16-read-from-replicas | 100504' logger.go:42: 15:00:05 | gr-self-healing/16-read-from-replicas | configmap/16-read-from-replicas-0 created logger.go:42: 15:00:05 | gr-self-healing/16-read-from-replicas | + for i in 0 1 2 logger.go:42: 15:00:05 | gr-self-healing/16-read-from-replicas | +++ get_cluster_name logger.go:42: 15:00:05 | gr-self-healing/16-read-from-replicas | +++ kubectl -n kuttl-test-secure-shiner get ps -o 'jsonpath={.items[0].metadata.name}' logger.go:42: 15:00:05 | gr-self-healing/16-read-from-replicas | ++ get_mysql_headless_fqdn gr-self-healing 1 logger.go:42: 15:00:05 | gr-self-healing/16-read-from-replicas | ++ local cluster=gr-self-healing logger.go:42: 15:00:05 | gr-self-healing/16-read-from-replicas | ++ local index=1 logger.go:42: 15:00:05 | gr-self-healing/16-read-from-replicas | ++ echo gr-self-healing-mysql-1.gr-self-healing-mysql logger.go:42: 15:00:05 | gr-self-healing/16-read-from-replicas | + host=gr-self-healing-mysql-1.gr-self-healing-mysql logger.go:42: 15:00:05 | gr-self-healing/16-read-from-replicas | ++ run_mysql 'SELECT * FROM myDB.myTable' '-h gr-self-healing-mysql-1.gr-self-healing-mysql -uroot -proot_password' logger.go:42: 15:00:05 | gr-self-healing/16-read-from-replicas | ++ local 'command=SELECT * FROM myDB.myTable' logger.go:42: 15:00:05 | gr-self-healing/16-read-from-replicas | ++ local 'uri=-h gr-self-healing-mysql-1.gr-self-healing-mysql -uroot -proot_password' logger.go:42: 15:00:05 | gr-self-healing/16-read-from-replicas | ++ local pod= logger.go:42: 15:00:05 | gr-self-healing/16-read-from-replicas | +++ get_client_pod logger.go:42: 15:00:05 | gr-self-healing/16-read-from-replicas | +++ kubectl -n kuttl-test-secure-shiner get pods --selector=name=mysql-client -o 'jsonpath={.items[].metadata.name}' logger.go:42: 15:00:06 | gr-self-healing/16-read-from-replicas | ++ client_pod=mysql-client logger.go:42: 15:00:06 | gr-self-healing/16-read-from-replicas | ++ wait_pod mysql-client logger.go:42: 15:00:06 | gr-self-healing/16-read-from-replicas | ++ local pod=mysql-client logger.go:42: 15:00:06 | gr-self-healing/16-read-from-replicas | ++ set +o xtrace logger.go:42: 15:00:07 | gr-self-healing/16-read-from-replicas | mysql-clienttrue logger.go:42: 15:00:07 | gr-self-healing/16-read-from-replicas | ++ kubectl -n kuttl-test-secure-shiner exec mysql-client -- bash -c 'printf '\''%s\n'\'' "SELECT * FROM myDB.myTable" | mysql -sN -h gr-self-healing-mysql-1.gr-self-healing-mysql -uroot -proot_password' logger.go:42: 15:00:07 | gr-self-healing/16-read-from-replicas | ++ sed -e 's/mysql: //' logger.go:42: 15:00:07 | gr-self-healing/16-read-from-replicas | ++ grep -v 'Using a password on the command line interface can be insecure.' logger.go:42: 15:00:08 | gr-self-healing/16-read-from-replicas | + data='100500 logger.go:42: 15:00:08 | gr-self-healing/16-read-from-replicas | 100501 logger.go:42: 15:00:08 | gr-self-healing/16-read-from-replicas | 100502 logger.go:42: 15:00:08 | gr-self-healing/16-read-from-replicas | 100503 logger.go:42: 15:00:08 | gr-self-healing/16-read-from-replicas | 100504' logger.go:42: 15:00:08 | gr-self-healing/16-read-from-replicas | + kubectl create configmap -n kuttl-test-secure-shiner 16-read-from-replicas-1 '--from-literal=data=100500 logger.go:42: 15:00:08 | gr-self-healing/16-read-from-replicas | 100501 logger.go:42: 15:00:08 | gr-self-healing/16-read-from-replicas | 100502 logger.go:42: 15:00:08 | gr-self-healing/16-read-from-replicas | 100503 logger.go:42: 15:00:08 | gr-self-healing/16-read-from-replicas | 100504' logger.go:42: 15:00:09 | gr-self-healing/16-read-from-replicas | configmap/16-read-from-replicas-1 created logger.go:42: 15:00:09 | gr-self-healing/16-read-from-replicas | + for i in 0 1 2 logger.go:42: 15:00:09 | gr-self-healing/16-read-from-replicas | +++ get_cluster_name logger.go:42: 15:00:09 | gr-self-healing/16-read-from-replicas | +++ kubectl -n kuttl-test-secure-shiner get ps -o 'jsonpath={.items[0].metadata.name}' logger.go:42: 15:00:10 | gr-self-healing/16-read-from-replicas | ++ get_mysql_headless_fqdn gr-self-healing 2 logger.go:42: 15:00:10 | gr-self-healing/16-read-from-replicas | ++ local cluster=gr-self-healing logger.go:42: 15:00:10 | gr-self-healing/16-read-from-replicas | ++ local index=2 logger.go:42: 15:00:10 | gr-self-healing/16-read-from-replicas | ++ echo gr-self-healing-mysql-2.gr-self-healing-mysql logger.go:42: 15:00:10 | gr-self-healing/16-read-from-replicas | + host=gr-self-healing-mysql-2.gr-self-healing-mysql logger.go:42: 15:00:10 | gr-self-healing/16-read-from-replicas | ++ run_mysql 'SELECT * FROM myDB.myTable' '-h gr-self-healing-mysql-2.gr-self-healing-mysql -uroot -proot_password' logger.go:42: 15:00:10 | gr-self-healing/16-read-from-replicas | ++ local 'command=SELECT * FROM myDB.myTable' logger.go:42: 15:00:10 | gr-self-healing/16-read-from-replicas | ++ local 'uri=-h gr-self-healing-mysql-2.gr-self-healing-mysql -uroot -proot_password' logger.go:42: 15:00:10 | gr-self-healing/16-read-from-replicas | ++ local pod= logger.go:42: 15:00:10 | gr-self-healing/16-read-from-replicas | +++ get_client_pod logger.go:42: 15:00:10 | gr-self-healing/16-read-from-replicas | +++ kubectl -n kuttl-test-secure-shiner get pods --selector=name=mysql-client -o 'jsonpath={.items[].metadata.name}' logger.go:42: 15:00:11 | gr-self-healing/16-read-from-replicas | ++ client_pod=mysql-client logger.go:42: 15:00:11 | gr-self-healing/16-read-from-replicas | ++ wait_pod mysql-client logger.go:42: 15:00:11 | gr-self-healing/16-read-from-replicas | ++ local pod=mysql-client logger.go:42: 15:00:11 | gr-self-healing/16-read-from-replicas | ++ set +o xtrace logger.go:42: 15:00:11 | gr-self-healing/16-read-from-replicas | mysql-clienttrue logger.go:42: 15:00:11 | gr-self-healing/16-read-from-replicas | ++ kubectl -n kuttl-test-secure-shiner exec mysql-client -- bash -c 'printf '\''%s\n'\'' "SELECT * FROM myDB.myTable" | mysql -sN -h gr-self-healing-mysql-2.gr-self-healing-mysql -uroot -proot_password' logger.go:42: 15:00:11 | gr-self-healing/16-read-from-replicas | ++ sed -e 's/mysql: //' logger.go:42: 15:00:11 | gr-self-healing/16-read-from-replicas | ++ grep -v 'Using a password on the command line interface can be insecure.' logger.go:42: 15:00:13 | gr-self-healing/16-read-from-replicas | + data='100500 logger.go:42: 15:00:13 | gr-self-healing/16-read-from-replicas | 100501 logger.go:42: 15:00:13 | gr-self-healing/16-read-from-replicas | 100502 logger.go:42: 15:00:13 | gr-self-healing/16-read-from-replicas | 100503 logger.go:42: 15:00:13 | gr-self-healing/16-read-from-replicas | 100504' logger.go:42: 15:00:13 | gr-self-healing/16-read-from-replicas | + kubectl create configmap -n kuttl-test-secure-shiner 16-read-from-replicas-2 '--from-literal=data=100500 logger.go:42: 15:00:13 | gr-self-healing/16-read-from-replicas | 100501 logger.go:42: 15:00:13 | gr-self-healing/16-read-from-replicas | 100502 logger.go:42: 15:00:13 | gr-self-healing/16-read-from-replicas | 100503 logger.go:42: 15:00:13 | gr-self-healing/16-read-from-replicas | 100504' logger.go:42: 15:00:13 | gr-self-healing/16-read-from-replicas | configmap/16-read-from-replicas-2 created logger.go:42: 15:00:15 | gr-self-healing/16-read-from-replicas | test step completed 16-read-from-replicas logger.go:42: 15:00:15 | gr-self-healing/17-destroy-chaos-mesh | starting test step 17-destroy-chaos-mesh logger.go:42: 15:00:15 | gr-self-healing/17-destroy-chaos-mesh | running command: [sh -c set -o errexit set -o xtrace source ../../functions destroy_chaos_mesh] logger.go:42: 15:00:15 | gr-self-healing/17-destroy-chaos-mesh | + source ../../functions logger.go:42: 15:00:15 | gr-self-healing/17-destroy-chaos-mesh | +++ realpath ../../.. logger.go:42: 15:00:15 | gr-self-healing/17-destroy-chaos-mesh | ++ ROOT_REPO=/mnt/jenkins/workspace/cloud-ps-operator_PR-424 logger.go:42: 15:00:15 | gr-self-healing/17-destroy-chaos-mesh | ++ source /mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests/vars.sh logger.go:42: 15:00:15 | gr-self-healing/17-destroy-chaos-mesh | +++ export ROOT_REPO=/mnt/jenkins/workspace/cloud-ps-operator_PR-424 logger.go:42: 15:00:15 | gr-self-healing/17-destroy-chaos-mesh | +++ ROOT_REPO=/mnt/jenkins/workspace/cloud-ps-operator_PR-424 logger.go:42: 15:00:15 | gr-self-healing/17-destroy-chaos-mesh | +++ export DEPLOY_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/deploy logger.go:42: 15:00:15 | gr-self-healing/17-destroy-chaos-mesh | +++ DEPLOY_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/deploy logger.go:42: 15:00:15 | gr-self-healing/17-destroy-chaos-mesh | +++ export TESTS_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests logger.go:42: 15:00:15 | gr-self-healing/17-destroy-chaos-mesh | +++ TESTS_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests logger.go:42: 15:00:15 | gr-self-healing/17-destroy-chaos-mesh | +++ export TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests/conf logger.go:42: 15:00:15 | gr-self-healing/17-destroy-chaos-mesh | +++ TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests/conf logger.go:42: 15:00:15 | gr-self-healing/17-destroy-chaos-mesh | ++++ mktemp -d logger.go:42: 15:00:15 | gr-self-healing/17-destroy-chaos-mesh | +++ export TEMP_DIR=/tmp/tmp.F62zqoEebg logger.go:42: 15:00:15 | gr-self-healing/17-destroy-chaos-mesh | +++ TEMP_DIR=/tmp/tmp.F62zqoEebg logger.go:42: 15:00:15 | gr-self-healing/17-destroy-chaos-mesh | ++++ git rev-parse --abbrev-ref HEAD logger.go:42: 15:00:15 | gr-self-healing/17-destroy-chaos-mesh | +++ export GIT_BRANCH=PR-424 logger.go:42: 15:00:15 | gr-self-healing/17-destroy-chaos-mesh | +++ GIT_BRANCH=PR-424 logger.go:42: 15:00:15 | gr-self-healing/17-destroy-chaos-mesh | +++ export VERSION=PR-424-70568ae logger.go:42: 15:00:15 | gr-self-healing/17-destroy-chaos-mesh | +++ VERSION=PR-424-70568ae logger.go:42: 15:00:15 | gr-self-healing/17-destroy-chaos-mesh | +++ export IMAGE=perconalab/percona-server-mysql-operator:PR-424-70568ae logger.go:42: 15:00:15 | gr-self-healing/17-destroy-chaos-mesh | +++ IMAGE=perconalab/percona-server-mysql-operator:PR-424-70568ae logger.go:42: 15:00:15 | gr-self-healing/17-destroy-chaos-mesh | +++ export IMAGE_MYSQL=perconalab/percona-server-mysql-operator:main-psmysql logger.go:42: 15:00:15 | gr-self-healing/17-destroy-chaos-mesh | +++ IMAGE_MYSQL=perconalab/percona-server-mysql-operator:main-psmysql logger.go:42: 15:00:15 | gr-self-healing/17-destroy-chaos-mesh | +++ export IMAGE_BACKUP=perconalab/percona-server-mysql-operator:main-backup logger.go:42: 15:00:15 | gr-self-healing/17-destroy-chaos-mesh | +++ IMAGE_BACKUP=perconalab/percona-server-mysql-operator:main-backup logger.go:42: 15:00:15 | gr-self-healing/17-destroy-chaos-mesh | +++ export IMAGE_ORCHESTRATOR=perconalab/percona-server-mysql-operator:main-orchestrator logger.go:42: 15:00:15 | gr-self-healing/17-destroy-chaos-mesh | +++ IMAGE_ORCHESTRATOR=perconalab/percona-server-mysql-operator:main-orchestrator logger.go:42: 15:00:15 | gr-self-healing/17-destroy-chaos-mesh | +++ export IMAGE_ROUTER=perconalab/percona-server-mysql-operator:main-router logger.go:42: 15:00:15 | gr-self-healing/17-destroy-chaos-mesh | +++ IMAGE_ROUTER=perconalab/percona-server-mysql-operator:main-router logger.go:42: 15:00:15 | gr-self-healing/17-destroy-chaos-mesh | +++ export IMAGE_TOOLKIT=perconalab/percona-server-mysql-operator:main-toolkit logger.go:42: 15:00:15 | gr-self-healing/17-destroy-chaos-mesh | +++ IMAGE_TOOLKIT=perconalab/percona-server-mysql-operator:main-toolkit logger.go:42: 15:00:15 | gr-self-healing/17-destroy-chaos-mesh | +++ export IMAGE_HAPROXY=perconalab/percona-server-mysql-operator:main-haproxy logger.go:42: 15:00:15 | gr-self-healing/17-destroy-chaos-mesh | +++ IMAGE_HAPROXY=perconalab/percona-server-mysql-operator:main-haproxy logger.go:42: 15:00:15 | gr-self-healing/17-destroy-chaos-mesh | +++ export IMAGE_PMM=perconalab/pmm-client:dev-latest logger.go:42: 15:00:15 | gr-self-healing/17-destroy-chaos-mesh | +++ IMAGE_PMM=perconalab/pmm-client:dev-latest logger.go:42: 15:00:15 | gr-self-healing/17-destroy-chaos-mesh | +++ export PMM_SERVER_VERSION=9.9.9 logger.go:42: 15:00:15 | gr-self-healing/17-destroy-chaos-mesh | +++ PMM_SERVER_VERSION=9.9.9 logger.go:42: 15:00:15 | gr-self-healing/17-destroy-chaos-mesh | +++ export IMAGE_PMM_SERVER_REPO=perconalab/pmm-server logger.go:42: 15:00:15 | gr-self-healing/17-destroy-chaos-mesh | +++ IMAGE_PMM_SERVER_REPO=perconalab/pmm-server logger.go:42: 15:00:15 | gr-self-healing/17-destroy-chaos-mesh | +++ export IMAGE_PMM_SERVER_TAG=dev-latest logger.go:42: 15:00:15 | gr-self-healing/17-destroy-chaos-mesh | +++ IMAGE_PMM_SERVER_TAG=dev-latest logger.go:42: 15:00:15 | gr-self-healing/17-destroy-chaos-mesh | ++++ which gdate logger.go:42: 15:00:15 | gr-self-healing/17-destroy-chaos-mesh | which: no gdate in (/mnt/jenkins/workspace/cloud-ps-operator_PR-424/bin/:/home/ec2-user/google-cloud-sdk/bin:/home/ec2-user/.krew/bin:/usr/local/bin:/usr/bin) logger.go:42: 15:00:15 | gr-self-healing/17-destroy-chaos-mesh | ++++ which date logger.go:42: 15:00:15 | gr-self-healing/17-destroy-chaos-mesh | +++ date=/usr/bin/date logger.go:42: 15:00:15 | gr-self-healing/17-destroy-chaos-mesh | +++ command -v oc logger.go:42: 15:00:15 | gr-self-healing/17-destroy-chaos-mesh | +++ oc get projects logger.go:42: 15:00:21 | gr-self-healing/17-destroy-chaos-mesh | error: the server doesn't have a resource type "projects" logger.go:42: 15:00:21 | gr-self-healing/17-destroy-chaos-mesh | +++ kubectl get nodes logger.go:42: 15:00:21 | gr-self-healing/17-destroy-chaos-mesh | +++ grep '^minikube' logger.go:42: 15:00:22 | gr-self-healing/17-destroy-chaos-mesh | ++++ pwd logger.go:42: 15:00:22 | gr-self-healing/17-destroy-chaos-mesh | +++ basename /mnt/jenkins/workspace/cloud-ps-operator_PR-424/e2e-tests/tests/gr-self-healing logger.go:42: 15:00:22 | gr-self-healing/17-destroy-chaos-mesh | ++ test_name=gr-self-healing logger.go:42: 15:00:22 | gr-self-healing/17-destroy-chaos-mesh | + destroy_chaos_mesh logger.go:42: 15:00:22 | gr-self-healing/17-destroy-chaos-mesh | ++ helm list --all-namespaces --filter chaos-mesh logger.go:42: 15:00:22 | gr-self-healing/17-destroy-chaos-mesh | ++ tail -n1 logger.go:42: 15:00:22 | gr-self-healing/17-destroy-chaos-mesh | ++ awk '-F ' '{print $2}' logger.go:42: 15:00:22 | gr-self-healing/17-destroy-chaos-mesh | ++ sed s/NAMESPACE// logger.go:42: 15:00:22 | gr-self-healing/17-destroy-chaos-mesh | WARNING: Kubernetes configuration file is group-readable. This is insecure. Location: /mnt/jenkins/workspace/cloud-ps-operator_PR-424/kubeconfig logger.go:42: 15:00:22 | gr-self-healing/17-destroy-chaos-mesh | WARNING: Kubernetes configuration file is world-readable. This is insecure. Location: /mnt/jenkins/workspace/cloud-ps-operator_PR-424/kubeconfig logger.go:42: 15:00:23 | gr-self-healing/17-destroy-chaos-mesh | + local chaos_mesh_ns=kuttl-test-secure-shiner logger.go:42: 15:00:23 | gr-self-healing/17-destroy-chaos-mesh | ++ kubectl api-resources logger.go:42: 15:00:23 | gr-self-healing/17-destroy-chaos-mesh | ++ grep chaos-mesh logger.go:42: 15:00:23 | gr-self-healing/17-destroy-chaos-mesh | ++ awk '{print $1}' logger.go:42: 15:00:24 | gr-self-healing/17-destroy-chaos-mesh | + for i in '$(kubectl api-resources | grep chaos-mesh | awk '\''{print $1}'\'')' logger.go:42: 15:00:24 | gr-self-healing/17-destroy-chaos-mesh | + timeout 30 kubectl delete awschaos --all --all-namespaces logger.go:42: 15:00:25 | gr-self-healing/17-destroy-chaos-mesh | No resources found logger.go:42: 15:00:25 | gr-self-healing/17-destroy-chaos-mesh | + for i in '$(kubectl api-resources | grep chaos-mesh | awk '\''{print $1}'\'')' logger.go:42: 15:00:25 | gr-self-healing/17-destroy-chaos-mesh | + timeout 30 kubectl delete azurechaos --all --all-namespaces logger.go:42: 15:00:25 | gr-self-healing/17-destroy-chaos-mesh | No resources found logger.go:42: 15:00:25 | gr-self-healing/17-destroy-chaos-mesh | + for i in '$(kubectl api-resources | grep chaos-mesh | awk '\''{print $1}'\'')' logger.go:42: 15:00:25 | gr-self-healing/17-destroy-chaos-mesh | + timeout 30 kubectl delete blockchaos --all --all-namespaces logger.go:42: 15:00:26 | gr-self-healing/17-destroy-chaos-mesh | No resources found logger.go:42: 15:00:26 | gr-self-healing/17-destroy-chaos-mesh | + for i in '$(kubectl api-resources | grep chaos-mesh | awk '\''{print $1}'\'')' logger.go:42: 15:00:26 | gr-self-healing/17-destroy-chaos-mesh | + timeout 30 kubectl delete dnschaos --all --all-namespaces logger.go:42: 15:00:27 | gr-self-healing/17-destroy-chaos-mesh | No resources found logger.go:42: 15:00:27 | gr-self-healing/17-destroy-chaos-mesh | + for i in '$(kubectl api-resources | grep chaos-mesh | awk '\''{print $1}'\'')' logger.go:42: 15:00:27 | gr-self-healing/17-destroy-chaos-mesh | + timeout 30 kubectl delete gcpchaos --all --all-namespaces logger.go:42: 15:00:27 | gr-self-healing/17-destroy-chaos-mesh | No resources found logger.go:42: 15:00:27 | gr-self-healing/17-destroy-chaos-mesh | + for i in '$(kubectl api-resources | grep chaos-mesh | awk '\''{print $1}'\'')' logger.go:42: 15:00:27 | gr-self-healing/17-destroy-chaos-mesh | + timeout 30 kubectl delete httpchaos --all --all-namespaces logger.go:42: 15:00:28 | gr-self-healing/17-destroy-chaos-mesh | No resources found logger.go:42: 15:00:28 | gr-self-healing/17-destroy-chaos-mesh | + for i in '$(kubectl api-resources | grep chaos-mesh | awk '\''{print $1}'\'')' logger.go:42: 15:00:28 | gr-self-healing/17-destroy-chaos-mesh | + timeout 30 kubectl delete iochaos --all --all-namespaces logger.go:42: 15:00:29 | gr-self-healing/17-destroy-chaos-mesh | No resources found logger.go:42: 15:00:29 | gr-self-healing/17-destroy-chaos-mesh | + for i in '$(kubectl api-resources | grep chaos-mesh | awk '\''{print $1}'\'')' logger.go:42: 15:00:29 | gr-self-healing/17-destroy-chaos-mesh | + timeout 30 kubectl delete jvmchaos --all --all-namespaces logger.go:42: 15:00:30 | gr-self-healing/17-destroy-chaos-mesh | No resources found logger.go:42: 15:00:30 | gr-self-healing/17-destroy-chaos-mesh | + for i in '$(kubectl api-resources | grep chaos-mesh | awk '\''{print $1}'\'')' logger.go:42: 15:00:30 | gr-self-healing/17-destroy-chaos-mesh | + timeout 30 kubectl delete kernelchaos --all --all-namespaces logger.go:42: 15:00:30 | gr-self-healing/17-destroy-chaos-mesh | No resources found logger.go:42: 15:00:30 | gr-self-healing/17-destroy-chaos-mesh | + for i in '$(kubectl api-resources | grep chaos-mesh | awk '\''{print $1}'\'')' logger.go:42: 15:00:30 | gr-self-healing/17-destroy-chaos-mesh | + timeout 30 kubectl delete networkchaos --all --all-namespaces logger.go:42: 15:00:31 | gr-self-healing/17-destroy-chaos-mesh | networkchaos.chaos-mesh.org "chaos-pod-network-loss-primary" deleted logger.go:42: 15:00:31 | gr-self-healing/17-destroy-chaos-mesh | + for i in '$(kubectl api-resources | grep chaos-mesh | awk '\''{print $1}'\'')' logger.go:42: 15:00:31 | gr-self-healing/17-destroy-chaos-mesh | + timeout 30 kubectl delete physicalmachinechaos --all --all-namespaces logger.go:42: 15:00:32 | gr-self-healing/17-destroy-chaos-mesh | No resources found logger.go:42: 15:00:32 | gr-self-healing/17-destroy-chaos-mesh | + for i in '$(kubectl api-resources | grep chaos-mesh | awk '\''{print $1}'\'')' logger.go:42: 15:00:32 | gr-self-healing/17-destroy-chaos-mesh | + timeout 30 kubectl delete physicalmachines --all --all-namespaces logger.go:42: 15:00:33 | gr-self-healing/17-destroy-chaos-mesh | No resources found logger.go:42: 15:00:33 | gr-self-healing/17-destroy-chaos-mesh | + for i in '$(kubectl api-resources | grep chaos-mesh | awk '\''{print $1}'\'')' logger.go:42: 15:00:33 | gr-self-healing/17-destroy-chaos-mesh | + timeout 30 kubectl delete podchaos --all --all-namespaces logger.go:42: 15:00:34 | gr-self-healing/17-destroy-chaos-mesh | podchaos.chaos-mesh.org "chaos-kill-label-cluster-crash" deleted logger.go:42: 15:00:34 | gr-self-healing/17-destroy-chaos-mesh | podchaos.chaos-mesh.org "chaos-pod-failure-primary" deleted logger.go:42: 15:00:34 | gr-self-healing/17-destroy-chaos-mesh | podchaos.chaos-mesh.org "chaos-pod-kill-primary" deleted logger.go:42: 15:00:34 | gr-self-healing/17-destroy-chaos-mesh | + for i in '$(kubectl api-resources | grep chaos-mesh | awk '\''{print $1}'\'')' logger.go:42: 15:00:34 | gr-self-healing/17-destroy-chaos-mesh | + timeout 30 kubectl delete podhttpchaos --all --all-namespaces logger.go:42: 15:00:35 | gr-self-healing/17-destroy-chaos-mesh | No resources found logger.go:42: 15:00:35 | gr-self-healing/17-destroy-chaos-mesh | + for i in '$(kubectl api-resources | grep chaos-mesh | awk '\''{print $1}'\'')' logger.go:42: 15:00:35 | gr-self-healing/17-destroy-chaos-mesh | + timeout 30 kubectl delete podiochaos --all --all-namespaces logger.go:42: 15:00:36 | gr-self-healing/17-destroy-chaos-mesh | No resources found logger.go:42: 15:00:36 | gr-self-healing/17-destroy-chaos-mesh | + for i in '$(kubectl api-resources | grep chaos-mesh | awk '\''{print $1}'\'')' logger.go:42: 15:00:36 | gr-self-healing/17-destroy-chaos-mesh | + timeout 30 kubectl delete podnetworkchaos --all --all-namespaces logger.go:42: 15:00:36 | gr-self-healing/17-destroy-chaos-mesh | No resources found logger.go:42: 15:00:36 | gr-self-healing/17-destroy-chaos-mesh | + for i in '$(kubectl api-resources | grep chaos-mesh | awk '\''{print $1}'\'')' logger.go:42: 15:00:36 | gr-self-healing/17-destroy-chaos-mesh | + timeout 30 kubectl delete remoteclusters --all --all-namespaces logger.go:42: 15:00:37 | gr-self-healing/17-destroy-chaos-mesh | No resources found logger.go:42: 15:00:37 | gr-self-healing/17-destroy-chaos-mesh | + for i in '$(kubectl api-resources | grep chaos-mesh | awk '\''{print $1}'\'')' logger.go:42: 15:00:37 | gr-self-healing/17-destroy-chaos-mesh | + timeout 30 kubectl delete schedules --all --all-namespaces logger.go:42: 15:00:38 | gr-self-healing/17-destroy-chaos-mesh | No resources found logger.go:42: 15:00:38 | gr-self-healing/17-destroy-chaos-mesh | + for i in '$(kubectl api-resources | grep chaos-mesh | awk '\''{print $1}'\'')' logger.go:42: 15:00:38 | gr-self-healing/17-destroy-chaos-mesh | + timeout 30 kubectl delete statuschecks --all --all-namespaces logger.go:42: 15:00:39 | gr-self-healing/17-destroy-chaos-mesh | No resources found logger.go:42: 15:00:39 | gr-self-healing/17-destroy-chaos-mesh | + for i in '$(kubectl api-resources | grep chaos-mesh | awk '\''{print $1}'\'')' logger.go:42: 15:00:39 | gr-self-healing/17-destroy-chaos-mesh | + timeout 30 kubectl delete stresschaos --all --all-namespaces logger.go:42: 15:00:39 | gr-self-healing/17-destroy-chaos-mesh | No resources found logger.go:42: 15:00:39 | gr-self-healing/17-destroy-chaos-mesh | + for i in '$(kubectl api-resources | grep chaos-mesh | awk '\''{print $1}'\'')' logger.go:42: 15:00:39 | gr-self-healing/17-destroy-chaos-mesh | + timeout 30 kubectl delete timechaos --all --all-namespaces logger.go:42: 15:00:40 | gr-self-healing/17-destroy-chaos-mesh | No resources found logger.go:42: 15:00:40 | gr-self-healing/17-destroy-chaos-mesh | + for i in '$(kubectl api-resources | grep chaos-mesh | awk '\''{print $1}'\'')' logger.go:42: 15:00:40 | gr-self-healing/17-destroy-chaos-mesh | + timeout 30 kubectl delete workflownodes --all --all-namespaces logger.go:42: 15:00:41 | gr-self-healing/17-destroy-chaos-mesh | No resources found logger.go:42: 15:00:41 | gr-self-healing/17-destroy-chaos-mesh | + for i in '$(kubectl api-resources | grep chaos-mesh | awk '\''{print $1}'\'')' logger.go:42: 15:00:41 | gr-self-healing/17-destroy-chaos-mesh | + timeout 30 kubectl delete workflows --all --all-namespaces logger.go:42: 15:00:41 | gr-self-healing/17-destroy-chaos-mesh | No resources found logger.go:42: 15:00:41 | gr-self-healing/17-destroy-chaos-mesh | + '[' -n kuttl-test-secure-shiner ']' logger.go:42: 15:00:41 | gr-self-healing/17-destroy-chaos-mesh | + helm uninstall chaos-mesh --namespace kuttl-test-secure-shiner logger.go:42: 15:00:41 | gr-self-healing/17-destroy-chaos-mesh | WARNING: Kubernetes configuration file is group-readable. This is insecure. Location: /mnt/jenkins/workspace/cloud-ps-operator_PR-424/kubeconfig logger.go:42: 15:00:41 | gr-self-healing/17-destroy-chaos-mesh | WARNING: Kubernetes configuration file is world-readable. This is insecure. Location: /mnt/jenkins/workspace/cloud-ps-operator_PR-424/kubeconfig logger.go:42: 15:00:46 | gr-self-healing/17-destroy-chaos-mesh | release "chaos-mesh" uninstalled logger.go:42: 15:00:46 | gr-self-healing/17-destroy-chaos-mesh | ++ kubectl get crd logger.go:42: 15:00:46 | gr-self-healing/17-destroy-chaos-mesh | ++ grep chaos-mesh.org logger.go:42: 15:00:46 | gr-self-healing/17-destroy-chaos-mesh | ++ awk '{print $1}' logger.go:42: 15:00:47 | gr-self-healing/17-destroy-chaos-mesh | + timeout 30 kubectl delete crd awschaos.chaos-mesh.org azurechaos.chaos-mesh.org blockchaos.chaos-mesh.org dnschaos.chaos-mesh.org gcpchaos.chaos-mesh.org httpchaos.chaos-mesh.org iochaos.chaos-mesh.org jvmchaos.chaos-mesh.org kernelchaos.chaos-mesh.org networkchaos.chaos-mesh.org physicalmachinechaos.chaos-mesh.org physicalmachines.chaos-mesh.org podchaos.chaos-mesh.org podhttpchaos.chaos-mesh.org podiochaos.chaos-mesh.org podnetworkchaos.chaos-mesh.org remoteclusters.chaos-mesh.org schedules.chaos-mesh.org statuschecks.chaos-mesh.org stresschaos.chaos-mesh.org timechaos.chaos-mesh.org workflownodes.chaos-mesh.org workflows.chaos-mesh.org logger.go:42: 15:00:48 | gr-self-healing/17-destroy-chaos-mesh | customresourcedefinition.apiextensions.k8s.io "awschaos.chaos-mesh.org" deleted logger.go:42: 15:00:48 | gr-self-healing/17-destroy-chaos-mesh | customresourcedefinition.apiextensions.k8s.io "azurechaos.chaos-mesh.org" deleted logger.go:42: 15:00:48 | gr-self-healing/17-destroy-chaos-mesh | customresourcedefinition.apiextensions.k8s.io "blockchaos.chaos-mesh.org" deleted logger.go:42: 15:00:49 | gr-self-healing/17-destroy-chaos-mesh | customresourcedefinition.apiextensions.k8s.io "dnschaos.chaos-mesh.org" deleted logger.go:42: 15:00:49 | gr-self-healing/17-destroy-chaos-mesh | customresourcedefinition.apiextensions.k8s.io "gcpchaos.chaos-mesh.org" deleted logger.go:42: 15:00:49 | gr-self-healing/17-destroy-chaos-mesh | customresourcedefinition.apiextensions.k8s.io "httpchaos.chaos-mesh.org" deleted logger.go:42: 15:00:49 | gr-self-healing/17-destroy-chaos-mesh | customresourcedefinition.apiextensions.k8s.io "iochaos.chaos-mesh.org" deleted logger.go:42: 15:00:49 | gr-self-healing/17-destroy-chaos-mesh | customresourcedefinition.apiextensions.k8s.io "jvmchaos.chaos-mesh.org" deleted logger.go:42: 15:00:49 | gr-self-healing/17-destroy-chaos-mesh | customresourcedefinition.apiextensions.k8s.io "kernelchaos.chaos-mesh.org" deleted logger.go:42: 15:00:50 | gr-self-healing/17-destroy-chaos-mesh | customresourcedefinition.apiextensions.k8s.io "networkchaos.chaos-mesh.org" deleted logger.go:42: 15:00:50 | gr-self-healing/17-destroy-chaos-mesh | customresourcedefinition.apiextensions.k8s.io "physicalmachinechaos.chaos-mesh.org" deleted logger.go:42: 15:00:50 | gr-self-healing/17-destroy-chaos-mesh | customresourcedefinition.apiextensions.k8s.io "physicalmachines.chaos-mesh.org" deleted logger.go:42: 15:00:50 | gr-self-healing/17-destroy-chaos-mesh | customresourcedefinition.apiextensions.k8s.io "podchaos.chaos-mesh.org" deleted logger.go:42: 15:00:50 | gr-self-healing/17-destroy-chaos-mesh | customresourcedefinition.apiextensions.k8s.io "podhttpchaos.chaos-mesh.org" deleted logger.go:42: 15:00:51 | gr-self-healing/17-destroy-chaos-mesh | customresourcedefinition.apiextensions.k8s.io "podiochaos.chaos-mesh.org" deleted logger.go:42: 15:00:51 | gr-self-healing/17-destroy-chaos-mesh | customresourcedefinition.apiextensions.k8s.io "podnetworkchaos.chaos-mesh.org" deleted logger.go:42: 15:00:51 | gr-self-healing/17-destroy-chaos-mesh | customresourcedefinition.apiextensions.k8s.io "remoteclusters.chaos-mesh.org" deleted logger.go:42: 15:00:52 | gr-self-healing/17-destroy-chaos-mesh | customresourcedefinition.apiextensions.k8s.io "schedules.chaos-mesh.org" deleted logger.go:42: 15:00:52 | gr-self-healing/17-destroy-chaos-mesh | customresourcedefinition.apiextensions.k8s.io "statuschecks.chaos-mesh.org" deleted logger.go:42: 15:00:52 | gr-self-healing/17-destroy-chaos-mesh | customresourcedefinition.apiextensions.k8s.io "stresschaos.chaos-mesh.org" deleted logger.go:42: 15:00:53 | gr-self-healing/17-destroy-chaos-mesh | customresourcedefinition.apiextensions.k8s.io "timechaos.chaos-mesh.org" deleted logger.go:42: 15:00:54 | gr-self-healing/17-destroy-chaos-mesh | customresourcedefinition.apiextensions.k8s.io "workflownodes.chaos-mesh.org" deleted logger.go:42: 15:00:56 | gr-self-healing/17-destroy-chaos-mesh | customresourcedefinition.apiextensions.k8s.io "workflows.chaos-mesh.org" deleted logger.go:42: 15:01:01 | gr-self-healing/17-destroy-chaos-mesh | ++ kubectl get clusterrolebinding logger.go:42: 15:01:01 | gr-self-healing/17-destroy-chaos-mesh | ++ grep chaos-mesh logger.go:42: 15:01:01 | gr-self-healing/17-destroy-chaos-mesh | ++ awk '{print $1}' logger.go:42: 15:01:02 | gr-self-healing/17-destroy-chaos-mesh | + timeout 30 kubectl delete clusterrolebinding logger.go:42: 15:01:02 | gr-self-healing/17-destroy-chaos-mesh | error: resource(s) were provided, but no name was specified logger.go:42: 15:01:02 | gr-self-healing/17-destroy-chaos-mesh | + : logger.go:42: 15:01:02 | gr-self-healing/17-destroy-chaos-mesh | ++ kubectl get clusterrole logger.go:42: 15:01:02 | gr-self-healing/17-destroy-chaos-mesh | ++ grep chaos-mesh logger.go:42: 15:01:02 | gr-self-healing/17-destroy-chaos-mesh | ++ awk '{print $1}' logger.go:42: 15:01:03 | gr-self-healing/17-destroy-chaos-mesh | + timeout 30 kubectl delete clusterrole logger.go:42: 15:01:04 | gr-self-healing/17-destroy-chaos-mesh | error: resource(s) were provided, but no name was specified logger.go:42: 15:01:04 | gr-self-healing/17-destroy-chaos-mesh | + : logger.go:42: 15:01:04 | gr-self-healing/17-destroy-chaos-mesh | ++ kubectl get MutatingWebhookConfiguration logger.go:42: 15:01:04 | gr-self-healing/17-destroy-chaos-mesh | ++ grep chaos-mesh logger.go:42: 15:01:04 | gr-self-healing/17-destroy-chaos-mesh | ++ awk '{print $1}' logger.go:42: 15:01:05 | gr-self-healing/17-destroy-chaos-mesh | + timeout 30 kubectl delete MutatingWebhookConfiguration logger.go:42: 15:01:05 | gr-self-healing/17-destroy-chaos-mesh | error: resource(s) were provided, but no name was specified logger.go:42: 15:01:05 | gr-self-healing/17-destroy-chaos-mesh | + : logger.go:42: 15:01:05 | gr-self-healing/17-destroy-chaos-mesh | ++ kubectl get ValidatingWebhookConfiguration logger.go:42: 15:01:05 | gr-self-healing/17-destroy-chaos-mesh | ++ grep chaos-mesh logger.go:42: 15:01:05 | gr-self-healing/17-destroy-chaos-mesh | ++ awk '{print $1}' logger.go:42: 15:01:06 | gr-self-healing/17-destroy-chaos-mesh | + timeout 30 kubectl delete ValidatingWebhookConfiguration logger.go:42: 15:01:06 | gr-self-healing/17-destroy-chaos-mesh | error: resource(s) were provided, but no name was specified logger.go:42: 15:01:06 | gr-self-healing/17-destroy-chaos-mesh | + : logger.go:42: 15:01:06 | gr-self-healing/17-destroy-chaos-mesh | ++ kubectl get ValidatingWebhookConfiguration logger.go:42: 15:01:06 | gr-self-healing/17-destroy-chaos-mesh | ++ grep validate-auth logger.go:42: 15:01:06 | gr-self-healing/17-destroy-chaos-mesh | ++ awk '{print $1}' logger.go:42: 15:01:07 | gr-self-healing/17-destroy-chaos-mesh | + timeout 30 kubectl delete ValidatingWebhookConfiguration logger.go:42: 15:01:07 | gr-self-healing/17-destroy-chaos-mesh | error: resource(s) were provided, but no name was specified logger.go:42: 15:01:07 | gr-self-healing/17-destroy-chaos-mesh | + : logger.go:42: 15:01:08 | gr-self-healing/17-destroy-chaos-mesh | test step completed 17-destroy-chaos-mesh logger.go:42: 15:01:08 | gr-self-healing/18-drop-finalizer | starting test step 18-drop-finalizer logger.go:42: 15:01:10 | gr-self-healing/18-drop-finalizer | PerconaServerMySQL:kuttl-test-secure-shiner/gr-self-healing updated logger.go:42: 15:01:10 | gr-self-healing/18-drop-finalizer | test step completed 18-drop-finalizer logger.go:42: 15:01:10 | gr-self-healing | gr-self-healing events from ns kuttl-test-secure-shiner: logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:40:30 +0000 UTC Normal Pod percona-server-mysql-operator-6b56d66f99-cpd8b Scheduled Successfully assigned kuttl-test-secure-shiner/percona-server-mysql-operator-6b56d66f99-cpd8b to gke-jen-ps-424-70568ae-7-default-pool-95e5ef23-brgf logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:40:30 +0000 UTC Normal ReplicaSet.apps percona-server-mysql-operator-6b56d66f99 SuccessfulCreate Created pod: percona-server-mysql-operator-6b56d66f99-cpd8b logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:40:30 +0000 UTC Normal Deployment.apps percona-server-mysql-operator ScalingReplicaSet Scaled up replica set percona-server-mysql-operator-6b56d66f99 to 1 logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:40:32 +0000 UTC Normal Pod percona-server-mysql-operator-6b56d66f99-cpd8b.spec.containers{manager} Pulling Pulling image "perconalab/percona-server-mysql-operator:PR-424-70568ae" logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:40:33 +0000 UTC Normal Lease.coordination.k8s.io 08db2feb.percona.com LeaderElection percona-server-mysql-operator-6b56d66f99-cpd8b_77d5815a-5cd0-4c35-9a1d-83836fc204e7 became leader logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:40:33 +0000 UTC Normal Pod percona-server-mysql-operator-6b56d66f99-cpd8b.spec.containers{manager} Pulled Successfully pulled image "perconalab/percona-server-mysql-operator:PR-424-70568ae" in 426.13436ms logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:40:33 +0000 UTC Normal Pod percona-server-mysql-operator-6b56d66f99-cpd8b.spec.containers{manager} Created Created container manager logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:40:33 +0000 UTC Normal Pod percona-server-mysql-operator-6b56d66f99-cpd8b.spec.containers{manager} Started Started container manager logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:40:34 +0000 UTC Normal Pod mysql-client Scheduled Successfully assigned kuttl-test-secure-shiner/mysql-client to gke-jen-ps-424-70568ae-7-default-pool-95e5ef23-qrsp logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:40:35 +0000 UTC Normal Pod mysql-client.spec.containers{mysql-client} Pulling Pulling image "percona/percona-server:8.0.25" logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:40:45 +0000 UTC Normal Pod mysql-client.spec.containers{mysql-client} Pulled Successfully pulled image "percona/percona-server:8.0.25" in 10.883013409s logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:40:45 +0000 UTC Normal Pod mysql-client.spec.containers{mysql-client} Created Created container mysql-client logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:40:46 +0000 UTC Normal Pod mysql-client.spec.containers{mysql-client} Started Started container mysql-client logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:41:23 +0000 UTC Normal Pod chaos-controller-manager-79b5ff6dcc-24k4z Scheduled Successfully assigned kuttl-test-secure-shiner/chaos-controller-manager-79b5ff6dcc-24k4z to gke-jen-ps-424-70568ae-7-default-pool-95e5ef23-4tf7 logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:41:23 +0000 UTC Normal Pod chaos-controller-manager-79b5ff6dcc-4m5tn Scheduled Successfully assigned kuttl-test-secure-shiner/chaos-controller-manager-79b5ff6dcc-4m5tn to gke-jen-ps-424-70568ae-7-default-pool-95e5ef23-qrsp logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:41:23 +0000 UTC Normal Pod chaos-controller-manager-79b5ff6dcc-glsgl Scheduled Successfully assigned kuttl-test-secure-shiner/chaos-controller-manager-79b5ff6dcc-glsgl to gke-jen-ps-424-70568ae-7-default-pool-95e5ef23-brgf logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:41:23 +0000 UTC Normal ReplicaSet.apps chaos-controller-manager-79b5ff6dcc SuccessfulCreate Created pod: chaos-controller-manager-79b5ff6dcc-glsgl logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:41:23 +0000 UTC Normal ReplicaSet.apps chaos-controller-manager-79b5ff6dcc SuccessfulCreate Created pod: chaos-controller-manager-79b5ff6dcc-24k4z logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:41:23 +0000 UTC Normal ReplicaSet.apps chaos-controller-manager-79b5ff6dcc SuccessfulCreate Created pod: chaos-controller-manager-79b5ff6dcc-4m5tn logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:41:23 +0000 UTC Normal Deployment.apps chaos-controller-manager ScalingReplicaSet Scaled up replica set chaos-controller-manager-79b5ff6dcc to 3 logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:41:23 +0000 UTC Normal Pod chaos-daemon-6cjrn Scheduled Successfully assigned kuttl-test-secure-shiner/chaos-daemon-6cjrn to gke-jen-ps-424-70568ae-7-default-pool-95e5ef23-qrsp logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:41:23 +0000 UTC Normal Pod chaos-daemon-6cjrn.spec.containers{chaos-daemon} Pulling Pulling image "ghcr.io/chaos-mesh/chaos-daemon:v2.5.1" logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:41:23 +0000 UTC Normal Pod chaos-daemon-x294b Scheduled Successfully assigned kuttl-test-secure-shiner/chaos-daemon-x294b to gke-jen-ps-424-70568ae-7-default-pool-95e5ef23-4tf7 logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:41:23 +0000 UTC Normal Pod chaos-daemon-x294b.spec.containers{chaos-daemon} Pulling Pulling image "ghcr.io/chaos-mesh/chaos-daemon:v2.5.1" logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:41:23 +0000 UTC Normal Pod chaos-daemon-xpgrm Scheduled Successfully assigned kuttl-test-secure-shiner/chaos-daemon-xpgrm to gke-jen-ps-424-70568ae-7-default-pool-95e5ef23-brgf logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:41:23 +0000 UTC Normal DaemonSet.apps chaos-daemon SuccessfulCreate Created pod: chaos-daemon-6cjrn logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:41:23 +0000 UTC Normal DaemonSet.apps chaos-daemon SuccessfulCreate Created pod: chaos-daemon-x294b logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:41:23 +0000 UTC Normal DaemonSet.apps chaos-daemon SuccessfulCreate Created pod: chaos-daemon-xpgrm logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:41:24 +0000 UTC Normal Pod chaos-controller-manager-79b5ff6dcc-24k4z.spec.containers{chaos-mesh} Pulling Pulling image "ghcr.io/chaos-mesh/chaos-mesh:v2.5.1" logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:41:24 +0000 UTC Normal Pod chaos-controller-manager-79b5ff6dcc-4m5tn.spec.containers{chaos-mesh} Pulling Pulling image "ghcr.io/chaos-mesh/chaos-mesh:v2.5.1" logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:41:24 +0000 UTC Normal Pod chaos-controller-manager-79b5ff6dcc-glsgl.spec.containers{chaos-mesh} Pulling Pulling image "ghcr.io/chaos-mesh/chaos-mesh:v2.5.1" logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:41:24 +0000 UTC Normal Pod chaos-daemon-xpgrm.spec.containers{chaos-daemon} Pulling Pulling image "ghcr.io/chaos-mesh/chaos-daemon:v2.5.1" logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:41:26 +0000 UTC Normal Pod chaos-controller-manager-79b5ff6dcc-glsgl.spec.containers{chaos-mesh} Pulled Successfully pulled image "ghcr.io/chaos-mesh/chaos-mesh:v2.5.1" in 1.930296237s logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:41:26 +0000 UTC Normal Pod chaos-controller-manager-79b5ff6dcc-glsgl.spec.containers{chaos-mesh} Created Created container chaos-mesh logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:41:26 +0000 UTC Normal Pod chaos-controller-manager-79b5ff6dcc-glsgl.spec.containers{chaos-mesh} Started Started container chaos-mesh logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:41:26 +0000 UTC Normal ConfigMap chaos-mesh LeaderElection chaos-controller-manager-79b5ff6dcc-glsgl_1fd745bc-4902-490f-adbf-07a8e951473b became leader logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:41:26 +0000 UTC Normal Lease.coordination.k8s.io chaos-mesh LeaderElection chaos-controller-manager-79b5ff6dcc-glsgl_1fd745bc-4902-490f-adbf-07a8e951473b became leader logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:41:33 +0000 UTC Normal Pod chaos-daemon-6cjrn.spec.containers{chaos-daemon} Pulled Successfully pulled image "ghcr.io/chaos-mesh/chaos-daemon:v2.5.1" in 10.344747474s logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:41:33 +0000 UTC Normal Pod chaos-daemon-6cjrn.spec.containers{chaos-daemon} Created Created container chaos-daemon logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:41:33 +0000 UTC Normal Pod chaos-daemon-6cjrn.spec.containers{chaos-daemon} Started Started container chaos-daemon logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:41:33 +0000 UTC Normal Pod chaos-daemon-x294b.spec.containers{chaos-daemon} Pulled Successfully pulled image "ghcr.io/chaos-mesh/chaos-daemon:v2.5.1" in 10.019348251s logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:41:33 +0000 UTC Normal Pod chaos-daemon-x294b.spec.containers{chaos-daemon} Created Created container chaos-daemon logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:41:34 +0000 UTC Normal Pod chaos-daemon-x294b.spec.containers{chaos-daemon} Started Started container chaos-daemon logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:41:35 +0000 UTC Normal Pod chaos-controller-manager-79b5ff6dcc-24k4z.spec.containers{chaos-mesh} Pulled Successfully pulled image "ghcr.io/chaos-mesh/chaos-mesh:v2.5.1" in 11.408593968s logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:41:35 +0000 UTC Normal Pod chaos-controller-manager-79b5ff6dcc-24k4z.spec.containers{chaos-mesh} Created Created container chaos-mesh logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:41:35 +0000 UTC Normal Pod chaos-controller-manager-79b5ff6dcc-24k4z.spec.containers{chaos-mesh} Started Started container chaos-mesh logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:41:35 +0000 UTC Normal Pod chaos-controller-manager-79b5ff6dcc-4m5tn.spec.containers{chaos-mesh} Pulled Successfully pulled image "ghcr.io/chaos-mesh/chaos-mesh:v2.5.1" in 10.874948593s logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:41:35 +0000 UTC Normal Pod chaos-controller-manager-79b5ff6dcc-4m5tn.spec.containers{chaos-mesh} Created Created container chaos-mesh logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:41:35 +0000 UTC Normal Pod chaos-controller-manager-79b5ff6dcc-4m5tn.spec.containers{chaos-mesh} Started Started container chaos-mesh logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:41:36 +0000 UTC Normal Pod chaos-daemon-xpgrm.spec.containers{chaos-daemon} Pulled Successfully pulled image "ghcr.io/chaos-mesh/chaos-daemon:v2.5.1" in 11.795657725s logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:41:36 +0000 UTC Normal Pod chaos-daemon-xpgrm.spec.containers{chaos-daemon} Created Created container chaos-daemon logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:41:36 +0000 UTC Normal Pod chaos-daemon-xpgrm.spec.containers{chaos-daemon} Started Started container chaos-daemon logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:41:50 +0000 UTC Normal PersistentVolumeClaim datadir-gr-self-healing-mysql-0 WaitForFirstConsumer waiting for first consumer to be created before binding logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:41:50 +0000 UTC Normal PersistentVolumeClaim datadir-gr-self-healing-mysql-0 Provisioning External provisioner is provisioning volume for claim "kuttl-test-secure-shiner/datadir-gr-self-healing-mysql-0" logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:41:50 +0000 UTC Normal PersistentVolumeClaim datadir-gr-self-healing-mysql-0 ExternalProvisioning waiting for a volume to be created, either by external provisioner "pd.csi.storage.gke.io" or manually created by system administrator logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:41:50 +0000 UTC Normal StatefulSet.apps gr-self-healing-mysql SuccessfulCreate create Claim datadir-gr-self-healing-mysql-0 Pod gr-self-healing-mysql-0 in StatefulSet gr-self-healing-mysql success logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:41:50 +0000 UTC Normal StatefulSet.apps gr-self-healing-mysql SuccessfulCreate create Pod gr-self-healing-mysql-0 in StatefulSet gr-self-healing-mysql successful logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:41:54 +0000 UTC Normal PersistentVolumeClaim datadir-gr-self-healing-mysql-0 ProvisioningSucceeded Successfully provisioned volume pvc-9e501573-12ef-4b7f-b5ed-4b7746a4da5d logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:41:54 +0000 UTC Normal Pod gr-self-healing-mysql-0 Scheduled Successfully assigned kuttl-test-secure-shiner/gr-self-healing-mysql-0 to gke-jen-ps-424-70568ae-7-default-pool-95e5ef23-4tf7 logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:41:56 +0000 UTC Warning Pod gr-self-healing-mysql-0 FailedMount MountVolume.SetUp failed for volume "tls" : failed to sync secret cache: timed out waiting for the condition logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:41:56 +0000 UTC Warning Pod gr-self-healing-mysql-0 FailedMount MountVolume.SetUp failed for volume "config" : failed to sync configmap cache: timed out waiting for the condition logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:42:02 +0000 UTC Normal Pod gr-self-healing-mysql-0 SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-9e501573-12ef-4b7f-b5ed-4b7746a4da5d" logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:42:05 +0000 UTC Normal Pod gr-self-healing-mysql-0.spec.initContainers{mysql-init} Pulling Pulling image "perconalab/percona-server-mysql-operator:PR-424-70568ae" logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:42:06 +0000 UTC Normal Pod gr-self-healing-mysql-0.spec.initContainers{mysql-init} Pulled Successfully pulled image "perconalab/percona-server-mysql-operator:PR-424-70568ae" in 644.895921ms logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:42:06 +0000 UTC Normal Pod gr-self-healing-mysql-0.spec.initContainers{mysql-init} Created Created container mysql-init logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:42:06 +0000 UTC Normal Pod gr-self-healing-mysql-0.spec.initContainers{mysql-init} Started Started container mysql-init logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:42:08 +0000 UTC Normal Pod gr-self-healing-mysql-0.spec.containers{mysql} Pulling Pulling image "perconalab/percona-server-mysql-operator:main-psmysql" logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:42:09 +0000 UTC Normal Pod gr-self-healing-mysql-0.spec.containers{mysql} Pulled Successfully pulled image "perconalab/percona-server-mysql-operator:main-psmysql" in 364.392124ms logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:42:09 +0000 UTC Normal Pod gr-self-healing-mysql-0.spec.containers{mysql} Created Created container mysql logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:42:09 +0000 UTC Normal Pod gr-self-healing-mysql-0.spec.containers{mysql} Started Started container mysql logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:42:09 +0000 UTC Normal Pod gr-self-healing-mysql-0.spec.containers{xtrabackup} Pulling Pulling image "perconalab/percona-server-mysql-operator:main-backup" logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:42:09 +0000 UTC Normal Pod gr-self-healing-mysql-0.spec.containers{xtrabackup} Pulled Successfully pulled image "perconalab/percona-server-mysql-operator:main-backup" in 396.572276ms logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:42:09 +0000 UTC Normal Pod gr-self-healing-mysql-0.spec.containers{xtrabackup} Created Created container xtrabackup logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:42:09 +0000 UTC Normal Pod gr-self-healing-mysql-0.spec.containers{xtrabackup} Started Started container xtrabackup logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:42:44 +0000 UTC Normal PersistentVolumeClaim datadir-gr-self-healing-mysql-1 WaitForFirstConsumer waiting for first consumer to be created before binding logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:42:44 +0000 UTC Normal PersistentVolumeClaim datadir-gr-self-healing-mysql-1 ExternalProvisioning waiting for a volume to be created, either by external provisioner "pd.csi.storage.gke.io" or manually created by system administrator logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:42:44 +0000 UTC Normal PersistentVolumeClaim datadir-gr-self-healing-mysql-1 Provisioning External provisioner is provisioning volume for claim "kuttl-test-secure-shiner/datadir-gr-self-healing-mysql-1" logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:42:44 +0000 UTC Normal StatefulSet.apps gr-self-healing-mysql SuccessfulCreate create Claim datadir-gr-self-healing-mysql-1 Pod gr-self-healing-mysql-1 in StatefulSet gr-self-healing-mysql success logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:42:44 +0000 UTC Normal StatefulSet.apps gr-self-healing-mysql SuccessfulCreate create Pod gr-self-healing-mysql-1 in StatefulSet gr-self-healing-mysql successful logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:42:48 +0000 UTC Normal PersistentVolumeClaim datadir-gr-self-healing-mysql-1 ProvisioningSucceeded Successfully provisioned volume pvc-921dea6d-03d0-43c8-8e0d-df778a9a539d logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:42:49 +0000 UTC Normal Pod gr-self-healing-mysql-1 Scheduled Successfully assigned kuttl-test-secure-shiner/gr-self-healing-mysql-1 to gke-jen-ps-424-70568ae-7-default-pool-95e5ef23-qrsp logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:42:54 +0000 UTC Normal Pod gr-self-healing-mysql-1 SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-921dea6d-03d0-43c8-8e0d-df778a9a539d" logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:42:55 +0000 UTC Normal Pod gr-self-healing-mysql-1.spec.initContainers{mysql-init} Pulling Pulling image "perconalab/percona-server-mysql-operator:PR-424-70568ae" logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:42:56 +0000 UTC Normal Pod gr-self-healing-mysql-1.spec.initContainers{mysql-init} Pulled Successfully pulled image "perconalab/percona-server-mysql-operator:PR-424-70568ae" in 381.594065ms logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:42:56 +0000 UTC Normal Pod gr-self-healing-mysql-1.spec.initContainers{mysql-init} Created Created container mysql-init logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:42:56 +0000 UTC Normal Pod gr-self-healing-mysql-1.spec.initContainers{mysql-init} Started Started container mysql-init logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:42:57 +0000 UTC Normal Pod gr-self-healing-mysql-1.spec.containers{mysql} Pulling Pulling image "perconalab/percona-server-mysql-operator:main-psmysql" logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:42:57 +0000 UTC Normal Pod gr-self-healing-mysql-1.spec.containers{mysql} Pulled Successfully pulled image "perconalab/percona-server-mysql-operator:main-psmysql" in 351.183582ms logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:42:57 +0000 UTC Normal Pod gr-self-healing-mysql-1.spec.containers{mysql} Created Created container mysql logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:42:58 +0000 UTC Normal Pod gr-self-healing-mysql-1.spec.containers{mysql} Started Started container mysql logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:42:58 +0000 UTC Normal Pod gr-self-healing-mysql-1.spec.containers{xtrabackup} Pulling Pulling image "perconalab/percona-server-mysql-operator:main-backup" logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:42:58 +0000 UTC Normal Pod gr-self-healing-mysql-1.spec.containers{xtrabackup} Pulled Successfully pulled image "perconalab/percona-server-mysql-operator:main-backup" in 361.455169ms logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:42:58 +0000 UTC Normal Pod gr-self-healing-mysql-1.spec.containers{xtrabackup} Created Created container xtrabackup logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:42:58 +0000 UTC Normal Pod gr-self-healing-mysql-1.spec.containers{xtrabackup} Started Started container xtrabackup logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:43:27 +0000 UTC Warning Pod gr-self-healing-mysql-1.spec.containers{mysql} Unhealthy Startup probe failed: logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:43:27 +0000 UTC Normal Pod gr-self-healing-mysql-1.spec.containers{mysql} Killing Container mysql failed startup probe, will be restarted logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:43:27 +0000 UTC Normal Pod gr-self-healing-mysql-1.spec.containers{mysql} Pulled Successfully pulled image "perconalab/percona-server-mysql-operator:main-psmysql" in 393.3531ms logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:44:00 +0000 UTC Normal PersistentVolumeClaim datadir-gr-self-healing-mysql-2 WaitForFirstConsumer waiting for first consumer to be created before binding logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:44:00 +0000 UTC Normal PersistentVolumeClaim datadir-gr-self-healing-mysql-2 ExternalProvisioning waiting for a volume to be created, either by external provisioner "pd.csi.storage.gke.io" or manually created by system administrator logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:44:00 +0000 UTC Normal PersistentVolumeClaim datadir-gr-self-healing-mysql-2 Provisioning External provisioner is provisioning volume for claim "kuttl-test-secure-shiner/datadir-gr-self-healing-mysql-2" logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:44:00 +0000 UTC Normal StatefulSet.apps gr-self-healing-mysql SuccessfulCreate create Claim datadir-gr-self-healing-mysql-2 Pod gr-self-healing-mysql-2 in StatefulSet gr-self-healing-mysql success logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:44:00 +0000 UTC Normal StatefulSet.apps gr-self-healing-mysql SuccessfulCreate create Pod gr-self-healing-mysql-2 in StatefulSet gr-self-healing-mysql successful logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:44:04 +0000 UTC Normal PersistentVolumeClaim datadir-gr-self-healing-mysql-2 ProvisioningSucceeded Successfully provisioned volume pvc-bc9ef768-10ae-4ec9-8167-bb39c05fe755 logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:44:04 +0000 UTC Normal Pod gr-self-healing-mysql-2 Scheduled Successfully assigned kuttl-test-secure-shiner/gr-self-healing-mysql-2 to gke-jen-ps-424-70568ae-7-default-pool-95e5ef23-brgf logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:44:12 +0000 UTC Normal Pod gr-self-healing-mysql-2 SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-bc9ef768-10ae-4ec9-8167-bb39c05fe755" logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:44:13 +0000 UTC Normal Pod gr-self-healing-mysql-2.spec.initContainers{mysql-init} Pulling Pulling image "perconalab/percona-server-mysql-operator:PR-424-70568ae" logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:44:13 +0000 UTC Normal Pod gr-self-healing-mysql-2.spec.initContainers{mysql-init} Pulled Successfully pulled image "perconalab/percona-server-mysql-operator:PR-424-70568ae" in 387.934011ms logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:44:13 +0000 UTC Normal Pod gr-self-healing-mysql-2.spec.initContainers{mysql-init} Created Created container mysql-init logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:44:13 +0000 UTC Normal Pod gr-self-healing-mysql-2.spec.initContainers{mysql-init} Started Started container mysql-init logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:44:15 +0000 UTC Normal Pod gr-self-healing-mysql-2.spec.containers{mysql} Pulling Pulling image "perconalab/percona-server-mysql-operator:main-psmysql" logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:44:15 +0000 UTC Normal Pod gr-self-healing-mysql-2.spec.containers{mysql} Pulled Successfully pulled image "perconalab/percona-server-mysql-operator:main-psmysql" in 369.533004ms logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:44:15 +0000 UTC Normal Pod gr-self-healing-mysql-2.spec.containers{mysql} Created Created container mysql logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:44:16 +0000 UTC Normal Pod gr-self-healing-mysql-2.spec.containers{mysql} Started Started container mysql logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:44:16 +0000 UTC Normal Pod gr-self-healing-mysql-2.spec.containers{xtrabackup} Pulling Pulling image "perconalab/percona-server-mysql-operator:main-backup" logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:44:16 +0000 UTC Normal Pod gr-self-healing-mysql-2.spec.containers{xtrabackup} Pulled Successfully pulled image "perconalab/percona-server-mysql-operator:main-backup" in 391.730574ms logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:44:16 +0000 UTC Normal Pod gr-self-healing-mysql-2.spec.containers{xtrabackup} Created Created container xtrabackup logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:44:16 +0000 UTC Normal Pod gr-self-healing-mysql-2.spec.containers{xtrabackup} Started Started container xtrabackup logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:44:45 +0000 UTC Warning Pod gr-self-healing-mysql-2.spec.containers{mysql} Unhealthy Startup probe failed: logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:44:45 +0000 UTC Normal Pod gr-self-healing-mysql-2.spec.containers{mysql} Killing Container mysql failed startup probe, will be restarted logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:44:45 +0000 UTC Normal Pod gr-self-healing-mysql-2.spec.containers{mysql} Pulled Successfully pulled image "perconalab/percona-server-mysql-operator:main-psmysql" in 375.584469ms logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:45:33 +0000 UTC Normal Pod gr-self-healing-router-64c5f89d45-6dgrd Scheduled Successfully assigned kuttl-test-secure-shiner/gr-self-healing-router-64c5f89d45-6dgrd to gke-jen-ps-424-70568ae-7-default-pool-95e5ef23-qrsp logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:45:33 +0000 UTC Normal Pod gr-self-healing-router-64c5f89d45-wpr78 Scheduled Successfully assigned kuttl-test-secure-shiner/gr-self-healing-router-64c5f89d45-wpr78 to gke-jen-ps-424-70568ae-7-default-pool-95e5ef23-4tf7 logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:45:33 +0000 UTC Normal Pod gr-self-healing-router-64c5f89d45-xg7kn Scheduled Successfully assigned kuttl-test-secure-shiner/gr-self-healing-router-64c5f89d45-xg7kn to gke-jen-ps-424-70568ae-7-default-pool-95e5ef23-brgf logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:45:33 +0000 UTC Normal ReplicaSet.apps gr-self-healing-router-64c5f89d45 SuccessfulCreate Created pod: gr-self-healing-router-64c5f89d45-wpr78 logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:45:33 +0000 UTC Normal ReplicaSet.apps gr-self-healing-router-64c5f89d45 SuccessfulCreate Created pod: gr-self-healing-router-64c5f89d45-6dgrd logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:45:33 +0000 UTC Normal ReplicaSet.apps gr-self-healing-router-64c5f89d45 SuccessfulCreate Created pod: gr-self-healing-router-64c5f89d45-xg7kn logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:45:33 +0000 UTC Normal Deployment.apps gr-self-healing-router ScalingReplicaSet Scaled up replica set gr-self-healing-router-64c5f89d45 to 3 logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:45:34 +0000 UTC Normal Pod gr-self-healing-router-64c5f89d45-6dgrd.spec.initContainers{router-init} Pulling Pulling image "perconalab/percona-server-mysql-operator:PR-424-70568ae" logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:45:34 +0000 UTC Normal Pod gr-self-healing-router-64c5f89d45-wpr78.spec.initContainers{router-init} Pulling Pulling image "perconalab/percona-server-mysql-operator:PR-424-70568ae" logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:45:34 +0000 UTC Normal Pod gr-self-healing-router-64c5f89d45-xg7kn.spec.initContainers{router-init} Pulling Pulling image "perconalab/percona-server-mysql-operator:PR-424-70568ae" logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:45:35 +0000 UTC Normal Pod gr-self-healing-router-64c5f89d45-6dgrd.spec.initContainers{router-init} Pulled Successfully pulled image "perconalab/percona-server-mysql-operator:PR-424-70568ae" in 1.444507056s logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:45:35 +0000 UTC Normal Pod gr-self-healing-router-64c5f89d45-6dgrd.spec.initContainers{router-init} Created Created container router-init logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:45:35 +0000 UTC Normal Pod gr-self-healing-router-64c5f89d45-6dgrd.spec.initContainers{router-init} Started Started container router-init logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:45:35 +0000 UTC Normal Pod gr-self-healing-router-64c5f89d45-wpr78.spec.initContainers{router-init} Pulled Successfully pulled image "perconalab/percona-server-mysql-operator:PR-424-70568ae" in 1.561169696s logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:45:35 +0000 UTC Normal Pod gr-self-healing-router-64c5f89d45-wpr78.spec.initContainers{router-init} Created Created container router-init logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:45:35 +0000 UTC Normal Pod gr-self-healing-router-64c5f89d45-wpr78.spec.initContainers{router-init} Started Started container router-init logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:45:35 +0000 UTC Normal Pod gr-self-healing-router-64c5f89d45-xg7kn.spec.initContainers{router-init} Pulled Successfully pulled image "perconalab/percona-server-mysql-operator:PR-424-70568ae" in 1.515931544s logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:45:35 +0000 UTC Normal Pod gr-self-healing-router-64c5f89d45-xg7kn.spec.initContainers{router-init} Created Created container router-init logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:45:35 +0000 UTC Normal Pod gr-self-healing-router-64c5f89d45-xg7kn.spec.initContainers{router-init} Started Started container router-init logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:45:36 +0000 UTC Normal Pod gr-self-healing-router-64c5f89d45-6dgrd.spec.containers{router} Pulling Pulling image "perconalab/percona-server-mysql-operator:main-router" logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:45:36 +0000 UTC Normal Pod gr-self-healing-router-64c5f89d45-xg7kn.spec.containers{router} Pulling Pulling image "perconalab/percona-server-mysql-operator:main-router" logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:45:37 +0000 UTC Normal Pod gr-self-healing-router-64c5f89d45-6dgrd.spec.containers{router} Pulled Successfully pulled image "perconalab/percona-server-mysql-operator:main-router" in 369.646742ms logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:45:37 +0000 UTC Normal Pod gr-self-healing-router-64c5f89d45-6dgrd.spec.containers{router} Created Created container router logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:45:37 +0000 UTC Normal Pod gr-self-healing-router-64c5f89d45-6dgrd.spec.containers{router} Started Started container router logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:45:37 +0000 UTC Normal Pod gr-self-healing-router-64c5f89d45-wpr78.spec.containers{router} Pulling Pulling image "perconalab/percona-server-mysql-operator:main-router" logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:45:37 +0000 UTC Normal Pod gr-self-healing-router-64c5f89d45-wpr78.spec.containers{router} Pulled Successfully pulled image "perconalab/percona-server-mysql-operator:main-router" in 379.692394ms logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:45:37 +0000 UTC Normal Pod gr-self-healing-router-64c5f89d45-wpr78.spec.containers{router} Created Created container router logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:45:37 +0000 UTC Normal Pod gr-self-healing-router-64c5f89d45-xg7kn.spec.containers{router} Pulled Successfully pulled image "perconalab/percona-server-mysql-operator:main-router" in 353.095275ms logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:45:37 +0000 UTC Normal Pod gr-self-healing-router-64c5f89d45-xg7kn.spec.containers{router} Created Created container router logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:45:37 +0000 UTC Normal Pod gr-self-healing-router-64c5f89d45-xg7kn.spec.containers{router} Started Started container router logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:45:38 +0000 UTC Normal Pod gr-self-healing-router-64c5f89d45-wpr78.spec.containers{router} Started Started container router logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:46:27 +0000 UTC Normal PodChaos.chaos-mesh.org chaos-pod-kill-primary FinalizerInited Finalizer has been inited logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:46:28 +0000 UTC Normal PodChaos.chaos-mesh.org chaos-pod-kill-primary Updated Successfully update finalizer of resource logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:46:28 +0000 UTC Normal PodChaos.chaos-mesh.org chaos-pod-kill-primary Updated Successfully update desiredPhase of resource logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:46:28 +0000 UTC Normal PodChaos.chaos-mesh.org chaos-pod-kill-primary Applied Successfully apply chaos for kuttl-test-secure-shiner/gr-self-healing-mysql-0 logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:46:28 +0000 UTC Normal PodChaos.chaos-mesh.org chaos-pod-kill-primary Updated Successfully update records of resource logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:46:28 +0000 UTC Normal Pod gr-self-healing-mysql-0.spec.containers{mysql} Killing Stopping container mysql logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:46:28 +0000 UTC Normal Pod gr-self-healing-mysql-0.spec.containers{xtrabackup} Killing Stopping container xtrabackup logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:46:28 +0000 UTC Normal Pod gr-self-healing-mysql-0 Scheduled Successfully assigned kuttl-test-secure-shiner/gr-self-healing-mysql-0 to gke-jen-ps-424-70568ae-7-default-pool-95e5ef23-4tf7 logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:46:29 +0000 UTC Normal Pod gr-self-healing-mysql-0.spec.initContainers{mysql-init} Pulling Pulling image "perconalab/percona-server-mysql-operator:PR-424-70568ae" logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:46:29 +0000 UTC Normal Pod gr-self-healing-mysql-0.spec.initContainers{mysql-init} Pulled Successfully pulled image "perconalab/percona-server-mysql-operator:PR-424-70568ae" in 394.005021ms logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:46:29 +0000 UTC Normal Pod gr-self-healing-mysql-0.spec.initContainers{mysql-init} Created Created container mysql-init logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:46:29 +0000 UTC Normal Pod gr-self-healing-mysql-0.spec.initContainers{mysql-init} Started Started container mysql-init logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:46:30 +0000 UTC Normal Pod gr-self-healing-mysql-0.spec.containers{mysql} Pulling Pulling image "perconalab/percona-server-mysql-operator:main-psmysql" logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:46:31 +0000 UTC Normal Pod gr-self-healing-mysql-0.spec.containers{mysql} Pulled Successfully pulled image "perconalab/percona-server-mysql-operator:main-psmysql" in 358.2325ms logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:46:31 +0000 UTC Normal Pod gr-self-healing-mysql-0.spec.containers{mysql} Created Created container mysql logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:46:31 +0000 UTC Normal Pod gr-self-healing-mysql-0.spec.containers{mysql} Started Started container mysql logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:46:31 +0000 UTC Normal Pod gr-self-healing-mysql-0.spec.containers{xtrabackup} Pulling Pulling image "perconalab/percona-server-mysql-operator:main-backup" logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:46:31 +0000 UTC Warning Pod gr-self-healing-mysql-0.spec.containers{mysql} Unhealthy Readiness probe failed: [mysql] 2023/08/17 14:46:31 packets.go:37: read tcp 10.67.90.21:35366->10.67.90.21:33062: read: connection reset by peer 2023/08/17 14:46:31 readiness check failed: connect to db: ping database: dial tcp 10.67.90.21:33062: connect: connection refused logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:46:31 +0000 UTC Normal Pod gr-self-healing-mysql-0.spec.containers{xtrabackup} Pulled Successfully pulled image "perconalab/percona-server-mysql-operator:main-backup" in 592.971535ms logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:46:31 +0000 UTC Normal Pod gr-self-healing-mysql-0.spec.containers{xtrabackup} Created Created container xtrabackup logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:46:31 +0000 UTC Normal Pod gr-self-healing-mysql-0.spec.containers{xtrabackup} Started Started container xtrabackup logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:47:56 +0000 UTC Normal PodChaos.chaos-mesh.org chaos-pod-failure-primary FinalizerInited Finalizer has been inited logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:47:56 +0000 UTC Normal PodChaos.chaos-mesh.org chaos-pod-failure-primary Updated Successfully update finalizer of resource logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:47:56 +0000 UTC Normal PodChaos.chaos-mesh.org chaos-pod-failure-primary Started Experiment has started logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:47:56 +0000 UTC Normal PodChaos.chaos-mesh.org chaos-pod-failure-primary Updated Successfully update desiredPhase of resource logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:47:56 +0000 UTC Normal PodChaos.chaos-mesh.org chaos-pod-failure-primary Applied Successfully apply chaos for kuttl-test-secure-shiner/gr-self-healing-mysql-1 logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:47:56 +0000 UTC Normal PodChaos.chaos-mesh.org chaos-pod-failure-primary Updated Successfully update records of resource logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:47:56 +0000 UTC Normal Pod gr-self-healing-mysql-1.spec.containers{mysql} Killing Container mysql definition changed, will be restarted logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:48:00 +0000 UTC Warning Pod gr-self-healing-mysql-1.spec.containers{mysql} Unhealthy Readiness probe failed: 2023/08/17 14:48:00 readiness check failed: connect to db: ping database: dial tcp 10.67.89.18:33062: connect: connection refused logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:48:02 +0000 UTC Normal Pod gr-self-healing-mysql-1.spec.containers{xtrabackup} Killing Container xtrabackup definition changed, will be restarted logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:48:02 +0000 UTC Normal Pod gr-self-healing-mysql-1.spec.containers{mysql} Pulling Pulling image "gcr.io/google-containers/pause:latest" logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:48:03 +0000 UTC Normal Pod gr-self-healing-mysql-1.spec.containers{mysql} Pulled Successfully pulled image "gcr.io/google-containers/pause:latest" in 434.561893ms logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:48:03 +0000 UTC Normal Pod gr-self-healing-mysql-1.spec.containers{xtrabackup} Pulling Pulling image "gcr.io/google-containers/pause:latest" logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:48:56 +0000 UTC Normal PodChaos.chaos-mesh.org chaos-pod-failure-primary TimeUp Time up according to the duration logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:48:56 +0000 UTC Normal PodChaos.chaos-mesh.org chaos-pod-failure-primary Updated Successfully update desiredPhase of resource logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:48:56 +0000 UTC Normal PodChaos.chaos-mesh.org chaos-pod-failure-primary Recovered Successfully recover chaos for kuttl-test-secure-shiner/gr-self-healing-mysql-1 logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:48:56 +0000 UTC Normal PodChaos.chaos-mesh.org chaos-pod-failure-primary Updated Successfully update records of resource logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:50:24 +0000 UTC Normal NetworkChaos.chaos-mesh.org chaos-pod-network-loss-primary FinalizerInited Finalizer has been inited logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:50:24 +0000 UTC Normal NetworkChaos.chaos-mesh.org chaos-pod-network-loss-primary Updated Successfully update finalizer of resource logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:50:24 +0000 UTC Normal NetworkChaos.chaos-mesh.org chaos-pod-network-loss-primary Started Experiment has started logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:50:24 +0000 UTC Normal NetworkChaos.chaos-mesh.org chaos-pod-network-loss-primary Updated Successfully update desiredPhase of resource logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:50:24 +0000 UTC Normal NetworkChaos.chaos-mesh.org chaos-pod-network-loss-primary Updated Successfully update records of resource logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:50:25 +0000 UTC Normal NetworkChaos.chaos-mesh.org chaos-pod-network-loss-primary Applied Successfully apply chaos for kuttl-test-secure-shiner/gr-self-healing-mysql-0 logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:50:25 +0000 UTC Normal NetworkChaos.chaos-mesh.org chaos-pod-network-loss-primary Updated Successfully update records of resource logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:50:25 +0000 UTC Normal PodNetworkChaos.chaos-mesh.org gr-self-healing-mysql-0 Updated Successfully update ObservedGeneration and FailedMessage of resource logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:50:38 +0000 UTC Warning Pod gr-self-healing-mysql-0.spec.containers{mysql} Unhealthy Liveness probe failed: 2023/08/17 14:50:38 in primary partition: false 2023/08/17 14:50:38 liveness check failed: possible split brain! logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:50:38 +0000 UTC Warning Pod gr-self-healing-router-64c5f89d45-6dgrd.spec.containers{router} Unhealthy Readiness probe failed: Read-write route is not healthy logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:50:38 +0000 UTC Warning Pod gr-self-healing-router-64c5f89d45-wpr78.spec.containers{router} Unhealthy Readiness probe failed: Read-write route is not healthy logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:50:38 +0000 UTC Warning Pod gr-self-healing-router-64c5f89d45-xg7kn.spec.containers{router} Unhealthy Readiness probe failed: Read-write route is not healthy logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:50:48 +0000 UTC Warning Pod gr-self-healing-mysql-0.spec.containers{mysql} Unhealthy Liveness probe failed: 2023/08/17 14:50:48 in primary partition: false 2023/08/17 14:50:48 liveness check failed: possible split brain! logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:50:58 +0000 UTC Warning Pod gr-self-healing-mysql-0.spec.containers{mysql} Unhealthy Liveness probe failed: 2023/08/17 14:50:58 in primary partition: false 2023/08/17 14:50:58 liveness check failed: possible split brain! logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:50:58 +0000 UTC Normal Pod gr-self-healing-mysql-0.spec.containers{mysql} Killing Container mysql failed liveness probe, will be restarted logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:50:59 +0000 UTC Normal Pod gr-self-healing-mysql-0.spec.containers{mysql} Pulled Successfully pulled image "perconalab/percona-server-mysql-operator:main-psmysql" in 372.64985ms logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:51:24 +0000 UTC Normal NetworkChaos.chaos-mesh.org chaos-pod-network-loss-primary TimeUp Time up according to the duration logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:51:24 +0000 UTC Normal NetworkChaos.chaos-mesh.org chaos-pod-network-loss-primary Updated Successfully update desiredPhase of resource logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:51:24 +0000 UTC Normal NetworkChaos.chaos-mesh.org chaos-pod-network-loss-primary Updated Successfully update records of resource logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:51:24 +0000 UTC Normal NetworkChaos.chaos-mesh.org chaos-pod-network-loss-primary Recovered Successfully recover chaos for kuttl-test-secure-shiner/gr-self-healing-mysql-0 logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:51:24 +0000 UTC Normal NetworkChaos.chaos-mesh.org chaos-pod-network-loss-primary Updated Successfully update records of resource logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:51:24 +0000 UTC Normal PodNetworkChaos.chaos-mesh.org gr-self-healing-mysql-0 Updated Successfully update ObservedGeneration and FailedMessage of resource logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:51:29 +0000 UTC Warning Pod gr-self-healing-mysql-0.spec.containers{mysql} Unhealthy Startup probe failed: logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:51:29 +0000 UTC Normal Pod gr-self-healing-mysql-0.spec.containers{mysql} Killing Container mysql failed startup probe, will be restarted logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:51:59 +0000 UTC Normal Pod gr-self-healing-mysql-0.spec.containers{mysql} Pulled Successfully pulled image "perconalab/percona-server-mysql-operator:main-psmysql" in 402.322358ms logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:53:19 +0000 UTC Normal PodChaos.chaos-mesh.org chaos-kill-label-cluster-crash FinalizerInited Finalizer has been inited logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:53:20 +0000 UTC Normal PodChaos.chaos-mesh.org chaos-kill-label-cluster-crash Updated Successfully update finalizer of resource logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:53:20 +0000 UTC Normal PodChaos.chaos-mesh.org chaos-kill-label-cluster-crash Updated Successfully update desiredPhase of resource logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:53:20 +0000 UTC Normal PodChaos.chaos-mesh.org chaos-kill-label-cluster-crash Applied Successfully apply chaos for kuttl-test-secure-shiner/gr-self-healing-router-64c5f89d45-wpr78 logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:53:20 +0000 UTC Normal PodChaos.chaos-mesh.org chaos-kill-label-cluster-crash Applied Successfully apply chaos for kuttl-test-secure-shiner/gr-self-healing-router-64c5f89d45-6dgrd logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:53:20 +0000 UTC Normal PodChaos.chaos-mesh.org chaos-kill-label-cluster-crash Applied Successfully apply chaos for kuttl-test-secure-shiner/gr-self-healing-router-64c5f89d45-xg7kn logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:53:20 +0000 UTC Normal PodChaos.chaos-mesh.org chaos-kill-label-cluster-crash Applied Successfully apply chaos for kuttl-test-secure-shiner/gr-self-healing-mysql-2 logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:53:20 +0000 UTC Normal PodChaos.chaos-mesh.org chaos-kill-label-cluster-crash Applied Successfully apply chaos for kuttl-test-secure-shiner/gr-self-healing-mysql-1 logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:53:20 +0000 UTC Normal PodChaos.chaos-mesh.org chaos-kill-label-cluster-crash Applied Successfully apply chaos for kuttl-test-secure-shiner/gr-self-healing-mysql-0 logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:53:20 +0000 UTC Normal PodChaos.chaos-mesh.org chaos-kill-label-cluster-crash Updated Successfully update records of resource logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:53:20 +0000 UTC Normal Pod gr-self-healing-mysql-0 Scheduled Successfully assigned kuttl-test-secure-shiner/gr-self-healing-mysql-0 to gke-jen-ps-424-70568ae-7-default-pool-95e5ef23-4tf7 logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:53:20 +0000 UTC Normal Pod gr-self-healing-mysql-1.spec.containers{mysql} Killing Stopping container mysql logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:53:20 +0000 UTC Normal Pod gr-self-healing-mysql-2.spec.containers{xtrabackup} Killing Stopping container xtrabackup logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:53:20 +0000 UTC Normal Pod gr-self-healing-mysql-2.spec.containers{mysql} Killing Stopping container mysql logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:53:20 +0000 UTC Normal Pod gr-self-healing-router-64c5f89d45-586lr Scheduled Successfully assigned kuttl-test-secure-shiner/gr-self-healing-router-64c5f89d45-586lr to gke-jen-ps-424-70568ae-7-default-pool-95e5ef23-qrsp logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:53:20 +0000 UTC Normal Pod gr-self-healing-router-64c5f89d45-6dgrd.spec.containers{router} Killing Stopping container router logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:53:20 +0000 UTC Normal Pod gr-self-healing-router-64c5f89d45-8djzd Scheduled Successfully assigned kuttl-test-secure-shiner/gr-self-healing-router-64c5f89d45-8djzd to gke-jen-ps-424-70568ae-7-default-pool-95e5ef23-brgf logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:53:20 +0000 UTC Normal Pod gr-self-healing-router-64c5f89d45-qd2h5 Scheduled Successfully assigned kuttl-test-secure-shiner/gr-self-healing-router-64c5f89d45-qd2h5 to gke-jen-ps-424-70568ae-7-default-pool-95e5ef23-4tf7 logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:53:20 +0000 UTC Normal Pod gr-self-healing-router-64c5f89d45-wpr78.spec.containers{router} Killing Stopping container router logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:53:20 +0000 UTC Normal Pod gr-self-healing-router-64c5f89d45-xg7kn.spec.containers{router} Killing Stopping container router logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:53:20 +0000 UTC Normal ReplicaSet.apps gr-self-healing-router-64c5f89d45 SuccessfulCreate Created pod: gr-self-healing-router-64c5f89d45-8djzd logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:53:20 +0000 UTC Normal ReplicaSet.apps gr-self-healing-router-64c5f89d45 SuccessfulCreate Created pod: gr-self-healing-router-64c5f89d45-586lr logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:53:20 +0000 UTC Normal ReplicaSet.apps gr-self-healing-router-64c5f89d45 SuccessfulCreate Created pod: gr-self-healing-router-64c5f89d45-qd2h5 logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:53:21 +0000 UTC Normal Pod gr-self-healing-router-64c5f89d45-586lr.spec.initContainers{router-init} Pulling Pulling image "perconalab/percona-server-mysql-operator:PR-424-70568ae" logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:53:21 +0000 UTC Normal Pod gr-self-healing-router-64c5f89d45-586lr.spec.initContainers{router-init} Pulled Successfully pulled image "perconalab/percona-server-mysql-operator:PR-424-70568ae" in 394.516269ms logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:53:21 +0000 UTC Normal Pod gr-self-healing-router-64c5f89d45-586lr.spec.initContainers{router-init} Created Created container router-init logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:53:21 +0000 UTC Normal Pod gr-self-healing-router-64c5f89d45-586lr.spec.initContainers{router-init} Started Started container router-init logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:53:21 +0000 UTC Normal Pod gr-self-healing-router-64c5f89d45-8djzd.spec.initContainers{router-init} Pulling Pulling image "perconalab/percona-server-mysql-operator:PR-424-70568ae" logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:53:21 +0000 UTC Normal Pod gr-self-healing-router-64c5f89d45-8djzd.spec.initContainers{router-init} Pulled Successfully pulled image "perconalab/percona-server-mysql-operator:PR-424-70568ae" in 391.580551ms logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:53:21 +0000 UTC Normal Pod gr-self-healing-router-64c5f89d45-8djzd.spec.initContainers{router-init} Created Created container router-init logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:53:21 +0000 UTC Normal Pod gr-self-healing-router-64c5f89d45-8djzd.spec.initContainers{router-init} Started Started container router-init logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:53:21 +0000 UTC Normal Pod gr-self-healing-router-64c5f89d45-qd2h5.spec.initContainers{router-init} Pulling Pulling image "perconalab/percona-server-mysql-operator:PR-424-70568ae" logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:53:21 +0000 UTC Normal Pod gr-self-healing-router-64c5f89d45-qd2h5.spec.initContainers{router-init} Pulled Successfully pulled image "perconalab/percona-server-mysql-operator:PR-424-70568ae" in 392.267758ms logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:53:21 +0000 UTC Normal Pod gr-self-healing-router-64c5f89d45-qd2h5.spec.initContainers{router-init} Created Created container router-init logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:53:22 +0000 UTC Normal Pod gr-self-healing-mysql-0.spec.initContainers{mysql-init} Pulling Pulling image "perconalab/percona-server-mysql-operator:PR-424-70568ae" logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:53:22 +0000 UTC Normal Pod gr-self-healing-router-64c5f89d45-qd2h5.spec.initContainers{router-init} Started Started container router-init logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:53:23 +0000 UTC Normal Pod gr-self-healing-mysql-0.spec.initContainers{mysql-init} Pulled Successfully pulled image "perconalab/percona-server-mysql-operator:PR-424-70568ae" in 585.958705ms logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:53:23 +0000 UTC Normal Pod gr-self-healing-mysql-0.spec.initContainers{mysql-init} Created Created container mysql-init logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:53:23 +0000 UTC Normal Pod gr-self-healing-mysql-0.spec.initContainers{mysql-init} Started Started container mysql-init logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:53:23 +0000 UTC Normal Pod gr-self-healing-router-64c5f89d45-586lr.spec.containers{router} Pulling Pulling image "perconalab/percona-server-mysql-operator:main-router" logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:53:23 +0000 UTC Normal Pod gr-self-healing-router-64c5f89d45-586lr.spec.containers{router} Pulled Successfully pulled image "perconalab/percona-server-mysql-operator:main-router" in 373.203145ms logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:53:23 +0000 UTC Normal Pod gr-self-healing-router-64c5f89d45-586lr.spec.containers{router} Created Created container router logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:53:23 +0000 UTC Normal Pod gr-self-healing-router-64c5f89d45-586lr.spec.containers{router} Started Started container router logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:53:23 +0000 UTC Normal Pod gr-self-healing-router-64c5f89d45-8djzd.spec.containers{router} Pulling Pulling image "perconalab/percona-server-mysql-operator:main-router" logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:53:23 +0000 UTC Normal Pod gr-self-healing-router-64c5f89d45-8djzd.spec.containers{router} Pulled Successfully pulled image "perconalab/percona-server-mysql-operator:main-router" in 358.355725ms logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:53:23 +0000 UTC Normal Pod gr-self-healing-router-64c5f89d45-8djzd.spec.containers{router} Created Created container router logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:53:23 +0000 UTC Normal Pod gr-self-healing-router-64c5f89d45-8djzd.spec.containers{router} Started Started container router logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:53:23 +0000 UTC Normal Pod gr-self-healing-router-64c5f89d45-qd2h5.spec.containers{router} Pulling Pulling image "perconalab/percona-server-mysql-operator:main-router" logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:53:24 +0000 UTC Normal Pod gr-self-healing-mysql-0.spec.containers{mysql} Pulling Pulling image "perconalab/percona-server-mysql-operator:main-psmysql" logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:53:24 +0000 UTC Normal Pod gr-self-healing-router-64c5f89d45-qd2h5.spec.containers{router} Pulled Successfully pulled image "perconalab/percona-server-mysql-operator:main-router" in 555.954023ms logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:53:24 +0000 UTC Normal Pod gr-self-healing-router-64c5f89d45-qd2h5.spec.containers{router} Created Created container router logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:53:24 +0000 UTC Normal Pod gr-self-healing-router-64c5f89d45-qd2h5.spec.containers{router} Started Started container router logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:53:25 +0000 UTC Normal Pod gr-self-healing-mysql-0.spec.containers{mysql} Pulled Successfully pulled image "perconalab/percona-server-mysql-operator:main-psmysql" in 449.026341ms logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:53:25 +0000 UTC Normal Pod gr-self-healing-mysql-0.spec.containers{mysql} Created Created container mysql logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:53:25 +0000 UTC Normal Pod gr-self-healing-mysql-0.spec.containers{mysql} Started Started container mysql logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:53:25 +0000 UTC Normal Pod gr-self-healing-mysql-0.spec.containers{xtrabackup} Pulling Pulling image "perconalab/percona-server-mysql-operator:main-backup" logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:53:25 +0000 UTC Normal Pod gr-self-healing-mysql-0.spec.containers{xtrabackup} Pulled Successfully pulled image "perconalab/percona-server-mysql-operator:main-backup" in 460.907097ms logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:53:25 +0000 UTC Normal Pod gr-self-healing-mysql-0.spec.containers{xtrabackup} Created Created container xtrabackup logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:53:26 +0000 UTC Normal Pod gr-self-healing-mysql-0.spec.containers{xtrabackup} Started Started container xtrabackup logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:53:29 +0000 UTC Normal Pod gr-self-healing-router-64c5f89d45-586lr.spec.containers{router} Pulled Successfully pulled image "perconalab/percona-server-mysql-operator:main-router" in 339.104556ms logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:53:29 +0000 UTC Normal Pod gr-self-healing-router-64c5f89d45-8djzd.spec.containers{router} Pulled Successfully pulled image "perconalab/percona-server-mysql-operator:main-router" in 620.869494ms logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:53:30 +0000 UTC Normal Pod gr-self-healing-router-64c5f89d45-qd2h5.spec.containers{router} Pulled Successfully pulled image "perconalab/percona-server-mysql-operator:main-router" in 397.686173ms logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:53:35 +0000 UTC Warning Pod gr-self-healing-router-64c5f89d45-586lr.spec.containers{router} BackOff Back-off restarting failed container logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:53:35 +0000 UTC Warning Pod gr-self-healing-router-64c5f89d45-8djzd.spec.containers{router} BackOff Back-off restarting failed container logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:53:35 +0000 UTC Warning Pod gr-self-healing-router-64c5f89d45-qd2h5.spec.containers{router} BackOff Back-off restarting failed container logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:53:43 +0000 UTC Warning Pod gr-self-healing-mysql-0.spec.containers{mysql} Unhealthy Startup probe failed: logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:53:43 +0000 UTC Normal Pod gr-self-healing-mysql-0.spec.containers{mysql} Killing Container mysql failed startup probe, will be restarted logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:53:47 +0000 UTC Normal Pod gr-self-healing-mysql-0.spec.containers{mysql} Pulled Successfully pulled image "perconalab/percona-server-mysql-operator:main-psmysql" in 376.942565ms logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:53:48 +0000 UTC Normal Pod gr-self-healing-router-64c5f89d45-8djzd.spec.containers{router} Pulled Successfully pulled image "perconalab/percona-server-mysql-operator:main-router" in 392.482322ms logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:53:49 +0000 UTC Normal Pod gr-self-healing-router-64c5f89d45-586lr.spec.containers{router} Pulled Successfully pulled image "perconalab/percona-server-mysql-operator:main-router" in 395.224594ms logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:53:50 +0000 UTC Normal Pod gr-self-healing-router-64c5f89d45-qd2h5.spec.containers{router} Pulled Successfully pulled image "perconalab/percona-server-mysql-operator:main-router" in 345.310984ms logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:54:17 +0000 UTC Normal Pod gr-self-healing-mysql-1 Scheduled Successfully assigned kuttl-test-secure-shiner/gr-self-healing-mysql-1 to gke-jen-ps-424-70568ae-7-default-pool-95e5ef23-qrsp logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:54:19 +0000 UTC Normal Pod gr-self-healing-router-64c5f89d45-586lr.spec.containers{router} Pulled Successfully pulled image "perconalab/percona-server-mysql-operator:main-router" in 393.269125ms logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:54:19 +0000 UTC Normal Pod gr-self-healing-router-64c5f89d45-8djzd.spec.containers{router} Pulled Successfully pulled image "perconalab/percona-server-mysql-operator:main-router" in 553.752146ms logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:54:23 +0000 UTC Normal Pod gr-self-healing-router-64c5f89d45-qd2h5.spec.containers{router} Pulled Successfully pulled image "perconalab/percona-server-mysql-operator:main-router" in 380.33876ms logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:54:23 +0000 UTC Warning PerconaServerMySQL.ps.percona.com gr-self-healing FullClusterCrashDetected Full cluster crash detected logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:54:25 +0000 UTC Normal Pod gr-self-healing-mysql-1 SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-921dea6d-03d0-43c8-8e0d-df778a9a539d" logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:54:26 +0000 UTC Normal Pod gr-self-healing-mysql-1.spec.initContainers{mysql-init} Pulling Pulling image "perconalab/percona-server-mysql-operator:PR-424-70568ae" logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:54:26 +0000 UTC Normal Pod gr-self-healing-mysql-1.spec.initContainers{mysql-init} Pulled Successfully pulled image "perconalab/percona-server-mysql-operator:PR-424-70568ae" in 398.894497ms logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:54:26 +0000 UTC Normal Pod gr-self-healing-mysql-1.spec.initContainers{mysql-init} Created Created container mysql-init logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:54:26 +0000 UTC Normal Pod gr-self-healing-mysql-1.spec.initContainers{mysql-init} Started Started container mysql-init logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:54:28 +0000 UTC Normal Pod gr-self-healing-mysql-1.spec.containers{mysql} Pulling Pulling image "perconalab/percona-server-mysql-operator:main-psmysql" logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:54:28 +0000 UTC Normal Pod gr-self-healing-mysql-1.spec.containers{mysql} Pulled Successfully pulled image "perconalab/percona-server-mysql-operator:main-psmysql" in 368.000786ms logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:54:28 +0000 UTC Normal Pod gr-self-healing-mysql-1.spec.containers{mysql} Created Created container mysql logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:54:28 +0000 UTC Normal Pod gr-self-healing-mysql-1.spec.containers{mysql} Started Started container mysql logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:54:28 +0000 UTC Normal Pod gr-self-healing-mysql-1.spec.containers{xtrabackup} Pulling Pulling image "perconalab/percona-server-mysql-operator:main-backup" logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:54:29 +0000 UTC Normal Pod gr-self-healing-mysql-1.spec.containers{xtrabackup} Pulled Successfully pulled image "perconalab/percona-server-mysql-operator:main-backup" in 362.413192ms logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:54:29 +0000 UTC Normal Pod gr-self-healing-mysql-1.spec.containers{xtrabackup} Created Created container xtrabackup logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:54:29 +0000 UTC Normal Pod gr-self-healing-mysql-1.spec.containers{xtrabackup} Started Started container xtrabackup logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:54:47 +0000 UTC Warning Pod gr-self-healing-mysql-1.spec.containers{mysql} Unhealthy Startup probe failed: logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:54:47 +0000 UTC Normal Pod gr-self-healing-mysql-1.spec.containers{mysql} Killing Container mysql failed startup probe, will be restarted logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:55:18 +0000 UTC Normal Pod gr-self-healing-mysql-1.spec.containers{mysql} Pulled Successfully pulled image "perconalab/percona-server-mysql-operator:main-psmysql" in 373.330145ms logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:55:51 +0000 UTC Normal Pod gr-self-healing-mysql-2 Scheduled Successfully assigned kuttl-test-secure-shiner/gr-self-healing-mysql-2 to gke-jen-ps-424-70568ae-7-default-pool-95e5ef23-brgf logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:55:55 +0000 UTC Normal Pod gr-self-healing-mysql-2 SuccessfulAttachVolume AttachVolume.Attach succeeded for volume "pvc-bc9ef768-10ae-4ec9-8167-bb39c05fe755" logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:56:02 +0000 UTC Normal Pod gr-self-healing-mysql-2.spec.initContainers{mysql-init} Pulling Pulling image "perconalab/percona-server-mysql-operator:PR-424-70568ae" logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:56:03 +0000 UTC Normal Pod gr-self-healing-mysql-2.spec.initContainers{mysql-init} Pulled Successfully pulled image "perconalab/percona-server-mysql-operator:PR-424-70568ae" in 425.132106ms logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:56:03 +0000 UTC Normal Pod gr-self-healing-mysql-2.spec.initContainers{mysql-init} Created Created container mysql-init logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:56:03 +0000 UTC Normal Pod gr-self-healing-mysql-2.spec.initContainers{mysql-init} Started Started container mysql-init logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:56:04 +0000 UTC Normal Pod gr-self-healing-mysql-2.spec.containers{mysql} Pulling Pulling image "perconalab/percona-server-mysql-operator:main-psmysql" logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:56:04 +0000 UTC Normal Pod gr-self-healing-mysql-2.spec.containers{mysql} Pulled Successfully pulled image "perconalab/percona-server-mysql-operator:main-psmysql" in 375.982495ms logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:56:04 +0000 UTC Normal Pod gr-self-healing-mysql-2.spec.containers{mysql} Created Created container mysql logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:56:05 +0000 UTC Normal Pod gr-self-healing-mysql-2.spec.containers{mysql} Started Started container mysql logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:56:05 +0000 UTC Normal Pod gr-self-healing-mysql-2.spec.containers{xtrabackup} Pulling Pulling image "perconalab/percona-server-mysql-operator:main-backup" logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:56:05 +0000 UTC Normal Pod gr-self-healing-mysql-2.spec.containers{xtrabackup} Pulled Successfully pulled image "perconalab/percona-server-mysql-operator:main-backup" in 381.910967ms logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:56:05 +0000 UTC Normal Pod gr-self-healing-mysql-2.spec.containers{xtrabackup} Created Created container xtrabackup logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:56:05 +0000 UTC Normal Pod gr-self-healing-mysql-2.spec.containers{xtrabackup} Started Started container xtrabackup logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:56:24 +0000 UTC Warning Pod gr-self-healing-mysql-2.spec.containers{mysql} Unhealthy Startup probe failed: logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:56:24 +0000 UTC Normal Pod gr-self-healing-mysql-2.spec.containers{mysql} Killing Container mysql failed startup probe, will be restarted logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 14:56:55 +0000 UTC Normal Pod gr-self-healing-mysql-2.spec.containers{mysql} Pulled Successfully pulled image "perconalab/percona-server-mysql-operator:main-psmysql" in 349.642452ms logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 15:00:31 +0000 UTC Normal NetworkChaos.chaos-mesh.org chaos-pod-network-loss-primary FinalizerInited Finalizer has been removed logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 15:00:31 +0000 UTC Normal NetworkChaos.chaos-mesh.org chaos-pod-network-loss-primary Updated Successfully update finalizer of resource logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 15:00:34 +0000 UTC Normal PodChaos.chaos-mesh.org chaos-kill-label-cluster-crash Deleted Experiment has been deleted logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 15:00:34 +0000 UTC Normal PodChaos.chaos-mesh.org chaos-kill-label-cluster-crash Updated Successfully update desiredPhase of resource logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 15:00:34 +0000 UTC Normal PodChaos.chaos-mesh.org chaos-kill-label-cluster-crash Recovered Successfully recover chaos for kuttl-test-secure-shiner/gr-self-healing-router-64c5f89d45-wpr78 logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 15:00:34 +0000 UTC Normal PodChaos.chaos-mesh.org chaos-kill-label-cluster-crash Recovered Successfully recover chaos for kuttl-test-secure-shiner/gr-self-healing-router-64c5f89d45-6dgrd logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 15:00:34 +0000 UTC Normal PodChaos.chaos-mesh.org chaos-kill-label-cluster-crash Recovered Successfully recover chaos for kuttl-test-secure-shiner/gr-self-healing-router-64c5f89d45-xg7kn logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 15:00:34 +0000 UTC Normal PodChaos.chaos-mesh.org chaos-kill-label-cluster-crash Recovered Successfully recover chaos for kuttl-test-secure-shiner/gr-self-healing-mysql-2 logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 15:00:34 +0000 UTC Normal PodChaos.chaos-mesh.org chaos-kill-label-cluster-crash Recovered Successfully recover chaos for kuttl-test-secure-shiner/gr-self-healing-mysql-1 logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 15:00:34 +0000 UTC Normal PodChaos.chaos-mesh.org chaos-kill-label-cluster-crash Recovered Successfully recover chaos for kuttl-test-secure-shiner/gr-self-healing-mysql-0 logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 15:00:34 +0000 UTC Normal PodChaos.chaos-mesh.org chaos-kill-label-cluster-crash Updated Successfully update records of resource logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 15:00:34 +0000 UTC Normal PodChaos.chaos-mesh.org chaos-kill-label-cluster-crash FinalizerInited Finalizer has been removed logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 15:00:34 +0000 UTC Normal PodChaos.chaos-mesh.org chaos-kill-label-cluster-crash Updated Successfully update finalizer of resource logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 15:00:34 +0000 UTC Normal PodChaos.chaos-mesh.org chaos-pod-failure-primary FinalizerInited Finalizer has been removed logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 15:00:34 +0000 UTC Normal PodChaos.chaos-mesh.org chaos-pod-failure-primary Updated Successfully update finalizer of resource logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 15:00:34 +0000 UTC Normal PodChaos.chaos-mesh.org chaos-pod-kill-primary Deleted Experiment has been deleted logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 15:00:34 +0000 UTC Normal PodChaos.chaos-mesh.org chaos-pod-kill-primary Updated Successfully update desiredPhase of resource logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 15:00:34 +0000 UTC Normal PodChaos.chaos-mesh.org chaos-pod-kill-primary Recovered Successfully recover chaos for kuttl-test-secure-shiner/gr-self-healing-mysql-0 logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 15:00:34 +0000 UTC Normal PodChaos.chaos-mesh.org chaos-pod-kill-primary Updated Successfully update records of resource logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 15:00:34 +0000 UTC Normal PodChaos.chaos-mesh.org chaos-pod-kill-primary FinalizerInited Finalizer has been removed logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 15:00:34 +0000 UTC Normal PodChaos.chaos-mesh.org chaos-pod-kill-primary Updated Successfully update finalizer of resource logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 15:00:45 +0000 UTC Normal Pod chaos-controller-manager-79b5ff6dcc-24k4z.spec.containers{chaos-mesh} Killing Stopping container chaos-mesh logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 15:00:45 +0000 UTC Normal Pod chaos-controller-manager-79b5ff6dcc-4m5tn.spec.containers{chaos-mesh} Killing Stopping container chaos-mesh logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 15:00:45 +0000 UTC Normal Pod chaos-controller-manager-79b5ff6dcc-glsgl.spec.containers{chaos-mesh} Killing Stopping container chaos-mesh logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 15:00:45 +0000 UTC Normal Pod chaos-daemon-6cjrn.spec.containers{chaos-daemon} Killing Stopping container chaos-daemon logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 15:00:45 +0000 UTC Normal Pod chaos-daemon-x294b.spec.containers{chaos-daemon} Killing Stopping container chaos-daemon logger.go:42: 15:01:10 | gr-self-healing | 2023-08-17 15:00:45 +0000 UTC Normal Pod chaos-daemon-xpgrm.spec.containers{chaos-daemon} Killing Stopping container chaos-daemon logger.go:42: 15:01:10 | gr-self-healing | Deleting namespace: kuttl-test-secure-shiner === CONT kuttl harness.go:405: run tests finished harness.go:513: cleaning up harness.go:570: removing temp folder: "" --- PASS: kuttl (1297.60s) --- PASS: kuttl/harness (0.00s) --- PASS: kuttl/harness/gr-self-healing (1294.22s) PASS