=== RUN kuttl harness.go:459: starting setup harness.go:254: running tests using configured kubeconfig. harness.go:277: Successful connection to cluster at: https://34.28.83.67 harness.go:362: running tests harness.go:74: going to run test suite with timeout of 180 seconds for each step harness.go:374: testsuite: e2e-tests/tests has 43 tests === RUN kuttl/harness === RUN kuttl/harness/gr-self-healing === PAUSE kuttl/harness/gr-self-healing === CONT kuttl/harness/gr-self-healing logger.go:42: 04:09:42 | gr-self-healing | Creating namespace: kuttl-test-concrete-skylark logger.go:42: 04:09:43 | gr-self-healing/0-deploy-operator | starting test step 0-deploy-operator logger.go:42: 04:09:43 | gr-self-healing/0-deploy-operator | running command: [sh -c set -o errexit set -o xtrace source ../../functions init_temp_dir # do this only in the first TestStep deploy_operator deploy_non_tls_cluster_secrets deploy_tls_cluster_secrets deploy_client] logger.go:42: 04:09:43 | gr-self-healing/0-deploy-operator | + source ../../functions logger.go:42: 04:09:43 | gr-self-healing/0-deploy-operator | +++ realpath ../../.. logger.go:42: 04:09:43 | gr-self-healing/0-deploy-operator | ++ ROOT_REPO=/mnt/jenkins/workspace/cloud-ps-operator_PR-1108 logger.go:42: 04:09:43 | gr-self-healing/0-deploy-operator | ++++ pwd logger.go:42: 04:09:43 | gr-self-healing/0-deploy-operator | +++ basename /mnt/jenkins/workspace/cloud-ps-operator_PR-1108/e2e-tests/tests/gr-self-healing logger.go:42: 04:09:43 | gr-self-healing/0-deploy-operator | ++ test_name=gr-self-healing logger.go:42: 04:09:43 | gr-self-healing/0-deploy-operator | ++ source /mnt/jenkins/workspace/cloud-ps-operator_PR-1108/e2e-tests/vars.sh logger.go:42: 04:09:43 | gr-self-healing/0-deploy-operator | +++ export ROOT_REPO=/mnt/jenkins/workspace/cloud-ps-operator_PR-1108 logger.go:42: 04:09:43 | gr-self-healing/0-deploy-operator | +++ ROOT_REPO=/mnt/jenkins/workspace/cloud-ps-operator_PR-1108 logger.go:42: 04:09:43 | gr-self-healing/0-deploy-operator | +++ export DEPLOY_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-1108/deploy logger.go:42: 04:09:43 | gr-self-healing/0-deploy-operator | +++ DEPLOY_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-1108/deploy logger.go:42: 04:09:43 | gr-self-healing/0-deploy-operator | +++ export TESTS_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-1108/e2e-tests logger.go:42: 04:09:43 | gr-self-healing/0-deploy-operator | +++ TESTS_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-1108/e2e-tests logger.go:42: 04:09:43 | gr-self-healing/0-deploy-operator | +++ export TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-1108/e2e-tests/conf logger.go:42: 04:09:43 | gr-self-healing/0-deploy-operator | +++ TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-1108/e2e-tests/conf logger.go:42: 04:09:43 | gr-self-healing/0-deploy-operator | +++ export TEMP_DIR=/tmp/kuttl/ps/gr-self-healing logger.go:42: 04:09:43 | gr-self-healing/0-deploy-operator | +++ TEMP_DIR=/tmp/kuttl/ps/gr-self-healing logger.go:42: 04:09:43 | gr-self-healing/0-deploy-operator | ++++ git rev-parse --abbrev-ref HEAD logger.go:42: 04:09:43 | gr-self-healing/0-deploy-operator | +++ export GIT_BRANCH=PR-1108 logger.go:42: 04:09:43 | gr-self-healing/0-deploy-operator | +++ GIT_BRANCH=PR-1108 logger.go:42: 04:09:43 | gr-self-healing/0-deploy-operator | +++ export VERSION=PR-1108-c411fa10 logger.go:42: 04:09:43 | gr-self-healing/0-deploy-operator | +++ VERSION=PR-1108-c411fa10 logger.go:42: 04:09:43 | gr-self-healing/0-deploy-operator | +++ export IMAGE=perconalab/percona-server-mysql-operator:PR-1108-c411fa10 logger.go:42: 04:09:43 | gr-self-healing/0-deploy-operator | +++ IMAGE=perconalab/percona-server-mysql-operator:PR-1108-c411fa10 logger.go:42: 04:09:43 | gr-self-healing/0-deploy-operator | +++ [[ -z '' ]] logger.go:42: 04:09:43 | gr-self-healing/0-deploy-operator | +++ [[ -n '' ]] logger.go:42: 04:09:43 | gr-self-healing/0-deploy-operator | +++ export MYSQL_VERSION=8.4 logger.go:42: 04:09:43 | gr-self-healing/0-deploy-operator | +++ MYSQL_VERSION=8.4 logger.go:42: 04:09:43 | gr-self-healing/0-deploy-operator | +++ export IMAGE_MYSQL=perconalab/percona-server-mysql-operator:main-psmysql8.4 logger.go:42: 04:09:43 | gr-self-healing/0-deploy-operator | +++ IMAGE_MYSQL=perconalab/percona-server-mysql-operator:main-psmysql8.4 logger.go:42: 04:09:43 | gr-self-healing/0-deploy-operator | +++ export IMAGE_BACKUP=perconalab/percona-server-mysql-operator:main-backup8.4 logger.go:42: 04:09:43 | gr-self-healing/0-deploy-operator | +++ IMAGE_BACKUP=perconalab/percona-server-mysql-operator:main-backup8.4 logger.go:42: 04:09:43 | gr-self-healing/0-deploy-operator | +++ export IMAGE_ORCHESTRATOR=perconalab/percona-server-mysql-operator:main-orchestrator logger.go:42: 04:09:43 | gr-self-healing/0-deploy-operator | +++ IMAGE_ORCHESTRATOR=perconalab/percona-server-mysql-operator:main-orchestrator logger.go:42: 04:09:43 | gr-self-healing/0-deploy-operator | +++ export IMAGE_ROUTER=perconalab/percona-server-mysql-operator:main-router8.4 logger.go:42: 04:09:43 | gr-self-healing/0-deploy-operator | +++ IMAGE_ROUTER=perconalab/percona-server-mysql-operator:main-router8.4 logger.go:42: 04:09:43 | gr-self-healing/0-deploy-operator | +++ export IMAGE_TOOLKIT=perconalab/percona-server-mysql-operator:main-toolkit logger.go:42: 04:09:43 | gr-self-healing/0-deploy-operator | +++ IMAGE_TOOLKIT=perconalab/percona-server-mysql-operator:main-toolkit logger.go:42: 04:09:43 | gr-self-healing/0-deploy-operator | +++ export IMAGE_HAPROXY=perconalab/percona-server-mysql-operator:main-haproxy logger.go:42: 04:09:43 | gr-self-healing/0-deploy-operator | +++ IMAGE_HAPROXY=perconalab/percona-server-mysql-operator:main-haproxy logger.go:42: 04:09:43 | gr-self-healing/0-deploy-operator | +++ export PMM_SERVER_VERSION=1.4.3 logger.go:42: 04:09:43 | gr-self-healing/0-deploy-operator | +++ PMM_SERVER_VERSION=1.4.3 logger.go:42: 04:09:43 | gr-self-healing/0-deploy-operator | +++ export IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 04:09:43 | gr-self-healing/0-deploy-operator | +++ IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 04:09:43 | gr-self-healing/0-deploy-operator | +++ export IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 04:09:43 | gr-self-healing/0-deploy-operator | +++ IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 04:09:43 | gr-self-healing/0-deploy-operator | +++ export CERT_MANAGER_VER=1.18.2 logger.go:42: 04:09:43 | gr-self-healing/0-deploy-operator | +++ CERT_MANAGER_VER=1.18.2 logger.go:42: 04:09:43 | gr-self-healing/0-deploy-operator | +++ export MINIO_VER=5.4.0 logger.go:42: 04:09:43 | gr-self-healing/0-deploy-operator | +++ MINIO_VER=5.4.0 logger.go:42: 04:09:43 | gr-self-healing/0-deploy-operator | +++ export CHAOS_MESH_VER=2.7.2 logger.go:42: 04:09:43 | gr-self-healing/0-deploy-operator | +++ CHAOS_MESH_VER=2.7.2 logger.go:42: 04:09:43 | gr-self-healing/0-deploy-operator | +++ export VAULT_VER=0.16.1 logger.go:42: 04:09:43 | gr-self-healing/0-deploy-operator | +++ VAULT_VER=0.16.1 logger.go:42: 04:09:43 | gr-self-healing/0-deploy-operator | ++++ which gdate logger.go:42: 04:09:43 | gr-self-healing/0-deploy-operator | which: no gdate in (/mnt/jenkins/workspace/cloud-ps-operator_PR-1108/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 04:09:43 | gr-self-healing/0-deploy-operator | ++++ which date logger.go:42: 04:09:43 | gr-self-healing/0-deploy-operator | +++ date=/usr/sbin/date logger.go:42: 04:09:43 | gr-self-healing/0-deploy-operator | +++ oc get projects logger.go:42: 04:09:43 | gr-self-healing/0-deploy-operator | +++ : logger.go:42: 04:09:43 | gr-self-healing/0-deploy-operator | +++ kubectl get nodes logger.go:42: 04:09:43 | gr-self-healing/0-deploy-operator | +++ grep '^minikube' logger.go:42: 04:09:43 | gr-self-healing/0-deploy-operator | +++ which gsed logger.go:42: 04:09:43 | gr-self-healing/0-deploy-operator | which: no gsed in (/mnt/jenkins/workspace/cloud-ps-operator_PR-1108/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 04:09:43 | gr-self-healing/0-deploy-operator | +++ which sed logger.go:42: 04:09:43 | gr-self-healing/0-deploy-operator | ++ sed=/usr/sbin/sed logger.go:42: 04:09:43 | gr-self-healing/0-deploy-operator | ++ oc get projects logger.go:42: 04:09:43 | gr-self-healing/0-deploy-operator | +++ kubectl version -o json logger.go:42: 04:09:43 | gr-self-healing/0-deploy-operator | +++ jq -r .serverVersion.gitVersion logger.go:42: 04:09:43 | gr-self-healing/0-deploy-operator | +++ grep '\-eks\-' logger.go:42: 04:09:43 | gr-self-healing/0-deploy-operator | grep: warning: stray \ before - logger.go:42: 04:09:44 | gr-self-healing/0-deploy-operator | Warning: version difference between client (1.34) and server (1.31) exceeds the supported minor version skew of +/-1 logger.go:42: 04:09:44 | gr-self-healing/0-deploy-operator | ++ '[' ']' logger.go:42: 04:09:44 | gr-self-healing/0-deploy-operator | ++ EKS=0 logger.go:42: 04:09:44 | gr-self-healing/0-deploy-operator | + init_temp_dir logger.go:42: 04:09:44 | gr-self-healing/0-deploy-operator | + rm -rf /tmp/kuttl/ps/gr-self-healing logger.go:42: 04:09:44 | gr-self-healing/0-deploy-operator | + mkdir -p /tmp/kuttl/ps/gr-self-healing logger.go:42: 04:09:44 | gr-self-healing/0-deploy-operator | + deploy_operator logger.go:42: 04:09:44 | gr-self-healing/0-deploy-operator | + destroy_operator logger.go:42: 04:09:44 | gr-self-healing/0-deploy-operator | + kubectl -n ps-operator delete deployment percona-server-mysql-operator --force --grace-period=0 logger.go:42: 04:09:44 | gr-self-healing/0-deploy-operator | Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. logger.go:42: 04:09:44 | gr-self-healing/0-deploy-operator | deployment.apps "percona-server-mysql-operator" force deleted from ps-operator namespace logger.go:42: 04:09:45 | gr-self-healing/0-deploy-operator | + [[ -n ps-operator ]] logger.go:42: 04:09:45 | gr-self-healing/0-deploy-operator | + kubectl delete namespace ps-operator --force --grace-period=0 logger.go:42: 04:09:45 | gr-self-healing/0-deploy-operator | Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. logger.go:42: 04:09:45 | gr-self-healing/0-deploy-operator | namespace "ps-operator" force deleted logger.go:42: 04:09:59 | gr-self-healing/0-deploy-operator | + [[ -n ps-operator ]] logger.go:42: 04:09:59 | gr-self-healing/0-deploy-operator | + create_namespace ps-operator logger.go:42: 04:09:59 | gr-self-healing/0-deploy-operator | + local namespace=ps-operator logger.go:42: 04:09:59 | gr-self-healing/0-deploy-operator | + [[ -n '' ]] logger.go:42: 04:09:59 | gr-self-healing/0-deploy-operator | + kubectl delete namespace ps-operator --ignore-not-found logger.go:42: 04:10:00 | gr-self-healing/0-deploy-operator | + kubectl wait --for=delete namespace ps-operator logger.go:42: 04:10:00 | gr-self-healing/0-deploy-operator | + kubectl create namespace ps-operator logger.go:42: 04:10:01 | gr-self-healing/0-deploy-operator | namespace/ps-operator created logger.go:42: 04:10:01 | gr-self-healing/0-deploy-operator | + kubectl -n ps-operator apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-ps-operator_PR-1108/deploy/crd.yaml logger.go:42: 04:10:02 | gr-self-healing/0-deploy-operator | customresourcedefinition.apiextensions.k8s.io/perconaservermysqlbackups.ps.percona.com serverside-applied logger.go:42: 04:10:02 | gr-self-healing/0-deploy-operator | customresourcedefinition.apiextensions.k8s.io/perconaservermysqlrestores.ps.percona.com serverside-applied logger.go:42: 04:10:04 | gr-self-healing/0-deploy-operator | customresourcedefinition.apiextensions.k8s.io/perconaservermysqls.ps.percona.com serverside-applied logger.go:42: 04:10:04 | gr-self-healing/0-deploy-operator | + '[' -n ps-operator ']' logger.go:42: 04:10:04 | gr-self-healing/0-deploy-operator | + kubectl -n ps-operator apply -f /mnt/jenkins/workspace/cloud-ps-operator_PR-1108/deploy/cw-rbac.yaml logger.go:42: 04:10:05 | gr-self-healing/0-deploy-operator | serviceaccount/percona-server-mysql-operator created logger.go:42: 04:10:06 | gr-self-healing/0-deploy-operator | role.rbac.authorization.k8s.io/percona-server-mysql-operator-leaderelection created logger.go:42: 04:10:06 | gr-self-healing/0-deploy-operator | clusterrole.rbac.authorization.k8s.io/percona-server-mysql-operator unchanged logger.go:42: 04:10:06 | gr-self-healing/0-deploy-operator | rolebinding.rbac.authorization.k8s.io/percona-server-mysql-operator-leaderelection created logger.go:42: 04:10:07 | gr-self-healing/0-deploy-operator | clusterrolebinding.rbac.authorization.k8s.io/percona-server-mysql-operator unchanged logger.go:42: 04:10:07 | gr-self-healing/0-deploy-operator | + yq eval '(select(documentIndex==1).spec.template.spec.containers[] | select(.name=="manager").env[] | select(.name=="DISABLE_TELEMETRY").value) = "true"' logger.go:42: 04:10:07 | gr-self-healing/0-deploy-operator | + yq eval '(select(documentIndex==1).spec.template.spec.containers[] | select(.name=="manager").env[] | select(.name=="LOG_LEVEL").value) = "DEBUG"' logger.go:42: 04:10:07 | gr-self-healing/0-deploy-operator | + kubectl -n ps-operator apply -f - logger.go:42: 04:10:07 | gr-self-healing/0-deploy-operator | ++ printf 'select(documentIndex==1).spec.template.spec.containers[0].image="%s"' perconalab/percona-server-mysql-operator:PR-1108-c411fa10 logger.go:42: 04:10:07 | gr-self-healing/0-deploy-operator | + yq eval 'select(documentIndex==1).spec.template.spec.containers[0].image="perconalab/percona-server-mysql-operator:PR-1108-c411fa10"' /mnt/jenkins/workspace/cloud-ps-operator_PR-1108/deploy/cw-operator.yaml logger.go:42: 04:10:08 | gr-self-healing/0-deploy-operator | configmap/percona-server-mysql-operator-config created logger.go:42: 04:10:08 | gr-self-healing/0-deploy-operator | deployment.apps/percona-server-mysql-operator created logger.go:42: 04:10:08 | gr-self-healing/0-deploy-operator | + deploy_non_tls_cluster_secrets logger.go:42: 04:10:08 | gr-self-healing/0-deploy-operator | + kubectl -n kuttl-test-concrete-skylark apply -f /mnt/jenkins/workspace/cloud-ps-operator_PR-1108/e2e-tests/conf/secrets.yaml logger.go:42: 04:10:09 | gr-self-healing/0-deploy-operator | secret/test-secrets created logger.go:42: 04:10:09 | gr-self-healing/0-deploy-operator | + deploy_tls_cluster_secrets logger.go:42: 04:10:09 | gr-self-healing/0-deploy-operator | + kubectl -n kuttl-test-concrete-skylark apply -f /mnt/jenkins/workspace/cloud-ps-operator_PR-1108/e2e-tests/conf/ssl-secret.yaml logger.go:42: 04:10:10 | gr-self-healing/0-deploy-operator | secret/test-ssl created logger.go:42: 04:10:10 | gr-self-healing/0-deploy-operator | + deploy_client logger.go:42: 04:10:10 | gr-self-healing/0-deploy-operator | + kubectl -n kuttl-test-concrete-skylark apply -f - logger.go:42: 04:10:10 | gr-self-healing/0-deploy-operator | ++ printf '.spec.containers[0].image="%s"' perconalab/percona-server-mysql-operator:main-psmysql8.4 logger.go:42: 04:10:10 | gr-self-healing/0-deploy-operator | + yq eval '.spec.containers[0].image="perconalab/percona-server-mysql-operator:main-psmysql8.4"' /mnt/jenkins/workspace/cloud-ps-operator_PR-1108/e2e-tests/conf/client.yaml logger.go:42: 04:10:12 | gr-self-healing/0-deploy-operator | pod/mysql-client created logger.go:42: 04:10:12 | gr-self-healing/0-deploy-operator | running command: [sh -c kubectl assert exist-enhanced deployment percona-server-mysql-operator -n ${OPERATOR_NS:-$NAMESPACE} --field-selector status.readyReplicas=1] logger.go:42: 04:10:12 | gr-self-healing/0-deploy-operator | ASSERT deployment percona-server-mysql-operator matching field criteria 'status.readyReplicas=1' should exist. logger.go:42: 04:10:13 | gr-self-healing/0-deploy-operator | ASSERT FAIL Resource(s) not found. logger.go:42: 04:10:14 | gr-self-healing/0-deploy-operator | running command: [sh -c kubectl assert exist-enhanced deployment percona-server-mysql-operator -n ${OPERATOR_NS:-$NAMESPACE} --field-selector status.readyReplicas=1] logger.go:42: 04:10:14 | gr-self-healing/0-deploy-operator | ASSERT deployment percona-server-mysql-operator matching field criteria 'status.readyReplicas=1' should exist. logger.go:42: 04:10:15 | gr-self-healing/0-deploy-operator | ASSERT FAIL Resource(s) not found. logger.go:42: 04:10:16 | gr-self-healing/0-deploy-operator | running command: [sh -c kubectl assert exist-enhanced deployment percona-server-mysql-operator -n ${OPERATOR_NS:-$NAMESPACE} --field-selector status.readyReplicas=1] logger.go:42: 04:10:16 | gr-self-healing/0-deploy-operator | ASSERT deployment percona-server-mysql-operator matching field criteria 'status.readyReplicas=1' should exist. logger.go:42: 04:10:17 | gr-self-healing/0-deploy-operator | ASSERT FAIL Resource(s) not found. logger.go:42: 04:10:18 | gr-self-healing/0-deploy-operator | running command: [sh -c kubectl assert exist-enhanced deployment percona-server-mysql-operator -n ${OPERATOR_NS:-$NAMESPACE} --field-selector status.readyReplicas=1] logger.go:42: 04:10:18 | gr-self-healing/0-deploy-operator | ASSERT deployment percona-server-mysql-operator matching field criteria 'status.readyReplicas=1' should exist. logger.go:42: 04:10:18 | gr-self-healing/0-deploy-operator | ASSERT FAIL Resource(s) not found. logger.go:42: 04:10:20 | gr-self-healing/0-deploy-operator | running command: [sh -c kubectl assert exist-enhanced deployment percona-server-mysql-operator -n ${OPERATOR_NS:-$NAMESPACE} --field-selector status.readyReplicas=1] logger.go:42: 04:10:20 | gr-self-healing/0-deploy-operator | ASSERT deployment percona-server-mysql-operator matching field criteria 'status.readyReplicas=1' should exist. logger.go:42: 04:10:20 | gr-self-healing/0-deploy-operator | INFO Found 1 resource(s). logger.go:42: 04:10:20 | gr-self-healing/0-deploy-operator | NAME NAMESPACE COL0 logger.go:42: 04:10:20 | gr-self-healing/0-deploy-operator | percona-server-mysql-operator ps-operator 1 logger.go:42: 04:10:20 | gr-self-healing/0-deploy-operator | ASSERT PASS logger.go:42: 04:10:20 | gr-self-healing/0-deploy-operator | test step completed 0-deploy-operator logger.go:42: 04:10:20 | gr-self-healing/1-deploy-chaos-mesh | starting test step 1-deploy-chaos-mesh logger.go:42: 04:10:20 | gr-self-healing/1-deploy-chaos-mesh | running command: [sh -c set -o errexit set -o xtrace source ../../functions deploy_chaos_mesh] logger.go:42: 04:10:20 | gr-self-healing/1-deploy-chaos-mesh | + source ../../functions logger.go:42: 04:10:20 | gr-self-healing/1-deploy-chaos-mesh | +++ realpath ../../.. logger.go:42: 04:10:20 | gr-self-healing/1-deploy-chaos-mesh | ++ ROOT_REPO=/mnt/jenkins/workspace/cloud-ps-operator_PR-1108 logger.go:42: 04:10:20 | gr-self-healing/1-deploy-chaos-mesh | ++++ pwd logger.go:42: 04:10:20 | gr-self-healing/1-deploy-chaos-mesh | +++ basename /mnt/jenkins/workspace/cloud-ps-operator_PR-1108/e2e-tests/tests/gr-self-healing logger.go:42: 04:10:20 | gr-self-healing/1-deploy-chaos-mesh | ++ test_name=gr-self-healing logger.go:42: 04:10:20 | gr-self-healing/1-deploy-chaos-mesh | ++ source /mnt/jenkins/workspace/cloud-ps-operator_PR-1108/e2e-tests/vars.sh logger.go:42: 04:10:20 | gr-self-healing/1-deploy-chaos-mesh | +++ export ROOT_REPO=/mnt/jenkins/workspace/cloud-ps-operator_PR-1108 logger.go:42: 04:10:20 | gr-self-healing/1-deploy-chaos-mesh | +++ ROOT_REPO=/mnt/jenkins/workspace/cloud-ps-operator_PR-1108 logger.go:42: 04:10:20 | gr-self-healing/1-deploy-chaos-mesh | +++ export DEPLOY_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-1108/deploy logger.go:42: 04:10:20 | gr-self-healing/1-deploy-chaos-mesh | +++ DEPLOY_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-1108/deploy logger.go:42: 04:10:20 | gr-self-healing/1-deploy-chaos-mesh | +++ export TESTS_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-1108/e2e-tests logger.go:42: 04:10:20 | gr-self-healing/1-deploy-chaos-mesh | +++ TESTS_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-1108/e2e-tests logger.go:42: 04:10:20 | gr-self-healing/1-deploy-chaos-mesh | +++ export TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-1108/e2e-tests/conf logger.go:42: 04:10:20 | gr-self-healing/1-deploy-chaos-mesh | +++ TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-1108/e2e-tests/conf logger.go:42: 04:10:20 | gr-self-healing/1-deploy-chaos-mesh | +++ export TEMP_DIR=/tmp/kuttl/ps/gr-self-healing logger.go:42: 04:10:20 | gr-self-healing/1-deploy-chaos-mesh | +++ TEMP_DIR=/tmp/kuttl/ps/gr-self-healing logger.go:42: 04:10:20 | gr-self-healing/1-deploy-chaos-mesh | ++++ git rev-parse --abbrev-ref HEAD logger.go:42: 04:10:20 | gr-self-healing/1-deploy-chaos-mesh | +++ export GIT_BRANCH=PR-1108 logger.go:42: 04:10:20 | gr-self-healing/1-deploy-chaos-mesh | +++ GIT_BRANCH=PR-1108 logger.go:42: 04:10:20 | gr-self-healing/1-deploy-chaos-mesh | +++ export VERSION=PR-1108-c411fa10 logger.go:42: 04:10:20 | gr-self-healing/1-deploy-chaos-mesh | +++ VERSION=PR-1108-c411fa10 logger.go:42: 04:10:20 | gr-self-healing/1-deploy-chaos-mesh | +++ export IMAGE=perconalab/percona-server-mysql-operator:PR-1108-c411fa10 logger.go:42: 04:10:20 | gr-self-healing/1-deploy-chaos-mesh | +++ IMAGE=perconalab/percona-server-mysql-operator:PR-1108-c411fa10 logger.go:42: 04:10:20 | gr-self-healing/1-deploy-chaos-mesh | +++ [[ -z '' ]] logger.go:42: 04:10:20 | gr-self-healing/1-deploy-chaos-mesh | +++ [[ -n '' ]] logger.go:42: 04:10:20 | gr-self-healing/1-deploy-chaos-mesh | +++ export MYSQL_VERSION=8.4 logger.go:42: 04:10:20 | gr-self-healing/1-deploy-chaos-mesh | +++ MYSQL_VERSION=8.4 logger.go:42: 04:10:20 | gr-self-healing/1-deploy-chaos-mesh | +++ export IMAGE_MYSQL=perconalab/percona-server-mysql-operator:main-psmysql8.4 logger.go:42: 04:10:20 | gr-self-healing/1-deploy-chaos-mesh | +++ IMAGE_MYSQL=perconalab/percona-server-mysql-operator:main-psmysql8.4 logger.go:42: 04:10:20 | gr-self-healing/1-deploy-chaos-mesh | +++ export IMAGE_BACKUP=perconalab/percona-server-mysql-operator:main-backup8.4 logger.go:42: 04:10:20 | gr-self-healing/1-deploy-chaos-mesh | +++ IMAGE_BACKUP=perconalab/percona-server-mysql-operator:main-backup8.4 logger.go:42: 04:10:20 | gr-self-healing/1-deploy-chaos-mesh | +++ export IMAGE_ORCHESTRATOR=perconalab/percona-server-mysql-operator:main-orchestrator logger.go:42: 04:10:20 | gr-self-healing/1-deploy-chaos-mesh | +++ IMAGE_ORCHESTRATOR=perconalab/percona-server-mysql-operator:main-orchestrator logger.go:42: 04:10:20 | gr-self-healing/1-deploy-chaos-mesh | +++ export IMAGE_ROUTER=perconalab/percona-server-mysql-operator:main-router8.4 logger.go:42: 04:10:20 | gr-self-healing/1-deploy-chaos-mesh | +++ IMAGE_ROUTER=perconalab/percona-server-mysql-operator:main-router8.4 logger.go:42: 04:10:20 | gr-self-healing/1-deploy-chaos-mesh | +++ export IMAGE_TOOLKIT=perconalab/percona-server-mysql-operator:main-toolkit logger.go:42: 04:10:20 | gr-self-healing/1-deploy-chaos-mesh | +++ IMAGE_TOOLKIT=perconalab/percona-server-mysql-operator:main-toolkit logger.go:42: 04:10:20 | gr-self-healing/1-deploy-chaos-mesh | +++ export IMAGE_HAPROXY=perconalab/percona-server-mysql-operator:main-haproxy logger.go:42: 04:10:20 | gr-self-healing/1-deploy-chaos-mesh | +++ IMAGE_HAPROXY=perconalab/percona-server-mysql-operator:main-haproxy logger.go:42: 04:10:20 | gr-self-healing/1-deploy-chaos-mesh | +++ export PMM_SERVER_VERSION=1.4.3 logger.go:42: 04:10:20 | gr-self-healing/1-deploy-chaos-mesh | +++ PMM_SERVER_VERSION=1.4.3 logger.go:42: 04:10:20 | gr-self-healing/1-deploy-chaos-mesh | +++ export IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 04:10:20 | gr-self-healing/1-deploy-chaos-mesh | +++ IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 04:10:20 | gr-self-healing/1-deploy-chaos-mesh | +++ export IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 04:10:20 | gr-self-healing/1-deploy-chaos-mesh | +++ IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 04:10:20 | gr-self-healing/1-deploy-chaos-mesh | +++ export CERT_MANAGER_VER=1.18.2 logger.go:42: 04:10:20 | gr-self-healing/1-deploy-chaos-mesh | +++ CERT_MANAGER_VER=1.18.2 logger.go:42: 04:10:20 | gr-self-healing/1-deploy-chaos-mesh | +++ export MINIO_VER=5.4.0 logger.go:42: 04:10:20 | gr-self-healing/1-deploy-chaos-mesh | +++ MINIO_VER=5.4.0 logger.go:42: 04:10:20 | gr-self-healing/1-deploy-chaos-mesh | +++ export CHAOS_MESH_VER=2.7.2 logger.go:42: 04:10:20 | gr-self-healing/1-deploy-chaos-mesh | +++ CHAOS_MESH_VER=2.7.2 logger.go:42: 04:10:20 | gr-self-healing/1-deploy-chaos-mesh | +++ export VAULT_VER=0.16.1 logger.go:42: 04:10:20 | gr-self-healing/1-deploy-chaos-mesh | +++ VAULT_VER=0.16.1 logger.go:42: 04:10:20 | gr-self-healing/1-deploy-chaos-mesh | ++++ which gdate logger.go:42: 04:10:20 | gr-self-healing/1-deploy-chaos-mesh | which: no gdate in (/mnt/jenkins/workspace/cloud-ps-operator_PR-1108/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 04:10:20 | gr-self-healing/1-deploy-chaos-mesh | ++++ which date logger.go:42: 04:10:20 | gr-self-healing/1-deploy-chaos-mesh | +++ date=/usr/sbin/date logger.go:42: 04:10:20 | gr-self-healing/1-deploy-chaos-mesh | +++ oc get projects logger.go:42: 04:10:20 | gr-self-healing/1-deploy-chaos-mesh | +++ : logger.go:42: 04:10:20 | gr-self-healing/1-deploy-chaos-mesh | +++ kubectl get nodes logger.go:42: 04:10:20 | gr-self-healing/1-deploy-chaos-mesh | +++ grep '^minikube' logger.go:42: 04:10:21 | gr-self-healing/1-deploy-chaos-mesh | +++ which gsed logger.go:42: 04:10:21 | gr-self-healing/1-deploy-chaos-mesh | which: no gsed in (/mnt/jenkins/workspace/cloud-ps-operator_PR-1108/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 04:10:21 | gr-self-healing/1-deploy-chaos-mesh | +++ which sed logger.go:42: 04:10:21 | gr-self-healing/1-deploy-chaos-mesh | ++ sed=/usr/sbin/sed logger.go:42: 04:10:21 | gr-self-healing/1-deploy-chaos-mesh | ++ oc get projects logger.go:42: 04:10:21 | gr-self-healing/1-deploy-chaos-mesh | +++ kubectl version -o json logger.go:42: 04:10:21 | gr-self-healing/1-deploy-chaos-mesh | +++ jq -r .serverVersion.gitVersion logger.go:42: 04:10:21 | gr-self-healing/1-deploy-chaos-mesh | +++ grep '\-eks\-' logger.go:42: 04:10:21 | gr-self-healing/1-deploy-chaos-mesh | grep: warning: stray \ before - logger.go:42: 04:10:21 | gr-self-healing/1-deploy-chaos-mesh | Warning: version difference between client (1.34) and server (1.31) exceeds the supported minor version skew of +/-1 logger.go:42: 04:10:21 | gr-self-healing/1-deploy-chaos-mesh | ++ '[' ']' logger.go:42: 04:10:21 | gr-self-healing/1-deploy-chaos-mesh | ++ EKS=0 logger.go:42: 04:10:21 | gr-self-healing/1-deploy-chaos-mesh | + deploy_chaos_mesh logger.go:42: 04:10:21 | gr-self-healing/1-deploy-chaos-mesh | ++ kubectl get nodes -o 'jsonpath={range .items[0]}{.status.nodeInfo.containerRuntimeVersion}{"\n"}{end}' logger.go:42: 04:10:21 | gr-self-healing/1-deploy-chaos-mesh | ++ awk -F: '{print $1}' logger.go:42: 04:10:22 | gr-self-healing/1-deploy-chaos-mesh | + local current_runtime=containerd logger.go:42: 04:10:22 | gr-self-healing/1-deploy-chaos-mesh | + [[ containerd == \c\o\n\t\a\i\n\e\r\d ]] logger.go:42: 04:10:22 | gr-self-healing/1-deploy-chaos-mesh | + container_runtime=containerd logger.go:42: 04:10:22 | gr-self-healing/1-deploy-chaos-mesh | + socket_path=/run/containerd/containerd.sock logger.go:42: 04:10:22 | gr-self-healing/1-deploy-chaos-mesh | + destroy_chaos_mesh logger.go:42: 04:10:22 | gr-self-healing/1-deploy-chaos-mesh | ++ helm list --all-namespaces --filter chaos-mesh logger.go:42: 04:10:22 | gr-self-healing/1-deploy-chaos-mesh | ++ tail -n1 logger.go:42: 04:10:22 | gr-self-healing/1-deploy-chaos-mesh | ++ awk '-F ' '{print $2}' logger.go:42: 04:10:22 | gr-self-healing/1-deploy-chaos-mesh | ++ /usr/sbin/sed s/NAMESPACE// logger.go:42: 04:10:23 | gr-self-healing/1-deploy-chaos-mesh | + local chaos_mesh_ns= logger.go:42: 04:10:23 | gr-self-healing/1-deploy-chaos-mesh | + '[' -n '' ']' logger.go:42: 04:10:23 | gr-self-healing/1-deploy-chaos-mesh | ++ kubectl get MutatingWebhookConfiguration logger.go:42: 04:10:23 | gr-self-healing/1-deploy-chaos-mesh | ++ grep chaos-mesh logger.go:42: 04:10:23 | gr-self-healing/1-deploy-chaos-mesh | ++ awk '{print $1}' logger.go:42: 04:10:23 | gr-self-healing/1-deploy-chaos-mesh | + timeout 30 kubectl delete MutatingWebhookConfiguration chaos-mesh-mutation logger.go:42: 04:10:24 | gr-self-healing/1-deploy-chaos-mesh | mutatingwebhookconfiguration.admissionregistration.k8s.io "chaos-mesh-mutation" deleted logger.go:42: 04:10:24 | gr-self-healing/1-deploy-chaos-mesh | ++ kubectl get ValidatingWebhookConfiguration logger.go:42: 04:10:24 | gr-self-healing/1-deploy-chaos-mesh | ++ grep chaos-mesh logger.go:42: 04:10:24 | gr-self-healing/1-deploy-chaos-mesh | ++ awk '{print $1}' logger.go:42: 04:10:24 | gr-self-healing/1-deploy-chaos-mesh | + timeout 30 kubectl delete ValidatingWebhookConfiguration chaos-mesh-validation chaos-mesh-validation-auth logger.go:42: 04:10:25 | gr-self-healing/1-deploy-chaos-mesh | validatingwebhookconfiguration.admissionregistration.k8s.io "chaos-mesh-validation" deleted logger.go:42: 04:10:25 | gr-self-healing/1-deploy-chaos-mesh | validatingwebhookconfiguration.admissionregistration.k8s.io "chaos-mesh-validation-auth" deleted logger.go:42: 04:10:25 | gr-self-healing/1-deploy-chaos-mesh | ++ kubectl get ValidatingWebhookConfiguration logger.go:42: 04:10:25 | gr-self-healing/1-deploy-chaos-mesh | ++ grep validate-auth logger.go:42: 04:10:25 | gr-self-healing/1-deploy-chaos-mesh | ++ awk '{print $1}' logger.go:42: 04:10:26 | gr-self-healing/1-deploy-chaos-mesh | + timeout 30 kubectl delete ValidatingWebhookConfiguration logger.go:42: 04:10:26 | gr-self-healing/1-deploy-chaos-mesh | error: resource(s) were provided, but no name was specified logger.go:42: 04:10:26 | gr-self-healing/1-deploy-chaos-mesh | + : logger.go:42: 04:10:26 | gr-self-healing/1-deploy-chaos-mesh | ++ kubectl api-resources logger.go:42: 04:10:26 | gr-self-healing/1-deploy-chaos-mesh | ++ grep chaos-mesh logger.go:42: 04:10:26 | gr-self-healing/1-deploy-chaos-mesh | ++ awk '{print $1}' logger.go:42: 04:10:26 | gr-self-healing/1-deploy-chaos-mesh | + for i in $(kubectl api-resources | grep chaos-mesh | awk '{print $1}') logger.go:42: 04:10:26 | gr-self-healing/1-deploy-chaos-mesh | + kubectl get awschaos --all-namespaces --no-headers -o custom-columns=Kind:.kind,Name:.metadata.name,NAMESPACE:.metadata.namespace logger.go:42: 04:10:26 | gr-self-healing/1-deploy-chaos-mesh | + read -r line logger.go:42: 04:10:27 | gr-self-healing/1-deploy-chaos-mesh | + timeout 30 kubectl delete awschaos --all --all-namespaces logger.go:42: 04:10:27 | gr-self-healing/1-deploy-chaos-mesh | No resources found logger.go:42: 04:10:27 | gr-self-healing/1-deploy-chaos-mesh | + for i in $(kubectl api-resources | grep chaos-mesh | awk '{print $1}') logger.go:42: 04:10:27 | gr-self-healing/1-deploy-chaos-mesh | + kubectl get azurechaos --all-namespaces --no-headers -o custom-columns=Kind:.kind,Name:.metadata.name,NAMESPACE:.metadata.namespace logger.go:42: 04:10:27 | gr-self-healing/1-deploy-chaos-mesh | + read -r line logger.go:42: 04:10:28 | gr-self-healing/1-deploy-chaos-mesh | + timeout 30 kubectl delete azurechaos --all --all-namespaces logger.go:42: 04:10:28 | gr-self-healing/1-deploy-chaos-mesh | No resources found logger.go:42: 04:10:28 | gr-self-healing/1-deploy-chaos-mesh | + for i in $(kubectl api-resources | grep chaos-mesh | awk '{print $1}') logger.go:42: 04:10:28 | gr-self-healing/1-deploy-chaos-mesh | + kubectl get blockchaos --all-namespaces --no-headers -o custom-columns=Kind:.kind,Name:.metadata.name,NAMESPACE:.metadata.namespace logger.go:42: 04:10:28 | gr-self-healing/1-deploy-chaos-mesh | + read -r line logger.go:42: 04:10:29 | gr-self-healing/1-deploy-chaos-mesh | + timeout 30 kubectl delete blockchaos --all --all-namespaces logger.go:42: 04:10:29 | gr-self-healing/1-deploy-chaos-mesh | No resources found logger.go:42: 04:10:29 | gr-self-healing/1-deploy-chaos-mesh | + for i in $(kubectl api-resources | grep chaos-mesh | awk '{print $1}') logger.go:42: 04:10:29 | gr-self-healing/1-deploy-chaos-mesh | + read -r line logger.go:42: 04:10:29 | gr-self-healing/1-deploy-chaos-mesh | + kubectl get dnschaos --all-namespaces --no-headers -o custom-columns=Kind:.kind,Name:.metadata.name,NAMESPACE:.metadata.namespace logger.go:42: 04:10:30 | gr-self-healing/1-deploy-chaos-mesh | + timeout 30 kubectl delete dnschaos --all --all-namespaces logger.go:42: 04:10:30 | gr-self-healing/1-deploy-chaos-mesh | No resources found logger.go:42: 04:10:30 | gr-self-healing/1-deploy-chaos-mesh | + for i in $(kubectl api-resources | grep chaos-mesh | awk '{print $1}') logger.go:42: 04:10:30 | gr-self-healing/1-deploy-chaos-mesh | + kubectl get gcpchaos --all-namespaces --no-headers -o custom-columns=Kind:.kind,Name:.metadata.name,NAMESPACE:.metadata.namespace logger.go:42: 04:10:30 | gr-self-healing/1-deploy-chaos-mesh | + read -r line logger.go:42: 04:10:31 | gr-self-healing/1-deploy-chaos-mesh | + timeout 30 kubectl delete gcpchaos --all --all-namespaces logger.go:42: 04:10:31 | gr-self-healing/1-deploy-chaos-mesh | No resources found logger.go:42: 04:10:31 | gr-self-healing/1-deploy-chaos-mesh | + for i in $(kubectl api-resources | grep chaos-mesh | awk '{print $1}') logger.go:42: 04:10:31 | gr-self-healing/1-deploy-chaos-mesh | + kubectl get httpchaos --all-namespaces --no-headers -o custom-columns=Kind:.kind,Name:.metadata.name,NAMESPACE:.metadata.namespace logger.go:42: 04:10:31 | gr-self-healing/1-deploy-chaos-mesh | + read -r line logger.go:42: 04:10:31 | gr-self-healing/1-deploy-chaos-mesh | + timeout 30 kubectl delete httpchaos --all --all-namespaces logger.go:42: 04:10:32 | gr-self-healing/1-deploy-chaos-mesh | No resources found logger.go:42: 04:10:32 | gr-self-healing/1-deploy-chaos-mesh | + for i in $(kubectl api-resources | grep chaos-mesh | awk '{print $1}') logger.go:42: 04:10:32 | gr-self-healing/1-deploy-chaos-mesh | + kubectl get iochaos --all-namespaces --no-headers -o custom-columns=Kind:.kind,Name:.metadata.name,NAMESPACE:.metadata.namespace logger.go:42: 04:10:32 | gr-self-healing/1-deploy-chaos-mesh | + read -r line logger.go:42: 04:10:32 | gr-self-healing/1-deploy-chaos-mesh | + timeout 30 kubectl delete iochaos --all --all-namespaces logger.go:42: 04:10:33 | gr-self-healing/1-deploy-chaos-mesh | No resources found logger.go:42: 04:10:33 | gr-self-healing/1-deploy-chaos-mesh | + for i in $(kubectl api-resources | grep chaos-mesh | awk '{print $1}') logger.go:42: 04:10:33 | gr-self-healing/1-deploy-chaos-mesh | + kubectl get jvmchaos --all-namespaces --no-headers -o custom-columns=Kind:.kind,Name:.metadata.name,NAMESPACE:.metadata.namespace logger.go:42: 04:10:33 | gr-self-healing/1-deploy-chaos-mesh | + read -r line logger.go:42: 04:10:33 | gr-self-healing/1-deploy-chaos-mesh | + timeout 30 kubectl delete jvmchaos --all --all-namespaces logger.go:42: 04:10:34 | gr-self-healing/1-deploy-chaos-mesh | No resources found logger.go:42: 04:10:34 | gr-self-healing/1-deploy-chaos-mesh | + for i in $(kubectl api-resources | grep chaos-mesh | awk '{print $1}') logger.go:42: 04:10:34 | gr-self-healing/1-deploy-chaos-mesh | + kubectl get kernelchaos --all-namespaces --no-headers -o custom-columns=Kind:.kind,Name:.metadata.name,NAMESPACE:.metadata.namespace logger.go:42: 04:10:34 | gr-self-healing/1-deploy-chaos-mesh | + read -r line logger.go:42: 04:10:34 | gr-self-healing/1-deploy-chaos-mesh | + timeout 30 kubectl delete kernelchaos --all --all-namespaces logger.go:42: 04:10:35 | gr-self-healing/1-deploy-chaos-mesh | No resources found logger.go:42: 04:10:35 | gr-self-healing/1-deploy-chaos-mesh | + for i in $(kubectl api-resources | grep chaos-mesh | awk '{print $1}') logger.go:42: 04:10:35 | gr-self-healing/1-deploy-chaos-mesh | + kubectl get networkchaos --all-namespaces --no-headers -o custom-columns=Kind:.kind,Name:.metadata.name,NAMESPACE:.metadata.namespace logger.go:42: 04:10:35 | gr-self-healing/1-deploy-chaos-mesh | + read -r line logger.go:42: 04:10:35 | gr-self-healing/1-deploy-chaos-mesh | ++ echo 'NetworkChaos chaos-pod-network-loss-primary kuttl-test-teaching-sawfish' logger.go:42: 04:10:35 | gr-self-healing/1-deploy-chaos-mesh | ++ awk '{print $1}' logger.go:42: 04:10:35 | gr-self-healing/1-deploy-chaos-mesh | + local kind=NetworkChaos logger.go:42: 04:10:35 | gr-self-healing/1-deploy-chaos-mesh | ++ echo 'NetworkChaos chaos-pod-network-loss-primary kuttl-test-teaching-sawfish' logger.go:42: 04:10:35 | gr-self-healing/1-deploy-chaos-mesh | ++ awk '{print $2}' logger.go:42: 04:10:35 | gr-self-healing/1-deploy-chaos-mesh | + local name=chaos-pod-network-loss-primary logger.go:42: 04:10:35 | gr-self-healing/1-deploy-chaos-mesh | ++ echo 'NetworkChaos chaos-pod-network-loss-primary kuttl-test-teaching-sawfish' logger.go:42: 04:10:35 | gr-self-healing/1-deploy-chaos-mesh | ++ awk '{print $3}' logger.go:42: 04:10:35 | gr-self-healing/1-deploy-chaos-mesh | + local namespace=kuttl-test-teaching-sawfish logger.go:42: 04:10:35 | gr-self-healing/1-deploy-chaos-mesh | + kubectl patch NetworkChaos chaos-pod-network-loss-primary -n kuttl-test-teaching-sawfish --type=merge -p '{"metadata":{"finalizers":[]}}' logger.go:42: 04:10:36 | gr-self-healing/1-deploy-chaos-mesh | networkchaos.chaos-mesh.org/chaos-pod-network-loss-primary patched logger.go:42: 04:10:36 | gr-self-healing/1-deploy-chaos-mesh | + read -r line logger.go:42: 04:10:36 | gr-self-healing/1-deploy-chaos-mesh | + timeout 30 kubectl delete networkchaos --all --all-namespaces logger.go:42: 04:10:36 | gr-self-healing/1-deploy-chaos-mesh | No resources found logger.go:42: 04:10:36 | gr-self-healing/1-deploy-chaos-mesh | + for i in $(kubectl api-resources | grep chaos-mesh | awk '{print $1}') logger.go:42: 04:10:36 | gr-self-healing/1-deploy-chaos-mesh | + kubectl get physicalmachinechaos --all-namespaces --no-headers -o custom-columns=Kind:.kind,Name:.metadata.name,NAMESPACE:.metadata.namespace logger.go:42: 04:10:36 | gr-self-healing/1-deploy-chaos-mesh | + read -r line logger.go:42: 04:10:37 | gr-self-healing/1-deploy-chaos-mesh | + timeout 30 kubectl delete physicalmachinechaos --all --all-namespaces logger.go:42: 04:10:38 | gr-self-healing/1-deploy-chaos-mesh | No resources found logger.go:42: 04:10:38 | gr-self-healing/1-deploy-chaos-mesh | + for i in $(kubectl api-resources | grep chaos-mesh | awk '{print $1}') logger.go:42: 04:10:38 | gr-self-healing/1-deploy-chaos-mesh | + kubectl get physicalmachines --all-namespaces --no-headers -o custom-columns=Kind:.kind,Name:.metadata.name,NAMESPACE:.metadata.namespace logger.go:42: 04:10:38 | gr-self-healing/1-deploy-chaos-mesh | + read -r line logger.go:42: 04:10:38 | gr-self-healing/1-deploy-chaos-mesh | + timeout 30 kubectl delete physicalmachines --all --all-namespaces logger.go:42: 04:10:39 | gr-self-healing/1-deploy-chaos-mesh | No resources found logger.go:42: 04:10:39 | gr-self-healing/1-deploy-chaos-mesh | + for i in $(kubectl api-resources | grep chaos-mesh | awk '{print $1}') logger.go:42: 04:10:39 | gr-self-healing/1-deploy-chaos-mesh | + kubectl get podchaos --all-namespaces --no-headers -o custom-columns=Kind:.kind,Name:.metadata.name,NAMESPACE:.metadata.namespace logger.go:42: 04:10:39 | gr-self-healing/1-deploy-chaos-mesh | + read -r line logger.go:42: 04:10:39 | gr-self-healing/1-deploy-chaos-mesh | ++ echo 'PodChaos chaos-kill-label-cluster-crash kuttl-test-teaching-sawfish' logger.go:42: 04:10:39 | gr-self-healing/1-deploy-chaos-mesh | ++ awk '{print $1}' logger.go:42: 04:10:39 | gr-self-healing/1-deploy-chaos-mesh | + local kind=PodChaos logger.go:42: 04:10:39 | gr-self-healing/1-deploy-chaos-mesh | ++ echo 'PodChaos chaos-kill-label-cluster-crash kuttl-test-teaching-sawfish' logger.go:42: 04:10:39 | gr-self-healing/1-deploy-chaos-mesh | ++ awk '{print $2}' logger.go:42: 04:10:39 | gr-self-healing/1-deploy-chaos-mesh | + local name=chaos-kill-label-cluster-crash logger.go:42: 04:10:39 | gr-self-healing/1-deploy-chaos-mesh | ++ echo 'PodChaos chaos-kill-label-cluster-crash kuttl-test-teaching-sawfish' logger.go:42: 04:10:39 | gr-self-healing/1-deploy-chaos-mesh | ++ awk '{print $3}' logger.go:42: 04:10:39 | gr-self-healing/1-deploy-chaos-mesh | + local namespace=kuttl-test-teaching-sawfish logger.go:42: 04:10:39 | gr-self-healing/1-deploy-chaos-mesh | + kubectl patch PodChaos chaos-kill-label-cluster-crash -n kuttl-test-teaching-sawfish --type=merge -p '{"metadata":{"finalizers":[]}}' logger.go:42: 04:10:40 | gr-self-healing/1-deploy-chaos-mesh | podchaos.chaos-mesh.org/chaos-kill-label-cluster-crash patched logger.go:42: 04:10:40 | gr-self-healing/1-deploy-chaos-mesh | + read -r line logger.go:42: 04:10:40 | gr-self-healing/1-deploy-chaos-mesh | ++ echo 'PodChaos chaos-pod-failure-primary kuttl-test-teaching-sawfish' logger.go:42: 04:10:40 | gr-self-healing/1-deploy-chaos-mesh | ++ awk '{print $1}' logger.go:42: 04:10:40 | gr-self-healing/1-deploy-chaos-mesh | + local kind=PodChaos logger.go:42: 04:10:40 | gr-self-healing/1-deploy-chaos-mesh | ++ echo 'PodChaos chaos-pod-failure-primary kuttl-test-teaching-sawfish' logger.go:42: 04:10:40 | gr-self-healing/1-deploy-chaos-mesh | ++ awk '{print $2}' logger.go:42: 04:10:40 | gr-self-healing/1-deploy-chaos-mesh | + local name=chaos-pod-failure-primary logger.go:42: 04:10:40 | gr-self-healing/1-deploy-chaos-mesh | ++ echo 'PodChaos chaos-pod-failure-primary kuttl-test-teaching-sawfish' logger.go:42: 04:10:40 | gr-self-healing/1-deploy-chaos-mesh | ++ awk '{print $3}' logger.go:42: 04:10:40 | gr-self-healing/1-deploy-chaos-mesh | + local namespace=kuttl-test-teaching-sawfish logger.go:42: 04:10:40 | gr-self-healing/1-deploy-chaos-mesh | + kubectl patch PodChaos chaos-pod-failure-primary -n kuttl-test-teaching-sawfish --type=merge -p '{"metadata":{"finalizers":[]}}' logger.go:42: 04:10:40 | gr-self-healing/1-deploy-chaos-mesh | podchaos.chaos-mesh.org/chaos-pod-failure-primary patched logger.go:42: 04:10:40 | gr-self-healing/1-deploy-chaos-mesh | + read -r line logger.go:42: 04:10:40 | gr-self-healing/1-deploy-chaos-mesh | ++ echo 'PodChaos chaos-pod-kill-primary kuttl-test-teaching-sawfish' logger.go:42: 04:10:40 | gr-self-healing/1-deploy-chaos-mesh | ++ awk '{print $1}' logger.go:42: 04:10:40 | gr-self-healing/1-deploy-chaos-mesh | + local kind=PodChaos logger.go:42: 04:10:40 | gr-self-healing/1-deploy-chaos-mesh | ++ echo 'PodChaos chaos-pod-kill-primary kuttl-test-teaching-sawfish' logger.go:42: 04:10:40 | gr-self-healing/1-deploy-chaos-mesh | ++ awk '{print $2}' logger.go:42: 04:10:40 | gr-self-healing/1-deploy-chaos-mesh | + local name=chaos-pod-kill-primary logger.go:42: 04:10:40 | gr-self-healing/1-deploy-chaos-mesh | ++ echo 'PodChaos chaos-pod-kill-primary kuttl-test-teaching-sawfish' logger.go:42: 04:10:40 | gr-self-healing/1-deploy-chaos-mesh | ++ awk '{print $3}' logger.go:42: 04:10:40 | gr-self-healing/1-deploy-chaos-mesh | + local namespace=kuttl-test-teaching-sawfish logger.go:42: 04:10:40 | gr-self-healing/1-deploy-chaos-mesh | + kubectl patch PodChaos chaos-pod-kill-primary -n kuttl-test-teaching-sawfish --type=merge -p '{"metadata":{"finalizers":[]}}' logger.go:42: 04:10:41 | gr-self-healing/1-deploy-chaos-mesh | podchaos.chaos-mesh.org/chaos-pod-kill-primary patched logger.go:42: 04:10:41 | gr-self-healing/1-deploy-chaos-mesh | + read -r line logger.go:42: 04:10:41 | gr-self-healing/1-deploy-chaos-mesh | + timeout 30 kubectl delete podchaos --all --all-namespaces logger.go:42: 04:10:42 | gr-self-healing/1-deploy-chaos-mesh | No resources found logger.go:42: 04:10:42 | gr-self-healing/1-deploy-chaos-mesh | + for i in $(kubectl api-resources | grep chaos-mesh | awk '{print $1}') logger.go:42: 04:10:42 | gr-self-healing/1-deploy-chaos-mesh | + kubectl get podhttpchaos --all-namespaces --no-headers -o custom-columns=Kind:.kind,Name:.metadata.name,NAMESPACE:.metadata.namespace logger.go:42: 04:10:42 | gr-self-healing/1-deploy-chaos-mesh | + read -r line logger.go:42: 04:10:42 | gr-self-healing/1-deploy-chaos-mesh | + timeout 30 kubectl delete podhttpchaos --all --all-namespaces logger.go:42: 04:10:42 | gr-self-healing/1-deploy-chaos-mesh | No resources found logger.go:42: 04:10:42 | gr-self-healing/1-deploy-chaos-mesh | + for i in $(kubectl api-resources | grep chaos-mesh | awk '{print $1}') logger.go:42: 04:10:42 | gr-self-healing/1-deploy-chaos-mesh | + kubectl get podiochaos --all-namespaces --no-headers -o custom-columns=Kind:.kind,Name:.metadata.name,NAMESPACE:.metadata.namespace logger.go:42: 04:10:42 | gr-self-healing/1-deploy-chaos-mesh | + read -r line logger.go:42: 04:10:43 | gr-self-healing/1-deploy-chaos-mesh | + timeout 30 kubectl delete podiochaos --all --all-namespaces logger.go:42: 04:10:43 | gr-self-healing/1-deploy-chaos-mesh | No resources found logger.go:42: 04:10:43 | gr-self-healing/1-deploy-chaos-mesh | + for i in $(kubectl api-resources | grep chaos-mesh | awk '{print $1}') logger.go:42: 04:10:43 | gr-self-healing/1-deploy-chaos-mesh | + kubectl get podnetworkchaos --all-namespaces --no-headers -o custom-columns=Kind:.kind,Name:.metadata.name,NAMESPACE:.metadata.namespace logger.go:42: 04:10:43 | gr-self-healing/1-deploy-chaos-mesh | + read -r line logger.go:42: 04:10:44 | gr-self-healing/1-deploy-chaos-mesh | + timeout 30 kubectl delete podnetworkchaos --all --all-namespaces logger.go:42: 04:10:44 | gr-self-healing/1-deploy-chaos-mesh | No resources found logger.go:42: 04:10:44 | gr-self-healing/1-deploy-chaos-mesh | + for i in $(kubectl api-resources | grep chaos-mesh | awk '{print $1}') logger.go:42: 04:10:44 | gr-self-healing/1-deploy-chaos-mesh | + kubectl get remoteclusters --all-namespaces --no-headers -o custom-columns=Kind:.kind,Name:.metadata.name,NAMESPACE:.metadata.namespace logger.go:42: 04:10:44 | gr-self-healing/1-deploy-chaos-mesh | + read -r line logger.go:42: 04:10:45 | gr-self-healing/1-deploy-chaos-mesh | + timeout 30 kubectl delete remoteclusters --all --all-namespaces logger.go:42: 04:10:45 | gr-self-healing/1-deploy-chaos-mesh | No resources found logger.go:42: 04:10:45 | gr-self-healing/1-deploy-chaos-mesh | + for i in $(kubectl api-resources | grep chaos-mesh | awk '{print $1}') logger.go:42: 04:10:45 | gr-self-healing/1-deploy-chaos-mesh | + kubectl get schedules --all-namespaces --no-headers -o custom-columns=Kind:.kind,Name:.metadata.name,NAMESPACE:.metadata.namespace logger.go:42: 04:10:45 | gr-self-healing/1-deploy-chaos-mesh | + read -r line logger.go:42: 04:10:46 | gr-self-healing/1-deploy-chaos-mesh | + timeout 30 kubectl delete schedules --all --all-namespaces logger.go:42: 04:10:46 | gr-self-healing/1-deploy-chaos-mesh | No resources found logger.go:42: 04:10:46 | gr-self-healing/1-deploy-chaos-mesh | + for i in $(kubectl api-resources | grep chaos-mesh | awk '{print $1}') logger.go:42: 04:10:46 | gr-self-healing/1-deploy-chaos-mesh | + kubectl get statuschecks --all-namespaces --no-headers -o custom-columns=Kind:.kind,Name:.metadata.name,NAMESPACE:.metadata.namespace logger.go:42: 04:10:46 | gr-self-healing/1-deploy-chaos-mesh | + read -r line logger.go:42: 04:10:47 | gr-self-healing/1-deploy-chaos-mesh | + timeout 30 kubectl delete statuschecks --all --all-namespaces logger.go:42: 04:10:47 | gr-self-healing/1-deploy-chaos-mesh | No resources found logger.go:42: 04:10:47 | gr-self-healing/1-deploy-chaos-mesh | + for i in $(kubectl api-resources | grep chaos-mesh | awk '{print $1}') logger.go:42: 04:10:47 | gr-self-healing/1-deploy-chaos-mesh | + kubectl get stresschaos --all-namespaces --no-headers -o custom-columns=Kind:.kind,Name:.metadata.name,NAMESPACE:.metadata.namespace logger.go:42: 04:10:47 | gr-self-healing/1-deploy-chaos-mesh | + read -r line logger.go:42: 04:10:48 | gr-self-healing/1-deploy-chaos-mesh | + timeout 30 kubectl delete stresschaos --all --all-namespaces logger.go:42: 04:10:48 | gr-self-healing/1-deploy-chaos-mesh | No resources found logger.go:42: 04:10:48 | gr-self-healing/1-deploy-chaos-mesh | + for i in $(kubectl api-resources | grep chaos-mesh | awk '{print $1}') logger.go:42: 04:10:48 | gr-self-healing/1-deploy-chaos-mesh | + kubectl get timechaos --all-namespaces --no-headers -o custom-columns=Kind:.kind,Name:.metadata.name,NAMESPACE:.metadata.namespace logger.go:42: 04:10:48 | gr-self-healing/1-deploy-chaos-mesh | + read -r line logger.go:42: 04:10:48 | gr-self-healing/1-deploy-chaos-mesh | + timeout 30 kubectl delete timechaos --all --all-namespaces logger.go:42: 04:10:49 | gr-self-healing/1-deploy-chaos-mesh | No resources found logger.go:42: 04:10:49 | gr-self-healing/1-deploy-chaos-mesh | + for i in $(kubectl api-resources | grep chaos-mesh | awk '{print $1}') logger.go:42: 04:10:49 | gr-self-healing/1-deploy-chaos-mesh | + kubectl get workflownodes --all-namespaces --no-headers -o custom-columns=Kind:.kind,Name:.metadata.name,NAMESPACE:.metadata.namespace logger.go:42: 04:10:49 | gr-self-healing/1-deploy-chaos-mesh | + read -r line logger.go:42: 04:10:49 | gr-self-healing/1-deploy-chaos-mesh | + timeout 30 kubectl delete workflownodes --all --all-namespaces logger.go:42: 04:10:50 | gr-self-healing/1-deploy-chaos-mesh | No resources found logger.go:42: 04:10:50 | gr-self-healing/1-deploy-chaos-mesh | + for i in $(kubectl api-resources | grep chaos-mesh | awk '{print $1}') logger.go:42: 04:10:50 | gr-self-healing/1-deploy-chaos-mesh | + kubectl get workflows --all-namespaces --no-headers -o custom-columns=Kind:.kind,Name:.metadata.name,NAMESPACE:.metadata.namespace logger.go:42: 04:10:50 | gr-self-healing/1-deploy-chaos-mesh | + read -r line logger.go:42: 04:10:50 | gr-self-healing/1-deploy-chaos-mesh | + timeout 30 kubectl delete workflows --all --all-namespaces logger.go:42: 04:10:51 | gr-self-healing/1-deploy-chaos-mesh | No resources found logger.go:42: 04:10:51 | gr-self-healing/1-deploy-chaos-mesh | ++ kubectl get crd logger.go:42: 04:10:51 | gr-self-healing/1-deploy-chaos-mesh | ++ grep chaos-mesh.org logger.go:42: 04:10:51 | gr-self-healing/1-deploy-chaos-mesh | ++ awk '{print $1}' logger.go:42: 04:10:52 | gr-self-healing/1-deploy-chaos-mesh | + timeout 30 kubectl delete crd awschaos.chaos-mesh.org azurechaos.chaos-mesh.org blockchaos.chaos-mesh.org dnschaos.chaos-mesh.org gcpchaos.chaos-mesh.org httpchaos.chaos-mesh.org iochaos.chaos-mesh.org jvmchaos.chaos-mesh.org kernelchaos.chaos-mesh.org networkchaos.chaos-mesh.org physicalmachinechaos.chaos-mesh.org physicalmachines.chaos-mesh.org podchaos.chaos-mesh.org podhttpchaos.chaos-mesh.org podiochaos.chaos-mesh.org podnetworkchaos.chaos-mesh.org remoteclusters.chaos-mesh.org schedules.chaos-mesh.org statuschecks.chaos-mesh.org stresschaos.chaos-mesh.org timechaos.chaos-mesh.org workflownodes.chaos-mesh.org workflows.chaos-mesh.org logger.go:42: 04:10:52 | gr-self-healing/1-deploy-chaos-mesh | customresourcedefinition.apiextensions.k8s.io "awschaos.chaos-mesh.org" deleted logger.go:42: 04:10:53 | gr-self-healing/1-deploy-chaos-mesh | customresourcedefinition.apiextensions.k8s.io "azurechaos.chaos-mesh.org" deleted logger.go:42: 04:10:53 | gr-self-healing/1-deploy-chaos-mesh | customresourcedefinition.apiextensions.k8s.io "blockchaos.chaos-mesh.org" deleted logger.go:42: 04:10:53 | gr-self-healing/1-deploy-chaos-mesh | customresourcedefinition.apiextensions.k8s.io "dnschaos.chaos-mesh.org" deleted logger.go:42: 04:10:53 | gr-self-healing/1-deploy-chaos-mesh | customresourcedefinition.apiextensions.k8s.io "gcpchaos.chaos-mesh.org" deleted logger.go:42: 04:10:53 | gr-self-healing/1-deploy-chaos-mesh | customresourcedefinition.apiextensions.k8s.io "httpchaos.chaos-mesh.org" deleted logger.go:42: 04:10:54 | gr-self-healing/1-deploy-chaos-mesh | customresourcedefinition.apiextensions.k8s.io "iochaos.chaos-mesh.org" deleted logger.go:42: 04:10:54 | gr-self-healing/1-deploy-chaos-mesh | customresourcedefinition.apiextensions.k8s.io "jvmchaos.chaos-mesh.org" deleted logger.go:42: 04:10:54 | gr-self-healing/1-deploy-chaos-mesh | customresourcedefinition.apiextensions.k8s.io "kernelchaos.chaos-mesh.org" deleted logger.go:42: 04:10:54 | gr-self-healing/1-deploy-chaos-mesh | customresourcedefinition.apiextensions.k8s.io "networkchaos.chaos-mesh.org" deleted logger.go:42: 04:10:55 | gr-self-healing/1-deploy-chaos-mesh | customresourcedefinition.apiextensions.k8s.io "physicalmachinechaos.chaos-mesh.org" deleted logger.go:42: 04:10:55 | gr-self-healing/1-deploy-chaos-mesh | customresourcedefinition.apiextensions.k8s.io "physicalmachines.chaos-mesh.org" deleted logger.go:42: 04:10:55 | gr-self-healing/1-deploy-chaos-mesh | customresourcedefinition.apiextensions.k8s.io "podchaos.chaos-mesh.org" deleted logger.go:42: 04:10:55 | gr-self-healing/1-deploy-chaos-mesh | customresourcedefinition.apiextensions.k8s.io "podhttpchaos.chaos-mesh.org" deleted logger.go:42: 04:10:55 | gr-self-healing/1-deploy-chaos-mesh | customresourcedefinition.apiextensions.k8s.io "podiochaos.chaos-mesh.org" deleted logger.go:42: 04:10:56 | gr-self-healing/1-deploy-chaos-mesh | customresourcedefinition.apiextensions.k8s.io "podnetworkchaos.chaos-mesh.org" deleted logger.go:42: 04:10:56 | gr-self-healing/1-deploy-chaos-mesh | customresourcedefinition.apiextensions.k8s.io "remoteclusters.chaos-mesh.org" deleted logger.go:42: 04:10:56 | gr-self-healing/1-deploy-chaos-mesh | customresourcedefinition.apiextensions.k8s.io "schedules.chaos-mesh.org" deleted logger.go:42: 04:10:57 | gr-self-healing/1-deploy-chaos-mesh | customresourcedefinition.apiextensions.k8s.io "statuschecks.chaos-mesh.org" deleted logger.go:42: 04:10:57 | gr-self-healing/1-deploy-chaos-mesh | customresourcedefinition.apiextensions.k8s.io "stresschaos.chaos-mesh.org" deleted logger.go:42: 04:10:57 | gr-self-healing/1-deploy-chaos-mesh | customresourcedefinition.apiextensions.k8s.io "timechaos.chaos-mesh.org" deleted logger.go:42: 04:10:59 | gr-self-healing/1-deploy-chaos-mesh | customresourcedefinition.apiextensions.k8s.io "workflownodes.chaos-mesh.org" deleted logger.go:42: 04:11:01 | gr-self-healing/1-deploy-chaos-mesh | customresourcedefinition.apiextensions.k8s.io "workflows.chaos-mesh.org" deleted logger.go:42: 04:11:05 | gr-self-healing/1-deploy-chaos-mesh | ++ grep chaos-mesh logger.go:42: 04:11:05 | gr-self-healing/1-deploy-chaos-mesh | ++ kubectl get clusterrolebinding logger.go:42: 04:11:05 | gr-self-healing/1-deploy-chaos-mesh | ++ awk '{print $1}' logger.go:42: 04:11:05 | gr-self-healing/1-deploy-chaos-mesh | + timeout 30 kubectl delete clusterrolebinding chaos-mesh-chaos-controller-manager-cluster-level chaos-mesh-chaos-controller-manager-target-namespace chaos-mesh-chaos-dashboard-cluster-level chaos-mesh-chaos-dashboard-target-namespace chaos-mesh-chaos-dns-server-cluster-level chaos-mesh-chaos-dns-server-target-namespace logger.go:42: 04:11:06 | gr-self-healing/1-deploy-chaos-mesh | clusterrolebinding.rbac.authorization.k8s.io "chaos-mesh-chaos-controller-manager-cluster-level" deleted logger.go:42: 04:11:06 | gr-self-healing/1-deploy-chaos-mesh | clusterrolebinding.rbac.authorization.k8s.io "chaos-mesh-chaos-controller-manager-target-namespace" deleted logger.go:42: 04:11:06 | gr-self-healing/1-deploy-chaos-mesh | clusterrolebinding.rbac.authorization.k8s.io "chaos-mesh-chaos-dashboard-cluster-level" deleted logger.go:42: 04:11:06 | gr-self-healing/1-deploy-chaos-mesh | clusterrolebinding.rbac.authorization.k8s.io "chaos-mesh-chaos-dashboard-target-namespace" deleted logger.go:42: 04:11:06 | gr-self-healing/1-deploy-chaos-mesh | clusterrolebinding.rbac.authorization.k8s.io "chaos-mesh-chaos-dns-server-cluster-level" deleted logger.go:42: 04:11:07 | gr-self-healing/1-deploy-chaos-mesh | clusterrolebinding.rbac.authorization.k8s.io "chaos-mesh-chaos-dns-server-target-namespace" deleted logger.go:42: 04:11:07 | gr-self-healing/1-deploy-chaos-mesh | ++ kubectl get clusterrole logger.go:42: 04:11:07 | gr-self-healing/1-deploy-chaos-mesh | ++ grep chaos-mesh logger.go:42: 04:11:07 | gr-self-healing/1-deploy-chaos-mesh | ++ awk '{print $1}' logger.go:42: 04:11:08 | gr-self-healing/1-deploy-chaos-mesh | + timeout 30 kubectl delete clusterrole chaos-mesh-chaos-controller-manager-cluster-level chaos-mesh-chaos-controller-manager-target-namespace chaos-mesh-chaos-dashboard-cluster-level chaos-mesh-chaos-dashboard-target-namespace chaos-mesh-chaos-dns-server-cluster-level chaos-mesh-chaos-dns-server-target-namespace logger.go:42: 04:11:09 | gr-self-healing/1-deploy-chaos-mesh | clusterrole.rbac.authorization.k8s.io "chaos-mesh-chaos-controller-manager-cluster-level" deleted logger.go:42: 04:11:09 | gr-self-healing/1-deploy-chaos-mesh | clusterrole.rbac.authorization.k8s.io "chaos-mesh-chaos-controller-manager-target-namespace" deleted logger.go:42: 04:11:09 | gr-self-healing/1-deploy-chaos-mesh | clusterrole.rbac.authorization.k8s.io "chaos-mesh-chaos-dashboard-cluster-level" deleted logger.go:42: 04:11:09 | gr-self-healing/1-deploy-chaos-mesh | clusterrole.rbac.authorization.k8s.io "chaos-mesh-chaos-dashboard-target-namespace" deleted logger.go:42: 04:11:09 | gr-self-healing/1-deploy-chaos-mesh | clusterrole.rbac.authorization.k8s.io "chaos-mesh-chaos-dns-server-cluster-level" deleted logger.go:42: 04:11:10 | gr-self-healing/1-deploy-chaos-mesh | clusterrole.rbac.authorization.k8s.io "chaos-mesh-chaos-dns-server-target-namespace" deleted logger.go:42: 04:11:10 | gr-self-healing/1-deploy-chaos-mesh | + helm repo add chaos-mesh https://charts.chaos-mesh.org logger.go:42: 04:11:10 | gr-self-healing/1-deploy-chaos-mesh | "chaos-mesh" already exists with the same configuration, skipping logger.go:42: 04:11:10 | gr-self-healing/1-deploy-chaos-mesh | + helm install chaos-mesh chaos-mesh/chaos-mesh --namespace=kuttl-test-concrete-skylark --set chaosDaemon.runtime=containerd --set chaosDaemon.socketPath=/run/containerd/containerd.sock --set dashboard.create=false --version 2.7.2 logger.go:42: 04:11:38 | gr-self-healing/1-deploy-chaos-mesh | NAME: chaos-mesh logger.go:42: 04:11:38 | gr-self-healing/1-deploy-chaos-mesh | LAST DEPLOYED: Wed Oct 1 04:11:24 2025 logger.go:42: 04:11:38 | gr-self-healing/1-deploy-chaos-mesh | NAMESPACE: kuttl-test-concrete-skylark logger.go:42: 04:11:38 | gr-self-healing/1-deploy-chaos-mesh | STATUS: deployed logger.go:42: 04:11:38 | gr-self-healing/1-deploy-chaos-mesh | REVISION: 1 logger.go:42: 04:11:38 | gr-self-healing/1-deploy-chaos-mesh | TEST SUITE: None logger.go:42: 04:11:38 | gr-self-healing/1-deploy-chaos-mesh | NOTES: logger.go:42: 04:11:38 | gr-self-healing/1-deploy-chaos-mesh | 1. Make sure chaos-mesh components are running logger.go:42: 04:11:38 | gr-self-healing/1-deploy-chaos-mesh | kubectl get pods --namespace kuttl-test-concrete-skylark -l app.kubernetes.io/instance=chaos-mesh logger.go:42: 04:11:38 | gr-self-healing/1-deploy-chaos-mesh | + '[' -n '' ']' logger.go:42: 04:11:38 | gr-self-healing/1-deploy-chaos-mesh | + echo 'Waiting for chaos-mesh DaemonSet to be ready...' logger.go:42: 04:11:38 | gr-self-healing/1-deploy-chaos-mesh | Waiting for chaos-mesh DaemonSet to be ready... logger.go:42: 04:11:38 | gr-self-healing/1-deploy-chaos-mesh | ++ kubectl get daemonset chaos-daemon -n kuttl-test-concrete-skylark -o 'jsonpath={.status.numberReady}' logger.go:42: 04:11:39 | gr-self-healing/1-deploy-chaos-mesh | ++ kubectl get daemonset chaos-daemon -n kuttl-test-concrete-skylark -o 'jsonpath={.status.desiredNumberScheduled}' logger.go:42: 04:11:39 | gr-self-healing/1-deploy-chaos-mesh | + '[' 3 = 3 ']' logger.go:42: 04:11:40 | gr-self-healing/1-deploy-chaos-mesh | test step completed 1-deploy-chaos-mesh logger.go:42: 04:11:40 | gr-self-healing/2-create-cluster | starting test step 2-create-cluster logger.go:42: 04:11:40 | gr-self-healing/2-create-cluster | running command: [sh -c set -o errexit set -o xtrace source ../../functions get_cr \ | yq eval '.spec.mysql.clusterType="group-replication"' - \ | yq eval '.spec.mysql.size=3' - \ | yq eval '.spec.mysql.affinity.antiAffinityTopologyKey="none"' - \ | yq eval '.spec.proxy.haproxy.enabled=false' - \ | yq eval '.spec.proxy.router.enabled=true' - \ | yq eval '.spec.proxy.router.size=3' - \ | yq eval '.spec.proxy.router.affinity.antiAffinityTopologyKey="none"' - \ | yq eval '.spec.orchestrator.enabled=false' - \ | kubectl -n "${NAMESPACE}" apply -f -] logger.go:42: 04:11:40 | gr-self-healing/2-create-cluster | + source ../../functions logger.go:42: 04:11:40 | gr-self-healing/2-create-cluster | +++ realpath ../../.. logger.go:42: 04:11:40 | gr-self-healing/2-create-cluster | ++ ROOT_REPO=/mnt/jenkins/workspace/cloud-ps-operator_PR-1108 logger.go:42: 04:11:40 | gr-self-healing/2-create-cluster | ++++ pwd logger.go:42: 04:11:40 | gr-self-healing/2-create-cluster | +++ basename /mnt/jenkins/workspace/cloud-ps-operator_PR-1108/e2e-tests/tests/gr-self-healing logger.go:42: 04:11:40 | gr-self-healing/2-create-cluster | ++ test_name=gr-self-healing logger.go:42: 04:11:40 | gr-self-healing/2-create-cluster | ++ source /mnt/jenkins/workspace/cloud-ps-operator_PR-1108/e2e-tests/vars.sh logger.go:42: 04:11:40 | gr-self-healing/2-create-cluster | +++ export ROOT_REPO=/mnt/jenkins/workspace/cloud-ps-operator_PR-1108 logger.go:42: 04:11:40 | gr-self-healing/2-create-cluster | +++ ROOT_REPO=/mnt/jenkins/workspace/cloud-ps-operator_PR-1108 logger.go:42: 04:11:40 | gr-self-healing/2-create-cluster | +++ export DEPLOY_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-1108/deploy logger.go:42: 04:11:40 | gr-self-healing/2-create-cluster | +++ DEPLOY_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-1108/deploy logger.go:42: 04:11:40 | gr-self-healing/2-create-cluster | +++ export TESTS_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-1108/e2e-tests logger.go:42: 04:11:40 | gr-self-healing/2-create-cluster | +++ TESTS_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-1108/e2e-tests logger.go:42: 04:11:40 | gr-self-healing/2-create-cluster | +++ export TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-1108/e2e-tests/conf logger.go:42: 04:11:40 | gr-self-healing/2-create-cluster | +++ TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-1108/e2e-tests/conf logger.go:42: 04:11:40 | gr-self-healing/2-create-cluster | +++ export TEMP_DIR=/tmp/kuttl/ps/gr-self-healing logger.go:42: 04:11:40 | gr-self-healing/2-create-cluster | +++ TEMP_DIR=/tmp/kuttl/ps/gr-self-healing logger.go:42: 04:11:40 | gr-self-healing/2-create-cluster | ++++ git rev-parse --abbrev-ref HEAD logger.go:42: 04:11:40 | gr-self-healing/2-create-cluster | +++ export GIT_BRANCH=PR-1108 logger.go:42: 04:11:40 | gr-self-healing/2-create-cluster | +++ GIT_BRANCH=PR-1108 logger.go:42: 04:11:40 | gr-self-healing/2-create-cluster | +++ export VERSION=PR-1108-c411fa10 logger.go:42: 04:11:40 | gr-self-healing/2-create-cluster | +++ VERSION=PR-1108-c411fa10 logger.go:42: 04:11:40 | gr-self-healing/2-create-cluster | +++ export IMAGE=perconalab/percona-server-mysql-operator:PR-1108-c411fa10 logger.go:42: 04:11:40 | gr-self-healing/2-create-cluster | +++ IMAGE=perconalab/percona-server-mysql-operator:PR-1108-c411fa10 logger.go:42: 04:11:40 | gr-self-healing/2-create-cluster | +++ [[ -z '' ]] logger.go:42: 04:11:40 | gr-self-healing/2-create-cluster | +++ [[ -n '' ]] logger.go:42: 04:11:40 | gr-self-healing/2-create-cluster | +++ export MYSQL_VERSION=8.4 logger.go:42: 04:11:40 | gr-self-healing/2-create-cluster | +++ MYSQL_VERSION=8.4 logger.go:42: 04:11:40 | gr-self-healing/2-create-cluster | +++ export IMAGE_MYSQL=perconalab/percona-server-mysql-operator:main-psmysql8.4 logger.go:42: 04:11:40 | gr-self-healing/2-create-cluster | +++ IMAGE_MYSQL=perconalab/percona-server-mysql-operator:main-psmysql8.4 logger.go:42: 04:11:40 | gr-self-healing/2-create-cluster | +++ export IMAGE_BACKUP=perconalab/percona-server-mysql-operator:main-backup8.4 logger.go:42: 04:11:40 | gr-self-healing/2-create-cluster | +++ IMAGE_BACKUP=perconalab/percona-server-mysql-operator:main-backup8.4 logger.go:42: 04:11:40 | gr-self-healing/2-create-cluster | +++ export IMAGE_ORCHESTRATOR=perconalab/percona-server-mysql-operator:main-orchestrator logger.go:42: 04:11:40 | gr-self-healing/2-create-cluster | +++ IMAGE_ORCHESTRATOR=perconalab/percona-server-mysql-operator:main-orchestrator logger.go:42: 04:11:40 | gr-self-healing/2-create-cluster | +++ export IMAGE_ROUTER=perconalab/percona-server-mysql-operator:main-router8.4 logger.go:42: 04:11:40 | gr-self-healing/2-create-cluster | +++ IMAGE_ROUTER=perconalab/percona-server-mysql-operator:main-router8.4 logger.go:42: 04:11:40 | gr-self-healing/2-create-cluster | +++ export IMAGE_TOOLKIT=perconalab/percona-server-mysql-operator:main-toolkit logger.go:42: 04:11:40 | gr-self-healing/2-create-cluster | +++ IMAGE_TOOLKIT=perconalab/percona-server-mysql-operator:main-toolkit logger.go:42: 04:11:40 | gr-self-healing/2-create-cluster | +++ export IMAGE_HAPROXY=perconalab/percona-server-mysql-operator:main-haproxy logger.go:42: 04:11:40 | gr-self-healing/2-create-cluster | +++ IMAGE_HAPROXY=perconalab/percona-server-mysql-operator:main-haproxy logger.go:42: 04:11:40 | gr-self-healing/2-create-cluster | +++ export PMM_SERVER_VERSION=1.4.3 logger.go:42: 04:11:40 | gr-self-healing/2-create-cluster | +++ PMM_SERVER_VERSION=1.4.3 logger.go:42: 04:11:40 | gr-self-healing/2-create-cluster | +++ export IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 04:11:40 | gr-self-healing/2-create-cluster | +++ IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 04:11:40 | gr-self-healing/2-create-cluster | +++ export IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 04:11:40 | gr-self-healing/2-create-cluster | +++ IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 04:11:40 | gr-self-healing/2-create-cluster | +++ export CERT_MANAGER_VER=1.18.2 logger.go:42: 04:11:40 | gr-self-healing/2-create-cluster | +++ CERT_MANAGER_VER=1.18.2 logger.go:42: 04:11:40 | gr-self-healing/2-create-cluster | +++ export MINIO_VER=5.4.0 logger.go:42: 04:11:40 | gr-self-healing/2-create-cluster | +++ MINIO_VER=5.4.0 logger.go:42: 04:11:40 | gr-self-healing/2-create-cluster | +++ export CHAOS_MESH_VER=2.7.2 logger.go:42: 04:11:40 | gr-self-healing/2-create-cluster | +++ CHAOS_MESH_VER=2.7.2 logger.go:42: 04:11:40 | gr-self-healing/2-create-cluster | +++ export VAULT_VER=0.16.1 logger.go:42: 04:11:40 | gr-self-healing/2-create-cluster | +++ VAULT_VER=0.16.1 logger.go:42: 04:11:40 | gr-self-healing/2-create-cluster | ++++ which gdate logger.go:42: 04:11:40 | gr-self-healing/2-create-cluster | which: no gdate in (/mnt/jenkins/workspace/cloud-ps-operator_PR-1108/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 04:11:40 | gr-self-healing/2-create-cluster | ++++ which date logger.go:42: 04:11:40 | gr-self-healing/2-create-cluster | +++ date=/usr/sbin/date logger.go:42: 04:11:40 | gr-self-healing/2-create-cluster | +++ oc get projects logger.go:42: 04:11:40 | gr-self-healing/2-create-cluster | +++ : logger.go:42: 04:11:40 | gr-self-healing/2-create-cluster | +++ kubectl get nodes logger.go:42: 04:11:40 | gr-self-healing/2-create-cluster | +++ grep '^minikube' logger.go:42: 04:11:40 | gr-self-healing/2-create-cluster | +++ which gsed logger.go:42: 04:11:40 | gr-self-healing/2-create-cluster | which: no gsed in (/mnt/jenkins/workspace/cloud-ps-operator_PR-1108/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 04:11:40 | gr-self-healing/2-create-cluster | +++ which sed logger.go:42: 04:11:40 | gr-self-healing/2-create-cluster | ++ sed=/usr/sbin/sed logger.go:42: 04:11:40 | gr-self-healing/2-create-cluster | ++ oc get projects logger.go:42: 04:11:40 | gr-self-healing/2-create-cluster | +++ kubectl version -o json logger.go:42: 04:11:40 | gr-self-healing/2-create-cluster | +++ jq -r .serverVersion.gitVersion logger.go:42: 04:11:40 | gr-self-healing/2-create-cluster | +++ grep '\-eks\-' logger.go:42: 04:11:40 | gr-self-healing/2-create-cluster | grep: warning: stray \ before - logger.go:42: 04:11:41 | gr-self-healing/2-create-cluster | Warning: version difference between client (1.34) and server (1.31) exceeds the supported minor version skew of +/-1 logger.go:42: 04:11:41 | gr-self-healing/2-create-cluster | ++ '[' ']' logger.go:42: 04:11:41 | gr-self-healing/2-create-cluster | ++ EKS=0 logger.go:42: 04:11:41 | gr-self-healing/2-create-cluster | + get_cr logger.go:42: 04:11:41 | gr-self-healing/2-create-cluster | + yq eval '.spec.mysql.clusterType="group-replication"' - logger.go:42: 04:11:41 | gr-self-healing/2-create-cluster | + local name_suffix= logger.go:42: 04:11:41 | gr-self-healing/2-create-cluster | + yq eval .spec.mysql.size=3 - logger.go:42: 04:11:41 | gr-self-healing/2-create-cluster | + yq eval '.spec.mysql.affinity.antiAffinityTopologyKey="none"' - logger.go:42: 04:11:41 | gr-self-healing/2-create-cluster | + yq eval .spec.proxy.haproxy.enabled=false - logger.go:42: 04:11:41 | gr-self-healing/2-create-cluster | + yq eval .spec.proxy.router.enabled=true - logger.go:42: 04:11:41 | gr-self-healing/2-create-cluster | + yq eval '.spec.secretsName="test-secrets"' - logger.go:42: 04:11:41 | gr-self-healing/2-create-cluster | ++ printf '.metadata.name="%s"' gr-self-healing logger.go:42: 04:11:41 | gr-self-healing/2-create-cluster | + yq eval .spec.orchestrator.enabled=false - logger.go:42: 04:11:41 | gr-self-healing/2-create-cluster | + yq eval '.spec.upgradeOptions.apply="disabled"' - logger.go:42: 04:11:41 | gr-self-healing/2-create-cluster | + yq eval .spec.proxy.router.size=3 - logger.go:42: 04:11:41 | gr-self-healing/2-create-cluster | + yq eval '.spec.sslSecretName="test-ssl"' - logger.go:42: 04:11:41 | gr-self-healing/2-create-cluster | + yq eval '.spec.mysql.clusterType="async"' - logger.go:42: 04:11:41 | gr-self-healing/2-create-cluster | ++ printf '.spec.initContainer.image="%s"' perconalab/percona-server-mysql-operator:PR-1108-c411fa10 logger.go:42: 04:11:41 | gr-self-healing/2-create-cluster | + kubectl -n kuttl-test-concrete-skylark apply -f - logger.go:42: 04:11:41 | gr-self-healing/2-create-cluster | + yq eval .spec.mysql.gracePeriod=30 - logger.go:42: 04:11:41 | gr-self-healing/2-create-cluster | + yq eval '.metadata.name="gr-self-healing"' /mnt/jenkins/workspace/cloud-ps-operator_PR-1108/deploy/cr.yaml logger.go:42: 04:11:41 | gr-self-healing/2-create-cluster | + yq eval .spec.orchestrator.enabled=true - logger.go:42: 04:11:41 | gr-self-healing/2-create-cluster | + yq eval '.spec.proxy.router.affinity.antiAffinityTopologyKey="none"' - logger.go:42: 04:11:41 | gr-self-healing/2-create-cluster | + yq eval '.spec.initContainer.image="perconalab/percona-server-mysql-operator:PR-1108-c411fa10"' - logger.go:42: 04:11:41 | gr-self-healing/2-create-cluster | ++ printf '.spec.pmm.image="%s"' perconalab/pmm-client:3-dev-latest logger.go:42: 04:11:41 | gr-self-healing/2-create-cluster | ++ printf '.spec.mysql.image="%s"' perconalab/percona-server-mysql-operator:main-psmysql8.4 logger.go:42: 04:11:41 | gr-self-healing/2-create-cluster | + '[' -n '' ']' logger.go:42: 04:11:41 | gr-self-healing/2-create-cluster | + yq eval - logger.go:42: 04:11:41 | gr-self-healing/2-create-cluster | ++ printf '.spec.backup.image="%s"' perconalab/percona-server-mysql-operator:main-backup8.4 logger.go:42: 04:11:41 | gr-self-healing/2-create-cluster | ++ printf '.spec.toolkit.image="%s"' perconalab/percona-server-mysql-operator:main-toolkit logger.go:42: 04:11:41 | gr-self-healing/2-create-cluster | ++ printf '.spec.proxy.haproxy.image="%s"' perconalab/percona-server-mysql-operator:main-haproxy logger.go:42: 04:11:41 | gr-self-healing/2-create-cluster | ++ printf '.spec.proxy.router.image="%s"' perconalab/percona-server-mysql-operator:main-router8.4 logger.go:42: 04:11:41 | gr-self-healing/2-create-cluster | ++ printf '.spec.orchestrator.image="%s"' perconalab/percona-server-mysql-operator:main-orchestrator logger.go:42: 04:11:41 | gr-self-healing/2-create-cluster | + yq eval '.spec.mysql.image="perconalab/percona-server-mysql-operator:main-psmysql8.4"' - logger.go:42: 04:11:41 | gr-self-healing/2-create-cluster | + yq eval '.spec.pmm.image="perconalab/pmm-client:3-dev-latest"' - logger.go:42: 04:11:41 | gr-self-healing/2-create-cluster | + yq eval '.spec.orchestrator.image="perconalab/percona-server-mysql-operator:main-orchestrator"' - logger.go:42: 04:11:41 | gr-self-healing/2-create-cluster | + yq eval '.spec.proxy.router.image="perconalab/percona-server-mysql-operator:main-router8.4"' - logger.go:42: 04:11:41 | gr-self-healing/2-create-cluster | + yq eval '.spec.backup.image="perconalab/percona-server-mysql-operator:main-backup8.4"' - logger.go:42: 04:11:41 | gr-self-healing/2-create-cluster | + yq eval '.spec.proxy.haproxy.image="perconalab/percona-server-mysql-operator:main-haproxy"' - logger.go:42: 04:11:41 | gr-self-healing/2-create-cluster | + yq eval '.spec.toolkit.image="perconalab/percona-server-mysql-operator:main-toolkit"' - logger.go:42: 04:11:42 | gr-self-healing/2-create-cluster | perconaservermysql.ps.percona.com/gr-self-healing created logger.go:42: 04:14:32 | gr-self-healing/2-create-cluster | test step completed 2-create-cluster logger.go:42: 04:14:32 | gr-self-healing/3-write-data | starting test step 3-write-data logger.go:42: 04:14:32 | gr-self-healing/3-write-data | running command: [sh -c set -o errexit set -o xtrace source ../../functions run_mysql \ "CREATE DATABASE IF NOT EXISTS myDB; CREATE TABLE IF NOT EXISTS myDB.myTable (id int PRIMARY KEY)" \ "-h $(get_mysql_router_service $(get_cluster_name)) -P 6446 -uroot -proot_password" run_mysql \ "INSERT myDB.myTable (id) VALUES (100500)" \ "-h $(get_mysql_router_service $(get_cluster_name)) -P 6446 -uroot -proot_password" sleep 5] logger.go:42: 04:14:32 | gr-self-healing/3-write-data | + source ../../functions logger.go:42: 04:14:32 | gr-self-healing/3-write-data | +++ realpath ../../.. logger.go:42: 04:14:33 | gr-self-healing/3-write-data | ++ ROOT_REPO=/mnt/jenkins/workspace/cloud-ps-operator_PR-1108 logger.go:42: 04:14:33 | gr-self-healing/3-write-data | ++++ pwd logger.go:42: 04:14:33 | gr-self-healing/3-write-data | +++ basename /mnt/jenkins/workspace/cloud-ps-operator_PR-1108/e2e-tests/tests/gr-self-healing logger.go:42: 04:14:33 | gr-self-healing/3-write-data | ++ test_name=gr-self-healing logger.go:42: 04:14:33 | gr-self-healing/3-write-data | ++ source /mnt/jenkins/workspace/cloud-ps-operator_PR-1108/e2e-tests/vars.sh logger.go:42: 04:14:33 | gr-self-healing/3-write-data | +++ export ROOT_REPO=/mnt/jenkins/workspace/cloud-ps-operator_PR-1108 logger.go:42: 04:14:33 | gr-self-healing/3-write-data | +++ ROOT_REPO=/mnt/jenkins/workspace/cloud-ps-operator_PR-1108 logger.go:42: 04:14:33 | gr-self-healing/3-write-data | +++ export DEPLOY_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-1108/deploy logger.go:42: 04:14:33 | gr-self-healing/3-write-data | +++ DEPLOY_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-1108/deploy logger.go:42: 04:14:33 | gr-self-healing/3-write-data | +++ export TESTS_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-1108/e2e-tests logger.go:42: 04:14:33 | gr-self-healing/3-write-data | +++ TESTS_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-1108/e2e-tests logger.go:42: 04:14:33 | gr-self-healing/3-write-data | +++ export TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-1108/e2e-tests/conf logger.go:42: 04:14:33 | gr-self-healing/3-write-data | +++ TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-1108/e2e-tests/conf logger.go:42: 04:14:33 | gr-self-healing/3-write-data | +++ export TEMP_DIR=/tmp/kuttl/ps/gr-self-healing logger.go:42: 04:14:33 | gr-self-healing/3-write-data | +++ TEMP_DIR=/tmp/kuttl/ps/gr-self-healing logger.go:42: 04:14:33 | gr-self-healing/3-write-data | ++++ git rev-parse --abbrev-ref HEAD logger.go:42: 04:14:33 | gr-self-healing/3-write-data | +++ export GIT_BRANCH=PR-1108 logger.go:42: 04:14:33 | gr-self-healing/3-write-data | +++ GIT_BRANCH=PR-1108 logger.go:42: 04:14:33 | gr-self-healing/3-write-data | +++ export VERSION=PR-1108-c411fa10 logger.go:42: 04:14:33 | gr-self-healing/3-write-data | +++ VERSION=PR-1108-c411fa10 logger.go:42: 04:14:33 | gr-self-healing/3-write-data | +++ export IMAGE=perconalab/percona-server-mysql-operator:PR-1108-c411fa10 logger.go:42: 04:14:33 | gr-self-healing/3-write-data | +++ IMAGE=perconalab/percona-server-mysql-operator:PR-1108-c411fa10 logger.go:42: 04:14:33 | gr-self-healing/3-write-data | +++ [[ -z '' ]] logger.go:42: 04:14:33 | gr-self-healing/3-write-data | +++ [[ -n '' ]] logger.go:42: 04:14:33 | gr-self-healing/3-write-data | +++ export MYSQL_VERSION=8.4 logger.go:42: 04:14:33 | gr-self-healing/3-write-data | +++ MYSQL_VERSION=8.4 logger.go:42: 04:14:33 | gr-self-healing/3-write-data | +++ export IMAGE_MYSQL=perconalab/percona-server-mysql-operator:main-psmysql8.4 logger.go:42: 04:14:33 | gr-self-healing/3-write-data | +++ IMAGE_MYSQL=perconalab/percona-server-mysql-operator:main-psmysql8.4 logger.go:42: 04:14:33 | gr-self-healing/3-write-data | +++ export IMAGE_BACKUP=perconalab/percona-server-mysql-operator:main-backup8.4 logger.go:42: 04:14:33 | gr-self-healing/3-write-data | +++ IMAGE_BACKUP=perconalab/percona-server-mysql-operator:main-backup8.4 logger.go:42: 04:14:33 | gr-self-healing/3-write-data | +++ export IMAGE_ORCHESTRATOR=perconalab/percona-server-mysql-operator:main-orchestrator logger.go:42: 04:14:33 | gr-self-healing/3-write-data | +++ IMAGE_ORCHESTRATOR=perconalab/percona-server-mysql-operator:main-orchestrator logger.go:42: 04:14:33 | gr-self-healing/3-write-data | +++ export IMAGE_ROUTER=perconalab/percona-server-mysql-operator:main-router8.4 logger.go:42: 04:14:33 | gr-self-healing/3-write-data | +++ IMAGE_ROUTER=perconalab/percona-server-mysql-operator:main-router8.4 logger.go:42: 04:14:33 | gr-self-healing/3-write-data | +++ export IMAGE_TOOLKIT=perconalab/percona-server-mysql-operator:main-toolkit logger.go:42: 04:14:33 | gr-self-healing/3-write-data | +++ IMAGE_TOOLKIT=perconalab/percona-server-mysql-operator:main-toolkit logger.go:42: 04:14:33 | gr-self-healing/3-write-data | +++ export IMAGE_HAPROXY=perconalab/percona-server-mysql-operator:main-haproxy logger.go:42: 04:14:33 | gr-self-healing/3-write-data | +++ IMAGE_HAPROXY=perconalab/percona-server-mysql-operator:main-haproxy logger.go:42: 04:14:33 | gr-self-healing/3-write-data | +++ export PMM_SERVER_VERSION=1.4.3 logger.go:42: 04:14:33 | gr-self-healing/3-write-data | +++ PMM_SERVER_VERSION=1.4.3 logger.go:42: 04:14:33 | gr-self-healing/3-write-data | +++ export IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 04:14:33 | gr-self-healing/3-write-data | +++ IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 04:14:33 | gr-self-healing/3-write-data | +++ export IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 04:14:33 | gr-self-healing/3-write-data | +++ IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 04:14:33 | gr-self-healing/3-write-data | +++ export CERT_MANAGER_VER=1.18.2 logger.go:42: 04:14:33 | gr-self-healing/3-write-data | +++ CERT_MANAGER_VER=1.18.2 logger.go:42: 04:14:33 | gr-self-healing/3-write-data | +++ export MINIO_VER=5.4.0 logger.go:42: 04:14:33 | gr-self-healing/3-write-data | +++ MINIO_VER=5.4.0 logger.go:42: 04:14:33 | gr-self-healing/3-write-data | +++ export CHAOS_MESH_VER=2.7.2 logger.go:42: 04:14:33 | gr-self-healing/3-write-data | +++ CHAOS_MESH_VER=2.7.2 logger.go:42: 04:14:33 | gr-self-healing/3-write-data | +++ export VAULT_VER=0.16.1 logger.go:42: 04:14:33 | gr-self-healing/3-write-data | +++ VAULT_VER=0.16.1 logger.go:42: 04:14:33 | gr-self-healing/3-write-data | ++++ which gdate logger.go:42: 04:14:33 | gr-self-healing/3-write-data | which: no gdate in (/mnt/jenkins/workspace/cloud-ps-operator_PR-1108/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 04:14:33 | gr-self-healing/3-write-data | ++++ which date logger.go:42: 04:14:33 | gr-self-healing/3-write-data | +++ date=/usr/sbin/date logger.go:42: 04:14:33 | gr-self-healing/3-write-data | +++ oc get projects logger.go:42: 04:14:33 | gr-self-healing/3-write-data | +++ : logger.go:42: 04:14:33 | gr-self-healing/3-write-data | +++ kubectl get nodes logger.go:42: 04:14:33 | gr-self-healing/3-write-data | +++ grep '^minikube' logger.go:42: 04:14:33 | gr-self-healing/3-write-data | +++ which gsed logger.go:42: 04:14:33 | gr-self-healing/3-write-data | which: no gsed in (/mnt/jenkins/workspace/cloud-ps-operator_PR-1108/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 04:14:33 | gr-self-healing/3-write-data | +++ which sed logger.go:42: 04:14:33 | gr-self-healing/3-write-data | ++ sed=/usr/sbin/sed logger.go:42: 04:14:33 | gr-self-healing/3-write-data | ++ oc get projects logger.go:42: 04:14:33 | gr-self-healing/3-write-data | +++ kubectl version -o json logger.go:42: 04:14:33 | gr-self-healing/3-write-data | +++ jq -r .serverVersion.gitVersion logger.go:42: 04:14:33 | gr-self-healing/3-write-data | +++ grep '\-eks\-' logger.go:42: 04:14:33 | gr-self-healing/3-write-data | grep: warning: stray \ before - logger.go:42: 04:14:34 | gr-self-healing/3-write-data | Warning: version difference between client (1.34) and server (1.31) exceeds the supported minor version skew of +/-1 logger.go:42: 04:14:34 | gr-self-healing/3-write-data | ++ '[' ']' logger.go:42: 04:14:34 | gr-self-healing/3-write-data | ++ EKS=0 logger.go:42: 04:14:34 | gr-self-healing/3-write-data | +++ get_cluster_name logger.go:42: 04:14:34 | gr-self-healing/3-write-data | +++ kubectl -n kuttl-test-concrete-skylark get ps -o 'jsonpath={.items[0].metadata.name}' logger.go:42: 04:14:34 | gr-self-healing/3-write-data | ++ get_mysql_router_service gr-self-healing logger.go:42: 04:14:34 | gr-self-healing/3-write-data | ++ local cluster=gr-self-healing logger.go:42: 04:14:34 | gr-self-healing/3-write-data | ++ echo gr-self-healing-router logger.go:42: 04:14:34 | gr-self-healing/3-write-data | + run_mysql 'CREATE DATABASE IF NOT EXISTS myDB; CREATE TABLE IF NOT EXISTS myDB.myTable (id int PRIMARY KEY)' '-h gr-self-healing-router -P 6446 -uroot -proot_password' logger.go:42: 04:14:34 | gr-self-healing/3-write-data | + local 'command=CREATE DATABASE IF NOT EXISTS myDB; CREATE TABLE IF NOT EXISTS myDB.myTable (id int PRIMARY KEY)' logger.go:42: 04:14:34 | gr-self-healing/3-write-data | + local 'uri=-h gr-self-healing-router -P 6446 -uroot -proot_password' logger.go:42: 04:14:34 | gr-self-healing/3-write-data | + local pod= logger.go:42: 04:14:34 | gr-self-healing/3-write-data | ++ get_client_pod logger.go:42: 04:14:34 | gr-self-healing/3-write-data | ++ kubectl -n kuttl-test-concrete-skylark get pods --selector=name=mysql-client -o 'jsonpath={.items[].metadata.name}' logger.go:42: 04:14:35 | gr-self-healing/3-write-data | + client_pod=mysql-client logger.go:42: 04:14:35 | gr-self-healing/3-write-data | + wait_pod mysql-client logger.go:42: 04:14:35 | gr-self-healing/3-write-data | + local pod=mysql-client logger.go:42: 04:14:35 | gr-self-healing/3-write-data | + set +o xtrace logger.go:42: 04:14:35 | gr-self-healing/3-write-data | mysql-clienttrue logger.go:42: 04:14:35 | gr-self-healing/3-write-data | + /usr/sbin/sed -e 's/mysql: //' logger.go:42: 04:14:35 | gr-self-healing/3-write-data | + kubectl -n kuttl-test-concrete-skylark exec mysql-client -- bash -c 'printf '\''%s\n'\'' "CREATE DATABASE IF NOT EXISTS myDB; CREATE TABLE IF NOT EXISTS myDB.myTable (id int PRIMARY KEY)" | mysql -sN -h gr-self-healing-router -P 6446 -uroot -proot_password' logger.go:42: 04:14:35 | gr-self-healing/3-write-data | + grep -v 'Using a password on the command line interface can be insecure.' logger.go:42: 04:14:36 | gr-self-healing/3-write-data | + : logger.go:42: 04:14:36 | gr-self-healing/3-write-data | +++ get_cluster_name logger.go:42: 04:14:36 | gr-self-healing/3-write-data | +++ kubectl -n kuttl-test-concrete-skylark get ps -o 'jsonpath={.items[0].metadata.name}' logger.go:42: 04:14:36 | gr-self-healing/3-write-data | ++ get_mysql_router_service gr-self-healing logger.go:42: 04:14:36 | gr-self-healing/3-write-data | ++ local cluster=gr-self-healing logger.go:42: 04:14:36 | gr-self-healing/3-write-data | ++ echo gr-self-healing-router logger.go:42: 04:14:36 | gr-self-healing/3-write-data | + run_mysql 'INSERT myDB.myTable (id) VALUES (100500)' '-h gr-self-healing-router -P 6446 -uroot -proot_password' logger.go:42: 04:14:36 | gr-self-healing/3-write-data | + local 'command=INSERT myDB.myTable (id) VALUES (100500)' logger.go:42: 04:14:36 | gr-self-healing/3-write-data | + local 'uri=-h gr-self-healing-router -P 6446 -uroot -proot_password' logger.go:42: 04:14:36 | gr-self-healing/3-write-data | + local pod= logger.go:42: 04:14:36 | gr-self-healing/3-write-data | ++ get_client_pod logger.go:42: 04:14:36 | gr-self-healing/3-write-data | ++ kubectl -n kuttl-test-concrete-skylark get pods --selector=name=mysql-client -o 'jsonpath={.items[].metadata.name}' logger.go:42: 04:14:37 | gr-self-healing/3-write-data | + client_pod=mysql-client logger.go:42: 04:14:37 | gr-self-healing/3-write-data | + wait_pod mysql-client logger.go:42: 04:14:37 | gr-self-healing/3-write-data | + local pod=mysql-client logger.go:42: 04:14:37 | gr-self-healing/3-write-data | + set +o xtrace logger.go:42: 04:14:38 | gr-self-healing/3-write-data | mysql-clienttrue logger.go:42: 04:14:38 | gr-self-healing/3-write-data | + kubectl -n kuttl-test-concrete-skylark exec mysql-client -- bash -c 'printf '\''%s\n'\'' "INSERT myDB.myTable (id) VALUES (100500)" | mysql -sN -h gr-self-healing-router -P 6446 -uroot -proot_password' logger.go:42: 04:14:38 | gr-self-healing/3-write-data | + /usr/sbin/sed -e 's/mysql: //' logger.go:42: 04:14:38 | gr-self-healing/3-write-data | + grep -v 'Using a password on the command line interface can be insecure.' logger.go:42: 04:14:39 | gr-self-healing/3-write-data | + : logger.go:42: 04:14:39 | gr-self-healing/3-write-data | + sleep 5 logger.go:42: 04:14:44 | gr-self-healing/3-write-data | test step completed 3-write-data logger.go:42: 04:14:44 | gr-self-healing/4-read-from-primary | starting test step 4-read-from-primary logger.go:42: 04:14:44 | gr-self-healing/4-read-from-primary | running command: [sh -c set -o errexit set -o xtrace source ../../functions data=$(run_mysql "SELECT * FROM myDB.myTable" "-h $(get_mysql_router_service $(get_cluster_name)) -P 6446 -uroot -proot_password") kubectl create configmap -n "${NAMESPACE}" 04-read-from-primary --from-literal=data="${data}"] logger.go:42: 04:14:44 | gr-self-healing/4-read-from-primary | + source ../../functions logger.go:42: 04:14:44 | gr-self-healing/4-read-from-primary | +++ realpath ../../.. logger.go:42: 04:14:44 | gr-self-healing/4-read-from-primary | ++ ROOT_REPO=/mnt/jenkins/workspace/cloud-ps-operator_PR-1108 logger.go:42: 04:14:44 | gr-self-healing/4-read-from-primary | ++++ pwd logger.go:42: 04:14:44 | gr-self-healing/4-read-from-primary | +++ basename /mnt/jenkins/workspace/cloud-ps-operator_PR-1108/e2e-tests/tests/gr-self-healing logger.go:42: 04:14:44 | gr-self-healing/4-read-from-primary | ++ test_name=gr-self-healing logger.go:42: 04:14:44 | gr-self-healing/4-read-from-primary | ++ source /mnt/jenkins/workspace/cloud-ps-operator_PR-1108/e2e-tests/vars.sh logger.go:42: 04:14:44 | gr-self-healing/4-read-from-primary | +++ export ROOT_REPO=/mnt/jenkins/workspace/cloud-ps-operator_PR-1108 logger.go:42: 04:14:44 | gr-self-healing/4-read-from-primary | +++ ROOT_REPO=/mnt/jenkins/workspace/cloud-ps-operator_PR-1108 logger.go:42: 04:14:44 | gr-self-healing/4-read-from-primary | +++ export DEPLOY_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-1108/deploy logger.go:42: 04:14:44 | gr-self-healing/4-read-from-primary | +++ DEPLOY_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-1108/deploy logger.go:42: 04:14:44 | gr-self-healing/4-read-from-primary | +++ export TESTS_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-1108/e2e-tests logger.go:42: 04:14:44 | gr-self-healing/4-read-from-primary | +++ TESTS_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-1108/e2e-tests logger.go:42: 04:14:44 | gr-self-healing/4-read-from-primary | +++ export TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-1108/e2e-tests/conf logger.go:42: 04:14:44 | gr-self-healing/4-read-from-primary | +++ TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-1108/e2e-tests/conf logger.go:42: 04:14:44 | gr-self-healing/4-read-from-primary | +++ export TEMP_DIR=/tmp/kuttl/ps/gr-self-healing logger.go:42: 04:14:44 | gr-self-healing/4-read-from-primary | +++ TEMP_DIR=/tmp/kuttl/ps/gr-self-healing logger.go:42: 04:14:44 | gr-self-healing/4-read-from-primary | ++++ git rev-parse --abbrev-ref HEAD logger.go:42: 04:14:44 | gr-self-healing/4-read-from-primary | +++ export GIT_BRANCH=PR-1108 logger.go:42: 04:14:44 | gr-self-healing/4-read-from-primary | +++ GIT_BRANCH=PR-1108 logger.go:42: 04:14:44 | gr-self-healing/4-read-from-primary | +++ export VERSION=PR-1108-c411fa10 logger.go:42: 04:14:44 | gr-self-healing/4-read-from-primary | +++ VERSION=PR-1108-c411fa10 logger.go:42: 04:14:44 | gr-self-healing/4-read-from-primary | +++ export IMAGE=perconalab/percona-server-mysql-operator:PR-1108-c411fa10 logger.go:42: 04:14:44 | gr-self-healing/4-read-from-primary | +++ IMAGE=perconalab/percona-server-mysql-operator:PR-1108-c411fa10 logger.go:42: 04:14:44 | gr-self-healing/4-read-from-primary | +++ [[ -z '' ]] logger.go:42: 04:14:44 | gr-self-healing/4-read-from-primary | +++ [[ -n '' ]] logger.go:42: 04:14:44 | gr-self-healing/4-read-from-primary | +++ export MYSQL_VERSION=8.4 logger.go:42: 04:14:44 | gr-self-healing/4-read-from-primary | +++ MYSQL_VERSION=8.4 logger.go:42: 04:14:44 | gr-self-healing/4-read-from-primary | +++ export IMAGE_MYSQL=perconalab/percona-server-mysql-operator:main-psmysql8.4 logger.go:42: 04:14:44 | gr-self-healing/4-read-from-primary | +++ IMAGE_MYSQL=perconalab/percona-server-mysql-operator:main-psmysql8.4 logger.go:42: 04:14:44 | gr-self-healing/4-read-from-primary | +++ export IMAGE_BACKUP=perconalab/percona-server-mysql-operator:main-backup8.4 logger.go:42: 04:14:44 | gr-self-healing/4-read-from-primary | +++ IMAGE_BACKUP=perconalab/percona-server-mysql-operator:main-backup8.4 logger.go:42: 04:14:44 | gr-self-healing/4-read-from-primary | +++ export IMAGE_ORCHESTRATOR=perconalab/percona-server-mysql-operator:main-orchestrator logger.go:42: 04:14:44 | gr-self-healing/4-read-from-primary | +++ IMAGE_ORCHESTRATOR=perconalab/percona-server-mysql-operator:main-orchestrator logger.go:42: 04:14:44 | gr-self-healing/4-read-from-primary | +++ export IMAGE_ROUTER=perconalab/percona-server-mysql-operator:main-router8.4 logger.go:42: 04:14:44 | gr-self-healing/4-read-from-primary | +++ IMAGE_ROUTER=perconalab/percona-server-mysql-operator:main-router8.4 logger.go:42: 04:14:44 | gr-self-healing/4-read-from-primary | +++ export IMAGE_TOOLKIT=perconalab/percona-server-mysql-operator:main-toolkit logger.go:42: 04:14:44 | gr-self-healing/4-read-from-primary | +++ IMAGE_TOOLKIT=perconalab/percona-server-mysql-operator:main-toolkit logger.go:42: 04:14:44 | gr-self-healing/4-read-from-primary | +++ export IMAGE_HAPROXY=perconalab/percona-server-mysql-operator:main-haproxy logger.go:42: 04:14:44 | gr-self-healing/4-read-from-primary | +++ IMAGE_HAPROXY=perconalab/percona-server-mysql-operator:main-haproxy logger.go:42: 04:14:44 | gr-self-healing/4-read-from-primary | +++ export PMM_SERVER_VERSION=1.4.3 logger.go:42: 04:14:44 | gr-self-healing/4-read-from-primary | +++ PMM_SERVER_VERSION=1.4.3 logger.go:42: 04:14:44 | gr-self-healing/4-read-from-primary | +++ export IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 04:14:44 | gr-self-healing/4-read-from-primary | +++ IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 04:14:44 | gr-self-healing/4-read-from-primary | +++ export IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 04:14:44 | gr-self-healing/4-read-from-primary | +++ IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 04:14:44 | gr-self-healing/4-read-from-primary | +++ export CERT_MANAGER_VER=1.18.2 logger.go:42: 04:14:44 | gr-self-healing/4-read-from-primary | +++ CERT_MANAGER_VER=1.18.2 logger.go:42: 04:14:44 | gr-self-healing/4-read-from-primary | +++ export MINIO_VER=5.4.0 logger.go:42: 04:14:44 | gr-self-healing/4-read-from-primary | +++ MINIO_VER=5.4.0 logger.go:42: 04:14:44 | gr-self-healing/4-read-from-primary | +++ export CHAOS_MESH_VER=2.7.2 logger.go:42: 04:14:44 | gr-self-healing/4-read-from-primary | +++ CHAOS_MESH_VER=2.7.2 logger.go:42: 04:14:44 | gr-self-healing/4-read-from-primary | +++ export VAULT_VER=0.16.1 logger.go:42: 04:14:44 | gr-self-healing/4-read-from-primary | +++ VAULT_VER=0.16.1 logger.go:42: 04:14:44 | gr-self-healing/4-read-from-primary | ++++ which gdate logger.go:42: 04:14:44 | gr-self-healing/4-read-from-primary | which: no gdate in (/mnt/jenkins/workspace/cloud-ps-operator_PR-1108/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 04:14:44 | gr-self-healing/4-read-from-primary | ++++ which date logger.go:42: 04:14:44 | gr-self-healing/4-read-from-primary | +++ date=/usr/sbin/date logger.go:42: 04:14:44 | gr-self-healing/4-read-from-primary | +++ oc get projects logger.go:42: 04:14:44 | gr-self-healing/4-read-from-primary | +++ : logger.go:42: 04:14:44 | gr-self-healing/4-read-from-primary | +++ kubectl get nodes logger.go:42: 04:14:44 | gr-self-healing/4-read-from-primary | +++ grep '^minikube' logger.go:42: 04:14:44 | gr-self-healing/4-read-from-primary | +++ which gsed logger.go:42: 04:14:44 | gr-self-healing/4-read-from-primary | which: no gsed in (/mnt/jenkins/workspace/cloud-ps-operator_PR-1108/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 04:14:44 | gr-self-healing/4-read-from-primary | +++ which sed logger.go:42: 04:14:44 | gr-self-healing/4-read-from-primary | ++ sed=/usr/sbin/sed logger.go:42: 04:14:44 | gr-self-healing/4-read-from-primary | ++ oc get projects logger.go:42: 04:14:44 | gr-self-healing/4-read-from-primary | +++ kubectl version -o json logger.go:42: 04:14:44 | gr-self-healing/4-read-from-primary | +++ jq -r .serverVersion.gitVersion logger.go:42: 04:14:44 | gr-self-healing/4-read-from-primary | +++ grep '\-eks\-' logger.go:42: 04:14:44 | gr-self-healing/4-read-from-primary | grep: warning: stray \ before - logger.go:42: 04:14:45 | gr-self-healing/4-read-from-primary | Warning: version difference between client (1.34) and server (1.31) exceeds the supported minor version skew of +/-1 logger.go:42: 04:14:45 | gr-self-healing/4-read-from-primary | ++ '[' ']' logger.go:42: 04:14:45 | gr-self-healing/4-read-from-primary | ++ EKS=0 logger.go:42: 04:14:45 | gr-self-healing/4-read-from-primary | ++++ get_cluster_name logger.go:42: 04:14:45 | gr-self-healing/4-read-from-primary | ++++ kubectl -n kuttl-test-concrete-skylark get ps -o 'jsonpath={.items[0].metadata.name}' logger.go:42: 04:14:45 | gr-self-healing/4-read-from-primary | +++ get_mysql_router_service gr-self-healing logger.go:42: 04:14:45 | gr-self-healing/4-read-from-primary | +++ local cluster=gr-self-healing logger.go:42: 04:14:45 | gr-self-healing/4-read-from-primary | +++ echo gr-self-healing-router logger.go:42: 04:14:45 | gr-self-healing/4-read-from-primary | ++ run_mysql 'SELECT * FROM myDB.myTable' '-h gr-self-healing-router -P 6446 -uroot -proot_password' logger.go:42: 04:14:45 | gr-self-healing/4-read-from-primary | ++ local 'command=SELECT * FROM myDB.myTable' logger.go:42: 04:14:45 | gr-self-healing/4-read-from-primary | ++ local 'uri=-h gr-self-healing-router -P 6446 -uroot -proot_password' logger.go:42: 04:14:45 | gr-self-healing/4-read-from-primary | ++ local pod= logger.go:42: 04:14:45 | gr-self-healing/4-read-from-primary | +++ get_client_pod logger.go:42: 04:14:45 | gr-self-healing/4-read-from-primary | +++ kubectl -n kuttl-test-concrete-skylark get pods --selector=name=mysql-client -o 'jsonpath={.items[].metadata.name}' logger.go:42: 04:14:46 | gr-self-healing/4-read-from-primary | ++ client_pod=mysql-client logger.go:42: 04:14:46 | gr-self-healing/4-read-from-primary | ++ wait_pod mysql-client logger.go:42: 04:14:46 | gr-self-healing/4-read-from-primary | ++ local pod=mysql-client logger.go:42: 04:14:46 | gr-self-healing/4-read-from-primary | ++ set +o xtrace logger.go:42: 04:14:46 | gr-self-healing/4-read-from-primary | mysql-clienttrue logger.go:42: 04:14:46 | gr-self-healing/4-read-from-primary | ++ kubectl -n kuttl-test-concrete-skylark exec mysql-client -- bash -c 'printf '\''%s\n'\'' "SELECT * FROM myDB.myTable" | mysql -sN -h gr-self-healing-router -P 6446 -uroot -proot_password' logger.go:42: 04:14:46 | gr-self-healing/4-read-from-primary | ++ /usr/sbin/sed -e 's/mysql: //' logger.go:42: 04:14:46 | gr-self-healing/4-read-from-primary | ++ grep -v 'Using a password on the command line interface can be insecure.' logger.go:42: 04:14:47 | gr-self-healing/4-read-from-primary | + data=100500 logger.go:42: 04:14:47 | gr-self-healing/4-read-from-primary | + kubectl create configmap -n kuttl-test-concrete-skylark 04-read-from-primary --from-literal=data=100500 logger.go:42: 04:14:47 | gr-self-healing/4-read-from-primary | configmap/04-read-from-primary created logger.go:42: 04:14:48 | gr-self-healing/4-read-from-primary | test step completed 4-read-from-primary logger.go:42: 04:14:48 | gr-self-healing/5-kill-primary | starting test step 5-kill-primary logger.go:42: 04:14:48 | gr-self-healing/5-kill-primary | running command: [sh -c set -o errexit set -o xtrace source ../../functions check_primary_chaos "kill" ${NAMESPACE} $(get_primary_from_group_replication)] logger.go:42: 04:14:48 | gr-self-healing/5-kill-primary | + source ../../functions logger.go:42: 04:14:48 | gr-self-healing/5-kill-primary | +++ realpath ../../.. logger.go:42: 04:14:48 | gr-self-healing/5-kill-primary | ++ ROOT_REPO=/mnt/jenkins/workspace/cloud-ps-operator_PR-1108 logger.go:42: 04:14:48 | gr-self-healing/5-kill-primary | ++++ pwd logger.go:42: 04:14:48 | gr-self-healing/5-kill-primary | +++ basename /mnt/jenkins/workspace/cloud-ps-operator_PR-1108/e2e-tests/tests/gr-self-healing logger.go:42: 04:14:48 | gr-self-healing/5-kill-primary | ++ test_name=gr-self-healing logger.go:42: 04:14:48 | gr-self-healing/5-kill-primary | ++ source /mnt/jenkins/workspace/cloud-ps-operator_PR-1108/e2e-tests/vars.sh logger.go:42: 04:14:48 | gr-self-healing/5-kill-primary | +++ export ROOT_REPO=/mnt/jenkins/workspace/cloud-ps-operator_PR-1108 logger.go:42: 04:14:48 | gr-self-healing/5-kill-primary | +++ ROOT_REPO=/mnt/jenkins/workspace/cloud-ps-operator_PR-1108 logger.go:42: 04:14:48 | gr-self-healing/5-kill-primary | +++ export DEPLOY_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-1108/deploy logger.go:42: 04:14:48 | gr-self-healing/5-kill-primary | +++ DEPLOY_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-1108/deploy logger.go:42: 04:14:48 | gr-self-healing/5-kill-primary | +++ export TESTS_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-1108/e2e-tests logger.go:42: 04:14:48 | gr-self-healing/5-kill-primary | +++ TESTS_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-1108/e2e-tests logger.go:42: 04:14:48 | gr-self-healing/5-kill-primary | +++ export TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-1108/e2e-tests/conf logger.go:42: 04:14:48 | gr-self-healing/5-kill-primary | +++ TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-1108/e2e-tests/conf logger.go:42: 04:14:48 | gr-self-healing/5-kill-primary | +++ export TEMP_DIR=/tmp/kuttl/ps/gr-self-healing logger.go:42: 04:14:48 | gr-self-healing/5-kill-primary | +++ TEMP_DIR=/tmp/kuttl/ps/gr-self-healing logger.go:42: 04:14:48 | gr-self-healing/5-kill-primary | ++++ git rev-parse --abbrev-ref HEAD logger.go:42: 04:14:48 | gr-self-healing/5-kill-primary | +++ export GIT_BRANCH=PR-1108 logger.go:42: 04:14:48 | gr-self-healing/5-kill-primary | +++ GIT_BRANCH=PR-1108 logger.go:42: 04:14:48 | gr-self-healing/5-kill-primary | +++ export VERSION=PR-1108-c411fa10 logger.go:42: 04:14:48 | gr-self-healing/5-kill-primary | +++ VERSION=PR-1108-c411fa10 logger.go:42: 04:14:48 | gr-self-healing/5-kill-primary | +++ export IMAGE=perconalab/percona-server-mysql-operator:PR-1108-c411fa10 logger.go:42: 04:14:48 | gr-self-healing/5-kill-primary | +++ IMAGE=perconalab/percona-server-mysql-operator:PR-1108-c411fa10 logger.go:42: 04:14:48 | gr-self-healing/5-kill-primary | +++ [[ -z '' ]] logger.go:42: 04:14:48 | gr-self-healing/5-kill-primary | +++ [[ -n '' ]] logger.go:42: 04:14:48 | gr-self-healing/5-kill-primary | +++ export MYSQL_VERSION=8.4 logger.go:42: 04:14:48 | gr-self-healing/5-kill-primary | +++ MYSQL_VERSION=8.4 logger.go:42: 04:14:48 | gr-self-healing/5-kill-primary | +++ export IMAGE_MYSQL=perconalab/percona-server-mysql-operator:main-psmysql8.4 logger.go:42: 04:14:48 | gr-self-healing/5-kill-primary | +++ IMAGE_MYSQL=perconalab/percona-server-mysql-operator:main-psmysql8.4 logger.go:42: 04:14:48 | gr-self-healing/5-kill-primary | +++ export IMAGE_BACKUP=perconalab/percona-server-mysql-operator:main-backup8.4 logger.go:42: 04:14:48 | gr-self-healing/5-kill-primary | +++ IMAGE_BACKUP=perconalab/percona-server-mysql-operator:main-backup8.4 logger.go:42: 04:14:48 | gr-self-healing/5-kill-primary | +++ export IMAGE_ORCHESTRATOR=perconalab/percona-server-mysql-operator:main-orchestrator logger.go:42: 04:14:48 | gr-self-healing/5-kill-primary | +++ IMAGE_ORCHESTRATOR=perconalab/percona-server-mysql-operator:main-orchestrator logger.go:42: 04:14:48 | gr-self-healing/5-kill-primary | +++ export IMAGE_ROUTER=perconalab/percona-server-mysql-operator:main-router8.4 logger.go:42: 04:14:48 | gr-self-healing/5-kill-primary | +++ IMAGE_ROUTER=perconalab/percona-server-mysql-operator:main-router8.4 logger.go:42: 04:14:48 | gr-self-healing/5-kill-primary | +++ export IMAGE_TOOLKIT=perconalab/percona-server-mysql-operator:main-toolkit logger.go:42: 04:14:48 | gr-self-healing/5-kill-primary | +++ IMAGE_TOOLKIT=perconalab/percona-server-mysql-operator:main-toolkit logger.go:42: 04:14:48 | gr-self-healing/5-kill-primary | +++ export IMAGE_HAPROXY=perconalab/percona-server-mysql-operator:main-haproxy logger.go:42: 04:14:48 | gr-self-healing/5-kill-primary | +++ IMAGE_HAPROXY=perconalab/percona-server-mysql-operator:main-haproxy logger.go:42: 04:14:48 | gr-self-healing/5-kill-primary | +++ export PMM_SERVER_VERSION=1.4.3 logger.go:42: 04:14:48 | gr-self-healing/5-kill-primary | +++ PMM_SERVER_VERSION=1.4.3 logger.go:42: 04:14:48 | gr-self-healing/5-kill-primary | +++ export IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 04:14:48 | gr-self-healing/5-kill-primary | +++ IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 04:14:48 | gr-self-healing/5-kill-primary | +++ export IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 04:14:48 | gr-self-healing/5-kill-primary | +++ IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 04:14:48 | gr-self-healing/5-kill-primary | +++ export CERT_MANAGER_VER=1.18.2 logger.go:42: 04:14:48 | gr-self-healing/5-kill-primary | +++ CERT_MANAGER_VER=1.18.2 logger.go:42: 04:14:48 | gr-self-healing/5-kill-primary | +++ export MINIO_VER=5.4.0 logger.go:42: 04:14:48 | gr-self-healing/5-kill-primary | +++ MINIO_VER=5.4.0 logger.go:42: 04:14:48 | gr-self-healing/5-kill-primary | +++ export CHAOS_MESH_VER=2.7.2 logger.go:42: 04:14:48 | gr-self-healing/5-kill-primary | +++ CHAOS_MESH_VER=2.7.2 logger.go:42: 04:14:48 | gr-self-healing/5-kill-primary | +++ export VAULT_VER=0.16.1 logger.go:42: 04:14:48 | gr-self-healing/5-kill-primary | +++ VAULT_VER=0.16.1 logger.go:42: 04:14:48 | gr-self-healing/5-kill-primary | ++++ which gdate logger.go:42: 04:14:48 | gr-self-healing/5-kill-primary | which: no gdate in (/mnt/jenkins/workspace/cloud-ps-operator_PR-1108/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 04:14:48 | gr-self-healing/5-kill-primary | ++++ which date logger.go:42: 04:14:48 | gr-self-healing/5-kill-primary | +++ date=/usr/sbin/date logger.go:42: 04:14:48 | gr-self-healing/5-kill-primary | +++ oc get projects logger.go:42: 04:14:48 | gr-self-healing/5-kill-primary | +++ : logger.go:42: 04:14:48 | gr-self-healing/5-kill-primary | +++ kubectl get nodes logger.go:42: 04:14:48 | gr-self-healing/5-kill-primary | +++ grep '^minikube' logger.go:42: 04:14:48 | gr-self-healing/5-kill-primary | +++ which gsed logger.go:42: 04:14:48 | gr-self-healing/5-kill-primary | which: no gsed in (/mnt/jenkins/workspace/cloud-ps-operator_PR-1108/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 04:14:48 | gr-self-healing/5-kill-primary | +++ which sed logger.go:42: 04:14:48 | gr-self-healing/5-kill-primary | ++ sed=/usr/sbin/sed logger.go:42: 04:14:48 | gr-self-healing/5-kill-primary | ++ oc get projects logger.go:42: 04:14:48 | gr-self-healing/5-kill-primary | +++ kubectl version -o json logger.go:42: 04:14:48 | gr-self-healing/5-kill-primary | +++ jq -r .serverVersion.gitVersion logger.go:42: 04:14:48 | gr-self-healing/5-kill-primary | +++ grep '\-eks\-' logger.go:42: 04:14:48 | gr-self-healing/5-kill-primary | grep: warning: stray \ before - logger.go:42: 04:14:49 | gr-self-healing/5-kill-primary | Warning: version difference between client (1.34) and server (1.31) exceeds the supported minor version skew of +/-1 logger.go:42: 04:14:49 | gr-self-healing/5-kill-primary | ++ '[' ']' logger.go:42: 04:14:49 | gr-self-healing/5-kill-primary | ++ EKS=0 logger.go:42: 04:14:49 | gr-self-healing/5-kill-primary | ++ get_primary_from_group_replication logger.go:42: 04:14:49 | gr-self-healing/5-kill-primary | ++ cut -d. -f1 logger.go:42: 04:14:49 | gr-self-healing/5-kill-primary | ++++ get_cluster_name logger.go:42: 04:14:49 | gr-self-healing/5-kill-primary | ++++ kubectl -n kuttl-test-concrete-skylark get ps -o 'jsonpath={.items[0].metadata.name}' logger.go:42: 04:14:49 | gr-self-healing/5-kill-primary | +++ get_mysql_router_service gr-self-healing logger.go:42: 04:14:49 | gr-self-healing/5-kill-primary | +++ local cluster=gr-self-healing logger.go:42: 04:14:49 | gr-self-healing/5-kill-primary | +++ echo gr-self-healing-router logger.go:42: 04:14:49 | gr-self-healing/5-kill-primary | ++ run_mysql 'SELECT MEMBER_HOST FROM performance_schema.replication_group_members where MEMBER_ROLE='\''PRIMARY'\'';' '-h gr-self-healing-router -P 6446 -uroot -proot_password' logger.go:42: 04:14:49 | gr-self-healing/5-kill-primary | ++ local 'command=SELECT MEMBER_HOST FROM performance_schema.replication_group_members where MEMBER_ROLE='\''PRIMARY'\'';' logger.go:42: 04:14:49 | gr-self-healing/5-kill-primary | ++ local 'uri=-h gr-self-healing-router -P 6446 -uroot -proot_password' logger.go:42: 04:14:49 | gr-self-healing/5-kill-primary | ++ local pod= logger.go:42: 04:14:49 | gr-self-healing/5-kill-primary | +++ get_client_pod logger.go:42: 04:14:49 | gr-self-healing/5-kill-primary | +++ kubectl -n kuttl-test-concrete-skylark get pods --selector=name=mysql-client -o 'jsonpath={.items[].metadata.name}' logger.go:42: 04:14:50 | gr-self-healing/5-kill-primary | ++ client_pod=mysql-client logger.go:42: 04:14:50 | gr-self-healing/5-kill-primary | ++ wait_pod mysql-client logger.go:42: 04:14:50 | gr-self-healing/5-kill-primary | ++ local pod=mysql-client logger.go:42: 04:14:50 | gr-self-healing/5-kill-primary | ++ set +o xtrace logger.go:42: 04:14:50 | gr-self-healing/5-kill-primary | mysql-clienttrue logger.go:42: 04:14:50 | gr-self-healing/5-kill-primary | ++ kubectl -n kuttl-test-concrete-skylark exec mysql-client -- bash -c 'printf '\''%s\n'\'' "SELECT MEMBER_HOST FROM performance_schema.replication_group_members where MEMBER_ROLE='\''PRIMARY'\'';" | mysql -sN -h gr-self-healing-router -P 6446 -uroot -proot_password' logger.go:42: 04:14:50 | gr-self-healing/5-kill-primary | ++ /usr/sbin/sed -e 's/mysql: //' logger.go:42: 04:14:50 | gr-self-healing/5-kill-primary | ++ grep -v 'Using a password on the command line interface can be insecure.' logger.go:42: 04:14:51 | gr-self-healing/5-kill-primary | + check_primary_chaos kill kuttl-test-concrete-skylark gr-self-healing-mysql-0 logger.go:42: 04:14:51 | gr-self-healing/5-kill-primary | + local chaos_type=kill logger.go:42: 04:14:51 | gr-self-healing/5-kill-primary | + local ns=kuttl-test-concrete-skylark logger.go:42: 04:14:51 | gr-self-healing/5-kill-primary | + local primary_before_failure=gr-self-healing-mysql-0 logger.go:42: 04:14:51 | gr-self-healing/5-kill-primary | + local chaos_name logger.go:42: 04:14:51 | gr-self-healing/5-kill-primary | + case ${chaos_type} in logger.go:42: 04:14:51 | gr-self-healing/5-kill-primary | + chaos_name=chaos-pod-kill-primary logger.go:42: 04:14:51 | gr-self-healing/5-kill-primary | + kill_pods kuttl-test-concrete-skylark pod gr-self-healing-mysql-0 '' chaos-pod-kill-primary logger.go:42: 04:14:51 | gr-self-healing/5-kill-primary | + local ns=kuttl-test-concrete-skylark logger.go:42: 04:14:51 | gr-self-healing/5-kill-primary | + local selector=pod logger.go:42: 04:14:51 | gr-self-healing/5-kill-primary | + local pod_label=gr-self-healing-mysql-0 logger.go:42: 04:14:51 | gr-self-healing/5-kill-primary | + local label_value= logger.go:42: 04:14:51 | gr-self-healing/5-kill-primary | + local chaos_name=chaos-pod-kill-primary logger.go:42: 04:14:51 | gr-self-healing/5-kill-primary | + '[' pod == pod ']' logger.go:42: 04:14:51 | gr-self-healing/5-kill-primary | + yq eval ' logger.go:42: 04:14:51 | gr-self-healing/5-kill-primary | .metadata.name = "chaos-pod-kill-primary" | logger.go:42: 04:14:51 | gr-self-healing/5-kill-primary | del(.spec.selector.pods.test-namespace) | logger.go:42: 04:14:51 | gr-self-healing/5-kill-primary | .spec.selector.pods.kuttl-test-concrete-skylark[0] = "gr-self-healing-mysql-0"' /mnt/jenkins/workspace/cloud-ps-operator_PR-1108/e2e-tests/conf/chaos-pod-kill.yml logger.go:42: 04:14:51 | gr-self-healing/5-kill-primary | + kubectl apply --namespace kuttl-test-concrete-skylark -f - logger.go:42: 04:14:53 | gr-self-healing/5-kill-primary | podchaos.chaos-mesh.org/chaos-pod-kill-primary created logger.go:42: 04:14:53 | gr-self-healing/5-kill-primary | + sleep 5 logger.go:42: 04:14:58 | gr-self-healing/5-kill-primary | + wait_until_chaos_applied kill chaos-pod-kill-primary logger.go:42: 04:14:58 | gr-self-healing/5-kill-primary | + local chaos_type=kill logger.go:42: 04:14:58 | gr-self-healing/5-kill-primary | + local chaos_name=chaos-pod-kill-primary logger.go:42: 04:14:58 | gr-self-healing/5-kill-primary | + local resource logger.go:42: 04:14:58 | gr-self-healing/5-kill-primary | + case ${chaos_type} in logger.go:42: 04:14:58 | gr-self-healing/5-kill-primary | + resource=podchaos/chaos-pod-kill-primary logger.go:42: 04:14:58 | gr-self-healing/5-kill-primary | + local retry=0 logger.go:42: 04:14:58 | gr-self-healing/5-kill-primary | + [[ 0 == 30 ]] logger.go:42: 04:14:58 | gr-self-healing/5-kill-primary | + sleep 10 logger.go:42: 04:15:08 | gr-self-healing/5-kill-primary | + retry=1 logger.go:42: 04:15:08 | gr-self-healing/5-kill-primary | ++ kubectl -n kuttl-test-concrete-skylark get podchaos/chaos-pod-kill-primary -o yaml logger.go:42: 04:15:08 | gr-self-healing/5-kill-primary | ++ yq '.status.experiment.containerRecords[].events[] logger.go:42: 04:15:08 | gr-self-healing/5-kill-primary | | select(.operation == "Apply" and .type == "Succeeded")' logger.go:42: 04:15:09 | gr-self-healing/5-kill-primary | + succeeded='operation: Apply logger.go:42: 04:15:09 | gr-self-healing/5-kill-primary | timestamp: "2025-10-01T04:14:53Z" logger.go:42: 04:15:09 | gr-self-healing/5-kill-primary | type: Succeeded' logger.go:42: 04:15:09 | gr-self-healing/5-kill-primary | + [[ -n operation: Apply logger.go:42: 04:15:09 | gr-self-healing/5-kill-primary | timestamp: "2025-10-01T04:14:53Z" logger.go:42: 04:15:09 | gr-self-healing/5-kill-primary | type: Succeeded ]] logger.go:42: 04:15:09 | gr-self-healing/5-kill-primary | + return logger.go:42: 04:15:09 | gr-self-healing/5-kill-primary | + [[ kill == \f\a\i\l\u\r\e ]] logger.go:42: 04:15:09 | gr-self-healing/5-kill-primary | + [[ kill == \n\e\t\w\o\r\k ]] logger.go:42: 04:15:09 | gr-self-healing/5-kill-primary | ++ get_cluster_name logger.go:42: 04:15:09 | gr-self-healing/5-kill-primary | ++ kubectl -n kuttl-test-concrete-skylark get ps -o 'jsonpath={.items[0].metadata.name}' logger.go:42: 04:15:09 | gr-self-healing/5-kill-primary | + wait_cluster_consistency_gr gr-self-healing 3 3 logger.go:42: 04:15:09 | gr-self-healing/5-kill-primary | + local cluster_name=gr-self-healing logger.go:42: 04:15:09 | gr-self-healing/5-kill-primary | + local cluster_size=3 logger.go:42: 04:15:09 | gr-self-healing/5-kill-primary | + local router_size=3 logger.go:42: 04:15:09 | gr-self-healing/5-kill-primary | + '[' -z 3 ']' logger.go:42: 04:15:09 | gr-self-healing/5-kill-primary | + sleep 7 logger.go:42: 04:15:16 | gr-self-healing/5-kill-primary | ++ kubectl get ps gr-self-healing -n kuttl-test-concrete-skylark -o 'jsonpath={.status.mysql.state}' logger.go:42: 04:15:17 | gr-self-healing/5-kill-primary | + [[ initializing == \r\e\a\d\y ]] logger.go:42: 04:15:17 | gr-self-healing/5-kill-primary | + echo 'waiting for cluster readyness (group replication)' logger.go:42: 04:15:17 | gr-self-healing/5-kill-primary | waiting for cluster readyness (group replication) logger.go:42: 04:15:17 | gr-self-healing/5-kill-primary | + sleep 15 logger.go:42: 04:15:32 | gr-self-healing/5-kill-primary | ++ kubectl get ps gr-self-healing -n kuttl-test-concrete-skylark -o 'jsonpath={.status.mysql.state}' logger.go:42: 04:15:32 | gr-self-healing/5-kill-primary | + [[ initializing == \r\e\a\d\y ]] logger.go:42: 04:15:32 | gr-self-healing/5-kill-primary | + echo 'waiting for cluster readyness (group replication)' logger.go:42: 04:15:32 | gr-self-healing/5-kill-primary | waiting for cluster readyness (group replication) logger.go:42: 04:15:32 | gr-self-healing/5-kill-primary | + sleep 15 logger.go:42: 04:15:47 | gr-self-healing/5-kill-primary | ++ kubectl get ps gr-self-healing -n kuttl-test-concrete-skylark -o 'jsonpath={.status.mysql.state}' logger.go:42: 04:15:48 | gr-self-healing/5-kill-primary | + [[ initializing == \r\e\a\d\y ]] logger.go:42: 04:15:48 | gr-self-healing/5-kill-primary | + echo 'waiting for cluster readyness (group replication)' logger.go:42: 04:15:48 | gr-self-healing/5-kill-primary | waiting for cluster readyness (group replication) logger.go:42: 04:15:48 | gr-self-healing/5-kill-primary | + sleep 15 logger.go:42: 04:16:03 | gr-self-healing/5-kill-primary | ++ kubectl get ps gr-self-healing -n kuttl-test-concrete-skylark -o 'jsonpath={.status.mysql.state}' logger.go:42: 04:16:03 | gr-self-healing/5-kill-primary | + [[ initializing == \r\e\a\d\y ]] logger.go:42: 04:16:03 | gr-self-healing/5-kill-primary | + echo 'waiting for cluster readyness (group replication)' logger.go:42: 04:16:03 | gr-self-healing/5-kill-primary | waiting for cluster readyness (group replication) logger.go:42: 04:16:03 | gr-self-healing/5-kill-primary | + sleep 15 logger.go:42: 04:16:18 | gr-self-healing/5-kill-primary | ++ kubectl get ps gr-self-healing -n kuttl-test-concrete-skylark -o 'jsonpath={.status.mysql.state}' logger.go:42: 04:16:19 | gr-self-healing/5-kill-primary | + [[ ready == \r\e\a\d\y ]] logger.go:42: 04:16:19 | gr-self-healing/5-kill-primary | ++ kubectl get ps gr-self-healing -n kuttl-test-concrete-skylark -o 'jsonpath={.status.mysql.ready}' logger.go:42: 04:16:20 | gr-self-healing/5-kill-primary | + [[ 3 == \3 ]] logger.go:42: 04:16:20 | gr-self-healing/5-kill-primary | ++ kubectl get ps gr-self-healing -n kuttl-test-concrete-skylark -o 'jsonpath={.status.router.ready}' logger.go:42: 04:16:20 | gr-self-healing/5-kill-primary | + [[ 3 == \3 ]] logger.go:42: 04:16:20 | gr-self-healing/5-kill-primary | ++ kubectl get ps gr-self-healing -n kuttl-test-concrete-skylark -o 'jsonpath={.status.router.state}' logger.go:42: 04:16:21 | gr-self-healing/5-kill-primary | + [[ ready == \r\e\a\d\y ]] logger.go:42: 04:16:21 | gr-self-healing/5-kill-primary | ++ kubectl get ps gr-self-healing -n kuttl-test-concrete-skylark -o 'jsonpath={.status.state}' logger.go:42: 04:16:21 | gr-self-healing/5-kill-primary | + [[ ready == \r\e\a\d\y ]] logger.go:42: 04:16:21 | gr-self-healing/5-kill-primary | ++ get_primary_from_group_replication logger.go:42: 04:16:21 | gr-self-healing/5-kill-primary | ++ cut -d. -f1 logger.go:42: 04:16:21 | gr-self-healing/5-kill-primary | ++++ get_cluster_name logger.go:42: 04:16:21 | gr-self-healing/5-kill-primary | ++++ kubectl -n kuttl-test-concrete-skylark get ps -o 'jsonpath={.items[0].metadata.name}' logger.go:42: 04:16:22 | gr-self-healing/5-kill-primary | +++ get_mysql_router_service gr-self-healing logger.go:42: 04:16:22 | gr-self-healing/5-kill-primary | +++ local cluster=gr-self-healing logger.go:42: 04:16:22 | gr-self-healing/5-kill-primary | +++ echo gr-self-healing-router logger.go:42: 04:16:22 | gr-self-healing/5-kill-primary | ++ run_mysql 'SELECT MEMBER_HOST FROM performance_schema.replication_group_members where MEMBER_ROLE='\''PRIMARY'\'';' '-h gr-self-healing-router -P 6446 -uroot -proot_password' logger.go:42: 04:16:22 | gr-self-healing/5-kill-primary | ++ local 'command=SELECT MEMBER_HOST FROM performance_schema.replication_group_members where MEMBER_ROLE='\''PRIMARY'\'';' logger.go:42: 04:16:22 | gr-self-healing/5-kill-primary | ++ local 'uri=-h gr-self-healing-router -P 6446 -uroot -proot_password' logger.go:42: 04:16:22 | gr-self-healing/5-kill-primary | ++ local pod= logger.go:42: 04:16:22 | gr-self-healing/5-kill-primary | +++ get_client_pod logger.go:42: 04:16:22 | gr-self-healing/5-kill-primary | +++ kubectl -n kuttl-test-concrete-skylark get pods --selector=name=mysql-client -o 'jsonpath={.items[].metadata.name}' logger.go:42: 04:16:23 | gr-self-healing/5-kill-primary | ++ client_pod=mysql-client logger.go:42: 04:16:23 | gr-self-healing/5-kill-primary | ++ wait_pod mysql-client logger.go:42: 04:16:23 | gr-self-healing/5-kill-primary | ++ local pod=mysql-client logger.go:42: 04:16:23 | gr-self-healing/5-kill-primary | ++ set +o xtrace logger.go:42: 04:16:23 | gr-self-healing/5-kill-primary | mysql-clienttrue logger.go:42: 04:16:23 | gr-self-healing/5-kill-primary | ++ kubectl -n kuttl-test-concrete-skylark exec mysql-client -- bash -c 'printf '\''%s\n'\'' "SELECT MEMBER_HOST FROM performance_schema.replication_group_members where MEMBER_ROLE='\''PRIMARY'\'';" | mysql -sN -h gr-self-healing-router -P 6446 -uroot -proot_password' logger.go:42: 04:16:23 | gr-self-healing/5-kill-primary | ++ /usr/sbin/sed -e 's/mysql: //' logger.go:42: 04:16:23 | gr-self-healing/5-kill-primary | ++ grep -v 'Using a password on the command line interface can be insecure.' logger.go:42: 04:16:24 | gr-self-healing/5-kill-primary | + primary_after_failure=gr-self-healing-mysql-2 logger.go:42: 04:16:24 | gr-self-healing/5-kill-primary | ++ get_mysqlsh_uri_for_pod gr-self-healing-mysql-2 logger.go:42: 04:16:24 | gr-self-healing/5-kill-primary | ++ local pod=gr-self-healing-mysql-2 logger.go:42: 04:16:24 | gr-self-healing/5-kill-primary | +++ get_cluster_name logger.go:42: 04:16:24 | gr-self-healing/5-kill-primary | +++ kubectl -n kuttl-test-concrete-skylark get ps -o 'jsonpath={.items[0].metadata.name}' logger.go:42: 04:16:24 | gr-self-healing/5-kill-primary | ++ echo root:root_password@gr-self-healing-mysql-2.gr-self-healing-mysql.kuttl-test-concrete-skylark logger.go:42: 04:16:24 | gr-self-healing/5-kill-primary | + uri=root:root_password@gr-self-healing-mysql-2.gr-self-healing-mysql.kuttl-test-concrete-skylark logger.go:42: 04:16:24 | gr-self-healing/5-kill-primary | + wait_until_innodb_ok root:root_password@gr-self-healing-mysql-2.gr-self-healing-mysql.kuttl-test-concrete-skylark logger.go:42: 04:16:24 | gr-self-healing/5-kill-primary | + local uri=root:root_password@gr-self-healing-mysql-2.gr-self-healing-mysql.kuttl-test-concrete-skylark logger.go:42: 04:16:24 | gr-self-healing/5-kill-primary | + local retry=0 logger.go:42: 04:16:24 | gr-self-healing/5-kill-primary | ++ get_innodb_cluster_status root:root_password@gr-self-healing-mysql-2.gr-self-healing-mysql.kuttl-test-concrete-skylark logger.go:42: 04:16:24 | gr-self-healing/5-kill-primary | ++ local uri=root:root_password@gr-self-healing-mysql-2.gr-self-healing-mysql.kuttl-test-concrete-skylark logger.go:42: 04:16:24 | gr-self-healing/5-kill-primary | ++ jq -r .defaultReplicaSet.status logger.go:42: 04:16:24 | gr-self-healing/5-kill-primary | +++ get_client_pod logger.go:42: 04:16:24 | gr-self-healing/5-kill-primary | +++ kubectl -n kuttl-test-concrete-skylark get pods --selector=name=mysql-client -o 'jsonpath={.items[].metadata.name}' logger.go:42: 04:16:25 | gr-self-healing/5-kill-primary | ++ client_pod=mysql-client logger.go:42: 04:16:25 | gr-self-healing/5-kill-primary | ++ wait_pod mysql-client logger.go:42: 04:16:25 | gr-self-healing/5-kill-primary | ++ local pod=mysql-client logger.go:42: 04:16:25 | gr-self-healing/5-kill-primary | ++ set +o xtrace logger.go:42: 04:16:25 | gr-self-healing/5-kill-primary | mysql-clienttrue logger.go:42: 04:16:25 | gr-self-healing/5-kill-primary | ++ kubectl -n kuttl-test-concrete-skylark exec mysql-client -- mysqlsh --js --quiet-start=2 --uri root:root_password@gr-self-healing-mysql-2.gr-self-healing-mysql.kuttl-test-concrete-skylark -- cluster status logger.go:42: 04:16:26 | gr-self-healing/5-kill-primary | Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory logger.go:42: 04:16:27 | gr-self-healing/5-kill-primary | + [[ OK == \O\K ]] logger.go:42: 04:16:27 | gr-self-healing/5-kill-primary | + [[ gr-self-healing-mysql-0 == \g\r\-\s\e\l\f\-\h\e\a\l\i\n\g\-\m\y\s\q\l\-\2 ]] logger.go:42: 04:16:27 | gr-self-healing/5-kill-primary | +++ get_primary_from_group_replication logger.go:42: 04:16:27 | gr-self-healing/5-kill-primary | +++ cut -d. -f1 logger.go:42: 04:16:27 | gr-self-healing/5-kill-primary | +++++ get_cluster_name logger.go:42: 04:16:27 | gr-self-healing/5-kill-primary | +++++ kubectl -n kuttl-test-concrete-skylark get ps -o 'jsonpath={.items[0].metadata.name}' logger.go:42: 04:16:28 | gr-self-healing/5-kill-primary | ++++ get_mysql_router_service gr-self-healing logger.go:42: 04:16:28 | gr-self-healing/5-kill-primary | ++++ local cluster=gr-self-healing logger.go:42: 04:16:28 | gr-self-healing/5-kill-primary | ++++ echo gr-self-healing-router logger.go:42: 04:16:28 | gr-self-healing/5-kill-primary | +++ run_mysql 'SELECT MEMBER_HOST FROM performance_schema.replication_group_members where MEMBER_ROLE='\''PRIMARY'\'';' '-h gr-self-healing-router -P 6446 -uroot -proot_password' logger.go:42: 04:16:28 | gr-self-healing/5-kill-primary | +++ local 'command=SELECT MEMBER_HOST FROM performance_schema.replication_group_members where MEMBER_ROLE='\''PRIMARY'\'';' logger.go:42: 04:16:28 | gr-self-healing/5-kill-primary | +++ local 'uri=-h gr-self-healing-router -P 6446 -uroot -proot_password' logger.go:42: 04:16:28 | gr-self-healing/5-kill-primary | +++ local pod= logger.go:42: 04:16:28 | gr-self-healing/5-kill-primary | ++++ get_client_pod logger.go:42: 04:16:28 | gr-self-healing/5-kill-primary | ++++ kubectl -n kuttl-test-concrete-skylark get pods --selector=name=mysql-client -o 'jsonpath={.items[].metadata.name}' logger.go:42: 04:16:28 | gr-self-healing/5-kill-primary | +++ client_pod=mysql-client logger.go:42: 04:16:28 | gr-self-healing/5-kill-primary | +++ wait_pod mysql-client logger.go:42: 04:16:28 | gr-self-healing/5-kill-primary | +++ local pod=mysql-client logger.go:42: 04:16:28 | gr-self-healing/5-kill-primary | +++ set +o xtrace logger.go:42: 04:16:28 | gr-self-healing/5-kill-primary | mysql-clienttrue logger.go:42: 04:16:28 | gr-self-healing/5-kill-primary | +++ kubectl -n kuttl-test-concrete-skylark exec mysql-client -- bash -c 'printf '\''%s\n'\'' "SELECT MEMBER_HOST FROM performance_schema.replication_group_members where MEMBER_ROLE='\''PRIMARY'\'';" | mysql -sN -h gr-self-healing-router -P 6446 -uroot -proot_password' logger.go:42: 04:16:28 | gr-self-healing/5-kill-primary | +++ /usr/sbin/sed -e 's/mysql: //' logger.go:42: 04:16:28 | gr-self-healing/5-kill-primary | +++ grep -v 'Using a password on the command line interface can be insecure.' logger.go:42: 04:16:29 | gr-self-healing/5-kill-primary | ++ get_mysqlsh_uri_for_pod gr-self-healing-mysql-2 logger.go:42: 04:16:29 | gr-self-healing/5-kill-primary | ++ local pod=gr-self-healing-mysql-2 logger.go:42: 04:16:29 | gr-self-healing/5-kill-primary | +++ get_cluster_name logger.go:42: 04:16:29 | gr-self-healing/5-kill-primary | +++ kubectl -n kuttl-test-concrete-skylark get ps -o 'jsonpath={.items[0].metadata.name}' logger.go:42: 04:16:30 | gr-self-healing/5-kill-primary | ++ echo root:root_password@gr-self-healing-mysql-2.gr-self-healing-mysql.kuttl-test-concrete-skylark logger.go:42: 04:16:30 | gr-self-healing/5-kill-primary | + uri=root:root_password@gr-self-healing-mysql-2.gr-self-healing-mysql.kuttl-test-concrete-skylark logger.go:42: 04:16:30 | gr-self-healing/5-kill-primary | ++ get_innodb_cluster_status root:root_password@gr-self-healing-mysql-2.gr-self-healing-mysql.kuttl-test-concrete-skylark logger.go:42: 04:16:30 | gr-self-healing/5-kill-primary | ++ jq '.defaultReplicaSet.topology[].status' logger.go:42: 04:16:30 | gr-self-healing/5-kill-primary | ++ local uri=root:root_password@gr-self-healing-mysql-2.gr-self-healing-mysql.kuttl-test-concrete-skylark logger.go:42: 04:16:30 | gr-self-healing/5-kill-primary | ++ grep ONLINE logger.go:42: 04:16:30 | gr-self-healing/5-kill-primary | ++ wc -l logger.go:42: 04:16:30 | gr-self-healing/5-kill-primary | ++ /usr/sbin/sed 's/ //g' logger.go:42: 04:16:30 | gr-self-healing/5-kill-primary | +++ get_client_pod logger.go:42: 04:16:30 | gr-self-healing/5-kill-primary | +++ kubectl -n kuttl-test-concrete-skylark get pods --selector=name=mysql-client -o 'jsonpath={.items[].metadata.name}' logger.go:42: 04:16:30 | gr-self-healing/5-kill-primary | ++ client_pod=mysql-client logger.go:42: 04:16:30 | gr-self-healing/5-kill-primary | ++ wait_pod mysql-client logger.go:42: 04:16:30 | gr-self-healing/5-kill-primary | ++ local pod=mysql-client logger.go:42: 04:16:30 | gr-self-healing/5-kill-primary | ++ set +o xtrace logger.go:42: 04:16:31 | gr-self-healing/5-kill-primary | mysql-clienttrue logger.go:42: 04:16:31 | gr-self-healing/5-kill-primary | ++ kubectl -n kuttl-test-concrete-skylark exec mysql-client -- mysqlsh --js --quiet-start=2 --uri root:root_password@gr-self-healing-mysql-2.gr-self-healing-mysql.kuttl-test-concrete-skylark -- cluster status logger.go:42: 04:16:32 | gr-self-healing/5-kill-primary | Cannot set LC_ALL to locale en_US.UTF-8: No such file or directory logger.go:42: 04:16:33 | gr-self-healing/5-kill-primary | + online_members=3 logger.go:42: 04:16:33 | gr-self-healing/5-kill-primary | + [[ 3 != 3 ]] logger.go:42: 04:16:35 | gr-self-healing/5-kill-primary | test step completed 5-kill-primary logger.go:42: 04:16:35 | gr-self-healing/6-write-data | starting test step 6-write-data logger.go:42: 04:16:35 | gr-self-healing/6-write-data | running command: [sh -c set -o errexit set -o xtrace source ../../functions run_mysql \ "INSERT myDB.myTable (id) VALUES (100501)" \ "-h $(get_mysql_router_service $(get_cluster_name)) -P 6446 -uroot -proot_password" sleep 5] logger.go:42: 04:16:35 | gr-self-healing/6-write-data | + source ../../functions logger.go:42: 04:16:35 | gr-self-healing/6-write-data | +++ realpath ../../.. logger.go:42: 04:16:35 | gr-self-healing/6-write-data | ++ ROOT_REPO=/mnt/jenkins/workspace/cloud-ps-operator_PR-1108 logger.go:42: 04:16:35 | gr-self-healing/6-write-data | ++++ pwd logger.go:42: 04:16:35 | gr-self-healing/6-write-data | +++ basename /mnt/jenkins/workspace/cloud-ps-operator_PR-1108/e2e-tests/tests/gr-self-healing logger.go:42: 04:16:35 | gr-self-healing/6-write-data | ++ test_name=gr-self-healing logger.go:42: 04:16:35 | gr-self-healing/6-write-data | ++ source /mnt/jenkins/workspace/cloud-ps-operator_PR-1108/e2e-tests/vars.sh logger.go:42: 04:16:35 | gr-self-healing/6-write-data | +++ export ROOT_REPO=/mnt/jenkins/workspace/cloud-ps-operator_PR-1108 logger.go:42: 04:16:35 | gr-self-healing/6-write-data | +++ ROOT_REPO=/mnt/jenkins/workspace/cloud-ps-operator_PR-1108 logger.go:42: 04:16:35 | gr-self-healing/6-write-data | +++ export DEPLOY_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-1108/deploy logger.go:42: 04:16:35 | gr-self-healing/6-write-data | +++ DEPLOY_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-1108/deploy logger.go:42: 04:16:35 | gr-self-healing/6-write-data | +++ export TESTS_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-1108/e2e-tests logger.go:42: 04:16:35 | gr-self-healing/6-write-data | +++ TESTS_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-1108/e2e-tests logger.go:42: 04:16:35 | gr-self-healing/6-write-data | +++ export TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-1108/e2e-tests/conf logger.go:42: 04:16:35 | gr-self-healing/6-write-data | +++ TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-1108/e2e-tests/conf logger.go:42: 04:16:35 | gr-self-healing/6-write-data | +++ export TEMP_DIR=/tmp/kuttl/ps/gr-self-healing logger.go:42: 04:16:35 | gr-self-healing/6-write-data | +++ TEMP_DIR=/tmp/kuttl/ps/gr-self-healing logger.go:42: 04:16:35 | gr-self-healing/6-write-data | ++++ git rev-parse --abbrev-ref HEAD logger.go:42: 04:16:35 | gr-self-healing/6-write-data | +++ export GIT_BRANCH=PR-1108 logger.go:42: 04:16:35 | gr-self-healing/6-write-data | +++ GIT_BRANCH=PR-1108 logger.go:42: 04:16:35 | gr-self-healing/6-write-data | +++ export VERSION=PR-1108-c411fa10 logger.go:42: 04:16:35 | gr-self-healing/6-write-data | +++ VERSION=PR-1108-c411fa10 logger.go:42: 04:16:35 | gr-self-healing/6-write-data | +++ export IMAGE=perconalab/percona-server-mysql-operator:PR-1108-c411fa10 logger.go:42: 04:16:35 | gr-self-healing/6-write-data | +++ IMAGE=perconalab/percona-server-mysql-operator:PR-1108-c411fa10 logger.go:42: 04:16:35 | gr-self-healing/6-write-data | +++ [[ -z '' ]] logger.go:42: 04:16:35 | gr-self-healing/6-write-data | +++ [[ -n '' ]] logger.go:42: 04:16:35 | gr-self-healing/6-write-data | +++ export MYSQL_VERSION=8.4 logger.go:42: 04:16:35 | gr-self-healing/6-write-data | +++ MYSQL_VERSION=8.4 logger.go:42: 04:16:35 | gr-self-healing/6-write-data | +++ export IMAGE_MYSQL=perconalab/percona-server-mysql-operator:main-psmysql8.4 logger.go:42: 04:16:35 | gr-self-healing/6-write-data | +++ IMAGE_MYSQL=perconalab/percona-server-mysql-operator:main-psmysql8.4 logger.go:42: 04:16:35 | gr-self-healing/6-write-data | +++ export IMAGE_BACKUP=perconalab/percona-server-mysql-operator:main-backup8.4 logger.go:42: 04:16:35 | gr-self-healing/6-write-data | +++ IMAGE_BACKUP=perconalab/percona-server-mysql-operator:main-backup8.4 logger.go:42: 04:16:35 | gr-self-healing/6-write-data | +++ export IMAGE_ORCHESTRATOR=perconalab/percona-server-mysql-operator:main-orchestrator logger.go:42: 04:16:35 | gr-self-healing/6-write-data | +++ IMAGE_ORCHESTRATOR=perconalab/percona-server-mysql-operator:main-orchestrator logger.go:42: 04:16:35 | gr-self-healing/6-write-data | +++ export IMAGE_ROUTER=perconalab/percona-server-mysql-operator:main-router8.4 logger.go:42: 04:16:35 | gr-self-healing/6-write-data | +++ IMAGE_ROUTER=perconalab/percona-server-mysql-operator:main-router8.4 logger.go:42: 04:16:35 | gr-self-healing/6-write-data | +++ export IMAGE_TOOLKIT=perconalab/percona-server-mysql-operator:main-toolkit logger.go:42: 04:16:35 | gr-self-healing/6-write-data | +++ IMAGE_TOOLKIT=perconalab/percona-server-mysql-operator:main-toolkit logger.go:42: 04:16:35 | gr-self-healing/6-write-data | +++ export IMAGE_HAPROXY=perconalab/percona-server-mysql-operator:main-haproxy logger.go:42: 04:16:35 | gr-self-healing/6-write-data | +++ IMAGE_HAPROXY=perconalab/percona-server-mysql-operator:main-haproxy logger.go:42: 04:16:35 | gr-self-healing/6-write-data | +++ export PMM_SERVER_VERSION=1.4.3 logger.go:42: 04:16:35 | gr-self-healing/6-write-data | +++ PMM_SERVER_VERSION=1.4.3 logger.go:42: 04:16:35 | gr-self-healing/6-write-data | +++ export IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 04:16:35 | gr-self-healing/6-write-data | +++ IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 04:16:35 | gr-self-healing/6-write-data | +++ export IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 04:16:35 | gr-self-healing/6-write-data | +++ IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 04:16:35 | gr-self-healing/6-write-data | +++ export CERT_MANAGER_VER=1.18.2 logger.go:42: 04:16:35 | gr-self-healing/6-write-data | +++ CERT_MANAGER_VER=1.18.2 logger.go:42: 04:16:35 | gr-self-healing/6-write-data | +++ export MINIO_VER=5.4.0 logger.go:42: 04:16:35 | gr-self-healing/6-write-data | +++ MINIO_VER=5.4.0 logger.go:42: 04:16:35 | gr-self-healing/6-write-data | +++ export CHAOS_MESH_VER=2.7.2 logger.go:42: 04:16:35 | gr-self-healing/6-write-data | +++ CHAOS_MESH_VER=2.7.2 logger.go:42: 04:16:35 | gr-self-healing/6-write-data | +++ export VAULT_VER=0.16.1 logger.go:42: 04:16:35 | gr-self-healing/6-write-data | +++ VAULT_VER=0.16.1 logger.go:42: 04:16:35 | gr-self-healing/6-write-data | ++++ which gdate logger.go:42: 04:16:35 | gr-self-healing/6-write-data | which: no gdate in (/mnt/jenkins/workspace/cloud-ps-operator_PR-1108/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 04:16:35 | gr-self-healing/6-write-data | ++++ which date logger.go:42: 04:16:35 | gr-self-healing/6-write-data | +++ date=/usr/sbin/date logger.go:42: 04:16:35 | gr-self-healing/6-write-data | +++ oc get projects logger.go:42: 04:16:35 | gr-self-healing/6-write-data | +++ : logger.go:42: 04:16:35 | gr-self-healing/6-write-data | +++ kubectl get nodes logger.go:42: 04:16:35 | gr-self-healing/6-write-data | +++ grep '^minikube' logger.go:42: 04:16:35 | gr-self-healing/6-write-data | +++ which gsed logger.go:42: 04:16:35 | gr-self-healing/6-write-data | which: no gsed in (/mnt/jenkins/workspace/cloud-ps-operator_PR-1108/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 04:16:35 | gr-self-healing/6-write-data | +++ which sed logger.go:42: 04:16:35 | gr-self-healing/6-write-data | ++ sed=/usr/sbin/sed logger.go:42: 04:16:35 | gr-self-healing/6-write-data | ++ oc get projects logger.go:42: 04:16:35 | gr-self-healing/6-write-data | +++ kubectl version -o json logger.go:42: 04:16:35 | gr-self-healing/6-write-data | +++ jq -r .serverVersion.gitVersion logger.go:42: 04:16:35 | gr-self-healing/6-write-data | +++ grep '\-eks\-' logger.go:42: 04:16:35 | gr-self-healing/6-write-data | grep: warning: stray \ before - logger.go:42: 04:16:36 | gr-self-healing/6-write-data | Warning: version difference between client (1.34) and server (1.31) exceeds the supported minor version skew of +/-1 logger.go:42: 04:16:36 | gr-self-healing/6-write-data | ++ '[' ']' logger.go:42: 04:16:36 | gr-self-healing/6-write-data | ++ EKS=0 logger.go:42: 04:16:36 | gr-self-healing/6-write-data | +++ get_cluster_name logger.go:42: 04:16:36 | gr-self-healing/6-write-data | +++ kubectl -n kuttl-test-concrete-skylark get ps -o 'jsonpath={.items[0].metadata.name}' logger.go:42: 04:16:36 | gr-self-healing/6-write-data | ++ get_mysql_router_service gr-self-healing logger.go:42: 04:16:36 | gr-self-healing/6-write-data | ++ local cluster=gr-self-healing logger.go:42: 04:16:36 | gr-self-healing/6-write-data | ++ echo gr-self-healing-router logger.go:42: 04:16:36 | gr-self-healing/6-write-data | + run_mysql 'INSERT myDB.myTable (id) VALUES (100501)' '-h gr-self-healing-router -P 6446 -uroot -proot_password' logger.go:42: 04:16:36 | gr-self-healing/6-write-data | + local 'command=INSERT myDB.myTable (id) VALUES (100501)' logger.go:42: 04:16:36 | gr-self-healing/6-write-data | + local 'uri=-h gr-self-healing-router -P 6446 -uroot -proot_password' logger.go:42: 04:16:36 | gr-self-healing/6-write-data | + local pod= logger.go:42: 04:16:36 | gr-self-healing/6-write-data | ++ get_client_pod logger.go:42: 04:16:36 | gr-self-healing/6-write-data | ++ kubectl -n kuttl-test-concrete-skylark get pods --selector=name=mysql-client -o 'jsonpath={.items[].metadata.name}' logger.go:42: 04:16:37 | gr-self-healing/6-write-data | + client_pod=mysql-client logger.go:42: 04:16:37 | gr-self-healing/6-write-data | + wait_pod mysql-client logger.go:42: 04:16:37 | gr-self-healing/6-write-data | + local pod=mysql-client logger.go:42: 04:16:37 | gr-self-healing/6-write-data | + set +o xtrace logger.go:42: 04:16:37 | gr-self-healing/6-write-data | mysql-clienttrue logger.go:42: 04:16:37 | gr-self-healing/6-write-data | + kubectl -n kuttl-test-concrete-skylark exec mysql-client -- bash -c 'printf '\''%s\n'\'' "INSERT myDB.myTable (id) VALUES (100501)" | mysql -sN -h gr-self-healing-router -P 6446 -uroot -proot_password' logger.go:42: 04:16:37 | gr-self-healing/6-write-data | + /usr/sbin/sed -e 's/mysql: //' logger.go:42: 04:16:37 | gr-self-healing/6-write-data | + grep -v 'Using a password on the command line interface can be insecure.' logger.go:42: 04:16:38 | gr-self-healing/6-write-data | + : logger.go:42: 04:16:38 | gr-self-healing/6-write-data | + sleep 5 logger.go:42: 04:16:43 | gr-self-healing/6-write-data | test step completed 6-write-data logger.go:42: 04:16:43 | gr-self-healing/7-read-from-replicas | starting test step 7-read-from-replicas logger.go:42: 04:16:43 | gr-self-healing/7-read-from-replicas | running command: [sh -c set -o errexit set -o xtrace source ../../functions for i in 0 1 2; do host=$(get_mysql_headless_fqdn $(get_cluster_name) $i) data=$(run_mysql "SELECT * FROM myDB.myTable" "-h ${host} -uroot -proot_password") kubectl create configmap -n "${NAMESPACE}" 07-read-from-replicas-${i} --from-literal=data="${data}" done] logger.go:42: 04:16:43 | gr-self-healing/7-read-from-replicas | + source ../../functions logger.go:42: 04:16:43 | gr-self-healing/7-read-from-replicas | +++ realpath ../../.. logger.go:42: 04:16:43 | gr-self-healing/7-read-from-replicas | ++ ROOT_REPO=/mnt/jenkins/workspace/cloud-ps-operator_PR-1108 logger.go:42: 04:16:43 | gr-self-healing/7-read-from-replicas | ++++ pwd logger.go:42: 04:16:43 | gr-self-healing/7-read-from-replicas | +++ basename /mnt/jenkins/workspace/cloud-ps-operator_PR-1108/e2e-tests/tests/gr-self-healing logger.go:42: 04:16:43 | gr-self-healing/7-read-from-replicas | ++ test_name=gr-self-healing logger.go:42: 04:16:43 | gr-self-healing/7-read-from-replicas | ++ source /mnt/jenkins/workspace/cloud-ps-operator_PR-1108/e2e-tests/vars.sh logger.go:42: 04:16:43 | gr-self-healing/7-read-from-replicas | +++ export ROOT_REPO=/mnt/jenkins/workspace/cloud-ps-operator_PR-1108 logger.go:42: 04:16:43 | gr-self-healing/7-read-from-replicas | +++ ROOT_REPO=/mnt/jenkins/workspace/cloud-ps-operator_PR-1108 logger.go:42: 04:16:43 | gr-self-healing/7-read-from-replicas | +++ export DEPLOY_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-1108/deploy logger.go:42: 04:16:43 | gr-self-healing/7-read-from-replicas | +++ DEPLOY_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-1108/deploy logger.go:42: 04:16:43 | gr-self-healing/7-read-from-replicas | +++ export TESTS_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-1108/e2e-tests logger.go:42: 04:16:43 | gr-self-healing/7-read-from-replicas | +++ TESTS_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-1108/e2e-tests logger.go:42: 04:16:43 | gr-self-healing/7-read-from-replicas | +++ export TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-1108/e2e-tests/conf logger.go:42: 04:16:43 | gr-self-healing/7-read-from-replicas | +++ TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-1108/e2e-tests/conf logger.go:42: 04:16:43 | gr-self-healing/7-read-from-replicas | +++ export TEMP_DIR=/tmp/kuttl/ps/gr-self-healing logger.go:42: 04:16:43 | gr-self-healing/7-read-from-replicas | +++ TEMP_DIR=/tmp/kuttl/ps/gr-self-healing logger.go:42: 04:16:43 | gr-self-healing/7-read-from-replicas | ++++ git rev-parse --abbrev-ref HEAD logger.go:42: 04:16:43 | gr-self-healing/7-read-from-replicas | +++ export GIT_BRANCH=PR-1108 logger.go:42: 04:16:43 | gr-self-healing/7-read-from-replicas | +++ GIT_BRANCH=PR-1108 logger.go:42: 04:16:43 | gr-self-healing/7-read-from-replicas | +++ export VERSION=PR-1108-c411fa10 logger.go:42: 04:16:43 | gr-self-healing/7-read-from-replicas | +++ VERSION=PR-1108-c411fa10 logger.go:42: 04:16:43 | gr-self-healing/7-read-from-replicas | +++ export IMAGE=perconalab/percona-server-mysql-operator:PR-1108-c411fa10 logger.go:42: 04:16:43 | gr-self-healing/7-read-from-replicas | +++ IMAGE=perconalab/percona-server-mysql-operator:PR-1108-c411fa10 logger.go:42: 04:16:43 | gr-self-healing/7-read-from-replicas | +++ [[ -z '' ]] logger.go:42: 04:16:43 | gr-self-healing/7-read-from-replicas | +++ [[ -n '' ]] logger.go:42: 04:16:43 | gr-self-healing/7-read-from-replicas | +++ export MYSQL_VERSION=8.4 logger.go:42: 04:16:43 | gr-self-healing/7-read-from-replicas | +++ MYSQL_VERSION=8.4 logger.go:42: 04:16:43 | gr-self-healing/7-read-from-replicas | +++ export IMAGE_MYSQL=perconalab/percona-server-mysql-operator:main-psmysql8.4 logger.go:42: 04:16:43 | gr-self-healing/7-read-from-replicas | +++ IMAGE_MYSQL=perconalab/percona-server-mysql-operator:main-psmysql8.4 logger.go:42: 04:16:43 | gr-self-healing/7-read-from-replicas | +++ export IMAGE_BACKUP=perconalab/percona-server-mysql-operator:main-backup8.4 logger.go:42: 04:16:43 | gr-self-healing/7-read-from-replicas | +++ IMAGE_BACKUP=perconalab/percona-server-mysql-operator:main-backup8.4 logger.go:42: 04:16:43 | gr-self-healing/7-read-from-replicas | +++ export IMAGE_ORCHESTRATOR=perconalab/percona-server-mysql-operator:main-orchestrator logger.go:42: 04:16:43 | gr-self-healing/7-read-from-replicas | +++ IMAGE_ORCHESTRATOR=perconalab/percona-server-mysql-operator:main-orchestrator logger.go:42: 04:16:43 | gr-self-healing/7-read-from-replicas | +++ export IMAGE_ROUTER=perconalab/percona-server-mysql-operator:main-router8.4 logger.go:42: 04:16:43 | gr-self-healing/7-read-from-replicas | +++ IMAGE_ROUTER=perconalab/percona-server-mysql-operator:main-router8.4 logger.go:42: 04:16:43 | gr-self-healing/7-read-from-replicas | +++ export IMAGE_TOOLKIT=perconalab/percona-server-mysql-operator:main-toolkit logger.go:42: 04:16:43 | gr-self-healing/7-read-from-replicas | +++ IMAGE_TOOLKIT=perconalab/percona-server-mysql-operator:main-toolkit logger.go:42: 04:16:43 | gr-self-healing/7-read-from-replicas | +++ export IMAGE_HAPROXY=perconalab/percona-server-mysql-operator:main-haproxy logger.go:42: 04:16:43 | gr-self-healing/7-read-from-replicas | +++ IMAGE_HAPROXY=perconalab/percona-server-mysql-operator:main-haproxy logger.go:42: 04:16:43 | gr-self-healing/7-read-from-replicas | +++ export PMM_SERVER_VERSION=1.4.3 logger.go:42: 04:16:43 | gr-self-healing/7-read-from-replicas | +++ PMM_SERVER_VERSION=1.4.3 logger.go:42: 04:16:43 | gr-self-healing/7-read-from-replicas | +++ export IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 04:16:43 | gr-self-healing/7-read-from-replicas | +++ IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 04:16:43 | gr-self-healing/7-read-from-replicas | +++ export IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 04:16:43 | gr-self-healing/7-read-from-replicas | +++ IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 04:16:43 | gr-self-healing/7-read-from-replicas | +++ export CERT_MANAGER_VER=1.18.2 logger.go:42: 04:16:43 | gr-self-healing/7-read-from-replicas | +++ CERT_MANAGER_VER=1.18.2 logger.go:42: 04:16:43 | gr-self-healing/7-read-from-replicas | +++ export MINIO_VER=5.4.0 logger.go:42: 04:16:43 | gr-self-healing/7-read-from-replicas | +++ MINIO_VER=5.4.0 logger.go:42: 04:16:43 | gr-self-healing/7-read-from-replicas | +++ export CHAOS_MESH_VER=2.7.2 logger.go:42: 04:16:43 | gr-self-healing/7-read-from-replicas | +++ CHAOS_MESH_VER=2.7.2 logger.go:42: 04:16:43 | gr-self-healing/7-read-from-replicas | +++ export VAULT_VER=0.16.1 logger.go:42: 04:16:43 | gr-self-healing/7-read-from-replicas | +++ VAULT_VER=0.16.1 logger.go:42: 04:16:43 | gr-self-healing/7-read-from-replicas | ++++ which gdate logger.go:42: 04:16:43 | gr-self-healing/7-read-from-replicas | which: no gdate in (/mnt/jenkins/workspace/cloud-ps-operator_PR-1108/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 04:16:43 | gr-self-healing/7-read-from-replicas | ++++ which date logger.go:42: 04:16:43 | gr-self-healing/7-read-from-replicas | +++ date=/usr/sbin/date logger.go:42: 04:16:43 | gr-self-healing/7-read-from-replicas | +++ oc get projects logger.go:42: 04:16:43 | gr-self-healing/7-read-from-replicas | +++ : logger.go:42: 04:16:43 | gr-self-healing/7-read-from-replicas | +++ kubectl get nodes logger.go:42: 04:16:43 | gr-self-healing/7-read-from-replicas | +++ grep '^minikube' logger.go:42: 04:16:44 | gr-self-healing/7-read-from-replicas | +++ which gsed logger.go:42: 04:16:44 | gr-self-healing/7-read-from-replicas | which: no gsed in (/mnt/jenkins/workspace/cloud-ps-operator_PR-1108/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 04:16:44 | gr-self-healing/7-read-from-replicas | +++ which sed logger.go:42: 04:16:44 | gr-self-healing/7-read-from-replicas | ++ sed=/usr/sbin/sed logger.go:42: 04:16:44 | gr-self-healing/7-read-from-replicas | ++ oc get projects logger.go:42: 04:16:44 | gr-self-healing/7-read-from-replicas | +++ kubectl version -o json logger.go:42: 04:16:44 | gr-self-healing/7-read-from-replicas | +++ jq -r .serverVersion.gitVersion logger.go:42: 04:16:44 | gr-self-healing/7-read-from-replicas | +++ grep '\-eks\-' logger.go:42: 04:16:44 | gr-self-healing/7-read-from-replicas | grep: warning: stray \ before - logger.go:42: 04:16:44 | gr-self-healing/7-read-from-replicas | Warning: version difference between client (1.34) and server (1.31) exceeds the supported minor version skew of +/-1 logger.go:42: 04:16:44 | gr-self-healing/7-read-from-replicas | ++ '[' ']' logger.go:42: 04:16:44 | gr-self-healing/7-read-from-replicas | ++ EKS=0 logger.go:42: 04:16:44 | gr-self-healing/7-read-from-replicas | + for i in 0 1 2 logger.go:42: 04:16:44 | gr-self-healing/7-read-from-replicas | +++ get_cluster_name logger.go:42: 04:16:44 | gr-self-healing/7-read-from-replicas | +++ kubectl -n kuttl-test-concrete-skylark get ps -o 'jsonpath={.items[0].metadata.name}' logger.go:42: 04:16:45 | gr-self-healing/7-read-from-replicas | ++ get_mysql_headless_fqdn gr-self-healing 0 logger.go:42: 04:16:45 | gr-self-healing/7-read-from-replicas | ++ local cluster=gr-self-healing logger.go:42: 04:16:45 | gr-self-healing/7-read-from-replicas | ++ local index=0 logger.go:42: 04:16:45 | gr-self-healing/7-read-from-replicas | ++ echo gr-self-healing-mysql-0.gr-self-healing-mysql logger.go:42: 04:16:45 | gr-self-healing/7-read-from-replicas | + host=gr-self-healing-mysql-0.gr-self-healing-mysql logger.go:42: 04:16:45 | gr-self-healing/7-read-from-replicas | ++ run_mysql 'SELECT * FROM myDB.myTable' '-h gr-self-healing-mysql-0.gr-self-healing-mysql -uroot -proot_password' logger.go:42: 04:16:45 | gr-self-healing/7-read-from-replicas | ++ local 'command=SELECT * FROM myDB.myTable' logger.go:42: 04:16:45 | gr-self-healing/7-read-from-replicas | ++ local 'uri=-h gr-self-healing-mysql-0.gr-self-healing-mysql -uroot -proot_password' logger.go:42: 04:16:45 | gr-self-healing/7-read-from-replicas | ++ local pod= logger.go:42: 04:16:45 | gr-self-healing/7-read-from-replicas | +++ get_client_pod logger.go:42: 04:16:45 | gr-self-healing/7-read-from-replicas | +++ kubectl -n kuttl-test-concrete-skylark get pods --selector=name=mysql-client -o 'jsonpath={.items[].metadata.name}' logger.go:42: 04:16:45 | gr-self-healing/7-read-from-replicas | ++ client_pod=mysql-client logger.go:42: 04:16:45 | gr-self-healing/7-read-from-replicas | ++ wait_pod mysql-client logger.go:42: 04:16:45 | gr-self-healing/7-read-from-replicas | ++ local pod=mysql-client logger.go:42: 04:16:45 | gr-self-healing/7-read-from-replicas | ++ set +o xtrace logger.go:42: 04:16:46 | gr-self-healing/7-read-from-replicas | mysql-clienttrue logger.go:42: 04:16:46 | gr-self-healing/7-read-from-replicas | ++ /usr/sbin/sed -e 's/mysql: //' logger.go:42: 04:16:46 | gr-self-healing/7-read-from-replicas | ++ kubectl -n kuttl-test-concrete-skylark exec mysql-client -- bash -c 'printf '\''%s\n'\'' "SELECT * FROM myDB.myTable" | mysql -sN -h gr-self-healing-mysql-0.gr-self-healing-mysql -uroot -proot_password' logger.go:42: 04:16:46 | gr-self-healing/7-read-from-replicas | ++ grep -v 'Using a password on the command line interface can be insecure.' logger.go:42: 04:16:47 | gr-self-healing/7-read-from-replicas | + data='100500 logger.go:42: 04:16:47 | gr-self-healing/7-read-from-replicas | 100501' logger.go:42: 04:16:47 | gr-self-healing/7-read-from-replicas | + kubectl create configmap -n kuttl-test-concrete-skylark 07-read-from-replicas-0 '--from-literal=data=100500 logger.go:42: 04:16:47 | gr-self-healing/7-read-from-replicas | 100501' logger.go:42: 04:16:47 | gr-self-healing/7-read-from-replicas | configmap/07-read-from-replicas-0 created logger.go:42: 04:16:47 | gr-self-healing/7-read-from-replicas | + for i in 0 1 2 logger.go:42: 04:16:47 | gr-self-healing/7-read-from-replicas | +++ get_cluster_name logger.go:42: 04:16:47 | gr-self-healing/7-read-from-replicas | +++ kubectl -n kuttl-test-concrete-skylark get ps -o 'jsonpath={.items[0].metadata.name}' logger.go:42: 04:16:48 | gr-self-healing/7-read-from-replicas | ++ get_mysql_headless_fqdn gr-self-healing 1 logger.go:42: 04:16:48 | gr-self-healing/7-read-from-replicas | ++ local cluster=gr-self-healing logger.go:42: 04:16:48 | gr-self-healing/7-read-from-replicas | ++ local index=1 logger.go:42: 04:16:48 | gr-self-healing/7-read-from-replicas | ++ echo gr-self-healing-mysql-1.gr-self-healing-mysql logger.go:42: 04:16:48 | gr-self-healing/7-read-from-replicas | + host=gr-self-healing-mysql-1.gr-self-healing-mysql logger.go:42: 04:16:48 | gr-self-healing/7-read-from-replicas | ++ run_mysql 'SELECT * FROM myDB.myTable' '-h gr-self-healing-mysql-1.gr-self-healing-mysql -uroot -proot_password' logger.go:42: 04:16:48 | gr-self-healing/7-read-from-replicas | ++ local 'command=SELECT * FROM myDB.myTable' logger.go:42: 04:16:48 | gr-self-healing/7-read-from-replicas | ++ local 'uri=-h gr-self-healing-mysql-1.gr-self-healing-mysql -uroot -proot_password' logger.go:42: 04:16:48 | gr-self-healing/7-read-from-replicas | ++ local pod= logger.go:42: 04:16:48 | gr-self-healing/7-read-from-replicas | +++ get_client_pod logger.go:42: 04:16:48 | gr-self-healing/7-read-from-replicas | +++ kubectl -n kuttl-test-concrete-skylark get pods --selector=name=mysql-client -o 'jsonpath={.items[].metadata.name}' logger.go:42: 04:16:48 | gr-self-healing/7-read-from-replicas | ++ client_pod=mysql-client logger.go:42: 04:16:48 | gr-self-healing/7-read-from-replicas | ++ wait_pod mysql-client logger.go:42: 04:16:48 | gr-self-healing/7-read-from-replicas | ++ local pod=mysql-client logger.go:42: 04:16:48 | gr-self-healing/7-read-from-replicas | ++ set +o xtrace logger.go:42: 04:16:48 | gr-self-healing/7-read-from-replicas | mysql-clienttrue logger.go:42: 04:16:48 | gr-self-healing/7-read-from-replicas | ++ kubectl -n kuttl-test-concrete-skylark exec mysql-client -- bash -c 'printf '\''%s\n'\'' "SELECT * FROM myDB.myTable" | mysql -sN -h gr-self-healing-mysql-1.gr-self-healing-mysql -uroot -proot_password' logger.go:42: 04:16:48 | gr-self-healing/7-read-from-replicas | ++ /usr/sbin/sed -e 's/mysql: //' logger.go:42: 04:16:48 | gr-self-healing/7-read-from-replicas | ++ grep -v 'Using a password on the command line interface can be insecure.' logger.go:42: 04:16:49 | gr-self-healing/7-read-from-replicas | + data='100500 logger.go:42: 04:16:49 | gr-self-healing/7-read-from-replicas | 100501' logger.go:42: 04:16:49 | gr-self-healing/7-read-from-replicas | + kubectl create configmap -n kuttl-test-concrete-skylark 07-read-from-replicas-1 '--from-literal=data=100500 logger.go:42: 04:16:49 | gr-self-healing/7-read-from-replicas | 100501' logger.go:42: 04:16:50 | gr-self-healing/7-read-from-replicas | configmap/07-read-from-replicas-1 created logger.go:42: 04:16:50 | gr-self-healing/7-read-from-replicas | + for i in 0 1 2 logger.go:42: 04:16:50 | gr-self-healing/7-read-from-replicas | +++ get_cluster_name logger.go:42: 04:16:50 | gr-self-healing/7-read-from-replicas | +++ kubectl -n kuttl-test-concrete-skylark get ps -o 'jsonpath={.items[0].metadata.name}' logger.go:42: 04:16:50 | gr-self-healing/7-read-from-replicas | ++ get_mysql_headless_fqdn gr-self-healing 2 logger.go:42: 04:16:50 | gr-self-healing/7-read-from-replicas | ++ local cluster=gr-self-healing logger.go:42: 04:16:50 | gr-self-healing/7-read-from-replicas | ++ local index=2 logger.go:42: 04:16:50 | gr-self-healing/7-read-from-replicas | ++ echo gr-self-healing-mysql-2.gr-self-healing-mysql logger.go:42: 04:16:50 | gr-self-healing/7-read-from-replicas | + host=gr-self-healing-mysql-2.gr-self-healing-mysql logger.go:42: 04:16:50 | gr-self-healing/7-read-from-replicas | ++ run_mysql 'SELECT * FROM myDB.myTable' '-h gr-self-healing-mysql-2.gr-self-healing-mysql -uroot -proot_password' logger.go:42: 04:16:50 | gr-self-healing/7-read-from-replicas | ++ local 'command=SELECT * FROM myDB.myTable' logger.go:42: 04:16:50 | gr-self-healing/7-read-from-replicas | ++ local 'uri=-h gr-self-healing-mysql-2.gr-self-healing-mysql -uroot -proot_password' logger.go:42: 04:16:50 | gr-self-healing/7-read-from-replicas | ++ local pod= logger.go:42: 04:16:50 | gr-self-healing/7-read-from-replicas | +++ get_client_pod logger.go:42: 04:16:50 | gr-self-healing/7-read-from-replicas | +++ kubectl -n kuttl-test-concrete-skylark get pods --selector=name=mysql-client -o 'jsonpath={.items[].metadata.name}' logger.go:42: 04:16:51 | gr-self-healing/7-read-from-replicas | ++ client_pod=mysql-client logger.go:42: 04:16:51 | gr-self-healing/7-read-from-replicas | ++ wait_pod mysql-client logger.go:42: 04:16:51 | gr-self-healing/7-read-from-replicas | ++ local pod=mysql-client logger.go:42: 04:16:51 | gr-self-healing/7-read-from-replicas | ++ set +o xtrace logger.go:42: 04:16:51 | gr-self-healing/7-read-from-replicas | mysql-clienttrue logger.go:42: 04:16:51 | gr-self-healing/7-read-from-replicas | ++ kubectl -n kuttl-test-concrete-skylark exec mysql-client -- bash -c 'printf '\''%s\n'\'' "SELECT * FROM myDB.myTable" | mysql -sN -h gr-self-healing-mysql-2.gr-self-healing-mysql -uroot -proot_password' logger.go:42: 04:16:51 | gr-self-healing/7-read-from-replicas | ++ /usr/sbin/sed -e 's/mysql: //' logger.go:42: 04:16:51 | gr-self-healing/7-read-from-replicas | ++ grep -v 'Using a password on the command line interface can be insecure.' logger.go:42: 04:16:53 | gr-self-healing/7-read-from-replicas | + data='100500 logger.go:42: 04:16:53 | gr-self-healing/7-read-from-replicas | 100501' logger.go:42: 04:16:53 | gr-self-healing/7-read-from-replicas | + kubectl create configmap -n kuttl-test-concrete-skylark 07-read-from-replicas-2 '--from-literal=data=100500 logger.go:42: 04:16:53 | gr-self-healing/7-read-from-replicas | 100501' logger.go:42: 04:16:53 | gr-self-healing/7-read-from-replicas | configmap/07-read-from-replicas-2 created logger.go:42: 04:16:54 | gr-self-healing/7-read-from-replicas | test step completed 7-read-from-replicas logger.go:42: 04:16:54 | gr-self-healing/8-failure-primary | starting test step 8-failure-primary logger.go:42: 04:16:54 | gr-self-healing/8-failure-primary | running command: [sh -c set -o errexit set -o xtrace source ../../functions check_primary_chaos "failure" ${NAMESPACE} $(get_primary_from_group_replication)] logger.go:42: 04:16:54 | gr-self-healing/8-failure-primary | + source ../../functions logger.go:42: 04:16:54 | gr-self-healing/8-failure-primary | +++ realpath ../../.. logger.go:42: 04:16:54 | gr-self-healing/8-failure-primary | ++ ROOT_REPO=/mnt/jenkins/workspace/cloud-ps-operator_PR-1108 logger.go:42: 04:16:54 | gr-self-healing/8-failure-primary | ++++ pwd logger.go:42: 04:16:54 | gr-self-healing/8-failure-primary | +++ basename /mnt/jenkins/workspace/cloud-ps-operator_PR-1108/e2e-tests/tests/gr-self-healing logger.go:42: 04:16:54 | gr-self-healing/8-failure-primary | ++ test_name=gr-self-healing logger.go:42: 04:16:54 | gr-self-healing/8-failure-primary | ++ source /mnt/jenkins/workspace/cloud-ps-operator_PR-1108/e2e-tests/vars.sh logger.go:42: 04:16:54 | gr-self-healing/8-failure-primary | +++ export ROOT_REPO=/mnt/jenkins/workspace/cloud-ps-operator_PR-1108 logger.go:42: 04:16:54 | gr-self-healing/8-failure-primary | +++ ROOT_REPO=/mnt/jenkins/workspace/cloud-ps-operator_PR-1108 logger.go:42: 04:16:54 | gr-self-healing/8-failure-primary | +++ export DEPLOY_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-1108/deploy logger.go:42: 04:16:54 | gr-self-healing/8-failure-primary | +++ DEPLOY_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-1108/deploy logger.go:42: 04:16:54 | gr-self-healing/8-failure-primary | +++ export TESTS_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-1108/e2e-tests logger.go:42: 04:16:54 | gr-self-healing/8-failure-primary | +++ TESTS_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-1108/e2e-tests logger.go:42: 04:16:54 | gr-self-healing/8-failure-primary | +++ export TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-1108/e2e-tests/conf logger.go:42: 04:16:54 | gr-self-healing/8-failure-primary | +++ TESTS_CONFIG_DIR=/mnt/jenkins/workspace/cloud-ps-operator_PR-1108/e2e-tests/conf logger.go:42: 04:16:54 | gr-self-healing/8-failure-primary | +++ export TEMP_DIR=/tmp/kuttl/ps/gr-self-healing logger.go:42: 04:16:54 | gr-self-healing/8-failure-primary | +++ TEMP_DIR=/tmp/kuttl/ps/gr-self-healing logger.go:42: 04:16:54 | gr-self-healing/8-failure-primary | ++++ git rev-parse --abbrev-ref HEAD logger.go:42: 04:16:54 | gr-self-healing/8-failure-primary | +++ export GIT_BRANCH=PR-1108 logger.go:42: 04:16:54 | gr-self-healing/8-failure-primary | +++ GIT_BRANCH=PR-1108 logger.go:42: 04:16:54 | gr-self-healing/8-failure-primary | +++ export VERSION=PR-1108-c411fa10 logger.go:42: 04:16:54 | gr-self-healing/8-failure-primary | +++ VERSION=PR-1108-c411fa10 logger.go:42: 04:16:54 | gr-self-healing/8-failure-primary | +++ export IMAGE=perconalab/percona-server-mysql-operator:PR-1108-c411fa10 logger.go:42: 04:16:54 | gr-self-healing/8-failure-primary | +++ IMAGE=perconalab/percona-server-mysql-operator:PR-1108-c411fa10 logger.go:42: 04:16:54 | gr-self-healing/8-failure-primary | +++ [[ -z '' ]] logger.go:42: 04:16:54 | gr-self-healing/8-failure-primary | +++ [[ -n '' ]] logger.go:42: 04:16:54 | gr-self-healing/8-failure-primary | +++ export MYSQL_VERSION=8.4 logger.go:42: 04:16:54 | gr-self-healing/8-failure-primary | +++ MYSQL_VERSION=8.4 logger.go:42: 04:16:54 | gr-self-healing/8-failure-primary | +++ export IMAGE_MYSQL=perconalab/percona-server-mysql-operator:main-psmysql8.4 logger.go:42: 04:16:54 | gr-self-healing/8-failure-primary | +++ IMAGE_MYSQL=perconalab/percona-server-mysql-operator:main-psmysql8.4 logger.go:42: 04:16:54 | gr-self-healing/8-failure-primary | +++ export IMAGE_BACKUP=perconalab/percona-server-mysql-operator:main-backup8.4 logger.go:42: 04:16:54 | gr-self-healing/8-failure-primary | +++ IMAGE_BACKUP=perconalab/percona-server-mysql-operator:main-backup8.4 logger.go:42: 04:16:54 | gr-self-healing/8-failure-primary | +++ export IMAGE_ORCHESTRATOR=perconalab/percona-server-mysql-operator:main-orchestrator logger.go:42: 04:16:54 | gr-self-healing/8-failure-primary | +++ IMAGE_ORCHESTRATOR=perconalab/percona-server-mysql-operator:main-orchestrator logger.go:42: 04:16:54 | gr-self-healing/8-failure-primary | +++ export IMAGE_ROUTER=perconalab/percona-server-mysql-operator:main-router8.4 logger.go:42: 04:16:54 | gr-self-healing/8-failure-primary | +++ IMAGE_ROUTER=perconalab/percona-server-mysql-operator:main-router8.4 logger.go:42: 04:16:54 | gr-self-healing/8-failure-primary | +++ export IMAGE_TOOLKIT=perconalab/percona-server-mysql-operator:main-toolkit logger.go:42: 04:16:54 | gr-self-healing/8-failure-primary | +++ IMAGE_TOOLKIT=perconalab/percona-server-mysql-operator:main-toolkit logger.go:42: 04:16:54 | gr-self-healing/8-failure-primary | +++ export IMAGE_HAPROXY=perconalab/percona-server-mysql-operator:main-haproxy logger.go:42: 04:16:54 | gr-self-healing/8-failure-primary | +++ IMAGE_HAPROXY=perconalab/percona-server-mysql-operator:main-haproxy logger.go:42: 04:16:54 | gr-self-healing/8-failure-primary | +++ export PMM_SERVER_VERSION=1.4.3 logger.go:42: 04:16:54 | gr-self-healing/8-failure-primary | +++ PMM_SERVER_VERSION=1.4.3 logger.go:42: 04:16:54 | gr-self-healing/8-failure-primary | +++ export IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 04:16:54 | gr-self-healing/8-failure-primary | +++ IMAGE_PMM_CLIENT=perconalab/pmm-client:3-dev-latest logger.go:42: 04:16:54 | gr-self-healing/8-failure-primary | +++ export IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 04:16:54 | gr-self-healing/8-failure-primary | +++ IMAGE_PMM_SERVER=perconalab/pmm-server:3-dev-latest logger.go:42: 04:16:54 | gr-self-healing/8-failure-primary | +++ export CERT_MANAGER_VER=1.18.2 logger.go:42: 04:16:54 | gr-self-healing/8-failure-primary | +++ CERT_MANAGER_VER=1.18.2 logger.go:42: 04:16:54 | gr-self-healing/8-failure-primary | +++ export MINIO_VER=5.4.0 logger.go:42: 04:16:54 | gr-self-healing/8-failure-primary | +++ MINIO_VER=5.4.0 logger.go:42: 04:16:54 | gr-self-healing/8-failure-primary | +++ export CHAOS_MESH_VER=2.7.2 logger.go:42: 04:16:54 | gr-self-healing/8-failure-primary | +++ CHAOS_MESH_VER=2.7.2 logger.go:42: 04:16:54 | gr-self-healing/8-failure-primary | +++ export VAULT_VER=0.16.1 logger.go:42: 04:16:54 | gr-self-healing/8-failure-primary | +++ VAULT_VER=0.16.1 logger.go:42: 04:16:54 | gr-self-healing/8-failure-primary | ++++ which gdate logger.go:42: 04:16:54 | gr-self-healing/8-failure-primary | which: no gdate in (/mnt/jenkins/workspace/cloud-ps-operator_PR-1108/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 04:16:54 | gr-self-healing/8-failure-primary | ++++ which date logger.go:42: 04:16:54 | gr-self-healing/8-failure-primary | +++ date=/usr/sbin/date logger.go:42: 04:16:54 | gr-self-healing/8-failure-primary | +++ oc get projects logger.go:42: 04:16:54 | gr-self-healing/8-failure-primary | +++ : logger.go:42: 04:16:54 | gr-self-healing/8-failure-primary | +++ kubectl get nodes logger.go:42: 04:16:54 | gr-self-healing/8-failure-primary | +++ grep '^minikube' logger.go:42: 04:16:55 | gr-self-healing/8-failure-primary | +++ which gsed logger.go:42: 04:16:55 | gr-self-healing/8-failure-primary | which: no gsed in (/mnt/jenkins/workspace/cloud-ps-operator_PR-1108/bin/:/root/.krew/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/var/lib/snapd/snap/bin) logger.go:42: 04:16:55 | gr-self-healing/8-failure-primary | +++ which sed logger.go:42: 04:16:55 | gr-self-healing/8-failure-primary | ++ sed=/usr/sbin/sed logger.go:42: 04:16:55 | gr-self-healing/8-failure-primary | ++ oc get projects logger.go:42: 04:16:55 | gr-self-healing/8-failure-primary | +++ kubectl version -o json logger.go:42: 04:16:55 | gr-self-healing/8-failure-primary | +++ jq -r .serverVersion.gitVersion logger.go:42: 04:16:55 | gr-self-healing/8-failure-primary | +++ grep '\-eks\-' logger.go:42: 04:16:55 | gr-self-healing/8-failure-primary | grep: warning: stray \ before - logger.go:42: 04:16:55 | gr-self-healing/8-failure-primary | Warning: version difference between client (1.34) and server (1.31) exceeds the supported minor version skew of +/-1 logger.go:42: 04:16:55 | gr-self-healing/8-failure-primary | ++ '[' ']' logger.go:42: 04:16:55 | gr-self-healing/8-failure-primary | ++ EKS=0 logger.go:42: 04:16:55 | gr-self-healing/8-failure-primary | ++ get_primary_from_group_replication logger.go:42: 04:16:55 | gr-self-healing/8-failure-primary | ++ cut -d. -f1 logger.go:42: 04:16:55 | gr-self-healing/8-failure-primary | ++++ get_cluster_name logger.go:42: 04:16:55 | gr-self-healing/8-failure-primary | ++++ kubectl -n kuttl-test-concrete-skylark get ps -o 'jsonpath={.items[0].metadata.name}' logger.go:42: 04:16:56 | gr-self-healing/8-failure-primary | +++ get_mysql_router_service gr-self-healing logger.go:42: 04:16:56 | gr-self-healing/8-failure-primary | +++ local cluster=gr-self-healing logger.go:42: 04:16:56 | gr-self-healing/8-failure-primary | +++ echo gr-self-healing-router logger.go:42: 04:16:56 | gr-self-healing/8-failure-primary | ++ run_mysql 'SELECT MEMBER_HOST FROM performance_schema.replication_group_members where MEMBER_ROLE='\''PRIMARY'\'';' '-h gr-self-healing-router -P 6446 -uroot -proot_password' logger.go:42: 04:16:56 | gr-self-healing/8-failure-primary | ++ local 'command=SELECT MEMBER_HOST FROM performance_schema.replication_group_members where MEMBER_ROLE='\''PRIMARY'\'';' logger.go:42: 04:16:56 | gr-self-healing/8-failure-primary | ++ local 'uri=-h gr-self-healing-router -P 6446 -uroot -proot_password' logger.go:42: 04:16:56 | gr-self-healing/8-failure-primary | ++ local pod= logger.go:42: 04:16:56 | gr-self-healing/8-failure-primary | +++ get_client_pod logger.go:42: 04:16:56 | gr-self-healing/8-failure-primary | +++ kubectl -n kuttl-test-concrete-skylark get pods --selector=name=mysql-client -o 'jsonpath={.items[].metadata.name}' logger.go:42: 04:16:56 | gr-self-healing/8-failure-primary | ++ client_pod=mysql-client logger.go:42: 04:16:56 | gr-self-healing/8-failure-primary | ++ wait_pod mysql-client logger.go:42: 04:16:56 | gr-self-healing/8-failure-primary | ++ local pod=mysql-client logger.go:42: 04:16:56 | gr-self-healing/8-failure-primary | ++ set +o xtrace logger.go:42: 04:16:57 | gr-self-healing/8-failure-primary | mysql-clienttrue logger.go:42: 04:16:57 | gr-self-healing/8-failure-primary | ++ kubectl -n kuttl-test-concrete-skylark exec mysql-client -- bash -c 'printf '\''%s\n'\'' "SELECT MEMBER_HOST FROM performance_schema.replication_group_members where MEMBER_ROLE='\''PRIMARY'\'';" | mysql -sN -h gr-self-healing-router -P 6446 -uroot -proot_password' logger.go:42: 04:16:57 | gr-self-healing/8-failure-primary | ++ /usr/sbin/sed -e 's/mysql: //' logger.go:42: 04:16:57 | gr-self-healing/8-failure-primary | ++ grep -v 'Using a password on the command line interface can be insecure.' logger.go:42: 04:16:57 | gr-self-healing/8-failure-primary | + check_primary_chaos failure kuttl-test-concrete-skylark gr-self-healing-mysql-2 logger.go:42: 04:16:57 | gr-self-healing/8-failure-primary | + local chaos_type=failure logger.go:42: 04:16:57 | gr-self-healing/8-failure-primary | + local ns=kuttl-test-concrete-skylark logger.go:42: 04:16:57 | gr-self-healing/8-failure-primary | + local primary_before_failure=gr-self-healing-mysql-2 logger.go:42: 04:16:57 | gr-self-healing/8-failure-primary | + local chaos_name logger.go:42: 04:16:57 | gr-self-healing/8-failure-primary | + case ${chaos_type} in logger.go:42: 04:16:57 | gr-self-healing/8-failure-primary | + chaos_name=chaos-pod-failure-primary logger.go:42: 04:16:57 | gr-self-healing/8-failure-primary | + failure_pod kuttl-test-concrete-skylark gr-self-healing-mysql-2 chaos-pod-failure-primary logger.go:42: 04:16:57 | gr-self-healing/8-failure-primary | + local ns=kuttl-test-concrete-skylark logger.go:42: 04:16:57 | gr-self-healing/8-failure-primary | + local pod=gr-self-healing-mysql-2 logger.go:42: 04:16:57 | gr-self-healing/8-failure-primary | + local chaos_name=chaos-pod-failure-primary logger.go:42: 04:16:57 | gr-self-healing/8-failure-primary | + yq eval ' logger.go:42: 04:16:57 | gr-self-healing/8-failure-primary | .metadata.name = "chaos-pod-failure-primary" | logger.go:42: 04:16:57 | gr-self-healing/8-failure-primary | del(.spec.selector.pods.test-namespace) | logger.go:42: 04:16:57 | gr-self-healing/8-failure-primary | .spec.selector.pods.kuttl-test-concrete-skylark[0] = "gr-self-healing-mysql-2"' /mnt/jenkins/workspace/cloud-ps-operator_PR-1108/e2e-tests/conf/chaos-pod-failure.yml logger.go:42: 04:16:57 | gr-self-healing/8-failure-primary | + kubectl apply --namespace kuttl-test-concrete-skylark -f - logger.go:42: 04:16:59 | gr-self-healing/8-failure-primary | podchaos.chaos-mesh.org/chaos-pod-failure-primary created logger.go:42: 04:16:59 | gr-self-healing/8-failure-primary | + sleep 5 logger.go:42: 04:17:04 | gr-self-healing/8-failure-primary | + wait_until_chaos_applied failure chaos-pod-failure-primary logger.go:42: 04:17:04 | gr-self-healing/8-failure-primary | + local chaos_type=failure logger.go:42: 04:17:04 | gr-self-healing/8-failure-primary | + local chaos_name=chaos-pod-failure-primary logger.go:42: 04:17:04 | gr-self-healing/8-failure-primary | + local resource logger.go:42: 04:17:04 | gr-self-healing/8-failure-primary | + case ${chaos_type} in logger.go:42: 04:17:04 | gr-self-healing/8-failure-primary | + resource=podchaos/chaos-pod-failure-primary logger.go:42: 04:17:04 | gr-self-healing/8-failure-primary | + local retry=0 logger.go:42: 04:17:04 | gr-self-healing/8-failure-primary | + [[ 0 == 30 ]] logger.go:42: 04:17:04 | gr-self-healing/8-failure-primary | + sleep 10 logger.go:42: 04:17:14 | gr-self-healing/8-failure-primary | + retry=1 logger.go:42: 04:17:14 | gr-self-healing/8-failure-primary | ++ kubectl -n kuttl-test-concrete-skylark get podchaos/chaos-pod-failure-primary -o yaml logger.go:42: 04:17:14 | gr-self-healing/8-failure-primary | ++ yq '.status.experiment.containerRecords[].events[] logger.go:42: 04:17:14 | gr-self-healing/8-failure-primary | | select(.operation == "Apply" and .type == "Succeeded")' logger.go:42: 04:17:14 | gr-self-healing/8-failure-primary | + succeeded='operation: Apply logger.go:42: 04:17:14 | gr-self-healing/8-failure-primary | timestamp: "2025-10-01T04:16:59Z" logger.go:42: 04:17:14 | gr-self-healing/8-failure-primary | type: Succeeded' logger.go:42: 04:17:14 | gr-self-healing/8-failure-primary | + [[ -n operation: Apply logger.go:42: 04:17:14 | gr-self-healing/8-failure-primary | timestamp: "2025-10-01T04:16:59Z" logger.go:42: 04:17:14 | gr-self-healing/8-failure-primary | type: Succeeded ]] logger.go:42: 04:17:14 | gr-self-healing/8-failure-primary | + return logger.go:42: 04:17:14 | gr-self-healing/8-failure-primary | + [[ failure == \f\a\i\l\u\r\e ]] logger.go:42: 04:17:14 | gr-self-healing/8-failure-primary | + wait_until_chaos_recovered failure chaos-pod-failure-primary logger.go:42: 04:17:14 | gr-self-healing/8-failure-primary | + local chaos_type=failure logger.go:42: 04:17:14 | gr-self-healing/8-failure-primary | + local chaos_name=chaos-pod-failure-primary logger.go:42: 04:17:14 | gr-self-healing/8-failure-primary | + local resource logger.go:42: 04:17:14 | gr-self-healing/8-failure-primary | + case ${chaos_type} in logger.go:42: 04:17:14 | gr-self-healing/8-failure-primary | + resource=podchaos/chaos-pod-failure-primary logger.go:42: 04:17:14 | gr-self-healing/8-failure-primary | + local retry=0 logger.go:42: 04:17:14 | gr-self-healing/8-failure-primary | + [[ 0 == 30 ]] logger.go:42: 04:17:14 | gr-self-healing/8-failure-primary | + sleep 10 logger.go:42: 04:17:24 | gr-self-healing/8-failure-primary | + retry=1 logger.go:42: 04:17:24 | gr-self-healing/8-failure-primary | ++ kubectl -n kuttl-test-concrete-skylark get podchaos/chaos-pod-failure-primary -o yaml logger.go:42: 04:17:24 | gr-self-healing/8-failure-primary | ++ yq '.status.experiment.containerRecords[].events[] logger.go:42: 04:17:24 | gr-self-healing/8-failure-primary | | select(.operation == "Recover" and .type == "Succeeded")' logger.go:42: 04:17:25 | gr-self-healing/8-failure-primary | + succeeded= logger.go:42: 04:17:25 | gr-self-healing/8-failure-primary | + [[ -n '' ]] logger.go:42: 04:17:25 | gr-self-healing/8-failure-primary | + [[ 1 == 30 ]] logger.go:42: 04:17:25 | gr-self-healing/8-failure-primary | + sleep 10 logger.go:42: 04:17:35 | gr-self-healing/8-failure-primary | + retry=2 logger.go:42: 04:17:35 | gr-self-healing/8-failure-primary | ++ kubectl -n kuttl-test-concrete-skylark get podchaos/chaos-pod-failure-primary -o yaml logger.go:42: 04:17:35 | gr-self-healing/8-failure-primary | ++ yq '.status.experiment.containerRecords[].events[] logger.go:42: 04:17:35 | gr-self-healing/8-failure-primary | | select(.operation == "Recover" and .type == "Succeeded")' logger.go:42: 04:17:35 | gr-self-healing/8-failure-primary | + succeeded= logger.go:42: 04:17:35 | gr-self-healing/8-failure-primary | + [[ -n '' ]] logger.go:42: 04:17:35 | gr-self-healing/8-failure-primary | + [[ 2 == 30 ]] logger.go:42: 04:17:35 | gr-self-healing/8-failure-primary | + sleep 10 logger.go:42: 04:17:45 | gr-self-healing/8-failure-primary | + retry=3 logger.go:42: 04:17:45 | gr-self-healing/8-failure-primary | ++ kubectl -n kuttl-test-concrete-skylark get podchaos/chaos-pod-failure-primary -o yaml logger.go:42: 04:17:45 | gr-self-healing/8-failure-primary | ++ yq '.status.experiment.containerRecords[].events[] logger.go:42: 04:17:45 | gr-self-healing/8-failure-primary | | select(.operation == "Recover" and .type == "Succeeded")' logger.go:42: 04:17:46 | gr-self-healing/8-failure-primary | + succeeded= logger.go:42: 04:17:46 | gr-self-healing/8-failure-primary | + [[ -n '' ]] logger.go:42: 04:17:46 | gr-self-healing/8-failure-primary | + [[ 3 == 30 ]] logger.go:42: 04:17:46 | gr-self-healing/8-failure-primary | + sleep 10 logger.go:42: 04:17:56 | gr-self-healing/8-failure-primary | + retry=4 logger.go:42: 04:17:56 | gr-self-healing/8-failure-primary | ++ kubectl -n kuttl-test-concrete-skylark get podchaos/chaos-pod-failure-primary -o yaml logger.go:42: 04:17:56 | gr-self-healing/8-failure-primary | ++ yq '.status.experiment.containerRecords[].events[] logger.go:42: 04:17:56 | gr-self-healing/8-failure-primary | | select(.operation == "Recover" and .type == "Succeeded")' logger.go:42: 04:17:56 | gr-self-healing/8-failure-primary | + succeeded= logger.go:42: 04:17:56 | gr-self-healing/8-failure-primary | + [[ -n '' ]] logger.go:42: 04:17:56 | gr-self-healing/8-failure-primary | + [[ 4 == 30 ]] logger.go:42: 04:17:56 | gr-self-healing/8-failure-primary | + sleep 10 case.go:396: failed in step 8-failure-primary case.go:398: command "set -o xtrace\\n source ../../functions\\n check_primary_chaos \"failu..." failed, exit status 143