++ echo 'Log: /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/logs/self-healing-chaos.log' Log: /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/logs/self-healing-chaos.log ++ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/conf/cloud-secret.yml ']' ++ SKIP_BACKUPS_TO_AWS_GCP= ++ oc get projects error: No Auth Provider found for name "gcp" +++ kubectl version -o json +++ grep '\-eks\-' +++ jq -r .serverVersion.gitVersion ++ '[' ']' ++ EKS=0 +++ kubectl version -o json +++ /usr/bin/sed -r 's/[^0-9.]+//g' +++ jq -r '.serverVersion.major + "." + .serverVersion.minor' ++ KUBE_VERSION=1.20 +++ helm version -c +++ /usr/bin/sed -re 's/.*SemVer:"([^"]+)".*/\1/; s/.*\bVersion:"([^"]+)".*/\1/' ++ HELM_VERSION=v3.8.2 ++ '[' v3 == v2 ']' + cluster=some-name + main + create_infra self-healing-chaos-24626 + local ns=self-healing-chaos-24626 + '[' -n pxc-operator ']' + xargs -L 1 sh -xc 'kubectl patch pxc -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' + grep -v NAMESPACE + kubectl get pxc --all-namespaces -o wide No resources found + kubectl patch pxc -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: resource(s) were provided, but no name, label selector, or --all flag specified + : + kubectl_bin delete pxc --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.LCu25e8gzT ++ mktemp + local LAST_ERR=/tmp/tmp.NWrrra1Vmp + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl delete pxc --all --all-namespaces + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.LCu25e8gzT No resources found + cat /tmp/tmp.NWrrra1Vmp + rm /tmp/tmp.LCu25e8gzT /tmp/tmp.NWrrra1Vmp + return 0 + kubectl_bin delete pxc-backup --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.a7iD95Y2it ++ mktemp + local LAST_ERR=/tmp/tmp.hqodCpKmEj + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl delete pxc-backup --all --all-namespaces + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.a7iD95Y2it No resources found + cat /tmp/tmp.hqodCpKmEj + rm /tmp/tmp.a7iD95Y2it /tmp/tmp.hqodCpKmEj + return 0 + kubectl_bin delete pxc-restore --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.yH05wGMnZm ++ mktemp + local LAST_ERR=/tmp/tmp.akaYYQhXPM + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl delete pxc-restore --all --all-namespaces + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.yH05wGMnZm No resources found + cat /tmp/tmp.akaYYQhXPM + rm /tmp/tmp.yH05wGMnZm /tmp/tmp.akaYYQhXPM + return 0 + create_namespace pxc-operator + local namespace=pxc-operator + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + kubectl_bin get ns + '[' '!' -z '' ']' + kubectl_bin delete namespace pxc-operator ++ mktemp + awk '{print$1}' + egrep -v '^kube-|^default|Terminating|pxc-operator|openshift|^NAME' + local LAST_OUT=/tmp/tmp.KUCKvBL1HN + xargs kubectl delete ns ++ mktemp ++ mktemp + local LAST_ERR=/tmp/tmp.K0sNmYcV34 + local exit_status=0 ++ seq 0 2 + local LAST_OUT=/tmp/tmp.dOFxYEOfnO + for i in '$(seq 0 2)' + kubectl get ns ++ mktemp + local LAST_ERR=/tmp/tmp.tc5rtX39KK + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl delete namespace pxc-operator + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.KUCKvBL1HN + cat /tmp/tmp.K0sNmYcV34 + rm /tmp/tmp.KUCKvBL1HN /tmp/tmp.K0sNmYcV34 + return 0 namespace "self-healing-chaos-20651" deleted + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.dOFxYEOfnO namespace "pxc-operator" deleted + cat /tmp/tmp.tc5rtX39KK + rm /tmp/tmp.dOFxYEOfnO /tmp/tmp.tc5rtX39KK + return 0 + wait_for_delete namespace/pxc-operator + local res=namespace/pxc-operator + set +o xtrace namespace/pxc-operator - Error from server (NotFound): namespaces "pxc-operator" not found + kubectl_bin create namespace pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.uaaLSJ2vA9 ++ mktemp + local LAST_ERR=/tmp/tmp.eyk8qOdu5s + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl create namespace pxc-operator + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.uaaLSJ2vA9 namespace/pxc-operator created + cat /tmp/tmp.eyk8qOdu5s + rm /tmp/tmp.uaaLSJ2vA9 /tmp/tmp.eyk8qOdu5s + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.nPLk4jVbG2 +++ mktemp ++ local LAST_ERR=/tmp/tmp.AF9Ppgo01T ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl config current-context ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.nPLk4jVbG2 ++ cat /tmp/tmp.AF9Ppgo01T ++ rm /tmp/tmp.nPLk4jVbG2 /tmp/tmp.AF9Ppgo01T ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jenkins-pxc-706f792a-selfhealing --namespace=pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.vD4Qq9d8sZ ++ mktemp + local LAST_ERR=/tmp/tmp.drtv08Zh69 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jenkins-pxc-706f792a-selfhealing --namespace=pxc-operator + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.vD4Qq9d8sZ Context "gke_cloud-dev-112233_us-central1-a_jenkins-pxc-706f792a-selfhealing" modified. + cat /tmp/tmp.drtv08Zh69 + rm /tmp/tmp.vD4Qq9d8sZ /tmp/tmp.drtv08Zh69 + return 0 + deploy_operator + desc 'start operator' + set +o xtrace ----------------------------------------------------------------------------------- start operator ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.Er14SAQv6x ++ mktemp + local LAST_ERR=/tmp/tmp.wma0MBDVAF + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/deploy/crd.yaml + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.Er14SAQv6x customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com unchanged customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterbackups.pxc.percona.com unchanged customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterrestores.pxc.percona.com unchanged customresourcedefinition.apiextensions.k8s.io/perconaxtradbbackups.pxc.percona.com configured + cat /tmp/tmp.wma0MBDVAF + rm /tmp/tmp.Er14SAQv6x /tmp/tmp.wma0MBDVAF + return 0 + '[' -n pxc-operator ']' + apply_rbac cw-rbac + local operator_namespace=pxc-operator + local rbac=cw-rbac + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/deploy/cw-rbac.yaml + kubectl_bin apply -f - + sed -e 's^namespace: .*^namespace: pxc-operator^' ++ mktemp + local LAST_OUT=/tmp/tmp.PEVd9HpT3E ++ mktemp + local LAST_ERR=/tmp/tmp.K5EtoV4ObO + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl apply -f - + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.PEVd9HpT3E clusterrole.rbac.authorization.k8s.io/percona-xtradb-cluster-operator unchanged serviceaccount/percona-xtradb-cluster-operator created clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-xtradb-cluster-operator unchanged + cat /tmp/tmp.K5EtoV4ObO + rm /tmp/tmp.PEVd9HpT3E /tmp/tmp.K5EtoV4ObO + return 0 + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/deploy/cw-operator.yaml + sed -e 's^failureThreshold: .*^failureThreshold: 10^' + kubectl_bin apply -f - ++ mktemp + sed -e 's^image: .*^image: perconalab/percona-xtradb-cluster-operator:PR-1125-706f792a^' + local LAST_OUT=/tmp/tmp.ANeAk5NnPz ++ mktemp + local LAST_ERR=/tmp/tmp.nzUHZWuJCf + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl apply -f - + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.ANeAk5NnPz deployment.apps/percona-xtradb-cluster-operator created service/percona-xtradb-cluster-operator created + cat /tmp/tmp.nzUHZWuJCf + rm /tmp/tmp.ANeAk5NnPz /tmp/tmp.nzUHZWuJCf + return 0 + sleep 10 ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator +++ grep -c percona-xtradb-cluster-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.S1SqJsPJfw +++ mktemp ++ local LAST_ERR=/tmp/tmp.v2YzU0hxhR ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.S1SqJsPJfw ++ cat /tmp/tmp.v2YzU0hxhR ++ rm /tmp/tmp.S1SqJsPJfw /tmp/tmp.v2YzU0hxhR ++ return 0 + wait_pod percona-xtradb-cluster-operator-5699d7755d-z5px6 480 pxc-operator + local pod=percona-xtradb-cluster-operator-5699d7755d-z5px6 + local max_retry=480 + local ns=pxc-operator ++ echo percona-xtradb-cluster-operator-5699d7755d-z5px6 ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' ++ egrep '^(pxc|proxysql)$' + local container= + set +o xtrace percona-xtradb-cluster-operator-5699d7755d-z5px6.Ok + sleep 3 + create_namespace self-healing-chaos-24626 + local namespace=self-healing-chaos-24626 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + kubectl_bin get ns + '[' '!' -z '' ']' + kubectl_bin delete namespace self-healing-chaos-24626 + xargs kubectl delete ns + awk '{print$1}' + egrep -v '^kube-|^default|Terminating|pxc-operator|openshift|^NAME' ++ mktemp ++ mktemp + local LAST_OUT=/tmp/tmp.H9DXekzX5M + local LAST_OUT=/tmp/tmp.u3g8amWj0I ++ mktemp ++ mktemp + local LAST_ERR=/tmp/tmp.eDQc9MdiQc + local exit_status=0 + local LAST_ERR=/tmp/tmp.4SqX6CMsGd + local exit_status=0 ++ seq 0 2 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl delete namespace self-healing-chaos-24626 + for i in '$(seq 0 2)' + kubectl get ns + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.u3g8amWj0I + cat /tmp/tmp.4SqX6CMsGd + rm /tmp/tmp.u3g8amWj0I /tmp/tmp.4SqX6CMsGd + return 0 + exit_status=1 + [[ 1 != 0 ]] + sleep 0 + for i in '$(seq 0 2)' + kubectl delete namespace self-healing-chaos-24626 error: resource(s) were provided, but no name, label selector, or --all flag specified + exit_status=1 + [[ 1 != 0 ]] + sleep 0 + for i in '$(seq 0 2)' + kubectl delete namespace self-healing-chaos-24626 + exit_status=1 + [[ 1 != 0 ]] + sleep 0 + cat /tmp/tmp.H9DXekzX5M + cat /tmp/tmp.eDQc9MdiQc Error from server (NotFound): namespaces "self-healing-chaos-24626" not found + rm /tmp/tmp.H9DXekzX5M /tmp/tmp.eDQc9MdiQc + return 1 + : + wait_for_delete namespace/self-healing-chaos-24626 + local res=namespace/self-healing-chaos-24626 + set +o xtrace namespace/self-healing-chaos-24626 - Error from server (NotFound): namespaces "self-healing-chaos-24626" not found + kubectl_bin create namespace self-healing-chaos-24626 ++ mktemp + local LAST_OUT=/tmp/tmp.1Hqt6d6NmX ++ mktemp + local LAST_ERR=/tmp/tmp.LmUqX3TLUr + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl create namespace self-healing-chaos-24626 + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.1Hqt6d6NmX namespace/self-healing-chaos-24626 created + cat /tmp/tmp.LmUqX3TLUr + rm /tmp/tmp.1Hqt6d6NmX /tmp/tmp.LmUqX3TLUr + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.iFQO44ABO2 +++ mktemp ++ local LAST_ERR=/tmp/tmp.xqhDorDYlR ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl config current-context ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.iFQO44ABO2 ++ cat /tmp/tmp.xqhDorDYlR ++ rm /tmp/tmp.iFQO44ABO2 /tmp/tmp.xqhDorDYlR ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jenkins-pxc-706f792a-selfhealing --namespace=self-healing-chaos-24626 ++ mktemp + local LAST_OUT=/tmp/tmp.7OkIMnH5w5 ++ mktemp + local LAST_ERR=/tmp/tmp.uTz3YgTBug + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jenkins-pxc-706f792a-selfhealing --namespace=self-healing-chaos-24626 + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.7OkIMnH5w5 Context "gke_cloud-dev-112233_us-central1-a_jenkins-pxc-706f792a-selfhealing" modified. + cat /tmp/tmp.uTz3YgTBug + rm /tmp/tmp.7OkIMnH5w5 /tmp/tmp.uTz3YgTBug + return 0 + apply_secrets + '[' -z '' ']' + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/conf/cloud-secret.yml ++ mktemp + local LAST_OUT=/tmp/tmp.UE9hLprBq2 ++ mktemp + local LAST_ERR=/tmp/tmp.7zZJBn54wD + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/conf/cloud-secret.yml + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.UE9hLprBq2 secret/minio-secret created secret/aws-s3-secret created secret/gcp-cs-secret created secret/azure-secret created + cat /tmp/tmp.7zZJBn54wD + rm /tmp/tmp.UE9hLprBq2 /tmp/tmp.7zZJBn54wD + return 0 + deploy_chaos_mesh self-healing-chaos-24626 + local chaos_mesh_ns=self-healing-chaos-24626 + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ tail -n1 ++ awk '-F ' '{print $2}' ++ sed s/NAMESPACE// + local chaos_mesh_ns= + desc 'destroy chaos-mesh' + set +o xtrace ----------------------------------------------------------------------------------- destroy chaos-mesh ----------------------------------------------------------------------------------- + timeout 30 kubectl delete podchaos --all --all-namespaces podchaos.chaos-mesh.org "chaos-pod-failure" deleted podchaos.chaos-mesh.org "chaos-pod-kill" deleted + : + timeout 30 kubectl delete networkchaos --all --all-namespaces networkchaos.chaos-mesh.org "chaos-pod-network-loss" deleted + : + '[' -n '' ']' + timeout 30 kubectl delete crd awschaos.chaos-mesh.org dnschaos.chaos-mesh.org gcpchaos.chaos-mesh.org httpchaos.chaos-mesh.org iochaos.chaos-mesh.org jvmchaos.chaos-mesh.org kernelchaos.chaos-mesh.org networkchaos.chaos-mesh.org podchaos.chaos-mesh.org podhttpchaos.chaos-mesh.org podiochaos.chaos-mesh.org podnetworkchaos.chaos-mesh.org schedules.chaos-mesh.org stresschaos.chaos-mesh.org timechaos.chaos-mesh.org workflownodes.chaos-mesh.org workflows.chaos-mesh.org customresourcedefinition.apiextensions.k8s.io "awschaos.chaos-mesh.org" deleted customresourcedefinition.apiextensions.k8s.io "dnschaos.chaos-mesh.org" deleted customresourcedefinition.apiextensions.k8s.io "gcpchaos.chaos-mesh.org" deleted customresourcedefinition.apiextensions.k8s.io "httpchaos.chaos-mesh.org" deleted customresourcedefinition.apiextensions.k8s.io "iochaos.chaos-mesh.org" deleted customresourcedefinition.apiextensions.k8s.io "jvmchaos.chaos-mesh.org" deleted customresourcedefinition.apiextensions.k8s.io "kernelchaos.chaos-mesh.org" deleted customresourcedefinition.apiextensions.k8s.io "networkchaos.chaos-mesh.org" deleted customresourcedefinition.apiextensions.k8s.io "podchaos.chaos-mesh.org" deleted customresourcedefinition.apiextensions.k8s.io "podhttpchaos.chaos-mesh.org" deleted customresourcedefinition.apiextensions.k8s.io "podiochaos.chaos-mesh.org" deleted customresourcedefinition.apiextensions.k8s.io "podnetworkchaos.chaos-mesh.org" deleted customresourcedefinition.apiextensions.k8s.io "schedules.chaos-mesh.org" deleted customresourcedefinition.apiextensions.k8s.io "stresschaos.chaos-mesh.org" deleted customresourcedefinition.apiextensions.k8s.io "timechaos.chaos-mesh.org" deleted customresourcedefinition.apiextensions.k8s.io "workflownodes.chaos-mesh.org" deleted customresourcedefinition.apiextensions.k8s.io "workflows.chaos-mesh.org" deleted + : + timeout 30 kubectl delete clusterrolebinding chaos-mesh-chaos-controller-manager-cluster-level Error from server (NotFound): clusterrolebindings.rbac.authorization.k8s.io "chaos-mesh-chaos-controller-manager-cluster-level" not found + : + timeout 30 kubectl delete clusterrole chaos-mesh-chaos-controller-manager-cluster-level chaos-mesh-chaos-controller-manager-target-namespace Error from server (NotFound): clusterroles.rbac.authorization.k8s.io "chaos-mesh-chaos-controller-manager-cluster-level" not found Error from server (NotFound): clusterroles.rbac.authorization.k8s.io "chaos-mesh-chaos-controller-manager-target-namespace" not found + : + desc 'install chaos-mesh' + set +o xtrace ----------------------------------------------------------------------------------- install chaos-mesh ----------------------------------------------------------------------------------- + helm repo add chaos-mesh https://charts.chaos-mesh.org "chaos-mesh" already exists with the same configuration, skipping + version_gt 1.19 ++ echo '1.20 >= 1.19' ++ bc -l + '[' 1 -eq 1 ']' + return 0 + helm install chaos-mesh chaos-mesh/chaos-mesh --namespace=self-healing-chaos-24626 --set chaosDaemon.runtime=containerd --set chaosDaemon.socketPath=/run/containerd/containerd.sock --set dashboard.create=false --version 2.0.4 --set clusterScoped=false --set controllerManager.targetNamespace=self-healing-chaos-24626 Error: INSTALLATION FAILED: rendered manifests contain a resource that already exists. Unable to continue with install: MutatingWebhookConfiguration "chaos-mesh-mutation" in namespace "" exists and cannot be imported into the current release: invalid ownership metadata; annotation validation error: key "meta.helm.sh/release-namespace" must equal "self-healing-chaos-24626": current value is "self-healing-chaos-27971"