++ echo 'Log: /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/logs/self-healing-advanced-chaos.log' Log: /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/logs/self-healing-advanced-chaos.log ++ '[' -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/conf/cloud-secret.yml ']' ++ SKIP_BACKUPS_TO_AWS_GCP= ++ oc get projects error: No Auth Provider found for name "gcp" +++ kubectl version -o json +++ grep '\-eks\-' +++ jq -r .serverVersion.gitVersion ++ '[' ']' ++ EKS=0 +++ /usr/bin/sed -r 's/[^0-9.]+//g' +++ jq -r '.serverVersion.major + "." + .serverVersion.minor' +++ kubectl version -o json ++ KUBE_VERSION=1.20 +++ /usr/bin/sed -re 's/.*SemVer:"([^"]+)".*/\1/; s/.*\bVersion:"([^"]+)".*/\1/' +++ helm version -c ++ HELM_VERSION=v3.8.2 ++ '[' v3 == v2 ']' + cluster=self-healing-advanced + main + create_infra self-healing-advanced-chaos-8775 + local ns=self-healing-advanced-chaos-8775 + '[' -n pxc-operator ']' + xargs -L 1 sh -xc 'kubectl patch pxc -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' + grep -v NAMESPACE + kubectl get pxc --all-namespaces -o wide No resources found + kubectl patch pxc -n sh --type=merge -p '{"metadata":{"finalizers":[]}}' error: resource(s) were provided, but no name, label selector, or --all flag specified + : + kubectl_bin delete pxc --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.smDTEKsi1y ++ mktemp + local LAST_ERR=/tmp/tmp.9JYbXcA3cB + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl delete pxc --all --all-namespaces + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.smDTEKsi1y No resources found + cat /tmp/tmp.9JYbXcA3cB + rm /tmp/tmp.smDTEKsi1y /tmp/tmp.9JYbXcA3cB + return 0 + kubectl_bin delete pxc-backup --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.i8AEOBAJwH ++ mktemp + local LAST_ERR=/tmp/tmp.KzlQRkpY1q + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl delete pxc-backup --all --all-namespaces + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.i8AEOBAJwH No resources found + cat /tmp/tmp.KzlQRkpY1q + rm /tmp/tmp.i8AEOBAJwH /tmp/tmp.KzlQRkpY1q + return 0 + kubectl_bin delete pxc-restore --all --all-namespaces ++ mktemp + local LAST_OUT=/tmp/tmp.w7xI9llsoJ ++ mktemp + local LAST_ERR=/tmp/tmp.TBYYRVA8DH + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl delete pxc-restore --all --all-namespaces + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.w7xI9llsoJ No resources found + cat /tmp/tmp.TBYYRVA8DH + rm /tmp/tmp.w7xI9llsoJ /tmp/tmp.TBYYRVA8DH + return 0 + create_namespace pxc-operator + local namespace=pxc-operator + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + '[' '!' -z '' ']' + kubectl_bin delete namespace pxc-operator + egrep -v '^kube-|^default|Terminating|pxc-operator|openshift|^NAME' ++ mktemp + awk '{print$1}' + xargs kubectl delete ns + local LAST_OUT=/tmp/tmp.kNsBYaTaFK + kubectl_bin get ns ++ mktemp ++ mktemp + local LAST_ERR=/tmp/tmp.uorhY9Tr8W + local exit_status=0 + local LAST_OUT=/tmp/tmp.t3poTpLTQi ++ seq 0 2 ++ mktemp + local LAST_ERR=/tmp/tmp.p4s2EIr0A1 + local exit_status=0 + for i in '$(seq 0 2)' + kubectl delete namespace pxc-operator ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl get ns + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.t3poTpLTQi + cat /tmp/tmp.p4s2EIr0A1 + rm /tmp/tmp.t3poTpLTQi /tmp/tmp.p4s2EIr0A1 + return 0 namespace "self-healing-advanced-chaos-15192" deleted + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.kNsBYaTaFK namespace "pxc-operator" deleted + cat /tmp/tmp.uorhY9Tr8W + rm /tmp/tmp.kNsBYaTaFK /tmp/tmp.uorhY9Tr8W + return 0 + wait_for_delete namespace/pxc-operator + local res=namespace/pxc-operator + set +o xtrace namespace/pxc-operator - Error from server (NotFound): namespaces "pxc-operator" not found + kubectl_bin create namespace pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.fMJP1mhYX4 ++ mktemp + local LAST_ERR=/tmp/tmp.8w3CuYddSc + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl create namespace pxc-operator + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.fMJP1mhYX4 namespace/pxc-operator created + cat /tmp/tmp.8w3CuYddSc + rm /tmp/tmp.fMJP1mhYX4 /tmp/tmp.8w3CuYddSc + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.q6eRTEeREL +++ mktemp ++ local LAST_ERR=/tmp/tmp.cyHQWNYkmy ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl config current-context ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.q6eRTEeREL ++ cat /tmp/tmp.cyHQWNYkmy ++ rm /tmp/tmp.q6eRTEeREL /tmp/tmp.cyHQWNYkmy ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jenkins-pxc-706f792a-selfhealing --namespace=pxc-operator ++ mktemp + local LAST_OUT=/tmp/tmp.mJFWlBZAem ++ mktemp + local LAST_ERR=/tmp/tmp.FK2YU3KiCe + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jenkins-pxc-706f792a-selfhealing --namespace=pxc-operator + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.mJFWlBZAem Context "gke_cloud-dev-112233_us-central1-a_jenkins-pxc-706f792a-selfhealing" modified. + cat /tmp/tmp.FK2YU3KiCe + rm /tmp/tmp.mJFWlBZAem /tmp/tmp.FK2YU3KiCe + return 0 + deploy_operator + desc 'start operator' + set +o xtrace ----------------------------------------------------------------------------------- start operator ----------------------------------------------------------------------------------- + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/deploy/crd.yaml ++ mktemp + local LAST_OUT=/tmp/tmp.UrRglIz9hb ++ mktemp + local LAST_ERR=/tmp/tmp.AyONcpvbcm + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/deploy/crd.yaml + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.UrRglIz9hb customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com unchanged customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterbackups.pxc.percona.com unchanged customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterrestores.pxc.percona.com unchanged customresourcedefinition.apiextensions.k8s.io/perconaxtradbbackups.pxc.percona.com configured + cat /tmp/tmp.AyONcpvbcm + rm /tmp/tmp.UrRglIz9hb /tmp/tmp.AyONcpvbcm + return 0 + '[' -n pxc-operator ']' + apply_rbac cw-rbac + local operator_namespace=pxc-operator + local rbac=cw-rbac + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/deploy/cw-rbac.yaml + kubectl_bin apply -f - + sed -e 's^namespace: .*^namespace: pxc-operator^' ++ mktemp + local LAST_OUT=/tmp/tmp.7vkrNS18qH ++ mktemp + local LAST_ERR=/tmp/tmp.NgdlZ1Pf3Q + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl apply -f - + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.7vkrNS18qH clusterrole.rbac.authorization.k8s.io/percona-xtradb-cluster-operator unchanged serviceaccount/percona-xtradb-cluster-operator created clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-xtradb-cluster-operator unchanged + cat /tmp/tmp.NgdlZ1Pf3Q + rm /tmp/tmp.7vkrNS18qH /tmp/tmp.NgdlZ1Pf3Q + return 0 + cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/deploy/cw-operator.yaml + sed -e 's^failureThreshold: .*^failureThreshold: 10^' + kubectl_bin apply -f - ++ mktemp + sed -e 's^image: .*^image: perconalab/percona-xtradb-cluster-operator:PR-1125-706f792a^' + local LAST_OUT=/tmp/tmp.Tg4sIXtFY8 ++ mktemp + local LAST_ERR=/tmp/tmp.h1sT7f427v + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl apply -f - + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.Tg4sIXtFY8 deployment.apps/percona-xtradb-cluster-operator created service/percona-xtradb-cluster-operator created + cat /tmp/tmp.h1sT7f427v + rm /tmp/tmp.Tg4sIXtFY8 /tmp/tmp.h1sT7f427v + return 0 + sleep 10 ++ get_operator_pod ++ local label_prefix=app.kubernetes.io/ +++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator +++ grep -c percona-xtradb-cluster-operator ++ local check_label=1 ++ [[ 1 -eq 0 ]] ++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator +++ mktemp ++ local LAST_OUT=/tmp/tmp.6m2PahGnAI +++ mktemp ++ local LAST_ERR=/tmp/tmp.rdUGblrAua ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.6m2PahGnAI ++ cat /tmp/tmp.rdUGblrAua ++ rm /tmp/tmp.6m2PahGnAI /tmp/tmp.rdUGblrAua ++ return 0 + wait_pod percona-xtradb-cluster-operator-5699d7755d-stzvv 480 pxc-operator + local pod=percona-xtradb-cluster-operator-5699d7755d-stzvv + local max_retry=480 + local ns=pxc-operator ++ echo percona-xtradb-cluster-operator-5699d7755d-stzvv ++ egrep '^(pxc|proxysql)$' ++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' + local container= + set +o xtrace percona-xtradb-cluster-operator-5699d7755d-stzvv.Ok + sleep 3 + create_namespace self-healing-advanced-chaos-8775 + local namespace=self-healing-advanced-chaos-8775 + local skip_clean_namespace= + [[ 1 == 1 ]] + [[ -z '' ]] + '[' '!' -z '' ']' + kubectl_bin delete namespace self-healing-advanced-chaos-8775 + kubectl_bin get ns + egrep -v '^kube-|^default|Terminating|pxc-operator|openshift|^NAME' + awk '{print$1}' + xargs kubectl delete ns ++ mktemp ++ mktemp + local LAST_OUT=/tmp/tmp.kgJUtEjP38 ++ mktemp + local LAST_OUT=/tmp/tmp.UxsUZPe26o ++ mktemp + local LAST_ERR=/tmp/tmp.Hny7gKSWkQ + local exit_status=0 ++ seq 0 2 + local LAST_ERR=/tmp/tmp.aHOZyZtkS5 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl delete namespace self-healing-advanced-chaos-8775 + for i in '$(seq 0 2)' + kubectl get ns + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.UxsUZPe26o + cat /tmp/tmp.aHOZyZtkS5 + rm /tmp/tmp.UxsUZPe26o /tmp/tmp.aHOZyZtkS5 + return 0 + exit_status=1 + [[ 1 != 0 ]] + sleep 0 + for i in '$(seq 0 2)' + kubectl delete namespace self-healing-advanced-chaos-8775 error: resource(s) were provided, but no name, label selector, or --all flag specified + exit_status=1 + [[ 1 != 0 ]] + sleep 0 + for i in '$(seq 0 2)' + kubectl delete namespace self-healing-advanced-chaos-8775 + exit_status=1 + [[ 1 != 0 ]] + sleep 0 + cat /tmp/tmp.kgJUtEjP38 + cat /tmp/tmp.Hny7gKSWkQ Error from server (NotFound): namespaces "self-healing-advanced-chaos-8775" not found + rm /tmp/tmp.kgJUtEjP38 /tmp/tmp.Hny7gKSWkQ + return 1 + : + wait_for_delete namespace/self-healing-advanced-chaos-8775 + local res=namespace/self-healing-advanced-chaos-8775 + set +o xtrace namespace/self-healing-advanced-chaos-8775 - Error from server (NotFound): namespaces "self-healing-advanced-chaos-8775" not found + kubectl_bin create namespace self-healing-advanced-chaos-8775 ++ mktemp + local LAST_OUT=/tmp/tmp.G16RutJkaU ++ mktemp + local LAST_ERR=/tmp/tmp.rrLUNVZ2Tk + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl create namespace self-healing-advanced-chaos-8775 + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.G16RutJkaU namespace/self-healing-advanced-chaos-8775 created + cat /tmp/tmp.rrLUNVZ2Tk + rm /tmp/tmp.G16RutJkaU /tmp/tmp.rrLUNVZ2Tk + return 0 ++ kubectl_bin config current-context +++ mktemp ++ local LAST_OUT=/tmp/tmp.lp6P2eZHwH +++ mktemp ++ local LAST_ERR=/tmp/tmp.KWXFmt5jl0 ++ local exit_status=0 +++ seq 0 2 ++ for i in '$(seq 0 2)' ++ kubectl config current-context ++ exit_status=0 ++ [[ 0 != 0 ]] ++ break ++ cat /tmp/tmp.lp6P2eZHwH ++ cat /tmp/tmp.KWXFmt5jl0 ++ rm /tmp/tmp.lp6P2eZHwH /tmp/tmp.KWXFmt5jl0 ++ return 0 + kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jenkins-pxc-706f792a-selfhealing --namespace=self-healing-advanced-chaos-8775 ++ mktemp + local LAST_OUT=/tmp/tmp.zCfHJzudxi ++ mktemp + local LAST_ERR=/tmp/tmp.4CuDH1a4zu + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl config set-context gke_cloud-dev-112233_us-central1-a_jenkins-pxc-706f792a-selfhealing --namespace=self-healing-advanced-chaos-8775 + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.zCfHJzudxi Context "gke_cloud-dev-112233_us-central1-a_jenkins-pxc-706f792a-selfhealing" modified. + cat /tmp/tmp.4CuDH1a4zu + rm /tmp/tmp.zCfHJzudxi /tmp/tmp.4CuDH1a4zu + return 0 + apply_secrets + '[' -z '' ']' + kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/conf/cloud-secret.yml ++ mktemp + local LAST_OUT=/tmp/tmp.s5UiJvBJBb ++ mktemp + local LAST_ERR=/tmp/tmp.tU13YsYQp1 + local exit_status=0 ++ seq 0 2 + for i in '$(seq 0 2)' + kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-1125/e2e-tests/conf/cloud-secret.yml + exit_status=0 + [[ 0 != 0 ]] + break + cat /tmp/tmp.s5UiJvBJBb secret/minio-secret created secret/aws-s3-secret created secret/gcp-cs-secret created secret/azure-secret created + cat /tmp/tmp.tU13YsYQp1 + rm /tmp/tmp.s5UiJvBJBb /tmp/tmp.tU13YsYQp1 + return 0 + deploy_chaos_mesh self-healing-advanced-chaos-8775 + local chaos_mesh_ns=self-healing-advanced-chaos-8775 + destroy_chaos_mesh ++ helm list --all-namespaces --filter chaos-mesh ++ awk '-F ' '{print $2}' ++ sed s/NAMESPACE// ++ tail -n1 + local chaos_mesh_ns= + desc 'destroy chaos-mesh' + set +o xtrace ----------------------------------------------------------------------------------- destroy chaos-mesh ----------------------------------------------------------------------------------- + timeout 30 kubectl delete podchaos --all --all-namespaces podchaos.chaos-mesh.org "chaos-pod-failure" deleted podchaos.chaos-mesh.org "chaos-pod-kill" deleted + : + timeout 30 kubectl delete networkchaos --all --all-namespaces networkchaos.chaos-mesh.org "chaos-pod-network-loss" deleted + : + '[' -n '' ']' + timeout 30 kubectl delete crd awschaos.chaos-mesh.org dnschaos.chaos-mesh.org gcpchaos.chaos-mesh.org httpchaos.chaos-mesh.org iochaos.chaos-mesh.org jvmchaos.chaos-mesh.org kernelchaos.chaos-mesh.org networkchaos.chaos-mesh.org podchaos.chaos-mesh.org podhttpchaos.chaos-mesh.org podiochaos.chaos-mesh.org podnetworkchaos.chaos-mesh.org schedules.chaos-mesh.org stresschaos.chaos-mesh.org timechaos.chaos-mesh.org workflownodes.chaos-mesh.org workflows.chaos-mesh.org customresourcedefinition.apiextensions.k8s.io "awschaos.chaos-mesh.org" deleted customresourcedefinition.apiextensions.k8s.io "dnschaos.chaos-mesh.org" deleted customresourcedefinition.apiextensions.k8s.io "gcpchaos.chaos-mesh.org" deleted customresourcedefinition.apiextensions.k8s.io "httpchaos.chaos-mesh.org" deleted customresourcedefinition.apiextensions.k8s.io "iochaos.chaos-mesh.org" deleted customresourcedefinition.apiextensions.k8s.io "jvmchaos.chaos-mesh.org" deleted customresourcedefinition.apiextensions.k8s.io "kernelchaos.chaos-mesh.org" deleted customresourcedefinition.apiextensions.k8s.io "networkchaos.chaos-mesh.org" deleted customresourcedefinition.apiextensions.k8s.io "podchaos.chaos-mesh.org" deleted customresourcedefinition.apiextensions.k8s.io "podhttpchaos.chaos-mesh.org" deleted customresourcedefinition.apiextensions.k8s.io "podiochaos.chaos-mesh.org" deleted customresourcedefinition.apiextensions.k8s.io "podnetworkchaos.chaos-mesh.org" deleted customresourcedefinition.apiextensions.k8s.io "schedules.chaos-mesh.org" deleted customresourcedefinition.apiextensions.k8s.io "stresschaos.chaos-mesh.org" deleted customresourcedefinition.apiextensions.k8s.io "timechaos.chaos-mesh.org" deleted customresourcedefinition.apiextensions.k8s.io "workflownodes.chaos-mesh.org" deleted customresourcedefinition.apiextensions.k8s.io "workflows.chaos-mesh.org" deleted + : + timeout 30 kubectl delete clusterrolebinding chaos-mesh-chaos-controller-manager-cluster-level Error from server (NotFound): clusterrolebindings.rbac.authorization.k8s.io "chaos-mesh-chaos-controller-manager-cluster-level" not found + : + timeout 30 kubectl delete clusterrole chaos-mesh-chaos-controller-manager-cluster-level chaos-mesh-chaos-controller-manager-target-namespace Error from server (NotFound): clusterroles.rbac.authorization.k8s.io "chaos-mesh-chaos-controller-manager-cluster-level" not found Error from server (NotFound): clusterroles.rbac.authorization.k8s.io "chaos-mesh-chaos-controller-manager-target-namespace" not found + : + desc 'install chaos-mesh' + set +o xtrace ----------------------------------------------------------------------------------- install chaos-mesh ----------------------------------------------------------------------------------- + helm repo add chaos-mesh https://charts.chaos-mesh.org "chaos-mesh" already exists with the same configuration, skipping + version_gt 1.19 ++ bc -l ++ echo '1.20 >= 1.19' + '[' 1 -eq 1 ']' + return 0 + helm install chaos-mesh chaos-mesh/chaos-mesh --namespace=self-healing-advanced-chaos-8775 --set chaosDaemon.runtime=containerd --set chaosDaemon.socketPath=/run/containerd/containerd.sock --set dashboard.create=false --version 2.0.4 --set clusterScoped=false --set controllerManager.targetNamespace=self-healing-advanced-chaos-8775 Error: INSTALLATION FAILED: rendered manifests contain a resource that already exists. Unable to continue with install: MutatingWebhookConfiguration "chaos-mesh-mutation" in namespace "" exists and cannot be imported into the current release: invalid ownership metadata; annotation validation error: key "meta.helm.sh/release-namespace" must equal "self-healing-advanced-chaos-8775": current value is "self-healing-chaos-27971"