Log: /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/logs/recreate-8-0.log
Warning: version difference between client (1.34) and server (1.31) exceeds the supported minor version skew of +/-1
Warning: version difference between client (1.34) and server (1.31) exceeds the supported minor version skew of +/-1
+ create_infra recreate-20426
+ local ns=recreate-20426
+ '[' -n pxc-operator ']'
+ kubectl get pxc --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch pxc -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
No resources found
+ kubectl patch pxc -n sh --type=merge -p '{"metadata":{"finalizers":[]}}'
error: resource(s) were provided, but no name was specified
+ :
+ kubectl_bin delete pxc --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.lXAo5zi8lL
++ mktemp
+ local LAST_ERR=/tmp/tmp.cRGlb0oSgG
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.lXAo5zi8lL
No resources found
+ cat /tmp/tmp.cRGlb0oSgG
+ rm /tmp/tmp.lXAo5zi8lL /tmp/tmp.cRGlb0oSgG
+ return 0
+ kubectl_bin delete pxc-backup --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.jbwppGAbHv
++ mktemp
+ local LAST_ERR=/tmp/tmp.q75tyFRgce
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc-backup --all --all-namespaces
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc-backup --all --all-namespaces
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc-backup --all --all-namespaces
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ cat /tmp/tmp.jbwppGAbHv
+ cat /tmp/tmp.q75tyFRgce
error: the server doesn't have a resource type "pxc-backup"
+ rm /tmp/tmp.jbwppGAbHv /tmp/tmp.q75tyFRgce
+ return 1
+ :
+ kubectl_bin delete pxc-restore --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.fbMxHdFBiq
++ mktemp
+ local LAST_ERR=/tmp/tmp.ytIWL6yGaW
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc-restore --all --all-namespaces
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc-restore --all --all-namespaces
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc-restore --all --all-namespaces
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ cat /tmp/tmp.fbMxHdFBiq
+ cat /tmp/tmp.ytIWL6yGaW
error: the server doesn't have a resource type "pxc-restore"
+ rm /tmp/tmp.fbMxHdFBiq /tmp/tmp.ytIWL6yGaW
+ return 1
+ :
+ create_namespace pxc-operator
+ local namespace=pxc-operator
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ sed s/NAMESPACE//
++ awk '-F ' '{print $2}'
++ tail -n1
+ local chaos_mesh_ns=
+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ egrep -v '^kube-|^default|Terminating|pxc-operator|openshift|^gke-|^gmp-|^NAME'
+ awk '{print$1}'
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces pxc-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces pxc-operator
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace pxc-operator
+ kubectl_bin get ns
++ mktemp
+ local LAST_OUT=/tmp/tmp.As27Ztmms7
++ mktemp
+ local LAST_ERR=/tmp/tmp.1lQG9sFN92
+ local exit_status=0
++ seq 0 2
++ mktemp
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get ns
+ local LAST_OUT=/tmp/tmp.SiT8npZ2BQ
+ xargs kubectl delete ns
++ mktemp
+ local LAST_ERR=/tmp/tmp.hQkXxfuNKg
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace pxc-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.As27Ztmms7
+ cat /tmp/tmp.1lQG9sFN92
+ rm /tmp/tmp.As27Ztmms7 /tmp/tmp.1lQG9sFN92
+ return 0
namespace "recreate-22622" deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.SiT8npZ2BQ
namespace "pxc-operator" deleted
+ cat /tmp/tmp.hQkXxfuNKg
+ rm /tmp/tmp.SiT8npZ2BQ /tmp/tmp.hQkXxfuNKg
+ return 0
+ wait_for_delete namespace/pxc-operator
+ local res=namespace/pxc-operator
+ echo -n 'waiting for namespace/pxc-operator to be deleted'
waiting for namespace/pxc-operator to be deleted+ set +o xtrace
Error from server (NotFound): namespaces "pxc-operator" not found
+ desc 'create namespace pxc-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace pxc-operator
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace pxc-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.jrbGN7qv1T
++ mktemp
+ local LAST_ERR=/tmp/tmp.TnPNwkSFsP
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace pxc-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.jrbGN7qv1T
namespace/pxc-operator created
+ cat /tmp/tmp.TnPNwkSFsP
+ rm /tmp/tmp.jrbGN7qv1T /tmp/tmp.TnPNwkSFsP
+ return 0
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.zIUsSSA0II
+++ mktemp
++ local LAST_ERR=/tmp/tmp.T7MYvPPAwd
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.zIUsSSA0II
++ cat /tmp/tmp.T7MYvPPAwd
++ rm /tmp/tmp.zIUsSSA0II /tmp/tmp.T7MYvPPAwd
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2207-f8e092d7-9-cluster7 --namespace=pxc-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.6c7N24Ei3g
++ mktemp
+ local LAST_ERR=/tmp/tmp.HmO2crZc8Q
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2207-f8e092d7-9-cluster7 --namespace=pxc-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.6c7N24Ei3g
Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2207-f8e092d7-9-cluster7" modified.
+ cat /tmp/tmp.HmO2crZc8Q
+ rm /tmp/tmp.6c7N24Ei3g /tmp/tmp.HmO2crZc8Q
+ return 0
+ deploy_operator
+ desc 'start PXC operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
start PXC operator
-----------------------------------------------------------------------------------

+ kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/deploy/crd.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.7XqqXtCIHp
++ mktemp
+ local LAST_ERR=/tmp/tmp.1V8raic5Id
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/deploy/crd.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.7XqqXtCIHp
customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com serverside-applied
+ cat /tmp/tmp.1V8raic5Id
+ rm /tmp/tmp.7XqqXtCIHp /tmp/tmp.1V8raic5Id
+ return 0
+ '[' -n pxc-operator ']'
+ apply_rbac cw-rbac
+ local operator_namespace=pxc-operator
+ local rbac=cw-rbac
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/deploy/cw-rbac.yaml
+ sed -e 's^namespace: .*^namespace: pxc-operator^'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.3dxIGh9Ego
++ mktemp
+ local LAST_ERR=/tmp/tmp.XU08FnJooe
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.3dxIGh9Ego
clusterrole.rbac.authorization.k8s.io/percona-xtradb-cluster-operator unchanged
serviceaccount/percona-xtradb-cluster-operator created
clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-xtradb-cluster-operator unchanged
+ cat /tmp/tmp.XU08FnJooe
+ rm /tmp/tmp.3dxIGh9Ego /tmp/tmp.XU08FnJooe
+ return 0
+ sed -e 's^image: .*^image: perconalab/percona-xtradb-cluster-operator:PR-2207-f8e092d7^'
+ sed -e 's^failureThreshold: .*^failureThreshold: 10^'
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/deploy/cw-operator.yaml
+ yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "DISABLE_TELEMETRY").value) = "true"' -
+ kubectl_bin apply -f -
+ yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "LOG_LEVEL").value) = "VERBOSE"' -
++ mktemp
+ local LAST_OUT=/tmp/tmp.kwxBWCa0Hc
++ mktemp
+ local LAST_ERR=/tmp/tmp.S6t8I1nvGA
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.kwxBWCa0Hc
deployment.apps/percona-xtradb-cluster-operator created
service/percona-xtradb-cluster-operator created
+ cat /tmp/tmp.S6t8I1nvGA
+ rm /tmp/tmp.kwxBWCa0Hc /tmp/tmp.S6t8I1nvGA
+ return 0
+ sleep 10
+ kubectl_bin wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s
++ mktemp
+ local LAST_OUT=/tmp/tmp.WdUXOEGr2O
++ mktemp
+ local LAST_ERR=/tmp/tmp.WMGhnH8kEE
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.WdUXOEGr2O
pod/percona-xtradb-cluster-operator-859595f865-d7sg4 condition met
+ cat /tmp/tmp.WMGhnH8kEE
+ rm /tmp/tmp.WdUXOEGr2O /tmp/tmp.WMGhnH8kEE
+ return 0
++ get_operator_pod
++ local label_prefix=app.kubernetes.io/
+++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator
+++ grep -c percona-xtradb-cluster-operator
++ local check_label=1
++ [[ 1 -eq 0 ]]
++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.0xstePkXKS
+++ mktemp
++ local LAST_ERR=/tmp/tmp.JTwV4L1omC
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.0xstePkXKS
++ cat /tmp/tmp.JTwV4L1omC
++ rm /tmp/tmp.0xstePkXKS /tmp/tmp.JTwV4L1omC
++ return 0
+ wait_pod percona-xtradb-cluster-operator-859595f865-d7sg4 480 pxc-operator
+ local pod=percona-xtradb-cluster-operator-859595f865-d7sg4
+ local max_retry=480
+ local ns=pxc-operator
++ echo percona-xtradb-cluster-operator-859595f865-d7sg4
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/percona-xtradb-cluster-operator-859595f865-d7sg4 condition met
waiting for pod/percona-xtradb-cluster-operator-859595f865-d7sg4 to become Ready.Ok
+ sleep 3
+ create_namespace recreate-20426
+ local namespace=recreate-20426
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ helm list --all-namespaces --filter chaos-mesh
++ tail -n1
++ sed s/NAMESPACE//
++ awk '-F ' '{print $2}'
+ local chaos_mesh_ns=
+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get ValidatingWebhookConfiguration
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ grep validate-auth
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl get crd
++ grep chaos-mesh.org
++ awk '{print $1}'
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrole
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ kubectl_bin get ns
+ awk '{print$1}'
+ '[' -n '' ']'
+ desc 'cleaned up old namespaces recreate-20426'
+ xargs kubectl delete ns
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces recreate-20426
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace recreate-20426
+ egrep -v '^kube-|^default|Terminating|pxc-operator|openshift|^gke-|^gmp-|^NAME'
++ mktemp
++ mktemp
+ local LAST_OUT=/tmp/tmp.tJytmbTVfH
++ mktemp
+ local LAST_OUT=/tmp/tmp.VwiFt2eBKd
++ mktemp
+ local LAST_ERR=/tmp/tmp.fgfNpLFYwJ
+ local exit_status=0
+ local LAST_ERR=/tmp/tmp.9Cp5JjIKvm
+ local exit_status=0
++ seq 0 2
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace recreate-20426
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get ns
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.VwiFt2eBKd
+ cat /tmp/tmp.9Cp5JjIKvm
+ rm /tmp/tmp.VwiFt2eBKd /tmp/tmp.9Cp5JjIKvm
+ return 0
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace recreate-20426
error: resource(s) were provided, but no name was specified
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace recreate-20426
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ cat /tmp/tmp.tJytmbTVfH
+ cat /tmp/tmp.fgfNpLFYwJ
Error from server (NotFound): namespaces "recreate-20426" not found
+ rm /tmp/tmp.tJytmbTVfH /tmp/tmp.fgfNpLFYwJ
+ return 1
+ :
+ wait_for_delete namespace/recreate-20426
+ local res=namespace/recreate-20426
+ echo -n 'waiting for namespace/recreate-20426 to be deleted'
waiting for namespace/recreate-20426 to be deleted+ set +o xtrace
Error from server (NotFound): namespaces "recreate-20426" not found
+ desc 'create namespace recreate-20426'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace recreate-20426
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace recreate-20426
++ mktemp
+ local LAST_OUT=/tmp/tmp.byJ1rNZwO2
++ mktemp
+ local LAST_ERR=/tmp/tmp.BGX6XRhETn
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace recreate-20426
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.byJ1rNZwO2
namespace/recreate-20426 created
+ cat /tmp/tmp.BGX6XRhETn
+ rm /tmp/tmp.byJ1rNZwO2 /tmp/tmp.BGX6XRhETn
+ return 0
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.Q3zs5105xm
+++ mktemp
++ local LAST_ERR=/tmp/tmp.vpgaZKGSeX
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.Q3zs5105xm
++ cat /tmp/tmp.vpgaZKGSeX
++ rm /tmp/tmp.Q3zs5105xm /tmp/tmp.vpgaZKGSeX
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2207-f8e092d7-9-cluster7 --namespace=recreate-20426
++ mktemp
+ local LAST_OUT=/tmp/tmp.zLIy7oOJvs
++ mktemp
+ local LAST_ERR=/tmp/tmp.36cxwTi4vd
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2207-f8e092d7-9-cluster7 --namespace=recreate-20426
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.zLIy7oOJvs
Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2207-f8e092d7-9-cluster7" modified.
+ cat /tmp/tmp.36cxwTi4vd
+ rm /tmp/tmp.zLIy7oOJvs /tmp/tmp.36cxwTi4vd
+ return 0
+ apply_secrets
+ desc 'create secrets for cloud storages'
+ set +o xtrace


-----------------------------------------------------------------------------------
create secrets for cloud storages
-----------------------------------------------------------------------------------

+ '[' -z '' ']'
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/conf/cloud-secret.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.A61ATVIk0l
++ mktemp
+ local LAST_ERR=/tmp/tmp.wI4vsGFLfw
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/conf/cloud-secret.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.A61ATVIk0l
secret/minio-secret created
secret/aws-s3-secret created
secret/gcp-cs-secret created
secret/azure-secret created
+ cat /tmp/tmp.wI4vsGFLfw
+ rm /tmp/tmp.A61ATVIk0l /tmp/tmp.wI4vsGFLfw
+ return 0
+ desc 'create first PXC cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
create first PXC cluster
-----------------------------------------------------------------------------------

+ cluster=some-name
+ spinup_pxc some-name /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/conf/some-name.yml
+ local cluster=some-name
+ local config=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/conf/some-name.yml
+ local size=3
+ local sleep=10
+ local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/conf/secrets.yml
+ local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/conf/client.yml
+ local port=3306
+ desc 'create first PXC cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
create first PXC cluster
-----------------------------------------------------------------------------------

+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/conf/secrets.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.x1YDPqogqv
++ mktemp
+ local LAST_ERR=/tmp/tmp.GTPQOlan61
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/conf/secrets.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.x1YDPqogqv
secret/my-cluster-secrets created
secret/some-name-ssl created
secret/some-name-ssl-internal created
+ cat /tmp/tmp.GTPQOlan61
+ rm /tmp/tmp.x1YDPqogqv /tmp/tmp.GTPQOlan61
+ return 0
+ apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/conf/client.yml
+ '[' -z '' ']'
+ cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/conf/client.yml
+ kubectl_bin apply -f -
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/conf/client.yml
+ /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#'
++ mktemp
+ /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#'
+ local LAST_OUT=/tmp/tmp.tf5dlOrrQC
+ /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#'
+ /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#'
+ /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#'
+ /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#'
+ /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2207-f8e092d7#'
++ mktemp
+ /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#'
+ /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#'
+ /usr/bin/sed -e s~minio-service.#namespace~minio-service.recreate-20426~
+ /usr/bin/sed -e 's#apply:.*#apply: Never#'
+ local LAST_ERR=/tmp/tmp.RyjLRxEJF0
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.tf5dlOrrQC
deployment.apps/pxc-client created
+ cat /tmp/tmp.RyjLRxEJF0
+ rm /tmp/tmp.tf5dlOrrQC /tmp/tmp.RyjLRxEJF0
+ return 0
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]]
+ apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/conf/some-name.yml
+ '[' -z '' ']'
+ kubectl_bin apply -f -
+ cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/conf/some-name.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.CJUqCEVtTI
+ /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#'
++ mktemp
+ local LAST_ERR=/tmp/tmp.0NvLAwYLUH
+ local exit_status=0
++ seq 0 2
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/conf/some-name.yml
+ /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2207-f8e092d7#'
+ /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#'
+ /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#'
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#'
+ /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#'
+ /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#'
+ /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#'
+ /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#'
+ /usr/bin/sed -e s~minio-service.#namespace~minio-service.recreate-20426~
+ /usr/bin/sed -e 's#apply:.*#apply: Never#'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.CJUqCEVtTI
perconaxtradbcluster.pxc.percona.com/some-name created
+ cat /tmp/tmp.0NvLAwYLUH
+ rm /tmp/tmp.CJUqCEVtTI /tmp/tmp.0NvLAwYLUH
+ return 0
+ desc 'check if all 3 Pods started'
+ set +o xtrace


-----------------------------------------------------------------------------------
check if all 3 Pods started
-----------------------------------------------------------------------------------

++ get_proxy some-name
++ local target_cluster=some-name
+++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.fJSFqNIkcj
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.Rs2LPs6ChQ
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pxc some-name -o 'jsonpath={.spec.haproxy.enabled}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.fJSFqNIkcj
+++ cat /tmp/tmp.Rs2LPs6ChQ
+++ rm /tmp/tmp.fJSFqNIkcj /tmp/tmp.Rs2LPs6ChQ
+++ return 0
++ [[ '' == \t\r\u\e ]]
+++ kubectl_bin get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.YK6K9V0tBh
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.txGdy35Lmp
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pxc some-name -o 'jsonpath={.spec.proxysql.enabled}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.YK6K9V0tBh
+++ cat /tmp/tmp.txGdy35Lmp
+++ rm /tmp/tmp.YK6K9V0tBh /tmp/tmp.txGdy35Lmp
+++ return 0
++ [[ true == \t\r\u\e ]]
++ echo some-name-proxysql
++ return
+ local proxy=some-name-proxysql
+ kubectl_bin wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n recreate-20426
++ mktemp
+ local LAST_OUT=/tmp/tmp.jWeqvNltKf
++ mktemp
+ local LAST_ERR=/tmp/tmp.BhVWbZEdWV
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n recreate-20426
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n recreate-20426
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=monitoring,app.kubernetes.io/managed-by=percona-xtradb-cluster-operator --timeout=300s -n recreate-20426
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ cat /tmp/tmp.jWeqvNltKf
+ cat /tmp/tmp.BhVWbZEdWV
error: no matching resources found
+ rm /tmp/tmp.jWeqvNltKf /tmp/tmp.BhVWbZEdWV
+ return 1
+ true
+ wait_for_running some-name-proxysql 1
+ local name=some-name-proxysql
+ let last_pod=0
+ :
+ local max_retry=480
+ desc 'wait for running cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait for running cluster
-----------------------------------------------------------------------------------

++ seq 0 0
+ for i in '$(seq 0 $last_pod)'
+ wait_pod some-name-proxysql-0 480
+ local pod=some-name-proxysql-0
+ local max_retry=480
+ local ns=
++ echo some-name-proxysql-0
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=proxysql
+ set +o xtrace
pod/some-name-proxysql-0 condition met
waiting for pod/some-name-proxysql-0 to become Ready.Ok
+ wait_for_running some-name-pxc 3
+ local name=some-name-pxc
+ let last_pod=2
+ local max_retry=480
+ desc 'wait for running cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait for running cluster
-----------------------------------------------------------------------------------

++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ wait_pod some-name-pxc-0 480
+ local pod=some-name-pxc-0
+ local max_retry=480
+ local ns=
++ echo some-name-pxc-0
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=pxc
+ set +o xtrace
pod/some-name-pxc-0 condition met
waiting for pod/some-name-pxc-0 to become Ready.Ok
+ for i in '$(seq 0 $last_pod)'
+ wait_pod some-name-pxc-1 480
+ local pod=some-name-pxc-1
+ local max_retry=480
+ local ns=
++ echo some-name-pxc-1
++ egrep '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+ local container=pxc
+ set +o xtrace
pod/some-name-pxc-1 condition met
waiting for pod/some-name-pxc-1 to become Ready.Ok
+ for i in '$(seq 0 $last_pod)'
+ wait_pod some-name-pxc-2 480
+ local pod=some-name-pxc-2
+ local max_retry=480
+ local ns=
++ echo some-name-pxc-2
++ egrep '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+ local container=pxc
+ set +o xtrace
pod/some-name-pxc-2 condition met
waiting for pod/some-name-pxc-2 to become Ready.Ok
+ sleep 10
++ kubectl get pxc some-name -o 'jsonpath={.spec.secretsName}'
+ local secret_name=my-cluster-secrets
++ getSecretData my-cluster-secrets root
++ local secretName=my-cluster-secrets
++ local dataKey=root
++ kubectl_bin get secrets/my-cluster-secrets '--template={{.data.root}}'
++ base64 --decode
+++ mktemp
++ local LAST_OUT=/tmp/tmp.2aMufADCiY
+++ mktemp
++ local LAST_ERR=/tmp/tmp.b80ypYohoe
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get secrets/my-cluster-secrets '--template={{.data.root}}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.2aMufADCiY
++ cat /tmp/tmp.b80ypYohoe
++ rm /tmp/tmp.2aMufADCiY /tmp/tmp.b80ypYohoe
++ return 0
+ local root_pass=root_password
+ desc 'write data'
+ set +o xtrace


-----------------------------------------------------------------------------------
write data
-----------------------------------------------------------------------------------

+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]]
+ run_mysql 'CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;' '-h some-name-proxysql -uroot -p'\''root_password'\'' -P3306'
+ local 'command=CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) ;'
+ local 'uri=-h some-name-proxysql -uroot -p'\''root_password'\'' -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.TI8uBrom5V
+++ mktemp
++ local LAST_ERR=/tmp/tmp.6fiYKnvkIF
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.TI8uBrom5V
++ cat /tmp/tmp.6fiYKnvkIF
++ rm /tmp/tmp.TI8uBrom5V /tmp/tmp.6fiYKnvkIF
++ return 0
+ client_pod=pxc-client-59944c5bbf-wxzvx
+ wait_pod pxc-client-59944c5bbf-wxzvx
+ local pod=pxc-client-59944c5bbf-wxzvx
+ local max_retry=480
+ local ns=
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ echo pxc-client-59944c5bbf-wxzvx
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-59944c5bbf-wxzvx condition met
waiting for pod/pxc-client-59944c5bbf-wxzvx to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ run_mysql 'INSERT myApp.myApp (id) VALUES (100500)' '-h some-name-proxysql -uroot -p'\''root_password'\'' -P3306'
+ local 'command=INSERT myApp.myApp (id) VALUES (100500)'
+ local 'uri=-h some-name-proxysql -uroot -p'\''root_password'\'' -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.p3qjCwWA7O
+++ mktemp
++ local LAST_ERR=/tmp/tmp.YRwhAhfyAj
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.p3qjCwWA7O
++ cat /tmp/tmp.YRwhAhfyAj
++ rm /tmp/tmp.p3qjCwWA7O /tmp/tmp.YRwhAhfyAj
++ return 0
+ client_pod=pxc-client-59944c5bbf-wxzvx
+ wait_pod pxc-client-59944c5bbf-wxzvx
+ local pod=pxc-client-59944c5bbf-wxzvx
+ local max_retry=480
+ local ns=
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ echo pxc-client-59944c5bbf-wxzvx
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-59944c5bbf-wxzvx condition met
waiting for pod/pxc-client-59944c5bbf-wxzvx to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ sleep 30
++ seq 0 2
+ for i in '$(seq 0 $((size - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h some-name-pxc-0.some-name-pxc -uroot -p'\''root_password'\'' -P3306'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h some-name-pxc-0.some-name-pxc -uroot -p'\''root_password'\'' -P3306'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/recreate/compare/select-1.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/recreate/compare/select-1-80.sql ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h some-name-pxc-0.some-name-pxc -uroot -p'\''root_password'\'' -P3306'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h some-name-pxc-0.some-name-pxc -uroot -p'\''root_password'\'' -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.WrrpkDTl5U
+++ mktemp
++ local LAST_ERR=/tmp/tmp.iw48opr6eS
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.WrrpkDTl5U
++ cat /tmp/tmp.iw48opr6eS
++ rm /tmp/tmp.WrrpkDTl5U /tmp/tmp.iw48opr6eS
++ return 0
+ client_pod=pxc-client-59944c5bbf-wxzvx
+ wait_pod pxc-client-59944c5bbf-wxzvx
+ local pod=pxc-client-59944c5bbf-wxzvx
+ local max_retry=480
+ local ns=
++ echo pxc-client-59944c5bbf-wxzvx
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-59944c5bbf-wxzvx condition met
waiting for pod/pxc-client-59944c5bbf-wxzvx to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.NTdMUKjlbU/select-1.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/recreate/compare/select-1.sql /tmp/tmp.NTdMUKjlbU/select-1.sql
+ for i in '$(seq 0 $((size - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h some-name-pxc-1.some-name-pxc -uroot -p'\''root_password'\'' -P3306'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h some-name-pxc-1.some-name-pxc -uroot -p'\''root_password'\'' -P3306'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/recreate/compare/select-1.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/recreate/compare/select-1-80.sql ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h some-name-pxc-1.some-name-pxc -uroot -p'\''root_password'\'' -P3306'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h some-name-pxc-1.some-name-pxc -uroot -p'\''root_password'\'' -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.zU447GdsSe
+++ mktemp
++ local LAST_ERR=/tmp/tmp.hBQdNKd9aF
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.zU447GdsSe
++ cat /tmp/tmp.hBQdNKd9aF
++ rm /tmp/tmp.zU447GdsSe /tmp/tmp.hBQdNKd9aF
++ return 0
+ client_pod=pxc-client-59944c5bbf-wxzvx
+ wait_pod pxc-client-59944c5bbf-wxzvx
+ local pod=pxc-client-59944c5bbf-wxzvx
+ local max_retry=480
+ local ns=
++ echo pxc-client-59944c5bbf-wxzvx
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-59944c5bbf-wxzvx condition met
waiting for pod/pxc-client-59944c5bbf-wxzvx to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.NTdMUKjlbU/select-1.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/recreate/compare/select-1.sql /tmp/tmp.NTdMUKjlbU/select-1.sql
+ for i in '$(seq 0 $((size - 1)))'
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h some-name-pxc-2.some-name-pxc -uroot -p'\''root_password'\'' -P3306'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h some-name-pxc-2.some-name-pxc -uroot -p'\''root_password'\'' -P3306'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/recreate/compare/select-1.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/recreate/compare/select-1-80.sql ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h some-name-pxc-2.some-name-pxc -uroot -p'\''root_password'\'' -P3306'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h some-name-pxc-2.some-name-pxc -uroot -p'\''root_password'\'' -P3306'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.D5FIZzjjSd
+++ mktemp
++ local LAST_ERR=/tmp/tmp.JTKqJUJ3Eq
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.D5FIZzjjSd
++ cat /tmp/tmp.JTKqJUJ3Eq
++ rm /tmp/tmp.D5FIZzjjSd /tmp/tmp.JTKqJUJ3Eq
++ return 0
+ client_pod=pxc-client-59944c5bbf-wxzvx
+ wait_pod pxc-client-59944c5bbf-wxzvx
+ local pod=pxc-client-59944c5bbf-wxzvx
+ local max_retry=480
+ local ns=
++ echo pxc-client-59944c5bbf-wxzvx
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-59944c5bbf-wxzvx condition met
waiting for pod/pxc-client-59944c5bbf-wxzvx to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.NTdMUKjlbU/select-1.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/recreate/compare/select-1.sql /tmp/tmp.NTdMUKjlbU/select-1.sql
++ is_keyring_plugin_in_use some-name
++ local cluster=some-name
++ kubectl_bin exec -it some-name-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf'
++ egrep -o 'early-plugin-load=keyring_\w+.so'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.h9VPO1gSAL
+++ mktemp
++ local LAST_ERR=/tmp/tmp.mvm9ExpQYm
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl exec -it some-name-pxc-0 -c pxc -- bash -c 'cat /etc/mysql/node.cnf'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.h9VPO1gSAL
++ cat /tmp/tmp.mvm9ExpQYm
Unable to use a TTY - input is not a terminal or the right kind of file
++ rm /tmp/tmp.h9VPO1gSAL /tmp/tmp.mvm9ExpQYm
++ return 0
+ '[' '' ']'
+ desc pause
+ set +o xtrace


-----------------------------------------------------------------------------------
pause
-----------------------------------------------------------------------------------

+ cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/conf/some-name.yml
+ sed -e 's/pause: false/pause: true/'
+ kubectl_bin apply -f-
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/conf/some-name.yml
+ /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#'
+ /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#'
+ /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#'
++ mktemp
+ /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#'
+ /usr/bin/sed -e s~minio-service.#namespace~minio-service.recreate-20426~
+ /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2207-f8e092d7#'
+ /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#'
+ /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#'
+ /usr/bin/sed -e 's#apply:.*#apply: Never#'
+ /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#'
+ local LAST_OUT=/tmp/tmp.kzgHnKYPxu
+ /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#'
++ mktemp
+ local LAST_ERR=/tmp/tmp.0hh4pS72BS
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f-
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.kzgHnKYPxu
perconaxtradbcluster.pxc.percona.com/some-name configured
+ cat /tmp/tmp.0hh4pS72BS
+ rm /tmp/tmp.kzgHnKYPxu /tmp/tmp.0hh4pS72BS
+ return 0
+ wait_for_delete pod/some-name-proxysql-0
+ local res=pod/some-name-proxysql-0
+ echo -n 'waiting for pod/some-name-proxysql-0 to be deleted'
waiting for pod/some-name-proxysql-0 to be deleted+ set +o xtrace
.Error from server (NotFound): pods "some-name-proxysql-0" not found
+ wait_for_delete pod/some-name-pxc-2
+ local res=pod/some-name-pxc-2
+ echo -n 'waiting for pod/some-name-pxc-2 to be deleted'
waiting for pod/some-name-pxc-2 to be deleted+ set +o xtrace
.....Error from server (NotFound): pods "some-name-pxc-2" not found
+ wait_for_delete pod/some-name-pxc-1
+ local res=pod/some-name-pxc-1
+ echo -n 'waiting for pod/some-name-pxc-1 to be deleted'
waiting for pod/some-name-pxc-1 to be deleted+ set +o xtrace
.....Error from server (NotFound): pods "some-name-pxc-1" not found
+ wait_for_delete pod/some-name-pxc-0
+ local res=pod/some-name-pxc-0
+ echo -n 'waiting for pod/some-name-pxc-0 to be deleted'
waiting for pod/some-name-pxc-0 to be deleted+ set +o xtrace
......Error from server (NotFound): pods "some-name-pxc-0" not found
+ desc 'unpause, check data'
+ set +o xtrace


-----------------------------------------------------------------------------------
unpause, check data
-----------------------------------------------------------------------------------

+ sed -e 's/pause: true/pause: false/'
+ cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/conf/some-name.yml
+ /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#'
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/conf/some-name.yml
+ /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#'
+ /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#'
+ /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#'
+ kubectl_bin apply -f-
+ /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#'
++ mktemp
+ local LAST_OUT=/tmp/tmp.B2bT0mvnmi
++ mktemp
+ /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2207-f8e092d7#'
+ /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#'
+ local LAST_ERR=/tmp/tmp.g7RYJr4pb1
+ local exit_status=0
+ /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#'
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f-
+ /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#'
+ /usr/bin/sed -e s~minio-service.#namespace~minio-service.recreate-20426~
+ /usr/bin/sed -e 's#apply:.*#apply: Never#'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.B2bT0mvnmi
perconaxtradbcluster.pxc.percona.com/some-name configured
+ cat /tmp/tmp.g7RYJr4pb1
+ rm /tmp/tmp.B2bT0mvnmi /tmp/tmp.g7RYJr4pb1
+ return 0
+ wait_for_running some-name-proxysql 1
+ local name=some-name-proxysql
+ let last_pod=0
+ :
+ local max_retry=480
+ desc 'wait for running cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait for running cluster
-----------------------------------------------------------------------------------

++ seq 0 0
+ for i in '$(seq 0 $last_pod)'
+ wait_pod some-name-proxysql-0 480
+ local pod=some-name-proxysql-0
+ local max_retry=480
+ local ns=
++ echo some-name-proxysql-0
++ egrep '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+ local container=proxysql
+ set +o xtrace
pod/some-name-proxysql-0 condition met
waiting for pod/some-name-proxysql-0 to become Ready.Ok
+ wait_for_running some-name-pxc 3
+ local name=some-name-pxc
+ let last_pod=2
+ local max_retry=480
+ desc 'wait for running cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait for running cluster
-----------------------------------------------------------------------------------

++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ wait_pod some-name-pxc-0 480
+ local pod=some-name-pxc-0
+ local max_retry=480
+ local ns=
++ echo some-name-pxc-0
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=pxc
+ set +o xtrace
pod/some-name-pxc-0 condition met
waiting for pod/some-name-pxc-0 to become Ready.Ok
+ for i in '$(seq 0 $last_pod)'
+ wait_pod some-name-pxc-1 480
+ local pod=some-name-pxc-1
+ local max_retry=480
+ local ns=
++ echo some-name-pxc-1
++ egrep '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+ local container=pxc
+ set +o xtrace
pod/some-name-pxc-1 condition met
waiting for pod/some-name-pxc-1 to become Ready.Ok
+ for i in '$(seq 0 $last_pod)'
+ wait_pod some-name-pxc-2 480
+ local pod=some-name-pxc-2
+ local max_retry=480
+ local ns=
++ echo some-name-pxc-2
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=pxc
+ set +o xtrace
pod/some-name-pxc-2 condition met
waiting for pod/some-name-pxc-2 to become Ready.Ok
+ desc 'check data'
+ set +o xtrace


-----------------------------------------------------------------------------------
check data
-----------------------------------------------------------------------------------

+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h some-name-pxc-0.some-name-pxc -uroot -proot_password'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h some-name-pxc-0.some-name-pxc -uroot -proot_password'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/recreate/compare/select-1.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/recreate/compare/select-1-80.sql ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h some-name-pxc-0.some-name-pxc -uroot -proot_password'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h some-name-pxc-0.some-name-pxc -uroot -proot_password'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.2vnD2PpB1L
+++ mktemp
++ local LAST_ERR=/tmp/tmp.liDX4aI7ah
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.2vnD2PpB1L
++ cat /tmp/tmp.liDX4aI7ah
++ rm /tmp/tmp.2vnD2PpB1L /tmp/tmp.liDX4aI7ah
++ return 0
+ client_pod=pxc-client-59944c5bbf-wxzvx
+ wait_pod pxc-client-59944c5bbf-wxzvx
+ local pod=pxc-client-59944c5bbf-wxzvx
+ local max_retry=480
+ local ns=
++ echo pxc-client-59944c5bbf-wxzvx
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-59944c5bbf-wxzvx condition met
waiting for pod/pxc-client-59944c5bbf-wxzvx to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.NTdMUKjlbU/select-1.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/recreate/compare/select-1.sql /tmp/tmp.NTdMUKjlbU/select-1.sql
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h some-name-pxc-1.some-name-pxc -uroot -proot_password'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h some-name-pxc-1.some-name-pxc -uroot -proot_password'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/recreate/compare/select-1.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/recreate/compare/select-1-80.sql ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h some-name-pxc-1.some-name-pxc -uroot -proot_password'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h some-name-pxc-1.some-name-pxc -uroot -proot_password'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.GrUlnAmush
+++ mktemp
++ local LAST_ERR=/tmp/tmp.2mUkBLzUeu
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.GrUlnAmush
++ cat /tmp/tmp.2mUkBLzUeu
++ rm /tmp/tmp.GrUlnAmush /tmp/tmp.2mUkBLzUeu
++ return 0
+ client_pod=pxc-client-59944c5bbf-wxzvx
+ wait_pod pxc-client-59944c5bbf-wxzvx
+ local pod=pxc-client-59944c5bbf-wxzvx
+ local max_retry=480
+ local ns=
++ echo pxc-client-59944c5bbf-wxzvx
++ egrep '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+ local container=
+ set +o xtrace
pod/pxc-client-59944c5bbf-wxzvx condition met
waiting for pod/pxc-client-59944c5bbf-wxzvx to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.NTdMUKjlbU/select-1.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/recreate/compare/select-1.sql /tmp/tmp.NTdMUKjlbU/select-1.sql
+ compare_mysql_cmd select-1 'SELECT * from myApp.myApp;' '-h some-name-pxc-2.some-name-pxc -uroot -proot_password'
+ local command_id=select-1
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h some-name-pxc-2.some-name-pxc -uroot -proot_password'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/recreate/compare/select-1.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/recreate/compare/select-1-80.sql ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h some-name-pxc-2.some-name-pxc -uroot -proot_password'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h some-name-pxc-2.some-name-pxc -uroot -proot_password'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.wWodhfFKqG
+++ mktemp
++ local LAST_ERR=/tmp/tmp.ow9YPs0sHW
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.wWodhfFKqG
++ cat /tmp/tmp.ow9YPs0sHW
++ rm /tmp/tmp.wWodhfFKqG /tmp/tmp.ow9YPs0sHW
++ return 0
+ client_pod=pxc-client-59944c5bbf-wxzvx
+ wait_pod pxc-client-59944c5bbf-wxzvx
+ local pod=pxc-client-59944c5bbf-wxzvx
+ local max_retry=480
+ local ns=
++ echo pxc-client-59944c5bbf-wxzvx
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-59944c5bbf-wxzvx condition met
waiting for pod/pxc-client-59944c5bbf-wxzvx to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.NTdMUKjlbU/select-1.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/recreate/compare/select-1.sql /tmp/tmp.NTdMUKjlbU/select-1.sql
+ run_mysql 'INSERT myApp.myApp (id) VALUES (100501)' '-h some-name-proxysql -uroot -proot_password'
+ local 'command=INSERT myApp.myApp (id) VALUES (100501)'
+ local 'uri=-h some-name-proxysql -uroot -proot_password'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.UqLKIgKW3T
+++ mktemp
++ local LAST_ERR=/tmp/tmp.3p8vR7bY6q
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.UqLKIgKW3T
++ cat /tmp/tmp.3p8vR7bY6q
++ rm /tmp/tmp.UqLKIgKW3T /tmp/tmp.3p8vR7bY6q
++ return 0
+ client_pod=pxc-client-59944c5bbf-wxzvx
+ wait_pod pxc-client-59944c5bbf-wxzvx
+ local pod=pxc-client-59944c5bbf-wxzvx
+ local max_retry=480
+ local ns=
++ echo pxc-client-59944c5bbf-wxzvx
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-59944c5bbf-wxzvx condition met
waiting for pod/pxc-client-59944c5bbf-wxzvx to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ compare_mysql_cmd select-2 'SELECT * from myApp.myApp;' '-h some-name-pxc-0.some-name-pxc -uroot -proot_password'
+ local command_id=select-2
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h some-name-pxc-0.some-name-pxc -uroot -proot_password'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/recreate/compare/select-2.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/recreate/compare/select-2-80.sql ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h some-name-pxc-0.some-name-pxc -uroot -proot_password'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h some-name-pxc-0.some-name-pxc -uroot -proot_password'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.nw5ODkr6AJ
+++ mktemp
++ local LAST_ERR=/tmp/tmp.8UORhQoPNC
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.nw5ODkr6AJ
++ cat /tmp/tmp.8UORhQoPNC
++ rm /tmp/tmp.nw5ODkr6AJ /tmp/tmp.8UORhQoPNC
++ return 0
+ client_pod=pxc-client-59944c5bbf-wxzvx
+ wait_pod pxc-client-59944c5bbf-wxzvx
+ local pod=pxc-client-59944c5bbf-wxzvx
+ local max_retry=480
+ local ns=
++ echo pxc-client-59944c5bbf-wxzvx
++ egrep '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+ local container=
+ set +o xtrace
pod/pxc-client-59944c5bbf-wxzvx condition met
waiting for pod/pxc-client-59944c5bbf-wxzvx to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.NTdMUKjlbU/select-2.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/recreate/compare/select-2.sql /tmp/tmp.NTdMUKjlbU/select-2.sql
+ compare_mysql_cmd select-2 'SELECT * from myApp.myApp;' '-h some-name-pxc-1.some-name-pxc -uroot -proot_password'
+ local command_id=select-2
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h some-name-pxc-1.some-name-pxc -uroot -proot_password'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/recreate/compare/select-2.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/recreate/compare/select-2-80.sql ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h some-name-pxc-1.some-name-pxc -uroot -proot_password'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h some-name-pxc-1.some-name-pxc -uroot -proot_password'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.2DEuAthftu
+++ mktemp
++ local LAST_ERR=/tmp/tmp.bSDLf38Nqx
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.2DEuAthftu
++ cat /tmp/tmp.bSDLf38Nqx
++ rm /tmp/tmp.2DEuAthftu /tmp/tmp.bSDLf38Nqx
++ return 0
+ client_pod=pxc-client-59944c5bbf-wxzvx
+ wait_pod pxc-client-59944c5bbf-wxzvx
+ local pod=pxc-client-59944c5bbf-wxzvx
+ local max_retry=480
+ local ns=
++ echo pxc-client-59944c5bbf-wxzvx
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-59944c5bbf-wxzvx condition met
waiting for pod/pxc-client-59944c5bbf-wxzvx to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.NTdMUKjlbU/select-2.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/recreate/compare/select-2.sql /tmp/tmp.NTdMUKjlbU/select-2.sql
+ compare_mysql_cmd select-2 'SELECT * from myApp.myApp;' '-h some-name-pxc-2.some-name-pxc -uroot -proot_password'
+ local command_id=select-2
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h some-name-pxc-2.some-name-pxc -uroot -proot_password'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/recreate/compare/select-2.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/recreate/compare/select-2-80.sql ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h some-name-pxc-2.some-name-pxc -uroot -proot_password'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h some-name-pxc-2.some-name-pxc -uroot -proot_password'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.LoJ9Rb4tbV
+++ mktemp
++ local LAST_ERR=/tmp/tmp.rEwqlFqRk5
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.LoJ9Rb4tbV
++ cat /tmp/tmp.rEwqlFqRk5
++ rm /tmp/tmp.LoJ9Rb4tbV /tmp/tmp.rEwqlFqRk5
++ return 0
+ client_pod=pxc-client-59944c5bbf-wxzvx
+ wait_pod pxc-client-59944c5bbf-wxzvx
+ local pod=pxc-client-59944c5bbf-wxzvx
+ local max_retry=480
+ local ns=
++ echo pxc-client-59944c5bbf-wxzvx
++ egrep '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+ local container=
+ set +o xtrace
pod/pxc-client-59944c5bbf-wxzvx condition met
waiting for pod/pxc-client-59944c5bbf-wxzvx to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.NTdMUKjlbU/select-2.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/recreate/compare/select-2.sql /tmp/tmp.NTdMUKjlbU/select-2.sql
+ desc 'delete cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
delete cluster
-----------------------------------------------------------------------------------

+ kubectl_bin delete -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/conf/some-name.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.e8nibfAaWk
++ mktemp
+ local LAST_ERR=/tmp/tmp.8weQPTXNZU
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/conf/some-name.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.e8nibfAaWk
perconaxtradbcluster.pxc.percona.com "some-name" deleted from recreate-20426 namespace
+ cat /tmp/tmp.8weQPTXNZU
+ rm /tmp/tmp.e8nibfAaWk /tmp/tmp.8weQPTXNZU
+ return 0
+ wait_for_delete pod/some-name-pxc-2
+ local res=pod/some-name-pxc-2
+ echo -n 'waiting for pod/some-name-pxc-2 to be deleted'
waiting for pod/some-name-pxc-2 to be deleted+ set +o xtrace
Error from server (NotFound): pods "some-name-pxc-2" not found
+ wait_for_delete pod/some-name-pxc-1
+ local res=pod/some-name-pxc-1
+ echo -n 'waiting for pod/some-name-pxc-1 to be deleted'
waiting for pod/some-name-pxc-1 to be deleted+ set +o xtrace
Error from server (NotFound): pods "some-name-pxc-1" not found
+ wait_for_delete pod/some-name-pxc-0
+ local res=pod/some-name-pxc-0
+ echo -n 'waiting for pod/some-name-pxc-0 to be deleted'
waiting for pod/some-name-pxc-0 to be deleted+ set +o xtrace
.....Error from server (NotFound): pods "some-name-pxc-0" not found
+ desc 'recreate cluster, check data'
+ set +o xtrace


-----------------------------------------------------------------------------------
recreate cluster, check data
-----------------------------------------------------------------------------------

+ cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/conf/some-name.yml
+ sed -e 's/storage: 2Gi/storage: 3Gi/; s/size: 3/size: 5/; s/antiAffinityTopologyKey:.*/antiAffinityTopologyKey: none/'
+ kubectl_bin apply -f-
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/conf/some-name.yml
++ mktemp
+ /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#'
+ /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#'
+ local LAST_OUT=/tmp/tmp.84XB2J90bv
+ /usr/bin/sed -e 's#apply:.*#apply: Never#'
+ /usr/bin/sed -e s~minio-service.#namespace~minio-service.recreate-20426~
+ /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/percona-xtradb-cluster-operator:main-logcollector#'
++ mktemp
+ /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#'
+ /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2207-f8e092d7#'
+ local LAST_ERR=/tmp/tmp.3yrCVPqGtx
+ local exit_status=0
+ /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#'
+ /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#'
+ /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#'
+ /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#'
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f-
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.84XB2J90bv
perconaxtradbcluster.pxc.percona.com/some-name created
+ cat /tmp/tmp.3yrCVPqGtx
+ rm /tmp/tmp.84XB2J90bv /tmp/tmp.3yrCVPqGtx
+ return 0
+ wait_for_running some-name-proxysql 1
+ local name=some-name-proxysql
+ let last_pod=0
+ :
+ local max_retry=480
+ desc 'wait for running cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait for running cluster
-----------------------------------------------------------------------------------

++ seq 0 0
+ for i in '$(seq 0 $last_pod)'
+ wait_pod some-name-proxysql-0 480
+ local pod=some-name-proxysql-0
+ local max_retry=480
+ local ns=
++ echo some-name-proxysql-0
++ egrep '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+ local container=proxysql
+ set +o xtrace
pod/some-name-proxysql-0 condition met
waiting for pod/some-name-proxysql-0 to become Ready.Ok
+ wait_for_running some-name-pxc 5
+ local name=some-name-pxc
+ let last_pod=4
+ local max_retry=480
+ desc 'wait for running cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
wait for running cluster
-----------------------------------------------------------------------------------

++ seq 0 4
+ for i in '$(seq 0 $last_pod)'
+ wait_pod some-name-pxc-0 480
+ local pod=some-name-pxc-0
+ local max_retry=480
+ local ns=
++ echo some-name-pxc-0
++ egrep '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+ local container=pxc
+ set +o xtrace
pod/some-name-pxc-0 condition met
waiting for pod/some-name-pxc-0 to become Ready.Ok
+ for i in '$(seq 0 $last_pod)'
+ wait_pod some-name-pxc-1 480
+ local pod=some-name-pxc-1
+ local max_retry=480
+ local ns=
++ echo some-name-pxc-1
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=pxc
+ set +o xtrace
pod/some-name-pxc-1 condition met
waiting for pod/some-name-pxc-1 to become Ready.Ok
+ for i in '$(seq 0 $last_pod)'
+ wait_pod some-name-pxc-2 480
+ local pod=some-name-pxc-2
+ local max_retry=480
+ local ns=
++ echo some-name-pxc-2
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=pxc
+ set +o xtrace
pod/some-name-pxc-2 condition met
waiting for pod/some-name-pxc-2 to become Ready.Ok
+ for i in '$(seq 0 $last_pod)'
+ wait_pod some-name-pxc-3 480
+ local pod=some-name-pxc-3
+ local max_retry=480
+ local ns=
++ echo some-name-pxc-3
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=pxc
+ set +o xtrace
pod/some-name-pxc-3 condition met
waiting for pod/some-name-pxc-3 to become Ready.Ok
+ for i in '$(seq 0 $last_pod)'
+ wait_pod some-name-pxc-4 480
+ local pod=some-name-pxc-4
+ local max_retry=480
+ local ns=
++ echo some-name-pxc-4
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=pxc
+ set +o xtrace
pod/some-name-pxc-4 condition met
waiting for pod/some-name-pxc-4 to become Ready.Ok
+ compare_mysql_cmd select-2 'SELECT * from myApp.myApp;' '-h some-name-pxc-0.some-name-pxc -uroot -proot_password'
+ local command_id=select-2
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h some-name-pxc-0.some-name-pxc -uroot -proot_password'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/recreate/compare/select-2.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/recreate/compare/select-2-80.sql ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h some-name-pxc-0.some-name-pxc -uroot -proot_password'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h some-name-pxc-0.some-name-pxc -uroot -proot_password'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.k4UkHKt63y
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Ld1mYexQmn
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.k4UkHKt63y
++ cat /tmp/tmp.Ld1mYexQmn
++ rm /tmp/tmp.k4UkHKt63y /tmp/tmp.Ld1mYexQmn
++ return 0
+ client_pod=pxc-client-59944c5bbf-wxzvx
+ wait_pod pxc-client-59944c5bbf-wxzvx
+ local pod=pxc-client-59944c5bbf-wxzvx
+ local max_retry=480
+ local ns=
++ echo pxc-client-59944c5bbf-wxzvx
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-59944c5bbf-wxzvx condition met
waiting for pod/pxc-client-59944c5bbf-wxzvx to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.NTdMUKjlbU/select-2.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/recreate/compare/select-2.sql /tmp/tmp.NTdMUKjlbU/select-2.sql
+ compare_mysql_cmd select-2 'SELECT * from myApp.myApp;' '-h some-name-pxc-1.some-name-pxc -uroot -proot_password'
+ local command_id=select-2
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h some-name-pxc-1.some-name-pxc -uroot -proot_password'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/recreate/compare/select-2.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/recreate/compare/select-2-80.sql ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h some-name-pxc-1.some-name-pxc -uroot -proot_password'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h some-name-pxc-1.some-name-pxc -uroot -proot_password'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.lYbvPiavUC
+++ mktemp
++ local LAST_ERR=/tmp/tmp.Dz9aAGsrfT
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.lYbvPiavUC
++ cat /tmp/tmp.Dz9aAGsrfT
++ rm /tmp/tmp.lYbvPiavUC /tmp/tmp.Dz9aAGsrfT
++ return 0
+ client_pod=pxc-client-59944c5bbf-wxzvx
+ wait_pod pxc-client-59944c5bbf-wxzvx
+ local pod=pxc-client-59944c5bbf-wxzvx
+ local max_retry=480
+ local ns=
++ echo pxc-client-59944c5bbf-wxzvx
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-59944c5bbf-wxzvx condition met
waiting for pod/pxc-client-59944c5bbf-wxzvx to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.NTdMUKjlbU/select-2.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/recreate/compare/select-2.sql /tmp/tmp.NTdMUKjlbU/select-2.sql
+ compare_mysql_cmd select-2 'SELECT * from myApp.myApp;' '-h some-name-pxc-2.some-name-pxc -uroot -proot_password'
+ local command_id=select-2
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h some-name-pxc-2.some-name-pxc -uroot -proot_password'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/recreate/compare/select-2.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/recreate/compare/select-2-80.sql ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h some-name-pxc-2.some-name-pxc -uroot -proot_password'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h some-name-pxc-2.some-name-pxc -uroot -proot_password'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.p3mwnXCML6
+++ mktemp
++ local LAST_ERR=/tmp/tmp.DRmC17rBvN
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.p3mwnXCML6
++ cat /tmp/tmp.DRmC17rBvN
++ rm /tmp/tmp.p3mwnXCML6 /tmp/tmp.DRmC17rBvN
++ return 0
+ client_pod=pxc-client-59944c5bbf-wxzvx
+ wait_pod pxc-client-59944c5bbf-wxzvx
+ local pod=pxc-client-59944c5bbf-wxzvx
+ local max_retry=480
+ local ns=
++ echo pxc-client-59944c5bbf-wxzvx
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-59944c5bbf-wxzvx condition met
waiting for pod/pxc-client-59944c5bbf-wxzvx to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.NTdMUKjlbU/select-2.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/recreate/compare/select-2.sql /tmp/tmp.NTdMUKjlbU/select-2.sql
+ compare_mysql_cmd select-2 'SELECT * from myApp.myApp;' '-h some-name-pxc-3.some-name-pxc -uroot -proot_password'
+ local command_id=select-2
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h some-name-pxc-3.some-name-pxc -uroot -proot_password'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/recreate/compare/select-2.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/recreate/compare/select-2-80.sql ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h some-name-pxc-3.some-name-pxc -uroot -proot_password'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h some-name-pxc-3.some-name-pxc -uroot -proot_password'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.SNBzM5SQpB
+++ mktemp
++ local LAST_ERR=/tmp/tmp.EPTDAIFYVb
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.SNBzM5SQpB
++ cat /tmp/tmp.EPTDAIFYVb
++ rm /tmp/tmp.SNBzM5SQpB /tmp/tmp.EPTDAIFYVb
++ return 0
+ client_pod=pxc-client-59944c5bbf-wxzvx
+ wait_pod pxc-client-59944c5bbf-wxzvx
+ local pod=pxc-client-59944c5bbf-wxzvx
+ local max_retry=480
+ local ns=
++ echo pxc-client-59944c5bbf-wxzvx
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-59944c5bbf-wxzvx condition met
waiting for pod/pxc-client-59944c5bbf-wxzvx to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.NTdMUKjlbU/select-2.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/recreate/compare/select-2.sql /tmp/tmp.NTdMUKjlbU/select-2.sql
+ compare_mysql_cmd select-2 'SELECT * from myApp.myApp;' '-h some-name-pxc-4.some-name-pxc -uroot -proot_password'
+ local command_id=select-2
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h some-name-pxc-4.some-name-pxc -uroot -proot_password'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/recreate/compare/select-2.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/recreate/compare/select-2-80.sql ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h some-name-pxc-4.some-name-pxc -uroot -proot_password'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h some-name-pxc-4.some-name-pxc -uroot -proot_password'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.wXnoqnBWQE
+++ mktemp
++ local LAST_ERR=/tmp/tmp.rkWADHBPZr
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.wXnoqnBWQE
++ cat /tmp/tmp.rkWADHBPZr
++ rm /tmp/tmp.wXnoqnBWQE /tmp/tmp.rkWADHBPZr
++ return 0
+ client_pod=pxc-client-59944c5bbf-wxzvx
+ wait_pod pxc-client-59944c5bbf-wxzvx
+ local pod=pxc-client-59944c5bbf-wxzvx
+ local max_retry=480
+ local ns=
++ echo pxc-client-59944c5bbf-wxzvx
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-59944c5bbf-wxzvx condition met
waiting for pod/pxc-client-59944c5bbf-wxzvx to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.NTdMUKjlbU/select-2.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/recreate/compare/select-2.sql /tmp/tmp.NTdMUKjlbU/select-2.sql
+ desc 'write data to ProxySQL, read from all'
+ set +o xtrace


-----------------------------------------------------------------------------------
write data to ProxySQL, read from all
-----------------------------------------------------------------------------------

+ run_mysql 'INSERT myApp.myApp (id) VALUES (100502)' '-h some-name-proxysql -uroot -proot_password'
+ local 'command=INSERT myApp.myApp (id) VALUES (100502)'
+ local 'uri=-h some-name-proxysql -uroot -proot_password'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.SeEP9WwUM7
+++ mktemp
++ local LAST_ERR=/tmp/tmp.zcmFhYTfsG
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.SeEP9WwUM7
++ cat /tmp/tmp.zcmFhYTfsG
++ rm /tmp/tmp.SeEP9WwUM7 /tmp/tmp.zcmFhYTfsG
++ return 0
+ client_pod=pxc-client-59944c5bbf-wxzvx
+ wait_pod pxc-client-59944c5bbf-wxzvx
+ local pod=pxc-client-59944c5bbf-wxzvx
+ local max_retry=480
+ local ns=
++ echo pxc-client-59944c5bbf-wxzvx
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-59944c5bbf-wxzvx condition met
waiting for pod/pxc-client-59944c5bbf-wxzvx to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ compare_mysql_cmd select-3 'SELECT * from myApp.myApp;' '-h some-name-pxc-0.some-name-pxc -uroot -proot_password'
+ local command_id=select-3
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h some-name-pxc-0.some-name-pxc -uroot -proot_password'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/recreate/compare/select-3.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/recreate/compare/select-3-80.sql ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h some-name-pxc-0.some-name-pxc -uroot -proot_password'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h some-name-pxc-0.some-name-pxc -uroot -proot_password'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.3T28iAqn9w
+++ mktemp
++ local LAST_ERR=/tmp/tmp.LXqfFtpiec
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.3T28iAqn9w
++ cat /tmp/tmp.LXqfFtpiec
++ rm /tmp/tmp.3T28iAqn9w /tmp/tmp.LXqfFtpiec
++ return 0
+ client_pod=pxc-client-59944c5bbf-wxzvx
+ wait_pod pxc-client-59944c5bbf-wxzvx
+ local pod=pxc-client-59944c5bbf-wxzvx
+ local max_retry=480
+ local ns=
++ echo pxc-client-59944c5bbf-wxzvx
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-59944c5bbf-wxzvx condition met
waiting for pod/pxc-client-59944c5bbf-wxzvx to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.NTdMUKjlbU/select-3.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/recreate/compare/select-3.sql /tmp/tmp.NTdMUKjlbU/select-3.sql
+ compare_mysql_cmd select-3 'SELECT * from myApp.myApp;' '-h some-name-pxc-1.some-name-pxc -uroot -proot_password'
+ local command_id=select-3
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h some-name-pxc-1.some-name-pxc -uroot -proot_password'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/recreate/compare/select-3.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/recreate/compare/select-3-80.sql ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h some-name-pxc-1.some-name-pxc -uroot -proot_password'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h some-name-pxc-1.some-name-pxc -uroot -proot_password'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.ymY2Gop1U3
+++ mktemp
++ local LAST_ERR=/tmp/tmp.YkFz7PZZte
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.ymY2Gop1U3
++ cat /tmp/tmp.YkFz7PZZte
++ rm /tmp/tmp.ymY2Gop1U3 /tmp/tmp.YkFz7PZZte
++ return 0
+ client_pod=pxc-client-59944c5bbf-wxzvx
+ wait_pod pxc-client-59944c5bbf-wxzvx
+ local pod=pxc-client-59944c5bbf-wxzvx
+ local max_retry=480
+ local ns=
++ echo pxc-client-59944c5bbf-wxzvx
++ egrep '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+ local container=
+ set +o xtrace
pod/pxc-client-59944c5bbf-wxzvx condition met
waiting for pod/pxc-client-59944c5bbf-wxzvx to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.NTdMUKjlbU/select-3.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/recreate/compare/select-3.sql /tmp/tmp.NTdMUKjlbU/select-3.sql
+ compare_mysql_cmd select-3 'SELECT * from myApp.myApp;' '-h some-name-pxc-2.some-name-pxc -uroot -proot_password'
+ local command_id=select-3
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h some-name-pxc-2.some-name-pxc -uroot -proot_password'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/recreate/compare/select-3.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/recreate/compare/select-3-80.sql ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h some-name-pxc-2.some-name-pxc -uroot -proot_password'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h some-name-pxc-2.some-name-pxc -uroot -proot_password'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.aZESbVavHA
+++ mktemp
++ local LAST_ERR=/tmp/tmp.NcWo1FrLT0
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.aZESbVavHA
++ cat /tmp/tmp.NcWo1FrLT0
++ rm /tmp/tmp.aZESbVavHA /tmp/tmp.NcWo1FrLT0
++ return 0
+ client_pod=pxc-client-59944c5bbf-wxzvx
+ wait_pod pxc-client-59944c5bbf-wxzvx
+ local pod=pxc-client-59944c5bbf-wxzvx
+ local max_retry=480
+ local ns=
++ echo pxc-client-59944c5bbf-wxzvx
++ egrep '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
+ local container=
+ set +o xtrace
pod/pxc-client-59944c5bbf-wxzvx condition met
waiting for pod/pxc-client-59944c5bbf-wxzvx to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.NTdMUKjlbU/select-3.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/recreate/compare/select-3.sql /tmp/tmp.NTdMUKjlbU/select-3.sql
+ compare_mysql_cmd select-3 'SELECT * from myApp.myApp;' '-h some-name-pxc-3.some-name-pxc -uroot -proot_password'
+ local command_id=select-3
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h some-name-pxc-3.some-name-pxc -uroot -proot_password'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/recreate/compare/select-3.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/recreate/compare/select-3-80.sql ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h some-name-pxc-3.some-name-pxc -uroot -proot_password'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h some-name-pxc-3.some-name-pxc -uroot -proot_password'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.SGM3YygpSe
+++ mktemp
++ local LAST_ERR=/tmp/tmp.d4xb2W4w3G
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.SGM3YygpSe
++ cat /tmp/tmp.d4xb2W4w3G
++ rm /tmp/tmp.SGM3YygpSe /tmp/tmp.d4xb2W4w3G
++ return 0
+ client_pod=pxc-client-59944c5bbf-wxzvx
+ wait_pod pxc-client-59944c5bbf-wxzvx
+ local pod=pxc-client-59944c5bbf-wxzvx
+ local max_retry=480
+ local ns=
++ echo pxc-client-59944c5bbf-wxzvx
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-59944c5bbf-wxzvx condition met
waiting for pod/pxc-client-59944c5bbf-wxzvx to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.NTdMUKjlbU/select-3.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/recreate/compare/select-3.sql /tmp/tmp.NTdMUKjlbU/select-3.sql
+ compare_mysql_cmd select-3 'SELECT * from myApp.myApp;' '-h some-name-pxc-4.some-name-pxc -uroot -proot_password'
+ local command_id=select-3
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h some-name-pxc-4.some-name-pxc -uroot -proot_password'
+ local postfix=
+ local expected_result=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/recreate/compare/select-3.sql
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.4 ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 8\.0 ]]
+ [[ -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/recreate/compare/select-3-80.sql ]]
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]]
+ run_mysql 'SELECT * from myApp.myApp;' '-h some-name-pxc-4.some-name-pxc -uroot -proot_password'
+ local 'command=SELECT * from myApp.myApp;'
+ local 'uri=-h some-name-pxc-4.some-name-pxc -uroot -proot_password'
++ get_client_pod
++ kubectl_bin get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.6WrIrfxcjM
+++ mktemp
++ local LAST_ERR=/tmp/tmp.q0x9fKjETP
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=name=pxc-client -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.6WrIrfxcjM
++ cat /tmp/tmp.q0x9fKjETP
++ rm /tmp/tmp.6WrIrfxcjM /tmp/tmp.q0x9fKjETP
++ return 0
+ client_pod=pxc-client-59944c5bbf-wxzvx
+ wait_pod pxc-client-59944c5bbf-wxzvx
+ local pod=pxc-client-59944c5bbf-wxzvx
+ local max_retry=480
+ local ns=
++ echo pxc-client-59944c5bbf-wxzvx
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ egrep '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/pxc-client-59944c5bbf-wxzvx condition met
waiting for pod/pxc-client-59944c5bbf-wxzvx to become ReadyDefaulted container "pxc-client" out of: pxc-client, backup
.Ok
+ set +o xtrace
+ '[' '!' -s /tmp/tmp.NTdMUKjlbU/select-3.sql ']'
+ diff -u /mnt/jenkins/workspace/cloud-pxc-operator_PR-2207/e2e-tests/recreate/compare/select-3.sql /tmp/tmp.NTdMUKjlbU/select-3.sql
+ destroy recreate-20426
+ local namespace=recreate-20426
+ local ignore_logs=true
+ desc 'destroy cluster/operator and all other resources'
+ set +o xtrace


-----------------------------------------------------------------------------------
destroy cluster/operator and all other resources
-----------------------------------------------------------------------------------

+ '[' true == false -o 1 == 1 ']'
+ grep -v level=info
+ grep -v 'get backup status: Job.batch'
+ tee /tmp/tmp.NTdMUKjlbU/operator.log
+ grep -v 'the object has been modified'
++ get_operator_pod
++ local label_prefix=app.kubernetes.io/
+++ grep -c percona-xtradb-cluster-operator
+++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator
+ /usr/bin/sed -r 's/"ts":[0-9.]+//; s^limits-[0-9.]+/^^g'
+ sort -u
++ local check_label=1
++ [[ 1 -eq 0 ]]
++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.A8mUv7YruC
+++ mktemp
++ local LAST_ERR=/tmp/tmp.egB1qtbtzl
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -o 'jsonpath={.items[].metadata.name}' -n pxc-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.A8mUv7YruC
++ cat /tmp/tmp.egB1qtbtzl
++ rm /tmp/tmp.A8mUv7YruC /tmp/tmp.egB1qtbtzl
++ return 0
+ kubectl_bin logs -n pxc-operator percona-xtradb-cluster-operator-859595f865-d7sg4
++ mktemp
+ local LAST_OUT=/tmp/tmp.wHJUSaRkhg
++ mktemp
+ local LAST_ERR=/tmp/tmp.MZg6ntE5s6
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl logs -n pxc-operator percona-xtradb-cluster-operator-859595f865-d7sg4
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.wHJUSaRkhg
+ cat /tmp/tmp.MZg6ntE5s6
+ rm /tmp/tmp.wHJUSaRkhg /tmp/tmp.MZg6ntE5s6
+ return 0
2025-10-10T09:59:13.642Z	INFO	setup	Manager starting up	{"gitCommit": "f8e092d7031c2931d0d1772ac8045385e44e3ff9", "gitBranch": "PR-2207-f8e092d7", "buildTime": "2025-10-10T08:21:44Z", "goVersion": "go1.24.8", "os": "linux", "arch": "amd64"}
2025-10-10T09:59:13.642Z	INFO	setup	Runs on	{"platform": "kubernetes", "version": "v1.31.13-gke.1008000"}
2025-10-10T09:59:13.645Z	INFO	setup	Registering Components.
2025-10-10T09:59:15.108Z	INFO	controller-runtime.certwatcher	Updated current TLS certificate	{"cert": "/tmp/k8s-webhook-server/serving-certs/tls.crt", "key": "/tmp/k8s-webhook-server/serving-certs/tls.key"}
2025-10-10T09:59:15.108Z	INFO	controller-runtime.metrics	Serving metrics server	{"bindAddress": ":8080", "secure": false}
2025-10-10T09:59:15.108Z	INFO	controller-runtime.metrics	Starting metrics server
2025-10-10T09:59:15.108Z	INFO	controller-runtime.webhook	Registering webhook	{"path": "/validate-percona-xtradbcluster"}
2025-10-10T09:59:15.108Z	INFO	controller-runtime.webhook	Serving webhook server	{"host": "", "port": 9443}
2025-10-10T09:59:15.108Z	INFO	controller-runtime.webhook	Starting webhook server
2025-10-10T09:59:15.108Z	INFO	setup	Starting the Cmd.
2025-10-10T09:59:15.108Z	INFO	starting server	{"name": "health probe", "addr": "[::]:8081"}
2025-10-10T09:59:15.109Z	INFO	controller-runtime.certwatcher	Starting certificate poll+watcher	{"cert": "/tmp/k8s-webhook-server/serving-certs/tls.crt", "key": "/tmp/k8s-webhook-server/serving-certs/tls.key", "interval": "10s"}
2025-10-10T09:59:15.209Z	INFO	attempting to acquire leader lease pxc-operator/08db1feb.percona.com...
2025-10-10T09:59:32.449Z	DEBUG	events	percona-xtradb-cluster-operator-859595f865-d7sg4_9ab1b153-3a55-4e21-aad0-4f5e73a55fc9 became leader	{"type": "Normal", "object": {"kind":"Lease","namespace":"pxc-operator","name":"08db1feb.percona.com","uid":"7b663237-7d00-43a7-812d-c2debe145e90","apiVersion":"coordination.k8s.io/v1","resourceVersion":"1760090372439391009"}, "reason": "LeaderElection"}
2025-10-10T09:59:32.449Z	INFO	successfully acquired lease pxc-operator/08db1feb.percona.com
2025-10-10T09:59:32.450Z	INFO	Starting EventSource	{"controller": "pxcbackup-controller", "source": "kind source: *v1.PerconaXtraDBClusterBackup"}
2025-10-10T09:59:32.450Z	INFO	Starting EventSource	{"controller": "pxc-controller", "source": "kind source: *v1.PerconaXtraDBCluster"}
2025-10-10T09:59:32.450Z	INFO	Starting EventSource	{"controller": "pxc-controller", "source": "kind source: *v1.Secret"}
2025-10-10T09:59:32.450Z	INFO	Starting EventSource	{"controller": "pxcrestore-controller", "source": "kind source: *v1.PerconaXtraDBClusterRestore"}
2025-10-10T09:59:32.463Z	ERROR	controller-runtime.source.Kind	if kind is a CRD, it should be installed before calling Start	{"kind": "PerconaXtraDBClusterRestore.pxc.percona.com", "error": "no matches for kind \"PerconaXtraDBClusterRestore\" in version \"pxc.percona.com/v1\""}
2025-10-10T09:59:32.469Z	ERROR	controller-runtime.source.Kind	if kind is a CRD, it should be installed before calling Start	{"kind": "PerconaXtraDBClusterBackup.pxc.percona.com", "error": "no matches for kind \"PerconaXtraDBClusterBackup\" in version \"pxc.percona.com/v1\""}
2025-10-10T09:59:32.550Z	INFO	Starting Controller	{"controller": "pxc-controller"}
2025-10-10T09:59:32.550Z	INFO	Starting workers	{"controller": "pxc-controller", "worker count": 1}
2025-10-10T09:59:33.686Z	ERROR	Reconciler error	{"controller": "pxc-controller", "namespace": "recreate-20426", "name": "some-name", "reconcileID": "31baf6e5-e115-4476-aaf0-a139beab786e", "error": "failed to check if restore is running: failed to get restore list: no matches for kind \"PerconaXtraDBClusterRestore\" in version \"pxc.percona.com/v1\"", "errorVerbose": "no matches for kind \"PerconaXtraDBClusterRestore\" in version \"pxc.percona.com/v1\"\nfailed to get restore list\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).isRestoreRunning\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/upgrade.go:680\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileBackups\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/backup.go:37\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:473\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700\nfailed to check if restore is running\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileBackups\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/backup.go:39\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:473\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700"}
2025-10-10T09:59:33.957Z	ERROR	Reconciler error	{"controller": "pxc-controller", "namespace": "recreate-20426", "name": "some-name", "reconcileID": "de747256-b0f5-43a4-8e95-1bebebd11976", "error": "failed to check if restore is running: failed to get restore list: no matches for kind \"PerconaXtraDBClusterRestore\" in version \"pxc.percona.com/v1\"", "errorVerbose": "no matches for kind \"PerconaXtraDBClusterRestore\" in version \"pxc.percona.com/v1\"\nfailed to get restore list\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).isRestoreRunning\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/upgrade.go:680\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileBackups\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/backup.go:37\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:473\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700\nfailed to check if restore is running\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileBackups\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/backup.go:39\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:473\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700"}
2025-10-10T09:59:34.227Z	ERROR	Reconciler error	{"controller": "pxc-controller", "namespace": "recreate-20426", "name": "some-name", "reconcileID": "90366102-f447-43ac-8eeb-ef054c563c43", "error": "failed to check if restore is running: failed to get restore list: no matches for kind \"PerconaXtraDBClusterRestore\" in version \"pxc.percona.com/v1\"", "errorVerbose": "no matches for kind \"PerconaXtraDBClusterRestore\" in version \"pxc.percona.com/v1\"\nfailed to get restore list\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).isRestoreRunning\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/upgrade.go:680\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileBackups\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/backup.go:37\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:473\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700\nfailed to check if restore is running\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileBackups\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/backup.go:39\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:473\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700"}
2025-10-10T09:59:35.439Z	ERROR	Reconciler error	{"controller": "pxc-controller", "namespace": "recreate-20426", "name": "some-name", "reconcileID": "863b0dea-cb53-490b-9f25-efa913353fbf", "error": "failed to check if restore is running: failed to get restore list: no matches for kind \"PerconaXtraDBClusterRestore\" in version \"pxc.percona.com/v1\"", "errorVerbose": "no matches for kind \"PerconaXtraDBClusterRestore\" in version \"pxc.percona.com/v1\"\nfailed to get restore list\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).isRestoreRunning\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/upgrade.go:680\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileBackups\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/backup.go:37\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:473\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700\nfailed to check if restore is running\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileBackups\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/backup.go:39\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:473\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700"}
2025-10-10T09:59:36.618Z	ERROR	Reconciler error	{"controller": "pxc-controller", "namespace": "recreate-20426", "name": "some-name", "reconcileID": "25fda4f6-1b65-4368-a4a5-e308f3ab7e38", "error": "failed to check if restore is running: failed to get restore list: no matches for kind \"PerconaXtraDBClusterRestore\" in version \"pxc.percona.com/v1\"", "errorVerbose": "no matches for kind \"PerconaXtraDBClusterRestore\" in version \"pxc.percona.com/v1\"\nfailed to get restore list\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).isRestoreRunning\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/upgrade.go:680\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileBackups\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/backup.go:37\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:473\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700\nfailed to check if restore is running\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileBackups\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/backup.go:39\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:473\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700"}
2025-10-10T09:59:37.818Z	ERROR	Reconciler error	{"controller": "pxc-controller", "namespace": "recreate-20426", "name": "some-name", "reconcileID": "597fc379-fc8e-4041-8bfa-5aeada37b14d", "error": "failed to check if restore is running: failed to get restore list: no matches for kind \"PerconaXtraDBClusterRestore\" in version \"pxc.percona.com/v1\"", "errorVerbose": "no matches for kind \"PerconaXtraDBClusterRestore\" in version \"pxc.percona.com/v1\"\nfailed to get restore list\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).isRestoreRunning\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/upgrade.go:680\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileBackups\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/backup.go:37\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:473\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700\nfailed to check if restore is running\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileBackups\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/backup.go:39\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:473\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700"}
2025-10-10T09:59:39.025Z	ERROR	Reconciler error	{"controller": "pxc-controller", "namespace": "recreate-20426", "name": "some-name", "reconcileID": "61f06d1e-7d67-4dd6-b3ef-72f82b851ed9", "error": "failed to check if restore is running: failed to get restore list: no matches for kind \"PerconaXtraDBClusterRestore\" in version \"pxc.percona.com/v1\"", "errorVerbose": "no matches for kind \"PerconaXtraDBClusterRestore\" in version \"pxc.percona.com/v1\"\nfailed to get restore list\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).isRestoreRunning\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/upgrade.go:680\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileBackups\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/backup.go:37\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:473\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700\nfailed to check if restore is running\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileBackups\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/backup.go:39\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:473\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700"}
2025-10-10T09:59:40.233Z	ERROR	Reconciler error	{"controller": "pxc-controller", "namespace": "recreate-20426", "name": "some-name", "reconcileID": "618d5a0c-879d-433e-a3d4-1de9abfd494c", "error": "failed to check if restore is running: failed to get restore list: no matches for kind \"PerconaXtraDBClusterRestore\" in version \"pxc.percona.com/v1\"", "errorVerbose": "no matches for kind \"PerconaXtraDBClusterRestore\" in version \"pxc.percona.com/v1\"\nfailed to get restore list\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).isRestoreRunning\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/upgrade.go:680\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileBackups\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/backup.go:37\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:473\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700\nfailed to check if restore is running\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileBackups\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/backup.go:39\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:473\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700"}
2025-10-10T09:59:41.435Z	ERROR	Reconciler error	{"controller": "pxc-controller", "namespace": "recreate-20426", "name": "some-name", "reconcileID": "a0f37e4a-f128-4b43-b361-f4486cb95e29", "error": "failed to check if restore is running: failed to get restore list: no matches for kind \"PerconaXtraDBClusterRestore\" in version \"pxc.percona.com/v1\"", "errorVerbose": "no matches for kind \"PerconaXtraDBClusterRestore\" in version \"pxc.percona.com/v1\"\nfailed to get restore list\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).isRestoreRunning\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/upgrade.go:680\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileBackups\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/backup.go:37\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:473\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700\nfailed to check if restore is running\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileBackups\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/backup.go:39\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:473\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700"}
2025-10-10T09:59:42.457Z	ERROR	controller-runtime.source.Kind	if kind is a CRD, it should be installed before calling Start	{"kind": "PerconaXtraDBClusterRestore.pxc.percona.com", "error": "no matches for kind \"PerconaXtraDBClusterRestore\" in version \"pxc.percona.com/v1\""}
2025-10-10T09:59:42.459Z	ERROR	controller-runtime.source.Kind	if kind is a CRD, it should be installed before calling Start	{"kind": "PerconaXtraDBClusterBackup.pxc.percona.com", "error": "no matches for kind \"PerconaXtraDBClusterBackup\" in version \"pxc.percona.com/v1\""}
2025-10-10T09:59:42.623Z	ERROR	Reconciler error	{"controller": "pxc-controller", "namespace": "recreate-20426", "name": "some-name", "reconcileID": "c0cd6cdb-8b18-4c57-ba67-ae9141902962", "error": "failed to check if restore is running: failed to get restore list: no matches for kind \"PerconaXtraDBClusterRestore\" in version \"pxc.percona.com/v1\"", "errorVerbose": "no matches for kind \"PerconaXtraDBClusterRestore\" in version \"pxc.percona.com/v1\"\nfailed to get restore list\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).isRestoreRunning\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/upgrade.go:680\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileBackups\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/backup.go:37\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:473\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700\nfailed to check if restore is running\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileBackups\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/backup.go:39\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:473\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700"}
2025-10-10T09:59:43.867Z	ERROR	Reconciler error	{"controller": "pxc-controller", "namespace": "recreate-20426", "name": "some-name", "reconcileID": "e6a667a3-bb31-4427-a911-01ec1aff6984", "error": "failed to check if restore is running: failed to get restore list: no matches for kind \"PerconaXtraDBClusterRestore\" in version \"pxc.percona.com/v1\"", "errorVerbose": "no matches for kind \"PerconaXtraDBClusterRestore\" in version \"pxc.percona.com/v1\"\nfailed to get restore list\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).isRestoreRunning\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/upgrade.go:680\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileBackups\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/backup.go:37\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:473\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700\nfailed to check if restore is running\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileBackups\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/backup.go:39\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:473\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700"}
2025-10-10T09:59:49.140Z	ERROR	Reconciler error	{"controller": "pxc-controller", "namespace": "recreate-20426", "name": "some-name", "reconcileID": "9af3e8ab-ac41-4f46-86f4-561c5b036461", "error": "failed to check if restore is running: failed to get restore list: no matches for kind \"PerconaXtraDBClusterRestore\" in version \"pxc.percona.com/v1\"", "errorVerbose": "no matches for kind \"PerconaXtraDBClusterRestore\" in version \"pxc.percona.com/v1\"\nfailed to get restore list\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).isRestoreRunning\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/upgrade.go:680\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileBackups\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/backup.go:37\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:473\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700\nfailed to check if restore is running\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).reconcileBackups\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/backup.go:39\ngithub.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc.(*ReconcilePerconaXtraDBCluster).Reconcile\n\t/go/src/github.com/percona/percona-xtradb-cluster-operator/pkg/controller/pxc/controller.go:473\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1700"}
2025-10-10T09:59:52.456Z	ERROR	controller-runtime.source.Kind	if kind is a CRD, it should be installed before calling Start	{"kind": "PerconaXtraDBClusterRestore.pxc.percona.com", "error": "no matches for kind \"PerconaXtraDBClusterRestore\" in version \"pxc.percona.com/v1\""}
2025-10-10T09:59:52.460Z	ERROR	controller-runtime.source.Kind	if kind is a CRD, it should be installed before calling Start	{"kind": "PerconaXtraDBClusterBackup.pxc.percona.com", "error": "no matches for kind \"PerconaXtraDBClusterBackup\" in version \"pxc.percona.com/v1\""}
	/go/pkg/mod/k8s.io/apimachinery@v0.34.1/pkg/util/wait/loop.go:53
	/go/pkg/mod/k8s.io/apimachinery@v0.34.1/pkg/util/wait/loop.go:54
	/go/pkg/mod/k8s.io/apimachinery@v0.34.1/pkg/util/wait/loop.go:87
	/go/pkg/mod/k8s.io/apimachinery@v0.34.1/pkg/util/wait/loop.go:88
	/go/pkg/mod/k8s.io/apimachinery@v0.34.1/pkg/util/wait/poll.go:33
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/source/kind.go:68
	/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/source/kind.go:75
k8s.io/apimachinery/pkg/util/wait.loopConditionUntilContext
k8s.io/apimachinery/pkg/util/wait.loopConditionUntilContext.func1
k8s.io/apimachinery/pkg/util/wait.loopConditionUntilContext.func2
k8s.io/apimachinery/pkg/util/wait.PollUntilContextCancel
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1
sigs.k8s.io/controller-runtime/pkg/internal/source.(*Kind[...]).Start.func1
sigs.k8s.io/controller-runtime/pkg/internal/source.(*Kind[...]).Start.func1.1
+ kubectl get pxc --all-namespaces -o wide
+ grep -v NAMESPACE
+ xargs -L 1 sh -xc 'kubectl patch pxc -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"'
+ kubectl patch pxc -n recreate-20426 some-name --type=merge -p '{"metadata":{"finalizers":[]}}'
perconaxtradbcluster.pxc.percona.com/some-name patched
+ kubectl_bin delete pxc --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.YUG3YyGH7Y
++ mktemp
+ local LAST_ERR=/tmp/tmp.ELPhWXDygY
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc --all --all-namespaces
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.YUG3YyGH7Y
perconaxtradbcluster.pxc.percona.com "some-name" deleted from recreate-20426 namespace
+ cat /tmp/tmp.ELPhWXDygY
+ rm /tmp/tmp.YUG3YyGH7Y /tmp/tmp.ELPhWXDygY
+ return 0
+ kubectl_bin delete pxc-backup --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.zYR2bmKjKp
++ mktemp
+ local LAST_ERR=/tmp/tmp.6s2RO2WXDA
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc-backup --all --all-namespaces
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc-backup --all --all-namespaces
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc-backup --all --all-namespaces
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ cat /tmp/tmp.zYR2bmKjKp
+ cat /tmp/tmp.6s2RO2WXDA
error: the server doesn't have a resource type "pxc-backup"
+ rm /tmp/tmp.zYR2bmKjKp /tmp/tmp.6s2RO2WXDA
+ return 1
+ :
+ kubectl_bin delete pxc-restore --all --all-namespaces
++ mktemp
+ local LAST_OUT=/tmp/tmp.q7p9MajpdE
++ mktemp
+ local LAST_ERR=/tmp/tmp.VrPGKb3Qfk
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc-restore --all --all-namespaces
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc-restore --all --all-namespaces
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete pxc-restore --all --all-namespaces
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ cat /tmp/tmp.q7p9MajpdE
+ cat /tmp/tmp.VrPGKb3Qfk
error: the server doesn't have a resource type "pxc-restore"
+ rm /tmp/tmp.q7p9MajpdE /tmp/tmp.VrPGKb3Qfk
+ return 1
+ :
+ kubectl_bin delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook
++ mktemp
+ local LAST_OUT=/tmp/tmp.DOe68LKJt0
++ mktemp
+ local LAST_ERR=/tmp/tmp.mSUCjFWQpY
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.DOe68LKJt0
validatingwebhookconfiguration.admissionregistration.k8s.io "percona-xtradbcluster-webhook" deleted
+ cat /tmp/tmp.mSUCjFWQpY
+ rm /tmp/tmp.DOe68LKJt0 /tmp/tmp.mSUCjFWQpY
+ return 0
+ kubectl_bin delete -f https://github.com/jetstack/cert-manager/releases/download/v1.18.2/cert-manager.yaml
+ :
+ '[' '!' -z '' ']'
+ '[' -n pxc-operator ']'
+ kubectl_bin delete --grace-period=0 --force=true namespace recreate-20426
+ rm -rf /tmp/tmp.NTdMUKjlbU
++ mktemp
+ kubectl_bin delete --grace-period=0 --force=true namespace pxc-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.9LH7hRDCJ8
+ desc 'test passed'
+ set +o xtrace


-----------------------------------------------------------------------------------
test passed
-----------------------------------------------------------------------------------

++ mktemp
+ local LAST_OUT=/tmp/tmp.1xwmVlQc9K
++ mktemp
+ local LAST_ERR=/tmp/tmp.gb6ylPKuH6
+ local exit_status=0
+ local LAST_ERR=/tmp/tmp.PjziCk6oN0
+ local exit_status=0
++ seq 0 2
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete --grace-period=0 --force=true namespace recreate-20426
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete --grace-period=0 --force=true namespace pxc-operator