Log: /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/logs/pitr-gap-errors-8-0.log
Warning: version difference between client (1.36) and server (1.33) exceeds the supported minor version skew of +/-1
Warning: version difference between client (1.36) and server (1.33) exceeds the supported minor version skew of +/-1
+ GTID_PATTERN='[A-F0-9a-f]{8}-[A-F0-9a-f]{4}-[A-F0-9a-f]{4}-[A-F0-9a-f]{4}-[A-F0-9a-f]{12}:[0-9]+'
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]]
+ main
+ create_infra pitr-gap-errors-17705
+ local ns=pitr-gap-errors-17705
+ '[' -n pxc-operator ']'
+ read -r pns name
+ kubectl_bin get pxc --all-namespaces -o 'jsonpath={range .items[*]}{.metadata.namespace} {.metadata.name}{"\n"}{end}'
+ [[ -z pitr-gap-errors-12146 ]]
+ [[ -z pitr-gap-errors ]]
+ kubectl_bin patch pxc -n pitr-gap-errors-12146 pitr-gap-errors --type=merge -p '{"metadata":{"finalizers":[]}}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.1whKbhoalz
++ mktemp
+ local LAST_ERR=/tmp/tmp.AE5goGp2VE
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch pxc -n pitr-gap-errors-12146 pitr-gap-errors --type=merge -p '{"metadata":{"finalizers":[]}}'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.1whKbhoalz
perconaxtradbcluster.pxc.percona.com/pitr-gap-errors patched
+ cat /tmp/tmp.AE5goGp2VE
+ rm /tmp/tmp.1whKbhoalz /tmp/tmp.AE5goGp2VE
+ return 0
+ read -r pns name
+ read -r pns name
+ kubectl_bin get pxc-backup --all-namespaces -o 'jsonpath={range .items[*]}{.metadata.namespace} {.metadata.name}{"\n"}{end}'
+ [[ -z pitr-gap-errors-12146 ]]
+ [[ -z on-pitr-minio ]]
+ kubectl_bin patch pxc-backup -n pitr-gap-errors-12146 on-pitr-minio --type=merge -p '{"metadata":{"finalizers":[]}}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.U5q8XdGnJv
++ mktemp
+ local LAST_ERR=/tmp/tmp.vqhFFDq5oR
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch pxc-backup -n pitr-gap-errors-12146 on-pitr-minio --type=merge -p '{"metadata":{"finalizers":[]}}'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.U5q8XdGnJv
perconaxtradbclusterbackup.pxc.percona.com/on-pitr-minio patched (no change)
+ cat /tmp/tmp.vqhFFDq5oR
+ rm /tmp/tmp.U5q8XdGnJv /tmp/tmp.vqhFFDq5oR
+ return 0
+ read -r pns name
+ [[ -z pitr-gap-errors-12146 ]]
+ [[ -z on-pitr-minio-gap ]]
+ kubectl_bin patch pxc-backup -n pitr-gap-errors-12146 on-pitr-minio-gap --type=merge -p '{"metadata":{"finalizers":[]}}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.AbL0dC9H0k
++ mktemp
+ local LAST_ERR=/tmp/tmp.Xb6dfxjOfw
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch pxc-backup -n pitr-gap-errors-12146 on-pitr-minio-gap --type=merge -p '{"metadata":{"finalizers":[]}}'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.AbL0dC9H0k
perconaxtradbclusterbackup.pxc.percona.com/on-pitr-minio-gap patched (no change)
+ cat /tmp/tmp.Xb6dfxjOfw
+ rm /tmp/tmp.AbL0dC9H0k /tmp/tmp.Xb6dfxjOfw
+ return 0
+ read -r pns name
+ [[ -z pitr-gap-errors-12146 ]]
+ [[ -z on-pitr-minio1 ]]
+ kubectl_bin patch pxc-backup -n pitr-gap-errors-12146 on-pitr-minio1 --type=merge -p '{"metadata":{"finalizers":[]}}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.GBux3MsgdM
++ mktemp
+ local LAST_ERR=/tmp/tmp.WVyvQECVut
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl patch pxc-backup -n pitr-gap-errors-12146 on-pitr-minio1 --type=merge -p '{"metadata":{"finalizers":[]}}'
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.GBux3MsgdM
perconaxtradbclusterbackup.pxc.percona.com/on-pitr-minio1 patched (no change)
+ cat /tmp/tmp.WVyvQECVut
+ rm /tmp/tmp.GBux3MsgdM /tmp/tmp.WVyvQECVut
+ return 0
+ read -r pns name
+ timeout 300 kubectl delete pxc --all --all-namespaces --wait=false --ignore-not-found
perconaxtradbcluster.pxc.percona.com "pitr-gap-errors" deleted from pitr-gap-errors-12146 namespace
+ timeout 300 kubectl delete pxc-backup --all --all-namespaces --wait=false --ignore-not-found
perconaxtradbclusterbackup.pxc.percona.com "on-pitr-minio" deleted from pitr-gap-errors-12146 namespace
perconaxtradbclusterbackup.pxc.percona.com "on-pitr-minio-gap" deleted from pitr-gap-errors-12146 namespace
perconaxtradbclusterbackup.pxc.percona.com "on-pitr-minio1" deleted from pitr-gap-errors-12146 namespace
+ timeout 300 kubectl delete pxc-restore --all --all-namespaces --wait=false --ignore-not-found
perconaxtradbclusterrestore.pxc.percona.com "restore-on-pitr-minio" deleted from pitr-gap-errors-12146 namespace
perconaxtradbclusterrestore.pxc.percona.com "restore-on-pitr-minio-gap-no-pitr" deleted from pitr-gap-errors-12146 namespace
+ create_namespace pxc-operator
+ local namespace=pxc-operator
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ awk '-F ' '{print $2}'
++ helm list --all-namespaces --filter chaos-mesh
++ sed s/NAMESPACE//
++ tail -n1
+ local chaos_mesh_ns=
+ '[' -n '' ']'
++ grep chaos-mesh
++ kubectl get MutatingWebhookConfiguration
++ awk '{print $1}'
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ awk '{print $1}'
++ grep chaos-mesh
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get ValidatingWebhookConfiguration
++ awk '{print $1}'
++ grep validate-auth
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ grep chaos-mesh
++ awk '{print $1}'
++ kubectl api-resources
++ kubectl get crd
++ awk '{print $1}'
++ grep chaos-mesh.org
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ awk '{print $1}'
++ grep chaos-mesh
++ kubectl get clusterrolebinding
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ awk '{print $1}'
++ grep chaos-mesh
++ kubectl get clusterrole
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
+ desc 'cleaned up old namespaces pxc-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces pxc-operator
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace pxc-operator
+ awk '{print$1}'
+ grep -E -v '^kube-|^default|Terminating|pxc-operator|openshift|^gke-|^gmp-|^NAME'
+ xargs kubectl delete ns
++ mktemp
+ kubectl_bin get ns
++ mktemp
+ local LAST_OUT=/tmp/tmp.UzKtDQ2fsC
+ local LAST_OUT=/tmp/tmp.2SpJmHXrom
++ mktemp
++ mktemp
+ local LAST_ERR=/tmp/tmp.WAHnHYpSeO
+ local exit_status=0
++ seq 0 2
+ local LAST_ERR=/tmp/tmp.pOLNTvWsQy
+ local exit_status=0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace pxc-operator
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get ns
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.2SpJmHXrom
+ cat /tmp/tmp.pOLNTvWsQy
+ rm /tmp/tmp.2SpJmHXrom /tmp/tmp.pOLNTvWsQy
+ return 0
namespace "cert-manager" deleted
namespace "pitr-gap-errors-12146" deleted
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.UzKtDQ2fsC
namespace "pxc-operator" deleted
+ cat /tmp/tmp.WAHnHYpSeO
+ rm /tmp/tmp.UzKtDQ2fsC /tmp/tmp.WAHnHYpSeO
+ return 0
+ wait_for_delete namespace/pxc-operator
+ local res=namespace/pxc-operator
+ echo -n 'waiting for namespace/pxc-operator to be deleted'
waiting for namespace/pxc-operator to be deleted+ set +o xtrace
Error from server (NotFound): namespaces "pxc-operator" not found
+ desc 'create namespace pxc-operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace pxc-operator
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace pxc-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.EBGECIGuy4
++ mktemp
+ local LAST_ERR=/tmp/tmp.2Qssib7tAC
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace pxc-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.EBGECIGuy4
namespace/pxc-operator created
+ cat /tmp/tmp.2Qssib7tAC
+ rm /tmp/tmp.EBGECIGuy4 /tmp/tmp.2Qssib7tAC
+ return 0
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.dTNqF9Jlhv
+++ mktemp
++ local LAST_ERR=/tmp/tmp.VhGbl38SEy
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.dTNqF9Jlhv
++ cat /tmp/tmp.VhGbl38SEy
++ rm /tmp/tmp.dTNqF9Jlhv /tmp/tmp.VhGbl38SEy
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2539-dcdf1e8a-1-cluster2 --namespace=pxc-operator
++ mktemp
+ local LAST_OUT=/tmp/tmp.JwVnkw9duU
++ mktemp
+ local LAST_ERR=/tmp/tmp.Dkqx6MkX8g
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2539-dcdf1e8a-1-cluster2 --namespace=pxc-operator
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.JwVnkw9duU
Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2539-dcdf1e8a-1-cluster2" modified.
+ cat /tmp/tmp.Dkqx6MkX8g
+ rm /tmp/tmp.JwVnkw9duU /tmp/tmp.Dkqx6MkX8g
+ return 0
+ deploy_operator
+ desc 'start PXC operator'
+ set +o xtrace


-----------------------------------------------------------------------------------
start PXC operator
-----------------------------------------------------------------------------------

+ kubectl_bin apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/deploy/crd.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.TeNF8tD28W
++ mktemp
+ local LAST_ERR=/tmp/tmp.3FChrDRWmy
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply --server-side --force-conflicts -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/deploy/crd.yaml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.TeNF8tD28W
customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterbackups.pxc.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterrestores.pxc.percona.com serverside-applied
customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com serverside-applied
+ cat /tmp/tmp.3FChrDRWmy
+ rm /tmp/tmp.TeNF8tD28W /tmp/tmp.3FChrDRWmy
+ return 0
+ kubectl_bin wait --for=condition=Established --timeout=120s crd/perconaxtradbclusters.pxc.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.19jPHKBN4L
++ mktemp
+ local LAST_ERR=/tmp/tmp.l6pEhN2pZK
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Established --timeout=120s crd/perconaxtradbclusters.pxc.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.19jPHKBN4L
customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusters.pxc.percona.com condition met
+ cat /tmp/tmp.l6pEhN2pZK
E0702 03:45:49.896462   16226 reflector.go:227] "Failed to watch" err="Get \"https://136.64.139.152/apis/apiextensions.k8s.io/v1/customresourcedefinitions?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dperconaxtradbclusters.pxc.percona.com&resourceVersion=1782963946166224000&timeoutSeconds=317&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured"
+ rm /tmp/tmp.19jPHKBN4L /tmp/tmp.l6pEhN2pZK
+ return 0
+ kubectl_bin wait --for=condition=Established --timeout=120s crd/perconaxtradbclusterbackups.pxc.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.x1A1TyFCOm
++ mktemp
+ local LAST_ERR=/tmp/tmp.c86RmxqEZy
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Established --timeout=120s crd/perconaxtradbclusterbackups.pxc.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.x1A1TyFCOm
customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterbackups.pxc.percona.com condition met
+ cat /tmp/tmp.c86RmxqEZy
E0702 03:45:50.938107   16637 reflector.go:227] "Failed to watch" err="Get \"https://136.64.139.152/apis/apiextensions.k8s.io/v1/customresourcedefinitions?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dperconaxtradbclusterbackups.pxc.percona.com&resourceVersion=1782963946166224000&timeoutSeconds=397&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured"
+ rm /tmp/tmp.x1A1TyFCOm /tmp/tmp.c86RmxqEZy
+ return 0
+ kubectl_bin wait --for=condition=Established --timeout=120s crd/perconaxtradbclusterrestores.pxc.percona.com
++ mktemp
+ local LAST_OUT=/tmp/tmp.xIGTg9iYAL
++ mktemp
+ local LAST_ERR=/tmp/tmp.HOGU2qeJ0Q
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Established --timeout=120s crd/perconaxtradbclusterrestores.pxc.percona.com
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.xIGTg9iYAL
customresourcedefinition.apiextensions.k8s.io/perconaxtradbclusterrestores.pxc.percona.com condition met
+ cat /tmp/tmp.HOGU2qeJ0Q
E0702 03:45:53.537479   16702 reflector.go:227] "Failed to watch" err="Get \"https://136.64.139.152/apis/apiextensions.k8s.io/v1/customresourcedefinitions?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dperconaxtradbclusterrestores.pxc.percona.com&resourceVersion=1782963951166289000&timeoutSeconds=577&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured"
+ rm /tmp/tmp.xIGTg9iYAL /tmp/tmp.HOGU2qeJ0Q
+ return 0
+ '[' -n pxc-operator ']'
+ apply_rbac cw-rbac
+ local operator_namespace=pxc-operator
+ local rbac=cw-rbac
+ sed -e 's^namespace: .*^namespace: pxc-operator^'
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/deploy/cw-rbac.yaml
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.RcCQvRlH1y
++ mktemp
+ local LAST_ERR=/tmp/tmp.CKDXxDWOvZ
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.RcCQvRlH1y
clusterrole.rbac.authorization.k8s.io/percona-xtradb-cluster-operator unchanged
serviceaccount/percona-xtradb-cluster-operator created
clusterrolebinding.rbac.authorization.k8s.io/service-account-percona-xtradb-cluster-operator unchanged
+ cat /tmp/tmp.CKDXxDWOvZ
+ rm /tmp/tmp.RcCQvRlH1y /tmp/tmp.CKDXxDWOvZ
+ return 0
+ sed -e 's^failureThreshold: .*^failureThreshold: 10^'
+ yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "DISABLE_TELEMETRY").value) = "true"' -
+ yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "LOG_LEVEL").value) = "VERBOSE"' -
+ yq eval '(select(.kind == "Deployment").spec.template.spec.containers[] | select(.name == "percona-xtradb-cluster-operator").env[] | select(.name == "PXCO_FEATURE_GATES").value) = ""' -
+ kubectl_bin apply -f -
+ sed -e 's^image: .*^image: perconalab/percona-xtradb-cluster-operator:PR-2539-dcdf1e8a^'
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/deploy/cw-operator.yaml
++ mktemp
+ local LAST_OUT=/tmp/tmp.caZswdLQLh
++ mktemp
+ local LAST_ERR=/tmp/tmp.ANqOQBLEn1
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.caZswdLQLh
deployment.apps/percona-xtradb-cluster-operator created
service/percona-xtradb-cluster-operator created
+ cat /tmp/tmp.ANqOQBLEn1
+ rm /tmp/tmp.caZswdLQLh /tmp/tmp.ANqOQBLEn1
+ return 0
+ sleep 10
+ kubectl_bin wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s
++ mktemp
+ local LAST_OUT=/tmp/tmp.wV2a9TkTy1
++ mktemp
+ local LAST_ERR=/tmp/tmp.4JqaMYv5Yp
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl wait --for=condition=Ready pods -l app.kubernetes.io/component=operator,app.kubernetes.io/instance=percona-xtradb-cluster-operator,app.kubernetes.io/name=percona-xtradb-cluster-operator --timeout=30s
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.wV2a9TkTy1
pod/percona-xtradb-cluster-operator-65b5d85575-6svw2 condition met
+ cat /tmp/tmp.4JqaMYv5Yp
E0702 03:46:14.287653   19861 reflector.go:227] "Failed to watch" err="Get \"https://136.64.139.152/api/v1/namespaces/pxc-operator/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpercona-xtradb-cluster-operator-65b5d85575-6svw2&resourceVersion=1782963973930241000&timeoutSeconds=470&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured"
+ rm /tmp/tmp.wV2a9TkTy1 /tmp/tmp.4JqaMYv5Yp
+ return 0
++ get_operator_pod
++ local label_prefix=app.kubernetes.io/
+++ grep -c percona-xtradb-cluster-operator
+++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator -n pxc-operator
++ local check_label=1
++ [[ 1 -eq 0 ]]
++ head -1
++ jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name'
++ kubectl_bin get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator
+++ mktemp
++ local LAST_OUT=/tmp/tmp.B9ajGQQ4Le
+++ mktemp
++ local LAST_ERR=/tmp/tmp.F32JQJpLQH
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator --field-selector=status.phase=Running -o json -n pxc-operator
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.B9ajGQQ4Le
++ cat /tmp/tmp.F32JQJpLQH
++ rm /tmp/tmp.B9ajGQQ4Le /tmp/tmp.F32JQJpLQH
++ return 0
+ wait_pod percona-xtradb-cluster-operator-65b5d85575-6svw2 480 pxc-operator
+ local pod=percona-xtradb-cluster-operator-65b5d85575-6svw2
+ local max_retry=480
+ local ns=pxc-operator
++ grep -E '^(pxc|proxysql)$'
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ echo percona-xtradb-cluster-operator-65b5d85575-6svw2
+ local container=
+ set +o xtrace
pod/percona-xtradb-cluster-operator-65b5d85575-6svw2 condition met
E0702 03:46:20.360400   20745 reflector.go:227] "Failed to watch" err="Get \"https://136.64.139.152/api/v1/namespaces/pxc-operator/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dpercona-xtradb-cluster-operator-65b5d85575-6svw2&resourceVersion=1782963977649597000&timeoutSeconds=446&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured"
waiting for pod/percona-xtradb-cluster-operator-65b5d85575-6svw2 to become Ready.Ok
+ sleep 3
+ create_namespace pitr-gap-errors-17705
+ local namespace=pitr-gap-errors-17705
+ local skip_clean_namespace=
+ [[ 1 == 1 ]]
+ [[ -z '' ]]
+ destroy_chaos_mesh
++ tail -n1
++ helm list --all-namespaces --filter chaos-mesh
++ awk '-F ' '{print $2}'
++ sed s/NAMESPACE//
+ local chaos_mesh_ns=
+ '[' -n '' ']'
++ kubectl get MutatingWebhookConfiguration
++ awk '{print $1}'
++ grep chaos-mesh
+ timeout 30 kubectl delete MutatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ awk '{print $1}'
++ kubectl get ValidatingWebhookConfiguration
++ grep chaos-mesh
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ grep validate-auth
++ kubectl get ValidatingWebhookConfiguration
++ awk '{print $1}'
+ timeout 30 kubectl delete ValidatingWebhookConfiguration
error: resource(s) were provided, but no name was specified
+ :
++ kubectl api-resources
++ grep chaos-mesh
++ awk '{print $1}'
++ awk '{print $1}'
++ grep chaos-mesh.org
++ kubectl get crd
+ timeout 30 kubectl delete crd
error: resource(s) were provided, but no name was specified
+ :
++ kubectl get clusterrolebinding
++ grep chaos-mesh
++ awk '{print $1}'
+ timeout 30 kubectl delete clusterrolebinding
error: resource(s) were provided, but no name was specified
+ :
++ awk '{print $1}'
++ kubectl get clusterrole
++ grep chaos-mesh
+ timeout 30 kubectl delete clusterrole
error: resource(s) were provided, but no name was specified
+ :
+ desc 'cleaned up all old namespaces'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up all old namespaces
-----------------------------------------------------------------------------------

+ '[' -n '' ']'
+ desc 'cleaned up old namespaces pitr-gap-errors-17705'
+ set +o xtrace


-----------------------------------------------------------------------------------
cleaned up old namespaces pitr-gap-errors-17705
-----------------------------------------------------------------------------------

+ kubectl_bin delete namespace pitr-gap-errors-17705
+ awk '{print$1}'
++ mktemp
+ local LAST_OUT=/tmp/tmp.gTkNcsOTLv
++ mktemp
+ local LAST_ERR=/tmp/tmp.qenQaY9ZHh
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace pitr-gap-errors-17705
+ xargs kubectl delete ns
+ grep -E -v '^kube-|^default|Terminating|pxc-operator|openshift|^gke-|^gmp-|^NAME'
+ kubectl_bin get ns
++ mktemp
+ local LAST_OUT=/tmp/tmp.nBhkawId9n
++ mktemp
+ local LAST_ERR=/tmp/tmp.z3S6HsTOOD
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl get ns
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace pitr-gap-errors-17705
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.nBhkawId9n
+ cat /tmp/tmp.z3S6HsTOOD
+ rm /tmp/tmp.nBhkawId9n /tmp/tmp.z3S6HsTOOD
+ return 0
error: resource(s) were provided, but no name was specified
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl delete namespace pitr-gap-errors-17705
+ exit_status=1
+ set -e
+ '[' 1 '!=' 0 ']'
+ '[' 1 == 1 ']'
+ sleep 0
+ cat /tmp/tmp.gTkNcsOTLv
+ cat /tmp/tmp.qenQaY9ZHh
Error from server (NotFound): namespaces "pitr-gap-errors-17705" not found
+ rm /tmp/tmp.gTkNcsOTLv /tmp/tmp.qenQaY9ZHh
+ return 1
+ :
+ wait_for_delete namespace/pitr-gap-errors-17705
+ local res=namespace/pitr-gap-errors-17705
+ echo -n 'waiting for namespace/pitr-gap-errors-17705 to be deleted'
waiting for namespace/pitr-gap-errors-17705 to be deleted+ set +o xtrace
Error from server (NotFound): namespaces "pitr-gap-errors-17705" not found
+ desc 'create namespace pitr-gap-errors-17705'
+ set +o xtrace


-----------------------------------------------------------------------------------
create namespace pitr-gap-errors-17705
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace pitr-gap-errors-17705
++ mktemp
+ local LAST_OUT=/tmp/tmp.ucZRd6xuSL
++ mktemp
+ local LAST_ERR=/tmp/tmp.zhLTUrdrLJ
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace pitr-gap-errors-17705
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.ucZRd6xuSL
namespace/pitr-gap-errors-17705 created
+ cat /tmp/tmp.zhLTUrdrLJ
+ rm /tmp/tmp.ucZRd6xuSL /tmp/tmp.zhLTUrdrLJ
+ return 0
++ kubectl_bin config current-context
+++ mktemp
++ local LAST_OUT=/tmp/tmp.qQdkuV78S5
+++ mktemp
++ local LAST_ERR=/tmp/tmp.zMqxXc0LTd
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl config current-context
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.qQdkuV78S5
++ cat /tmp/tmp.zMqxXc0LTd
++ rm /tmp/tmp.qQdkuV78S5 /tmp/tmp.zMqxXc0LTd
++ return 0
+ kubectl_bin config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2539-dcdf1e8a-1-cluster2 --namespace=pitr-gap-errors-17705
++ mktemp
+ local LAST_OUT=/tmp/tmp.CjhlKOjj0n
++ mktemp
+ local LAST_ERR=/tmp/tmp.HvDSyJ31be
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl config set-context gke_cloud-dev-112233_us-central1-a_jen-pxc-2539-dcdf1e8a-1-cluster2 --namespace=pitr-gap-errors-17705
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.CjhlKOjj0n
Context "gke_cloud-dev-112233_us-central1-a_jen-pxc-2539-dcdf1e8a-1-cluster2" modified.
+ cat /tmp/tmp.HvDSyJ31be
+ rm /tmp/tmp.CjhlKOjj0n /tmp/tmp.HvDSyJ31be
+ return 0
+ apply_secrets
+ desc 'create secrets for cloud storages'
+ set +o xtrace


-----------------------------------------------------------------------------------
create secrets for cloud storages
-----------------------------------------------------------------------------------

+ '[' -z '' ']'
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/cloud-secret.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.Ek2HLNa8RK
++ mktemp
+ local LAST_ERR=/tmp/tmp.hR93dX32bc
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/minio-secret.yml -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/cloud-secret.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.Ek2HLNa8RK
secret/minio-secret created
secret/aws-s3-secret created
secret/do-spaces-secret created
secret/gcp-cs-secret created
secret/azure-secret created
+ cat /tmp/tmp.hR93dX32bc
+ rm /tmp/tmp.Ek2HLNa8RK /tmp/tmp.hR93dX32bc
+ return 0
+ deploy_cert_manager
+ desc 'deploy cert manager'
+ set +o xtrace


-----------------------------------------------------------------------------------
deploy cert manager
-----------------------------------------------------------------------------------

+ kubectl_bin create namespace cert-manager
++ mktemp
+ local LAST_OUT=/tmp/tmp.W4M9SMZb5p
++ mktemp
+ local LAST_ERR=/tmp/tmp.ctxrYMH8UE
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl create namespace cert-manager
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.W4M9SMZb5p
namespace/cert-manager created
+ cat /tmp/tmp.ctxrYMH8UE
+ rm /tmp/tmp.W4M9SMZb5p /tmp/tmp.ctxrYMH8UE
+ return 0
+ kubectl_bin label namespace cert-manager certmanager.k8s.io/disable-validation=true
++ mktemp
+ local LAST_OUT=/tmp/tmp.QWBZNaW1S1
++ mktemp
+ local LAST_ERR=/tmp/tmp.uaTbHJKWfR
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl label namespace cert-manager certmanager.k8s.io/disable-validation=true
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.QWBZNaW1S1
namespace/cert-manager labeled
+ cat /tmp/tmp.uaTbHJKWfR
+ rm /tmp/tmp.QWBZNaW1S1 /tmp/tmp.uaTbHJKWfR
+ return 0
+ kubectl_bin apply -f https://github.com/jetstack/cert-manager/releases/download/v1.20.2/cert-manager.yaml --validate=false
++ mktemp
+ local LAST_OUT=/tmp/tmp.9CutHUBVkt
++ mktemp
+ local LAST_ERR=/tmp/tmp.uSr9Cr2rQV
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f https://github.com/jetstack/cert-manager/releases/download/v1.20.2/cert-manager.yaml --validate=false
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.9CutHUBVkt
namespace/cert-manager configured
customresourcedefinition.apiextensions.k8s.io/challenges.acme.cert-manager.io unchanged
customresourcedefinition.apiextensions.k8s.io/orders.acme.cert-manager.io unchanged
customresourcedefinition.apiextensions.k8s.io/certificaterequests.cert-manager.io unchanged
customresourcedefinition.apiextensions.k8s.io/certificates.cert-manager.io unchanged
customresourcedefinition.apiextensions.k8s.io/clusterissuers.cert-manager.io unchanged
customresourcedefinition.apiextensions.k8s.io/issuers.cert-manager.io unchanged
serviceaccount/cert-manager-cainjector created
serviceaccount/cert-manager created
serviceaccount/cert-manager-webhook created
clusterrole.rbac.authorization.k8s.io/cert-manager-cainjector unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-issuers unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-clusterissuers unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-certificates unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-orders unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-challenges unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-ingress-shim unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-cluster-view unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-view unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-edit unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-approve:cert-manager-io unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-controller-certificatesigningrequests unchanged
clusterrole.rbac.authorization.k8s.io/cert-manager-webhook:subjectaccessreviews unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-cainjector unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-issuers unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-clusterissuers unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-certificates unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-orders unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-challenges unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-ingress-shim unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-approve:cert-manager-io unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-controller-certificatesigningrequests unchanged
clusterrolebinding.rbac.authorization.k8s.io/cert-manager-webhook:subjectaccessreviews unchanged
role.rbac.authorization.k8s.io/cert-manager-cainjector:leaderelection unchanged
role.rbac.authorization.k8s.io/cert-manager:leaderelection unchanged
role.rbac.authorization.k8s.io/cert-manager-tokenrequest created
role.rbac.authorization.k8s.io/cert-manager-webhook:dynamic-serving created
rolebinding.rbac.authorization.k8s.io/cert-manager-cainjector:leaderelection unchanged
rolebinding.rbac.authorization.k8s.io/cert-manager:leaderelection unchanged
rolebinding.rbac.authorization.k8s.io/cert-manager-tokenrequest created
rolebinding.rbac.authorization.k8s.io/cert-manager-webhook:dynamic-serving created
service/cert-manager-cainjector created
service/cert-manager created
service/cert-manager-webhook created
deployment.apps/cert-manager-cainjector created
deployment.apps/cert-manager created
deployment.apps/cert-manager-webhook created
mutatingwebhookconfiguration.admissionregistration.k8s.io/cert-manager-webhook configured
validatingwebhookconfiguration.admissionregistration.k8s.io/cert-manager-webhook configured
+ cat /tmp/tmp.uSr9Cr2rQV
Warning: resource namespaces/cert-manager is missing the kubectl.kubernetes.io/last-applied-configuration annotation which is required by kubectl apply. kubectl apply should only be used on resources created declaratively by either kubectl create --save-config or kubectl apply. The missing annotation will be patched automatically.
+ rm /tmp/tmp.9CutHUBVkt /tmp/tmp.uSr9Cr2rQV
+ return 0
+ '[' '' == 4.10 ']'
+ kubectl_bin -n cert-manager rollout status deployment/cert-manager --timeout=120s
++ mktemp
+ local LAST_OUT=/tmp/tmp.QuoaPiMeoa
++ mktemp
+ local LAST_ERR=/tmp/tmp.fiPYBc6ZYA
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl -n cert-manager rollout status deployment/cert-manager --timeout=120s
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.QuoaPiMeoa
deployment "cert-manager" successfully rolled out
+ cat /tmp/tmp.fiPYBc6ZYA
+ rm /tmp/tmp.QuoaPiMeoa /tmp/tmp.fiPYBc6ZYA
+ return 0
+ kubectl_bin -n cert-manager rollout status deployment/cert-manager-cainjector --timeout=120s
++ mktemp
+ local LAST_OUT=/tmp/tmp.BdN4kPB3TX
++ mktemp
+ local LAST_ERR=/tmp/tmp.XksWUWZDXL
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl -n cert-manager rollout status deployment/cert-manager-cainjector --timeout=120s
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.BdN4kPB3TX
deployment "cert-manager-cainjector" successfully rolled out
+ cat /tmp/tmp.XksWUWZDXL
+ rm /tmp/tmp.BdN4kPB3TX /tmp/tmp.XksWUWZDXL
+ return 0
+ kubectl_bin -n cert-manager rollout status deployment/cert-manager-webhook --timeout=120s
++ mktemp
+ local LAST_OUT=/tmp/tmp.MgQUB62ndp
++ mktemp
+ local LAST_ERR=/tmp/tmp.CdsxNZ2adP
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl -n cert-manager rollout status deployment/cert-manager-webhook --timeout=120s
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.MgQUB62ndp
Waiting for deployment "cert-manager-webhook" rollout to finish: 0 of 1 updated replicas are available...
deployment "cert-manager-webhook" successfully rolled out
+ cat /tmp/tmp.CdsxNZ2adP
+ rm /tmp/tmp.MgQUB62ndp /tmp/tmp.CdsxNZ2adP
+ return 0
+ log 'waiting for cert-manager webhook caBundle to be injected'
++ date +%Y-%m-%dT%H:%M:%S%z
+ echo '[2026-07-02T03:47:25+0000]' waiting for cert-manager webhook caBundle to be injected
[2026-07-02T03:47:25+0000] waiting for cert-manager webhook caBundle to be injected
++ date +%s
+ local deadline=1782964165
+ kubectl_bin get validatingwebhookconfigurations cert-manager-webhook -o 'jsonpath={.webhooks[0].clientConfig.caBundle}'
+ grep -q .
+ log 'probing cert-manager webhook'
++ date +%Y-%m-%dT%H:%M:%S%z
+ echo '[2026-07-02T03:47:26+0000]' probing cert-manager webhook
[2026-07-02T03:47:26+0000] probing cert-manager webhook
+ local probe_file
++ mktemp
+ probe_file=/tmp/tmp.H1tYENCE9y
+ cat
+ local retries=0
+ kubectl_bin apply -f /tmp/tmp.H1tYENCE9y --dry-run=server
+ rm -f /tmp/tmp.H1tYENCE9y
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/pitr-gap-errors/conf/issuer.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.uViO4yxtoD
++ mktemp
+ local LAST_ERR=/tmp/tmp.fbS39cpK29
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/pitr-gap-errors/conf/issuer.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.uViO4yxtoD
issuer.cert-manager.io/selfsigning-issuer created
+ cat /tmp/tmp.fbS39cpK29
+ rm /tmp/tmp.uViO4yxtoD /tmp/tmp.fbS39cpK29
+ return 0
+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/pitr-gap-errors/conf/cert.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.91QREF5aNj
++ mktemp
+ local LAST_ERR=/tmp/tmp.1n06q8pEnv
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/pitr-gap-errors/conf/cert.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.91QREF5aNj
certificate.cert-manager.io/tls-minio created
+ cat /tmp/tmp.1n06q8pEnv
+ rm /tmp/tmp.91QREF5aNj /tmp/tmp.1n06q8pEnv
+ return 0
+ sleep_with_log 25
+ local d=25
+ log 'sleeping for 25 seconds'
++ date +%Y-%m-%dT%H:%M:%S%z
+ echo '[2026-07-02T03:47:32+0000]' sleeping for 25 seconds
[2026-07-02T03:47:32+0000] sleeping for 25 seconds
+ sleep 25
+ start_minio tls-minio
+ deploy_helm pitr-gap-errors-17705
+ helm repo add hashicorp https://helm.releases.hashicorp.com
"hashicorp" already exists with the same configuration, skipping
+ helm repo add minio https://charts.min.io/
"minio" already exists with the same configuration, skipping
+ helm repo update
Hang tight while we grab the latest from your chart repositories...
...Successfully got an update from the "chaos-mesh" chart repository
...Successfully got an update from the "minio" chart repository
...Successfully got an update from the "hashicorp" chart repository
...Successfully got an update from the "percona" chart repository
Update Complete. ⎈Happy Helming!⎈
+ local cert_secret=tls-minio
+ local endpoint=http://minio-service:9000
+ minio_args=(--version $MINIO_VER --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set "users[0].accessKey=some-access-key" --set "users[0].secretKey=some-secret-key" --set "users[0].policy=consoleAdmin" --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G)
+ local minio_args
+ [[ -n tls-minio ]]
+ endpoint=https://minio-service:9000
+ minio_args+=(--set tls.enabled=true --set tls.certSecret="$cert_secret" --set tls.publicCrt=tls.crt --set tls.privateKey=tls.key)
+ desc 'install Minio'
+ set +o xtrace


-----------------------------------------------------------------------------------
install Minio
-----------------------------------------------------------------------------------

+ helm uninstall minio-service
Error: uninstall: Release not loaded: minio-service: release: not found
+ :
+ retry 10 60 helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G --set tls.enabled=true --set tls.certSecret=tls-minio --set tls.publicCrt=tls.crt --set tls.privateKey=tls.key minio/minio
+ local max=10
+ local delay=60
+ shift 2
+ local n=1
+ helm install minio-service --version 5.4.0 --set replicas=1 --set mode=standalone --set resources.requests.memory=256Mi --set rootUser=rootuser --set rootPassword=rootpass123 --set 'users[0].accessKey=some-access-key' --set 'users[0].secretKey=some-secret-key' --set 'users[0].policy=consoleAdmin' --set service.type=ClusterIP --set configPathmc=/tmp/ --set securityContext.enabled=false --set persistence.size=2G --set tls.enabled=true --set tls.certSecret=tls-minio --set tls.publicCrt=tls.crt --set tls.privateKey=tls.key minio/minio
NAME: minio-service
LAST DEPLOYED: Thu Jul  2 03:48:03 2026
NAMESPACE: pitr-gap-errors-17705
STATUS: deployed
REVISION: 1
TEST SUITE: None
NOTES:
MinIO can be accessed via port 9000 on the following DNS name from within your cluster:
minio-service.pitr-gap-errors-17705.cluster.local

To access MinIO from localhost, run the below commands:

  1. export POD_NAME=$(kubectl get pods --namespace pitr-gap-errors-17705 -l "release=minio-service" -o jsonpath="{.items[0].metadata.name}")

  2. kubectl port-forward $POD_NAME 9000 --namespace pitr-gap-errors-17705

Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/

You can now access MinIO server on http://localhost:9000. Follow the below steps to connect to MinIO server with mc client:

  1. Download the MinIO mc client - https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart

  2. export MC_HOST_minio-service-local=http://$(kubectl get secret --namespace pitr-gap-errors-17705 minio-service -o jsonpath="{.data.rootUser}" | base64 --decode):$(kubectl get secret --namespace pitr-gap-errors-17705 minio-service -o jsonpath="{.data.rootPassword}" | base64 --decode)@localhost:9000

  3. mc ls minio-service-local
+ sleep 30
++ kubectl_bin get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}'
+++ mktemp
++ local LAST_OUT=/tmp/tmp.bKgRPmcexP
+++ mktemp
++ local LAST_ERR=/tmp/tmp.i5rgNO7SLT
++ local exit_status=0
+++ seq 0 2
++ for i in '$(seq 0 2)'
++ set +e
++ kubectl get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}'
++ exit_status=0
++ set -e
++ '[' 0 '!=' 0 ']'
++ break
++ cat /tmp/tmp.bKgRPmcexP
++ cat /tmp/tmp.i5rgNO7SLT
++ rm /tmp/tmp.bKgRPmcexP /tmp/tmp.i5rgNO7SLT
++ return 0
+ MINIO_POD=minio-service-757b74c96c-5d9tc
+ wait_pod minio-service-757b74c96c-5d9tc
+ local pod=minio-service-757b74c96c-5d9tc
+ local max_retry=480
+ local ns=
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ echo minio-service-757b74c96c-5d9tc
++ grep -E '^(pxc|proxysql)$'
+ local container=
+ set +o xtrace
pod/minio-service-757b74c96c-5d9tc condition met
E0702 03:49:06.795657   12777 reflector.go:227] "Failed to watch" err="Get \"https://136.64.139.152/api/v1/namespaces/pitr-gap-errors-17705/pods?allowWatchBookmarks=true&fieldSelector=metadata.name%3Dminio-service-757b74c96c-5d9tc&resourceVersion=1782964144532660000&timeoutSeconds=572&watch=true\": context canceled" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" type="*unstructured.Unstructured"
waiting for pod/minio-service-757b74c96c-5d9tc to become Ready.Ok
+ kubectl_bin run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url https://minio-service:9000 --no-verify-ssl s3 mb s3://operator-testing
++ mktemp
+ local LAST_OUT=/tmp/tmp.xPgBDnIoNc
++ mktemp
+ local LAST_ERR=/tmp/tmp.Wx6vxkiT8u
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- /usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 /usr/bin/aws --endpoint-url https://minio-service:9000 --no-verify-ssl s3 mb s3://operator-testing
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.xPgBDnIoNc
/usr/lib/python2.7/site-packages/botocore/vendored/requests/packages/urllib3/connectionpool.py:768: InsecureRequestWarning: Unverified HTTPS request is being made. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.org/en/latest/security.html
  InsecureRequestWarning)
make_bucket: operator-testing
/usr/lib/python2.7/site-packages/botocore/vendored/requests/packages/urllib3/connectionpool.py:768: InsecureRequestWarning: Unverified HTTPS request is being made. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.org/en/latest/security.html
  InsecureRequestWarning)
make_bucket: operator-testing
pod "aws-cli" deleted from pitr-gap-errors-17705 namespace
+ cat /tmp/tmp.Wx6vxkiT8u
All commands and output from this session will be recorded in container logs, including credentials and sensitive information passed through the command prompt.
If you don't see a command prompt, try pressing enter.
warning: couldn't attach to pod/aws-cli, falling back to streaming logs: Internal error occurred: unable to upgrade connection: container aws-cli not found in pod aws-cli_pitr-gap-errors-17705
+ rm /tmp/tmp.xPgBDnIoNc /tmp/tmp.Wx6vxkiT8u
+ return 0
+ cluster=pitr-gap-errors
+ spinup_pxc pitr-gap-errors /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/pitr-gap-errors/conf/pitr-gap-errors.yml
+ local cluster=pitr-gap-errors
+ local config=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/pitr-gap-errors/conf/pitr-gap-errors.yml
+ local size=3
+ local sleep=10
+ local secretsFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/secrets.yml
+ local pxcClientFile=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/client.yml
+ local port=3306
+ desc 'create first PXC cluster'
+ set +o xtrace


-----------------------------------------------------------------------------------
create first PXC cluster
-----------------------------------------------------------------------------------

+ kubectl_bin apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/secrets.yml
++ mktemp
+ local LAST_OUT=/tmp/tmp.Vl4Ju77C7F
++ mktemp
+ local LAST_ERR=/tmp/tmp.UgQdbao5gg
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/secrets.yml
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.Vl4Ju77C7F
secret/my-cluster-secrets created
secret/some-name-ssl created
secret/some-name-ssl-internal created
+ cat /tmp/tmp.UgQdbao5gg
+ rm /tmp/tmp.Vl4Ju77C7F /tmp/tmp.UgQdbao5gg
+ return 0
+ apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/client.yml
+ local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/client.yml
+ local pvc_name=
+ '[' -z '' ']'
+ kubectl_bin apply -f -
++ mktemp
+ cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/client.yml ''
+ local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/client.yml
+ local pvc_name=
+ local LAST_OUT=/tmp/tmp.DVIFVwkRxL
+ /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #'
+ /usr/bin/sed -e 's#apply:.*#apply: Never#'
+ /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#'
+ /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#'
+ /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#'
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/conf/client.yml
+ /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#'
+ /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#'
+ /usr/bin/sed -e s~minio-service.#namespace~minio-service.pitr-gap-errors-17705~
++ mktemp
+ /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#'
+ local LAST_ERR=/tmp/tmp.syUTOKModW
+ local exit_status=0
+ /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#'
+ /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#'
++ seq 0 2
+ /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2539-dcdf1e8a#'
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.DVIFVwkRxL
deployment.apps/pxc-client created
+ cat /tmp/tmp.syUTOKModW
+ rm /tmp/tmp.DVIFVwkRxL /tmp/tmp.syUTOKModW
+ return 0
+ [[ perconalab/percona-xtradb-cluster-operator:main-pxc8.0 =~ 5\.7 ]]
+ apply_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/pitr-gap-errors/conf/pitr-gap-errors.yml
+ local config_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/pitr-gap-errors/conf/pitr-gap-errors.yml
+ local pvc_name=
+ '[' -z '' ']'
+ kubectl_bin apply -f -
++ mktemp
+ local LAST_OUT=/tmp/tmp.5rsjuW0Qvp
+ cat_config /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/pitr-gap-errors/conf/pitr-gap-errors.yml ''
+ local input_file=/mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/pitr-gap-errors/conf/pitr-gap-errors.yml
+ local pvc_name=
+ /usr/bin/sed -e 's#image:.*-proxysql$#image: perconalab/percona-xtradb-cluster-operator:main-proxysql#'
+ /usr/bin/sed -e 's#image:.*-logcollector$#image: perconalab/fluentbit:main-logcollector#'
+ /usr/bin/sed -e 's#apply:.*#apply: Never#'
+ /usr/bin/sed -e 's#image:.*\/percona-xtradb-cluster:.*$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#'
+ /usr/bin/sed -e 's#image:.*-init$#image: perconalab/percona-xtradb-cluster-operator:PR-2539-dcdf1e8a#'
++ mktemp
+ local LAST_ERR=/tmp/tmp.rZvPPcIOoS
+ local exit_status=0
++ seq 0 2
+ for i in '$(seq 0 2)'
+ set +e
+ kubectl apply -f -
+ cat /mnt/jenkins/workspace/cloud-pxc-operator_PR-2539/e2e-tests/pitr-gap-errors/conf/pitr-gap-errors.yml
+ /usr/bin/sed -e 's#apiVersion: pxc.percona.com/v.*$#apiVersion: pxc.percona.com/v1#'
+ /usr/bin/sed -e 's#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0#'
+ /usr/bin/sed -e 's#image:.*-pmm$#image: perconalab/pmm-client:dev-latest#'
+ /usr/bin/sed -e 's#claimName:..*-backup-pvc$#claimName: #'
+ /usr/bin/sed -e 's#image:.*-backup$#image: perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup#'
+ /usr/bin/sed -e 's#image:.*-haproxy$#image: perconalab/percona-xtradb-cluster-operator:main-haproxy#'
+ /usr/bin/sed -e s~minio-service.#namespace~minio-service.pitr-gap-errors-17705~
+ exit_status=0
+ set -e
+ '[' 0 '!=' 0 ']'
+ break
+ cat /tmp/tmp.5rsjuW0Qvp
perconaxtradbcluster.pxc.percona.com/pitr-gap-errors created
+ cat /tmp/tmp.rZvPPcIOoS
+ rm /tmp/tmp.5rsjuW0Qvp /tmp/tmp.rZvPPcIOoS
+ return 0
+ desc 'waiting pods to be running for pitr-gap-errors'
+ set +o xtrace


-----------------------------------------------------------------------------------
waiting pods to be running for pitr-gap-errors
-----------------------------------------------------------------------------------

++ get_proxy pitr-gap-errors
++ local target_cluster=pitr-gap-errors
+++ kubectl_bin get pxc pitr-gap-errors -o 'jsonpath={.spec.haproxy.enabled}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.uMf4X6fW3c
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.1U2xoJhPj9
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pxc pitr-gap-errors -o 'jsonpath={.spec.haproxy.enabled}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.uMf4X6fW3c
+++ cat /tmp/tmp.1U2xoJhPj9
+++ rm /tmp/tmp.uMf4X6fW3c /tmp/tmp.1U2xoJhPj9
+++ return 0
++ [[ '' == \t\r\u\e ]]
+++ kubectl_bin get pxc pitr-gap-errors -o 'jsonpath={.spec.proxysql.enabled}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.2xzqOi6tYg
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.i1FSKbL9Nd
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pxc pitr-gap-errors -o 'jsonpath={.spec.proxysql.enabled}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.2xzqOi6tYg
+++ cat /tmp/tmp.i1FSKbL9Nd
+++ rm /tmp/tmp.2xzqOi6tYg /tmp/tmp.i1FSKbL9Nd
+++ return 0
++ [[ true == \t\r\u\e ]]
++ echo pitr-gap-errors-proxysql
++ return
+ local proxy=pitr-gap-errors-proxysql
++ get_proxy_size pitr-gap-errors
++ local cluster=pitr-gap-errors
++ local size=0
+++ kubectl_bin get pxc pitr-gap-errors -o 'jsonpath={.spec.haproxy.enabled}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.5DQP3FJlqG
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.vay1Cm3CKn
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pxc pitr-gap-errors -o 'jsonpath={.spec.haproxy.enabled}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.5DQP3FJlqG
+++ cat /tmp/tmp.vay1Cm3CKn
+++ rm /tmp/tmp.5DQP3FJlqG /tmp/tmp.vay1Cm3CKn
+++ return 0
++ [[ '' == \t\r\u\e ]]
+++ kubectl_bin get pxc pitr-gap-errors -o 'jsonpath={.spec.proxysql.enabled}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.eWi4FKEBgy
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.fDRykAA1Sp
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pxc pitr-gap-errors -o 'jsonpath={.spec.proxysql.enabled}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.eWi4FKEBgy
+++ cat /tmp/tmp.fDRykAA1Sp
+++ rm /tmp/tmp.eWi4FKEBgy /tmp/tmp.fDRykAA1Sp
+++ return 0
++ [[ true == \t\r\u\e ]]
+++ kubectl_bin get pxc pitr-gap-errors -o 'jsonpath={.spec.proxysql.size}'
++++ mktemp
+++ local LAST_OUT=/tmp/tmp.KH73M7JUYM
++++ mktemp
+++ local LAST_ERR=/tmp/tmp.28miBTX7bQ
+++ local exit_status=0
++++ seq 0 2
+++ for i in '$(seq 0 2)'
+++ set +e
+++ kubectl get pxc pitr-gap-errors -o 'jsonpath={.spec.proxysql.size}'
+++ exit_status=0
+++ set -e
+++ '[' 0 '!=' 0 ']'
+++ break
+++ cat /tmp/tmp.KH73M7JUYM
+++ cat /tmp/tmp.28miBTX7bQ
+++ rm /tmp/tmp.KH73M7JUYM /tmp/tmp.28miBTX7bQ
+++ return 0
++ size=2
++ echo 2
+ local proxy_size=2
+ wait_for_running pitr-gap-errors-pxc 3
+ local name=pitr-gap-errors-pxc
+ local size=3
+ local last_pod
+ local max_retry=480
+ [[ -z pitr-gap-errors-pxc ]]
+ [[ 3 -le 0 ]]
+ last_pod=2
+ desc 'Waiting for 3 running pods for pitr-gap-errors-pxc'
+ set +o xtrace


-----------------------------------------------------------------------------------
Waiting for 3 running pods for pitr-gap-errors-pxc
-----------------------------------------------------------------------------------

++ seq 0 2
+ for i in '$(seq 0 $last_pod)'
+ wait_pod pitr-gap-errors-pxc-0 480
+ local pod=pitr-gap-errors-pxc-0
+ local max_retry=480
+ local ns=
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
++ echo pitr-gap-errors-pxc-0
+ local container=pxc
+ set +o xtrace
pod/pitr-gap-errors-pxc-0 condition met
waiting for pod/pitr-gap-errors-pxc-0 to become Ready.Ok
+ for i in '$(seq 0 $last_pod)'
+ wait_pod pitr-gap-errors-pxc-1 480
+ local pod=pitr-gap-errors-pxc-1
+ local max_retry=480
+ local ns=
++ echo pitr-gap-errors-pxc-1
++ /usr/bin/sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/'
++ grep -E '^(pxc|proxysql)$'
+ local container=pxc
+ set +o xtrace
pod/pitr-gap-errors-pxc-1 condition met
waiting for pod/pitr-gap-errors-pxc-1 to become Ready.full cluster crash detected