diff --git a/.github/workflows/chaos.yml b/.github/workflows/chaos.yml new file mode 100644 index 000000000000..1b317308bbb4 --- /dev/null +++ b/.github/workflows/chaos.yml @@ -0,0 +1,52 @@ +name: Chaos Test + +on: [pull_request] + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: setup go + uses: actions/setup-go@v2.1.3 + with: + go-version: "1.14" + + - name: Creating minikube cluster + run: | + bash ./t/chaos/utils.sh start_minikube + + - name: Print cluster information + run: | + kubectl config view + kubectl cluster-info + kubectl get nodes + kubectl get pods -n kube-system + kubectl version + + # TODO: should not use personal repo, while it's hard to modify a lot in an archived repo. Could we use api7's repo? + - name: Deploy Etcd Operator + run: | + git clone https://github.com/yiyiyimu/etcd-operator.git --depth 1 + bash etcd-operator/example/rbac/create_role.sh + kubectl create -f etcd-operator/example/deployment.yaml + bash ./t/chaos/utils.sh ensure_pods_ready etcd-operator "True" 30 + kubectl create -f etcd-operator/example/example-etcd-cluster.yaml + bash ./t/chaos/utils.sh ensure_pods_ready etcd "True True True" 30 + + - name: Deploy APISIX + run: | + bash ./t/chaos/utils.sh modify_config + kubectl create configmap apisix-gw-config.yaml --from-file=./conf/config.yaml + kubectl apply -f ./kubernetes/deployment.yaml + kubectl apply -f ./kubernetes/service.yaml + bash ./t/chaos/utils.sh ensure_pods_ready apisix-gw "True" 30 + nohup kubectl port-forward svc/apisix-gw-lb 9080:9080 >/dev/null 2>&1 & + + - name: Deploy Chaos mesh + run: | + curl -sSL https://mirrors.chaos-mesh.org/v1.1.1/install.sh | bash + + - name: run test + working-directory: ./t/chaos + run: go test -v diff --git a/kubernetes/deployment.yaml b/kubernetes/deployment.yaml index 069f72e7f769..cb4dcdc539eb 100644 --- a/kubernetes/deployment.yaml +++ b/kubernetes/deployment.yaml @@ -23,7 +23,7 @@ metadata: name: apisix-gw-deployment # namespace: default spec: - replicas: 2 + replicas: 1 selector: matchLabels: app: apisix-gw @@ -57,6 +57,7 @@ spec: sysctl -w fs.inotify.max_user_watches=524288 sysctl -w fs.inotify.max_queued_events=16384 image: busybox:latest + imagePullPolicy: IfNotPresent name: init-sysctl resources: {} securityContext: diff --git a/t/chaos/go.mod b/t/chaos/go.mod new file mode 100644 index 000000000000..b24abf3454e3 --- /dev/null +++ b/t/chaos/go.mod @@ -0,0 +1,8 @@ +module github.com/apache/apisix/t/chaos + +require ( + github.com/gavv/httpexpect/v2 v2.1.0 + github.com/onsi/gomega v1.9.0 +) + +go 1.14 diff --git a/t/chaos/go.sum b/t/chaos/go.sum new file mode 100644 index 000000000000..5d88022bf1db --- /dev/null +++ b/t/chaos/go.sum @@ -0,0 +1,97 @@ +github.com/ajg/form v1.5.1 h1:t9c7v8JUKu/XxOGBU0yjNpaMloxGEJhUkqFRq0ibGeU= +github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= +github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/fasthttp/websocket v1.4.2 h1:AU/zSiIIAuJjBMf5o+vO0syGOnEfvZRu40xIhW/3RuM= +github.com/fasthttp/websocket v1.4.2/go.mod h1:smsv/h4PBEBaU0XDTY5UwJTpZv69fQ0FfcLJr21mA6Y= +github.com/fatih/structs v1.0.0 h1:BrX964Rv5uQ3wwS+KRUAJCBBw5PQmgJfJ6v4yly5QwU= +github.com/fatih/structs v1.0.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= +github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/gavv/httpexpect/v2 v2.1.0 h1:Q7xnFuKqBY2si4DsqxdbWBt9rfrbVTT2/9YSomc9tEw= +github.com/gavv/httpexpect/v2 v2.1.0/go.mod h1:lnd0TqJLrP+wkJk3SFwtrpSlOAZQ7HaaIFuOYbgqgUM= +github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= +github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= +github.com/gorilla/websocket v1.0.0 h1:J/mA+d2LqcDKjAEhQjXDHt9/e7Cnm+oBUwgHp5C6XDg= +github.com/gorilla/websocket v1.0.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/imkira/go-interpol v1.0.0 h1:HrmLyvOLJyjR0YofMw8QGdCIuYOs4TJUBDNU5sJC09E= +github.com/imkira/go-interpol v1.0.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA= +github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88 h1:uC1QfSlInpQF+M0ao65imhwqKnz3Q2z/d8PWZRMQvDM= +github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k= +github.com/klauspost/compress v1.8.2 h1:Bx0qjetmNjdFXASH02NSAREKpiaDwkO1DRZ3dV2KCcs= +github.com/klauspost/compress v1.8.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/cpuid v1.2.1 h1:vJi+O/nMdFt0vqm8NZBI6wzALWdA2X+egi0ogNyrC/w= +github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= +github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU= +github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/onsi/ginkgo v1.6.0 h1:Ix8l273rp3QzYgXSR+c8d1fTG7UPgYkOSELPhiY/YGw= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.10.1 h1:q/mM8GF/n0shIN8SaAZ0V+jnLPzen6WIVZdiwrRlMlo= +github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME= +github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.9.0 h1:R1uwffexN6Pr340GtYRIdZmAiN4J+iw6WG4wog1DUXg= +github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/savsgio/gotils v0.0.0-20200117113501-90175b0fbe3f h1:PgA+Olipyj258EIEYnpFFONrrCcAIWNUNoFhUfMqAGY= +github.com/savsgio/gotils v0.0.0-20200117113501-90175b0fbe3f/go.mod h1:lHhJedqxCoHN+zMtwGNTXWmF0u9Jt363FYRhV6g0CdY= +github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ= +github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasthttp v1.9.0 h1:hNpmUdy/+ZXYpGy0OBfm7K0UQTzb73W0T0U4iJIVrMw= +github.com/valyala/fasthttp v1.9.0/go.mod h1:FstJa9V+Pj9vQ7OJie2qMHdwemEDaDiSdBnvPM1Su9w= +github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= +github.com/xeipuuv/gojsonschema v1.1.0 h1:ngVtJC9TY/lg0AA/1k48FYhBrhRoFlEmWzsehpNAaZg= +github.com/xeipuuv/gojsonschema v1.1.0/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs= +github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0 h1:6fRhSjgLCkTD3JnJxvaJ4Sj+TYblw757bqYgZaOq5ZY= +github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI= +github.com/yudai/gojsondiff v1.0.0 h1:27cbfqXLVEJ1o8I6v3y9lg8Ydm53EKqHXAOMxEGlCOA= +github.com/yudai/gojsondiff v1.0.0/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FBNExI05xg= +github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82 h1:BHyfKlQyqbsFN5p3IfnEUduWvb9is428/nNb5L3U01M= +github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82/go.mod h1:lgjkn3NuSvDfVJdfcVVdX+jpBxNmX4rDAzaS45IcYoM= +github.com/yudai/pp v2.0.1+incompatible h1:Q4//iY4pNF6yPLZIigmvcl7k/bPgrcTPIFIcmawg5bI= +github.com/yudai/pp v2.0.1+incompatible/go.mod h1:PuxR/8QJ7cyCkFp/aUDS+JY727OFEZkTdatxwunjIkc= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd h1:nTDtHvHSdCn1m6ITfMRqtOd/9+7a3s8RBNOZ3eYZzJA= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297 h1:k7pJ2yAPLPgbskkFdhRCsA77k2fySZ1zf2zCjvQCiIM= +golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f h1:wMNYb4v58l5UBM7MYRLPG6ZhfOqbKu7X5eyFl8ZhKvA= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e h1:o3PsSEY8E4eXWkXrIP9YJALUkVZqzHJT5DOasTyn8Vs= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223 h1:DH4skfRX4EBpamg7iV4ZlCpblAHI6s6TDM39bFZumv8= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 h1:9zdDQZ7Thm29KFXgAX/+yaf3eVbP7djjWp/dXAppNCc= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +moul.io/http2curl v1.0.1-0.20190925090545-5cd742060b0e h1:C7q+e9M5nggAvWfVg9Nl66kebKeuJlP3FD58V4RR5wo= +moul.io/http2curl v1.0.1-0.20190925090545-5cd742060b0e/go.mod h1:nejbQVfXh96n9dSF6cH3Jsk/QI1Z2oEL7sSI2ifXFNA= diff --git a/t/chaos/kill-etcd.yaml b/t/chaos/kill-etcd.yaml new file mode 100644 index 000000000000..98c41f6fe855 --- /dev/null +++ b/t/chaos/kill-etcd.yaml @@ -0,0 +1,29 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +apiVersion: chaos-mesh.org/v1alpha1 +kind: PodChaos +metadata: + name: kill-etcd +spec: + action: pod-kill + mode: all + selector: + labelSelectors: + "app": "etcd" + scheduler: + cron: "@every 10m" diff --git a/t/chaos/kill-etcd_test.go b/t/chaos/kill-etcd_test.go new file mode 100644 index 000000000000..204249cc7ec1 --- /dev/null +++ b/t/chaos/kill-etcd_test.go @@ -0,0 +1,240 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package chaos + +import ( + "fmt" + "net/http" + "os/exec" + "strconv" + "strings" + "testing" + "time" + + "github.com/gavv/httpexpect/v2" + . "github.com/onsi/gomega" +) + +var ( + token = "edd1c9f034335f136f87ad84b625c8f1" + host = "http://127.0.0.1:9080" +) + +type httpTestCase struct { + E *httpexpect.Expect + Method string + Path string + Body string + Headers map[string]string + ExpectStatus int + ExpectBody string +} + +func caseCheck(tc httpTestCase) *httpexpect.Response { + e := tc.E + var req *httpexpect.Request + switch tc.Method { + case http.MethodGet: + req = e.GET(tc.Path) + case http.MethodPut: + req = e.PUT(tc.Path) + default: + } + + if req == nil { + panic("fail to init request") + } + for key, val := range tc.Headers { + req.WithHeader(key, val) + } + if tc.Body != "" { + req.WithText(tc.Body) + } + + resp := req.Expect() + if tc.ExpectStatus != 0 { + resp.Status(tc.ExpectStatus) + } + + if tc.ExpectBody != "" { + resp.Body().Contains(tc.ExpectBody) + } + + return resp +} + +func setRoute(e *httpexpect.Expect, expectStatus int) { + caseCheck(httpTestCase{ + E: e, + Method: http.MethodPut, + Path: "/apisix/admin/routes/1", + Headers: map[string]string{"X-API-KEY": token}, + Body: `{ + "uri": "/hello", + "host": "foo.com", + "plugins": { + "prometheus": {} + }, + "upstream": { + "nodes": { + "bar.org": 1 + }, + "type": "roundrobin" + } + }`, + ExpectStatus: expectStatus, + }) +} + +func getRoute(e *httpexpect.Expect, expectStatus int) { + caseCheck(httpTestCase{ + E: e, + Method: http.MethodGet, + Path: "/hello", + Headers: map[string]string{"Host": "foo.com"}, + ExpectStatus: expectStatus, + }) +} + +func deleteRoute(e *httpexpect.Expect, expectStatus int) { + caseCheck(httpTestCase{ + E: e, + Method: http.MethodDelete, + Path: "/apisix/admin/routes/1", + Headers: map[string]string{"X-API-KEY": token}, + ExpectStatus: expectStatus, + }) +} + +func testPrometheusEtcdMetric(e *httpexpect.Expect, expectEtcd int) { + caseCheck(httpTestCase{ + E: e, + Method: http.MethodGet, + Path: "/apisix/prometheus/metrics", + ExpectBody: fmt.Sprintf("apisix_etcd_reachable %d", expectEtcd), + }) +} + +// get the first line which contains the key +func getPrometheusMetric(e *httpexpect.Expect, g *WithT, key string) string { + resp := caseCheck(httpTestCase{ + E: e, + Method: http.MethodGet, + Path: "/apisix/prometheus/metrics", + }) + resps := strings.Split(resp.Body().Raw(), "\n") + var targetLine string + for _, line := range resps { + if strings.Contains(line, key) { + targetLine = line + break + } + } + targetSlice := strings.Fields(targetLine) + g.Expect(len(targetSlice) == 2).To(BeTrue()) + return targetSlice[1] +} + +func getIngressBandwidthPerSecond(e *httpexpect.Expect, g *WithT) float64 { + key := "apisix_bandwidth{type=\"ingress\"," + bandWidthString := getPrometheusMetric(e, g, key) + bandWidthStart, err := strconv.ParseFloat(bandWidthString, 64) + g.Expect(err).To(BeNil()) + // after etcd got killed, it would take longer time to get the metrics + // so need to calculate the duration + timeStart := time.Now() + + time.Sleep(10 * time.Second) + bandWidthString = getPrometheusMetric(e, g, key) + bandWidthEnd, err := strconv.ParseFloat(bandWidthString, 64) + g.Expect(err).To(BeNil()) + duration := time.Now().Sub(timeStart) + + return (bandWidthEnd - bandWidthStart) / duration.Seconds() +} + +func runCommand(t *testing.T, cmd string) string { + out, err := exec.Command("bash", "-c", cmd).CombinedOutput() + if err != nil { + t.Fatalf("fail to run command %s: %s, %s", cmd, err.Error(), out) + } + return string(out) +} + +func roughCompare(a float64, b float64) bool { + ratio := a / b + if ratio < 1.3 && ratio > 0.7 { + return true + } + return false +} + +func TestGetSuccessWhenEtcdKilled(t *testing.T) { + g := NewWithT(t) + e := httpexpect.New(t, host) + + // check if everything works + setRoute(e, http.StatusCreated) + + // to avoid route haven't been set yet + time.Sleep(1 * time.Second) + getRoute(e, http.StatusOK) + testPrometheusEtcdMetric(e, 1) + + // run in background + go func() { + for { + go getRoute(e, http.StatusOK) + time.Sleep(1000 * time.Millisecond) + } + }() + + // wait 5 second to let first route access returns + time.Sleep(5 * time.Second) + bpsBefore := getIngressBandwidthPerSecond(e, g) + g.Expect(bpsBefore).NotTo(BeZero()) + + podName := runCommand(t, "kubectl get pod -l app=apisix-gw -o 'jsonpath={..metadata.name}'") + t.Run("error log not contains etcd error", func(t *testing.T) { + errorLog := runCommand(t, fmt.Sprintf("kubectl exec -it %s -- cat logs/error.log", podName)) + g.Expect(strings.Contains(errorLog, "failed to fetch data from etcd")).To(BeFalse()) + }) + + // TODO: use client-go + // apply chaos to kill all etcd pods + t.Log("kill all etcd pods") + _ = runCommand(t, "kubectl apply -f kill-etcd.yaml") + time.Sleep(3 * time.Second) + + // fail to set route since etcd is all killed + // while get route could still succeed + setRoute(e, http.StatusInternalServerError) + getRoute(e, http.StatusOK) + testPrometheusEtcdMetric(e, 0) + + t.Run("error log contains etcd error", func(t *testing.T) { + errorLog := runCommand(t, fmt.Sprintf("kubectl exec -it %s -- cat logs/error.log", podName)) + g.Expect(strings.Contains(errorLog, "failed to fetch data from etcd")).To(BeTrue()) + }) + + bpsAfter := getIngressBandwidthPerSecond(e, g) + t.Run("ingress bandwidth per second not change much", func(t *testing.T) { + t.Logf("bps before: %f, after: %f", bpsBefore, bpsAfter) + g.Expect(roughCompare(bpsBefore, bpsAfter)).To(BeTrue()) + }) +} diff --git a/t/chaos/utils.sh b/t/chaos/utils.sh new file mode 100755 index 000000000000..7a250b813423 --- /dev/null +++ b/t/chaos/utils.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -ex + +start_minikube() { + curl -LO "https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl" + chmod +x ./kubectl + sudo mv ./kubectl /usr/local/bin/kubectl + + curl -LO https://storage.googleapis.com/minikube/releases/latest/minikube_latest_amd64.deb + sudo dpkg -i --force-architecture minikube_latest_amd64.deb + minikube start +} + +modify_config() { + DNS_IP=$(kubectl get svc -n kube-system -l k8s-app=kube-dns -o 'jsonpath={..spec.clusterIP}') + echo "dns_resolver: + - ${DNS_IP} +etcd: + host: + - \"http://etcd-cluster-client.default.svc.cluster.local:2379\" " > ./conf/config.yaml +} + +ensure_pods_ready() { + local app=$1 + local status=$2 + local retries=$3 + + count=0 + while [[ $(kubectl get pods -l app=${app} -o 'jsonpath={..status.conditions[?(@.type=="Ready")].status}') != ${status} ]]; + do + echo "Waiting for pod running" && sleep 10; + + ((count=count+1)) + if [ $count -gt ${retries} ]; then + printf "Waiting for pod status running timeout\n" + kubectl describe pod -l app=${app} + printf "\n\n" + kubectl logs -l app=${app} + exit 1 + fi + done +} + +"$@"