From 41374c16c5a2f82fffddc159876e38096e97d0b1 Mon Sep 17 00:00:00 2001 From: spacewander Date: Mon, 19 Jul 2021 20:57:36 +0800 Subject: [PATCH] feat: add gzip plugin Signed-off-by: spacewander --- apisix/plugins/gzip.lua | 158 ++++++++++++ conf/config-default.yaml | 1 + docs/en/latest/config.json | 1 + docs/en/latest/plugins/gzip.md | 112 +++++++++ t/admin/plugins.t | 2 +- t/debug/debug-mode.t | 1 + t/lib/server.lua | 2 +- t/plugin/gzip.t | 443 +++++++++++++++++++++++++++++++++ 8 files changed, 718 insertions(+), 2 deletions(-) create mode 100644 apisix/plugins/gzip.lua create mode 100644 docs/en/latest/plugins/gzip.md create mode 100644 t/plugin/gzip.t diff --git a/apisix/plugins/gzip.lua b/apisix/plugins/gzip.lua new file mode 100644 index 000000000000..a2263221be0d --- /dev/null +++ b/apisix/plugins/gzip.lua @@ -0,0 +1,158 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +local core = require("apisix.core") +local is_apisix_or, response = pcall(require, "resty.apisix.response") +local ngx_header = ngx.header +local req_http_version = ngx.req.http_version +local str_sub = string.sub +local ipairs = ipairs +local tonumber = tonumber + + +local schema = { + type = "object", + properties = { + types = { + type = "array", + minItems = 1, + items = { + type = "string", + minLength = 1, + }, + default = {"text/html"} + }, + min_length = { + type = "integer", + minimum = 1, + default = 20, + }, + comp_level = { + type = "integer", + minimum = 1, + maximum = 9, + default = 1, + }, + http_version = { + enum = {1.1, 1.0}, + default = 1.1, + }, + buffers = { + type = "object", + properties = { + number = { + type = "integer", + minimum = 1, + default = 32, + }, + size = { + type = "integer", + minimum = 1, + default = 4096, + } + }, + default = { + number = 32, + size = 4096, + } + }, + vary = { + type = "boolean", + } + }, +} + + +local plugin_name = "gzip" + + +local _M = { + version = 0.1, + priority = 995, + name = plugin_name, + schema = schema, +} + + +function _M.check_schema(conf) + return core.schema.check(schema, conf) +end + + +function _M.header_filter(conf, ctx) + if not is_apisix_or then + core.log.error("need to build APISIX-OpenResty to support setting gzip") + return 501 + end + + local types = conf.types + local content_type = ngx_header["Content-Type"] + if not content_type then + -- Like Nginx, don't gzip if Content-Type is missing + return + end + local from = core.string.find(content_type, ";") + if from then + content_type = str_sub(content_type, 1, from - 1) + end + + local matched = false + for _, ty in ipairs(types) do + if content_type == ty then + matched = true + break + end + end + if not matched then + return + end + + local content_length = tonumber(ngx_header["Content-Length"]) + if content_length then + local min_length = conf.min_length + if content_length < min_length then + return + end + -- Like Nginx, don't check min_length if Content-Length is missing + end + + local http_version = req_http_version() + if http_version < conf.http_version then + return + end + + local buffers = conf.buffers + + core.log.info("set gzip with buffers: ", buffers.number, " ", buffers.size, + ", level: ", conf.comp_level) + + local ok, err = response.set_gzip({ + buffer_num = buffers.number, + buffer_size = buffers.size, + compress_level = conf.comp_level, + }) + if not ok then + core.log.error("failed to set gzip: ", err) + return + end + + if conf.vary then + core.response.add_header("Vary", "Accept-Encoding") + end +end + + +return _M diff --git a/conf/config-default.yaml b/conf/config-default.yaml index 8db9af02f15f..84338674cfd9 100644 --- a/conf/config-default.yaml +++ b/conf/config-default.yaml @@ -300,6 +300,7 @@ plugins: # plugin list (sorted by priority) - limit-count # priority: 1002 - limit-req # priority: 1001 #- node-status # priority: 1000 + - gzip # priority: 995 - server-info # priority: 990 - traffic-split # priority: 966 - redirect # priority: 900 diff --git a/docs/en/latest/config.json b/docs/en/latest/config.json index 2f088781efff..718552af790a 100644 --- a/docs/en/latest/config.json +++ b/docs/en/latest/config.json @@ -38,6 +38,7 @@ "plugins/serverless", "plugins/redirect", "plugins/echo", + "plugins/gzip", "plugins/server-info", "plugins/ext-plugin-pre-req", "plugins/ext-plugin-post-req" diff --git a/docs/en/latest/plugins/gzip.md b/docs/en/latest/plugins/gzip.md new file mode 100644 index 000000000000..ef520301ad38 --- /dev/null +++ b/docs/en/latest/plugins/gzip.md @@ -0,0 +1,112 @@ +--- +title: gzip +--- + + + +## Summary + +- [**Name**](#name) +- [**Attributes**](#attributes) +- [**How To Enable**](#how-to-enable) +- [**Test Plugin**](#test-plugin) +- [**Disable Plugin**](#disable-plugin) + +## Name + +The `gzip` plugin dynamically set the gzip behavior of Nginx. + +This plugin requires APISIX to run on [APISIX-OpenResty](../how-to-build.md#6-build-openresty-for-apisix). + +## Attributes + +| Name | Type | Requirement | Default | Valid | Description | +| --------- | ------------- | ----------- | ---------- | ------------------------------------------------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------- | +| types | array | optional | ["text/html"] | | dynamically set the `gzip_types` directive | +| min_length | integer | optional | 20 | >= 1 | dynamically set the `gzip_min_length` directive | +| comp_level | integer | optional | 1 | [1, 9] | dynamically set the `gzip_comp_level` directive | +| http_version | number | optional | 1.1 | 1.1, 1.0 | dynamically set the `gzip_http_version` directive | +| buffers.number | integer | optional | 32 | >= 1 | dynamically set the `gzip_buffers` directive | +| buffers.size | integer | optional | 4096 | >= 1 | dynamically set the `gzip_buffers` directive | +| vary | boolean | optional | false | | dynamically set the `gzip_vary` directive | + +## How To Enable + +Here's an example, enable this plugin on the specified route: + +```shell +curl -i http://127.0.0.1:9080/apisix/admin/routes/1 -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d ' +{ + "uri": "/index.html", + "plugins": { + "gzip": { + "buffers": { + "number": 8 + } + } + }, + "upstream": { + "type": "roundrobin", + "nodes": { + "127.0.0.1:1980": 1 + } + } +}' +``` + +## Test Plugin + +Use curl to access: + +```shell +curl http://127.0.0.1:9080/index.html -i -H "Accept-Encoding: gzip" +HTTP/1.1 404 Not Found +Content-Type: text/html; charset=utf-8 +Transfer-Encoding: chunked +Connection: keep-alive +Date: Wed, 21 Jul 2021 03:52:55 GMT +Server: APISIX/2.7 +Content-Encoding: gzip + +Warning: Binary output can mess up your terminal. Use "--output -" to tell +Warning: curl to output it to your terminal anyway, or consider "--output +Warning: " to save to a file. +``` + +## Disable Plugin + +When you want to disable this plugin, it is very simple, +you can delete the corresponding JSON configuration in the plugin configuration, +no need to restart the service, it will take effect immediately: + +```shell +curl http://127.0.0.1:9080/apisix/admin/routes/1 -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d ' +{ + "uri": "/index.html", + "upstream": { + "type": "roundrobin", + "nodes": { + "127.0.0.1:1980": 1 + } + } +}' +``` + +This plugin has been disabled now. It works for other plugins. diff --git a/t/admin/plugins.t b/t/admin/plugins.t index 61553a35e343..3986fa22a40d 100644 --- a/t/admin/plugins.t +++ b/t/admin/plugins.t @@ -40,7 +40,7 @@ __DATA__ --- request GET /apisix/admin/plugins/list --- response_body_like eval -qr/\["client-control","ext-plugin-pre-req","zipkin","request-id","fault-injection","serverless-pre-function","batch-requests","cors","ip-restriction","referer-restriction","uri-blocker","request-validation","openid-connect","wolf-rbac","hmac-auth","basic-auth","jwt-auth","key-auth","consumer-restriction","authz-keycloak","proxy-mirror","proxy-cache","proxy-rewrite","api-breaker","limit-conn","limit-count","limit-req","server-info","traffic-split","redirect","response-rewrite","grpc-transcode","prometheus","echo","http-logger","sls-logger","tcp-logger","kafka-logger","syslog","udp-logger","example-plugin","serverless-post-function","ext-plugin-post-req"\]/ +qr/\["client-control","ext-plugin-pre-req","zipkin","request-id","fault-injection","serverless-pre-function","batch-requests","cors","ip-restriction","referer-restriction","uri-blocker","request-validation","openid-connect","wolf-rbac","hmac-auth","basic-auth","jwt-auth","key-auth","consumer-restriction","authz-keycloak","proxy-mirror","proxy-cache","proxy-rewrite","api-breaker","limit-conn","limit-count","limit-req","gzip","server-info","traffic-split","redirect","response-rewrite","grpc-transcode","prometheus","echo","http-logger","sls-logger","tcp-logger","kafka-logger","syslog","udp-logger","example-plugin","serverless-post-function","ext-plugin-post-req"\]/ --- no_error_log [error] diff --git a/t/debug/debug-mode.t b/t/debug/debug-mode.t index a3da9534c87e..ea847281a5a1 100644 --- a/t/debug/debug-mode.t +++ b/t/debug/debug-mode.t @@ -71,6 +71,7 @@ loaded plugin and sort by priority: 1005 name: api-breaker loaded plugin and sort by priority: 1003 name: limit-conn loaded plugin and sort by priority: 1002 name: limit-count loaded plugin and sort by priority: 1001 name: limit-req +loaded plugin and sort by priority: 995 name: gzip loaded plugin and sort by priority: 990 name: server-info loaded plugin and sort by priority: 966 name: traffic-split loaded plugin and sort by priority: 900 name: redirect diff --git a/t/lib/server.lua b/t/lib/server.lua index 006a4b415907..9f1849602595 100644 --- a/t/lib/server.lua +++ b/t/lib/server.lua @@ -376,7 +376,7 @@ function _M.echo() for k, v in pairs(hdrs) do ngx.header[k] = v end - ngx.say(ngx.req.get_body_data() or "") + ngx.print(ngx.req.get_body_data() or "") end diff --git a/t/plugin/gzip.t b/t/plugin/gzip.t new file mode 100644 index 000000000000..f29214934c8a --- /dev/null +++ b/t/plugin/gzip.t @@ -0,0 +1,443 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +use t::APISIX; + +my $nginx_binary = $ENV{'TEST_NGINX_BINARY'} || 'nginx'; +my $version = eval { `$nginx_binary -V 2>&1` }; + +if ($version !~ m/\/apisix-nginx-module/) { + plan(skip_all => "apisix-nginx-module not installed"); +} else { + plan('no_plan'); +} + +repeat_each(1); +log_level('info'); +no_root_location(); +no_shuffle(); + +add_block_preprocessor(sub { + my ($block) = @_; + + if (!$block->request) { + $block->set_value("request", "GET /t"); + } + + if ((!defined $block->error_log) && (!defined $block->no_error_log)) { + $block->set_value("no_error_log", "[error]"); + } +}); + +run_tests(); + +__DATA__ + +=== TEST 1: sanity +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "uri": "/echo", + "upstream": { + "type": "roundrobin", + "nodes": { + "127.0.0.1:1980": 1 + } + }, + "plugins": { + "gzip": { + } + } + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } +} +--- response_body +passed + + + +=== TEST 2: hit +--- request +POST /echo +0123456789 +012345678 +--- more_headers +Accept-Encoding: gzip +Content-Type: text/html +--- response_headers +Content-Encoding: gzip +Vary: + + + +=== TEST 3: default buffers and compress level +--- config + location /t { + content_by_lua_block { + local plugin = require("apisix.plugins.gzip") + local core = require("apisix.core") + local json = require("toolkit.json") + + for _, conf in ipairs({ + {}, + {buffers = {}}, + {buffers = {number = 1}}, + {buffers = {size = 1}}, + }) do + local ok, err = plugin.check_schema(conf) + if not ok then + ngx.say(err) + return + end + ngx.say(json.encode(conf.buffers)) + end + } + } +--- response_body +{"number":32,"size":4096} +{"number":32,"size":4096} +{"number":1,"size":4096} +{"number":32,"size":1} + + + +=== TEST 4: compress level +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [=[{ + "uri": "/echo", + "vars": [["http_x", "==", "1"]], + "upstream": { + "type": "roundrobin", + "nodes": { + "127.0.0.1:1980": 1 + } + }, + "plugins": { + "gzip": { + "comp_level": 1 + } + } + }]=] + ) + + if code >= 300 then + ngx.status = code + return + end + + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [=[{ + "uri": "/echo", + "vars": [["http_x", "==", "2"]], + "upstream": { + "type": "roundrobin", + "nodes": { + "127.0.0.1:1980": 1 + } + }, + "plugins": { + "gzip": { + "comp_level": 9 + } + } + }]=] + ) + + if code >= 300 then + ngx.status = code + return + end + ngx.say(body) + } +} +--- response_body +passed + + + +=== TEST 5: hit +--- config + location /t { + content_by_lua_block { + local http = require "resty.http" + local uri = "http://127.0.0.1:" .. ngx.var.server_port + .. "/echo" + local httpc = http.new() + local res, err = httpc:request_uri(uri, + {method = "POST", headers = {x = "1"}, body = ("0123"):rep(1024)}) + if not res then + ngx.say(err) + return + end + local less_compressed = res.body + local res, err = httpc:request_uri(uri, + {method = "POST", headers = {x = "2"}, body = ("0123"):rep(1024)}) + if not res then + ngx.say(err) + return + end + if #less_compressed < 4096 and #less_compressed < #res.body then + ngx.say("ok") + end + } + } +--- response_body +ok + + + +=== TEST 6: min length +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "uri": "/echo", + "upstream": { + "type": "roundrobin", + "nodes": { + "127.0.0.1:1980": 1 + } + }, + "plugins": { + "gzip": { + "min_length": 21 + } + } + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } +} +--- response_body +passed + + + +=== TEST 7: not hit +--- request +POST /echo +0123456789 +012345678 +--- more_headers +Accept-Encoding: gzip +Content-Type: text/html +--- response_headers +Content-Encoding: + + + +=== TEST 8: http version +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "uri": "/echo", + "upstream": { + "type": "roundrobin", + "nodes": { + "127.0.0.1:1980": 1 + } + }, + "plugins": { + "gzip": { + "http_version": 1.1 + } + } + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } +} +--- response_body +passed + + + +=== TEST 9: not hit +--- request +POST /echo HTTP/1.0 +0123456789 +012345678 +--- more_headers +Accept-Encoding: gzip +Content-Type: text/html +--- response_headers +Content-Encoding: + + + +=== TEST 10: hit again +--- request +POST /echo HTTP/1.1 +0123456789 +012345678 +--- more_headers +Accept-Encoding: gzip +Content-Type: text/html +--- response_headers +Content-Encoding: gzip + + + +=== TEST 11: types +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "uri": "/echo", + "upstream": { + "type": "roundrobin", + "nodes": { + "127.0.0.1:1980": 1 + } + }, + "plugins": { + "gzip": { + "types": ["text/plain", "text/xml"] + } + } + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } +} +--- response_body +passed + + + +=== TEST 12: not hit +--- request +POST /echo +0123456789 +012345678 +--- more_headers +Accept-Encoding: gzip +Content-Type: text/html +--- response_headers +Content-Encoding: + + + +=== TEST 13: hit again +--- request +POST /echo +0123456789 +012345678 +--- more_headers +Accept-Encoding: gzip +Content-Type: text/xml +--- response_headers +Content-Encoding: gzip + + + +=== TEST 14: hit with charset +--- request +POST /echo +0123456789 +012345678 +--- more_headers +Accept-Encoding: gzip +Content-Type: text/plain; charset=UTF-8 +--- response_headers +Content-Encoding: gzip + + + +=== TEST 15: vary +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "uri": "/echo", + "upstream": { + "type": "roundrobin", + "nodes": { + "127.0.0.1:1980": 1 + } + }, + "plugins": { + "gzip": { + "vary": true + } + } + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } +} +--- response_body +passed + + + +=== TEST 16: hit +--- request +POST /echo +0123456789 +012345678 +--- more_headers +Accept-Encoding: gzip +Vary: upstream +Content-Type: text/html +--- response_headers +Content-Encoding: gzip +Vary: upstream, Accept-Encoding