From eead3a0623007c21dbc776cef5d0558565292214 Mon Sep 17 00:00:00 2001
From: nic-chen <johz@163.com>
Date: Wed, 22 Jul 2020 21:16:28 +0800
Subject: [PATCH 01/18] feat: add more metrics for prometheus

---
 apisix/consumer.lua                    |  9 +++
 apisix/http/router/radixtree_sni.lua   |  9 +++
 apisix/plugins/prometheus/exporter.lua | 82 ++++++++++++++++++++++++++
 apisix/router.lua                      |  4 ++
 apisix/schema_def.lua                  |  1 +
 5 files changed, 105 insertions(+)

diff --git a/apisix/consumer.lua b/apisix/consumer.lua
index 31dfa05c80f3..35f01329d82b 100644
--- a/apisix/consumer.lua
+++ b/apisix/consumer.lua
@@ -74,6 +74,15 @@ function _M.plugin(plugin_name)
 end
 
 
+function _M.consumers()
+    if not consumers then
+        return nil, nil
+    end
+
+    return consumers.values, consumers.conf_version
+end
+
+
 function _M.init_worker()
     local err
     consumers, err = core.config.new("/consumers", {
diff --git a/apisix/http/router/radixtree_sni.lua b/apisix/http/router/radixtree_sni.lua
index d042aeedc957..07c689fa8033 100644
--- a/apisix/http/router/radixtree_sni.lua
+++ b/apisix/http/router/radixtree_sni.lua
@@ -204,6 +204,15 @@ function _M.match_and_set(api_ctx)
 end
 
 
+function _M.ssls()
+    if not ssl_certificates then
+        return nil, nil
+    end
+
+    return ssl_certificates.values, ssl_certificates.conf_version
+end
+
+
 function _M.init_worker()
     local err
     ssl_certificates, err = core.config.new("/ssl", {
diff --git a/apisix/plugins/prometheus/exporter.lua b/apisix/plugins/prometheus/exporter.lua
index 2c0b6fc618ab..8536c32d1ffe 100644
--- a/apisix/plugins/prometheus/exporter.lua
+++ b/apisix/plugins/prometheus/exporter.lua
@@ -23,6 +23,14 @@ local re_gmatch = ngx.re.gmatch
 local tonumber = tonumber
 local select = select
 local prometheus
+local router = require("apisix.router")
+local get_routes = router.http_routes
+local get_ssls   = router.ssls
+local get_services = require("apisix.http.service").services
+local get_consumers = require("apisix.consumer").consumers
+local get_upstreams = require("apisix.upstream").upstreams
+
+
 
 -- Default set of latency buckets, 1ms to 60s:
 local DEFAULT_BUCKETS = { 1, 2, 5, 7, 10, 15, 20, 25, 30, 40, 50, 60, 70,
@@ -64,8 +72,15 @@ function _M.init()
             "Number of HTTP connections",
             {"state"})
 
+    metrics.hostname = prometheus:gauge("node_hostname",
+            "Hostname of APISIX node") 
+
     metrics.etcd_reachable = prometheus:gauge("etcd_reachable",
             "Config server etcd reachable from APISIX, 0 is unreachable")
+    
+    metrics.etcd_modify_indexes = prometheus:gauge("etcd_modify_indexes",
+            "Etcd modify index for APISIX keys",
+            {"key"})
 
     -- per service
     metrics.status = prometheus:counter("http_status",
@@ -152,7 +167,68 @@ local function nginx_status()
 
         label_values[1] = name
         metrics.connections:set(val[0], label_values)
+
+    end
+end
+
+local key_values = {}
+function set_modify_index(key, items, items_ver, global_max_idx)
+    local max_idx = 0
+    
+    if items_ver and items then
+        for _, item in ipairs(items) do
+            if type(item) == "table" and item.modifiedIndex > max_idx then
+                max_idx = item.modifiedIndex
+            end
+        end
+    end
+
+    key_values[1] = key
+    metrics.etcd_modify_indexes:set(max_idx, key_values)
+
+
+    global_max_idx = max_idx > global_max_idx and max_idx or global_max_idx
+
+    return global_max_idx
+end
+local function etcd_modify_index()
+    local global_max_idx = 0
+
+    -- routes
+    local routes, routes_ver = get_routes()
+    global_max_idx = set_modify_index("routes", routes, routes_ver, global_max_idx)
+
+    -- services
+    local services, services_ver = get_services()
+    global_max_idx = set_modify_index("services", services, services_ver, global_max_idx)
+
+    -- ssls
+    local ssls, ssls_ver = get_ssls()
+    global_max_idx = set_modify_index("ssls", ssls, ssls_ver, global_max_idx)
+
+    -- consumers
+    local consumers, consumers_ver = get_consumers()
+    global_max_idx = set_modify_index("consumers", consumers, consumers_ver, global_max_idx)
+
+    -- consumers
+    local consumers, consumers_ver = get_consumers()
+    global_max_idx = set_modify_index("consumers", consumers, consumers_ver, global_max_idx)
+
+    -- global_rules
+    local global_rules = router.global_rules
+    if global_rules then
+        global_max_idx = set_modify_index("global_rules", global_rules.values, 
+            global_rules.conf_version, global_max_idx)
     end
+
+    -- upstreams
+    local upstreams, upstreams_ver = get_upstreams()
+    global_max_idx = set_modify_index("upstreams", upstreams, upstreams_ver, global_max_idx)
+    
+    -- global max
+    key_values[1] = "max_modify_index"
+    metrics.etcd_modify_indexes:set(global_max_idx, key_values)
+
 end
 
 
@@ -166,6 +242,9 @@ function _M.collect()
     -- across all services
     nginx_status()
 
+    -- etcd modify index
+    etcd_modify_index()
+
     -- config server status
     local config = core.config.new()
     local version, err = config:server_version()
@@ -178,6 +257,9 @@ function _M.collect()
                        "processing metrics endpoint: ", err)
     end
 
+    local vars = ngx.var
+    metrics.hostname:set(vars.hostname)
+
     core.response.set_header("content_type", "text/plain")
     return 200, core.table.concat(prometheus:metric_data())
 end
diff --git a/apisix/router.lua b/apisix/router.lua
index 4ba870993717..70ac80a5dba5 100644
--- a/apisix/router.lua
+++ b/apisix/router.lua
@@ -103,6 +103,10 @@ function _M.stream_init_worker()
 end
 
 
+function _M.ssls()
+    return _M.router_ssl.ssls()
+end
+
 function _M.http_routes()
     return _M.router_http.routes()
 end
diff --git a/apisix/schema_def.lua b/apisix/schema_def.lua
index d580be6982ed..34957ad8ae39 100644
--- a/apisix/schema_def.lua
+++ b/apisix/schema_def.lua
@@ -463,6 +463,7 @@ _M.service = {
 _M.consumer = {
     type = "object",
     properties = {
+        id = id_schema,
         username = {
             type = "string", minLength = 1, maxLength = 32,
             pattern = [[^[a-zA-Z0-9_]+$]]

From e46a64e379d4ae457e2c686e5d1b26d3c2c9ddf5 Mon Sep 17 00:00:00 2001
From: nic-chen <johz@163.com>
Date: Thu, 23 Jul 2020 08:53:46 +0800
Subject: [PATCH 02/18] test: add test cases

---
 t/plugin/prometheus.t | 24 ++++++++++++++++++++++--
 1 file changed, 22 insertions(+), 2 deletions(-)

diff --git a/t/plugin/prometheus.t b/t/plugin/prometheus.t
index 4e99b84d4a95..4f62fef4e526 100644
--- a/t/plugin/prometheus.t
+++ b/t/plugin/prometheus.t
@@ -620,7 +620,27 @@ qr/apisix_http_overhead_bucket.*service=\"3\".*le=\"00500.0.*/
 
 
 
-=== TEST 30: delete route 4
+=== TEST 30: fetch the prometheus metric data with `hostname`
+--- request
+GET /apisix/prometheus/metrics
+--- response_body_like
+TYPE apisix_node_hostname gauge
+--- no_error_log
+[error]
+
+
+
+=== TEST 31: fetch the prometheus metric data with `modify_indexes`
+--- request
+GET /apisix/prometheus/metrics
+--- response_body_like
+TYPE apisix_etcd_modify_indexes gauge
+--- no_error_log
+[error]
+
+
+
+=== TEST 32: delete route 4
 --- config
     location /t {
         content_by_lua_block {
@@ -643,7 +663,7 @@ passed
 
 
 
-=== TEST 31: delete service 3
+=== TEST 33: delete service 3
 --- config
     location /t {
         content_by_lua_block {

From e050e168d421beb679fd8f11c3368e3aba90318f Mon Sep 17 00:00:00 2001
From: nic-chen <johz@163.com>
Date: Thu, 23 Jul 2020 08:54:04 +0800
Subject: [PATCH 03/18] fix: check lint error

---
 apisix/plugins/prometheus/exporter.lua | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/apisix/plugins/prometheus/exporter.lua b/apisix/plugins/prometheus/exporter.lua
index 8536c32d1ffe..c22dea6261d9 100644
--- a/apisix/plugins/prometheus/exporter.lua
+++ b/apisix/plugins/prometheus/exporter.lua
@@ -22,6 +22,7 @@ local ngx_capture = ngx.location.capture
 local re_gmatch = ngx.re.gmatch
 local tonumber = tonumber
 local select = select
+local type = type
 local prometheus
 local router = require("apisix.router")
 local get_routes = router.http_routes
@@ -73,11 +74,11 @@ function _M.init()
             {"state"})
 
     metrics.hostname = prometheus:gauge("node_hostname",
-            "Hostname of APISIX node") 
+            "Hostname of APISIX node")
 
     metrics.etcd_reachable = prometheus:gauge("etcd_reachable",
             "Config server etcd reachable from APISIX, 0 is unreachable")
-    
+
     metrics.etcd_modify_indexes = prometheus:gauge("etcd_modify_indexes",
             "Etcd modify index for APISIX keys",
             {"key"})
@@ -172,9 +173,9 @@ local function nginx_status()
 end
 
 local key_values = {}
-function set_modify_index(key, items, items_ver, global_max_idx)
+local function set_modify_index(key, items, items_ver, global_max_index)
     local max_idx = 0
-    
+
     if items_ver and items then
         for _, item in ipairs(items) do
             if type(item) == "table" and item.modifiedIndex > max_idx then
@@ -187,9 +188,9 @@ function set_modify_index(key, items, items_ver, global_max_idx)
     metrics.etcd_modify_indexes:set(max_idx, key_values)
 
 
-    global_max_idx = max_idx > global_max_idx and max_idx or global_max_idx
+    global_max_index = max_idx > global_max_index and max_idx or global_max_index
 
-    return global_max_idx
+    return global_max_index
 end
 local function etcd_modify_index()
     local global_max_idx = 0
@@ -217,14 +218,14 @@ local function etcd_modify_index()
     -- global_rules
     local global_rules = router.global_rules
     if global_rules then
-        global_max_idx = set_modify_index("global_rules", global_rules.values, 
+        global_max_idx = set_modify_index("global_rules", global_rules.values,
             global_rules.conf_version, global_max_idx)
     end
 
     -- upstreams
     local upstreams, upstreams_ver = get_upstreams()
     global_max_idx = set_modify_index("upstreams", upstreams, upstreams_ver, global_max_idx)
-    
+
     -- global max
     key_values[1] = "max_modify_index"
     metrics.etcd_modify_indexes:set(global_max_idx, key_values)

From 50f9aba0907983d6ed43fb4623aba55dbd43e170 Mon Sep 17 00:00:00 2001
From: nic-chen <johz@163.com>
Date: Thu, 23 Jul 2020 10:31:34 +0800
Subject: [PATCH 04/18] remove hostname metrics. -> can't save string value.

---
 apisix/plugins/prometheus/exporter.lua |  6 ------
 t/plugin/prometheus.t                  | 16 +++-------------
 2 files changed, 3 insertions(+), 19 deletions(-)

diff --git a/apisix/plugins/prometheus/exporter.lua b/apisix/plugins/prometheus/exporter.lua
index c22dea6261d9..54d19ae54ffd 100644
--- a/apisix/plugins/prometheus/exporter.lua
+++ b/apisix/plugins/prometheus/exporter.lua
@@ -73,9 +73,6 @@ function _M.init()
             "Number of HTTP connections",
             {"state"})
 
-    metrics.hostname = prometheus:gauge("node_hostname",
-            "Hostname of APISIX node")
-
     metrics.etcd_reachable = prometheus:gauge("etcd_reachable",
             "Config server etcd reachable from APISIX, 0 is unreachable")
 
@@ -258,9 +255,6 @@ function _M.collect()
                        "processing metrics endpoint: ", err)
     end
 
-    local vars = ngx.var
-    metrics.hostname:set(vars.hostname)
-
     core.response.set_header("content_type", "text/plain")
     return 200, core.table.concat(prometheus:metric_data())
 end
diff --git a/t/plugin/prometheus.t b/t/plugin/prometheus.t
index 4f62fef4e526..c63aa80d5ce0 100644
--- a/t/plugin/prometheus.t
+++ b/t/plugin/prometheus.t
@@ -620,17 +620,7 @@ qr/apisix_http_overhead_bucket.*service=\"3\".*le=\"00500.0.*/
 
 
 
-=== TEST 30: fetch the prometheus metric data with `hostname`
---- request
-GET /apisix/prometheus/metrics
---- response_body_like
-TYPE apisix_node_hostname gauge
---- no_error_log
-[error]
-
-
-
-=== TEST 31: fetch the prometheus metric data with `modify_indexes`
+=== TEST 30: fetch the prometheus metric data with `modify_indexes`
 --- request
 GET /apisix/prometheus/metrics
 --- response_body_like
@@ -640,7 +630,7 @@ TYPE apisix_etcd_modify_indexes gauge
 
 
 
-=== TEST 32: delete route 4
+=== TEST 31: delete route 4
 --- config
     location /t {
         content_by_lua_block {
@@ -663,7 +653,7 @@ passed
 
 
 
-=== TEST 33: delete service 3
+=== TEST 32: delete service 3
 --- config
     location /t {
         content_by_lua_block {

From 3eacd698105a04100669913a06e8a59b072916ff Mon Sep 17 00:00:00 2001
From: nic-chen <johz@163.com>
Date: Tue, 28 Jul 2020 21:46:43 +0800
Subject: [PATCH 05/18] add more index

---
 apisix/plugins/grpc-transcode/proto.lua |  9 ++++++++
 apisix/plugins/prometheus/exporter.lua  | 28 +++++++++++++++++--------
 apisix/router.lua                       |  8 +++++++
 3 files changed, 36 insertions(+), 9 deletions(-)

diff --git a/apisix/plugins/grpc-transcode/proto.lua b/apisix/plugins/grpc-transcode/proto.lua
index 13f3060b1cd8..09240fa5a5b3 100644
--- a/apisix/plugins/grpc-transcode/proto.lua
+++ b/apisix/plugins/grpc-transcode/proto.lua
@@ -63,6 +63,15 @@ function _M.fetch(proto_id)
 end
 
 
+function _M.protos()
+    if not protos then
+        return nil, nil
+    end
+
+    return protos.values, protos.conf_version
+end
+
+
 function _M.init()
     local err
     protos, err = core.config.new("/proto", {
diff --git a/apisix/plugins/prometheus/exporter.lua b/apisix/plugins/prometheus/exporter.lua
index 54d19ae54ffd..7c9686d63a9f 100644
--- a/apisix/plugins/prometheus/exporter.lua
+++ b/apisix/plugins/prometheus/exporter.lua
@@ -30,6 +30,9 @@ local get_ssls   = router.ssls
 local get_services = require("apisix.http.service").services
 local get_consumers = require("apisix.consumer").consumers
 local get_upstreams = require("apisix.upstream").upstreams
+local clear_tab = core.table.clear
+local get_stream_routes = router.stream_routes
+local get_protos = require("apisix.plugins.grpc-transcode.proto").protos
 
 
 
@@ -43,7 +46,6 @@ local metrics = {}
 
 
     local inner_tab_arr = {}
-    local clear_tab = core.table.clear
 local function gen_arr(...)
     clear_tab(inner_tab_arr)
 
@@ -65,7 +67,7 @@ function _M.init()
         return
     end
 
-    core.table.clear(metrics)
+    clear_tab(metrics)
 
     -- across all services
     prometheus = base_prometheus.init("prometheus-metrics", "apisix_")
@@ -169,10 +171,11 @@ local function nginx_status()
     end
 end
 
+
 local key_values = {}
 local function set_modify_index(key, items, items_ver, global_max_index)
+    clear_tab(key_values)
     local max_idx = 0
-
     if items_ver and items then
         for _, item in ipairs(items) do
             if type(item) == "table" and item.modifiedIndex > max_idx then
@@ -189,7 +192,10 @@ local function set_modify_index(key, items, items_ver, global_max_index)
 
     return global_max_index
 end
+
+
 local function etcd_modify_index()
+    clear_tab(key_values)
     local global_max_idx = 0
 
     -- routes
@@ -208,21 +214,25 @@ local function etcd_modify_index()
     local consumers, consumers_ver = get_consumers()
     global_max_idx = set_modify_index("consumers", consumers, consumers_ver, global_max_idx)
 
-    -- consumers
-    local consumers, consumers_ver = get_consumers()
-    global_max_idx = set_modify_index("consumers", consumers, consumers_ver, global_max_idx)
-
     -- global_rules
     local global_rules = router.global_rules
     if global_rules then
         global_max_idx = set_modify_index("global_rules", global_rules.values,
-            global_rules.conf_version, global_max_idx)
+            global_rules.conf_version, global_max_idx)      
     end
 
     -- upstreams
     local upstreams, upstreams_ver = get_upstreams()
     global_max_idx = set_modify_index("upstreams", upstreams, upstreams_ver, global_max_idx)
 
+    -- stream_routes
+    local stream_routes, stream_routes_ver = get_stream_routes()
+    global_max_idx = set_modify_index("stream_routes", stream_routes, stream_routes_ver, global_max_idx)
+
+    -- proto
+    local protos, protos_ver = get_protos()
+    global_max_idx = set_modify_index("protos", protos, protos_ver, global_max_idx)
+
     -- global max
     key_values[1] = "max_modify_index"
     metrics.etcd_modify_indexes:set(global_max_idx, key_values)
@@ -232,7 +242,7 @@ end
 
 function _M.collect()
     if not prometheus or not metrics then
-        core.log.err("prometheus: plugin is not initialized, please make sure ",
+        core.log.error("prometheus: plugin is not initialized, please make sure ",
                      " 'prometheus_metrics' shared dict is present in nginx template")
         return 500, {message = "An unexpected error occurred"}
     end
diff --git a/apisix/router.lua b/apisix/router.lua
index 70ac80a5dba5..b94be0c42f42 100644
--- a/apisix/router.lua
+++ b/apisix/router.lua
@@ -111,6 +111,14 @@ function _M.http_routes()
     return _M.router_http.routes()
 end
 
+function _M.stream_routes()
+    -- maybe it's not inited.
+    if not _M.router_stream then
+        return nil, nil
+    end
+    return _M.router_stream.routes()
+end
+
 
 -- for test
 _M.filter_test = filter

From 79e6cbf7aae3dd626160d307140f8c1972e6b531 Mon Sep 17 00:00:00 2001
From: nic-chen <johz@163.com>
Date: Tue, 28 Jul 2020 22:40:17 +0800
Subject: [PATCH 06/18] add X-Etcd-Index

---
 apisix/core/config_etcd.lua            | 29 ++++++++++++++++++++++++++
 apisix/plugins/prometheus/exporter.lua |  9 ++++++++
 2 files changed, 38 insertions(+)

diff --git a/apisix/core/config_etcd.lua b/apisix/core/config_etcd.lua
index fbb6df8dbf5d..fe6a10d4e53b 100644
--- a/apisix/core/config_etcd.lua
+++ b/apisix/core/config_etcd.lua
@@ -49,6 +49,26 @@ local mt = {
     end
 }
 
+
+local function getkey(etcd_cli, key)
+    if not etcd_cli then
+        return nil, "not inited"
+    end
+
+    local res, err = etcd_cli:get(key)
+    if not res then
+        -- log.error("failed to get key from etcd: ", err)
+        return nil, err
+    end
+
+    if type(res.body) ~= "table" then
+        return nil, "failed to get key from etcd"
+    end
+
+    return res
+end
+
+
 local function readdir(etcd_cli, key)
     if not etcd_cli then
         return nil, nil, "not inited"
@@ -330,6 +350,15 @@ function _M.get(self, key)
 end
 
 
+function _M.getkey(self, key)
+    if not self.running then
+        return nil, "stoped"
+    end
+
+    return getkey(self.etcd_cli, key)
+end
+
+
 local function _automatic_fetch(premature, self)
     if premature then
         return
diff --git a/apisix/plugins/prometheus/exporter.lua b/apisix/plugins/prometheus/exporter.lua
index 7c9686d63a9f..7ba44303e6e1 100644
--- a/apisix/plugins/prometheus/exporter.lua
+++ b/apisix/plugins/prometheus/exporter.lua
@@ -265,6 +265,15 @@ function _M.collect()
                        "processing metrics endpoint: ", err)
     end
 
+    local res, err = config:getkey("/routes")
+    core.log.error("err:", err)
+    if res and res.headers then
+        clear_tab(key_values)
+        -- global max
+        key_values[1] = "x_etcd_index"
+        metrics.etcd_modify_indexes:set(res.headers["X-Etcd-Index"], key_values)
+    end
+
     core.response.set_header("content_type", "text/plain")
     return 200, core.table.concat(prometheus:metric_data())
 end

From 3e8e19ac07e98c2feb0fb031f3e6714fe5ee7cb4 Mon Sep 17 00:00:00 2001
From: nic-chen <johz@163.com>
Date: Tue, 28 Jul 2020 23:43:18 +0800
Subject: [PATCH 07/18] fix lint

---
 apisix/plugins/prometheus/exporter.lua | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/apisix/plugins/prometheus/exporter.lua b/apisix/plugins/prometheus/exporter.lua
index 7ba44303e6e1..9e14a91ac7b6 100644
--- a/apisix/plugins/prometheus/exporter.lua
+++ b/apisix/plugins/prometheus/exporter.lua
@@ -218,7 +218,7 @@ local function etcd_modify_index()
     local global_rules = router.global_rules
     if global_rules then
         global_max_idx = set_modify_index("global_rules", global_rules.values,
-            global_rules.conf_version, global_max_idx)      
+            global_rules.conf_version, global_max_idx)
     end
 
     -- upstreams
@@ -227,7 +227,8 @@ local function etcd_modify_index()
 
     -- stream_routes
     local stream_routes, stream_routes_ver = get_stream_routes()
-    global_max_idx = set_modify_index("stream_routes", stream_routes, stream_routes_ver, global_max_idx)
+    global_max_idx = set_modify_index("stream_routes", stream_routes,
+        stream_routes_ver, global_max_idx)
 
     -- proto
     local protos, protos_ver = get_protos()

From 895e94fa811f1c59baeaadc268216079c3823e71 Mon Sep 17 00:00:00 2001
From: nic-chen <johz@163.com>
Date: Wed, 29 Jul 2020 10:32:29 +0800
Subject: [PATCH 08/18] test: add more test cases

---
 apisix/plugins/prometheus/exporter.lua |   5 +-
 t/plugin/prometheus.t                  | 115 ++++++++++++++++++++++---
 2 files changed, 106 insertions(+), 14 deletions(-)

diff --git a/apisix/plugins/prometheus/exporter.lua b/apisix/plugins/prometheus/exporter.lua
index 9e14a91ac7b6..4053bb0cf04a 100644
--- a/apisix/plugins/prometheus/exporter.lua
+++ b/apisix/plugins/prometheus/exporter.lua
@@ -219,6 +219,8 @@ local function etcd_modify_index()
     if global_rules then
         global_max_idx = set_modify_index("global_rules", global_rules.values,
             global_rules.conf_version, global_max_idx)
+    else
+        global_max_idx = set_modify_index("global_rules", nil, nil, global_max_idx)
     end
 
     -- upstreams
@@ -266,8 +268,7 @@ function _M.collect()
                        "processing metrics endpoint: ", err)
     end
 
-    local res, err = config:getkey("/routes")
-    core.log.error("err:", err)
+    local res, _ = config:getkey("/routes")
     if res and res.headers then
         clear_tab(key_values)
         -- global max
diff --git a/t/plugin/prometheus.t b/t/plugin/prometheus.t
index c63aa80d5ce0..748e3b08b857 100644
--- a/t/plugin/prometheus.t
+++ b/t/plugin/prometheus.t
@@ -620,17 +620,7 @@ qr/apisix_http_overhead_bucket.*service=\"3\".*le=\"00500.0.*/
 
 
 
-=== TEST 30: fetch the prometheus metric data with `modify_indexes`
---- request
-GET /apisix/prometheus/metrics
---- response_body_like
-TYPE apisix_etcd_modify_indexes gauge
---- no_error_log
-[error]
-
-
-
-=== TEST 31: delete route 4
+=== TEST 30: delete route 4
 --- config
     location /t {
         content_by_lua_block {
@@ -653,7 +643,7 @@ passed
 
 
 
-=== TEST 32: delete service 3
+=== TEST 31: delete service 3
 --- config
     location /t {
         content_by_lua_block {
@@ -673,3 +663,104 @@ GET /t
 passed
 --- no_error_log
 [error]
+
+
+
+=== TEST 32: fetch the prometheus metric data with `modify_indexes consumers`
+--- request
+GET /apisix/prometheus/metrics
+--- response_body_like eval
+qr/apisix_etcd_modify_indexes\{key="consumers"\} \d+/
+--- no_error_log
+[error]
+
+
+
+=== TEST 33: fetch the prometheus metric data with `modify_indexes global_rules`
+--- request
+GET /apisix/prometheus/metrics
+--- response_body_like eval
+qr/apisix_etcd_modify_indexes\{key="global_rules"\} \d+/
+--- no_error_log
+[error]
+
+
+
+=== TEST 34: fetch the prometheus metric data with `modify_indexes max_modify_index`
+--- request
+GET /apisix/prometheus/metrics
+--- response_body_like eval
+qr/apisix_etcd_modify_indexes\{key="max_modify_index"\} \d+/
+--- no_error_log
+[error]
+
+
+
+=== TEST 35: fetch the prometheus metric data with `modify_indexes protos`
+--- request
+GET /apisix/prometheus/metrics
+--- response_body_like eval
+qr/apisix_etcd_modify_indexes\{key="protos"\} \d+/
+--- no_error_log
+[error]
+
+
+
+=== TEST 36: fetch the prometheus metric data with `modify_indexes routes`
+--- request
+GET /apisix/prometheus/metrics
+--- response_body_like eval
+qr/apisix_etcd_modify_indexes\{key="routes"\} \d+/
+--- no_error_log
+[error]
+
+
+
+=== TEST 37: fetch the prometheus metric data with `modify_indexes services`
+--- request
+GET /apisix/prometheus/metrics
+--- response_body_like eval
+qr/apisix_etcd_modify_indexes\{key="services"\} \d+/
+--- no_error_log
+[error]
+
+
+
+=== TEST 38: fetch the prometheus metric data with `modify_indexes ssls`
+--- request
+GET /apisix/prometheus/metrics
+--- response_body_like eval
+qr/apisix_etcd_modify_indexes\{key="ssls"\} \d+/
+--- no_error_log
+[error]
+
+
+
+=== TEST 39: fetch the prometheus metric data with `modify_indexes stream_routes`
+--- request
+GET /apisix/prometheus/metrics
+--- response_body_like eval
+qr/apisix_etcd_modify_indexes\{key="stream_routes"\} \d+/
+--- no_error_log
+[error]
+
+
+
+=== TEST 40: fetch the prometheus metric data with `modify_indexes upstreams`
+--- request
+GET /apisix/prometheus/metrics
+--- response_body_like eval
+qr/apisix_etcd_modify_indexes\{key="upstreams"\} \d+/
+--- no_error_log
+[error]
+
+
+
+=== TEST 41: fetch the prometheus metric data with `modify_indexes x_etcd_index`
+--- request
+GET /apisix/prometheus/metrics
+--- response_body_like eval
+qr/apisix_etcd_modify_indexes\{key="x_etcd_index"\} \d+/
+--- no_error_log
+[error]
+

From ccdb8c4879ae8949e818100ec4d87abc82334dbc Mon Sep 17 00:00:00 2001
From: nic-chen <johz@163.com>
Date: Wed, 29 Jul 2020 14:30:51 +0800
Subject: [PATCH 09/18] fix: Avoid the index to exceed 1000 by updating other
 keys that will causing a full reload

---
 apisix/core/config_etcd.lua |  20 +++++--
 t/core/etcd-sync.t          | 103 ++++++++++++++++++++++++++++++++++++
 2 files changed, 120 insertions(+), 3 deletions(-)
 create mode 100644 t/core/etcd-sync.t

diff --git a/apisix/core/config_etcd.lua b/apisix/core/config_etcd.lua
index fe6a10d4e53b..8144136331b0 100644
--- a/apisix/core/config_etcd.lua
+++ b/apisix/core/config_etcd.lua
@@ -87,12 +87,12 @@ local function readdir(etcd_cli, key)
     return res
 end
 
-local function waitdir(etcd_cli, key, modified_index)
+local function waitdir(etcd_cli, key, modified_index, timeout)
     if not etcd_cli then
         return nil, nil, "not inited"
     end
 
-    local res, err = etcd_cli:waitdir(key, modified_index)
+    local res, err = etcd_cli:waitdir(key, modified_index, timeout)
     if not res then
         -- log.error("failed to get key from etcd: ", err)
         return nil, err
@@ -221,10 +221,22 @@ local function sync_data(self)
         return true
     end
 
-    local dir_res, err = waitdir(self.etcd_cli, self.key, self.prev_index + 1)
+    local dir_res, err = waitdir(self.etcd_cli, self.key, self.prev_index + 1, self.timeout)
+
     log.info("waitdir key: ", self.key, " prev_index: ", self.prev_index + 1)
     log.info("res: ", json.delay_encode(dir_res, true))
     if not dir_res then
+        -- for fetch the last etcd index
+        local key_res, _ = getkey(self.etcd_cli, self.key)
+        if key_res and key_res.headers then
+            local key_index = key_res.headers["X-Etcd-Index"]
+            local key_idx = key_index and tonumber(key_index) or 0
+            if key_idx and key_idx > self.prev_index then
+                -- Avoid the index to exceed 1000 by updating other keys that will causing a full reload
+                self:upgrade_version(key_index)
+            end
+        end
+
         return false, err
     end
 
@@ -424,6 +436,7 @@ function _M.new(key, opts)
     local automatic = opts and opts.automatic
     local item_schema = opts and opts.item_schema
     local filter_fun = opts and opts.filter
+    local timeout = opts and opts.timeout
 
     local obj = setmetatable({
         etcd_cli = etcd_cli,
@@ -439,6 +452,7 @@ function _M.new(key, opts)
         prev_index = nil,
         last_err = nil,
         last_err_time = nil,
+        timeout = timeout,
         filter = filter_fun,
     }, mt)
 
diff --git a/t/core/etcd-sync.t b/t/core/etcd-sync.t
new file mode 100644
index 000000000000..56977c039a9d
--- /dev/null
+++ b/t/core/etcd-sync.t
@@ -0,0 +1,103 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+use t::APISIX 'no_plan';
+
+no_root_location();
+
+run_tests;
+
+__DATA__
+
+=== TEST 1: auto update prev_index when other keys update
+--- config
+    location /t {
+        content_by_lua_block {
+        	local core = require("apisix.core")
+
+		    local consumers, _ = core.config.new("/consumers", {
+		            automatic = true,
+		            item_schema = core.schema.consumer,
+		            timeout = 1
+		        })
+
+		    ngx.sleep(0.5)
+		    local idx = consumers.prev_index
+
+		    core.log.error("idx:", idx)
+
+            local key = "/test_key"
+            local val = "test_value"
+            core.etcd.set(key, val)
+
+            ngx.sleep(2)
+
+            local new_idx = consumers.prev_index
+            core.log.error("new_idx:", new_idx)
+
+            if new_idx > idx then
+            	ngx.say("prev_index updated")
+            end
+        }
+    }
+--- request
+GET /t
+--- response_body
+prev_index updated
+--- timeout: 3
+
+
+
+=== TEST 2: using default timeout
+--- config
+    location /t {
+        content_by_lua_block {
+        	local core = require("apisix.core")
+
+		    local consumers, _ = core.config.new("/consumers", {
+		            automatic = true,
+		            item_schema = core.schema.consumer
+		        })
+
+		    ngx.sleep(0.5)
+		    local idx = consumers.prev_index
+
+		    core.log.error("idx:", idx)
+
+            local key = "/test_key"
+            local val = "test_value"
+            core.etcd.set(key, val)
+
+            ngx.sleep(2)
+
+            local new_idx = consumers.prev_index
+            core.log.error("new_idx:", new_idx)
+
+            if new_idx > idx then
+            	ngx.say("prev_index updated")
+            else
+                ngx.say("prev_index not update")
+            end
+        }
+    }
+--- request
+GET /t
+--- response_body
+prev_index not update
+--- timeout: 3
+
+
+

From bb6f23dbe16038f17c8f70912b816d9abeef678b Mon Sep 17 00:00:00 2001
From: nic-chen <johz@163.com>
Date: Wed, 29 Jul 2020 14:31:42 +0800
Subject: [PATCH 10/18] add prev_index

---
 apisix/plugins/prometheus/exporter.lua |  5 +++++
 t/plugin/prometheus.t                  | 13 +++++++++++--
 2 files changed, 16 insertions(+), 2 deletions(-)

diff --git a/apisix/plugins/prometheus/exporter.lua b/apisix/plugins/prometheus/exporter.lua
index 4053bb0cf04a..7e3d63eae7da 100644
--- a/apisix/plugins/prometheus/exporter.lua
+++ b/apisix/plugins/prometheus/exporter.lua
@@ -219,6 +219,11 @@ local function etcd_modify_index()
     if global_rules then
         global_max_idx = set_modify_index("global_rules", global_rules.values,
             global_rules.conf_version, global_max_idx)
+
+        -- prev_index
+        key_values[1] = "prev_index"
+        metrics.etcd_modify_indexes:set(global_rules.prev_index, key_values)
+
     else
         global_max_idx = set_modify_index("global_rules", nil, nil, global_max_idx)
     end
diff --git a/t/plugin/prometheus.t b/t/plugin/prometheus.t
index 748e3b08b857..3a7d5c7584e1 100644
--- a/t/plugin/prometheus.t
+++ b/t/plugin/prometheus.t
@@ -756,11 +756,20 @@ qr/apisix_etcd_modify_indexes\{key="upstreams"\} \d+/
 
 
 
-=== TEST 41: fetch the prometheus metric data with `modify_indexes x_etcd_index`
+=== TEST 41: fetch the prometheus metric data with `modify_indexes prev_index`
 --- request
 GET /apisix/prometheus/metrics
 --- response_body_like eval
-qr/apisix_etcd_modify_indexes\{key="x_etcd_index"\} \d+/
+qr/apisix_etcd_modify_indexes\{key="prev_index"\} \d+/
 --- no_error_log
 [error]
 
+
+
+=== TEST 42: fetch the prometheus metric data with `modify_indexes x_etcd_index`
+--- request
+GET /apisix/prometheus/metrics
+--- response_body_like eval
+qr/apisix_etcd_modify_indexes\{key="x_etcd_index"\} \d+/
+--- no_error_log
+[error]

From c5fbdb746277425f84d8e4e0ffe9f12194c55f78 Mon Sep 17 00:00:00 2001
From: nic-chen <johz@163.com>
Date: Wed, 29 Jul 2020 14:33:33 +0800
Subject: [PATCH 11/18] fix lint

---
 apisix/core/config_etcd.lua | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/apisix/core/config_etcd.lua b/apisix/core/config_etcd.lua
index 8144136331b0..54f360d63390 100644
--- a/apisix/core/config_etcd.lua
+++ b/apisix/core/config_etcd.lua
@@ -232,7 +232,8 @@ local function sync_data(self)
             local key_index = key_res.headers["X-Etcd-Index"]
             local key_idx = key_index and tonumber(key_index) or 0
             if key_idx and key_idx > self.prev_index then
-                -- Avoid the index to exceed 1000 by updating other keys that will causing a full reload
+                -- Avoid the index to exceed 1000 by updating other keys
+                -- that will causing a full reload
                 self:upgrade_version(key_index)
             end
         end

From b17cdef2a8ccfdf937fe865f8257ab48f320762a Mon Sep 17 00:00:00 2001
From: nic-chen <johz@163.com>
Date: Wed, 29 Jul 2020 22:30:05 +0800
Subject: [PATCH 12/18] fix: fetch key first

---
 apisix/core/config_etcd.lua | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/apisix/core/config_etcd.lua b/apisix/core/config_etcd.lua
index 54f360d63390..aac8ab0ec58c 100644
--- a/apisix/core/config_etcd.lua
+++ b/apisix/core/config_etcd.lua
@@ -221,13 +221,14 @@ local function sync_data(self)
         return true
     end
 
+    -- for fetch the etcd index
+    local key_res, _ = getkey(self.etcd_cli, self.key)
+
     local dir_res, err = waitdir(self.etcd_cli, self.key, self.prev_index + 1, self.timeout)
 
     log.info("waitdir key: ", self.key, " prev_index: ", self.prev_index + 1)
     log.info("res: ", json.delay_encode(dir_res, true))
     if not dir_res then
-        -- for fetch the last etcd index
-        local key_res, _ = getkey(self.etcd_cli, self.key)
         if key_res and key_res.headers then
             local key_index = key_res.headers["X-Etcd-Index"]
             local key_idx = key_index and tonumber(key_index) or 0

From d2aec40834857f550de9b50d71828bae5f61ac25 Mon Sep 17 00:00:00 2001
From: nic-chen <johz@163.com>
Date: Thu, 30 Jul 2020 00:44:01 +0800
Subject: [PATCH 13/18] fix: test error

---
 t/core/etcd-sync.t | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/t/core/etcd-sync.t b/t/core/etcd-sync.t
index 56977c039a9d..4f72bc1abcb5 100644
--- a/t/core/etcd-sync.t
+++ b/t/core/etcd-sync.t
@@ -31,10 +31,10 @@ __DATA__
 		    local consumers, _ = core.config.new("/consumers", {
 		            automatic = true,
 		            item_schema = core.schema.consumer,
-		            timeout = 1
+		            timeout = 0.2
 		        })
 
-		    ngx.sleep(0.5)
+		    ngx.sleep(0.2)
 		    local idx = consumers.prev_index
 
 		    core.log.error("idx:", idx)
@@ -43,7 +43,7 @@ __DATA__
             local val = "test_value"
             core.etcd.set(key, val)
 
-            ngx.sleep(2)
+            ngx.sleep(1)
 
             local new_idx = consumers.prev_index
             core.log.error("new_idx:", new_idx)

From b771bd719bf9a0c4d5e0fb76bfb4917264ccf46a Mon Sep 17 00:00:00 2001
From: nic-chen <johz@163.com>
Date: Thu, 30 Jul 2020 07:35:02 +0800
Subject: [PATCH 14/18] fix update version only when waitdir error is timeout

---
 apisix/core/config_etcd.lua | 4 +++-
 t/core/etcd-sync.t          | 7 ++-----
 2 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/apisix/core/config_etcd.lua b/apisix/core/config_etcd.lua
index aac8ab0ec58c..ba61dcb3b6ba 100644
--- a/apisix/core/config_etcd.lua
+++ b/apisix/core/config_etcd.lua
@@ -228,7 +228,7 @@ local function sync_data(self)
 
     log.info("waitdir key: ", self.key, " prev_index: ", self.prev_index + 1)
     log.info("res: ", json.delay_encode(dir_res, true))
-    if not dir_res then
+    if err and err == "timeout" then
         if key_res and key_res.headers then
             local key_index = key_res.headers["X-Etcd-Index"]
             local key_idx = key_index and tonumber(key_index) or 0
@@ -238,7 +238,9 @@ local function sync_data(self)
                 self:upgrade_version(key_index)
             end
         end
+    end
 
+    if not dir_res then
         return false, err
     end
 
diff --git a/t/core/etcd-sync.t b/t/core/etcd-sync.t
index 4f72bc1abcb5..c869bbe9d570 100644
--- a/t/core/etcd-sync.t
+++ b/t/core/etcd-sync.t
@@ -72,7 +72,7 @@ prev_index updated
 		            item_schema = core.schema.consumer
 		        })
 
-		    ngx.sleep(0.5)
+		    ngx.sleep(0.2)
 		    local idx = consumers.prev_index
 
 		    core.log.error("idx:", idx)
@@ -81,7 +81,7 @@ prev_index updated
             local val = "test_value"
             core.etcd.set(key, val)
 
-            ngx.sleep(2)
+            ngx.sleep(1)
 
             local new_idx = consumers.prev_index
             core.log.error("new_idx:", new_idx)
@@ -98,6 +98,3 @@ GET /t
 --- response_body
 prev_index not update
 --- timeout: 3
-
-
-

From bca3d939afa5cf50651f9874a7257eac6a466e64 Mon Sep 17 00:00:00 2001
From: nic-chen <johz@163.com>
Date: Thu, 30 Jul 2020 09:57:00 +0800
Subject: [PATCH 15/18] test

---
 t/core/etcd-sync.t | 14 +++++---------
 1 file changed, 5 insertions(+), 9 deletions(-)

diff --git a/t/core/etcd-sync.t b/t/core/etcd-sync.t
index c869bbe9d570..9815850cbbbe 100644
--- a/t/core/etcd-sync.t
+++ b/t/core/etcd-sync.t
@@ -35,10 +35,8 @@ __DATA__
 		        })
 
 		    ngx.sleep(0.2)
+            
 		    local idx = consumers.prev_index
-
-		    core.log.error("idx:", idx)
-
             local key = "/test_key"
             local val = "test_value"
             core.etcd.set(key, val)
@@ -46,7 +44,6 @@ __DATA__
             ngx.sleep(1)
 
             local new_idx = consumers.prev_index
-            core.log.error("new_idx:", new_idx)
 
             if new_idx > idx then
             	ngx.say("prev_index updated")
@@ -57,7 +54,8 @@ __DATA__
 GET /t
 --- response_body
 prev_index updated
---- timeout: 3
+--- no_error_log
+[error]
 
 
 
@@ -75,8 +73,6 @@ prev_index updated
 		    ngx.sleep(0.2)
 		    local idx = consumers.prev_index
 
-		    core.log.error("idx:", idx)
-
             local key = "/test_key"
             local val = "test_value"
             core.etcd.set(key, val)
@@ -84,7 +80,6 @@ prev_index updated
             ngx.sleep(1)
 
             local new_idx = consumers.prev_index
-            core.log.error("new_idx:", new_idx)
 
             if new_idx > idx then
             	ngx.say("prev_index updated")
@@ -97,4 +92,5 @@ prev_index updated
 GET /t
 --- response_body
 prev_index not update
---- timeout: 3
+--- no_error_log
+[error]

From 3222bf42c32f4a48da998ed584de21228d063e6e Mon Sep 17 00:00:00 2001
From: nic-chen <johz@163.com>
Date: Thu, 30 Jul 2020 10:14:14 +0800
Subject: [PATCH 16/18] fix: sleep more time

---
 t/core/etcd-sync.t | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/t/core/etcd-sync.t b/t/core/etcd-sync.t
index 9815850cbbbe..53192b033ab0 100644
--- a/t/core/etcd-sync.t
+++ b/t/core/etcd-sync.t
@@ -34,14 +34,14 @@ __DATA__
 		            timeout = 0.2
 		        })
 
-		    ngx.sleep(0.2)
-            
+		    ngx.sleep(0.6)
+
 		    local idx = consumers.prev_index
             local key = "/test_key"
             local val = "test_value"
             core.etcd.set(key, val)
 
-            ngx.sleep(1)
+            ngx.sleep(2)
 
             local new_idx = consumers.prev_index
 
@@ -70,14 +70,14 @@ prev_index updated
 		            item_schema = core.schema.consumer
 		        })
 
-		    ngx.sleep(0.2)
+		    ngx.sleep(0.6)
 		    local idx = consumers.prev_index
 
             local key = "/test_key"
             local val = "test_value"
             core.etcd.set(key, val)
 
-            ngx.sleep(1)
+            ngx.sleep(2)
 
             local new_idx = consumers.prev_index
 

From 74012f23f6efd9c54039ce9e4c11b4a3bcfcde7a Mon Sep 17 00:00:00 2001
From: nic-chen <johz@163.com>
Date: Thu, 30 Jul 2020 11:24:16 +0800
Subject: [PATCH 17/18] fix: code style

---
 apisix/core/config_etcd.lua | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/apisix/core/config_etcd.lua b/apisix/core/config_etcd.lua
index ba61dcb3b6ba..6e616b08f906 100644
--- a/apisix/core/config_etcd.lua
+++ b/apisix/core/config_etcd.lua
@@ -228,7 +228,7 @@ local function sync_data(self)
 
     log.info("waitdir key: ", self.key, " prev_index: ", self.prev_index + 1)
     log.info("res: ", json.delay_encode(dir_res, true))
-    if err and err == "timeout" then
+    if err == "timeout" then
         if key_res and key_res.headers then
             local key_index = key_res.headers["X-Etcd-Index"]
             local key_idx = key_index and tonumber(key_index) or 0

From 58f651b052f6f80a4df75c3d24d18e78c6379b13 Mon Sep 17 00:00:00 2001
From: nic-chen <johz@163.com>
Date: Thu, 30 Jul 2020 13:37:02 +0800
Subject: [PATCH 18/18] fix: code style

---
 t/core/etcd-sync.t | 34 +++++++++++++++++-----------------
 1 file changed, 17 insertions(+), 17 deletions(-)

diff --git a/t/core/etcd-sync.t b/t/core/etcd-sync.t
index 53192b033ab0..ef0ea57cdf5b 100644
--- a/t/core/etcd-sync.t
+++ b/t/core/etcd-sync.t
@@ -26,17 +26,17 @@ __DATA__
 --- config
     location /t {
         content_by_lua_block {
-        	local core = require("apisix.core")
+            local core = require("apisix.core")
 
-		    local consumers, _ = core.config.new("/consumers", {
-		            automatic = true,
-		            item_schema = core.schema.consumer,
-		            timeout = 0.2
-		        })
+            local consumers, _ = core.config.new("/consumers", {
+                automatic = true,
+                item_schema = core.schema.consumer,
+                timeout = 0.2
+            })
 
-		    ngx.sleep(0.6)
+            ngx.sleep(0.6)
 
-		    local idx = consumers.prev_index
+            local idx = consumers.prev_index
             local key = "/test_key"
             local val = "test_value"
             core.etcd.set(key, val)
@@ -46,7 +46,7 @@ __DATA__
             local new_idx = consumers.prev_index
 
             if new_idx > idx then
-            	ngx.say("prev_index updated")
+                ngx.say("prev_index updated")
             end
         }
     }
@@ -63,15 +63,15 @@ prev_index updated
 --- config
     location /t {
         content_by_lua_block {
-        	local core = require("apisix.core")
+            local core = require("apisix.core")
 
-		    local consumers, _ = core.config.new("/consumers", {
-		            automatic = true,
-		            item_schema = core.schema.consumer
-		        })
+            local consumers, _ = core.config.new("/consumers", {
+                automatic = true,
+                item_schema = core.schema.consumer
+            })
 
-		    ngx.sleep(0.6)
-		    local idx = consumers.prev_index
+            ngx.sleep(0.6)
+            local idx = consumers.prev_index
 
             local key = "/test_key"
             local val = "test_value"
@@ -82,7 +82,7 @@ prev_index updated
             local new_idx = consumers.prev_index
 
             if new_idx > idx then
-            	ngx.say("prev_index updated")
+                ngx.say("prev_index updated")
             else
                 ngx.say("prev_index not update")
             end