Added Web Collector catcheable header valuers

dutoitc · Oct 12, 2017 · e4759bd · e4759bd
1 parent d45432f
commit e4759bd
Show file tree

Hide file tree

Showing 5 changed files with 74 additions and 18 deletions.
diff --git a/README.md b/README.md
@@ -84,8 +84,12 @@ RUN ON CRON */5 7-18 * * 1-5
 GIVEN COLLECTOR WEB WITH url=http://localhost:1530/ws/infra/status
     KEEP status AS WEB_STATUS
     KEEP lastReload AS WEB_LAST_RELOAD
+    KEEP body AS WEB_BODY
+    KEEP responseCode as WEB_RETURN_CODE
 THEN STORE VALUES
 ````
+Syntax: KEEP (* | body | contentLength | contentType | responseCode | json expression) AS variable
+
 
 ## JMX collector
 ````

diff --git a/TODO.txt b/TODO.txt
@@ -3,6 +3,7 @@ v1.1.0 Refactoring:
 [ ] Properties should be accessed only when creating connectors
 [ ]
 [ ] http: store statusCode, store executeTime, store responseLength ?
+[ ] WebCollector: keep header.x, json.path, body, ...
 
 
 Add database, keep history + retention time ?

diff --git a/src/main/java/ch/mno/copper/collect/WebCollector.java b/src/main/java/ch/mno/copper/collect/WebCollector.java
@@ -1,10 +1,11 @@
 package ch.mno.copper.collect;
 
 import ch.mno.copper.collect.connectors.HttpConnector;
+import ch.mno.copper.collect.connectors.HttpResponseData;
 import ch.mno.copper.collect.connectors.JmxConnector;
 import com.jayway.jsonpath.JsonPath;
+import com.jayway.jsonpath.PathNotFoundException;
 import org.apache.commons.lang3.tuple.Pair;
-import org.json.simple.JSONArray;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -51,11 +52,10 @@ public static List<String> query(String url, String username, String password, L
                 throw new RuntimeException("Username-Password is not yet supported");
             }
 
-            String json = conn.get(path);
-            //JSONParser jsonParser = new JSONParser();
-            //JSONObject jsonObject = (JSONObject) jsonParser.parse(json);
 
-            results = extractValues(json, valuesKept);
+            HttpResponseData<String> data = conn.get2(path);
+
+            results = extractValues(data, valuesKept);
         } catch (Exception e) {
             System.err.println("Connector exception (server " + url+ "): " + e.getMessage());
             if (results==null) {
@@ -72,20 +72,32 @@ public static List<String> query(String url, String username, String password, L
         return results;
     }
 
-    static List<String> extractValues(String json, List<Pair<String, String>> valuesKept) {
+    static List<String> extractValues(HttpResponseData<String> data, List<Pair<String, String>> valuesKept) {
         List<String> results = new ArrayList(valuesKept.size());
         for (Pair<String, String> value: valuesKept) {
-            if ("*".equals(value.getKey())) {
-                results.add(json);
+            String key = value.getKey();
+            if ("responseCode".equals(key)) {
+                results.add(String.valueOf(data.getResponseCode()));
+            } else if ("contentLength".equals(key)) {
+                results.add(String.valueOf(data.getContentLength()));}
+            else if ("contentType".equals(key)) {
+                results.add(String.valueOf(data.getContentType()));
+            } else if ("*".equals(key) || "body".equals(key)) {
+                results.add(data.getData());
             } else {
-                net.minidev.json.JSONArray res = JsonPath.read(json, value.getKey());
-                if (res == null || res.size() == 0) {
-                    LOG.info("Warning: jsonpath " + value.getKey() + " not found in " + json);
-                    results.add("ERR_NOT_FOUND");
-                } else if (res.size() > 1) {
-                    results.add("TOO_MUCH_VALUES_FOUND");
-                } else {
-                    results.add(res.get(0).toString());
+                try {
+                    net.minidev.json.JSONArray res = JsonPath.read(data.getData(), key);
+                    if (res == null || res.size() == 0) {
+                        LOG.info("Warning: jsonpath " + key + " not found in " + data);
+                        results.add("ERR_NOT_FOUND");
+                    } else if (res.size() > 1) {
+                        results.add("TOO_MUCH_VALUES_FOUND");
+                    } else {
+                        results.add(res.get(0).toString());
+                    }
+                } catch (PathNotFoundException e) {
+                    LOG.error("Path not  found: " + key);
+                    results.add("?");
                 }
             }
         }

diff --git a/src/main/java/ch/mno/copper/collect/connectors/HttpConnector.java b/src/main/java/ch/mno/copper/collect/connectors/HttpConnector.java
@@ -75,6 +75,28 @@ public String get(String uri) throws ConnectorException {
         }
     }
 
+    public HttpResponseData<String> get2(String uri) throws ConnectorException {
+
+        HttpGet request = new HttpGet(uri);
+
+        try (CloseableHttpResponse response = httpclient.execute(target, request)){
+
+            HttpResponseData<String> data = new HttpResponseData<>();
+            if (response.getStatusLine().getStatusCode()!=200) {
+                data.setData("Error " + response.getStatusLine().getStatusCode() + ":" + response.getStatusLine().getReasonPhrase());
+            } else {
+                data.setData(EntityUtils.toString(response.getEntity()).trim());
+            }
+            data.setResponseCode(response.getStatusLine().getStatusCode());
+            data.setContentLength(response.getLastHeader("Content-Length").getValue());
+            data.setContentType(response.getLastHeader("Content-Type").getValue());
+
+            return data;
+        } catch (IOException e) {
+            throw new ConnectorException("Exception: " + e.getMessage(), e);
+        }
+    }
+
     public void close() {
         try {
             httpclient.close();

diff --git a/src/test/java/ch/mno/copper/collect/WebCollectorTest.java b/src/test/java/ch/mno/copper/collect/WebCollectorTest.java
@@ -1,5 +1,6 @@
 package ch.mno.copper.collect;
 
+import ch.mno.copper.collect.connectors.HttpResponseData;
 import com.jayway.jsonpath.JsonPath;
 import org.apache.commons.lang3.tuple.ImmutablePair;
 import org.apache.commons.lang3.tuple.Pair;
@@ -77,9 +78,25 @@ public void test() {
         // TODO: json ko here, but ok in http://jsonpath.com/
         List<Pair<String, String>> valuesKept = new ArrayList<>();
         valuesKept.add(new ImmutablePair(".jobs[?(@.name=='BLOCK1-compile')].color", "ATEV_compile"));
-        List<String> res = WebCollector.extractValues(json, valuesKept);
-        Assert.assertEquals(1, res.size());
+        valuesKept.add(new ImmutablePair<>("responseCode", "code"));
+        valuesKept.add(new ImmutablePair<>("contentType", "contentType"));
+        valuesKept.add(new ImmutablePair<>("contentLength", "contentLength"));
+
+
+        HttpResponseData<String> d = new HttpResponseData<>();
+        d.setData(json);
+        d.setResponseCode(200);
+        d.setContentType("text/plain");
+        d.setContentLength("123");
+
+        List<String> res = WebCollector.extractValues(d, valuesKept);
+        Assert.assertEquals(4, res.size());
         Assert.assertEquals("blue", res.get(0));
+        Assert.assertEquals("200", res.get(1));
+        Assert.assertEquals("text/plain", res.get(2));
+        Assert.assertEquals("123", res.get(3));
     }
 
+
+
 }