Add new protocol for visualization response format (#251)

* added viz protocol, added tests and updated user manual Signed-off-by: chloe-zh <[email protected]> * changed version back to 1.1 Signed-off-by: chloe-zh <[email protected]> * update Signed-off-by: chloe-zh <[email protected]> * update Signed-off-by: chloe-zh <[email protected]>
opensearch-project · Oct 28, 2021 · 6dae7b2 · 6dae7b2
1 parent c88f643
commit 6dae7b2
Show file tree

Hide file tree

Showing 8 changed files with 618 additions and 1 deletion.
diff --git a/docs/user/interfaces/protocol.rst b/docs/user/interfaces/protocol.rst
@@ -382,3 +382,146 @@ Result set::
 
     '+firstname|'=lastname|address
     'Hattie|@Bond|"671 Bristol Street|, Dente, TN"
+
+
+Visualization Format
+====================
+
+To support the Observability visualizations we also provide a new protocol that formats the data in columns for PPL. You can specify the format as "viz" to apply this format to your response, the response is formatted as compact json by default, for example::
+
+    >> curl -H 'Content-Type: application/json -X POST localhost:9200/_plugins/_ppl?format=viz' -d '{
+      "query": "source=accounts"
+    }'
+
+Result set::
+
+    {"data":{"account_number":[1,6,13,18],"firstname":["Amber","Hattie","Nanette","Dale"],"address":["880 Holmes Lane","671 Bristol Street","789 Madison Street","467 Hutchinson Court"],"balance":[39225,5686,32838,4180],"gender":["M","M","F","M"],"city":["Brogan","Dante","Nogal","Orick"],"employer":["Pyrami","Netagy","Quility",null],"state":["IL","TN","VA","MD"],"age":[32,36,28,33],"email":["[email protected]","[email protected]","[email protected]","[email protected]"],"lastname":["Duke","Bond","Bates","Adams"]},"fields":[{"name":"account_number","type":"long"},{"name":"firstname","type":"text"},{"name":"address","type":"text"},{"name":"balance","type":"long"},{"name":"gender","type":"text"},{"name":"city","type":"text"},{"name":"employer","type":"text"},{"name":"state","type":"text"},{"name":"age","type":"long"},{"name":"email","type":"text"},{"name":"lastname","type":"text"}],"size":4,"status":200}
+
+
+You can also shape the format to pretty json by adding additional param ``pretty`` set it to true ``pretty=true``, for example::
+
+    >> curl -H 'Content-Type: application/json -X POST localhost:9200/_plugins/_ppl?format=viz&pretty' -d '{
+      "query": "source=accounts"
+    }'
+
+Result set::
+
+    {
+      "data": {
+        "account_number": [
+          1,
+          6,
+          13,
+          18
+        ],
+        "firstname": [
+          "Amber",
+          "Hattie",
+          "Nanette",
+          "Dale"
+        ],
+        "address": [
+          "880 Holmes Lane",
+          "671 Bristol Street",
+          "789 Madison Street",
+          "467 Hutchinson Court"
+        ],
+        "balance": [
+          39225,
+          5686,
+          32838,
+          4180
+        ],
+        "gender": [
+          "M",
+          "M",
+          "F",
+          "M"
+        ],
+        "city": [
+          "Brogan",
+          "Dante",
+          "Nogal",
+          "Orick"
+        ],
+        "employer": [
+          "Pyrami",
+          "Netagy",
+          "Quility",
+          null
+        ],
+        "state": [
+          "IL",
+          "TN",
+          "VA",
+          "MD"
+        ],
+        "age": [
+          32,
+          36,
+          28,
+          33
+        ],
+        "email": [
+          "[email protected]",
+          "[email protected]",
+          "[email protected]",
+          "[email protected]"
+        ],
+        "lastname": [
+          "Duke",
+          "Bond",
+          "Bates",
+          "Adams"
+        ]
+      },
+      "fields": [
+        {
+          "name": "account_number",
+          "type": "long"
+        },
+        {
+          "name": "firstname",
+          "type": "text"
+        },
+        {
+          "name": "address",
+          "type": "text"
+        },
+        {
+          "name": "balance",
+          "type": "long"
+        },
+        {
+          "name": "gender",
+          "type": "text"
+        },
+        {
+          "name": "city",
+          "type": "text"
+        },
+        {
+          "name": "employer",
+          "type": "text"
+        },
+        {
+          "name": "state",
+          "type": "text"
+        },
+        {
+          "name": "age",
+          "type": "long"
+        },
+        {
+          "name": "email",
+          "type": "text"
+        },
+        {
+          "name": "lastname",
+          "type": "text"
+        }
+      ],
+      "size": 4,
+      "status": 200
+    }
+
diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/VisualizationFormatIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/VisualizationFormatIT.java
@@ -0,0 +1,83 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ *  The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ *
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ *
+ */
+
+package org.opensearch.sql.ppl;
+
+import static org.opensearch.sql.legacy.TestUtils.getResponseBody;
+import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK;
+import static org.opensearch.sql.plugin.rest.RestPPLQueryAction.QUERY_API_ENDPOINT;
+
+import java.io.IOException;
+import java.util.Locale;
+import org.junit.Assert;
+import org.junit.jupiter.api.Test;
+import org.opensearch.client.Request;
+import org.opensearch.client.Response;
+
+public class VisualizationFormatIT extends PPLIntegTestCase {
+  @Override
+  public void init() throws IOException {
+    loadIndex(Index.BANK);
+  }
+
+  @Test
+  void format() throws IOException {
+    String result = executeVizQuery(
+        String.format(Locale.ROOT, "source=%s | fields firstname, age", TEST_INDEX_BANK), true);
+    assertEquals(
+        "{\n"
+            + "  \"data\": {\n"
+            + "    \"firstname\": [\n"
+            + "      \"Amber JOHnny\",\n"
+            + "      \"Hattie\",\n"
+            + "      \"Nanette\",\n"
+            + "      \"Dale\",\n"
+            + "      \"Elinor\",\n"
+            + "      \"Virginia\",\n"
+            + "      \"Dillard\"\n"
+            + "    ],\n"
+            + "    \"age\": [\n"
+            + "      32,\n"
+            + "      36,\n"
+            + "      28,\n"
+            + "      33,\n"
+            + "      36,\n"
+            + "      39,\n"
+            + "      34\n"
+            + "    ]\n"
+            + "  },\n"
+            + "  \"metadata\": {\n"
+            + "    \"fields\": [\n"
+            + "      {\n"
+            + "        \"name\": \"firstname\",\n"
+            + "        \"type\": \"keyword\"\n"
+            + "      },\n"
+            + "      {\n"
+            + "        \"name\": \"age\",\n"
+            + "        \"type\": \"integer\"\n"
+            + "      }\n"
+            + "    ]\n"
+            + "  },\n"
+            + "  \"size\": 7,\n"
+            + "  \"status\": 200\n"
+            + "}",
+        result);
+  }
+
+  private String executeVizQuery(String query, boolean pretty) throws IOException {
+    Request request = buildRequest(query,
+        QUERY_API_ENDPOINT + String.format(Locale.ROOT, "?format=csv&pretty=%b", pretty));
+    Response response = client().performRequest(request);
+    Assert.assertEquals(200, response.getStatusLine().getStatusCode());
+    return getResponseBody(response, true);
+  }
+}
diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/request/PPLQueryRequestFactory.java b/plugin/src/main/java/org/opensearch/sql/plugin/request/PPLQueryRequestFactory.java
@@ -33,6 +33,7 @@
 import org.opensearch.rest.RestRequest;
 import org.opensearch.sql.ppl.domain.PPLQueryRequest;
 import org.opensearch.sql.protocol.response.format.Format;
+import org.opensearch.sql.protocol.response.format.JsonResponseFormatter;
 
 /**
  * Factory of {@link PPLQueryRequest}.
@@ -43,6 +44,7 @@ public class PPLQueryRequestFactory {
   private static final String QUERY_PARAMS_FORMAT = "format";
   private static final String QUERY_PARAMS_SANITIZE = "sanitize";
   private static final String DEFAULT_RESPONSE_FORMAT = "jdbc";
+  private static final String QUERY_PARAMS_PRETTY = "pretty";
 
   /**
    * Build {@link PPLQueryRequest} from {@link RestRequest}.
@@ -75,6 +77,7 @@ private static PPLQueryRequest parsePPLRequestFromPayload(RestRequest restReques
     String content = restRequest.content().utf8ToString();
     JSONObject jsonContent;
     Format format = getFormat(restRequest.params());
+    boolean pretty = getPrettyOption(restRequest.params());
     try {
       jsonContent = new JSONObject(content);
     } catch (JSONException e) {
@@ -86,6 +89,10 @@ private static PPLQueryRequest parsePPLRequestFromPayload(RestRequest restReques
     if (format.equals(Format.CSV)) {
       pplRequest.sanitize(getSanitizeOption(restRequest.params()));
     }
+    // set pretty option
+    if (pretty) {
+      pplRequest.style(JsonResponseFormatter.Style.PRETTY);
+    }
     return pplRequest;
   }
 
@@ -109,4 +116,15 @@ private static boolean getSanitizeOption(Map<String, String> requestParams) {
     }
     return true;
   }
+
+  private static boolean getPrettyOption(Map<String, String> requestParams) {
+    if (requestParams.containsKey(QUERY_PARAMS_PRETTY)) {
+      String prettyValue = requestParams.get(QUERY_PARAMS_PRETTY);
+      if (prettyValue.isEmpty()) {
+        return true;
+      }
+      return Boolean.parseBoolean(prettyValue);
+    }
+    return false;
+  }
 }
diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestPPLQueryAction.java b/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestPPLQueryAction.java
@@ -75,6 +75,7 @@
 import org.opensearch.sql.protocol.response.format.RawResponseFormatter;
 import org.opensearch.sql.protocol.response.format.ResponseFormatter;
 import org.opensearch.sql.protocol.response.format.SimpleJsonResponseFormatter;
+import org.opensearch.sql.protocol.response.format.VisualizationResponseFormatter;
 import org.springframework.context.annotation.AnnotationConfigApplicationContext;
 
 public class RestPPLQueryAction extends BaseRestHandler {
@@ -221,6 +222,8 @@ private ResponseListener<QueryResponse> createListener(RestChannel channel,
       formatter = new CsvResponseFormatter(pplRequest.sanitize());
     } else if (format.equals(Format.RAW)) {
       formatter = new RawResponseFormatter();
+    } else if (format.equals(Format.VIZ)) {
+      formatter = new VisualizationResponseFormatter(pplRequest.style());
     } else {
       formatter = new SimpleJsonResponseFormatter(PRETTY);
     }

diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/domain/PPLQueryRequest.java b/ppl/src/main/java/org/opensearch/sql/ppl/domain/PPLQueryRequest.java
@@ -34,6 +34,7 @@
 import lombok.experimental.Accessors;
 import org.json.JSONObject;
 import org.opensearch.sql.protocol.response.format.Format;
+import org.opensearch.sql.protocol.response.format.JsonResponseFormatter;
 
 @RequiredArgsConstructor
 public class PPLQueryRequest {
@@ -49,6 +50,11 @@ public class PPLQueryRequest {
   @Accessors(fluent = true)
   private boolean sanitize = true;
 
+  @Setter
+  @Getter
+  @Accessors(fluent = true)
+  private JsonResponseFormatter.Style style = JsonResponseFormatter.Style.COMPACT;
+
   /**
    * Constructor of PPLQueryRequest.
    */

diff --git a/protocol/src/main/java/org/opensearch/sql/protocol/response/format/Format.java b/protocol/src/main/java/org/opensearch/sql/protocol/response/format/Format.java
@@ -37,7 +37,8 @@
 public enum Format {
   JDBC("jdbc"),
   CSV("csv"),
-  RAW("raw");
+  RAW("raw"),
+  VIZ("viz");
 
   @Getter
   private final String formatName;