Skip to content

Commit

Permalink
(#162) Add support in docdb to correctly evaluate json operators.
Browse files Browse the repository at this point in the history 
Summary:
This diff adds support at the docdb level to correctly evaluate the '->' and '->>' json
operators on json columns. Currently, these operators are only supported in the WHERE clause which
helps in filtering out data based on the internal structure of the json data stored in a column.

The only operator that works reliably right now is equality, since operators like >, < etc. would
typically need integer casts.

Test Plan: unit tests.

Reviewers: mihnea, neil, robert

Reviewed By: robert

Subscribers: yql

Differential Revision: https://phabricator.dev.yugabyte.com/D4646
  • Loading branch information
@pritamdamania87
pritamdamania87 committed May 1, 2018
1 parent 0bf5007 commit 68296cb
Show file tree
Hide file tree
Showing 20 changed files with 737 additions and 221 deletions.
84 changes: 76 additions & 8 deletions java/yb-cql/src/test/java/org/yb/cql/TestJson.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
//
package org.yb.cql;

import com.datastax.driver.core.ResultSet;
import com.datastax.driver.core.Row;
import org.junit.Test;
import org.json.*;
Expand All @@ -20,11 +21,31 @@

public class TestJson extends BaseCQLTest {

private void verifyResultSet(ResultSet rs) {
assertEquals(1, rs.getAvailableWithoutFetching());
Row row = rs.one();
JSONObject jsonObject = new JSONObject(row.getJson("c2"));
assertEquals(1, jsonObject.getInt("b"));
assertEquals(false, jsonObject.getJSONArray("a1").getBoolean(3));
assertEquals(3.0, jsonObject.getJSONArray("a1").getDouble(2), 1e-9);
assertEquals(200, jsonObject.getJSONArray("a1").getJSONObject(5).getJSONArray("k2").getInt(1));
assertEquals(2147483647, jsonObject.getJSONObject("a").getJSONObject("q").getInt("s"));
assertEquals("hello", jsonObject.getJSONObject("a").getString("f"));
}

private void testScalar(String json, int c1) {
Row row = session.execute(String.format("SELECT * FROM test_json WHERE c2 = '%s'", json)).one();
assertEquals(c1, row.getInt("c1"));
assertEquals(json, row.getJson("c2"));
}

@Test
public void testJson() throws Exception {
String json =
"{ " +
"\"b\" : 1," +
"\"a2\" : {}," +
"\"a3\" : \"\"," +
"\"a1\" : [1, 2, 3.0, false, true, { \"k1\" : 1, \"k2\" : [100, 200, 300], \"k3\" : true}]," +
"\"a\" :" +
"{" +
Expand All @@ -49,13 +70,60 @@ public void testJson() throws Exception {

session.execute("CREATE TABLE test_json(c1 int, c2 jsonb, PRIMARY KEY(c1))");
session.execute(String.format("INSERT INTO test_json(c1, c2) values (1, '%s');", json));
Row row = session.execute("SELECT * FROM test_json").one();
JSONObject jsonObject = new JSONObject(row.getJson("c2"));
assertEquals(1, jsonObject.getInt("b"));
assertEquals(false, jsonObject.getJSONArray("a1").getBoolean(3));
assertEquals(3.0, jsonObject.getJSONArray("a1").getDouble(2), 1e-9);
assertEquals(200, jsonObject.getJSONArray("a1").getJSONObject(5).getJSONArray("k2").getInt(1));
assertEquals(2147483647, jsonObject.getJSONObject("a").getJSONObject("q").getInt("s"));
assertEquals("hello", jsonObject.getJSONObject("a").getString("f"));
session.execute("INSERT INTO test_json(c1, c2) values (2, '\"abc\"');");
session.execute("INSERT INTO test_json(c1, c2) values (3, '3');");
session.execute("INSERT INTO test_json(c1, c2) values (4, 'true');");
session.execute("INSERT INTO test_json(c1, c2) values (5, 'false');");
session.execute("INSERT INTO test_json(c1, c2) values (6, 'null');");
session.execute("INSERT INTO test_json(c1, c2) values (7, '2.0');");
session.execute("INSERT INTO test_json(c1, c2) values (8, '{\"b\" : 1}');");
verifyResultSet(session.execute("SELECT * FROM test_json WHERE c1 = 1"));

// Test operators.
verifyResultSet(session.execute("SELECT * FROM test_json WHERE c2->'a'->'q'->'p' = " +
"'4294967295'"));
verifyResultSet(session.execute("SELECT * FROM test_json WHERE c2->'a'->'q'->>'p' = " +
"'4294967295'"));
verifyResultSet(session.execute("SELECT * FROM test_json WHERE c2->'a1'->5->'k2'->1 = '200'"));
verifyResultSet(session.execute("SELECT * FROM test_json WHERE c2->'a1'->5->'k3' = 'true'"));
verifyResultSet(session.execute("SELECT * FROM test_json WHERE c2->'a1'->0 = '1'"));
verifyResultSet(session.execute("SELECT * FROM test_json WHERE c2->'a2' = '{}'"));
verifyResultSet(session.execute("SELECT * FROM test_json WHERE c2->'a3' = '\"\"'"));
verifyResultSet(session.execute("SELECT * FROM test_json WHERE c2->'a'->'e' = 'null'"));
verifyResultSet(session.execute("SELECT * FROM test_json WHERE c2->'a'->'c' = 'false'"));
verifyResultSet(session.execute("SELECT * FROM test_json WHERE c2->'a'->>'f' = '\"hello\"'"));
verifyResultSet(session.execute("SELECT * FROM test_json WHERE c2->'a'->>'x' = '2.0'"));
verifyResultSet(session.execute("SELECT * FROM test_json WHERE c2->'a'->'q' = " +
"'{\"r\": -2147483648, \"p\": 4294967295, \"s\": 2147483647}'"));

testScalar("\"abc\"", 2);
testScalar("3", 3);
testScalar("true", 4);
testScalar("false", 5);
testScalar("null", 6);
testScalar("2.0", 7);
assertEquals(2, session.execute("SELECT * FROM test_json WHERE c2->'b' = '1'")
.getAvailableWithoutFetching());

// Test invalid operators. We should never return errors, just return an empty result (this
// is what postgres does).
assertEquals(0, session.execute("SELECT * FROM test_json WHERE c2->'b'->'c' = '1'")
.getAvailableWithoutFetching());
assertEquals(0, session.execute("SELECT * FROM test_json WHERE c2->'z' = '1'")
.getAvailableWithoutFetching());
assertEquals(0, session.execute("SELECT * FROM test_json WHERE c2->2 = '1'")
.getAvailableWithoutFetching());
assertEquals(0, session.execute("SELECT * FROM test_json WHERE c2->'a'->2 = '1'")
.getAvailableWithoutFetching());
assertEquals(0, session.execute("SELECT * FROM test_json WHERE c2->'a1'->'b' = '1'")
.getAvailableWithoutFetching());
assertEquals(0, session.execute("SELECT * FROM test_json WHERE c2->'a1'->6 = '1'")
.getAvailableWithoutFetching());
assertEquals(0, session.execute("SELECT * FROM test_json WHERE c2->'a2'->'a' = '1'")
.getAvailableWithoutFetching());
assertEquals(0, session.execute("SELECT * FROM test_json WHERE c2->'a3'->'a' = '1'")
.getAvailableWithoutFetching());
assertEquals(0, session.execute("SELECT * FROM test_json WHERE c2->'a3'->2 = '1'")
.getAvailableWithoutFetching());
}
}
41 changes: 41 additions & 0 deletions src/yb/common/ql_expr.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
// Copyright (c) YugaByte, Inc.
//--------------------------------------------------------------------------------------------------

#include <yb/util/jsonb.h>
#include "yb/common/ql_expr.h"
#include "yb/common/ql_bfunc.h"

Expand Down Expand Up @@ -29,6 +30,46 @@ CHECKED_STATUS QLExprExecutor::EvalExpr(const QLExpressionPB& ql_expr,
RETURN_NOT_OK(table_row.ReadColumn(ql_expr.column_id(), result));
break;

case QLExpressionPB::ExprCase::kJsonColumn: {
QLValue jsonb;
QLJsonColumnOperationsPB json_op = ql_expr.json_column();
RETURN_NOT_OK(table_row.ReadColumn(json_op.column_id(), &jsonb));
const int num_ops = json_op.json_operations().size();

Slice jsonop_result;
Slice operand(jsonb.jsonb_value());
util::JEntry element_metadata;
for (int i = 0; i < num_ops; i++) {
const QLJsonOperationPB &op = json_op.json_operations().Get(i);
const Status s = util::Jsonb::ApplyJsonbOperator(operand, op, &jsonop_result,
&element_metadata);
if (s.IsNotFound()) {
// We couldn't apply the operator to the operand and hence return null as the result.
result->SetNull();
return Status::OK();
}
RETURN_NOT_OK(s);

if (util::Jsonb::IsScalar(element_metadata) && i != num_ops - 1) {
// We have to apply another operation after this, but we received a scalar intermediate
// result.
result->SetNull();
return Status::OK();
}
operand = jsonop_result;
}

string jsonb_result = jsonop_result.ToBuffer();
if (util::Jsonb::IsScalar(element_metadata)) {
// In case of a scalar that is received from an operation, convert it to a jsonb scalar.
RETURN_NOT_OK(util::Jsonb::CreateScalar(jsonop_result,
element_metadata,
&jsonb_result));
}
result->set_jsonb_value(jsonb_result);
break;
}

case QLExpressionPB::ExprCase::kSubscriptedCol:
if (table_row.IsEmpty()) {
result->SetNull();
Expand Down
17 changes: 17 additions & 0 deletions src/yb/common/ql_protocol.proto
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,22 @@ message QLSubscriptedColPB {
repeated QLExpressionPB subscript_args = 2;
}

// Represents operations applied to a json column.
message QLJsonColumnOperationsPB {
optional int32 column_id = 1;
repeated QLJsonOperationPB json_operations = 2;
}

enum JsonOperatorPB {
JSON_OBJECT = 0; // The -> operator applied to a column.
JSON_TEXT = 1; // The ->> operator applied to a column.
}

message QLJsonOperationPB {
required JsonOperatorPB json_operator = 1;
required QLExpressionPB operand = 2;
}

// An expression in a WHERE condition
message QLExpressionPB {
oneof expr {
Expand All @@ -97,6 +113,7 @@ message QLExpressionPB {
QLBCallPB bfcall = 6; // Regular builtin calls.
QLBCallPB tscall = 7; // Tablet server builtin calls.
QLBCallPB bocall = 8; // Builtin operator calls.
QLJsonColumnOperationsPB json_column = 9; // Json column operators.
}
}

Expand Down
1 change: 1 addition & 0 deletions src/yb/util/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -238,6 +238,7 @@ set(UTIL_LIBS
histogram_proto
pb_util_proto
protobuf
ql_protocol_proto
version_info_proto
zlib)

Expand Down
Loading

0 comments on commit 68296cb

Please sign in to comment.