Skip to content

Commit

Permalink
Support semicolon and escaped strings in lark (#1339)
Browse files Browse the repository at this point in the history
Support semi-colons in string literals for queries of the form:
```
"""SELECT ChatGPT("Here's a; question", "This is the context") FROM TAIPAI;"""
```

Also support string escape to run ChatGPT queries more easily:
```
"""SELECT ChatGPT('Here\\'s a question', 'This is the context') FROM TAIPAI;"""
```
  • Loading branch information
pchunduri6 authored Nov 6, 2023
1 parent c296318 commit 6929103
Show file tree
Hide file tree
Showing 2 changed files with 36 additions and 2 deletions.
4 changes: 2 additions & 2 deletions evadb/parser/evadb.lark
Original file line number Diff line number Diff line change
Expand Up @@ -580,8 +580,8 @@ GLOBAL_ID: "@" "@" (/[A-Z0-9._$]+/ | BQUOTA_STRING)

EXPONENT_NUM_PART: /"E" "-"? DEC_DIGIT+/
ID_LITERAL: /[A-Za-z_$0-9]*?[A-Za-z_$]+?[A-Za-z_$0-9]*/
DQUOTA_STRING: /"[^";]*"/
SQUOTA_STRING: /'[^';]*'/
DQUOTA_STRING: "\"" /(?:[^"\\]|\\.)*"/
SQUOTA_STRING: "'" /(?:[^'\\]|\\.)*'/
BQUOTA_STRING: /`[^'`]*`/
QUERY_STRING: /[^{};]+/
DEC_DIGIT: /[0-9]/
Expand Down
34 changes: 34 additions & 0 deletions test/unit_tests/parser/test_parser.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
import unittest
from pathlib import Path

Expand Down Expand Up @@ -464,6 +465,39 @@ def test_select_with_empty_string_literal(self):
self.assertEqual(len(evadb_statement_list), 1)
self.assertEqual(evadb_statement_list[0].stmt_type, StatementType.SELECT)

def test_string_literal_with_escaped_single_quote(self):
parser = Parser()

select_query = """SELECT ChatGPT('Here\\'s a question', 'This is the context') FROM TAIPAI;"""

evadb_statement_list = parser.parse(select_query)
self.assertIsInstance(evadb_statement_list, list)
self.assertEqual(len(evadb_statement_list), 1)
self.assertEqual(evadb_statement_list[0].stmt_type, StatementType.SELECT)

def test_string_literal_with_semi_colon(self):
parser = Parser()

select_query = """SELECT ChatGPT("Here's a; question", "This is the context") FROM TAIPAI;"""

evadb_statement_list = parser.parse(select_query)
self.assertIsInstance(evadb_statement_list, list)
self.assertEqual(len(evadb_statement_list), 1)
self.assertEqual(evadb_statement_list[0].stmt_type, StatementType.SELECT)

def test_string_literal_with_single_quotes_from_variable(self):
parser = Parser()

question = json.dumps("Here's a question")
answer = json.dumps('This is "the" context')

select_query = f"""SELECT ChatGPT({question}, {answer}) FROM TAIPAI;"""

evadb_statement_list = parser.parse(select_query)
self.assertIsInstance(evadb_statement_list, list)
self.assertEqual(len(evadb_statement_list), 1)
self.assertEqual(evadb_statement_list[0].stmt_type, StatementType.SELECT)

def test_select_union_statement(self):
parser = Parser()
select_union_query = "SELECT CLASS, REDNESS FROM TAIPAI \
Expand Down

0 comments on commit 6929103

Please sign in to comment.