From 4b8f27b70306026f6bfaa15944dd129b9ac479ee Mon Sep 17 00:00:00 2001 From: W Chan Date: Thu, 9 Apr 2020 15:23:04 -0700 Subject: [PATCH 1/9] Fix the yaql/jinja vars extraction to ignore methods of base ctx() dict The ctx() method returns a dict and there are use cases where users call supported methods on the dict such as keys(), values(), and get(). The vars extraction would return false positives on these function calls. The regular expressions to extract variables from expressions is refactored to ignore these dict related function calls directly on ctx(). --- orquesta/expressions/base.py | 1 - orquesta/expressions/jinja.py | 34 +++++++++++++------ orquesta/expressions/yql.py | 34 +++++++++++++------ .../test_facade_jinja_ctx_by_dot_notation.py | 8 +++-- .../test_facade_jinja_ctx_by_function_arg.py | 18 ++++++++-- .../test_facade_yaql_ctx_by_dot_notation.py | 8 +++-- .../test_facade_yaql_ctx_by_function_arg.py | 15 ++++++-- .../test_jinja_ctx_by_dot_notation.py | 9 ++--- .../test_jinja_ctx_by_function_arg.py | 9 ++--- .../test_yaql_ctx_by_dot_notation.py | 9 ++--- .../test_yaql_ctx_by_function_arg.py | 9 ++--- 11 files changed, 105 insertions(+), 49 deletions(-) diff --git a/orquesta/expressions/base.py b/orquesta/expressions/base.py index 1aacfc24..af27263a 100644 --- a/orquesta/expressions/base.py +++ b/orquesta/expressions/base.py @@ -144,7 +144,6 @@ def evaluate(statement, data=None): def extract_vars(statement): - variables = [] if isinstance(statement, dict): diff --git a/orquesta/expressions/jinja.py b/orquesta/expressions/jinja.py index 296f0fb3..e2671d44 100644 --- a/orquesta/expressions/jinja.py +++ b/orquesta/expressions/jinja.py @@ -14,6 +14,7 @@ import functools import inspect +import itertools import logging import re import six @@ -54,15 +55,26 @@ class JinjaEvaluator(expr_base.Evaluator): _regex_parser = re.compile(_regex_pattern) _regex_dot_pattern = '[a-zA-Z0-9_\'"\.\[\]\(\)]*' - _regex_ctx_pattern_1 = 'ctx\(\)\.%s' % _regex_dot_pattern - _regex_ctx_pattern_2 = 'ctx\([\'|"]?{0}[\'|"]?\)[\.{0}]?'.format(_regex_dot_pattern) - _regex_var_pattern = '.*?(%s|%s).*?' % (_regex_ctx_pattern_1, _regex_ctx_pattern_2) + _regex_ctx_patterns = [ + '^ctx\(\)\.%s' % _regex_dot_pattern, # line start ctx().* + '^ctx\([\'|"]?{0}[\'|"]?\)[\.{0}]?'.format(_regex_dot_pattern), # line start ctx(*).* + '[\s]ctx\(\)\.%s' % _regex_dot_pattern, # whitespace ctx().* + '[\s]ctx\([\'|"]?{0}[\'|"]?\)[\.{0}]?'.format(_regex_dot_pattern) # whitespace ctx(*).* + ] + _regex_var_pattern = '.*?(%s).*?' % '|'.join(_regex_ctx_patterns) _regex_var_parser = re.compile(_regex_var_pattern) - _regex_dot_extract = '([a-zA-Z0-9_\-]*)' - _regex_ctx_extract_1 = 'ctx\(\)\.%s' % _regex_dot_extract - _regex_ctx_extract_2 = 'ctx\([\'|"]?%s(%s)' % (_regex_dot_extract, _regex_dot_pattern) - _regex_var_extracts = ['%s\.?' % _regex_ctx_extract_1, '%s\.?' % _regex_ctx_extract_2] + _regex_dot_extract = '[a-zA-Z0-9_\-]+' + _regex_var_extracts = [ + r'(?<=^ctx\(\)\.)(\b%s\b)(?!\()\.?' % _regex_dot_extract, # line start ctx().foobar + r'(?<=^ctx\()(\b%s\b)(?=\))\.?' % _regex_dot_extract, # line start ctx(foobar) + r'(?<=^ctx\(\')(\b%s\b)(?=\'\))\.?' % _regex_dot_extract, # line start ctx('foobar') + r'(?<=^ctx\(")(\b%s\b)(?="\))\.?' % _regex_dot_extract, # line start ctx("foobar") + r'(?<=[\s]ctx\(\)\.)(\b%s\b)(?!\()\.?' % _regex_dot_extract, # whitespace ctx().foobar + r'(?<=[\s]ctx\()(\b%s\b)(?=\))\.?' % _regex_dot_extract, # whitespace ctx(foobar) + r'(?<=[\s]ctx\(\')(\b%s\b)(?=\'\))\.?' % _regex_dot_extract, # whitespace ctx('foobar') + r'(?<=[\s]ctx\(")(\b%s\b)(?="\))\.?' % _regex_dot_extract # whitespace ctx("foobar") + ] _block_delimiter = '{%}' _regex_block_pattern = '{%.*?%}' @@ -234,9 +246,11 @@ def extract_vars(cls, text): if not isinstance(text, six.string_types): raise ValueError('Text to be evaluated is not typeof string.') - variables = [] + results = [ + cls._regex_var_parser.findall(expr.strip(cls._delimiter).strip()) + for expr in cls._regex_parser.findall(text) + ] - for expr in cls._regex_parser.findall(text): - variables.extend(cls._regex_var_parser.findall(expr)) + variables = [v.strip() for v in itertools.chain.from_iterable(results)] return sorted(list(set(variables))) diff --git a/orquesta/expressions/yql.py b/orquesta/expressions/yql.py index 0676e076..919d77ec 100644 --- a/orquesta/expressions/yql.py +++ b/orquesta/expressions/yql.py @@ -13,6 +13,7 @@ # limitations under the License. import inspect +import itertools import logging import re import six @@ -54,15 +55,26 @@ class YAQLEvaluator(expr_base.Evaluator): _regex_parser = re.compile(_regex_pattern) _regex_dot_pattern = '[a-zA-Z0-9_\'"\.\[\]\(\)]*' - _regex_ctx_pattern_1 = 'ctx\(\)\.%s' % _regex_dot_pattern - _regex_ctx_pattern_2 = 'ctx\([\'|"]?{0}[\'|"]?\)[\.{0}]?'.format(_regex_dot_pattern) - _regex_var_pattern = '.*?(%s|%s).*?' % (_regex_ctx_pattern_1, _regex_ctx_pattern_2) + _regex_ctx_patterns = [ + '^ctx\(\)\.%s' % _regex_dot_pattern, # line start ctx().* + '^ctx\([\'|"]?{0}[\'|"]?\)[\.{0}]?'.format(_regex_dot_pattern), # line start ctx(*).* + '[\s]ctx\(\)\.%s' % _regex_dot_pattern, # whitespace ctx().* + '[\s]ctx\([\'|"]?{0}[\'|"]?\)[\.{0}]?'.format(_regex_dot_pattern) # whitespace ctx(*).* + ] + _regex_var_pattern = '.*?(%s).*?' % '|'.join(_regex_ctx_patterns) _regex_var_parser = re.compile(_regex_var_pattern) - _regex_dot_extract = '([a-zA-Z0-9_\-]*)' - _regex_ctx_extract_1 = 'ctx\(\)\.%s' % _regex_dot_extract - _regex_ctx_extract_2 = 'ctx\([\'|"]?%s(%s)' % (_regex_dot_extract, _regex_dot_pattern) - _regex_var_extracts = ['%s\.?' % _regex_ctx_extract_1, '%s\.?' % _regex_ctx_extract_2] + _regex_dot_extract = '[a-zA-Z0-9_\-]+' + _regex_var_extracts = [ + r'(?<=^ctx\(\)\.)(\b%s\b)(?!\()\.?' % _regex_dot_extract, # line start ctx().foobar + r'(?<=^ctx\()(\b%s\b)(?=\))\.?' % _regex_dot_extract, # line start ctx(foobar) + r'(?<=^ctx\(\')(\b%s\b)(?=\'\))\.?' % _regex_dot_extract, # line start ctx('foobar') + r'(?<=^ctx\(")(\b%s\b)(?="\))\.?' % _regex_dot_extract, # line start ctx("foobar") + r'(?<=[\s]ctx\(\)\.)(\b%s\b)(?!\()\.?' % _regex_dot_extract, # whitespace ctx().foobar + r'(?<=[\s]ctx\()(\b%s\b)(?=\))\.?' % _regex_dot_extract, # whitespace ctx(foobar) + r'(?<=[\s]ctx\(\')(\b%s\b)(?=\'\))\.?' % _regex_dot_extract, # whitespace ctx('foobar') + r'(?<=[\s]ctx\(")(\b%s\b)(?="\))\.?' % _regex_dot_extract # whitespace ctx("foobar") + ] _engine = yaql.language.factory.YaqlFactory().create() _root_ctx = yaql.create_context() @@ -153,9 +165,11 @@ def extract_vars(cls, text): if not isinstance(text, six.string_types): raise ValueError('Text to be evaluated is not typeof string.') - variables = [] + results = [ + cls._regex_var_parser.findall(expr.strip(cls._delimiter).strip()) + for expr in cls._regex_parser.findall(text) + ] - for expr in cls._regex_parser.findall(text): - variables.extend(cls._regex_var_parser.findall(expr)) + variables = [v.strip() for v in itertools.chain.from_iterable(results)] return sorted(list(set(variables))) diff --git a/orquesta/tests/unit/expressions/test_facade_jinja_ctx_by_dot_notation.py b/orquesta/tests/unit/expressions/test_facade_jinja_ctx_by_dot_notation.py index 374fe7f9..2391d0f0 100644 --- a/orquesta/tests/unit/expressions/test_facade_jinja_ctx_by_dot_notation.py +++ b/orquesta/tests/unit/expressions/test_facade_jinja_ctx_by_dot_notation.py @@ -19,7 +19,7 @@ class JinjaFacadeVariableExtractionTest(test_base.ExpressionFacadeEvaluatorTest): def test_empty_extraction(self): - expr = '{{ just_text and _not_a_var }}' + expr = '{{ just_text and _not_a_var and fooctx(foo) and fooctx("bar") and fooctx(\'fu\') }}' self.assertListEqual([], expr_base.extract_vars(expr)) @@ -52,12 +52,14 @@ def test_single_functional_var_extraction(self): self.assertListEqual(expected_vars, expr_base.extract_vars(expr)) def test_multiple_vars_extraction(self): - expr = '{{ ctx().foobar ctx().foo.get(bar) ctx().fu.bar ctx().fu.bar[0] }}' + expr = '{{ctx().fubar ctx().foobar ctx().foo.get(bar) ctx().fu.bar ctx().foobaz.bar[0] }}' expected_vars = [ ('jinja', expr, 'foo'), ('jinja', expr, 'foobar'), - ('jinja', expr, 'fu') + ('jinja', expr, 'foobaz'), + ('jinja', expr, 'fu'), + ('jinja', expr, 'fubar') ] self.assertListEqual(expected_vars, expr_base.extract_vars(expr)) diff --git a/orquesta/tests/unit/expressions/test_facade_jinja_ctx_by_function_arg.py b/orquesta/tests/unit/expressions/test_facade_jinja_ctx_by_function_arg.py index 69d429ab..dc998eda 100644 --- a/orquesta/tests/unit/expressions/test_facade_jinja_ctx_by_function_arg.py +++ b/orquesta/tests/unit/expressions/test_facade_jinja_ctx_by_function_arg.py @@ -19,7 +19,7 @@ class JinjaFacadeVariableExtractionTest(test_base.ExpressionFacadeEvaluatorTest): def test_empty_extraction(self): - expr = '{{ just_text and _not_a_var }}' + expr = '{{ just_text and _not_a_var and fooctx(foo) and fooctx("bar") and fooctx(\'fu\') }}' self.assertListEqual([], expr_base.extract_vars(expr)) @@ -52,12 +52,17 @@ def test_single_functional_var_extraction(self): self.assertListEqual(expected_vars, expr_base.extract_vars(expr)) def test_multiple_vars_extraction(self): - expr = '{{ ctx("foobar") ctx("foo").get(bar) ctx("fu").bar ctx("fu").bar[0] }}' + expr = ( + '{{ctx("fubar") ctx("foobar") ctx("foo").get(bar) ' + 'ctx("fu").bar ctx("foobaz").bar[0] }}' + ) expected_vars = [ ('jinja', expr, 'foo'), ('jinja', expr, 'foobar'), - ('jinja', expr, 'fu') + ('jinja', expr, 'foobaz'), + ('jinja', expr, 'fu'), + ('jinja', expr, 'fubar') ] self.assertListEqual(expected_vars, expr_base.extract_vars(expr)) @@ -108,3 +113,10 @@ def test_vars_extraction_from_dict(self): ] self.assertListEqual(expected_vars, expr_base.extract_vars(expr)) + + def test_ignore_ctx_dict_funcs(self): + expr = '{{ctx().keys() and ctx().values() and ctx().set("b", 3) }}' + + expected_vars = [] + + self.assertListEqual(expected_vars, expr_base.extract_vars(expr)) diff --git a/orquesta/tests/unit/expressions/test_facade_yaql_ctx_by_dot_notation.py b/orquesta/tests/unit/expressions/test_facade_yaql_ctx_by_dot_notation.py index 1d51a3be..6dd9dc9a 100644 --- a/orquesta/tests/unit/expressions/test_facade_yaql_ctx_by_dot_notation.py +++ b/orquesta/tests/unit/expressions/test_facade_yaql_ctx_by_dot_notation.py @@ -19,7 +19,7 @@ class YAQLFacadeVariableExtractionTest(test_base.ExpressionFacadeEvaluatorTest): def test_empty_extraction(self): - expr = '<% just_text and $not_a_var %>' + expr = '<% just_text and $not_a_var and fooctx(foo) and fooctx("bar") and fooctx(\'fu\') %>' self.assertListEqual([], expr_base.extract_vars(expr)) @@ -52,12 +52,14 @@ def test_single_functional_var_extraction(self): self.assertListEqual(expected_vars, expr_base.extract_vars(expr)) def test_multiple_vars_extraction(self): - expr = '<% ctx().foobar ctx().foo.get(bar) ctx().fu.bar ctx().fu.bar[0] %>' + expr = '<%ctx().fubar ctx().foobar ctx().foo.get(bar) ctx().fu.bar ctx().foobaz.bar[0] %>' expected_vars = [ ('yaql', expr, 'foo'), ('yaql', expr, 'foobar'), - ('yaql', expr, 'fu') + ('yaql', expr, 'foobaz'), + ('yaql', expr, 'fu'), + ('yaql', expr, 'fubar') ] self.assertListEqual(expected_vars, expr_base.extract_vars(expr)) diff --git a/orquesta/tests/unit/expressions/test_facade_yaql_ctx_by_function_arg.py b/orquesta/tests/unit/expressions/test_facade_yaql_ctx_by_function_arg.py index 30af3f0a..c4c29648 100644 --- a/orquesta/tests/unit/expressions/test_facade_yaql_ctx_by_function_arg.py +++ b/orquesta/tests/unit/expressions/test_facade_yaql_ctx_by_function_arg.py @@ -19,7 +19,7 @@ class YAQLFacadeVariableExtractionTest(test_base.ExpressionFacadeEvaluatorTest): def test_empty_extraction(self): - expr = '<% just_text and $not_a_var %>' + expr = '<% just_text and $not_a_var and fooctx(foo) and fooctx("bar") and fooctx(\'fu\') %>' self.assertListEqual([], expr_base.extract_vars(expr)) @@ -52,12 +52,14 @@ def test_single_functional_var_extraction(self): self.assertListEqual(expected_vars, expr_base.extract_vars(expr)) def test_multiple_vars_extraction(self): - expr = '<% ctx(foobar) ctx(foo).get(bar) ctx(fu).bar ctx(fu).bar[0] %>' + expr = '<%ctx(fubar) ctx(foobar) ctx(foo).get(bar) ctx(fu).bar ctx(foobaz).bar[0] %>' expected_vars = [ ('yaql', expr, 'foo'), ('yaql', expr, 'foobar'), - ('yaql', expr, 'fu') + ('yaql', expr, 'foobaz'), + ('yaql', expr, 'fu'), + ('yaql', expr, 'fubar') ] self.assertListEqual(expected_vars, expr_base.extract_vars(expr)) @@ -108,3 +110,10 @@ def test_vars_extraction_from_dict(self): ] self.assertListEqual(expected_vars, expr_base.extract_vars(expr)) + + def test_ignore_ctx_dict_funcs(self): + expr = '<%ctx().keys() and ctx().values() and ctx().set("b", 3) %>' + + expected_vars = [] + + self.assertListEqual(expected_vars, expr_base.extract_vars(expr)) diff --git a/orquesta/tests/unit/expressions/test_jinja_ctx_by_dot_notation.py b/orquesta/tests/unit/expressions/test_jinja_ctx_by_dot_notation.py index 1fc86e1b..4712f24a 100644 --- a/orquesta/tests/unit/expressions/test_jinja_ctx_by_dot_notation.py +++ b/orquesta/tests/unit/expressions/test_jinja_ctx_by_dot_notation.py @@ -23,7 +23,7 @@ def setUpClass(cls): super(JinjaVariableExtractionTest, cls).setUpClass() def test_empty_extraction(self): - expr = '{{ just_text and _not_a_var }}' + expr = '{{ just_text and _not_a_var and fooctx().bar }}' self.assertListEqual([], self.evaluator.extract_vars(expr)) @@ -64,13 +64,14 @@ def test_single_functional_var_extraction(self): self.assertListEqual(expected_vars, self.evaluator.extract_vars(expr)) def test_multiple_vars_extraction(self): - expr = '{{ ctx().foobar ctx().foo.get(bar) ctx().fu.bar ctx().fu.bar[0] }}' + expr = '{{ctx().fubar ctx().foobar ctx().foo.get(bar) ctx().fu.bar ctx().foobaz.bar[0] }}' expected_vars = [ 'ctx().foobar', + 'ctx().foobaz.bar[0]', 'ctx().foo.get(bar)', - 'ctx().fu.bar', - 'ctx().fu.bar[0]' + 'ctx().fubar', + 'ctx().fu.bar' ] self.assertListEqual( diff --git a/orquesta/tests/unit/expressions/test_jinja_ctx_by_function_arg.py b/orquesta/tests/unit/expressions/test_jinja_ctx_by_function_arg.py index 47ccedc4..ab374682 100644 --- a/orquesta/tests/unit/expressions/test_jinja_ctx_by_function_arg.py +++ b/orquesta/tests/unit/expressions/test_jinja_ctx_by_function_arg.py @@ -23,7 +23,7 @@ def setUpClass(cls): super(JinjaVariableExtractionTest, cls).setUpClass() def test_empty_extraction(self): - expr = '{{ just_text and _not_a_var }}' + expr = '{{ just_text and _not_a_var and fooctx(foo) and fooctx("bar") and fooctx(\'fu\') }}' self.assertListEqual([], self.evaluator.extract_vars(expr)) @@ -82,13 +82,14 @@ def test_single_functional_var_extraction(self): self.assertListEqual(expected_vars, self.evaluator.extract_vars(expr)) def test_multiple_vars_extraction(self): - expr = '{{ ctx(foobar) ctx(foo).get(bar) ctx(fu).bar ctx(fu).bar[0] }}' + expr = '{{ctx(fubar) ctx(foobar) ctx(foo).get(bar) ctx(fu).bar ctx(foobaz).bar[0] }}' expected_vars = [ 'ctx(foobar)', + 'ctx(foobaz).bar[0]', 'ctx(foo).get(bar)', - 'ctx(fu).bar', - 'ctx(fu).bar[0]' + 'ctx(fubar)', + 'ctx(fu).bar' ] self.assertListEqual( diff --git a/orquesta/tests/unit/expressions/test_yaql_ctx_by_dot_notation.py b/orquesta/tests/unit/expressions/test_yaql_ctx_by_dot_notation.py index 7fac7a1c..10deecfe 100644 --- a/orquesta/tests/unit/expressions/test_yaql_ctx_by_dot_notation.py +++ b/orquesta/tests/unit/expressions/test_yaql_ctx_by_dot_notation.py @@ -23,7 +23,7 @@ def setUpClass(cls): super(YAQLVariableExtractionTest, cls).setUpClass() def test_empty_extraction(self): - expr = '<% just_text and $not_a_var %>' + expr = '<% just_text and $not_a_var and fooctx().bar %>' self.assertListEqual([], self.evaluator.extract_vars(expr)) @@ -64,13 +64,14 @@ def test_single_functional_var_extraction(self): self.assertListEqual(expected_vars, self.evaluator.extract_vars(expr)) def test_multiple_vars_extraction(self): - expr = '<% ctx().foobar ctx().foo.get(bar) ctx().fu.bar ctx().fu.bar[0] %>' + expr = '<%ctx().fubar ctx().foobar ctx().foo.get(bar) ctx().fu.bar ctx().foobaz.bar[0] %>' expected_vars = [ 'ctx().foobar', + 'ctx().foobaz.bar[0]', 'ctx().foo.get(bar)', - 'ctx().fu.bar', - 'ctx().fu.bar[0]' + 'ctx().fubar', + 'ctx().fu.bar' ] self.assertListEqual( diff --git a/orquesta/tests/unit/expressions/test_yaql_ctx_by_function_arg.py b/orquesta/tests/unit/expressions/test_yaql_ctx_by_function_arg.py index 9a675ad2..111db34a 100644 --- a/orquesta/tests/unit/expressions/test_yaql_ctx_by_function_arg.py +++ b/orquesta/tests/unit/expressions/test_yaql_ctx_by_function_arg.py @@ -23,7 +23,7 @@ def setUpClass(cls): super(YAQLVariableExtractionTest, cls).setUpClass() def test_empty_extraction(self): - expr = '<% just_text and $not_a_var %>' + expr = '<% just_text and $not_a_var and fooctx(foo) and fooctx("bar") and fooctx(\'fu\') %>' self.assertListEqual([], self.evaluator.extract_vars(expr)) @@ -82,13 +82,14 @@ def test_single_functional_var_extraction(self): self.assertListEqual(expected_vars, self.evaluator.extract_vars(expr)) def test_multiple_vars_extraction(self): - expr = '<% ctx(foobar) ctx(foo).get(bar) ctx(fu).bar ctx(fu).bar[0] %>' + expr = '<%ctx(fubar) ctx(foobar) ctx(foo).get(bar) ctx(fu).bar ctx(foobaz).bar[0] %>' expected_vars = [ 'ctx(foobar)', + 'ctx(foobaz).bar[0]', 'ctx(foo).get(bar)', - 'ctx(fu).bar', - 'ctx(fu).bar[0]' + 'ctx(fubar)', + 'ctx(fu).bar' ] self.assertListEqual( From b7e44daa2fb270fd2835a5ed5af3384f7f552267 Mon Sep 17 00:00:00 2001 From: W Chan Date: Thu, 9 Apr 2020 16:30:54 -0700 Subject: [PATCH 2/9] Add more unit tests to cover various ctx().get use cases --- orquesta/expressions/jinja.py | 34 +++++++++---------- orquesta/expressions/yql.py | 34 +++++++++---------- .../test_facade_jinja_ctx_by_function_arg.py | 12 ++++++- .../test_facade_yaql_ctx_by_function_arg.py | 12 ++++++- 4 files changed, 56 insertions(+), 36 deletions(-) diff --git a/orquesta/expressions/jinja.py b/orquesta/expressions/jinja.py index e2671d44..40e4299e 100644 --- a/orquesta/expressions/jinja.py +++ b/orquesta/expressions/jinja.py @@ -54,26 +54,26 @@ class JinjaEvaluator(expr_base.Evaluator): _regex_pattern = '{{.*?}}' _regex_parser = re.compile(_regex_pattern) - _regex_dot_pattern = '[a-zA-Z0-9_\'"\.\[\]\(\)]*' + _regex_ctx_pattern = '[a-zA-Z0-9_\'"\.\[\]\(\)]*' _regex_ctx_patterns = [ - '^ctx\(\)\.%s' % _regex_dot_pattern, # line start ctx().* - '^ctx\([\'|"]?{0}[\'|"]?\)[\.{0}]?'.format(_regex_dot_pattern), # line start ctx(*).* - '[\s]ctx\(\)\.%s' % _regex_dot_pattern, # whitespace ctx().* - '[\s]ctx\([\'|"]?{0}[\'|"]?\)[\.{0}]?'.format(_regex_dot_pattern) # whitespace ctx(*).* + '^ctx\(\)\.%s' % _regex_ctx_pattern, # line start ctx().* + '^ctx\([\'|"]?{0}[\'|"]?\)[\.{0}]?'.format(_regex_ctx_pattern), # line start ctx(*).* + '[\s]ctx\(\)\.%s' % _regex_ctx_pattern, # whitespace ctx().* + '[\s]ctx\([\'|"]?{0}[\'|"]?\)[\.{0}]?'.format(_regex_ctx_pattern) # whitespace ctx(*).* ] - _regex_var_pattern = '.*?(%s).*?' % '|'.join(_regex_ctx_patterns) - _regex_var_parser = re.compile(_regex_var_pattern) + _regex_ctx_var = '.*?(%s).*?' % '|'.join(_regex_ctx_patterns) + _regex_ctx_var_parser = re.compile(_regex_ctx_var) - _regex_dot_extract = '[a-zA-Z0-9_\-]+' + _regex_var = '[a-zA-Z0-9_\-]+' _regex_var_extracts = [ - r'(?<=^ctx\(\)\.)(\b%s\b)(?!\()\.?' % _regex_dot_extract, # line start ctx().foobar - r'(?<=^ctx\()(\b%s\b)(?=\))\.?' % _regex_dot_extract, # line start ctx(foobar) - r'(?<=^ctx\(\')(\b%s\b)(?=\'\))\.?' % _regex_dot_extract, # line start ctx('foobar') - r'(?<=^ctx\(")(\b%s\b)(?="\))\.?' % _regex_dot_extract, # line start ctx("foobar") - r'(?<=[\s]ctx\(\)\.)(\b%s\b)(?!\()\.?' % _regex_dot_extract, # whitespace ctx().foobar - r'(?<=[\s]ctx\()(\b%s\b)(?=\))\.?' % _regex_dot_extract, # whitespace ctx(foobar) - r'(?<=[\s]ctx\(\')(\b%s\b)(?=\'\))\.?' % _regex_dot_extract, # whitespace ctx('foobar') - r'(?<=[\s]ctx\(")(\b%s\b)(?="\))\.?' % _regex_dot_extract # whitespace ctx("foobar") + r'(?<=^ctx\(\)\.)(\b%s\b)(?!\()\.?' % _regex_var, # line start ctx().x + r'(?<=^ctx\()(\b%s\b)(?=\))\.?' % _regex_var, # line start ctx(x) + r'(?<=^ctx\(\')(\b%s\b)(?=\'\))\.?' % _regex_var, # line start ctx('x') + r'(?<=^ctx\(")(\b%s\b)(?="\))\.?' % _regex_var, # line start ctx("x") + r'(?<=[\s]ctx\(\)\.)(\b%s\b)(?!\()\.?' % _regex_var, # whitespace ctx().x + r'(?<=[\s]ctx\()(\b%s\b)(?=\))\.?' % _regex_var, # whitespace ctx(x) + r'(?<=[\s]ctx\(\')(\b%s\b)(?=\'\))\.?' % _regex_var, # whitespace ctx('x') + r'(?<=[\s]ctx\(")(\b%s\b)(?="\))\.?' % _regex_var # whitespace ctx("x") ] _block_delimiter = '{%}' @@ -247,7 +247,7 @@ def extract_vars(cls, text): raise ValueError('Text to be evaluated is not typeof string.') results = [ - cls._regex_var_parser.findall(expr.strip(cls._delimiter).strip()) + cls._regex_ctx_var_parser.findall(expr.strip(cls._delimiter).strip()) for expr in cls._regex_parser.findall(text) ] diff --git a/orquesta/expressions/yql.py b/orquesta/expressions/yql.py index 919d77ec..e59c8305 100644 --- a/orquesta/expressions/yql.py +++ b/orquesta/expressions/yql.py @@ -54,26 +54,26 @@ class YAQLEvaluator(expr_base.Evaluator): _regex_pattern = '<%.*?%>' _regex_parser = re.compile(_regex_pattern) - _regex_dot_pattern = '[a-zA-Z0-9_\'"\.\[\]\(\)]*' + _regex_ctx_pattern = '[a-zA-Z0-9_\'"\.\[\]\(\)]*' _regex_ctx_patterns = [ - '^ctx\(\)\.%s' % _regex_dot_pattern, # line start ctx().* - '^ctx\([\'|"]?{0}[\'|"]?\)[\.{0}]?'.format(_regex_dot_pattern), # line start ctx(*).* - '[\s]ctx\(\)\.%s' % _regex_dot_pattern, # whitespace ctx().* - '[\s]ctx\([\'|"]?{0}[\'|"]?\)[\.{0}]?'.format(_regex_dot_pattern) # whitespace ctx(*).* + '^ctx\(\)\.%s' % _regex_ctx_pattern, # line start ctx().* + '^ctx\([\'|"]?{0}[\'|"]?\)[\.{0}]?'.format(_regex_ctx_pattern), # line start ctx(*).* + '[\s]ctx\(\)\.%s' % _regex_ctx_pattern, # whitespace ctx().* + '[\s]ctx\([\'|"]?{0}[\'|"]?\)[\.{0}]?'.format(_regex_ctx_pattern) # whitespace ctx(*).* ] - _regex_var_pattern = '.*?(%s).*?' % '|'.join(_regex_ctx_patterns) - _regex_var_parser = re.compile(_regex_var_pattern) + _regex_ctx_var = '.*?(%s).*?' % '|'.join(_regex_ctx_patterns) + _regex_ctx_var_parser = re.compile(_regex_ctx_var) - _regex_dot_extract = '[a-zA-Z0-9_\-]+' + _regex_var = '[a-zA-Z0-9_\-]+' _regex_var_extracts = [ - r'(?<=^ctx\(\)\.)(\b%s\b)(?!\()\.?' % _regex_dot_extract, # line start ctx().foobar - r'(?<=^ctx\()(\b%s\b)(?=\))\.?' % _regex_dot_extract, # line start ctx(foobar) - r'(?<=^ctx\(\')(\b%s\b)(?=\'\))\.?' % _regex_dot_extract, # line start ctx('foobar') - r'(?<=^ctx\(")(\b%s\b)(?="\))\.?' % _regex_dot_extract, # line start ctx("foobar") - r'(?<=[\s]ctx\(\)\.)(\b%s\b)(?!\()\.?' % _regex_dot_extract, # whitespace ctx().foobar - r'(?<=[\s]ctx\()(\b%s\b)(?=\))\.?' % _regex_dot_extract, # whitespace ctx(foobar) - r'(?<=[\s]ctx\(\')(\b%s\b)(?=\'\))\.?' % _regex_dot_extract, # whitespace ctx('foobar') - r'(?<=[\s]ctx\(")(\b%s\b)(?="\))\.?' % _regex_dot_extract # whitespace ctx("foobar") + r'(?<=^ctx\(\)\.)(\b%s\b)(?!\()\.?' % _regex_var, # line start ctx().x + r'(?<=^ctx\()(\b%s\b)(?=\))\.?' % _regex_var, # line start ctx(x) + r'(?<=^ctx\(\')(\b%s\b)(?=\'\))\.?' % _regex_var, # line start ctx('x') + r'(?<=^ctx\(")(\b%s\b)(?="\))\.?' % _regex_var, # line start ctx("x") + r'(?<=[\s]ctx\(\)\.)(\b%s\b)(?!\()\.?' % _regex_var, # whitespace ctx().x + r'(?<=[\s]ctx\()(\b%s\b)(?=\))\.?' % _regex_var, # whitespace ctx(x) + r'(?<=[\s]ctx\(\')(\b%s\b)(?=\'\))\.?' % _regex_var, # whitespace ctx('x') + r'(?<=[\s]ctx\(")(\b%s\b)(?="\))\.?' % _regex_var # whitespace ctx("x") ] _engine = yaql.language.factory.YaqlFactory().create() @@ -166,7 +166,7 @@ def extract_vars(cls, text): raise ValueError('Text to be evaluated is not typeof string.') results = [ - cls._regex_var_parser.findall(expr.strip(cls._delimiter).strip()) + cls._regex_ctx_var_parser.findall(expr.strip(cls._delimiter).strip()) for expr in cls._regex_parser.findall(text) ] diff --git a/orquesta/tests/unit/expressions/test_facade_jinja_ctx_by_function_arg.py b/orquesta/tests/unit/expressions/test_facade_jinja_ctx_by_function_arg.py index dc998eda..dce7c7b7 100644 --- a/orquesta/tests/unit/expressions/test_facade_jinja_ctx_by_function_arg.py +++ b/orquesta/tests/unit/expressions/test_facade_jinja_ctx_by_function_arg.py @@ -19,7 +19,7 @@ class JinjaFacadeVariableExtractionTest(test_base.ExpressionFacadeEvaluatorTest): def test_empty_extraction(self): - expr = '{{ just_text and _not_a_var and fooctx(foo) and fooctx("bar") and fooctx(\'fu\') }}' + expr = '{{ just_text and $not_a_var and notctx(foo) and notctx("bar") and notctx(\'fu\') }}' self.assertListEqual([], expr_base.extract_vars(expr)) @@ -120,3 +120,13 @@ def test_ignore_ctx_dict_funcs(self): expected_vars = [] self.assertListEqual(expected_vars, expr_base.extract_vars(expr)) + + def test_ignore_ctx_get_func_calls(self): + expr = ( + '{{ctx().get(foo) and ctx().get(bar) and ctx().get("fu") and ctx().get(\'baz\') and ' + 'ctx().get(foo, "bar") and ctx().get("fu", "bar") and ctx().get(\'bar\', \'foo\') }}' + ) + + expected_vars = [] + + self.assertListEqual(expected_vars, expr_base.extract_vars(expr)) diff --git a/orquesta/tests/unit/expressions/test_facade_yaql_ctx_by_function_arg.py b/orquesta/tests/unit/expressions/test_facade_yaql_ctx_by_function_arg.py index c4c29648..daad8a52 100644 --- a/orquesta/tests/unit/expressions/test_facade_yaql_ctx_by_function_arg.py +++ b/orquesta/tests/unit/expressions/test_facade_yaql_ctx_by_function_arg.py @@ -19,7 +19,7 @@ class YAQLFacadeVariableExtractionTest(test_base.ExpressionFacadeEvaluatorTest): def test_empty_extraction(self): - expr = '<% just_text and $not_a_var and fooctx(foo) and fooctx("bar") and fooctx(\'fu\') %>' + expr = '<% just_text and $not_a_var and notctx(foo) and notctx("bar") and notctx(\'fu\') %>' self.assertListEqual([], expr_base.extract_vars(expr)) @@ -117,3 +117,13 @@ def test_ignore_ctx_dict_funcs(self): expected_vars = [] self.assertListEqual(expected_vars, expr_base.extract_vars(expr)) + + def test_ignore_ctx_get_func_calls(self): + expr = ( + '<%ctx().get(foo) and ctx().get(bar) and ctx().get("fu") and ctx().get(\'baz\') and ' + 'ctx().get(foo, "bar") and ctx().get("fu", "bar") and ctx().get(\'bar\', \'foo\') %>' + ) + + expected_vars = [] + + self.assertListEqual(expected_vars, expr_base.extract_vars(expr)) From d19e73f0874d4ea636b100eb4b72bff784f62ee9 Mon Sep 17 00:00:00 2001 From: W Chan Date: Thu, 9 Apr 2020 17:03:07 -0700 Subject: [PATCH 3/9] Minor cleanup to the list of regex patterns for var extraction --- orquesta/expressions/jinja.py | 12 ++++-------- orquesta/expressions/yql.py | 12 ++++-------- 2 files changed, 8 insertions(+), 16 deletions(-) diff --git a/orquesta/expressions/jinja.py b/orquesta/expressions/jinja.py index 40e4299e..c0fb7314 100644 --- a/orquesta/expressions/jinja.py +++ b/orquesta/expressions/jinja.py @@ -66,14 +66,10 @@ class JinjaEvaluator(expr_base.Evaluator): _regex_var = '[a-zA-Z0-9_\-]+' _regex_var_extracts = [ - r'(?<=^ctx\(\)\.)(\b%s\b)(?!\()\.?' % _regex_var, # line start ctx().x - r'(?<=^ctx\()(\b%s\b)(?=\))\.?' % _regex_var, # line start ctx(x) - r'(?<=^ctx\(\')(\b%s\b)(?=\'\))\.?' % _regex_var, # line start ctx('x') - r'(?<=^ctx\(")(\b%s\b)(?="\))\.?' % _regex_var, # line start ctx("x") - r'(?<=[\s]ctx\(\)\.)(\b%s\b)(?!\()\.?' % _regex_var, # whitespace ctx().x - r'(?<=[\s]ctx\()(\b%s\b)(?=\))\.?' % _regex_var, # whitespace ctx(x) - r'(?<=[\s]ctx\(\')(\b%s\b)(?=\'\))\.?' % _regex_var, # whitespace ctx('x') - r'(?<=[\s]ctx\(")(\b%s\b)(?="\))\.?' % _regex_var # whitespace ctx("x") + r'(?<=^ctx\(\)\.)(\b%s\b)(?!\()\.?' % _regex_var, # extract x in ctx().x + r'(?<=^ctx\()(\b%s\b)(?=\))\.?' % _regex_var, # extract x in ctx(x) + r'(?<=^ctx\(\')(\b%s\b)(?=\'\))\.?' % _regex_var, # extract x in ctx('x') + r'(?<=^ctx\(")(\b%s\b)(?="\))\.?' % _regex_var # extract x in ctx("x") ] _block_delimiter = '{%}' diff --git a/orquesta/expressions/yql.py b/orquesta/expressions/yql.py index e59c8305..12ab71ff 100644 --- a/orquesta/expressions/yql.py +++ b/orquesta/expressions/yql.py @@ -66,14 +66,10 @@ class YAQLEvaluator(expr_base.Evaluator): _regex_var = '[a-zA-Z0-9_\-]+' _regex_var_extracts = [ - r'(?<=^ctx\(\)\.)(\b%s\b)(?!\()\.?' % _regex_var, # line start ctx().x - r'(?<=^ctx\()(\b%s\b)(?=\))\.?' % _regex_var, # line start ctx(x) - r'(?<=^ctx\(\')(\b%s\b)(?=\'\))\.?' % _regex_var, # line start ctx('x') - r'(?<=^ctx\(")(\b%s\b)(?="\))\.?' % _regex_var, # line start ctx("x") - r'(?<=[\s]ctx\(\)\.)(\b%s\b)(?!\()\.?' % _regex_var, # whitespace ctx().x - r'(?<=[\s]ctx\()(\b%s\b)(?=\))\.?' % _regex_var, # whitespace ctx(x) - r'(?<=[\s]ctx\(\')(\b%s\b)(?=\'\))\.?' % _regex_var, # whitespace ctx('x') - r'(?<=[\s]ctx\(")(\b%s\b)(?="\))\.?' % _regex_var # whitespace ctx("x") + r'(?<=^ctx\(\)\.)(\b%s\b)(?!\()\.?' % _regex_var, # extract x in ctx().x + r'(?<=^ctx\()(\b%s\b)(?=\))\.?' % _regex_var, # extract x in ctx(x) + r'(?<=^ctx\(\')(\b%s\b)(?=\'\))\.?' % _regex_var, # extract x in ctx('x') + r'(?<=^ctx\(")(\b%s\b)(?="\))\.?' % _regex_var # extract x in ctx("x") ] _engine = yaql.language.factory.YaqlFactory().create() From 518ba0484529a5bb8fdcc7326f2adb992fef33c0 Mon Sep 17 00:00:00 2001 From: W Chan Date: Thu, 9 Apr 2020 17:07:37 -0700 Subject: [PATCH 4/9] Add changelog entry for the ctx().get() variable inspection fix --- CHANGELOG.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 58824bd2..d2214004 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -37,6 +37,8 @@ Fixed * When inspecting custom YAQL/Jinja function to see if there is a context arg, use getargspec for py2 and getfullargspec for py3. (bug fix) * Check syntax on with items task to ensure action is indented correctly. Fixes #184 (bug fix) +* Fix variable inspection where ctx().get() method calls are identified as errors. + Fixes StackStorm/st2#4866 (bug fix) 1.0.0 ----- From 6990e5d94e3c5f2ccff49afd1157b66b9d5a5418 Mon Sep 17 00:00:00 2001 From: W Chan Date: Thu, 9 Apr 2020 18:19:33 -0700 Subject: [PATCH 5/9] Designate regex pattern with backslash as rawstring --- orquesta/expressions/jinja.py | 14 +++++++------- orquesta/expressions/yql.py | 14 +++++++------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/orquesta/expressions/jinja.py b/orquesta/expressions/jinja.py index c0fb7314..544d80e1 100644 --- a/orquesta/expressions/jinja.py +++ b/orquesta/expressions/jinja.py @@ -54,17 +54,17 @@ class JinjaEvaluator(expr_base.Evaluator): _regex_pattern = '{{.*?}}' _regex_parser = re.compile(_regex_pattern) - _regex_ctx_pattern = '[a-zA-Z0-9_\'"\.\[\]\(\)]*' + _regex_ctx_pattern = r'[a-zA-Z0-9_\'"\.\[\]\(\)]*' _regex_ctx_patterns = [ - '^ctx\(\)\.%s' % _regex_ctx_pattern, # line start ctx().* - '^ctx\([\'|"]?{0}[\'|"]?\)[\.{0}]?'.format(_regex_ctx_pattern), # line start ctx(*).* - '[\s]ctx\(\)\.%s' % _regex_ctx_pattern, # whitespace ctx().* - '[\s]ctx\([\'|"]?{0}[\'|"]?\)[\.{0}]?'.format(_regex_ctx_pattern) # whitespace ctx(*).* + r'^ctx\(\)\.%s' % _regex_ctx_pattern, # line start ctx().* + r'^ctx\([\'|"]?{0}[\'|"]?\)[\.{0}]?'.format(_regex_ctx_pattern), # line start ctx(*).* + r'[\s]ctx\(\)\.%s' % _regex_ctx_pattern, # whitespace ctx().* + r'[\s]ctx\([\'|"]?{0}[\'|"]?\)[\.{0}]?'.format(_regex_ctx_pattern) # whitespace ctx(*).* ] - _regex_ctx_var = '.*?(%s).*?' % '|'.join(_regex_ctx_patterns) + _regex_ctx_var = r'.*?(%s).*?' % '|'.join(_regex_ctx_patterns) _regex_ctx_var_parser = re.compile(_regex_ctx_var) - _regex_var = '[a-zA-Z0-9_\-]+' + _regex_var = r'[a-zA-Z0-9_\-]+' _regex_var_extracts = [ r'(?<=^ctx\(\)\.)(\b%s\b)(?!\()\.?' % _regex_var, # extract x in ctx().x r'(?<=^ctx\()(\b%s\b)(?=\))\.?' % _regex_var, # extract x in ctx(x) diff --git a/orquesta/expressions/yql.py b/orquesta/expressions/yql.py index 12ab71ff..68a13113 100644 --- a/orquesta/expressions/yql.py +++ b/orquesta/expressions/yql.py @@ -54,17 +54,17 @@ class YAQLEvaluator(expr_base.Evaluator): _regex_pattern = '<%.*?%>' _regex_parser = re.compile(_regex_pattern) - _regex_ctx_pattern = '[a-zA-Z0-9_\'"\.\[\]\(\)]*' + _regex_ctx_pattern = r'[a-zA-Z0-9_\'"\.\[\]\(\)]*' _regex_ctx_patterns = [ - '^ctx\(\)\.%s' % _regex_ctx_pattern, # line start ctx().* - '^ctx\([\'|"]?{0}[\'|"]?\)[\.{0}]?'.format(_regex_ctx_pattern), # line start ctx(*).* - '[\s]ctx\(\)\.%s' % _regex_ctx_pattern, # whitespace ctx().* - '[\s]ctx\([\'|"]?{0}[\'|"]?\)[\.{0}]?'.format(_regex_ctx_pattern) # whitespace ctx(*).* + r'^ctx\(\)\.%s' % _regex_ctx_pattern, # line start ctx().* + r'^ctx\([\'|"]?{0}[\'|"]?\)[\.{0}]?'.format(_regex_ctx_pattern), # line start ctx(*).* + r'[\s]ctx\(\)\.%s' % _regex_ctx_pattern, # whitespace ctx().* + r'[\s]ctx\([\'|"]?{0}[\'|"]?\)[\.{0}]?'.format(_regex_ctx_pattern) # whitespace ctx(*).* ] - _regex_ctx_var = '.*?(%s).*?' % '|'.join(_regex_ctx_patterns) + _regex_ctx_var = r'.*?(%s).*?' % '|'.join(_regex_ctx_patterns) _regex_ctx_var_parser = re.compile(_regex_ctx_var) - _regex_var = '[a-zA-Z0-9_\-]+' + _regex_var = r'[a-zA-Z0-9_\-]+' _regex_var_extracts = [ r'(?<=^ctx\(\)\.)(\b%s\b)(?!\()\.?' % _regex_var, # extract x in ctx().x r'(?<=^ctx\()(\b%s\b)(?=\))\.?' % _regex_var, # extract x in ctx(x) From c5ef955393e4047d5023fb30aae6b917d753a7b2 Mon Sep 17 00:00:00 2001 From: W Chan Date: Thu, 9 Apr 2020 18:24:25 -0700 Subject: [PATCH 6/9] Add more test cases to capture edge cases for ctx().get() --- .../test_facade_jinja_ctx_by_function_arg.py | 10 ++++++++-- .../test_facade_yaql_ctx_by_function_arg.py | 10 ++++++++-- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/orquesta/tests/unit/expressions/test_facade_jinja_ctx_by_function_arg.py b/orquesta/tests/unit/expressions/test_facade_jinja_ctx_by_function_arg.py index dce7c7b7..4968cd64 100644 --- a/orquesta/tests/unit/expressions/test_facade_jinja_ctx_by_function_arg.py +++ b/orquesta/tests/unit/expressions/test_facade_jinja_ctx_by_function_arg.py @@ -19,7 +19,12 @@ class JinjaFacadeVariableExtractionTest(test_base.ExpressionFacadeEvaluatorTest): def test_empty_extraction(self): - expr = '{{ just_text and $not_a_var and notctx(foo) and notctx("bar") and notctx(\'fu\') }}' + expr = ( + '{{ just_text and $not_a_var and ' + 'notctx(foo) and notctx("bar") and notctx(\'fu\') ' + 'ctx("foo\') and ctx(\'foo") and ctx(foo") and ' + 'ctx("foo) and ctx(foo\') and ctx(\'foo) }}' + ) self.assertListEqual([], expr_base.extract_vars(expr)) @@ -124,7 +129,8 @@ def test_ignore_ctx_dict_funcs(self): def test_ignore_ctx_get_func_calls(self): expr = ( '{{ctx().get(foo) and ctx().get(bar) and ctx().get("fu") and ctx().get(\'baz\') and ' - 'ctx().get(foo, "bar") and ctx().get("fu", "bar") and ctx().get(\'bar\', \'foo\') }}' + 'ctx().get(foo, "bar") and ctx().get("fu", "bar") and ctx().get(\'bar\', \'foo\') and ' + 'ctx().get("foo\') and ctx().get(\'foo") and ctx().get("foo) and ctx().get(foo") }}' ) expected_vars = [] diff --git a/orquesta/tests/unit/expressions/test_facade_yaql_ctx_by_function_arg.py b/orquesta/tests/unit/expressions/test_facade_yaql_ctx_by_function_arg.py index daad8a52..a8832177 100644 --- a/orquesta/tests/unit/expressions/test_facade_yaql_ctx_by_function_arg.py +++ b/orquesta/tests/unit/expressions/test_facade_yaql_ctx_by_function_arg.py @@ -19,7 +19,12 @@ class YAQLFacadeVariableExtractionTest(test_base.ExpressionFacadeEvaluatorTest): def test_empty_extraction(self): - expr = '<% just_text and $not_a_var and notctx(foo) and notctx("bar") and notctx(\'fu\') %>' + expr = ( + '<% just_text and $not_a_var and ' + 'notctx(foo) and notctx("bar") and notctx(\'fu\') ' + 'ctx("foo\') and ctx(\'foo") and ctx(foo") and ' + 'ctx("foo) and ctx(foo\') and ctx(\'foo) %>' + ) self.assertListEqual([], expr_base.extract_vars(expr)) @@ -121,7 +126,8 @@ def test_ignore_ctx_dict_funcs(self): def test_ignore_ctx_get_func_calls(self): expr = ( '<%ctx().get(foo) and ctx().get(bar) and ctx().get("fu") and ctx().get(\'baz\') and ' - 'ctx().get(foo, "bar") and ctx().get("fu", "bar") and ctx().get(\'bar\', \'foo\') %>' + 'ctx().get(foo, "bar") and ctx().get("fu", "bar") and ctx().get(\'bar\', \'foo\') and ' + 'ctx().get("foo\') and ctx().get(\'foo") and ctx().get("foo) and ctx().get(foo") %>' ) expected_vars = [] From 4a5835a3e660147d03acbf8b8bc875d910cab964 Mon Sep 17 00:00:00 2001 From: blag Date: Fri, 10 Apr 2020 11:00:51 -0700 Subject: [PATCH 7/9] Simplify regexes a bit --- orquesta/expressions/jinja.py | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/orquesta/expressions/jinja.py b/orquesta/expressions/jinja.py index 544d80e1..592f6642 100644 --- a/orquesta/expressions/jinja.py +++ b/orquesta/expressions/jinja.py @@ -54,22 +54,22 @@ class JinjaEvaluator(expr_base.Evaluator): _regex_pattern = '{{.*?}}' _regex_parser = re.compile(_regex_pattern) - _regex_ctx_pattern = r'[a-zA-Z0-9_\'"\.\[\]\(\)]*' - _regex_ctx_patterns = [ - r'^ctx\(\)\.%s' % _regex_ctx_pattern, # line start ctx().* - r'^ctx\([\'|"]?{0}[\'|"]?\)[\.{0}]?'.format(_regex_ctx_pattern), # line start ctx(*).* - r'[\s]ctx\(\)\.%s' % _regex_ctx_pattern, # whitespace ctx().* - r'[\s]ctx\([\'|"]?{0}[\'|"]?\)[\.{0}]?'.format(_regex_ctx_pattern) # whitespace ctx(*).* - ] - _regex_ctx_var = r'.*?(%s).*?' % '|'.join(_regex_ctx_patterns) - _regex_ctx_var_parser = re.compile(_regex_ctx_var) - - _regex_var = r'[a-zA-Z0-9_\-]+' + _regex_reference_pattern = r'[][a-zA-Z0-9_\'"\.()]*' + # match any of: + # word boundary ctx(*) + # word boundary ctx()* + # word boundary ctx().* + # word boundary ctx(*)* + # word boundary ctx(*).* + _regex_ctx_pattern = r'\bctx\([\'"]?{0}[\'"]?\)\.?{0}'.format(_regex_reference_pattern) + _regex_ctx_var_parser = re.compile(_regex_ctx_pattern) + + _regex_var = r'[a-zA-Z0-9_-]+' _regex_var_extracts = [ - r'(?<=^ctx\(\)\.)(\b%s\b)(?!\()\.?' % _regex_var, # extract x in ctx().x - r'(?<=^ctx\()(\b%s\b)(?=\))\.?' % _regex_var, # extract x in ctx(x) - r'(?<=^ctx\(\')(\b%s\b)(?=\'\))\.?' % _regex_var, # extract x in ctx('x') - r'(?<=^ctx\(")(\b%s\b)(?="\))\.?' % _regex_var # extract x in ctx("x") + r'(?<=\bctx\(\)\.)({})\b(?!\()\.?'.format(_regex_var), # extract x in ctx().x + r'(?:\bctx\(({})\))'.format(_regex_var), # extract x in ctx(x) + r'(?:\bctx\(\'({})\'\))'.format(_regex_var), # extract x in ctx('x') + r'(?:\bctx\("({})"\))'.format(_regex_var) # extract x in ctx("x") ] _block_delimiter = '{%}' From 900c7c77f373bc082522d5efe41e42fef5e8e5ae Mon Sep 17 00:00:00 2001 From: blag Date: Fri, 10 Apr 2020 11:01:13 -0700 Subject: [PATCH 8/9] Add a few more expressions tests --- .../unit/expressions/test_facade_jinja_ctx_by_dot_notation.py | 4 +++- .../unit/expressions/test_facade_jinja_ctx_by_function_arg.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/orquesta/tests/unit/expressions/test_facade_jinja_ctx_by_dot_notation.py b/orquesta/tests/unit/expressions/test_facade_jinja_ctx_by_dot_notation.py index 2391d0f0..92a953fc 100644 --- a/orquesta/tests/unit/expressions/test_facade_jinja_ctx_by_dot_notation.py +++ b/orquesta/tests/unit/expressions/test_facade_jinja_ctx_by_dot_notation.py @@ -19,7 +19,9 @@ class JinjaFacadeVariableExtractionTest(test_base.ExpressionFacadeEvaluatorTest): def test_empty_extraction(self): - expr = '{{ just_text and _not_a_var and fooctx(foo) and fooctx("bar") and fooctx(\'fu\') }}' + expr = ( + '{{ just_text and _not_a_var and fooctx(foo) and fooctx("bar") and fooctx(\'fu\') ' + 'and ctx(). and ctx().() and ctx().-foobar and ctx().foobar() }}') self.assertListEqual([], expr_base.extract_vars(expr)) diff --git a/orquesta/tests/unit/expressions/test_facade_jinja_ctx_by_function_arg.py b/orquesta/tests/unit/expressions/test_facade_jinja_ctx_by_function_arg.py index 4968cd64..d7bfe6e5 100644 --- a/orquesta/tests/unit/expressions/test_facade_jinja_ctx_by_function_arg.py +++ b/orquesta/tests/unit/expressions/test_facade_jinja_ctx_by_function_arg.py @@ -23,7 +23,7 @@ def test_empty_extraction(self): '{{ just_text and $not_a_var and ' 'notctx(foo) and notctx("bar") and notctx(\'fu\') ' 'ctx("foo\') and ctx(\'foo") and ctx(foo") and ' - 'ctx("foo) and ctx(foo\') and ctx(\'foo) }}' + 'ctx("foo) and ctx(foo\') and ctx(\'foo) and ctx(-foobar) }}' ) self.assertListEqual([], expr_base.extract_vars(expr)) From 387f56c9d74be87e78bc4edeeb724eb0ff0fd85c Mon Sep 17 00:00:00 2001 From: W Chan Date: Fri, 10 Apr 2020 12:05:17 -0700 Subject: [PATCH 9/9] Simplify the YAQL expression context regexes Copy the simpliciation for Jinja evaluator to the YAQL evaluator. Add more test cases to ensure correctness. --- orquesta/expressions/jinja.py | 4 +-- orquesta/expressions/yql.py | 30 +++++++++---------- .../test_facade_jinja_ctx_by_dot_notation.py | 5 ++-- .../test_facade_jinja_ctx_by_function_arg.py | 5 +++- .../test_facade_yaql_ctx_by_dot_notation.py | 5 +++- .../test_facade_yaql_ctx_by_function_arg.py | 5 +++- .../test_jinja_ctx_by_dot_notation.py | 2 +- .../test_jinja_ctx_by_function_arg.py | 2 +- .../test_yaql_ctx_by_dot_notation.py | 2 +- .../test_yaql_ctx_by_function_arg.py | 2 +- 10 files changed, 36 insertions(+), 26 deletions(-) diff --git a/orquesta/expressions/jinja.py b/orquesta/expressions/jinja.py index 592f6642..8c4dd1de 100644 --- a/orquesta/expressions/jinja.py +++ b/orquesta/expressions/jinja.py @@ -54,14 +54,14 @@ class JinjaEvaluator(expr_base.Evaluator): _regex_pattern = '{{.*?}}' _regex_parser = re.compile(_regex_pattern) - _regex_reference_pattern = r'[][a-zA-Z0-9_\'"\.()]*' + _regex_ctx_ref_pattern = r'[][a-zA-Z0-9_\'"\.()]*' # match any of: # word boundary ctx(*) # word boundary ctx()* # word boundary ctx().* # word boundary ctx(*)* # word boundary ctx(*).* - _regex_ctx_pattern = r'\bctx\([\'"]?{0}[\'"]?\)\.?{0}'.format(_regex_reference_pattern) + _regex_ctx_pattern = r'\bctx\([\'"]?{0}[\'"]?\)\.?{0}'.format(_regex_ctx_ref_pattern) _regex_ctx_var_parser = re.compile(_regex_ctx_pattern) _regex_var = r'[a-zA-Z0-9_-]+' diff --git a/orquesta/expressions/yql.py b/orquesta/expressions/yql.py index 68a13113..a172d74c 100644 --- a/orquesta/expressions/yql.py +++ b/orquesta/expressions/yql.py @@ -54,22 +54,22 @@ class YAQLEvaluator(expr_base.Evaluator): _regex_pattern = '<%.*?%>' _regex_parser = re.compile(_regex_pattern) - _regex_ctx_pattern = r'[a-zA-Z0-9_\'"\.\[\]\(\)]*' - _regex_ctx_patterns = [ - r'^ctx\(\)\.%s' % _regex_ctx_pattern, # line start ctx().* - r'^ctx\([\'|"]?{0}[\'|"]?\)[\.{0}]?'.format(_regex_ctx_pattern), # line start ctx(*).* - r'[\s]ctx\(\)\.%s' % _regex_ctx_pattern, # whitespace ctx().* - r'[\s]ctx\([\'|"]?{0}[\'|"]?\)[\.{0}]?'.format(_regex_ctx_pattern) # whitespace ctx(*).* - ] - _regex_ctx_var = r'.*?(%s).*?' % '|'.join(_regex_ctx_patterns) - _regex_ctx_var_parser = re.compile(_regex_ctx_var) - - _regex_var = r'[a-zA-Z0-9_\-]+' + _regex_ctx_ref_pattern = r'[][a-zA-Z0-9_\'"\.()]*' + # match any of: + # word boundary ctx(*) + # word boundary ctx()* + # word boundary ctx().* + # word boundary ctx(*)* + # word boundary ctx(*).* + _regex_ctx_pattern = r'\bctx\([\'"]?{0}[\'"]?\)\.?{0}'.format(_regex_ctx_ref_pattern) + _regex_ctx_var_parser = re.compile(_regex_ctx_pattern) + + _regex_var = r'[a-zA-Z0-9_-]+' _regex_var_extracts = [ - r'(?<=^ctx\(\)\.)(\b%s\b)(?!\()\.?' % _regex_var, # extract x in ctx().x - r'(?<=^ctx\()(\b%s\b)(?=\))\.?' % _regex_var, # extract x in ctx(x) - r'(?<=^ctx\(\')(\b%s\b)(?=\'\))\.?' % _regex_var, # extract x in ctx('x') - r'(?<=^ctx\(")(\b%s\b)(?="\))\.?' % _regex_var # extract x in ctx("x") + r'(?<=\bctx\(\)\.)({})\b(?!\()\.?'.format(_regex_var), # extract x in ctx().x + r'(?:\bctx\(({})\))'.format(_regex_var), # extract x in ctx(x) + r'(?:\bctx\(\'({})\'\))'.format(_regex_var), # extract x in ctx('x') + r'(?:\bctx\("({})"\))'.format(_regex_var) # extract x in ctx("x") ] _engine = yaql.language.factory.YaqlFactory().create() diff --git a/orquesta/tests/unit/expressions/test_facade_jinja_ctx_by_dot_notation.py b/orquesta/tests/unit/expressions/test_facade_jinja_ctx_by_dot_notation.py index 92a953fc..d77ddf72 100644 --- a/orquesta/tests/unit/expressions/test_facade_jinja_ctx_by_dot_notation.py +++ b/orquesta/tests/unit/expressions/test_facade_jinja_ctx_by_dot_notation.py @@ -20,8 +20,9 @@ class JinjaFacadeVariableExtractionTest(test_base.ExpressionFacadeEvaluatorTest) def test_empty_extraction(self): expr = ( - '{{ just_text and _not_a_var and fooctx(foo) and fooctx("bar") and fooctx(\'fu\') ' - 'and ctx(). and ctx().() and ctx().-foobar and ctx().foobar() }}') + '{{ just_text and $not_a_var and notctx().bar and ' + 'ctx(). and ctx().() and ctx().-foobar and ctx().foobar() }}' + ) self.assertListEqual([], expr_base.extract_vars(expr)) diff --git a/orquesta/tests/unit/expressions/test_facade_jinja_ctx_by_function_arg.py b/orquesta/tests/unit/expressions/test_facade_jinja_ctx_by_function_arg.py index d7bfe6e5..b341da41 100644 --- a/orquesta/tests/unit/expressions/test_facade_jinja_ctx_by_function_arg.py +++ b/orquesta/tests/unit/expressions/test_facade_jinja_ctx_by_function_arg.py @@ -23,7 +23,10 @@ def test_empty_extraction(self): '{{ just_text and $not_a_var and ' 'notctx(foo) and notctx("bar") and notctx(\'fu\') ' 'ctx("foo\') and ctx(\'foo") and ctx(foo") and ' - 'ctx("foo) and ctx(foo\') and ctx(\'foo) and ctx(-foobar) }}' + 'ctx("foo) and ctx(foo\') and ctx(\'foo) and ' + 'ctx(-foo) and ctx("-bar") and ctx(\'-fu\') and ' + 'ctx(foo.bar) and ctx("foo.bar") and ctx(\'foo.bar\') and ' + 'ctx(foo()) and ctx("foo()") and ctx(\'foo()\') }}' ) self.assertListEqual([], expr_base.extract_vars(expr)) diff --git a/orquesta/tests/unit/expressions/test_facade_yaql_ctx_by_dot_notation.py b/orquesta/tests/unit/expressions/test_facade_yaql_ctx_by_dot_notation.py index 6dd9dc9a..9c6081a3 100644 --- a/orquesta/tests/unit/expressions/test_facade_yaql_ctx_by_dot_notation.py +++ b/orquesta/tests/unit/expressions/test_facade_yaql_ctx_by_dot_notation.py @@ -19,7 +19,10 @@ class YAQLFacadeVariableExtractionTest(test_base.ExpressionFacadeEvaluatorTest): def test_empty_extraction(self): - expr = '<% just_text and $not_a_var and fooctx(foo) and fooctx("bar") and fooctx(\'fu\') %>' + expr = ( + '<% just_text and $not_a_var and notctx().bar and ' + 'ctx(). and ctx().() and ctx().-foobar and ctx().foobar() %>' + ) self.assertListEqual([], expr_base.extract_vars(expr)) diff --git a/orquesta/tests/unit/expressions/test_facade_yaql_ctx_by_function_arg.py b/orquesta/tests/unit/expressions/test_facade_yaql_ctx_by_function_arg.py index a8832177..4880f9a7 100644 --- a/orquesta/tests/unit/expressions/test_facade_yaql_ctx_by_function_arg.py +++ b/orquesta/tests/unit/expressions/test_facade_yaql_ctx_by_function_arg.py @@ -23,7 +23,10 @@ def test_empty_extraction(self): '<% just_text and $not_a_var and ' 'notctx(foo) and notctx("bar") and notctx(\'fu\') ' 'ctx("foo\') and ctx(\'foo") and ctx(foo") and ' - 'ctx("foo) and ctx(foo\') and ctx(\'foo) %>' + 'ctx("foo) and ctx(foo\') and ctx(\'foo) and ' + 'ctx(-foo) and ctx("-bar") and ctx(\'-fu\') and ' + 'ctx(foo.bar) and ctx("foo.bar") and ctx(\'foo.bar\') and ' + 'ctx(foo()) and ctx("foo()") and ctx(\'foo()\') %>' ) self.assertListEqual([], expr_base.extract_vars(expr)) diff --git a/orquesta/tests/unit/expressions/test_jinja_ctx_by_dot_notation.py b/orquesta/tests/unit/expressions/test_jinja_ctx_by_dot_notation.py index 4712f24a..d3755a7e 100644 --- a/orquesta/tests/unit/expressions/test_jinja_ctx_by_dot_notation.py +++ b/orquesta/tests/unit/expressions/test_jinja_ctx_by_dot_notation.py @@ -23,7 +23,7 @@ def setUpClass(cls): super(JinjaVariableExtractionTest, cls).setUpClass() def test_empty_extraction(self): - expr = '{{ just_text and _not_a_var and fooctx().bar }}' + expr = '{{ just_text and $not_a_var and notctx().bar }}' self.assertListEqual([], self.evaluator.extract_vars(expr)) diff --git a/orquesta/tests/unit/expressions/test_jinja_ctx_by_function_arg.py b/orquesta/tests/unit/expressions/test_jinja_ctx_by_function_arg.py index ab374682..b1ed7b62 100644 --- a/orquesta/tests/unit/expressions/test_jinja_ctx_by_function_arg.py +++ b/orquesta/tests/unit/expressions/test_jinja_ctx_by_function_arg.py @@ -23,7 +23,7 @@ def setUpClass(cls): super(JinjaVariableExtractionTest, cls).setUpClass() def test_empty_extraction(self): - expr = '{{ just_text and _not_a_var and fooctx(foo) and fooctx("bar") and fooctx(\'fu\') }}' + expr = '{{ just_text and $not_a_var and notctx(foo) and notctx("bar") and notctx(\'fu\') }}' self.assertListEqual([], self.evaluator.extract_vars(expr)) diff --git a/orquesta/tests/unit/expressions/test_yaql_ctx_by_dot_notation.py b/orquesta/tests/unit/expressions/test_yaql_ctx_by_dot_notation.py index 10deecfe..d6456544 100644 --- a/orquesta/tests/unit/expressions/test_yaql_ctx_by_dot_notation.py +++ b/orquesta/tests/unit/expressions/test_yaql_ctx_by_dot_notation.py @@ -23,7 +23,7 @@ def setUpClass(cls): super(YAQLVariableExtractionTest, cls).setUpClass() def test_empty_extraction(self): - expr = '<% just_text and $not_a_var and fooctx().bar %>' + expr = '<% just_text and $not_a_var and notctx().bar %>' self.assertListEqual([], self.evaluator.extract_vars(expr)) diff --git a/orquesta/tests/unit/expressions/test_yaql_ctx_by_function_arg.py b/orquesta/tests/unit/expressions/test_yaql_ctx_by_function_arg.py index 111db34a..34c9daab 100644 --- a/orquesta/tests/unit/expressions/test_yaql_ctx_by_function_arg.py +++ b/orquesta/tests/unit/expressions/test_yaql_ctx_by_function_arg.py @@ -23,7 +23,7 @@ def setUpClass(cls): super(YAQLVariableExtractionTest, cls).setUpClass() def test_empty_extraction(self): - expr = '<% just_text and $not_a_var and fooctx(foo) and fooctx("bar") and fooctx(\'fu\') %>' + expr = '<% just_text and $not_a_var and notctx(foo) and notctx("bar") and notctx(\'fu\') %>' self.assertListEqual([], self.evaluator.extract_vars(expr))