DPNLPF-1924 add read jinja2 file templates (#188)

* DPNLPF-1924 add read jinja2 file templates DPNLPF-1924 add tests for UnifiedTemplateMultiLoader action and requirement remove support for 3.8 --------- Co-authored-by: Mark Chilingaryan <[email protected]> Co-authored-by: Makar Shevchenko <[email protected]>
salute-developers · Mar 19, 2024 · 663e218 · 663e218
1 parent 9f81083
commit 663e218
Show file tree

Hide file tree

Showing 15 changed files with 229 additions and 23 deletions.
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -14,7 +14,7 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v2
       with:
-        python-version: '3.8.x'
+        python-version: '3.11.x'
     - name: Install Poetry
       uses: snok/install-poetry@v1
     - name: Install Poetry plugins

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -16,7 +16,7 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v5
       with:
-        python-version: "3.8"
+        python-version: "3.11"
         cache: 'poetry'
     - name: Install dependencies
       run: poetry install --no-interaction --no-root --all-extras
@@ -27,7 +27,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['3.8', '3.9', '3.10', '3.11']
+        python-version: ['3.9', '3.10', '3.11']
 
     runs-on: ubuntu-20.04
     name: unittests with py-${{ matrix.python-version }}
@@ -61,7 +61,7 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v5
       with:
-        python-version: "3.8"
+        python-version: "3.11"
         cache: 'poetry'
     - name: Install Poetry plugins
       run: poetry self add "poetry-dynamic-versioning[plugin]"

diff --git a/README.md b/README.md
@@ -42,7 +42,7 @@ ____
 
 * Linux, Mac OS или Windows (необходима установка [Conda](https://docs.conda.io/en/latest/)).
 * 512 МБ свободной памяти.
-* Python 3.8.0 - 3.9.6.
+* Python 3.9 - 3.11.
 
 ____
 

diff --git a/core/basic_models/actions/string_actions.py b/core/basic_models/actions/string_actions.py
@@ -4,14 +4,17 @@
 from functools import cached_property
 from itertools import chain
 from typing import Union, Dict, List, Any, Optional, Tuple, TypeVar, Type, AsyncGenerator
+from lazy import lazy
+from scenarios.user.user_model import User
 
 from core.basic_models.actions.basic_actions import CommandAction
 from core.basic_models.actions.command import Command
 from core.basic_models.answer_items.answer_items import SdkAnswerItem
 from core.model.base_user import BaseUser
 from core.model.factory import list_factory
 from core.text_preprocessing.base import BaseTextPreprocessingResult
-from core.unified_template.unified_template import UnifiedTemplate, UNIFIED_TEMPLATE_TYPE_NAME
+from core.unified_template.unified_template import UnifiedTemplate, UNIFIED_TEMPLATE_TYPE_NAME, \
+    UnifiedTemplateMultiLoader
 
 T = TypeVar("T")
 
@@ -112,6 +115,7 @@ class StringAction(NodeAction):
       }
     }
     """
+
     def __init__(self, items: Dict[str, Any], id: Optional[str] = None):
         super(StringAction, self).__init__(items, id)
 
@@ -130,7 +134,7 @@ def _generate_command_context(self, user: BaseUser, text_preprocessing_result: B
 
     async def run(self, user: BaseUser, text_preprocessing_result: BaseTextPreprocessingResult,
                   params: Optional[Dict[str, Union[str, float, int]]] = None) -> AsyncGenerator[Command, None]:
-        # Example: Command("ANSWER_TO_USER", {"answer": {"key1": "string1", "keyN": "stringN"}})
+        # Result command format: Command("ANSWER_TO_USER", {"answer": {"key1": "string1", "keyN": "stringN"}})
         params = params or {}
         command_params = self._generate_command_context(user, text_preprocessing_result, params)
 
@@ -146,6 +150,69 @@ async def run(self, user: BaseUser, text_preprocessing_result: BaseTextPreproces
                       request_data=self.request_data)
 
 
+class StringFileUnifiedTemplateAction(StringAction):
+    @lazy
+    def nodes(self):
+        if self._nodes.get("type", "") == UNIFIED_TEMPLATE_TYPE_NAME:
+            return self._get_template_tree(self._nodes)
+        else:
+            return {k: self._get_template_tree(t) for k, t in self._nodes.items()}
+
+    async def run(self, user: User, text_preprocessing_result: BaseTextPreprocessingResult,
+                  params: Optional[Dict[str, Union[str, float, int]]] = None) -> List[Command]:
+        command_params = dict()
+        params = copy(params) or {}
+        collected = user.parametrizer.collect(text_preprocessing_result, filter_params={"command": self.command})
+        params.update(collected)
+        if type(self.nodes) == UnifiedTemplateMultiLoader:
+            command_params = self._get_rendered_tree(self.nodes, params, self.no_empty_nodes)
+        else:
+            for key, value in self.nodes.items():
+                rendered = self._get_rendered_tree(value, params, self.no_empty_nodes)
+                if rendered != "" or not self.no_empty_nodes:
+                    command_params[key] = rendered
+
+        yield Command(self.command, command_params, self.id, request_type=self.request_type,
+                      request_data=self.request_data)
+
+    def _get_template_tree(self, value):
+        is_dict_unified_template = isinstance(value, dict) and value.get("type") == UNIFIED_TEMPLATE_TYPE_NAME
+        if isinstance(value, str) or is_dict_unified_template:
+            result = UnifiedTemplateMultiLoader(value)
+        elif isinstance(value, dict):
+            result = {}
+            for inner_key, inner_value in value.items():
+                result[inner_key] = self._get_template_tree(inner_value)
+        elif isinstance(value, list):
+            result = []
+            for inner_value in value:
+                result.append(self._get_template_tree(inner_value))
+        else:
+            result = value
+        return result
+
+    def _get_rendered_tree_recursive(self, value, params, no_empty=False):
+        value_type = type(value)
+        if value_type is dict:
+            result = {}
+            for inner_key, inner_value in value.items():
+                rendered = self._get_rendered_tree_recursive(inner_value, params, no_empty=no_empty)
+                if rendered != "" or not no_empty:
+                    result[inner_key] = rendered
+        elif value_type is list:
+            result = []
+            for inner_value in value:
+                rendered = self._get_rendered_tree_recursive(inner_value, params, no_empty=no_empty)
+                if rendered != "" or not no_empty:
+                    result.append(rendered)
+
+        elif value_type is UnifiedTemplateMultiLoader:
+            result = value.render(params)
+        else:
+            result = value
+        return result
+
+
 class AfinaAnswerAction(NodeAction):
     """
     Example:
@@ -160,6 +227,7 @@ class AfinaAnswerAction(NodeAction):
     Output:
     [command1(pronounceText)]
     """
+
     def __init__(self, items: Dict[str, Any], id: Optional[str] = None):
         super(AfinaAnswerAction, self).__init__(items, id)
         self.command: str = ANSWER_TO_USER

diff --git a/core/basic_models/requirement/basic_requirements.py b/core/basic_models/requirement/basic_requirements.py
@@ -16,7 +16,7 @@
 from core.model.registered import Registered
 from core.text_preprocessing.base import BaseTextPreprocessingResult
 from core.text_preprocessing.preprocessing_result import TextPreprocessingResult
-from core.unified_template.unified_template import UnifiedTemplate
+from core.unified_template.unified_template import UnifiedTemplate, UnifiedTemplateMultiLoader
 from core.utils.stats_timer import StatsTimer
 from scenarios.scenario_models.field.field_filler_description import IntersectionFieldFiller
 from scenarios.user.user_model import User
@@ -208,7 +208,7 @@ def _check(self, text_preprocessing_result: BaseTextPreprocessingResult, user: B
 class TemplateRequirement(Requirement):
     def __init__(self, items: Dict[str, Any], id: Optional[str] = None) -> None:
         super().__init__(items, id)
-        self._template = UnifiedTemplate(items["template"])
+        self._template = UnifiedTemplateMultiLoader(items["template"])
 
     def _check(self, text_preprocessing_result: BaseTextPreprocessingResult, user: BaseUser,
                params: Dict[str, Any] = None) -> bool:

diff --git a/core/unified_template/unified_template.py b/core/unified_template/unified_template.py
@@ -3,6 +3,7 @@
 from copy import copy
 import jinja2
 from distutils.util import strtobool
+import os
 
 import core.logging.logger_constants as log_const
 from core.logging.logger_utils import log
@@ -84,3 +85,27 @@ def silent_render(self, params_dict):
 
     def __str__(self):
         return str(self.input)
+
+
+class UnifiedTemplateMultiLoader(UnifiedTemplate):
+    """
+    Загружает шаблон jinja из файла или строки
+    Файлы шаблонов jinja должны находиться в директории some-app.static.templates
+    """
+
+    def __init__(self, input):
+        if isinstance(input, dict) and input.get("file"):
+            file_name = input["file"]
+            from smart_kit.configs import get_app_config
+            app_config = get_app_config()
+            templates_dir = os.path.join(app_config.STATIC_PATH, "references/templates")
+            for root, dirs, files in os.walk(templates_dir):
+                if file_name in files:
+                    with open(os.path.join(root, file_name)) as f:
+                        input["template"] = f.read()
+                    super().__init__(input)
+                    return
+            raise Exception(f"Template {file_name} does not exist in templates directory {templates_dir}")
+
+        else:
+            super().__init__(input)
diff --git a/poetry.lock b/poetry.lock
diff --git a/pyproject.toml b/pyproject.toml
@@ -63,6 +63,7 @@ lxml = "4.9.2"
 twisted = "22.10.0"
 urllib3 = "1.26.16"
 certifi = "2023.07.22"
+lazy = "^1.6"
 
 [tool.poetry.extras]
 ml = ["keras", "scikit-learn", "tensorflow", "tensorflow-macos", "tensorflow-aarch64"]

diff --git a/smart_kit/resources/__init__.py b/smart_kit/resources/__init__.py
@@ -21,7 +21,7 @@
 from core.basic_models.actions.smartpay import SmartPayCreateAction, SmartPayPerformAction, SmartPayGetStatusAction, \
     SmartPayConfirmAction, SmartPayDeleteAction, SmartPayRefundAction
 from core.basic_models.actions.string_actions import StringAction, AfinaAnswerAction, SDKAnswer, \
-    SDKAnswerToUser
+    SDKAnswerToUser, StringFileUnifiedTemplateAction
 from core.basic_models.actions.variable_actions import ClearVariablesAction, DeleteVariableAction, \
     SetLocalVariableAction, SetVariableAction
 from core.basic_models.answer_items.answer_items import items_factory, SdkAnswerItem, answer_items, BubbleText, \
@@ -311,7 +311,7 @@ def init_actions(self):
         actions["self_service_with_state"] = SelfServiceActionWithState
         actions["set_local_variable"] = SetLocalVariableAction
         actions["set_variable"] = SetVariableAction
-        actions["string"] = StringAction
+        actions["string"] = StringFileUnifiedTemplateAction
         actions["push"] = PushAction
         actions["push_authentication"] = PushAuthenticationActionHttp
         actions["push_http"] = PushActionHttp

diff --git a/smart_kit/template/static/references/forms/hello_form.json b/smart_kit/template/static/references/forms/hello_form.json
@@ -61,13 +61,11 @@
             "command": "ANSWER_TO_USER",
             "nodes": {
               "pronounceText": "Сколько лет ты программируешь на Python?",
-              "items": [
-                {
-                  "bubble": {
-                    "text": "Сколько лет ты программируешь на Python?"
-                  }
-                }
-              ]
+              "items": {
+                "type": "unified_template",
+                "file": "experience_items_template.jinja2",
+                "loader": "json"
+              }
             }
           }
         ],

diff --git a/smart_kit/template/static/references/templates/experience_items_template.jinja2 b/smart_kit/template/static/references/templates/experience_items_template.jinja2
@@ -0,0 +1,9 @@
+{{
+    [
+        {
+            "bubble": {
+                "text": "Сколько лет ты программируешь на Python?"
+            }
+        }
+    ] |tojson
+}}