Merge pull request #465 from Crunch-io/add-strict-subvariable-syntax-…

…handler

Add strict subvariable syntax handler

integration/test_scripts.py

@@ -1,14 +1,137 @@
 # coding: utf-8
+import os
+from datetime import datetime
+
 import pytest
 
-from pycrunch.shoji import as_entity
+from pycrunch.shoji import as_entity, wait_progress, TaskError
 
+from scrunch.datasets import Project
+from scrunch.helpers import shoji_entity_wrapper
 from scrunch.scripts import ScriptExecutionError
 from scrunch.mutable_dataset import get_mutable_dataset
 from fixtures import BaseIntegrationTestCase
 
 
-class TestScripts(BaseIntegrationTestCase):
+@pytest.mark.skipif(os.environ.get("LOCAL_INTEGRATION") is None, reason="Do not run this test during CI/CD")
+class TestSystemScripts(BaseIntegrationTestCase):
+    def new_project(self, name):
+        res = self.site.projects.create(shoji_entity_wrapper({
+            "name": name + datetime.now().strftime("%Y%m%d%H%M%S")
+        })).refresh()
+        return Project(res)
+
+    def test_define_view_strict_subvariable_syntax(self):
+        project = self.new_project("test_view_strict_subvariable")
+        ds = self.site.datasets.create(as_entity({"name": "test_dataset_script"})).refresh()
+        categories = [
+            {"id": 2, "name": "Home"},
+            {"id": 3, "name": "Work"},
+            {"id": -1, "name": "No Data", "missing": True},
+        ]
+        subvariables = [
+            {"alias": "cat", "name": "Cat"},
+            {"alias": "dog", "name": "Dog"},
+            {"alias": "bird", "name": "Bird"},
+        ]
+
+        ds.variables.create(
+            as_entity(
+                dict(
+                    alias="pets",
+                    name="Pets",
+                    type="categorical_array",
+                    categories=categories,
+                    subvariables=subvariables,
+                    values=[[2, 3, 3], [3, 3, 2], [2, -1, 3], [3, 2, -1]],
+                )
+            )
+        )
+        ds.variables.create(
+            as_entity(
+                dict(
+                    alias="pets_2",
+                    name="Pets 2",
+                    type="categorical_array",
+                    categories=categories,
+                    subvariables=subvariables,
+                    values=[[2, 3, 3], [3, 3, 2], [2, -1, 3], [3, 2, -1]],
+                )
+            )
+        )
+        script_body = """
+                        DEFINE VIEW FROM DATASET_ID(`{}`)
+                        VARIABLES pets, pets_2
+                        NAME "My view";
+                        """.format(ds.body.id)
+
+        scrunch_dataset = get_mutable_dataset(ds.body.id, self.site)
+        project.move_here([scrunch_dataset])
+        resp = project.execute(script_body, strict_subvariable_syntax=True)
+        wait_progress(resp, self.site.session)
+        view = scrunch_dataset.views.get_by_name("My view")
+        assert view.project.name == project.name
+
+    def test_define_view_strict_subvariable_syntax_error(self):
+        project = self.new_project("test_view_strict_subvariable_false")
+        ds = self.site.datasets.create(as_entity({"name": "test_dataset_script_false"})).refresh()
+        categories = [
+            {"id": 2, "name": "Home"},
+            {"id": 3, "name": "Work"},
+            {"id": -1, "name": "No Data", "missing": True},
+        ]
+        subvariables = [
+            {"alias": "cat", "name": "Cat"},
+            {"alias": "dog", "name": "Dog"},
+            {"alias": "bird", "name": "Bird"},
+        ]
+
+        ds.variables.create(
+            as_entity(
+                dict(
+                    alias="pets",
+                    name="Pets",
+                    type="categorical_array",
+                    categories=categories,
+                    subvariables=subvariables,
+                    values=[[2, 3, 3], [3, 3, 2], [2, -1, 3], [3, 2, -1]],
+                )
+            )
+        )
+        ds.variables.create(
+            as_entity(
+                dict(
+                    alias="pets_2",
+                    name="Pets 2",
+                    type="categorical_array",
+                    categories=categories,
+                    subvariables=subvariables,
+                    values=[[2, 3, 3], [3, 3, 2], [2, -1, 3], [3, 2, -1]],
+                )
+            )
+        )
+        script_body = """
+                                DEFINE VIEW FROM DATASET_ID(`{}`)
+                                VARIABLES pets, pets_2
+                                NAME "My view";
+                                """.format(ds.body.id)
+
+        try:
+            scrunch_dataset = get_mutable_dataset(ds.body.id, self.site)
+            project.move_here([scrunch_dataset])
+            resp = project.execute(script_body)
+            with pytest.raises(TaskError) as err: 
+                wait_progress(resp, self.site.session)
+            err_value = err.value[0]
+            err_value["type"] == "script:validation"
+            err_value["description"] == "Errors processing the script"
+            err_value["resolutions"][0]["message"] == "The following subvariables: bird, cat, dog exist in multiple arrays: pets, pets_2"
+        finally:
+            ds.delete()
+            project.delete()
+
+
+class TestDatasetScripts(BaseIntegrationTestCase):
     def _create_ds(self):
         ds = self.site.datasets.create(as_entity({"name": "test_script"})).refresh()
         variable = ds.variables.create(
@@ -24,17 +147,19 @@ def _create_ds(self):
 
     def test_execute(self):
         ds, variable = self._create_ds()
-        scrunch_dataset = get_mutable_dataset(ds.body.id, self.site, editor=True)
-        script = """
-        RENAME pk TO varA;
+        try:
+            scrunch_dataset = get_mutable_dataset(ds.body.id, self.site, editor=True)
+            script = """
+            RENAME pk TO varA;
 
-        CHANGE TITLE IN varA WITH "Variable A";
-        """
-        scrunch_dataset.scripts.execute(script)
-        variable.refresh()
-        assert variable.body["alias"] == "varA"
-        assert variable.body["name"] == "Variable A"
-        ds.delete()
+            CHANGE TITLE IN varA WITH "Variable A";
+            """
+            scrunch_dataset.scripts.execute(script)
+            variable.refresh()
+            assert variable.body["alias"] == "varA"
+            assert variable.body["name"] == "Variable A"
+        finally:
+            ds.delete()
 
     def test_handle_error(self):
         ds, variable = self._create_ds()
@@ -79,23 +204,25 @@ def test_revert_script(self):
         assert variable.body["name"] == "pk"
         ds.delete()
 
+    @pytest.mark.skip(reason="Collapse is 504ing in the server.")
     def test_fetch_all_and_collapse(self):
-        raise self.skipTest("Collapse is 504ing in the server.")
         ds, variable = self._create_ds()
-        scrunch_dataset = get_mutable_dataset(ds.body.id, self.site)
-        s1 = "RENAME pk TO varA;"
-        s2 = 'CHANGE TITLE IN varA WITH "Variable A";'
+        try:
+            scrunch_dataset = get_mutable_dataset(ds.body.id, self.site)
+            s1 = "RENAME pk TO varA;"
+            s2 = 'CHANGE TITLE IN varA WITH "Variable A";'
 
-        scrunch_dataset.scripts.execute(s1)
-        scrunch_dataset.scripts.execute(s2)
+            scrunch_dataset.scripts.execute(s1)
+            scrunch_dataset.scripts.execute(s2)
 
-        r = scrunch_dataset.scripts.all()
-        assert len(r) == 2
-        assert r[0].body["body"] == s1
-        assert r[1].body["body"] == s2
+            r = scrunch_dataset.scripts.all()
+            assert len(r) == 2
+            assert r[0].body["body"] == s1
+            assert r[1].body["body"] == s2
 
-        scrunch_dataset.scripts.collapse()
+            scrunch_dataset.scripts.collapse()
 
-        r = scrunch_dataset.scripts.all()
-        assert len(r) == 1
-        ds.delete()
+            r = scrunch_dataset.scripts.all()
+            assert len(r) == 1
+        finally: 
+            ds.delete()

scrunch/accounts.py

@@ -2,7 +2,7 @@
 
 import pycrunch
 from scrunch.helpers import shoji_view_wrapper
-from scrunch.scripts import ScriptExecutionError
+from scrunch.scripts import ScriptExecutionError, SystemScript
 from scrunch.connections import _default_connection
 from scrunch.datasets import Project
 
@@ -50,20 +50,11 @@ def current_account(cls, connection=None):
         act_res = site_root.account
         return cls(act_res)
 
-    def execute(self, script_body):
-        """
-        Will run a system script on this account.
-
-        System scripts do not have a return value. If they execute correctly
-        they'll finish silently. Otherwise an error will raise.
-        """
+    def execute(self, script_body, strict_subvariable_syntax=None):
+        """Will run a system script on this account."""
         # The account execution endpoint is a shoji:view
-        payload = shoji_view_wrapper(script_body)
-        try:
-            self.resource.execute.post(payload)
-        except pycrunch.ClientError as err:
-            resolutions = err.args[2]["resolutions"]
-            raise ScriptExecutionError(err, resolutions)
+        system_script = SystemScript(self.resource)
+        return system_script.execute(script_body, strict_subvariable_syntax)
 
     @property
     def projects(self):

scrunch/datasets.py

@@ -33,13 +33,12 @@
 from scrunch.expressions import parse_expr, prettify, process_expr
 from scrunch.folders import DatasetFolders
 from scrunch.views import DatasetViews
-from scrunch.scripts import DatasetScripts, ScriptExecutionError
+from scrunch.scripts import DatasetScripts, SystemScript
 from scrunch.helpers import (ReadOnly, _validate_category_rules, abs_url,
                              case_expr, download_file, shoji_entity_wrapper,
                              subvar_alias, validate_categories, shoji_catalog_wrapper,
                              get_else_case, else_case_not_selected, SELECTED_ID,
                              NOT_SELECTED_ID, NO_DATA_ID, valid_categorical_date,
-                             shoji_view_wrapper, make_unique,
                              generate_subvariable_codes)
 from scrunch.order import DatasetVariablesOrder, ProjectDatasetsOrder
 from scrunch.subentity import Deck, Filter, Multitable
@@ -454,25 +453,11 @@ def __repr__(self):
     def __str__(self):
         return self.name
 
-    def execute(self, script_body):
-        """
-        Will run a system script on this project.
-
-        System scripts do not have a return value. If they execute correctly
-        they'll finish silently. Otherwise an error will raise.
-        """
+    def execute(self, script_body, strict_subvariable_syntax=None):
+        """Will run a system script on this project."""
         # The project execution endpoint is a shoji:view
-        payload = shoji_view_wrapper(script_body)
-        if "run" in self.resource.views:
-            exc_res = self.resource.run  # Backwards compat og API
-        else:
-            exc_res = self.resource.execute
-
-        try:
-            exc_res.post(payload)
-        except pycrunch.ClientError as err:
-            resolutions = err.args[2]["resolutions"]
-            raise ScriptExecutionError(err, resolutions)
+        system_script = SystemScript(self.resource)
+        return system_script.execute(script_body, strict_subvariable_syntax)
 
     @property
     def members(self):
@@ -2224,7 +2209,7 @@ def export(self, path, format='csv', filter=None, variables=None,
                     % (format, ','.join(k))
                 )
         if 'var_label_field' in options \
-                and not options['var_label_field'] in ('name', 'description'):
+                and options['var_label_field'] not in ('name', 'description'):
             raise ValueError(
                 'The "var_label_field" export option must be either "name" '
                 'or "description".'

scrunch/helpers.py

@@ -212,7 +212,7 @@ def _validate_category_rules(categories, rules):
 def shoji_view_wrapper(value, **kwargs):
     """
     receives a dictionary and wraps its content on a body keyed dictionary
-    with the appropiate shoji:<class> attribute
+    with the appropriate shoji:<class> attribute
     """
     payload = {
         'element': 'shoji:view',
@@ -225,7 +225,7 @@ def shoji_view_wrapper(value, **kwargs):
 def shoji_entity_wrapper(body, **kwargs):
     """
     receives a dictionary and wraps its content on a body keyed dictionary
-    with the appropiate shoji:<class> attribute
+    with the appropriate shoji:<class> attribute
     """
     payload = {
         'element': 'shoji:entity',
@@ -238,7 +238,7 @@ def shoji_entity_wrapper(body, **kwargs):
 def shoji_catalog_wrapper(index, **kwargs):
     """
     receives a dictionary and wraps its content on a body keyed dictionary
-    with the appropiate shoji:<class> attribute
+    with the appropriate shoji:<class> attribute
     """
     payload = {
         'element': 'shoji:catalog',