From ee8f8933e6b775cd5cfaf8e87a2893838f86d201 Mon Sep 17 00:00:00 2001 From: Maik Riechert Date: Wed, 2 Sep 2020 16:02:13 +0000 Subject: [PATCH 1/2] Update collections of JS modules in one proposal. Fixes #1479. --- python/ccf/proposal_generator.py | 35 ++++++++++++++++++++++ src/runtime_config/gov.lua | 23 ++++++++++++++ tests/modules.py | 51 +++++++++++++++++++++++++------- 3 files changed, 98 insertions(+), 11 deletions(-) diff --git a/python/ccf/proposal_generator.py b/python/ccf/proposal_generator.py index 67e454b042e3..491a11440071 100644 --- a/python/ccf/proposal_generator.py +++ b/python/ccf/proposal_generator.py @@ -5,6 +5,7 @@ import collections import inspect import json +import glob import os import sys from pathlib import PurePosixPath @@ -294,6 +295,40 @@ def remove_module(module_name: str, **kwargs): return build_proposal("remove_module", module_name, **kwargs) +@cli_proposal +def update_modules(module_name_prefix: str, modules_path: Optional[str], **kwargs): + LOG.debug("Generating update_modules proposal") + + # Validate module name prefix + module_name_prefix_ = PurePosixPath(module_name_prefix) + if not module_name_prefix_.is_absolute(): + raise ValueError("module name prefix must be an absolute path") + if any(folder in [".", ".."] for folder in module_name_prefix_.parents): + raise ValueError("module name prefix must not contain . or .. components") + if not module_name_prefix.endswith("/"): + raise ValueError("module name prefix must end with /") + + # Read module files and build relative module names + modules = [] + if modules_path: + for path in glob.glob(f"{modules_path}/**/*.js", recursive=True): + rel_module_name = os.path.relpath(path, modules_path) + rel_module_name = rel_module_name.replace('\\', '/') # Windows support + with open(path) as f: + js = f.read() + modules.append({"rel_name": rel_module_name, "module": {"js": js}}) + + proposal_args = {"prefix": module_name_prefix, "modules": modules} + + return build_proposal("update_modules", proposal_args, **kwargs) + + +@cli_proposal +def remove_modules(module_name_prefix: str, **kwargs): + LOG.debug("Generating update_modules proposal (remove only)") + return update_modules(module_name_prefix, modules_path=None) + + @cli_proposal def trust_node(node_id: int, **kwargs): return build_proposal("trust_node", node_id, **kwargs) diff --git a/src/runtime_config/gov.lua b/src/runtime_config/gov.lua index 12dda530ecaf..b497be6ba1ec 100644 --- a/src/runtime_config/gov.lua +++ b/src/runtime_config/gov.lua @@ -97,4 +97,27 @@ return { end end return true]], + + update_modules = [[ + tables, args = ... + function starts_with(str, start) + return str:sub(1, #start) == start + end + function remove_modules_with_prefix(prefix) + tables["ccf.modules"]:foreach(function(module_name, _) + if starts_with(module_name, prefix) then + tables["ccf.modules"]:remove(module_name) + end + end) + end + function add_modules_with_prefix(prefix, modules) + for _, module in pairs(modules) do + module_name = prefix .. module.rel_name + tables["ccf.modules"]:put(module_name, module.module) + end + end + remove_modules_with_prefix(args.prefix) + add_modules_with_prefix(args.prefix, args.modules) + return true + ]] } diff --git a/tests/modules.py b/tests/modules.py index 6e1ca7dfcf38..26cd4c5f4572 100644 --- a/tests/modules.py +++ b/tests/modules.py @@ -4,7 +4,6 @@ import http import subprocess import os -import glob import infra.network import infra.path import infra.proc @@ -18,6 +17,7 @@ THIS_DIR = os.path.dirname(__file__) +MODULE_PREFIX_1 = "/app/" MODULE_PATH_1 = "/app/foo.js" MODULE_RETURN_1 = "Hello world!" MODULE_CONTENT_1 = f""" @@ -85,7 +85,7 @@ def make_module_set_proposal(path, content, network): def test_module_set_and_remove(network, args): primary, _ = network.find_nodes() - LOG.info("Member makes a module update proposal") + LOG.info("Member makes a module set proposal") make_module_set_proposal(MODULE_PATH_1, MODULE_CONTENT_1, network) with primary.client( @@ -110,6 +110,35 @@ def test_module_set_and_remove(network, args): return network +@reqs.description("Test prefix-based modules remove") +def test_modules_remove(network, args): + primary, _ = network.find_nodes() + + LOG.info("Member makes a module set proposal") + make_module_set_proposal(MODULE_PATH_1, MODULE_CONTENT_1, network) + + with primary.client( + f"member{network.consortium.get_any_active_member().member_id}" + ) as c: + r = c.post("/gov/read", {"table": "ccf.modules", "key": MODULE_PATH_1}) + assert r.status_code == http.HTTPStatus.OK, r.status_code + assert r.body["js"] == MODULE_CONTENT_1, r.body + + LOG.info("Member makes a prefix-based modules remove proposal") + proposal_body, _ = ccf.proposal_generator.remove_modules(MODULE_PREFIX_1) + proposal = network.consortium.get_any_active_member().propose( + primary, proposal_body + ) + network.consortium.vote_using_majority(primary, proposal) + + with primary.client( + f"member{network.consortium.get_any_active_member().member_id}" + ) as c: + r = c.post("/gov/read", {"table": "ccf.modules", "key": MODULE_PATH_1}) + assert r.status_code == http.HTTPStatus.BAD_REQUEST, r.status_code + return network + + @reqs.description("Test module import") def test_module_import(network, args): primary, _ = network.find_nodes() @@ -132,7 +161,7 @@ def test_module_import(network, args): return network -@reqs.description("Test Node.js/npm app") +@reqs.description("Test Node.js/npm app with prefix-based modules update") def test_npm_app(network, args): primary, _ = network.find_nodes() @@ -142,15 +171,14 @@ def test_npm_app(network, args): subprocess.run(["npm", "run", "build"], cwd=app_dir, check=True) LOG.info("Deploying npm app modules") - kv_prefix = "/my-npm-app" + module_name_prefix = "/my-npm-app/" dist_dir = os.path.join(app_dir, "dist") - for module_path in glob.glob(os.path.join(dist_dir, "**", "*.js"), recursive=True): - module_name = os.path.join(kv_prefix, os.path.relpath(module_path, dist_dir)) - proposal_body, _ = ccf.proposal_generator.set_module(module_name, module_path) - proposal = network.consortium.get_any_active_member().propose( - primary, proposal_body - ) - network.consortium.vote_using_majority(primary, proposal) + + proposal_body, _ = ccf.proposal_generator.update_modules(module_name_prefix, dist_dir) + proposal = network.consortium.get_any_active_member().propose( + primary, proposal_body + ) + network.consortium.vote_using_majority(primary, proposal) LOG.info("Deploying endpoint script") with tempfile.NamedTemporaryFile("w") as f: @@ -186,6 +214,7 @@ def run(args): ) as network: network.start_and_join(args) network = test_module_set_and_remove(network, args) + network = test_modules_remove(network, args) network = test_module_import(network, args) network = test_npm_app(network, args) From ecb0cb85c910fc41f8bafdc7c4e6908877876e15 Mon Sep 17 00:00:00 2001 From: Maik Riechert Date: Wed, 2 Sep 2020 16:28:55 +0000 Subject: [PATCH 2/2] formatting --- python/ccf/proposal_generator.py | 2 +- tests/modules.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/python/ccf/proposal_generator.py b/python/ccf/proposal_generator.py index 491a11440071..4c5c01686818 100644 --- a/python/ccf/proposal_generator.py +++ b/python/ccf/proposal_generator.py @@ -313,7 +313,7 @@ def update_modules(module_name_prefix: str, modules_path: Optional[str], **kwarg if modules_path: for path in glob.glob(f"{modules_path}/**/*.js", recursive=True): rel_module_name = os.path.relpath(path, modules_path) - rel_module_name = rel_module_name.replace('\\', '/') # Windows support + rel_module_name = rel_module_name.replace("\\", "/") # Windows support with open(path) as f: js = f.read() modules.append({"rel_name": rel_module_name, "module": {"js": js}}) diff --git a/tests/modules.py b/tests/modules.py index 26cd4c5f4572..4e743126af0c 100644 --- a/tests/modules.py +++ b/tests/modules.py @@ -174,7 +174,9 @@ def test_npm_app(network, args): module_name_prefix = "/my-npm-app/" dist_dir = os.path.join(app_dir, "dist") - proposal_body, _ = ccf.proposal_generator.update_modules(module_name_prefix, dist_dir) + proposal_body, _ = ccf.proposal_generator.update_modules( + module_name_prefix, dist_dir + ) proposal = network.consortium.get_any_active_member().propose( primary, proposal_body )