From 56a217231386bd59cd72bd4c98faa2ce112c38f5 Mon Sep 17 00:00:00 2001 From: "kodiakhq[bot]" <49736102+kodiakhq[bot]@users.noreply.github.com> Date: Mon, 23 Jan 2023 13:41:18 +0000 Subject: [PATCH] Fix tutorial deployment to GitHub Pages (#4656) Fixes #4651 Description of changes: - use cloud providers to fetch JavaScript dependencies - cleanup test cases involved in the tutorial toolchain --- maintainer/CI/jupyter_warnings.py | 28 +++- requirements.txt | 1 + testsuite/scripts/CMakeLists.txt | 4 +- testsuite/scripts/importlib_wrapper.py | 132 ------------------ testsuite/scripts/test_importlib_wrapper.py | 62 -------- .../test_maintainer_CI_jupyter_warnings.py | 15 +- 6 files changed, 35 insertions(+), 207 deletions(-) diff --git a/maintainer/CI/jupyter_warnings.py b/maintainer/CI/jupyter_warnings.py index 9b1b6b3a639..520754b73ec 100755 --- a/maintainer/CI/jupyter_warnings.py +++ b/maintainer/CI/jupyter_warnings.py @@ -33,7 +33,7 @@ sphinx_docs = {} -def detect_invalid_urls(nb, sphinx_root='.'): +def detect_invalid_urls(nb, build_root='.', html_exporter=None): ''' Find all links. Check that links to the Sphinx documentation are valid (the target HTML files exist and contain the anchors). These links are @@ -44,7 +44,12 @@ def detect_invalid_urls(nb, sphinx_root='.'): Parameters ---------- nb: :obj:`nbformat.notebooknode.NotebookNode` - Jupyter notebook to process + Jupyter notebook to process. + build_root: :obj:`str` + Path to the ESPResSo build directory. The Sphinx files will be + searched in :file:`doc/sphinx/html`. + html_exporter: :obj:`nbformat.HTMLExporter` + Custom NB convert HTML exporter. Returns ------- @@ -52,7 +57,8 @@ def detect_invalid_urls(nb, sphinx_root='.'): List of warnings formatted as strings. ''' # convert notebooks to HTML - html_exporter = nbconvert.HTMLExporter() + if html_exporter is None: + html_exporter = nbconvert.HTMLExporter() html_exporter.template_name = 'classic' html_string = html_exporter.from_notebook_node(nb)[0] # parse HTML @@ -60,7 +66,7 @@ def detect_invalid_urls(nb, sphinx_root='.'): root = lxml.etree.fromstring(html_string, parser=html_parser) # process all links espressomd_website_root = 'https://espressomd.github.io/doc/' - sphinx_html_root = pathlib.Path(sphinx_root) / 'doc' / 'sphinx' / 'html' + sphinx_html_root = pathlib.Path(build_root) / 'doc' / 'sphinx' / 'html' broken_links = [] for link in root.xpath('//a'): url = link.attrib.get('href', '') @@ -76,7 +82,7 @@ def detect_invalid_urls(nb, sphinx_root='.'): basename = url.split(espressomd_website_root, 1)[1] filepath = sphinx_html_root / basename if not filepath.is_file(): - broken_links.append(f'{url} does not exist') + broken_links.append(f'"{url}" does not exist') continue # check anchor exists if anchor is not None: @@ -86,19 +92,27 @@ def detect_invalid_urls(nb, sphinx_root='.'): doc = sphinx_docs[filepath] nodes = doc.xpath(f'//*[@id="{anchor}"]') if not nodes: - broken_links.append(f'{url} has no anchor "{anchor}"') + broken_links.append(f'"{url}" has no anchor "{anchor}"') elif url.startswith('#'): # check anchor exists anchor = url[1:] nodes = root.xpath(f'//*[@id="{anchor}"]') if not nodes: broken_links.append(f'notebook has no anchor "{anchor}"') + elif url.startswith('file:///'): + broken_links.append(f'"{url}" is an absolute path to a local file') + for link in root.xpath('//script'): + url = link.attrib.get('src', '') + if url.startswith('file:///'): + broken_links.append(f'"{url}" is an absolute path to a local file') return broken_links if __name__ == '__main__': error_code = 0 - for nb_filepath in sorted(pathlib.Path().glob('doc/tutorials/*/*.ipynb')): + nb_filepaths = sorted(pathlib.Path().glob('doc/tutorials/*/*.ipynb')) + assert len(nb_filepaths) != 0, 'no Jupyter notebooks could be found!' + for nb_filepath in nb_filepaths: with open(nb_filepath, encoding='utf-8') as f: nb = nbformat.read(f, as_version=4) issues = detect_invalid_urls(nb) diff --git a/requirements.txt b/requirements.txt index 52c690e1fb7..ee273a89f50 100644 --- a/requirements.txt +++ b/requirements.txt @@ -18,6 +18,7 @@ sphinx>=2.3.0,!=3.0.0 sphinx-toggleprompt==0.0.5 sphinxcontrib-bibtex>=2.4.1 # jupyter dependencies +nbconvert==6.4.5 jupyter_contrib_nbextensions==0.5.1 tqdm>=4.30.0 # pep8 and its dependencies diff --git a/testsuite/scripts/CMakeLists.txt b/testsuite/scripts/CMakeLists.txt index 367d01a0e28..a00b6de1999 100644 --- a/testsuite/scripts/CMakeLists.txt +++ b/testsuite/scripts/CMakeLists.txt @@ -20,9 +20,9 @@ set(TEST_FILE_CONFIGURED_IMPORTLIB_WRAPPER ${CMAKE_CURRENT_BINARY_DIR}/test_importlib_wrapper.py) configure_file(importlib_wrapper.py - ${CMAKE_CURRENT_BINARY_DIR}/importlib_wrapper.py) + ${CMAKE_CURRENT_BINARY_DIR}/importlib_wrapper.py COPYONLY) configure_file(test_importlib_wrapper.py - ${TEST_FILE_CONFIGURED_IMPORTLIB_WRAPPER}) + ${TEST_FILE_CONFIGURED_IMPORTLIB_WRAPPER} COPYONLY) macro(PYTHON_SCRIPTS_TEST) cmake_parse_arguments(TEST "" "FILE;SUFFIX;TYPE" "DEPENDENCIES;LABELS" diff --git a/testsuite/scripts/importlib_wrapper.py b/testsuite/scripts/importlib_wrapper.py index 4076beeb9a5..8066ff08c10 100644 --- a/testsuite/scripts/importlib_wrapper.py +++ b/testsuite/scripts/importlib_wrapper.py @@ -237,138 +237,6 @@ def substitute_variable_values(code, strings_as_is=False, keep_original=True, return "\n".join(lines) -class GetPrngSeedEspressomdSystem(ast.NodeVisitor): - """ - Find all assignments of :class:`espressomd.system.System` in the global - namespace. Assignments made in classes or function raise an error. Detect - random seed setup of the numpy PRNG. - """ - - def __init__(self): - self.numpy_random_aliases = set() - self.es_system_aliases = set() - self.variable_system_aliases = set() - self.numpy_seeds = [] - self.abort_message = None - self.error_msg_multi_assign = "Cannot parse {} in a multiple assignment (line {})" - - def visit_Import(self, node): - # get system aliases - for child in node.names: - if child.name == "espressomd.system.System": - name = (child.asname or child.name) - self.es_system_aliases.add(name) - elif child.name == "espressomd.system": - name = (child.asname or child.name) - self.es_system_aliases.add(name + ".System") - elif child.name == "espressomd.System": - name = (child.asname or child.name) - self.es_system_aliases.add(name) - elif child.name == "espressomd": - name = (child.asname or "espressomd") - self.es_system_aliases.add(name + ".system.System") - self.es_system_aliases.add(name + ".System") - elif child.name == "numpy.random": - name = (child.asname or child.name) - self.numpy_random_aliases.add(name) - elif child.name == "numpy": - name = (child.asname or "numpy") - self.numpy_random_aliases.add(name + ".random") - - def visit_ImportFrom(self, node): - # get system aliases - for child in node.names: - if node.module == "espressomd" and child.name == "system": - name = (child.asname or child.name) - self.es_system_aliases.add(name + ".System") - elif node.module == "espressomd" and child.name == "System": - self.es_system_aliases.add(child.asname or child.name) - elif node.module == "espressomd.system" and child.name == "System": - self.es_system_aliases.add(child.asname or child.name) - elif node.module == "numpy" and child.name == "random": - self.numpy_random_aliases.add(child.asname or child.name) - elif node.module == "numpy.random": - self.numpy_random_aliases.add(child.asname or child.name) - - def is_es_system(self, child): - if hasattr(child, "value"): - if hasattr(child.value, "value") and hasattr( - child.value.value, "id"): - if (child.value.value.id + "." + child.value.attr + - "." + child.attr) in self.es_system_aliases: - return True - else: - if hasattr(child.value, "id") and (child.value.id + "." + - child.attr) in self.es_system_aliases: - return True - elif isinstance(child, ast.Call): - if hasattr(child, "id") and child.func.id in self.es_system_aliases: - return True - elif hasattr(child, "func") and hasattr(child.func, "value") and ( - hasattr(child.func.value, "value") and - (child.func.value.value.id + "." + - child.func.value.attr + "." + child.func.attr) - or (child.func.value.id + "." + child.func.attr)) in self.es_system_aliases: - return True - elif hasattr(child, "func") and hasattr(child.func, "id") and child.func.id in self.es_system_aliases: - return True - elif hasattr(child, "id") and child.id in self.es_system_aliases: - return True - return False - - def detect_es_system_instances(self, node): - varname = None - for target in node.targets: - if isinstance(target, ast.Name): - if hasattr(target, "id") and hasattr(node.value, "func") and \ - self.is_es_system(node.value.func): - varname = target.id - elif isinstance(target, ast.Tuple): - value = node.value - if (isinstance(value, ast.Tuple) or isinstance(value, ast.List)) \ - and any(map(self.is_es_system, node.value.elts)): - raise AssertionError(self.error_msg_multi_assign.format( - "espressomd.System", node.lineno)) - if varname is not None: - assert len(node.targets) == 1, self.error_msg_multi_assign.format( - "espressomd.System", node.lineno) - assert self.abort_message is None, \ - "Cannot process espressomd.System assignments in " + self.abort_message - self.variable_system_aliases.add(varname) - - def detect_np_random_expr_seed(self, node): - if hasattr(node.value, "func") and hasattr(node.value.func, "value") \ - and (hasattr(node.value.func.value, "id") and node.value.func.value.id in self.numpy_random_aliases - or hasattr(node.value.func.value, "value") - and hasattr(node.value.func.value.value, "id") - and hasattr(node.value.func.value, "attr") - and (node.value.func.value.value.id + "." + - node.value.func.value.attr) in self.numpy_random_aliases - ) and node.value.func.attr == "seed": - self.numpy_seeds.append(node.lineno) - - def visit_Assign(self, node): - self.detect_es_system_instances(node) - - def visit_Expr(self, node): - self.detect_np_random_expr_seed(node) - - def visit_ClassDef(self, node): - self.abort_message = "class definitions" - ast.NodeVisitor.generic_visit(self, node) - self.abort_message = None - - def visit_FunctionDef(self, node): - self.abort_message = "function definitions" - ast.NodeVisitor.generic_visit(self, node) - self.abort_message = None - - def visit_AsyncFunctionDef(self, node): - self.abort_message = "function definitions" - ast.NodeVisitor.generic_visit(self, node) - self.abort_message = None - - def delimit_statements(code): """ For every Python statement, map the line number where it starts to the diff --git a/testsuite/scripts/test_importlib_wrapper.py b/testsuite/scripts/test_importlib_wrapper.py index f325ff2dd40..7c64a4b8ec7 100644 --- a/testsuite/scripts/test_importlib_wrapper.py +++ b/testsuite/scripts/test_importlib_wrapper.py @@ -237,68 +237,6 @@ def test_matplotlib_pyplot_visitor(self): self.assertEqual(v.matplotlib_backend_linenos, [17, 18]) self.assertEqual(v.ipython_magic_linenos, [19]) - def test_prng_seed_espressomd_system_visitor(self): - import_stmt = [ - 'sys0 = espressomd.System() # nothing: espressomd not imported', - 'import espressomd as es1', - 'import espressomd.system as es2', - 'import espressomd.System as s1, espressomd.system.System as s2', - 'from espressomd import System as s3, electrostatics', - 'from espressomd.system import System as s4', - 'from espressomd import system as es5', - 'sys1 = es1.System()', - 'sys2 = es1.system.System()', - 'sys3 = es2.System()', - 'sys4 = s1()', - 'sys5 = s2()', - 'sys6 = s3()', - 'sys7 = s4()', - 'sys8 = es5.System()', - 'import numpy as np', - 'import numpy.random as npr1', - 'from numpy import random as npr2', - 'np.random.seed(1)', - 'npr1.seed(1)', - 'npr2.seed(1)', - ] - tree = ast.parse('\n'.join(import_stmt)) - v = iw.GetPrngSeedEspressomdSystem() - v.visit(tree) - # find all aliases for espressomd.system.System - expected_es_sys_aliases = {'es1.System', 'es1.system.System', - 'es2.System', 's1', 's2', 's3', 's4', - 'es5.System'} - self.assertEqual(v.es_system_aliases, expected_es_sys_aliases) - # find all variables of type espressomd.system.System - expected_es_sys_objs = set(f'sys{i}' for i in range(1, 9)) - self.assertEqual(v.variable_system_aliases, expected_es_sys_objs) - # find all seeds setup - self.assertEqual(v.numpy_seeds, [19, 20, 21]) - # test exceptions - str_es_sys_list = [ - 'import espressomd.System', - 'import espressomd.system.System', - 'from espressomd import System', - 'from espressomd.system import System', - ] - exception_stmt = [ - 's, var = System(), 5', - 'class A:\n\ts = System()', - 'def A():\n\ts = System()', - ] - for str_es_sys in str_es_sys_list: - for str_stmt in exception_stmt: - for alias in ['', ' as EsSystem']: - str_import = str_es_sys + alias + '\n' - alias = str_import.split()[-1] - code = str_import + str_stmt.replace('System', alias) - v = iw.GetPrngSeedEspressomdSystem() - tree = ast.parse(code) - err_msg = v.__class__.__name__ + \ - ' should fail on ' + repr(code) - with self.assertRaises(AssertionError, msg=err_msg): - v.visit(tree) - def test_delimit_statements(self): lines = [ 'a = 1 # NEWLINE becomes NL after a comment', diff --git a/testsuite/scripts/utils/test_maintainer_CI_jupyter_warnings.py b/testsuite/scripts/utils/test_maintainer_CI_jupyter_warnings.py index b2d288da830..6abc94d7ce6 100644 --- a/testsuite/scripts/utils/test_maintainer_CI_jupyter_warnings.py +++ b/testsuite/scripts/utils/test_maintainer_CI_jupyter_warnings.py @@ -1,3 +1,4 @@ +# # Copyright (C) 2020-2022 The ESPResSo project # # This file is part of ESPResSo. @@ -14,9 +15,11 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . +# import sys import nbformat +import nbconvert import importlib import unittest as ut @@ -36,18 +39,22 @@ class Test(ut.TestCase): invalid: https://espressomd.github.io/doc/index.html#unknown_anchor invalid: https://espressomd.github.io/doc/unknown_file.html invalid: [footnote 1](#unknown-footnote-1) +invalid: [resource](file:///home/espresso/image.png) ''' def test_detect_invalid_urls(self): + html_exporter = nbconvert.HTMLExporter() nb = nbformat.v4.new_notebook() cell_md = nbformat.v4.new_markdown_cell(source=self.cell_md_src) nb['cells'].append(cell_md) ref_issues = [ - 'https://espressomd.github.io/doc/index.html has no anchor "unknown_anchor"', - 'https://espressomd.github.io/doc/unknown_file.html does not exist', - 'notebook has no anchor "unknown-footnote-1"' + '"https://espressomd.github.io/doc/index.html" has no anchor "unknown_anchor"', + '"https://espressomd.github.io/doc/unknown_file.html" does not exist', + 'notebook has no anchor "unknown-footnote-1"', + '"file:///home/espresso/image.png" is an absolute path to a local file', ] - issues = module.detect_invalid_urls(nb, '@CMAKE_BINARY_DIR@') + issues = module.detect_invalid_urls( + nb, build_root='@CMAKE_BINARY_DIR@', html_exporter=html_exporter) self.assertEqual(issues, ref_issues)