Merge branch 't/28925/modify_find_python_sources__clean_stale_files_t…

…o_support_modularization_of_sagelib_by_native_namespace_packages__pep_420_' into t/29864/modularization-sage-tdlib
sagemath · Sep 10, 2020 · a315ba6 · a315ba6
2 parents 5ec24db + a967aca
commit a315ba6
Show file tree

Hide file tree

Showing 16 changed files with 77 additions and 17 deletions.
diff --git a/src/sage/interfaces/__init__.py → src/nonamespace b/src/sage/interfaces/__init__.py → src/nonamespace
diff --git a/src/sage/libs/__init__.py → src/sage/ext_data/nonamespace b/src/sage/libs/__init__.py → src/sage/ext_data/nonamespace
diff --git a/src/sage/graphs/__init__.py b/src/sage/graphs/__init__.py
diff --git a/src/sage/graphs/graph_decompositions/__init__.py b/src/sage/graphs/graph_decompositions/__init__.py
diff --git a/src/sage/graphs/graph_decompositions/namespace b/src/sage/graphs/graph_decompositions/namespace
diff --git a/src/sage/graphs/namespace b/src/sage/graphs/namespace
diff --git a/src/sage/interfaces/namespace b/src/sage/interfaces/namespace
diff --git a/src/sage/libs/namespace b/src/sage/libs/namespace
diff --git a/src/sage/matrix/__init__.py b/src/sage/matrix/__init__.py
diff --git a/src/sage/matrix/namespace b/src/sage/matrix/namespace
diff --git a/src/sage/numerical/backends/__init__.py b/src/sage/numerical/backends/__init__.py
diff --git a/src/sage/numerical/backends/namespace b/src/sage/numerical/backends/namespace
diff --git a/src/sage/tests/books/nonamespace b/src/sage/tests/books/nonamespace
diff --git a/src/sage_setup/command/sage_install.py b/src/sage_setup/command/sage_install.py
@@ -8,6 +8,7 @@
 from distutils.command.install import install
 
 class sage_install(install):
+
     def run(self):
         install.run(self)
         self.install_kernel_spec()
@@ -31,6 +32,14 @@ def install_kernel_spec(self):
         # the install_data directory for installing our Jupyter files.
         SageKernelSpec.update(prefix=self.install_data)
 
+class sage_install_and_clean(sage_install):
+
+    def run(self):
+        sage_install.run(self)
+        t = time.time()
+        self.clean_stale_files()
+        log.info('Finished cleaning, time: %.2f seconds.' % (time.time() - t))
+
     def clean_stale_files(self):
         """
         Remove stale installed files.

diff --git a/src/sage_setup/find.py b/src/sage_setup/find.py
@@ -80,7 +80,8 @@ def find_python_sources(src_dir, modules=['sage'], distributions=None):
     - the list of package names (corresponding to directories with
       ``__init__.py``),
 
-    - Python module names (corresponding to other ``*.py`` files).
+    - Python module names (corresponding to other ``*.py`` files except for
+      those below directories with a file named ``nonamespace`` in it).
 
     - Cython extensions (corresponding to ``*.pyx`` files).
 
@@ -112,6 +113,26 @@ def find_python_sources(src_dir, modules=['sage'], distributions=None):
         sage: ['sage.doctest.tests' in L for L in (py_packages, py_modules)]
         [False, False]
 
+    Native namespace package (no ``__init__.py``, PEP 420)::
+
+        sage: ['sage.graphs.graph_decompositions' in L for L in (py_packages, py_modules)]
+        [False, False]
+
+    Python module in a native namespace package::
+
+        sage: ['sage.graphs.graph_decompositions.modular_decomposition' in L for L in (py_packages, py_modules)]
+        [False, True]
+
+    Subdirectory marked with a ``nonamespace`` file::
+
+        sage: ['sage.extdata' in L for L in (py_packages, py_modules)]
+        [False, False]
+
+    Python file (not module) below a directory with a ``nonamespace`` file::
+
+        sage: ['sage.ext_data.nbconvert.postprocess' in L for L in (py_packages, py_modules)]
+        [False, False]
+
     Filtering by distribution (distutils package)::
 
         sage: find_python_sources(SAGE_SRC, distributions=['sage-tdlib'])
@@ -149,7 +170,10 @@ def find_python_sources(src_dir, modules=['sage'], distributions=None):
                     # Ordinary package.
                     if distributions is None or '' in distributions:
                         python_packages.append(package)
-                else:
+                if os.path.exists(os.path.join(dirpath, 'nonamespace')):
+                    # Marked as "not a namespace package"
+                    # (similar to nodoctest in sage.doctest.control)
+                    dirnames.clear()
                     continue
 
                 def is_in_distributions(filename):
@@ -172,6 +196,35 @@ def is_in_distributions(filename):
         os.chdir(cwd)
     return python_packages, python_modules, cython_modules
 
+def is_package_or_namespace_package_dir(dirpath):
+    """
+    True when ``dirpath`` is a regular or namespace package.
+
+    EXAMPLES::
+
+        sage: from sage.env import SAGE_SRC
+        sage: from sage_setup.find import is_package_or_namespace_package_dir
+        sage: is_package_or_namespace_package_dir(SAGE_SRC)
+        False
+
+    An ordinary package::
+
+        sage: is_package_or_namespace_package_dir(os.path.join(SAGE_SRC, 'sage', 'structure'))
+        True
+
+    A namespace package::
+
+        sage: is_package_or_namespace_package_dir(os.path.join(SAGE_SRC, 'sage', 'numerical', 'backends')
+        True
+
+    """
+    PACKAGE_FILES = ("__init__.py", "__init__.pyc", "__init__.pyx", "__init__.pxd")
+    for filename in PACKAGE_FILES:
+        path = os.path.join(dirpath, filename)
+        if os.path.exists(path):
+            return True
+    return os.path.exists(os.path.join(dirpath, 'namespace'))
+
 def find_extra_files(src_dir, modules, cythonized_dir, special_filenames=[]):
     """
     Find all extra files which should be installed.
@@ -280,7 +333,11 @@ def installed_files_by_module(site_packages, modules=('sage',)):
         sage: f2
         'sage/structure/....pyc'
 
-    This takes about 30ms with warm cache:
+    Namespace packages::
+
+        sage: files_by_module['sage.graphs.graph_decompositions']
+
+    This takes about 30ms with warm cache::
 
         sage: timeit('installed_files_by_module(site_packages)',       # random output
         ....:        number=1, repeat=1)

diff --git a/src/setup.py b/src/setup.py
@@ -70,16 +70,20 @@
 
 log.warn('distributions = {0}'.format(distributions))
 
-from sage_setup.find import find_python_sources
+from sage_setup.find import find_python_sources, is_package_or_namespace_package_dir
 python_packages, python_modules, cython_modules = find_python_sources(
     SAGE_SRC, ['sage', 'sage_setup'], distributions=distributions)
 
 log.debug('python_packages = {0}'.format(python_packages))
 
 print("Discovered Python/Cython sources, time: %.2f seconds." % (time.time() - t))
 
+import Cython.Build.Dependencies
+import Cython.Build.Cythonize
+import Cython.Utils
+Cython.Utils.is_package_dir = Cython.Build.Cythonize.is_package_dir = Cython.Build.Dependencies.is_package_dir = is_package_or_namespace_package_dir
 
-from sage_setup.command.sage_install import sage_install
+from sage_setup.command.sage_install import sage_install_and_clean
 
 #########################################################
 ### Distutils
@@ -93,6 +97,7 @@
       author_email= 'https://groups.google.com/group/sage-support',
       url         = 'https://www.sagemath.org',
       packages    = python_packages,
+      py_modules  = python_modules,
       package_data = {
           'sage.libs.gap': ['sage.gaprc'],
           'sage.interfaces': ['sage-maxima.lisp'],
@@ -177,5 +182,5 @@
       cmdclass = dict(build=sage_build,
                       build_cython=sage_build_cython,
                       build_ext=sage_build_ext,
-                      install=sage_install),
+                      install=sage_install_and_clean),
       ext_modules = cython_modules)