Merge branch '1.3.x'

.github/workflows/build.yml

@@ -19,34 +19,48 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ubuntu-latest, macos-latest]
-        python-version: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9, "3.10", pypy2, pypy3]
+        os: [ubuntu-22.04, macos-latest]
+        python-version: [3.8, 3.9, "3.10", "3.11", "3.12", "pypy3.9"]
         include:
+          - os: ubuntu-22.04
+            python-version: "2.7"
           - os: windows-latest
             python-version: 3.9
           - os: windows-latest
             python-version: "3.10"
+          - os: windows-latest
+            python-version: "3.11"
+          - os: windows-latest
+            python-version: "3.12"
         exclude:
           - os: macos-latest
             python-version: 3.5
           - os: macos-latest
             python-version: 3.6
           - os: macos-latest
-            python-version: pypy2
-          - os: macos-latest
-            python-version: pypy3
+            python-version: "pypy3.9"
 
 
     steps:
-    - uses: actions/checkout@v2
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
+    - uses: actions/checkout@v3
+    - name: Set up Python ${{ matrix.python-version }} via setup-python
+      if: matrix.python-version != '2.7'
+      uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python-version }}
+    - name: Set up Python ${{ matrix.python-version }} via apt-get
+      if: matrix.python-version == '2.7'
+      run: |
+        set -eux
+        sudo apt-get update
+        sudo apt-get install -y python2 python3-virtualenv
+        virtualenv -p python2 "${{ runner.temp }}/venv"
+        echo "${{ runner.temp }}/venv/bin" >> $GITHUB_PATH
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        python -m pip install coverage flake8
+        # must install all dependencies so the tab modules can be generated
+        python -m pip install coverage flake8 ply setuptools
         python -m pip install -e .
     - name: Lint with flake8
       run: |
@@ -55,9 +69,13 @@ jobs:
       run: |
         python -OO -m unittest calmjs.parse.tests.make_suite
         coverage run --include=src/* -m unittest calmjs.parse.tests.make_suite
+    # Python 3.12 on Windows resulted in MemoryError here, so optional.
+    - name: Coverage report
+      run: |
         coverage report -m
+      continue-on-error: true
     - name: Coveralls
-      if: ${{ matrix.os == 'ubuntu-latest' && matrix.python-version != '2.7' && matrix.python-version != 'pypy2' }}
+      if: ${{ matrix.os == 'ubuntu-22.04' && matrix.python-version != '2.7' && matrix.python-version != 'pypy2' }}
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       run: |

CHANGES.rst

@@ -1,6 +1,16 @@
 Changelog
 =========
 
+1.3.1 - 2023-10-28
+------------------
+
+- Modified existing ``setup.py`` hook from an install hook to a build
+  hook to ensure the generated module files are present.  Should any of
+  those modules are missing and the required dependencies for are not
+  present (i.e. ``ply`` and ``setuptools``), the build will result in a
+  non-zero exit status and the documented error message should reflect
+  which of the required dependencies are missing.
+
 1.3.0 - 2021-10-08
 ------------------
 

MANIFEST.in

@@ -6,3 +6,7 @@ include *.rst
 include LICENSE
 recursive-exclude src *.pyc
 recursive-exclude src *.pyo
+prune .github
+exclude .gitignore
+exclude appveyor.yml
+exclude .flake8

README.rst

@@ -78,9 +78,8 @@ As this package uses |ply|, it requires the generation of optimization
 modules for its lexer.  The wheel distribution of |calmjs.parse| does
 not require this extra step as it contains these pre-generated modules
 for |ply| up to version 3.11 (the latest version available at the time
-of previous release), however the source tarball or if |ply| version
-that is installed lies outside of the supported versions, the following
-caveats will apply.
+of previous release), however the version of |ply| that is installed is
+beyond the supported version, the following caveats will apply.
 
 If a more recent release of |ply| becomes available and the environment
 upgrades to that version, those pre-generated modules may become
@@ -89,26 +88,77 @@ A corrective action can be achieved through a `manual optimization`_
 step if a newer version of |calmjs.parse| is not available, or |ply| may
 be downgraded back to version 3.11 if possible.
 
+Alternatively, install a more recent version of |calmjs.parse| wheel
+that has the most complete set of pre-generated modules built.
+
 Once the package is installed, the installation may be `tested`_ or be
 `used directly`_.
 
-Alternative installation methods (for developers, advanced users)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Manual installation and packaging requirements
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+*This section is for developers and advanced users; contains important
+information for package maintainers for OS distributions (e.g. Linux)
+that will prevent less than ideal experiences for downstream users.*
 
 Development is still ongoing with |calmjs.parse|, for the latest
 features and bug fixes, the development version may be installed through
 git like so:
 
 .. code:: console
 
-    $ pip install git+https://github.com/calmjs/calmjs.parse.git#egg=calmjs.parse
+    $ pip install ply setuptools  # this MUST be done first; see below for reason
+    $ pip install -e git+https://github.com/calmjs/calmjs.parse.git#egg=calmjs.parse
 
-Alternatively, the git repository can be cloned directly and execute
-``python setup.py develop`` while inside the root of the source
-directory.
+Note that all dependencies MUST be pre-installed ``setup.py build`` step
+to run, otherwise the build step required to create the pre-generated
+modules will result in failure.
+
+If |ply| isn't installed:
+
+.. code:: console
+
+    $ python -m pip install -e .
+    ...
+    running egg_info
+    ...
+    WARNING: cannot find distribution for 'ply'; using default value,
+    assuming 'ply==3.11' for pre-generated modules
+    ERROR: cannot find pre-generated modules for the assumed 'ply'
+    version from above and/or cannot `import ply` to build generated
+    modules, aborting build; please either ensure that the source
+    archive containing the pre-generate modules is being used, or that
+    the python package 'ply' is installed and available for import
+    before attempting to use the setup.py to build this package; please
+    refer to the top level README for further details
+
+If ``setuptools`` isn't installed:
+
+.. code:: console
 
-A manual optimization step may need to be performed for platforms and
-systems that do not have utf8 as their default encoding.
+    $ python -m pip install -e .
+    ...
+    running egg_info
+    ...
+    Traceback (most recent call last):
+      ...
+    ModuleNotFoundError: No module named 'pkg_resources'
+
+Naturally, the git repository can be cloned directly and execute
+``python setup.py develop`` while inside the root of the source
+directory; again, both |ply| AND ``setuptools`` MUST already have be
+available for import.
+
+As the git repository does NOT contain any pre-generated modules or
+code, the above message is likely to be seen by developers or distro
+maintainers who are on their first try at interacting with this
+software.  However, the zip archives released on PyPI starting from
+version 1.3.0 do contain these modules fully pre-generated, thus they
+may be used as part of a standard installation step, i.e. without
+requiring |ply| be available for import before usage of the ``setup.py``
+for any purpose.  While the same warning message about |ply| being
+missing may be shown, the pre-generated modules will allow the build
+step to proceed as normal.
 
 Manual optimization
 ~~~~~~~~~~~~~~~~~~~
@@ -714,6 +764,7 @@ Object assignments from a given script file:
 Further details and example usage can be consulted from the various
 docstrings found within the module.
 
+
 Limitations
 -----------
 
@@ -735,6 +786,7 @@ comments.  Likewise, any comments before the ``:`` token in a ternary
 statement will also be discarded as that is the second token consumed
 by the production rule that produces a ``Conditional`` node.
 
+
 Troubleshooting
 ---------------
 
@@ -768,6 +820,59 @@ A workaround helper script is provided, it may be executed like so:
 Further details on this topic may be found in the `manual optimization`_
 section of this document.
 
+WARNING: There are unused tokens on import
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+This indicates that the installation method or source for this package
+being imported isn't optimized.  A quick workaround is to follow the
+instructions at the `manual optimization`_ section of this document to
+ensure these messages are no longer generated (and if this warning
+happens every time the module is imported, it means the symbol tables
+are regenerated every time that happens and this extra computational
+overhead should be corrected through the generation of that optimization
+module).
+
+The optimization modules are included with the wheel release and the
+source release on PyPI, but it is not part of the source repository as
+generated code are never committed.  Should a binary release made by
+a third-party results in this warning upon import, their release should
+be corrected to include the optimization module.
+
+Moreover, there are safeguards in place that prevent this warning from
+being generated for releases made for releases from 1.3.1 onwards by
+a more heavy handed enforcement of this optimization step at build time,
+but persistent (or careless) actors may circumvent this during the build
+process, but official releases made through PyPI should include the
+required optimization for all supported |ply| versions (which are
+versions 3.6 to 3.11, inclusive).
+
+Alternatively, this issue may also occur via usage of ``pyinstaller``
+if the package metadata is not copied for |ply| in versions prior to
+``calmjs.parse-1.3.1`` and will always occur if the hidden imports are
+not declared for those optimization modules.  The following hook should
+may be used to ensure |calmjs.parse| functions correctly in the compiled
+binary:
+
+.. code:: python
+
+    from PyInstaller.utils.hooks import collect_data_files, copy_metadata
+    from calmjs.parse.utils import generate_tab_names
+
+    datas = []
+    datas.extend(collect_data_files("ply"))
+    datas.extend(copy_metadata("ply"))
+    datas.extend(collect_data_files("calmjs.parse"))
+    datas.extend(copy_metadata("calmjs.parse"))
+
+    hiddenimports = []
+    hiddenimports.extend(generate_tab_names('calmjs.parse.parsers.es5'))
+
+    # if running under Python 3 with ply-3.11, above is equivalent to
+    # hiddenimports = [
+    #     "calmjs.parse.parsers.lextab_es5_py3_ply3_11",
+    #     "calmjs.parse.parsers.yacctab_es5_py3_ply3_11",
+    # ]
+
 Slow performance
 ~~~~~~~~~~~~~~~~
 
@@ -781,6 +886,18 @@ this will may require both the token and layout functions not having
 arguments with name collisions, and the new function will take in all
 of those arguments in one go.
 
+ERROR message about import error when trying to install
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+As noted in the error message, the |ply|_ and ``setuptools`` package
+must be installed before attempting to install build the package in the
+situation where the pre-generated modules are missing.  This situation
+may be caused by building directly using the source provided by the
+source code repository, or where there is no matching pre-generated
+module matching with the installed version of |ply|.  Please ensure that
+|ply| is installed and available first before installing from source if
+this error message is sighted.
+
 
 Contribute
 ----------

appveyor.yml

@@ -1,24 +1,21 @@
+image: Visual Studio 2022
+
 environment:
   matrix:
     - PYTHON: "C:\\Python27"
-      nodejs_version: "4.6"
     - PYTHON: "C:\\Python33"
-      nodejs_version: "4.6"
     - PYTHON: "C:\\Python34"
-      nodejs_version: "6.9"
     - PYTHON: "C:\\Python35"
-      nodejs_version: "6.9"
     - PYTHON: "C:\\Python36"
-      nodejs_version: "8"
     - PYTHON: "C:\\Python37"
-      nodejs_version: "10"
     - PYTHON: "C:\\Python38"
-      nodejs_version: "10"
+    - PYTHON: "C:\\Python39-x64"
+    - PYTHON: "C:\\Python310-x64"
+    - PYTHON: "C:\\Python311-x64"
 
 install:
   - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%"
-  - ps: Install-Product node $env:nodejs_version
-  - "%PYTHON%\\python.exe -m pip install coverage"
+  - "%PYTHON%\\python.exe -m pip install setuptools coverage ply"
   - "%PYTHON%\\python.exe setup.py install"
 
 test_script:

setup.py

@@ -1,16 +1,22 @@
-import atexit
+import os
 import sys
 from setuptools import setup, find_packages
-from setuptools.command.install import install
+from setuptools.command.build_py import build_py
 from subprocess import call
 
 
-class InstallHook(install):
-    """For hooking the optimizer when setup exits"""
+class BuildHook(build_py):
+    """Forcing the optimizer to run before the build step"""
     def __init__(self, *a, **kw):
-        install.__init__(self, *a, **kw)
-        atexit.register(
-            call, [sys.executable, '-m', 'calmjs.parse.parsers.optimize'])
+        # must use clone of this, otherwise Python on Windows gets sad.
+        env = os.environ.copy()
+        env['PYTHONPATH'] = 'src'
+        code = call([
+            sys.executable, '-m', 'calmjs.parse.parsers.optimize', '--build'
+        ], env=env)
+        if code:
+            sys.exit(1)
+        build_py.__init__(self, *a, **kw)
 
 
 # Attributes
@@ -32,6 +38,8 @@ def __init__(self, *a, **kw):
 Programming Language :: Python :: 3.8
 Programming Language :: Python :: 3.9
 Programming Language :: Python :: 3.10
+Programming Language :: Python :: 3.11
+Programming Language :: Python :: 3.12
 """.strip().splitlines()
 
 long_description = (
@@ -59,7 +67,7 @@ def __init__(self, *a, **kw):
     include_package_data=True,
     zip_safe=False,
     cmdclass={
-        'install': InstallHook,
+        'build_py': BuildHook,
     },
     install_requires=[
         'setuptools',

src/calmjs/parse/__init__.py

@@ -3,7 +3,14 @@
 Quick access helper functions
 """
 
-from calmjs.parse.factory import ParserUnparserFactory
+try:
+    from calmjs.parse.factory import ParserUnparserFactory
+except ImportError as e:  # pragma: no cover
+    exc = e
 
+    def import_error(*a, **kw):
+        raise exc
 
-es5 = ParserUnparserFactory('es5', 'pretty_print', 'minify_print')
+    es5 = import_error
+else:
+    es5 = ParserUnparserFactory('es5', 'pretty_print', 'minify_print')

src/calmjs/parse/parsers/es5.py

@@ -42,8 +42,10 @@
 
 asttypes = AstTypesFactory(pretty_print, ReprWalker())
 
-# The default values for the `Parser` constructor, passed on to ply; they must
-# be strings
+# These default values for the `Parser` constructor, passed on to ply;
+# they must be strings; these values are for reference only as
+# modifications to this value will not change what's been set up as
+# the Parser's default.
 lextab, yacctab = generate_tab_names(__name__)