Merge branch 'main' into aviyan-dev (2024-02-03)

.github/workflows/act.yml

@@ -33,6 +33,7 @@ jobs:
   build-and-test-runner:
     name: Build and Run Tests via Runner
     # skip during local actions testing (see https://github.com/nektos/act#skipping-jobs)
+    # note: enable by runnng ./local-workflows.sh with ACT_JSON=github-event.json in env
     if: ${{ !github.event.act }}        
     runs-on: ${{ matrix.os }}
 
@@ -41,7 +42,7 @@ jobs:
         ## TODO: os: [ubuntu-20.04, ubuntu-latest]
         os: [ubuntu-20.04]
         ## TODO: python-version: ["3.8", "3.9", "3.10"]
-        ## NOTE: only uses one version due to time to run tests locally
+        ## NOTE: only uses one version due to time to run tests locally as well as memory
         python-version: ["3.9"]
 
     steps:
@@ -73,6 +74,10 @@ jobs:
         run: |
           pip install --verbose --requirement requirements.txt
 
+      - name: Run Python Tests under Runner
+        run: |
+          PYTHONPATH="$PWD:$PWD/tests:$PYTHONPATH" ./tests/run_tests.bash
+          
   build-and-test-docker:
     name: Build and Run Tests via Docker
     # include unless skip-docker set (see skip-docker.json)

.github/workflows/debug.yml

@@ -76,6 +76,10 @@ jobs:
         run: |
           pip install --verbose --requirement requirements.txt
 
+      - name: Run Python Tests under Runner
+        run: |
+          PYTHONPATH="$PWD:$PWD/tests:$PYTHONPATH" ./tests/run_tests.bash
+          
   build-and-test-docker:
     name: Build and Run Tests via Docker
     # include unless skip-docker set (see skip-docker.json)

.github/workflows/github.yml

@@ -13,9 +13,10 @@ jobs:
 
   build-and-test-runner:
     name: Build and Run Tests via Runner
-    ## NOTE: need to track down differences between Docker and Runner runs
-    ## TEMP: disable tests (and just run Docker version below)
-    if: ${{ false }}
+    ## OLD:
+    ## ## NOTE: need to track down differences between Docker and Runner runs
+    ## ## TEMP: disable tests (and just run Docker version below)
+    ## if: ${{ false }}
     runs-on: ${{ matrix.os }}
 
     strategy:
@@ -27,6 +28,8 @@ jobs:
 
     steps:
       - name: Checkout Repository
+        ## TODO2: use checkout@v4
+        ## See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#example-using-versioned-actions
         uses: actions/checkout@v3
 
       - name: Setup Python

.gitignore

@@ -41,7 +41,7 @@ _temp-user-docker.env
 
 # Old or archived files
 **/old
-**/archive
+## OLD: **/archive
 
 # Output or log-file directories
 **/log-files

Dockerfile

@@ -9,7 +9,7 @@
 #   NOTE: --rm removes container afterwards; -it is for --interactive with --tty
 #   TODO2: --mount => --volume???
 # 3. [Optional] Run a bash shell using the created image:
-#   $ docker run -it --rm --entrypoint='/bin/bash' --mount type=bind,source="$(pwd)",target=/home/shell-scripts my-shell-scripts
+#   $ docker run -it --rm --entrypoint='/bin/bash' --mount type=bind,source="$(pwd)",target=/home/shell-scripts shell-scripts-dev
 #   # note: might need to tag the image (maldito docker): see local-workflows.sh
 # 4. Remove the image:
 #   $ docker rmi shell-scripts-dev
@@ -30,21 +30,17 @@ FROM catthehacker/ubuntu:act-20.04
 
 # Set default debug level (n.b., use docker build --build-arg "arg1=v1" to override)
 # Also optionally set the regex of tests to run.
-# Note: maldito act/nektos/docker not overriding properly
-## TODO2: fixme (see tests/run_tests.bash for workaround)
-## TODO4: ARG DEBUG_LEVEL=2
+# Note: can be overriden via _temp-user-docker.env
 ARG DEBUG_LEVEL=4
-## DEBUG: ARG DEBUG_LEVEL=5
 ARG TEST_REGEX=""
-## DEBUG: ARG TEST_REGEX="testing-tips"
 
 # [Work in progress]
 # Set branch override: this is not working due to subtle problem with the tfidf package
 #   ValueError: '/home/tomohara/python/tfidf' is not in the subpath of '/tmp/pip-req-build-4wdbom6g'
 #   OR one path is relative and the other is absolute.
 # Note: this is intended to avoid having to publish an update to PyPI.
 ARG GIT_BRANCH=""
-## HACK: hardcode branch to tom-dev due to stupid problems with act/docker
+## TEST: hardcode branch to tom-dev due to stupid problems with act/docker
 ## DEBUG: ARG GIT_BRANCH="tom-dev"
 
 # Trace overridable settings
@@ -66,20 +62,15 @@ RUN apt-get update && \
 RUN if [ "$DEBUG_LEVEL" -ge 4 ]; then apt-get install --yes emacs kdiff3 less tcsh zip || true; fi
 
 # Set the Python version to install
-# NOTE: The workflow yaml files now handle this (e.g., via matrix support)
-## TODO: keep in sync with .github/workflows
-## OLD: ARG PYTHON_VERSION=3.9.16
+# Note: The workflow yaml files only handle version for VM runner (e.g., via matrix support)
 ## TEST: ARG PYTHON_VERSION=3.8.12
 ARG PYTHON_VERSION=3.11.4
 
-
 # Download pre-compiled python build
-# TODO2: Make this step optional because it conflict with python matrix in yaml files (e.g., github/worflow/github.yml).
 # To find URL links, see https://github.com/actions/python-versions:
 # ex: https://github.com/actions/python-versions/releases/tag/3.11.4-5199054971
 #
 # maldito https://github.com/actions uses stupid idiosyncratic tag
-## OLD: ARG PYTHON_TAG=""
 ## TEST: ARG PYTHON_TAG="117929"
 ARG PYTHON_TAG="5199054971"
 #
@@ -104,10 +95,8 @@ RUN if [ "$PYTHON_VERSION" != "" ]; then
 # Copy the project's requirements file to the container
 ARG REQUIREMENTS=$WORKDIR/requirements.txt
 COPY requirements.txt $REQUIREMENTS
-## TODO3?: COPY tests/_test-config.bash $WORKDIR/tests
 
 # Install the project's dependencies
-## OLD: RUN pip install --verbose --no-cache-dir --requirement $REQUIREMENTS
 RUN if [ "$PYTHON_VERSION" != "" ]; then                                                \
         pip install --verbose --no-cache-dir --requirement $REQUIREMENTS;               \
     fi
@@ -134,14 +123,7 @@ fi
 # Enable github access
 # Note: This is not secure, but scrappycito only has access to
 # to dummy repo's like https://github.com/tomasohara/git-bash-test.
-## TODO:
-## RUN git config --global user.email "[email protected]" && \
-##     git config --global user.name "SCrappy Cito" && \
-##     export MY_GIT_TOKEN=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 && \
-##     git config --global url."https://api:[email protected]/".insteadOf "https://github.com/" && \
-##     git config --global url."https://ssh:[email protected]/".insteadOf "ssh://[email protected]/" && \
-##     git config --global url."https://git:[email protected]/".insteadOf "[email protected]:"
-
-## # Run the test, normally pytest over ./tests
+
+# Run the test, normally pytest over ./tests/*.py and batspp over ./tests/*.ipynb
 # Note: the status code (i.e., $?) determines whether docker run succeeds (e.h., OK if 0)
 ENTRYPOINT DEBUG_LEVEL=$DEBUG_LEVEL TEST_REGEX="$TEST_REGEX" './tests/run_tests.bash'

_temp-user-docker.env

@@ -5,6 +5,7 @@
 #   ENV2=value2
 #
 # Note:
+# - Consider using tests/_temp_test_settings.bash instead (also supports Github VM Runner).
 # - This normally is left empty.
 # - It can be used to just specific tests, such as by adding "TEST_REGEX=hello" below.
 # - It can also be used to increase the debugging level (e.g., by adding "DEBUG_LEVEL=5" below).
@@ -13,4 +14,4 @@
 # TODO:
 #   TEST_REGEX=pattern
 #
-DEBUG_LEVEL=6
+DEBUG_LEVEL=4

git-aliases.bash

@@ -313,7 +313,7 @@ function git-move-to-dir {    # Move files to specified directory
     for file in "$@"; do
         invoke-git-command mv "$file" "$dir"
         # TODO2: cut down on extraneous confirmations
-        git-commit-and-push "$file" "$dir/$file"
+        git-commit-and-push "$dir/$file"
     done
 }
 
@@ -686,10 +686,10 @@ function alt-invoke-next-single-checkin {
     # TODO: position cursor at start of ... (instead of pause)
     local is_text
     ## TODO: fix problem identifying scripts with UTF-8 as text (e.g., common.perl reported as data by file command)
-    is_text=$(file "$mod_file" | grep -i ':.*text')
+    is_text=$(file "$mod_file" | egrep -i ':.*(text|JSON|UTF-8)')
     ## HACK: add special case exceptions
     if [ "$is_text" = "" ]; then
-        case "$mod_file" in *.css | *.csv | *.html | *.java | *.js | *.perl | *.py | *.[a-z]*sh | *.text| *.txt) is_text="1"; echo "Special case hack for braindead file command (known program extension in $mod_file)" ;; esac
+        case "$mod_file" in *.css | *.csv | *.html | *.ipynb | *.java | *.js | *.perl | *.py | *.[a-z]*sh | *.text| *.txt) is_text="1"; echo "Special case hack for braindead file command (known program extension in $mod_file)" ;; esac
     fi;
     if [ "$is_text" != "" ]; then
         if [ "$GIT_NO_CONFIRM" = "1" ]; then
@@ -807,8 +807,9 @@ function git-alias-usage () {
     echo "Get changes from repository (set PRESERVE_GIT_STASH=1 to keep timestamps)":
     echo "    git-update-plus"
     echo ""
-    echo "To add regular files via git-add (n.b., ignored if matches .gitignore):"
+    echo "To add regular files to repo (via git-add):"
     echo "    GIT_MESSAGE='initial version' git-update-commit-push file..."
+    echo "Note: use GIT_FORCE=1 ... if file matches .gitignore patterns"
     echo ""
     echo "To check in specified changes:"
     echo "    GIT_MESSAGE='...' git-update-commit-push file..."

jupyter_to_batspp.py

@@ -105,6 +105,7 @@
 ADD_ANNOTS   = 'add-annots'
 STDOUT       = 'stdout'
 JUST_CODE    = 'just-code'
+REGULAR_OPT  = 'regular'
 AUTO_OUTPUT  = 'auto-output'
 
 # Other constants
@@ -133,7 +134,9 @@ def setup(self):
         # Check the command-line options
         # TODO2: make the defaults more intuitive (e.g., for auto_output and verbose)
         self.jupyter_file = self.get_parsed_argument(JUPYTER_FILE, self.jupyter_file)
-        self.just_code    = self.get_parsed_argument(JUST_CODE, self.just_code)
+        regular = self.get_parsed_argument(JUST_CODE, True)
+        self.just_code    = self.get_parsed_argument(REGULAR_OPT, self.just_code)
+        debug.assertion(not (regular and self.just_code))
         self.add_annots   = self.get_parsed_option(ADD_ANNOTS, self.add_annots)
         self.auto_output  = self.get_parsed_option(AUTO_OUTPUT, False)
         if self.auto_output:
@@ -261,6 +264,7 @@ def main():
         boolean_options      = [(VERBOSE,      'Show verbose debug'),
                                 (ADD_ANNOTS,   'Add annotations about current cells, etc.'),
                                 (JUST_CODE,    'Omit output cells'),
+                                (REGULAR_OPT,  'Regular mode--code and output)'),
                                 (AUTO_OUTPUT, f'Automatically save with .{BATSPP_EXTENSION}'),
                                 (STDOUT,      f'Print to standard output (default unless --{OUTPUT})')],
         manual_input         = True)

local-workflows.sh

@@ -145,8 +145,7 @@ done
 # Trace out main environment overrides
 #   CLONE_REPO, AUTO_REQS, RUN_BUILD, BUILD_OPTS, RUN_WORKFLOW, RUN_OPTS, WORKFLOW_FILE
 if [ "$DEBUG_LEVEL" -ge 4 ]; then
-    echo "in $0 $*"
-    echo "timestamp: $(date)"
+    echo "in $0 $* [$(date)]"
     src_dir=$(dirname "${BASH_SOURCE[0]}")
     source "${TOM_BIN:-"$src_dir"}/all-tomohara-aliases-etc.bash"
     trace-vars IMAGE_NAME ACT_PULL LOCAL_REPO_DIR DEBUG_LEVEL GIT_BRANCH BUILD_OPTS USER_ENV
@@ -247,6 +246,5 @@ fi
 
 # End processing
 if [ "$DEBUG_LEVEL" -ge 4 ]; then
-    echo "out $0"
-    echo "timestamp: $(date)"
+    echo "out $0 [$(date)]"
 fi

perl.sh

@@ -12,8 +12,9 @@
 
 ## DEBUG:
 ## echo "in "${BASH_SOURCE[0]}""
-## echo '$0 "$@":' "$0" "$@"
+## echo "$0 '$*':" "$0" "$@"
 ## echo "DEBUG_LEVEL=$DEBUG_LEVEL"
 ## echo "DURING_ALIAS=$DURING_ALIAS"
+## echo "perl: $(which perl)"
 
 perl "$@"

required-packages.txt

@@ -9,17 +9,21 @@
 #   debian-based Linux distributions.
 # - See requirements.txt for Python package requirements.
 #
+ansifilter
 diffutils
 git
 jupyter-core
 kdiff3
 less
+lynx
+perl
 python3-pip
 python3
 python-is-python3
 tar
 tcsh
 wget
+xmllint
 zip
 #
 #................................................................................

simple_batspp.py

@@ -200,7 +200,7 @@
 MERGE_CONTINUATION = system.getenv_bool("MERGE_CONTINUATION", False,
                                         "Merge function or backslash continuations in expected with actual")
 IGNORE_ALL_COMMENTS = system.getenv_bool("IGNORE_ALL_COMMENTS", False,
-                                         "Strip all comments from input")
+                                         "Strip all comments from input--blocks without commands or output")
 STRIP_COMMENTS = system.getenv_bool("STRIP_COMMENTS", False,
                                     "Strip comments from expected output")
 ALLOW_COMMENTS = system.getenv_bool("ALLOW_COMMENTS", False,
@@ -684,7 +684,6 @@ def _preprocess_command(self, field):
 
         # Remove comments (n.b., needs to be done after comment indicators checked)
         if STRIP_COMMENTS:
-            ## OLD: field = my_re.sub(r'^\s*\#.*\n', '', field, flags=re.MULTILINE|re.IGNORECASE)
             field = my_re.sub(r'^\s*\#.*\n', '', field, flags=re.MULTILINE)
         ## TODO2: debug.trace(T8, f"_preprocess_command({in_field!r}) ==  {field!r}")
         debug.trace(5, f"_preprocess_command({in_field!r}) == {field!r}")
@@ -888,9 +887,13 @@ def _convert_to_bats(self, test):
                 f'\t{actual_function} >| "$out_file"\n' +
                 f'\t{actual_var}="$(cat "$out_file")"\n')
             main_body += f'\t{expected_var}="$({expected_function})"\n'
+        ## TODO3: just normalize actual at runtime as expected can be done ahead of time
         if NORMALIZE_WHITESPACE:
             main_body += (f'\t{actual_var}="$(normalize-whitespace \"${actual_var}\")"\n' +
                           f'\t{expected_var}="$(normalize-whitespace \"${expected_var}\")"\n')
+        if STRIP_COMMENTS:
+            main_body += (f'\t{actual_var}="$(strip-comments \"${actual_var}\")"\n' +
+                          f'\t{expected_var}="$(strip-comments \"${expected_var}\")"\n')
         if IGNORE_SETUP_OUTPUT:
             if (has_setup_comment or has_continuation_comment or has_wrapup_comment):
                 # note: setup and wrapup output ignored; however, code run above for side effects
@@ -1065,6 +1068,18 @@ def get_bats_tests(self, text):
                 """)
             bats_tests += my_re.sub(r"^ +", "", helper_code, flags=re.MULTILINE)
             bats_tests += "\n"
+        if STRIP_COMMENTS:
+            tab = "\t"
+            helper_code = (rf"""# Helper functions
+                function strip-comments {{ 
+                   {tab}local text
+                   {tab}text=$(echo "$*" | perl -pe "s/^\s*\#.*\n//;"); 
+                   {tab}echo "$text";
+                }}
+                """)
+            bats_tests += my_re.sub(r"^ +", "", helper_code, flags=re.MULTILINE)
+            bats_tests += "\n"
+
 
         # Add global setup if using global test dir
         if GLOBAL_TEST_DIR:

template.bash

@@ -95,6 +95,7 @@
 # - Document file tests (e.g., -e fubar.txt).
 # - BASH_SOURCE usage for when source'd
 #     src_dir=$(dirname "${BASH_SOURCE[0]}")
+# - value=${value@L}                    # make lowercase
 #
 
 # Uncomment following line(s) for tracing: