From 747becda10c6289f9b7ba14f74256d08f2abf06e Mon Sep 17 00:00:00 2001 From: Mark Taylor <24257134+tmarktaylor@users.noreply.github.com> Date: Thu, 17 Aug 2023 08:06:27 -0400 Subject: [PATCH 01/27] Fix tests/test_docs.py. Don't expect 0.000s elapsed time. --- tests/test_docs.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/test_docs.py b/tests/test_docs.py index 52796da..71f9928 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -391,14 +391,18 @@ def test_unittest_stderr_printing(): number_of_deselected_blocks=1, ) assert want == phmresult.metrics - expected_std_err = """\ + expected_std_err = [ + """\ . ---------------------------------------------------------------------- -Ran 1 test in 0.000s +Ran 1 test in""", + """ OK -""" - assert expected_std_err == ferr.getvalue() +""", + ] + assert expected_std_err[0] in ferr.getvalue() + assert expected_std_err[1] in ferr.getvalue() assert output == fout.getvalue().lstrip() From cd09e345d4dbc212e8c5336a178da83ea148044f Mon Sep 17 00:00:00 2001 From: Mark Taylor <24257134+tmarktaylor@users.noreply.github.com> Date: Thu, 17 Aug 2023 08:36:15 -0400 Subject: [PATCH 02/27] Update wheel.yml - add missing key. --- .github/workflows/wheel.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index 9a83431..2576d01 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -4,6 +4,7 @@ on: workflow_dispatch: env: + project: phmutest version: 0.0.1 command: phmutest From 05a905a283a1158f12b975a853958d6177816ce2 Mon Sep 17 00:00:00 2001 From: Mark Taylor <24257134+tmarktaylor@users.noreply.github.com> Date: Thu, 17 Aug 2023 09:02:49 -0400 Subject: [PATCH 03/27] Fix action inspect job Formatting black command --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a1a0e86..1ef95ba 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -144,8 +144,8 @@ jobs: # Subsequent steps run with the modified files. # Don't run black on the generated test files. run: | - black **/*.py --check --force-exclude "tests.py.generated.*py" - black **/*.py --force-exclude "tests.py.generated.*py" + black **/*.py --check --force-exclude="tests/py" + black **/*.py --force-exclude="tests/py" continue-on-error: true - name: Code Style run: | From 5ffef7918611a134fb180bfb3dcdd617a80603c1 Mon Sep 17 00:00:00 2001 From: Mark Taylor <24257134+tmarktaylor@users.noreply.github.com> Date: Sun, 20 Aug 2023 10:18:08 -0400 Subject: [PATCH 04/27] Tests- Add missing import phmutest.summary. Removed unused import phmutest.tool. --- tests/test_cases.py | 1 + tests/test_config.py | 1 + tests/test_errors.py | 1 + tests/test_extra_args.py | 1 + tests/test_fenced.py | 1 + tests/test_globs.py | 1 + tests/test_patching.py | 1 + tests/test_rebind.py | 1 + tests/test_session.py | 1 + tests/test_sharing.py | 1 + tests/test_skip.py | 1 + tests/test_type_packaging.py | 1 - 12 files changed, 11 insertions(+), 1 deletion(-) diff --git a/tests/test_cases.py b/tests/test_cases.py index 47b018e..fc88d55 100644 --- a/tests/test_cases.py +++ b/tests/test_cases.py @@ -7,6 +7,7 @@ import phmutest.cases import phmutest.main import phmutest.subtest +import phmutest.summary def test_chop_final_newline(): diff --git a/tests/test_config.py b/tests/test_config.py index da0b3b2..59d09d3 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -7,6 +7,7 @@ import phmutest.config import phmutest.main +import phmutest.summary def process_args(commandline_args): diff --git a/tests/test_errors.py b/tests/test_errors.py index 77156a4..592aec4 100644 --- a/tests/test_errors.py +++ b/tests/test_errors.py @@ -5,6 +5,7 @@ import pytest import phmutest.main +import phmutest.summary def test_code_raises(): diff --git a/tests/test_extra_args.py b/tests/test_extra_args.py index c9b136f..e42c222 100644 --- a/tests/test_extra_args.py +++ b/tests/test_extra_args.py @@ -2,6 +2,7 @@ import pytest import phmutest.main +import phmutest.summary def test_noextras(): diff --git a/tests/test_fenced.py b/tests/test_fenced.py index 348b2e4..c4eb107 100644 --- a/tests/test_fenced.py +++ b/tests/test_fenced.py @@ -1,5 +1,6 @@ """Test Python info string matching, command line skip, and --report.""" import phmutest.main +import phmutest.summary def test_python_code_matcher(): diff --git a/tests/test_globs.py b/tests/test_globs.py index dcf1a75..91ce892 100644 --- a/tests/test_globs.py +++ b/tests/test_globs.py @@ -7,6 +7,7 @@ import pytest import phmutest.main +import phmutest.summary from phmutest.globs import AssignmentExtractor, Globals MYGLOBAL = 1 diff --git a/tests/test_patching.py b/tests/test_patching.py index 3d2c397..901e419 100644 --- a/tests/test_patching.py +++ b/tests/test_patching.py @@ -10,6 +10,7 @@ import phmutest.main import phmutest.reader import phmutest.session +import phmutest.summary from phmutest.direct import MarkerPattern # Note that the last block is included because it has the diff --git a/tests/test_rebind.py b/tests/test_rebind.py index 618f368..60a5adb 100644 --- a/tests/test_rebind.py +++ b/tests/test_rebind.py @@ -31,6 +31,7 @@ import unittest import phmutest.main +import phmutest.summary from phmutest.fixture import Fixture diff --git a/tests/test_session.py b/tests/test_session.py index 2b34eda..bf80f4b 100644 --- a/tests/test_session.py +++ b/tests/test_session.py @@ -1,5 +1,6 @@ """Test cases for session.py.""" import phmutest.main +import phmutest.summary from phmutest.fixture import Fixture diff --git a/tests/test_sharing.py b/tests/test_sharing.py index babce91..885cf91 100644 --- a/tests/test_sharing.py +++ b/tests/test_sharing.py @@ -4,6 +4,7 @@ import unittest import phmutest.main +import phmutest.summary def namelist(text, unwanted_prefix): diff --git a/tests/test_skip.py b/tests/test_skip.py index 5b8a038..5cb7358 100644 --- a/tests/test_skip.py +++ b/tests/test_skip.py @@ -1,6 +1,7 @@ """Check handling of --skip command line option.""" import phmutest.main +import phmutest.summary def test_skip_text1(): diff --git a/tests/test_type_packaging.py b/tests/test_type_packaging.py index 0735169..d621a78 100644 --- a/tests/test_type_packaging.py +++ b/tests/test_type_packaging.py @@ -4,7 +4,6 @@ import phmutest.fenced import phmutest.reader -import phmutest.tool def test_mypy_likes_imported_types() -> None: From 894749733166cdccacc6c2025269bebaca3b644e Mon Sep 17 00:00:00 2001 From: Mark Taylor <24257134+tmarktaylor@users.noreply.github.com> Date: Sat, 26 Aug 2023 17:02:11 -0400 Subject: [PATCH 05/27] Add main.command() for call from Python. --- README.md | 17 ++++++-- docs/callfrompython.md | 23 +++++++++-- docs/recent_changes.md | 4 ++ src/phmutest/main.py | 5 +++ tests/test_cases.py | 44 ++++++++++----------- tests/test_errors.py | 4 +- tests/test_examples.py | 90 +++++++++++++++++------------------------- tests/test_fenced.py | 12 +++--- tests/test_patching.py | 22 +++++------ tests/test_rebind.py | 5 +-- tests/test_sharing.py | 8 ++-- 11 files changed, 123 insertions(+), 111 deletions(-) diff --git a/README.md b/README.md index 733576f..2abb3db 100644 --- a/README.md +++ b/README.md @@ -246,6 +246,17 @@ The fixture file should be in the project directory tree. Fixture demos: - [fixture set globals](docs/fix/code/globdemo.md) - [fixture cleanup REPL Mode](docs/fix/repl/drink.md) +When calling phmutest from Python, mock.patch() patching +would typically be implemented as enclosing with statements. + +When invoking phmutest from a shell, +the --fixture function can be used to install patches. +See example near the end of tests/test_patching.py and +in tests/test_subprocess.py. The example shows how to patch to +apply doctest optionflags in --replmode. +Do patching cleanup when not in --replmode by calling +`unittest.addModuleCleanup(stack.pop_all().close)`. + ## Extend an example across files @@ -306,9 +317,9 @@ python -m phmutest README.md --log ## Call from Python -Call **phmutest.main.main()** with a list of strings for the usage arguments, -options, and option values like this: -`["/md/project.md", "--replmode"]` +Call **phmutest.main.command()** with a string that looks like a +command line less the phmutest, like this: +`"md/project.md --replmode"` - A `phmutest.summary.PhmResult` instance is returned. - When calling from Python there is no shell wildcard expansion. diff --git a/docs/callfrompython.md b/docs/callfrompython.md index c5098cb..b0bafa4 100644 --- a/docs/callfrompython.md +++ b/docs/callfrompython.md @@ -6,9 +6,8 @@ import phmutest.main ```python -command = "tests/md/project.md --replmode" -args = command.split() -phmresult = phmutest.main.main(args) +line = "tests/md/project.md --replmode" +phmresult = phmutest.main.command(line) ``` ```python @@ -19,7 +18,7 @@ assert phmresult.metrics.number_of_files == 1 ``` ## PhmResult -phmutest.main.main() returns a value of type PhmResult +phmutest.main.command() returns a value of type PhmResult defined in src/phmutest/summary.py. @@ -35,6 +34,22 @@ class PhmResult: log: List[List[str]] ``` +## Another way to call from Python + +This example shows how to call from Python +with the arguments as a list of strings. +phmutest.main.main() returns a value of type PhmResult as well. + +```python +args = ["tests/md/project.md", "--replmode"] +phmresult = phmutest.main.main(args) +assert phmresult.is_success +assert phmresult.metrics.number_blocks_run == 3 +assert phmresult.metrics.passed == 3 +assert phmresult.metrics.number_of_files == 1 +``` + + ## limitation The limitation described here applies to call from Python when diff --git a/docs/recent_changes.md b/docs/recent_changes.md index dbc8e74..38c32ca 100644 --- a/docs/recent_changes.md +++ b/docs/recent_changes.md @@ -3,3 +3,7 @@ - Initial upload to Python Package Index. +0.0.2 - 2023-xx-xx + +- add main.command(). + diff --git a/src/phmutest/main.py b/src/phmutest/main.py index fff4626..0935f0f 100644 --- a/src/phmutest/main.py +++ b/src/phmutest/main.py @@ -234,6 +234,11 @@ def main(argv: Optional[List[str]] = None) -> Optional[phmutest.summary.PhmResul return generate_and_run(known_args) +def command(line: str) -> Optional[phmutest.summary.PhmResult]: + """For call from Python with command line as a string and no sys.exit().""" + return main(line.split()) + + def entry_point(argv: Optional[List[str]] = None) -> None: """Entry point for command line invocation.""" phmresult = main(argv) diff --git a/tests/test_cases.py b/tests/test_cases.py index fc88d55..476429f 100644 --- a/tests/test_cases.py +++ b/tests/test_cases.py @@ -43,8 +43,8 @@ def test_deindent(): def test_no_files(): """Run with no files specified on the command line.""" # This covers the cases.py line near the end: test_classes += "\n" - command = "--log" - phmresult = phmutest.main.main(command.split()) + line = "--log" + phmresult = phmutest.main.command(line) want = phmutest.summary.Metrics( number_blocks_run=0, passed=0, @@ -62,8 +62,8 @@ def test_no_files(): def test_skipif_no_output(): """Source Code block with no ouput and skipif directive. Empty Python block.""" # This covers the cases.py line near the end: test_classes += "\n" - command = "tests/md/cases.md --log" - phmresult = phmutest.main.main(command.split()) + line = "tests/md/cases.md --log" + phmresult = phmutest.main.command(line) want = phmutest.summary.Metrics( number_blocks_run=2, passed=2, @@ -101,8 +101,8 @@ def test_duplicate_filename(): def test_deselected_blocks_report(capsys, endswith_checker): """See list of deselected blocks at the end of the report.""" - command = "tests/md/code_groups.md --deselect group-1 group-2 --report" - phmresult = phmutest.main.main(command.split()) + line = "tests/md/code_groups.md --deselect group-1 group-2 --report" + phmresult = phmutest.main.command(line) assert phmresult is None output = capsys.readouterr().out expected = """Deselected blocks: @@ -118,9 +118,9 @@ def test_print_captured_output(startswith_checker): # Note- While unittest is running Printer prints to stderr. # --quiet is passed through to unittest to prevent the progress dot printing. # Printing after unittest completes is to stdout. - command = "tests/md/printer.md --log --progress --quiet" + line = "tests/md/printer.md --log --progress --quiet" with contextlib.redirect_stderr(io.StringIO()) as ferr: - phmresult = phmutest.main.main(command.split()) + phmresult = phmutest.main.command(line) want = phmutest.summary.Metrics( number_blocks_run=3, passed=1, @@ -159,8 +159,8 @@ def test_skip_progress(): # This test exposed an issue with UnboundLocalError for sys when using # sys.stderr for verbose printing of a skipped block. # See src/phmutest/skip.py::make_replacements(). - command = "tests/md/directive1.md --log --progress" - phmresult = phmutest.main.main(command.split()) + line = "tests/md/directive1.md --log --progress" + phmresult = phmutest.main.command(line) want = phmutest.summary.Metrics( number_blocks_run=3, passed=2, @@ -177,11 +177,11 @@ def test_skip_progress(): def test_setup_module_progress(): """Test --progress printing in setUpModule has no UnboundLocalError for sys.""" - command = ( + line = ( "docs/setup/across1.md docs/setup/across2.md" " --setup-across-files docs/setup/across1.md --log --progress" ) - phmresult = phmutest.main.main(command.split()) + phmresult = phmutest.main.command(line) want = phmutest.summary.Metrics( number_blocks_run=6, passed=6, @@ -198,8 +198,8 @@ def test_setup_module_progress(): def test_setup_no_teardown(capsys): """Run setup without teardown and check the log.""" - command = "tests/md/setupnoteardown.md --log -f" - phmresult = phmutest.main.main(command.split()) + line = "tests/md/setupnoteardown.md --log -f" + phmresult = phmutest.main.command(line) want = phmutest.summary.Metrics( number_blocks_run=5, passed=5, @@ -222,11 +222,11 @@ def test_setup_no_teardown(capsys): def test_setup_across_no_teardown(capsys): """Run the setup across files example and check the log.""" - command = ( + line = ( "tests/md/setupnoteardown.md tests/md/setupto.md --log " "--setup-across-files tests/md/setupnoteardown.md" ) - phmresult = phmutest.main.main(command.split()) + phmresult = phmutest.main.command(line) want = phmutest.summary.Metrics( number_blocks_run=6, passed=6, @@ -252,8 +252,8 @@ def test_setup_across_no_teardown(capsys): def test_setup_across_share_across(capsys): """Run the setup across files example and check the log.""" - command = "--log --config tests/toml/acrossfiles.toml" - phmresult = phmutest.main.main(command.split()) + line = "--log --config tests/toml/acrossfiles.toml" + phmresult = phmutest.main.command(line) want = phmutest.summary.Metrics( number_blocks_run=7, passed=7, @@ -280,8 +280,8 @@ def test_setup_across_share_across(capsys): def test_share_across_with_setup(capsys): """Share across files a .md file that has (un-shared) setup blocks.""" - command = "--log --config tests/toml/acrossfiles2.toml" - phmresult = phmutest.main.main(command.split()) + line = "--log --config tests/toml/acrossfiles2.toml" + phmresult = phmutest.main.command(line) want = phmutest.summary.Metrics( number_blocks_run=6, passed=6, @@ -312,10 +312,10 @@ def test_progress_option(): # Using homemade stderr capture since not seeing it from pytest's # capsys.readouterr().err. with contextlib.redirect_stderr(io.StringIO()) as err: - command = ( + line = ( "docs/fix/code/chdir.md --fixture docs.fix.code.chdir.change_dir --progress" ) - phmresult = phmutest.main.main(command.split()) + phmresult = phmutest.main.command(line) want = phmutest.summary.Metrics( number_blocks_run=2, passed=2, diff --git a/tests/test_errors.py b/tests/test_errors.py index 592aec4..08c9351 100644 --- a/tests/test_errors.py +++ b/tests/test_errors.py @@ -31,8 +31,8 @@ def test_code_raises(): def test_unittest_fail_fast(): """Pass through callers -f option to unittest. (fail fast).""" - command = "tests/fail/raiser.md --log -f" - phmresult = phmutest.main.main(command.split()) + line = "tests/fail/raiser.md --log -f" + phmresult = phmutest.main.command(line) want = phmutest.summary.Metrics( number_blocks_run=5, passed=4, diff --git a/tests/test_examples.py b/tests/test_examples.py index e8a393a..0728660 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -5,9 +5,8 @@ def test_sample(): """Run Python code block with expected output block in project.md.""" - command = "tests/md/project.md" - args = command.split() - phmresult = phmutest.main.main(args) + line = "tests/md/project.md" + phmresult = phmutest.main.command(line) want = phmutest.summary.Metrics( number_blocks_run=2, passed=2, @@ -24,9 +23,8 @@ def test_sample(): def test_sample_replmode(): """Run Python interactive sessions (doctests) in project.md.""" - command = "tests/md/project.md --replmode" - args = command.split() - phmresult = phmutest.main.main(args) + line = "tests/md/project.md --replmode" + phmresult = phmutest.main.command(line) want = phmutest.summary.Metrics( number_blocks_run=3, passed=3, @@ -43,9 +41,8 @@ def test_sample_replmode(): def test_phmdoctest_example1(): """Run Python code block with expected output block in example1.md.""" - command = "tests/md/example1.md" - args = command.split() - phmresult = phmutest.main.main(args) + line = "tests/md/example1.md" + phmresult = phmutest.main.command(line) want = phmutest.summary.Metrics( number_blocks_run=1, passed=1, @@ -62,9 +59,8 @@ def test_phmdoctest_example1(): def test_example2(): """Run Python code block with expected output block in example2.md.""" - command = "tests/md/example2.md" - args = command.split() - phmresult = phmutest.main.main(args) + line = "tests/md/example2.md" + phmresult = phmutest.main.command(line) want = phmutest.summary.Metrics( number_blocks_run=5, passed=5, @@ -81,9 +77,8 @@ def test_example2(): def test_exmple2_replmode(): """Run Python interactive sessions (doctests) in example2.md.""" - command = "tests/md/example2.md --replmode" - args = command.split() - phmresult = phmutest.main.main(args) + line = "tests/md/example2.md --replmode" + phmresult = phmutest.main.command(line) want = phmutest.summary.Metrics( number_blocks_run=2, passed=2, @@ -100,9 +95,8 @@ def test_exmple2_replmode(): def test_directive1(): """Test label, skip, and skipif directives on code blocks.""" - command = "tests/md/directive1.md" - args = command.split() - phmresult = phmutest.main.main(args) + line = "tests/md/directive1.md" + phmresult = phmutest.main.command(line) want = phmutest.summary.Metrics( number_blocks_run=3, passed=2, @@ -121,9 +115,8 @@ def test_directive1(): def test_directive1_replmode(): """Test label, skip, and skipif directives on code blocks.""" - command = "tests/md/directive1.md --replmode" - args = command.split() - phmresult = phmutest.main.main(args) + line = "tests/md/directive1.md --replmode" + phmresult = phmutest.main.command(line) want = phmutest.summary.Metrics( number_blocks_run=1, passed=1, @@ -143,9 +136,8 @@ def test_directive1_replmode(): def test_directive2(): """Test single setup and teardown directives on code blocks.""" - command = "tests/md/directive2.md" - args = command.split() - phmresult = phmutest.main.main(args) + line = "tests/md/directive2.md" + phmresult = phmutest.main.command(line) want = phmutest.summary.Metrics( number_blocks_run=5, passed=5, @@ -164,9 +156,8 @@ def test_directive2(): def test_blank_output_lines(): """Embedded blank lines in Python code expected output block.""" - command = "tests/md/output_has_blank_lines.md" - args = command.split() - phmresult = phmutest.main.main(args) + line = "tests/md/output_has_blank_lines.md" + phmresult = phmutest.main.command(line) want = phmutest.summary.Metrics( number_blocks_run=1, passed=1, @@ -183,9 +174,8 @@ def test_blank_output_lines(): def test_no_code_blocks(): """Test file with no Python FCBs.""" - command = "tests/md/no_code_blocks.md" - args = command.split() - phmresult = phmutest.main.main(args) + line = "tests/md/no_code_blocks.md" + phmresult = phmutest.main.command(line) want = phmutest.summary.Metrics( number_blocks_run=0, passed=0, @@ -203,9 +193,8 @@ def test_no_code_blocks(): def test_no_sessions(): """Test file with no sessions in --replmode.""" - command = "tests/md/directive2.md --replmode" - args = command.split() - phmresult = phmutest.main.main(args) + line = "tests/md/directive2.md --replmode" + phmresult = phmutest.main.command(line) want = phmutest.summary.Metrics( number_blocks_run=0, passed=0, @@ -223,9 +212,8 @@ def test_no_sessions(): def test_bad_session_output(): """A Python interactive session example that has wrong output.""" - command = "tests/md/bad_session_output.md --replmode" - args = command.split() - phmresult = phmutest.main.main(args) + line = "tests/md/bad_session_output.md --replmode" + phmresult = phmutest.main.command(line) want = phmutest.summary.Metrics( number_blocks_run=1, passed=0, @@ -242,9 +230,8 @@ def test_bad_session_output(): def test_bad_skipif_number(): """Two different malformed phmutest-skipif directives are ignored.""" - command = "tests/md/bad_skipif_number.md" - args = command.split() - phmresult = phmutest.main.main(args) + line = "tests/md/bad_skipif_number.md" + phmresult = phmutest.main.command(line) want = phmutest.summary.Metrics( number_blocks_run=2, passed=2, @@ -261,9 +248,8 @@ def test_bad_skipif_number(): def test_does_not_print(): """Expected output expected but not produced.""" - command = "tests/md/does_not_print.md" - args = command.split() - phmresult = phmutest.main.main(args) + line = "tests/md/does_not_print.md" + phmresult = phmutest.main.command(line) want = phmutest.summary.Metrics( number_blocks_run=1, passed=0, @@ -280,9 +266,8 @@ def test_does_not_print(): def test_excess_printing(): """Excess expected output printed.""" - command = "tests/md/missing_some_output.md" - args = command.split() - phmresult = phmutest.main.main(args) + line = "tests/md/missing_some_output.md" + phmresult = phmutest.main.command(line) want = phmutest.summary.Metrics( number_blocks_run=1, passed=0, @@ -299,9 +284,8 @@ def test_excess_printing(): def test_no_fcbs(): """Test file with no FCBs.""" - command = "tests/md/no_fenced_code_blocks.md" - args = command.split() - phmresult = phmutest.main.main(args) + line = "tests/md/no_fenced_code_blocks.md" + phmresult = phmutest.main.command(line) want = phmutest.summary.Metrics( number_blocks_run=0, passed=0, @@ -319,9 +303,8 @@ def test_no_fcbs(): def test_phmdoctest_mark_skip(): """Test file with a single phmdoctest directive.""" - command = "tests/md/one_mark_skip.md" - args = command.split() - phmresult = phmutest.main.main(args) + line = "tests/md/one_mark_skip.md" + phmresult = phmutest.main.command(line) want = phmutest.summary.Metrics( number_blocks_run=0, passed=0, @@ -339,9 +322,8 @@ def test_phmdoctest_mark_skip(): def test_unexpected_output(): """Test code block with incorrect expected output.""" - command = "tests/md/unexpected_output.md" - args = command.split() - phmresult = phmutest.main.main(args) + line = "tests/md/unexpected_output.md" + phmresult = phmutest.main.command(line) want = phmutest.summary.Metrics( number_blocks_run=1, passed=0, diff --git a/tests/test_fenced.py b/tests/test_fenced.py index c4eb107..db8ed75 100644 --- a/tests/test_fenced.py +++ b/tests/test_fenced.py @@ -5,8 +5,8 @@ def test_python_code_matcher(): """Test Python block identification.""" - args = "tests/md/pythonmatch.md".split() - phmresult = phmutest.main.main(args) + line = "tests/md/pythonmatch.md" + phmresult = phmutest.main.command(line) want = phmutest.summary.Metrics( number_blocks_run=7, passed=7, @@ -22,8 +22,8 @@ def test_python_code_matcher(): def test_python_repl_matcher(): """Test Python block identification.""" - args = "tests/md/pythonmatch.md --replmode".split() - phmresult = phmutest.main.main(args) + line = "tests/md/pythonmatch.md --replmode" + phmresult = phmutest.main.command(line) want = phmutest.summary.Metrics( number_blocks_run=8, passed=8, @@ -99,9 +99,9 @@ def test_report(capsys, checker): Deselected blocks: """ - command = "tests/md/report.md --skip CHERRIES CHERRIES --report" + line = "tests/md/report.md --skip CHERRIES CHERRIES --report" # Note- duplicate skip pattern CHERRIES is not shown in list of # block skip patterns. - phmresult = phmutest.main.main(command.split()) + phmresult = phmutest.main.command(line) assert phmresult is None checker(expected, capsys.readouterr().out.rstrip()) diff --git a/tests/test_patching.py b/tests/test_patching.py index 901e419..41ab766 100644 --- a/tests/test_patching.py +++ b/tests/test_patching.py @@ -35,8 +35,8 @@ def test_infostring_patch(capsys): matcher.python_patterns.append("ladenpython") # Also match info string ladenpython. matcher.compile() with mock.patch("phmutest.fenced.python_matcher", matcher): - args = ["tests/md/patching1.md", "--log"] - phmresult = phmutest.main.main(args) + line = "tests/md/patching1.md --log" + phmresult = phmutest.main.command(line) want = phmutest.summary.Metrics( number_blocks_run=3, passed=3, # counts the ladenpython FCB @@ -89,8 +89,8 @@ def test_directive_patch(capsys): updated_finders = copy.copy(phmutest.direct.directive_finders) updated_finders.append(finder_alias) with mock.patch("phmutest.direct.directive_finders", updated_finders): - args = ["tests/md/patching1.md", "--log"] - phmresult = phmutest.main.main(args) + line = "tests/md/patching1.md --log" + phmresult = phmutest.main.command(line) want = phmutest.summary.Metrics( number_blocks_run=2, passed=2, # does not count the ladenpython FCB @@ -134,11 +134,8 @@ def test_modify_docstring_patch(capsys, endswith_checker): """Show modify_docstring patch changes the docstring that gets run.""" # Run twice, first time without the patch. The second time with the # patch that makes the 3 blocks fail. - command = ( - "tests/md/optionflags.md --replmode --fixture tests.test_patching.setflags" - ) - args = command.split() - phmresult1 = phmutest.main.main(args) + line = "tests/md/optionflags.md --replmode --fixture tests.test_patching.setflags" + phmresult1 = phmutest.main.command(line) want1 = phmutest.summary.Metrics( number_blocks_run=3, passed=3, @@ -153,7 +150,7 @@ def test_modify_docstring_patch(capsys, endswith_checker): assert phmresult1.is_success is True with mock.patch("phmutest.session.modify_docstring", rewrite_docstring): - phmresult2 = phmutest.main.main(args) + phmresult2 = phmutest.main.command(line) want2 = phmutest.summary.Metrics( number_blocks_run=3, passed=0, @@ -193,12 +190,11 @@ def setflags(**kwargs): def test_doctest_optionflags_patch(): """Test a --fixture that runs doctests with optionflags.""" - command = ( + line = ( "tests/md/optionflags.md --log --replmode " " --fixture tests.test_patching.setflags" ) - args = command.split() - phmresult = phmutest.main.main(args) + phmresult = phmutest.main.command(line) want = phmutest.summary.Metrics( number_blocks_run=3, passed=3, diff --git a/tests/test_rebind.py b/tests/test_rebind.py index 60a5adb..a14c818 100644 --- a/tests/test_rebind.py +++ b/tests/test_rebind.py @@ -49,15 +49,14 @@ def globsfixture(**kwargs): def test_share_across_rebind(): """Share across files re-assign of fixture glob name done in file1.md.""" - command = ( + line = ( "docs/share/file1.md docs/share/file2.md docs/share/file3.md " "--share-across-files docs/share/file1.md docs/share/file2.md --log " "--fixture tests.test_rebind.globsfixture " "--sharing . --quiet" ) # --quiet is passed through to unittest to prevent the progress dot printing. - args = command.split() - phmresult = phmutest.main.main(args) + phmresult = phmutest.main.command(line) want = phmutest.summary.Metrics( number_blocks_run=8, passed=8, diff --git a/tests/test_sharing.py b/tests/test_sharing.py index 885cf91..692c655 100644 --- a/tests/test_sharing.py +++ b/tests/test_sharing.py @@ -21,11 +21,11 @@ def test_share_across_with_setup_sharing(): """Show that setup blocks are not shared by share-across-files.""" # --quiet is passed through to unittest to prevent the progress dot printing. with contextlib.redirect_stderr(io.StringIO()) as err: - command = ( + line = ( "--log --config tests/toml/acrossfiles2.toml " "--sharing tests/md/setupnoteardown.md --quiet" ) - phmresult = phmutest.main.main(command.split()) + phmresult = phmutest.main.command(line) want = phmutest.summary.Metrics( number_blocks_run=6, passed=6, @@ -88,12 +88,12 @@ def test_share_across_with_setup_sharing(): def test_setup_across_sharing(): """Run the setup across files example with --sharing.""" with contextlib.redirect_stderr(io.StringIO()) as err: - command = ( + line = ( "docs/setup/across1.md docs/setup/across2.md " "--setup-across-files docs/setup/across1.md --log " "--sharing docs/setup/across1.md" ) - phmresult = phmutest.main.main(command.split()) + phmresult = phmutest.main.command(line) want = phmutest.summary.Metrics( number_blocks_run=6, passed=6, From 64e3b8fe06ca872a6cb98c63c41d6654986b69cb Mon Sep 17 00:00:00 2001 From: Mark Taylor <24257134+tmarktaylor@users.noreply.github.com> Date: Sat, 26 Aug 2023 18:57:11 -0400 Subject: [PATCH 06/27] Bump version to 0.0.2 --- .github/workflows/publish.yml | 2 +- .github/workflows/wheel.yml | 2 +- README.md | 2 +- docs/api.md | 2 +- docs/fixture_py.md | 2 +- mkdocs.yml | 2 +- setup.cfg | 2 +- src/phmutest/__init__.py | 2 +- src/phmutest/fixture.py | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index d10fc42..77e63ff 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -6,7 +6,7 @@ on: workflow_dispatch: env: - ref: v0.0.1 + ref: v0.0.2 jobs: builddist: diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index 2576d01..de46afd 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -5,7 +5,7 @@ on: env: project: phmutest - version: 0.0.1 + version: 0.0.2 command: phmutest jobs: diff --git a/README.md b/README.md index 2abb3db..0f586e1 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# phmutest 0.0.1 +# phmutest 0.0.2 ## Detect broken Python examples in Markdown diff --git a/docs/api.md b/docs/api.md index 0ef2395..d8c5c52 100644 --- a/docs/api.md +++ b/docs/api.md @@ -1,4 +1,4 @@ -# API version 0.0.1 +# API version 0.0.2 ## API - phmutest.tool. diff --git a/docs/fixture_py.md b/docs/fixture_py.md index 72d4c8f..d49bbd7 100644 --- a/docs/fixture_py.md +++ b/docs/fixture_py.md @@ -1,6 +1,6 @@ # src/phmutest/fixture.py ```python -"""v0.0.1 Keyword arguments passed to --fixture function and return type Fixture. +"""v0.0.2 Keyword arguments passed to --fixture function and return type Fixture. These are passed to the fixture function as keyword arguments: diff --git a/mkdocs.yml b/mkdocs.yml index e7efe85..5a7058c 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -1,4 +1,4 @@ -site_name: phmutest 0.0.1 +site_name: phmutest 0.0.2 site_description: Detect broken Python examples in Markdown copyright: Copyright (c) 2023 Mark Taylor docs_dir: _mkdocsin diff --git a/setup.cfg b/setup.cfg index 0905861..0f0b6b2 100644 --- a/setup.cfg +++ b/setup.cfg @@ -7,7 +7,7 @@ license_files = LICENSE # https://setuptools.readthedocs.io/en/latest/userguide/declarative_config.html name = phmutest -version = 0.0.1 +version = 0.0.2 author = Mark Taylor author_email = mark66547ta2@gmail.com description = Detect broken Python examples in Markdown. diff --git a/src/phmutest/__init__.py b/src/phmutest/__init__.py index f102a9c..3b93d0b 100644 --- a/src/phmutest/__init__.py +++ b/src/phmutest/__init__.py @@ -1 +1 @@ -__version__ = "0.0.1" +__version__ = "0.0.2" diff --git a/src/phmutest/fixture.py b/src/phmutest/fixture.py index 8eeebca..267ea41 100644 --- a/src/phmutest/fixture.py +++ b/src/phmutest/fixture.py @@ -1,4 +1,4 @@ -"""v0.0.1 Keyword arguments passed to --fixture function and return type Fixture. +"""v0.0.2 Keyword arguments passed to --fixture function and return type Fixture. These are passed to the fixture function as keyword arguments: From 7ae2d007b6f89660ec8a866994d168fef4c5b20f Mon Sep 17 00:00:00 2001 From: Mark Taylor <24257134+tmarktaylor@users.noreply.github.com> Date: Mon, 28 Aug 2023 09:50:44 -0400 Subject: [PATCH 07/27] Code updates. Comments, renames, unused code. --- .github/workflows/ci.yml | 1 + docs/make_wrapped_examples.py | 8 -------- src/phmutest/cases.py | 11 +++++------ src/phmutest/reader.py | 9 ++++----- src/phmutest/subtest.py | 1 + src/phmutest/summary.py | 8 ++++---- tests/test_patching.py | 6 +++--- 7 files changed, 18 insertions(+), 26 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1ef95ba..f210d80 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -78,6 +78,7 @@ jobs: phmutest docs/advanced/skipif.md --log phmutest docs/advanced/label.md --log phmutest docs/advanced/labelanyfcb.md --log + phmutest tests/md/optionflags.md --log --replmode --fixture tests.test_patching.setflags phmutest tests/md/project.md --report coverage: diff --git a/docs/make_wrapped_examples.py b/docs/make_wrapped_examples.py index e3ed2d8..66d87c8 100644 --- a/docs/make_wrapped_examples.py +++ b/docs/make_wrapped_examples.py @@ -6,14 +6,6 @@ bottom = """``` """ -raw_top = "# \n~~~\n" - -raw_bottom = """~~~ -""" - -text_bottom = """~~~ -""" - def nag(): print("If a new file...") diff --git a/src/phmutest/cases.py b/src/phmutest/cases.py index 1087706..1457def 100644 --- a/src/phmutest/cases.py +++ b/src/phmutest/cases.py @@ -42,7 +42,7 @@ def setUpModule(): _phm_log.append(["setUpModule", "", ""]) $showprogressenter _phm_globals = _phmGlobals(__name__, shareid=$shareid) - $usersetupupdate + $userfixtureglobs $setupblocks $showprogressexit """ @@ -82,9 +82,9 @@ def render_setup_module( ] = 'print("leaving setUpModule.", file=sys.stderr)' if args.fixture: - replacements["usersetupupdate"] = phmutest.subtest.justify( + replacements["userfixtureglobs"] = phmutest.subtest.justify( setup_module_form, - "$usersetupupdate", + "$userfixtureglobs", fixture_globs_update_code, ) return phmutest.subtest.fill_in( @@ -316,7 +316,7 @@ def markdown_file( from phmutest.globs import Globals as _phmGlobals from phmutest.printer import Printer as _phmPrinter from phmutest.skip import sys_tool as _phm_sys -$usersetupfunction +$importuserfixture _phm_globals = None _phm_testcase = unittest.TestCase() @@ -342,7 +342,7 @@ def testfile(args: argparse.Namespace, block_store: phmutest.select.BlockStore) if args.fixture: modulepackage, function_name = get_fixture_parts(args.fixture) replacements[ - "usersetupfunction" + "importuserfixture" ] = f"from {modulepackage} import {function_name} as _phm_user_setup_function" if args.setup_across_files or args.share_across_files or args.fixture: @@ -351,7 +351,6 @@ def testfile(args: argparse.Namespace, block_store: phmutest.select.BlockStore) teardown_code = render_teardown_module(args, block_store) replacements["teardownmodule"] = "\n\n" + teardown_code - sequence_number = 1 for sequence_number, path in enumerate(args.files, start=1): test_classes += "\n\n" + markdown_file(args, block_store, path, sequence_number) if not test_classes.endswith("\n"): diff --git a/src/phmutest/reader.py b/src/phmutest/reader.py index 228b8bf..51d1c37 100644 --- a/src/phmutest/reader.py +++ b/src/phmutest/reader.py @@ -39,7 +39,6 @@ def __init__(self, match: Any): self.payload = "" if self.ntype == NodeType.FENCED_CODE_BLOCK: self.info_string = match["info_string"].strip() - self.payload = match["contents"] if match["indent"]: self.payload = self.dedent(match["indent"], match["contents"]) else: @@ -63,7 +62,7 @@ def set_line_numbers(self, start_line: int, end_line: int) -> None: @staticmethod def dedent(indent: str, contents: str) -> str: - """De-indent FCB lines. All lines and fences assumed to have the indent.""" + """De-indent any indented FCB lines.""" dedented = [] lines = contents.splitlines() for line in lines: @@ -76,7 +75,7 @@ class PositionToLineNumber: """Given position in multiline string determine the line number.""" def __init__(self, text: str): - """Initialize the position to line number mapping.""" + """Remember the line number for some positions in the text string.""" self.maxpos = len(text) self.mapping = {} pos = 0 @@ -87,10 +86,10 @@ def __init__(self, text: str): pos += 1 def get_line(self, position: int) -> int: - """Return the line number that contains character position in the file.""" + """Return the line number that contains character position in the string.""" assert position <= ( self.maxpos - ), f"must not be a lot beyond EOF {position}/{self.maxpos}." + ), f"must not be a lot beyond the end {position}/{self.maxpos}." while position > 0: if position not in self.mapping: position -= 1 diff --git a/src/phmutest/subtest.py b/src/phmutest/subtest.py index 4a60792..eceb9ea 100644 --- a/src/phmutest/subtest.py +++ b/src/phmutest/subtest.py @@ -1,3 +1,4 @@ +"""Generate code to test Python FCBs.""" import argparse import re import textwrap diff --git a/src/phmutest/summary.py b/src/phmutest/summary.py index 930055c..88cf532 100644 --- a/src/phmutest/summary.py +++ b/src/phmutest/summary.py @@ -214,15 +214,15 @@ def show_log(log: Log) -> None: """Print a table of the log entries.""" if log: empty_3rd_col = not any([entry[2] for entry in log]) - colulmn_title = "location|label" + column_title = "location|label" if empty_3rd_col: - titles = [colulmn_title, "result"] + titles = [column_title, "result"] log2 = [[entry[0], entry[1]] for entry in log] else: - titles = [colulmn_title, "result", "skip reason"] + titles = [column_title, "result", "skip reason"] log2 = log titled_log = [titles] - filled_log = dot_fill_first_column(log2, len(colulmn_title)) + filled_log = dot_fill_first_column(log2, len(column_title)) titled_log.extend(filled_log) show_table(titled_log) diff --git a/tests/test_patching.py b/tests/test_patching.py index 41ab766..5b323c3 100644 --- a/tests/test_patching.py +++ b/tests/test_patching.py @@ -165,7 +165,7 @@ def test_modify_docstring_patch(capsys, endswith_checker): assert phmresult2.is_success is False -class FailFastRunner(phmutest.session.ExampleOutcomeRunner): +class OptionflagRunner(phmutest.session.ExampleOutcomeRunner): """Doctest Runner to set optionflags.""" myflags = doctest.NORMALIZE_WHITESPACE | doctest.ELLIPSIS @@ -177,10 +177,10 @@ def __init__(self, **kwargs): # type: ignore def setflags(**kwargs): - """phmutest fixture function replaces the doctest.DocTestRunner.""" + """phmutest fixture function to patch the doctest runner.""" with ExitStack() as stack: stack.enter_context( - mock.patch("phmutest.session.ExampleOutcomeRunner", FailFastRunner) + mock.patch("phmutest.session.ExampleOutcomeRunner", OptionflagRunner) ) fixture = phmutest.fixture.Fixture( globs=None, repl_cleanup=stack.pop_all().close From 5cac5d90699bc439b59c8fcb447bddc27d7e5657 Mon Sep 17 00:00:00 2001 From: Mark Taylor <24257134+tmarktaylor@users.noreply.github.com> Date: Mon, 28 Aug 2023 09:52:26 -0400 Subject: [PATCH 08/27] Documents updates. Fixes, clarifications. --- README.md | 40 ++++++++++++++++++++++++++++++++-------- checklist.txt | 1 - docs/advanced.md | 22 ++++++++++------------ docs/demos.md | 11 ++++++++++- docs/fix/code/chdir.md | 2 +- docs/howitworks.md | 6 +++--- docs/sessionmode.md | 3 ++- docs/setup/across1.md | 1 - docs/setup/setup.md | 18 ++++++++++-------- tests/test_subprocess.py | 25 +++++++++++++++++++++++++ 10 files changed, 93 insertions(+), 36 deletions(-) diff --git a/README.md b/README.md index 0f586e1..18bc22a 100644 --- a/README.md +++ b/README.md @@ -37,9 +37,9 @@ Designated and stable **patch points** for Python standard library These features require adding tool specific HTML comment **directives** to the Markdown. Because directives are HTML comments they are not visible in -rendered Markdown. View directives on [Latest README on GitHub][1] +rendered Markdown. View directives on GitHub by pressing the `Code` button in the banner at the top of the file. -| [Here](docs/advanced.md). +| [Advanced feature details](docs/advanced.md). - Assign test group names to blocks. Command line options select or deselect test groups by name. @@ -81,6 +81,8 @@ by pressing the `Code` button in the banner at the top of the file. [Patch points](#patch-points) | [Related projects](#related-projects) +[Sections](docs/demos.md#sections) | +[Demos](docs/demos.md#demos) | [Changes](docs/recent_changes.md) | [Contributions](CONTRIBUTING.md) @@ -137,9 +139,9 @@ args.log: 'True' location|label result -------------- ------ -README.md:92.. pass -README.md:98.. pass -README.md:111. pass +README.md:94.. pass +README.md:100. pass +README.md:113. pass -------------- ------ ``` @@ -334,15 +336,15 @@ command line less the phmutest, like this: Feel free to **unittest.mock.patch()** at these places in the code and not worry about breakage in future versions. Look for examples in tests/test_patching.py. +### List of patch points -| patched function | purpose +| patched function | purpose | :--------------------------------: | :----------: | phmutest.direct.directive_finders() | Add directive aliases | phmutest.fenced.python_matcher() | Add detect Python from FCB info string | phmutest.session.modify_docstring() | Inspect/modify REPL text before testing | phmutest.reader.post() | Inspect/modify DocNode detected in Markdown -List of patch points ## Related projects - phmdoctest @@ -355,6 +357,28 @@ List of patch points - pytest-phmdoctest - pytest-codeblocks + +Major differences between phmutest and phmdoctest: + +- phmutest treats each Markdown file as a single long example. phmdoctest + tests each FCB separately. Adding a share-names directive is necessary to + extend an example across FCBs within a file. +- Only phmutest can extend an example across files. +- phmutest uses Python standard library unittest and doctest as test runners. + phmdoctest writes a pytest testfile for each Markdown file + which requires a separate step to run. The testfiles then need to be discarded. +- phmdoctest offers two pytest fixtures that can be used in a pytest test case + to generate and run a testfile in one step. +- phmutest generates tests for multiple Markdown files in one step + and runs them internally so there are no leftover test files. +- The --fixture test suite initialization and cleanup is only available on phmutest. + phmdoctest offers some initialization behaviour using an FCB with a setup + directive and its --setup-doctest option and it only works with sessions. + See phmdoctest documentation "Execution Context" + section for an explanation. +- phmutest does not support inline annotations. + + [1]: https://github.com/tmarktaylor/phmutest/blob/master/README.md?plain=1 [3]: https://github.github.com/gfm/#fenced-code-blocks [11]: https://github.github.com/gfm/#info-string @@ -364,6 +388,6 @@ List of patch points [13]: https://ci.appveyor.com/project/tmarktaylor/phmutest [15]: https://docs.pytest.org/en/stable [16]: https://tmarktaylor.github.io/pytest-phmdoctest -[17]: https://pypi.python.org/pypi/phmutest +[17]: https://pypi.python.org/pypi/phmdoctest diff --git a/checklist.txt b/checklist.txt index d499983..b32cc76 100644 --- a/checklist.txt +++ b/checklist.txt @@ -31,7 +31,6 @@ Checklist Configure trusted publishing on both test PYPI and PYPI if this is the first release. - On GitHub manually dispatch publish.yml - - On GitHub manually dispatch wheel.yml to try installing from PYPI Alternate manual steps - On GitHub manually dispatch build.yml on the release ref diff --git a/docs/advanced.md b/docs/advanced.md index 31c5f99..9cb4ae4 100644 --- a/docs/advanced.md +++ b/docs/advanced.md @@ -6,7 +6,7 @@ Directives are HTML comments associated with fenced code blocks. They are edited into the Markdown file immediately before a fenced code block. It is OK if other HTML comments are present either before or after. A directive is recognized if it is in a continuous -series of either HTML comments or single blanks lines +series of either HTML comments or single blank lines ending at the fenced code block. The `` directive in the raw Markdown below @@ -25,8 +25,10 @@ Hello World! ~~~ Since directives are HTML comments they are not visible in rendered Markdown. -View directives on [Latest README on GitHub][1] -by pressing the `Code` button in the banner at the top of the file. +Reveal directives by navigating to the repository on GitHub [here][1] and +pressing the `Code` button in the banner at the top of the file. + +### List of directives | Directive HTML comment | Use on FCBs | Ok in REPL mode | :--------------------------------: | :----------: | :----------: @@ -37,10 +39,7 @@ by pressing the `Code` button in the banner at the top of the file. | `` | code | No | `` | code | yes -List of directives - - - +### phmdoctest directives recognized by phmutest | phmdoctest directive | phmutest equivalent | :---------------------------------: | :---------: @@ -52,8 +51,7 @@ List of directives | `` | `` | `` | `` -phmdoctest directives recognized by phmutest. - +### Directive Examples [Skip directive](advanced/skip.md) @@ -61,7 +59,7 @@ phmdoctest directives recognized by phmutest. [Label directive, label and skipif example](advanced/label.md) -### Test groups +## Test groups The test group directive identifies Python FCBs belonging to the group NAME. Test groups can be included or excluded from testing by the --select and @@ -72,7 +70,7 @@ Test groups can be included or excluded from testing by the --select and - The --report option lists excluded blocks. - The --summary option shows the number of deselected blocks. -### Setup and teardown +## Setup and teardown Blocks can be designated setup or teardown blocks by adding the `` and `` directives. @@ -97,6 +95,6 @@ setup blocks are shared to **all** FILEs. Teardown blocks in FILE are run by **unittest.tearDownModule()**. -[1]: https://github.com/tmarktaylor/phmutest/blob/master/README.md?plain=1 +[1]: https://github.com/tmarktaylor/phmutest/blob/master/tests/md/directive1.md?plain=1 [2]: https://tmarktaylor.github.io/phmdoctest diff --git a/docs/demos.md b/docs/demos.md index df204ef..fc36cb9 100644 --- a/docs/demos.md +++ b/docs/demos.md @@ -1,4 +1,13 @@ -# Index of Demos +# Sections + +- [Advanced feature details](advanced.md) +- [How it works](howitworks.md) +- [Code mode](codemode.md) +- [Session mode](sessionmode.md) +- [Call from Python example](callfrompython.md) +- [Api](api.md) + +# Demos - [REPL Mode](replmode.md) - [fixture change workdir](fix/code/chdir.md) diff --git a/docs/fix/code/chdir.md b/docs/fix/code/chdir.md index 8783447..394b7d3 100644 --- a/docs/fix/code/chdir.md +++ b/docs/fix/code/chdir.md @@ -12,7 +12,7 @@ note that dots separate the folders and the function. The function **change_dir()** changes the working directory to `docs/fix/code`. This allows the example to use a -pathname relative to `docs/init/fix/code` for the file. +pathname relative to `docs/fix/code` for the file. **change_dir()** also calls **unittest.addModuleCleanup()** to have unittest restore the working directory when unittest terminates. diff --git a/docs/howitworks.md b/docs/howitworks.md index a595789..44a9788 100644 --- a/docs/howitworks.md +++ b/docs/howitworks.md @@ -4,7 +4,7 @@ The Markdown [GFM fenced code blocks][1] are identified as Python by looking at the [info string][2]. -To be treated as Python the FCB [info string][2] should start +To be treated as Python the FCB info string should start with one of these: - python @@ -36,6 +36,8 @@ The --report option shows these fenced block details: - output block - directives +### role field values + | role | meaning | referred to in the docs as | :-------------| :-----------------: | :-----------------: | Role.CODE | Python code block | code blocks or Python code blocks @@ -43,8 +45,6 @@ The --report option shows these fenced block details: | Role.SESSION | Python REPL block | session or REPL blocks | Role.NOROLE | all other blocks -interpret the --report role field values - The report also shows a list called Deselected blocks: at the end that lists each block that was excluded by the --select or --deselect option. diff --git a/docs/sessionmode.md b/docs/sessionmode.md index 414ab3e..4975d54 100644 --- a/docs/sessionmode.md +++ b/docs/sessionmode.md @@ -14,7 +14,7 @@ file. The fixture function can return global variables that are used when running doctest. The fixture function returns a dict. -The items become globs as described by [doctest][1]. +The items become [globs][2] as described by doctest. Examples should avoid assignments to the fixture glob names. To see what happens see tests/test_rebind.py. @@ -54,4 +54,5 @@ to the standard output stream shared with doctest's verbose printing. [1]: https://docs.python.org/3/library/doctest.html +[2]: https://docs.python.org/3/library/doctest.html#doctest.DocTest.globs diff --git a/docs/setup/across1.md b/docs/setup/across1.md index b302a2f..a840e36 100644 --- a/docs/setup/across1.md +++ b/docs/setup/across1.md @@ -100,4 +100,3 @@ docs/setup/across1.md:53 teardown pass docs/setup/across1.md:68 teardown pass --------------------------------- ------ ``` -[1]: https://github.com/tmarktaylor/phmutest/docs/setup diff --git a/docs/setup/setup.md b/docs/setup/setup.md index 0e8ea32..832e3a9 100644 --- a/docs/setup/setup.md +++ b/docs/setup/setup.md @@ -10,8 +10,11 @@ in the Markdown file. The main reason to use setup blocks is so that the teardown blocks will run in the event a post setup block fails. Note that teardown will only run if all the setup blocks succeed. -We use ExitStack to create a single function call to cleanup the -temporary directory and change back to the original working directory. +We use ExitStack to create a single function call to do cleanup: + +- change back to the original working directory +- cleanup the temporary directory + We use with ExitStack to assure cleanup when a statement in the with suite raises an exception. @@ -91,11 +94,10 @@ args.log: 'True' location|label result ------------------------------- ------ -docs/setup/setup.md:19 setup... pass -docs/setup/setup.md:32 setup... pass -docs/setup/setup.md:44......... pass -docs/setup/setup.md:56 teardown pass -docs/setup/setup.md:71 teardown pass +docs/setup/setup.md:22 setup... pass +docs/setup/setup.md:35 setup... pass +docs/setup/setup.md:47......... pass +docs/setup/setup.md:59 teardown pass +docs/setup/setup.md:74 teardown pass ------------------------------- ------ ``` -[1]: https://github.com/tmarktaylor/phmutest/docs/setup diff --git a/tests/test_subprocess.py b/tests/test_subprocess.py index 92321bc..eda84df 100644 --- a/tests/test_subprocess.py +++ b/tests/test_subprocess.py @@ -38,3 +38,28 @@ def test_callfrompython(): ] completed = subprocess.run(commandline) assert completed.returncode == 0 + + +def test_fixture_patching(): + """Run phmutest in a subprocess, call a fixture that does mock.patch(). + + This shows a --fixture function using contextlib.ExitStack to install + a patch using unittest.mock.patch() with a shell invocation of + phmutest. It runs the same example as + test_patching::test_doctest_optionflags_patch. + + It should be possible to do the same patch when not in --replmode + by calling unittest.addModuleCleanup(stack.pop_all().close). + """ + commandline = [ + sys.executable, + "-m", + "phmutest", + "tests/md/optionflags.md", + "--log", + "--replmode", + "--fixture", + "tests.test_patching.setflags", + ] + completed = subprocess.run(commandline) + assert completed.returncode == 0 From ad70b4f178194e28fdd21097b70cb7300d060e3d Mon Sep 17 00:00:00 2001 From: Mark Taylor <24257134+tmarktaylor@users.noreply.github.com> Date: Tue, 29 Aug 2023 12:54:08 -0400 Subject: [PATCH 09/27] Update setup classifiers, fix docstring, renames. --- setup.cfg | 6 ++++-- src/phmutest/fenced.py | 2 +- src/phmutest/session.py | 24 ++++++++++++------------ 3 files changed, 17 insertions(+), 15 deletions(-) diff --git a/setup.cfg b/setup.cfg index 0f0b6b2..6fbcd86 100644 --- a/setup.cfg +++ b/setup.cfg @@ -19,18 +19,20 @@ project_urls = Source = https://github.com/tmarktaylor/phmutest/ classifiers = Development Status :: 4 - Beta + Environment :: Console Intended Audience :: Developers License :: OSI Approved :: MIT License + Operating System :: OS Independent Programming Language :: Python :: 3 :: Only Programming Language :: Python :: 3.8 Programming Language :: Python :: 3.9 Programming Language :: Python :: 3.10 Programming Language :: Python :: 3.11 Programming Language :: Python :: Implementation :: PyPy - Topic :: Software Development :: Testing Topic :: Software Development :: Documentation Topic :: Software Development :: Libraries :: Python Modules - Topic :: Utilities + Topic :: Software Development :: Quality Assurance + Topic :: Software Development :: Testing keywords = documentation markdown diff --git a/src/phmutest/fenced.py b/src/phmutest/fenced.py index 8af5003..b8c3ef3 100644 --- a/src/phmutest/fenced.py +++ b/src/phmutest/fenced.py @@ -115,7 +115,7 @@ def set_link_to_output(self, fenced_block: "FencedBlock") -> None: self.output = fenced_block def get_output_contents(self) -> str: - """Return contents of linked output block or empty str if no link.""" + """Return contents of linked output block.""" assert self.output, "Must be called on a block with Role.OUTPUT" return self.output.contents diff --git a/src/phmutest/session.py b/src/phmutest/session.py index 2e6c762..8c12cbd 100644 --- a/src/phmutest/session.py +++ b/src/phmutest/session.py @@ -23,22 +23,22 @@ class ExampleOutcomeRunner(doctest.DocTestRunner): def __init__(self, **kwargs): # type: ignore super().__init__(**kwargs) - self.outcomes = {} - self.number_of_failures = 0 - self.number_of_errors = 0 + self.phm_outcomes = {} + self.phm_number_of_failures = 0 + self.phm_number_of_errors = 0 def report_success(self, out, test, example, got): # type: ignore - self.outcomes[example.lineno + 1] = "pass" + self.phm_outcomes[example.lineno + 1] = "pass" super().report_success(out, test, example, got) def report_failure(self, out, test, example, got): # type: ignore - self.outcomes[example.lineno + 1] = "failed" - self.number_of_failures += 1 + self.phm_outcomes[example.lineno + 1] = "failed" + self.phm_number_of_failures += 1 super().report_failure(out, test, example, got) def report_unexpected_exception(self, out, test, example, exc_info): # type: ignore - self.outcomes[example.lineno + 1] = "error" - self.number_of_errors += 1 + self.phm_outcomes[example.lineno + 1] = "error" + self.phm_number_of_errors += 1 super().report_unexpected_exception(out, test, example, exc_info) @@ -156,10 +156,10 @@ def run_one_file( runner.run(tests[0]) # Determine each overall block result for the log from the file's Example outcomes. - runner_lineno = set(runner.outcomes) + runner_lineno = set(runner.phm_outcomes) for block, line_range in zip(tested_blocks, line_ranges): block_lineno = set(line_range) & runner_lineno - if block_outcomes := [runner.outcomes[line] for line in block_lineno]: + if block_outcomes := [runner.phm_outcomes[line] for line in block_lineno]: # Note- If there is a fail-fast there will be no outcomes for a # later block. result = get_result(block_outcomes) @@ -172,7 +172,7 @@ def run_one_file( lineno_log.sort() log = [entry for lineno, entry in lineno_log] return SessionResult( - log, runner.number_of_failures, runner.number_of_errors, docstring + log, runner.phm_number_of_failures, runner.phm_number_of_errors, docstring ) @@ -220,7 +220,7 @@ def process_user_fixture( globs = {} else: # If --sharing ".", show the fixture globs. - if phmutest.cases.is_verbose_sharing(args, Path("not_a_md_file")): + if phmutest.cases.is_verbose_sharing(args, Path("placeholder")): glob_names = ", ".join(globs.keys()) print(f"{args.fixture} is sharing: {glob_names}") From ab825284e643752358356c942d17d1a9b603a2cf Mon Sep 17 00:00:00 2001 From: Mark Taylor <24257134+tmarktaylor@users.noreply.github.com> Date: Wed, 30 Aug 2023 08:43:25 -0400 Subject: [PATCH 10/27] Remove import io, sys, contextlib from testfile. The imports io and contextlib were not needed. Replaced references to sys with phmutest.skip.SysTool. --- docs/codemode.md | 3 --- docs/generated_project_py.md | 3 --- docs/generated_share_demo_py.md | 3 --- docs/junit.md | 1 - docs/recent_changes.md | 4 +++- src/phmutest/cases.py | 13 ++++++------- src/phmutest/globs.py | 6 ------ tests/py/generated_project.py | 3 --- tests/py/generated_sharedemo.py | 3 --- tests/test_globs.py | 9 +++------ 10 files changed, 12 insertions(+), 36 deletions(-) diff --git a/docs/codemode.md b/docs/codemode.md index 671116f..32e0dfc 100644 --- a/docs/codemode.md +++ b/docs/codemode.md @@ -72,9 +72,6 @@ in its docstring. These imports will not be shown by --sharing because they are already imported at the top of the testfile. -- import contextlib -- import io -- import sys - import unittest ## Share across files diff --git a/docs/generated_project_py.md b/docs/generated_project_py.md index 21b7b81..7ec54be 100644 --- a/docs/generated_project_py.md +++ b/docs/generated_project_py.md @@ -1,9 +1,6 @@ # tests/py/generated_project.py ```python """Python unittest test file generated by Python Package phmutest.""" -import contextlib -import io -import sys import unittest from phmutest.globs import Globals as _phmGlobals diff --git a/docs/generated_share_demo_py.md b/docs/generated_share_demo_py.md index 846faf1..8f655e3 100644 --- a/docs/generated_share_demo_py.md +++ b/docs/generated_share_demo_py.md @@ -1,9 +1,6 @@ # tests/py/generated_sharedemo.py ```python """Python unittest test file generated by Python Package phmutest.""" -import contextlib -import io -import sys import unittest from phmutest.globs import Globals as _phmGlobals diff --git a/docs/junit.md b/docs/junit.md index e2b4d8e..d353024 100644 --- a/docs/junit.md +++ b/docs/junit.md @@ -17,7 +17,6 @@ FCB with an "error" status. ```python import contextlib import io -import sys import phmutest.main diff --git a/docs/recent_changes.md b/docs/recent_changes.md index 38c32ca..ae544b0 100644 --- a/docs/recent_changes.md +++ b/docs/recent_changes.md @@ -5,5 +5,7 @@ 0.0.2 - 2023-xx-xx -- add main.command(). +- Add main.command(). +- Remove import io, sys, contextlib from generated testfile. +- Setup classifiers, cleanups, Docs, renames. diff --git a/src/phmutest/cases.py b/src/phmutest/cases.py index 1457def..71ba3c7 100644 --- a/src/phmutest/cases.py +++ b/src/phmutest/cases.py @@ -76,10 +76,12 @@ def render_setup_module( replacements["shareid"] = shareid replacements["setupblocks"] = deindented_setup_blocks if args.progress: - replacements["showprogressenter"] = 'print("setUpModule()...", file=sys.stderr)' + replacements[ + "showprogressenter" + ] = '_phm_sys.stderr_printer("setUpModule()...")' replacements[ "showprogressexit" - ] = 'print("leaving setUpModule.", file=sys.stderr)' + ] = '_phm_sys.stderr_printer("leaving setUpModule.")' if args.fixture: replacements["userfixtureglobs"] = phmutest.subtest.justify( @@ -129,10 +131,10 @@ def render_teardown_module( if args.progress: replacements[ "showprogressenter" - ] = 'print("tearDownModule()...", file=sys.stderr)' + ] = '_phm_sys.stderr_printer("tearDownModule()...")' replacements[ "showprogressexit" - ] = 'print("leaving tearDownModule.", file=sys.stderr)' + ] = '_phm_sys.stderr_printer("leaving tearDownModule.")' return phmutest.subtest.fill_in( teardown_module_form, @@ -308,9 +310,6 @@ def markdown_file( testfile_form = '''\ """Python unittest test file generated by Python Package phmutest.""" -import contextlib -import io -import sys import unittest from phmutest.globs import Globals as _phmGlobals diff --git a/src/phmutest/globs.py b/src/phmutest/globs.py index 7352e0c..dbaa34d 100644 --- a/src/phmutest/globs.py +++ b/src/phmutest/globs.py @@ -125,12 +125,6 @@ def update( # cause check_attribute_name() to raise an exception. # Users might have one or more of them in their code block. # So we remove them from additions. - if "contextlib" in additions and "contextlib" in self.original_attributes: - _ = additions.pop("contextlib", None) - if "io" in additions and "io" in self.original_attributes: - _ = additions.pop("io", None) - if "sys" in additions and "sys" in self.original_attributes: - _ = additions.pop("sys", None) if "unittest" in additions and "unittest" in self.original_attributes: _ = additions.pop("unittest", None) added_names = ", ".join(additions.keys()) diff --git a/tests/py/generated_project.py b/tests/py/generated_project.py index 44ce6eb..371d116 100644 --- a/tests/py/generated_project.py +++ b/tests/py/generated_project.py @@ -1,7 +1,4 @@ """Python unittest test file generated by Python Package phmutest.""" -import contextlib -import io -import sys import unittest from phmutest.globs import Globals as _phmGlobals diff --git a/tests/py/generated_sharedemo.py b/tests/py/generated_sharedemo.py index 9806159..05b6d1b 100644 --- a/tests/py/generated_sharedemo.py +++ b/tests/py/generated_sharedemo.py @@ -1,7 +1,4 @@ """Python unittest test file generated by Python Package phmutest.""" -import contextlib -import io -import sys import unittest from phmutest.globs import Globals as _phmGlobals diff --git a/tests/test_globs.py b/tests/test_globs.py index 91ce892..65f1643 100644 --- a/tests/test_globs.py +++ b/tests/test_globs.py @@ -93,18 +93,15 @@ def test_update_name_ifpops(self): """Try an update with names that get conditionally popped before the update.""" # Exercises the if XXX in additions.pop('XXX', None) lines items = { - "contextlib": None, - "io": None, - "sys": None, + "unittest": None, "example_variable": 1111, # does not get popped } self.globs.update(additions=items) # Note get_names() does not include sys since sys is a global of this file. - assert self.globs.get_names() == {"example_variable", "contextlib", "io"} + assert self.globs.get_names() == {"example_variable", "unittest"} assert self.globs.copy() == { "example_variable": 1111, - "contextlib": None, - "io": None, + "unittest": None, } def test_more_additions(self): From 9af47788cecb3672ee15dd6ebdfa34c94ecf1806 Mon Sep 17 00:00:00 2001 From: Mark Taylor <24257134+tmarktaylor@users.noreply.github.com> Date: Wed, 13 Sep 2023 16:18:57 -0400 Subject: [PATCH 11/27] Changes due to Markdown linting. There may be some cleanups too. --- code_of_conduct.md | 1 - docs/advanced.md | 9 ++-- docs/advanced/label.md | 9 ++-- docs/advanced/labelanyfcb.md | 8 ++-- docs/advanced/skip.md | 32 ++++++++------ docs/advanced/skipif.md | 11 ++--- docs/fix/code/chdir.md | 20 ++++----- docs/fix/code/chdir_py.md | 1 + docs/fix/code/globdemo.md | 31 +++++++------- docs/fix/code/globdemo_py.md | 1 + docs/fix/repl/drink.md | 9 ++-- docs/fix/repl/drink_py.md | 1 + docs/fixture_py.md | 1 + docs/generated_project_py.md | 9 ++-- docs/generated_share_demo_py.md | 21 ++++----- docs/group/deselect.md | 21 ++++----- docs/group/select.md | 35 ++++++++------- docs/junit.md | 1 - docs/make_wrapped_examples.py | 2 +- docs/repl/repl1.md | 4 +- docs/repl/repl2.md | 3 +- docs/repl/repl3.md | 4 +- docs/repl/replshare_demo.md | 20 ++++----- docs/sessionmode.md | 3 -- docs/setup/across1.md | 52 ++++++++++++++--------- docs/setup/across2.md | 8 ++-- docs/setup/setup.md | 54 +++++++++++++++--------- docs/share/file1.md | 6 +-- docs/share/file2.md | 11 +++-- docs/share/file3.md | 9 ++-- docs/share/share_demo.md | 40 ++++++++++-------- tests/fail/raiser.md | 5 ++- tests/md/bad_session_output.md | 5 ++- tests/md/bad_skipif_number.md | 5 ++- tests/md/badsetup.md | 22 ++++++++-- tests/md/badteardown.md | 21 +++++++-- tests/md/cases.md | 7 ++- tests/md/code_groups.md | 1 - tests/md/directive1.md | 19 ++++++--- tests/md/directive2.md | 26 +++++++++--- tests/md/directives.md | 32 ++++++++++---- tests/md/example2.md | 33 ++++++++++++--- tests/md/extra_line_in_output.md | 9 +++- tests/md/legacy_directives.md | 41 +++++++++++++----- tests/md/missing_some_output.md | 7 ++- tests/md/more_directives.md | 24 ++++++++--- tests/md/multi_label.md | 22 ++++++++-- tests/md/no_code_blocks.md | 4 +- tests/md/no_fenced_code_blocks.md | 2 +- tests/md/noreadernodes.md | 7 +-- tests/md/one_mark_skip.md | 4 ++ tests/md/optionflags.md | 11 +++-- tests/md/output_has_blank_lines.md | 6 ++- tests/md/patching1.md | 12 ++++-- tests/md/printer.md | 13 +++--- tests/md/project.md | 12 +++--- tests/md/pythonmatch.md | 17 +++++--- tests/md/reader.md | 1 - tests/md/readerfcb.md | 3 +- tests/md/replerror.md | 17 ++++---- tests/md/report.md | 13 +++--- tests/md/setupnoteardown.md | 18 +++++--- tests/md/setupto.md | 6 +-- tests/md/sharedto.md | 13 +++--- tests/md/sharedto2.md | 8 ++-- tests/md/shareimports1.md | 5 +-- tests/md/shareimports2.md | 6 ++- tests/md/unexpected_output.md | 9 +++- tests/py/generated_project.py | 8 ++-- tests/py/generated_sharedemo.py | 20 ++++----- tests/test_docs.py | 68 +++++++++++++++++++++++------- tests/test_fenced.py | 26 ++++++------ tests/test_tool.py | 6 +-- 73 files changed, 644 insertions(+), 387 deletions(-) diff --git a/code_of_conduct.md b/code_of_conduct.md index 4980059..9eadd94 100644 --- a/code_of_conduct.md +++ b/code_of_conduct.md @@ -131,4 +131,3 @@ For answers to common questions about this code of conduct, see the FAQ at [Mozilla CoC]: https://github.com/mozilla/diversity [FAQ]: https://www.contributor-covenant.org/faq [translations]: https://www.contributor-covenant.org/translations - diff --git a/docs/advanced.md b/docs/advanced.md index 9cb4ae4..ec21c41 100644 --- a/docs/advanced.md +++ b/docs/advanced.md @@ -12,14 +12,17 @@ ending at the fenced code block. The `` directive in the raw Markdown below is associated with the FCB. -~~~ +~~~txt + ```python print("Hello World!") ``` + Expected Output -``` + +```expected-output Hello World! ``` ~~~ @@ -94,7 +97,5 @@ The file's setup blocks are run by **unittest.setUpModule()** and the names assi setup blocks are shared to **all** FILEs. Teardown blocks in FILE are run by **unittest.tearDownModule()**. - [1]: https://github.com/tmarktaylor/phmutest/blob/master/tests/md/directive1.md?plain=1 [2]: https://tmarktaylor.github.io/phmdoctest - diff --git a/docs/advanced/label.md b/docs/advanced/label.md index b9b8777..053b3ba 100644 --- a/docs/advanced/label.md +++ b/docs/advanced/label.md @@ -6,14 +6,14 @@ Here is the command and terminal output of the label, skip, and skipif directive example Please note this example is in a different file than this page's source. -``` +```shell phmutest tests/md/directive1.md --log --replmode ``` In the output, note the label "doctest_print_coffee" shows after tests/md/directive1.md:69 below. -``` +```txt log: args.files: 'tests/md/directive1.md' args.replmode: 'True' @@ -21,8 +21,7 @@ args.log: 'True' location|label result skip reason ---------------------------------------------- ------ ------------- -tests/md/directive1.md:35..................... skip phmutest-skip -tests/md/directive1.md:69 doctest_print_coffee pass +tests/md/directive1.md:41..................... skip phmutest-skip +tests/md/directive1.md:78 doctest_print_coffee pass ---------------------------------------------- ------ ------------- ``` - diff --git a/docs/advanced/labelanyfcb.md b/docs/advanced/labelanyfcb.md index c26cea7..777ed94 100644 --- a/docs/advanced/labelanyfcb.md +++ b/docs/advanced/labelanyfcb.md @@ -1,4 +1,5 @@ # label directive on any FCB + On any fenced code block, the label directive identifies the block for later retrieval by the class **phmdoctest.tool.FCBChooser()**. The `FCBChooser` is used separately from phmutest in @@ -9,13 +10,15 @@ phmutest. The directive value can be any string. [tool API](../api.md) Here is a YAML FCB with a `` label directive. + + ```yml theme: readthedocs ``` -Here is example code to retrieve the directive. Note that FCBs typically +Here is example code to retrieve the YAML FCB. Note that FCBs typically end with a newline. ```python @@ -25,7 +28,6 @@ fcb_contents = chooser.select(label="my-label") print(fcb_contents[0], end="") ``` -``` +```expected-output theme: readthedocs ``` - diff --git a/docs/advanced/skip.md b/docs/advanced/skip.md index b34ca5c..9ce0858 100644 --- a/docs/advanced/skip.md +++ b/docs/advanced/skip.md @@ -7,18 +7,21 @@ This example does not have a skip directive. ```python print("Hello World!") ``` -``` + +```expected-output Hello World! ``` This example has `` before the FCB. - + ```python print("Hello World!") + ``` -``` + +```expected-output Bad expected output. ``` @@ -30,27 +33,30 @@ It prevents the test from failing the expected output check. ```python print("Hello World!") ``` + -``` + +```expected-output Bad expected output. ``` ## phmutest command line -``` + +```shell phmutest docs/advanced/skip.md --log ``` ## phmutest output -``` + +```txt log: args.files: 'docs/advanced/skip.md' args.log: 'True' -location|label result skip reason ------------------------- ------ ------------- -docs/advanced/skip.md:7. pass -docs/advanced/skip.md:18 skip phmutest-skip -docs/advanced/skip.md:30 pass ------------------------- ------ ------------- +location|label result skip reason +------------------------- ------ ------------- +docs/advanced/skip.md:7 o pass +docs/advanced/skip.md:19. skip phmutest-skip +docs/advanced/skip.md:33. pass +------------------------- ------ ------------- ``` - diff --git a/docs/advanced/skipif.md b/docs/advanced/skipif.md index 76223a0..106b655 100644 --- a/docs/advanced/skipif.md +++ b/docs/advanced/skipif.md @@ -9,32 +9,33 @@ skip reason "requires Python >= 3.N". This test case will only run when Python is version 3.999 or higher. + ```python import sys b = 10 print(b) assert sys.version_info >= (3, 999) ``` -``` + +```expected-output 10 ``` ## phmutest command line -``` +```shell phmutest docs/advanced/skipif.md --log ``` ## phmutest expected output -``` +```txt log: args.files: 'docs/advanced/skipif.md' args.log: 'True' location|label result skip reason -------------------------- ------ ------------------------ -docs/advanced/skipif.md:12 skip requires Python >= 3.999 +docs/advanced/skipif.md:13 skip requires Python >= 3.999 -------------------------- ------ ------------------------ ``` - diff --git a/docs/fix/code/chdir.md b/docs/fix/code/chdir.md index 394b7d3..60f62cc 100644 --- a/docs/fix/code/chdir.md +++ b/docs/fix/code/chdir.md @@ -17,8 +17,7 @@ pathname relative to `docs/fix/code` for the file. **change_dir()** also calls **unittest.addModuleCleanup()** to have unittest restore the working directory when unittest terminates. - -## Python fenced code block and expected output under test. +## Python fenced code block and expected output under test This is the Python example we want to check. @@ -36,20 +35,22 @@ print(contents, end="") ``` Expected output: -``` + +```expected-output Demonstrate changed working directory. ``` -## phmutest command line. +## phmutest command line -``` +```shell phmutest docs/fix/code/chdir.md --fixture docs.fix.code.chdir.change_dir --log ``` -## phmutest output. +## phmutest output Terminal output after the `OK` line. -``` + +```txt log: args.files: 'docs/fix/code/chdir.md' args.fixture: 'docs.fix.code.chdir.change_dir' @@ -59,10 +60,9 @@ location|label result ------------------------- ------ setUpModule.............. change cwd............... -docs/fix/code/chdir.md:25 pass -docs/fix/code/chdir.md:29 pass +docs/fix/code/chdir.md:24 pass +docs/fix/code/chdir.md:28 pass tearDownModule........... restore cwd.............. ------------------------- ------ ``` - diff --git a/docs/fix/code/chdir_py.md b/docs/fix/code/chdir_py.md index 94c76a8..ea32d27 100644 --- a/docs/fix/code/chdir_py.md +++ b/docs/fix/code/chdir_py.md @@ -1,4 +1,5 @@ # docs/fix/code/chdir.py + ```python """User fixture to change the current working directory while running tests.""" import os diff --git a/docs/fix/code/globdemo.md b/docs/fix/code/globdemo.md index a8d43f6..b9e78d0 100644 --- a/docs/fix/code/globdemo.md +++ b/docs/fix/code/globdemo.md @@ -14,7 +14,7 @@ The function **init_globals()** assigns names and returns a mapping of them to the caller. The generated testfile copies them to the test module's module attributes. -## Python fenced code block and expected output under test. +## Python fenced code block and expected output under test This is the Python example we want to check. @@ -25,35 +25,36 @@ print(my_function(2)) ``` Expected output: -``` + +```expected-output 10 [1, 2, 3, 4, 'A'] 3 ``` -Note that Python prints the string value with single quotes. +Note that Python prints the string value with single quotes. -## phmutest command line. +## phmutest command line -``` +```shell phmutest docs/fix/code/globdemo.md --fixture docs.fix.code.globdemo.init_globals --log ``` -## phmutest output. +## phmutest output Terminal output after the `OK` line. -``` + +```txt log: args.files: 'docs/fix/code/globdemo.md' args.fixture: 'docs.fix.code.globdemo.init_globals' args.log: 'True' -location|label result ----------------------------- ------ -setUpModule................. -init_globals................ -docs/fix/code/globdemo.md:21 pass -tearDownModule.............. ----------------------------- ------ +location|label result +------------------------------ ------ +setUpModule................... +init_globals.................. +docs/fix/code/globdemo.md:21 o pass +tearDownModule................ +------------------------------ ------ ``` - diff --git a/docs/fix/code/globdemo_py.md b/docs/fix/code/globdemo_py.md index 4c73960..4794d79 100644 --- a/docs/fix/code/globdemo_py.md +++ b/docs/fix/code/globdemo_py.md @@ -1,4 +1,5 @@ # docs/fix/code/globdemo.py + ```python """User fixture to return mapping of values.""" from phmutest.fixture import Fixture diff --git a/docs/fix/repl/drink.md b/docs/fix/repl/drink.md index d9cb6c1..e3e0767 100644 --- a/docs/fix/repl/drink.md +++ b/docs/fix/repl/drink.md @@ -16,15 +16,15 @@ tea tea + sugar ``` -## phmutest command line. +## phmutest command line -``` +```shell phmutest docs/fix/repl/drink.md --fixture docs.fix.repl.drink.init --replmode --log ``` -## phmutest output. +## phmutest output -``` +```txt Acquiring Drink tea. ... Releasing Drink tea + sugar. ... @@ -44,4 +44,3 @@ docs/fix/repl/drink.md:13 pass Notice that 'Acquiring' and 'Releasing' lines in the output precede the log. In --replmode the --log printing happens after all testing and cleanup completes. - diff --git a/docs/fix/repl/drink_py.md b/docs/fix/repl/drink_py.md index 69de35b..6581cb0 100644 --- a/docs/fix/repl/drink_py.md +++ b/docs/fix/repl/drink_py.md @@ -1,4 +1,5 @@ # docs/fix/repl/drink.py + ```python """--replmode fixture acquires resource, passes to tests via globs, releases it.""" from phmutest.fixture import Fixture diff --git a/docs/fixture_py.md b/docs/fixture_py.md index d49bbd7..2776412 100644 --- a/docs/fixture_py.md +++ b/docs/fixture_py.md @@ -1,4 +1,5 @@ # src/phmutest/fixture.py + ```python """v0.0.2 Keyword arguments passed to --fixture function and return type Fixture. diff --git a/docs/generated_project_py.md b/docs/generated_project_py.md index 7ec54be..012d988 100644 --- a/docs/generated_project_py.md +++ b/docs/generated_project_py.md @@ -1,4 +1,5 @@ # tests/py/generated_project.py + ```python """Python unittest test file generated by Python Package phmutest.""" import unittest @@ -25,7 +26,7 @@ class Test001(unittest.TestCase): return "Hello" + "\n\n" + name print(greeting("World!")) - # line 20 + # line 21 _phm_expected_str = """\ Hello @@ -34,9 +35,9 @@ World! _phm_printer.cancel_print_capture_on_error() _phm_testcase.assertEqual(_phm_expected_str, _phm_printer.stdout()) - # ------ tests/md/project.md:27 ------ - with self.subTest(msg="tests/md/project.md:27"): - with _phmPrinter(_phm_log, "tests/md/project.md:27", False): + # ------ tests/md/project.md:29 ------ + with self.subTest(msg="tests/md/project.md:29"): + with _phmPrinter(_phm_log, "tests/md/project.md:29", False): text = greeting("Planet!") text = text.replace("\n\n", " ") assert text == "Hello Planet!" # this assert is in the Markdown example. diff --git a/docs/generated_share_demo_py.md b/docs/generated_share_demo_py.md index 8f655e3..61c759d 100644 --- a/docs/generated_share_demo_py.md +++ b/docs/generated_share_demo_py.md @@ -1,4 +1,5 @@ # tests/py/generated_sharedemo.py + ```python """Python unittest test file generated by Python Package phmutest.""" import unittest @@ -35,9 +36,9 @@ class Test001(unittest.TestCase): with _phmPrinter(_phm_log, "docs/share/file1.md:5", False): from dataclasses import dataclass - # ------ docs/share/file1.md:10 ------ - with self.subTest(msg="docs/share/file1.md:10"): - with _phmPrinter(_phm_log, "docs/share/file1.md:10", False): + # ------ docs/share/file1.md:9 ------ + with self.subTest(msg="docs/share/file1.md:9"): + with _phmPrinter(_phm_log, "docs/share/file1.md:9", False): @dataclass class BeverageActivity: beverage: str @@ -61,9 +62,9 @@ enjoyment _phm_printer.cancel_print_capture_on_error() _phm_testcase.assertEqual(_phm_expected_str, _phm_printer.stdout()) - # ------ docs/share/file1.md:34 ------ - with self.subTest(msg="docs/share/file1.md:34"): - with _phmPrinter(_phm_log, "docs/share/file1.md:34", False): + # ------ docs/share/file1.md:35 ------ + with self.subTest(msg="docs/share/file1.md:35"): + with _phmPrinter(_phm_log, "docs/share/file1.md:35", False): we = BeverageActivity("water", "exercise") _phm_globals.update(additions=locals(), built_from="docs/share/file1.md", existing_names=None) @@ -105,11 +106,11 @@ class Test003(unittest.TestCase): def tests(self): - # ------ docs/share/file3.md:8 ------ - with self.subTest(msg="docs/share/file3.md:8"): - with _phmPrinter(_phm_log, "docs/share/file3.md:8", False) as _phm_printer: + # ------ docs/share/file3.md:7 ------ + with self.subTest(msg="docs/share/file3.md:7"): + with _phmPrinter(_phm_log, "docs/share/file3.md:7", False) as _phm_printer: print(bp.combine()) - # line 12 + # line 11 _phm_expected_str = """\ beer-partying """ diff --git a/docs/group/deselect.md b/docs/group/deselect.md index af0c192..095ef5a 100644 --- a/docs/group/deselect.md +++ b/docs/group/deselect.md @@ -1,20 +1,22 @@ -# --deselect example. +# --deselect example Using the phmutest-group directive and --deselect command line option. -## phmutest command line. +## phmutest command line This command line selects all blocks in [select.md](select.md) that don't have a "phmutest-group slow" directive. -``` + +```shell phmutest docs/group/select.md --deselect slow --summary --log ``` -## phmutest output. +## phmutest output Terminal output after the `OK` line. Note in the log below that only the first block is tested. -``` + +```txt summary: metric -------------------- - @@ -34,9 +36,8 @@ args.deselect: 'slow' args.log: 'True' args.summary: 'True' -location|label result ------------------------ ------ -docs/group/select.md:10 pass ------------------------ ------ +location|label result +------------------------ ------ +docs/group/select.md:7 o pass +------------------------ ------ ``` - diff --git a/docs/group/select.md b/docs/group/select.md index 14eb48f..b73a53a 100644 --- a/docs/group/select.md +++ b/docs/group/select.md @@ -1,28 +1,26 @@ -# --select and group directive example. +# --select and group directive example Using the phmutest-group directive and --select command line option. -The command line below +The first FCB has no directives and expected output block. -## Fenced code block with no directives and expected output block. - -Example code adapted from the Python Tutorial: ```python squares = [1, 4, 9, 16, 25] print(squares) ``` + expected output: -``` + +```expected-output [1, 4, 9, 16, 25] ``` -## Code block with phmutest-group directive. - Look for the `` directive in the Markdown file. Note there is a space before the group name. The directive declares the block to be a member of test group "slow". + ```python from datetime import date @@ -30,23 +28,25 @@ d = date.fromordinal(730920) # 730920th day after 1. 1. 0001 print(d) ``` -``` +```expected-output 2002-03-11 ``` -## phmutest command line. +## phmutest command line This command line selectes all blocks in this file that have a `` directive. -``` + +```shell phmutest docs/group/select.md --select slow --summary --log ``` -## phmutest output. +## phmutest output Terminal output after the `OK` line. Note in the log below that only the second block is tested. -``` + +```txt summary: metric -------------------- - @@ -66,9 +66,8 @@ args.select: 'slow' args.log: 'True' args.summary: 'True' -location|label result ------------------------ ------ -docs/group/select.md:26 pass ------------------------ ------ +location|label result +------------------------- ------ +docs/group/select.md:24 o pass +------------------------- ------ ``` -[1]: https://github.com/tmarktaylor/phmutest/docs/group diff --git a/docs/junit.md b/docs/junit.md index d353024..1c17e0c 100644 --- a/docs/junit.md +++ b/docs/junit.md @@ -31,4 +31,3 @@ def test_broken_python_examples(capsys): phmresult = phmutest.main.main(args) assert phmresult.is_success, capsys.readouterr().out + "\n" + f.getvalue() ``` - diff --git a/docs/make_wrapped_examples.py b/docs/make_wrapped_examples.py index 66d87c8..1793e6d 100644 --- a/docs/make_wrapped_examples.py +++ b/docs/make_wrapped_examples.py @@ -1,7 +1,7 @@ """Create Markdown wrappers around the project's example .py files.""" from pathlib import Path -top = "# \n```python\n" +top = "# \n\n```python\n" bottom = """``` """ diff --git a/docs/repl/repl1.md b/docs/repl/repl1.md index 979a47a..3a17ffc 100644 --- a/docs/repl/repl1.md +++ b/docs/repl/repl1.md @@ -6,7 +6,6 @@ First file of share across files --replmode demo. >>> from dataclasses import dataclass ``` - ```python >>> @dataclass ... class BeverageActivity: @@ -21,6 +20,7 @@ First file of share across files --replmode demo. ``` Use `BeverageActivity` defined above to create the name `cc`. + ```python >>> cc = BeverageActivity("coffee", "coding") >>> cc.combine() @@ -28,7 +28,7 @@ Use `BeverageActivity` defined above to create the name `cc`. ``` Create the name `we` for use in later files. + ```python >>> we = BeverageActivity("water", "exercise") ``` - diff --git a/docs/repl/repl2.md b/docs/repl/repl2.md index 5d0325a..a804d56 100644 --- a/docs/repl/repl2.md +++ b/docs/repl/repl2.md @@ -5,6 +5,7 @@ Second file of --share-across-files --replmode demo. This file references the names shared from [repl1.md](repl1.md). Show the name `we` is visible. + ```python >>> we.combine() 'water-exercise' @@ -12,9 +13,9 @@ Show the name `we` is visible. Show the name `BeverageActivity` is visible. Create the name `bp`. + ```python >>> bp = BeverageActivity("beer", "partying") >>> bp.combine() 'beer-partying' ``` - diff --git a/docs/repl/repl3.md b/docs/repl/repl3.md index 38badbf..9aff4e3 100644 --- a/docs/repl/repl3.md +++ b/docs/repl/repl3.md @@ -3,17 +3,17 @@ Third file of --share-across-files --replmode demo. This file references the names shared from [repl1.md](repl1.md) and [repl2.md](repl2.md) +The block below uses `bp` defined in repl2.md -Use `bp` defined in repl2.md ```python >>> bp.combine() 'beer-partying' ``` Use the name `BeverageActivity` defined in repl1.md. + ```python >>> ss = BeverageActivity("soda", "snacking") >>> ss.combine() 'soda-snacking' ``` - diff --git a/docs/repl/replshare_demo.md b/docs/repl/replshare_demo.md index 175a56d..44e9b8d 100644 --- a/docs/repl/replshare_demo.md +++ b/docs/repl/replshare_demo.md @@ -19,16 +19,17 @@ All the names assigned at the top level of all the Python blocks in the file are shared. A later file might inadventently reassign a name that was shared. -## phmutest command line. +## phmutest command line -``` +```shell phmutest docs/repl/repl1.md docs/repl/repl2.md docs/repl/repl3.md --replmode --share-across-files docs/repl/repl1.md docs/repl/repl2.md --log --summary ``` -## phmutest output. +## phmutest output Terminal output after the `OK` line. -``` + +```txt summary: metric -------------------- - @@ -55,13 +56,12 @@ args.summary: 'True' location|label result --------------------- ------ docs/repl/repl1.md:5. pass -docs/repl/repl1.md:10 pass +docs/repl/repl1.md:9. pass docs/repl/repl1.md:24 pass -docs/repl/repl1.md:31 pass -docs/repl/repl2.md:8. pass -docs/repl/repl2.md:15 pass +docs/repl/repl1.md:32 pass +docs/repl/repl2.md:9. pass +docs/repl/repl2.md:17 pass docs/repl/repl3.md:8. pass -docs/repl/repl3.md:14 pass +docs/repl/repl3.md:15 pass --------------------- ------ ``` - diff --git a/docs/sessionmode.md b/docs/sessionmode.md index 4975d54..936befb 100644 --- a/docs/sessionmode.md +++ b/docs/sessionmode.md @@ -46,13 +46,10 @@ the docstring generated for the Markdown file. Add the FILE or `.` to the --sharing option to turn on verbose printing. `.` means show sharing for all files and will show fixture globs too. - ## --progress This option turns on per Markdown file verbose printing. The printing is directed to the standard output stream shared with doctest's verbose printing. - [1]: https://docs.python.org/3/library/doctest.html [2]: https://docs.python.org/3/library/doctest.html#doctest.DocTest.globs - diff --git a/docs/setup/across1.md b/docs/setup/across1.md index a840e36..05d528d 100644 --- a/docs/setup/across1.md +++ b/docs/setup/across1.md @@ -3,15 +3,14 @@ This example shows the use of the `` and `` directives. - **All** the names assigned in the setup blocks are made visible to the examples in all files. - [docs/setup/across1.md](across1.md) (This file) - [docs/setup/across2.md](across2.md) +## The next 2 blocks are marked as the setup blocks -## The next 2 blocks are marked as the setup blocks. The main reason to use setup blocks is so that the teardown blocks will run in the event a post setup block fails. Note that teardown will only run if all the setup blocks succeed. @@ -21,6 +20,7 @@ directory and restore the changed working directory. The function create_tmpdir() serves to hide the names 'stack' and 'td' from sharing. + ```python import os import tempfile @@ -33,7 +33,9 @@ original_cwd = Path.cwd() ``` Each setup block must have the `` directive. + + ```python def create_tmpdir(): with ExitStack() as stack: @@ -47,56 +49,64 @@ cleanup_tmpdir = create_tmpdir() Path(FILENAME).write_text(CONTENTS, encoding="utf-8") ``` -## The next 2 blocks are marked as the teardown blocks. +## The next 2 blocks are marked as the teardown blocks + Setup and teardown blocks can have an output block. + + ```python print("Removing tmpdir, restoring current working directory...") cleanup_tmpdir() ``` expected output: -``` + +```expected-output Removing tmpdir, restoring current working directory... ``` -## This block will be marked as the teardown code too. +## This block will be marked as the teardown code too + More than one setup or teardown block is allowed. Each block must have the `` directive. The block shows that cleanup_tmpdir() restored the initial cwd. + + ```python assert Path.cwd() == original_cwd ``` -## phmutest command line. +## phmutest command line Note that this fenced code block has `txt` as the info string. The txt tells phmutest that this is not an output block -for the python block immediately above. -```txt + +```shell phmutest docs/setup/across1.md docs/setup/across2.md --setup-across-files docs/setup/across1.md --log ``` -## phmutest output. +## phmutest output Terminal output after the `OK` line. -``` + +```txt log: args.files: 'docs/setup/across1.md' args.files: 'docs/setup/across2.md' args.setup_across_files: 'docs/setup/across1.md' args.log: 'True' -location|label result ---------------------------------- ------ -setUpModule...................... -docs/setup/across1.md:24 setup... pass -docs/setup/across1.md:37 setup... pass -docs/setup/across2.md:4.......... pass -docs/setup/across2.md:12......... pass -tearDownModule................... -docs/setup/across1.md:53 teardown pass -docs/setup/across1.md:68 teardown pass ---------------------------------- ------ +location|label result +----------------------------------- ------ +setUpModule........................ +docs/setup/across1.md:24 setup..... pass +docs/setup/across1.md:39 setup..... pass +docs/setup/across2.md:5 o.......... pass +docs/setup/across2.md:15........... pass +tearDownModule..................... +docs/setup/across1.md:58 teardown o pass +docs/setup/across1.md:77 teardown.. pass +----------------------------------- ------ ``` diff --git a/docs/setup/across2.md b/docs/setup/across2.md index f28425b..97e5909 100644 --- a/docs/setup/across2.md +++ b/docs/setup/across2.md @@ -1,15 +1,17 @@ # docs/setup/across2.md -## This block shows the temporary file exists. +## This block shows the temporary file exists + ```python print(Path(FILENAME).read_text(encoding="utf-8")) ``` + expected output: -``` + +```expected-output apples, cider, cherries, very small rocks. ``` ```python assert Path.cwd() != original_cwd, "In a different cwd, presumably tempdir." ``` - diff --git a/docs/setup/setup.md b/docs/setup/setup.md index 832e3a9..c47d6fb 100644 --- a/docs/setup/setup.md +++ b/docs/setup/setup.md @@ -1,4 +1,4 @@ -# Setup and teardown directive example. +# Setup and teardown directive example Setup and teardown directives. Directives are HTML comments @@ -6,7 +6,8 @@ and are not rendered. Look for the `` and `` directives in the Markdown file. -## The next 2 blocks are marked as the setup blocks. +## The next 2 blocks are marked as the setup blocks + The main reason to use setup blocks is so that the teardown blocks will run in the event a post setup block fails. Note that teardown will only run if all the setup blocks succeed. @@ -19,6 +20,7 @@ We use with ExitStack to assure cleanup when a statement in the with suite raises an exception. + ```python import os import tempfile @@ -31,7 +33,9 @@ workdir = Path.cwd() ``` Each setup block must have the `` directive. + + ```python # note callbacks are done in reverse order with ExitStack() as stack: @@ -43,61 +47,73 @@ with ExitStack() as stack: Path(FILENAME).write_text(CONTENTS, encoding="utf-8") ``` -## This block shows the temporary file exists. +## This block shows the temporary file exists + ```python print(Path(FILENAME).read_text(encoding="utf-8")) assert Path.cwd() != workdir, "In a different cwd, presumably tempdir." ``` + expected output: -``` + +```expected-output apples, cider, cherries, very small rocks. ``` -## The next 2 blocks are marked as the teardown blocks. +## The next 2 blocks are marked as the teardown blocks + Setup and teardown blocks can have an output block as well. + + ```python print("Restoring current working directory...") cleanup() ``` expected output: -``` + +```expected-output Restoring current working directory... ``` -## This block will be designated as teardown code too. +## This block will be designated as teardown code too + More than one setup or teardown block is allowed. Each block must have the `` directive. The block shows that cleanup() restored the initial cwd. + + ```python assert Path.cwd() == workdir, "The original cwd." ``` -## phmutest command line. +## phmutest command line Note that this fenced code block has `txt` as the info string. The txt tells phmutest that this is not an output block for the python block immediately above. -```txt + +```shell phmutest docs/setup/setup.md --log ``` -## phmutest output. +## phmutest output Terminal output after the `OK` line. -``` + +```txt log: args.files: 'docs/setup/setup.md' args.log: 'True' -location|label result -------------------------------- ------ -docs/setup/setup.md:22 setup... pass -docs/setup/setup.md:35 setup... pass -docs/setup/setup.md:47......... pass -docs/setup/setup.md:59 teardown pass -docs/setup/setup.md:74 teardown pass -------------------------------- ------ +location|label result +--------------------------------- ------ +docs/setup/setup.md:24 setup..... pass +docs/setup/setup.md:39 setup..... pass +docs/setup/setup.md:52 o......... pass +docs/setup/setup.md:69 teardown o pass +docs/setup/setup.md:88 teardown.. pass +--------------------------------- ------ ``` diff --git a/docs/share/file1.md b/docs/share/file1.md index 0b86f8d..c62e7dd 100644 --- a/docs/share/file1.md +++ b/docs/share/file1.md @@ -6,7 +6,6 @@ First file of share across files demo. from dataclasses import dataclass ``` - ```python @dataclass class BeverageActivity: @@ -21,17 +20,18 @@ class BeverageActivity: ``` Use `BeverageActivity` defined above to create the name `cc`. + ```python cc = BeverageActivity("coffee", "coding") print(cc.combine()) ``` -``` +```expected-output enjoyment ``` Create the name `we` for use in later files. + ```python we = BeverageActivity("water", "exercise") ``` - diff --git a/docs/share/file2.md b/docs/share/file2.md index b9c44eb..5283917 100644 --- a/docs/share/file2.md +++ b/docs/share/file2.md @@ -3,24 +3,23 @@ Second file of --share-across-files demo. This file references the names shared from [file1.md](file1.md). - Show the name `we` is visible. + ```python print(we.combine()) ``` -``` +```expected-output water-exercise ``` -Show the name `BeverageActivity` is visible. -Create the name `bp`. +Show the name `BeverageActivity` is visible. Create the name `bp`. + ```python bp = BeverageActivity("beer", "partying") print(bp.combine()) ``` -``` +```expected-output beer-partying ``` - diff --git a/docs/share/file3.md b/docs/share/file3.md index bb761ee..714f2ad 100644 --- a/docs/share/file3.md +++ b/docs/share/file3.md @@ -1,25 +1,24 @@ # docs/share/file3.md Third file of --share-across-files demo. - This file references the names shared from [file1.md](file1.md) and [file2.md](file2.md) - Use `bp` defined in file2.md + ```python print(bp.combine()) ``` -``` +```expected-output beer-partying ``` Use the name `BeverageActivity` defined in file1.md. + ```python ss = BeverageActivity("soda", "snacking") print(ss.combine()) ``` -``` +```expected-output soda-snacking ``` - diff --git a/docs/share/share_demo.md b/docs/share/share_demo.md index 8728600..68917fe 100644 --- a/docs/share/share_demo.md +++ b/docs/share/share_demo.md @@ -2,8 +2,9 @@ The --share-across-files option generates test code that shares the names assigned by the files specified by the option to the files specified later in the list -of input files (the input files are specified by FILE as shown in the usage.) -file1.md shares its names to file2.md and file3.md. file2.md shares its names +of input files (the input files are specified by the positional arg(s) +FILE as shown in the usage.) file1.md shares its names to +file2.md and file3.md. file2.md shares its names to file3.md. It is tempting to use shell file wildcards for the FILE names. Please be aware that @@ -11,6 +12,9 @@ the order of files in the wildcard expansion might not be the order that you wan That order may differ between shell and operating system combinations. If you are calling from Python there will be no shell wildcard expansion. +Sharing across files is useful to avoid repeating common setup examples +in subsequent Markdown files. + - [docs/share/file1.md](file1.md) - [docs/share/file2.md](file2.md) - [docs/share/file3.md](file3.md) @@ -19,15 +23,16 @@ All the names assigned at the top level of all the Python blocks in the file are shared. A later file might inadventently reassign a name that was shared. -## phmutest command line. +## phmutest command line ``` phmutest docs/share/file1.md docs/share/file2.md docs/share/file3.md --share-across-files docs/share/file1.md docs/share/file2.md --log --summary ``` -## phmutest output. +## phmutest output Terminal output after the `OK` line. + ``` summary: metric @@ -51,18 +56,17 @@ args.share_across_files: 'docs/share/file2.md' args.log: 'True' args.summary: 'True' -location|label result ----------------------- ------ -setUpModule........... -docs/share/file1.md:5. pass -docs/share/file1.md:10 pass -docs/share/file1.md:24 pass -docs/share/file1.md:34 pass -docs/share/file2.md:8. pass -docs/share/file2.md:18 pass -docs/share/file3.md:8. pass -docs/share/file3.md:17 pass -tearDownModule........ ----------------------- ------ +location|label result +------------------------ ------ +setUpModule............. +docs/share/file1.md:5... pass +docs/share/file1.md:9... pass +docs/share/file1.md:24 o pass +docs/share/file1.md:35.. pass +docs/share/file2.md:8 o. pass +docs/share/file2.md:18 o pass +docs/share/file3.md:7 o. pass +docs/share/file3.md:17 o pass +tearDownModule.......... +------------------------ ------ ``` - diff --git a/tests/fail/raiser.md b/tests/fail/raiser.md index d3c1cec..ef2b26e 100644 --- a/tests/fail/raiser.md +++ b/tests/fail/raiser.md @@ -1,4 +1,5 @@ -# +# Raise a TypeError + ```python from tests.fail.bumper import MyBumper ``` @@ -30,12 +31,14 @@ print(bumper2.bump()) This is incorrect expected output, but it is not checked because its code block raised an exception. + ``` aa ``` Show the test continues after catching the exception above. You can pass the unittest option --failfast to stop instead. + ```python print("Still going.") ``` diff --git a/tests/md/bad_session_output.md b/tests/md/bad_session_output.md index 35ef842..565cb56 100644 --- a/tests/md/bad_session_output.md +++ b/tests/md/bad_session_output.md @@ -1,9 +1,10 @@ -# The test generated from this file fails doctest. +# The test generated from this file fails doctest + The output `5` is not the expected value. + ```py >>> a = 2 >>> b = 2 >>> a + b 5 ``` - diff --git a/tests/md/bad_skipif_number.md b/tests/md/bad_skipif_number.md index 5e1f50b..ee3a477 100644 --- a/tests/md/bad_skipif_number.md +++ b/tests/md/bad_skipif_number.md @@ -1,7 +1,9 @@ -### Skipif directive has non-numeric or negative minor number. +# Skipif directive has non-numeric or negative minor number Malformed skipif directives silently ignored. Both blocks pass. + + ```python user = 'eric_idle' print(f"{user=}") @@ -12,6 +14,7 @@ user='eric_idle' ``` + ```python user = 'palin' print(f"{user=}") diff --git a/tests/md/badsetup.md b/tests/md/badsetup.md index 0934979..d2f9e4f 100644 --- a/tests/md/badsetup.md +++ b/tests/md/badsetup.md @@ -3,6 +3,7 @@ Setup code block raises an exception. + ```python import math @@ -15,14 +16,17 @@ def doubler(x): raise(TypeError("badsetup.md in setup block")) # <------ bad part here ``` -## This test case shows the setup names are visible. +## This test case shows the setup names are visible + ```python print("math.pi=", round(math.pi, 3)) print(mylist) print(a, b) print("doubler(16)=", doubler(16)) ``` + expected output: + ``` math.pi= 3.142 [1, 2, 3] @@ -30,30 +34,40 @@ math.pi= 3.142 doubler(16)= 32 ``` -## This test case modifies mylist. +## This test case modifies mylist + The objects created by the --setup code can be modified and blocks run afterward will see the changes. + ```python mylist.append(4) print(mylist) ``` + expected output: + ``` [1, 2, 3, 4] ``` -## This test case sees the modified mylist. +## This test case sees the modified mylist + ```python print(mylist == [1, 2, 3, 4]) ``` + expected output: + ``` True ``` -## This will be marked as the teardown code. +## This will be marked as the teardown code + Note `` directive in the Markdown file. + + ```python mylist.clear() assert not mylist, "mylist was not emptied" diff --git a/tests/md/badteardown.md b/tests/md/badteardown.md index ad29173..98a04c5 100644 --- a/tests/md/badteardown.md +++ b/tests/md/badteardown.md @@ -13,14 +13,17 @@ def doubler(x): return x * 2 ``` -## This test case shows the setup names are visible. +## This test case shows the setup names are visible + ```python print("math.pi=", round(math.pi, 3)) print(mylist) print(a, b) print("doubler(16)=", doubler(16)) ``` + expected output: + ``` math.pi= 3.142 [1, 2, 3] @@ -28,30 +31,40 @@ math.pi= 3.142 doubler(16)= 32 ``` -## This test case modifies mylist. +## This test case modifies mylist + The objects created by the --setup code can be modified and blocks run afterward will see the changes. + ```python mylist.append(4) print(mylist) ``` + expected output: + ``` [1, 2, 3, 4] ``` -## This test case sees the modified mylist. +## This test case sees the modified mylist + ```python print(mylist == [1, 2, 3, 4]) ``` + expected output: + ``` True ``` -## This will be marked as the teardown code. +## This will be marked as the teardown code + Note `` directive in the Markdown file. + + ```python mylist.clear() assert not mylist, "mylist was not emptied" diff --git a/tests/md/cases.md b/tests/md/cases.md index 7737104..9f8103c 100644 --- a/tests/md/cases.md +++ b/tests/md/cases.md @@ -1,9 +1,7 @@ -# - - -## Source Code block with no ouput and skipif directive. +# Source Code block with no ouput and skipif directive + ```python from enum import Enum @@ -15,5 +13,6 @@ class Floats(Enum): ``` Empty Python FCB. + ```python ``` diff --git a/tests/md/code_groups.md b/tests/md/code_groups.md index a11904d..0eee77d 100644 --- a/tests/md/code_groups.md +++ b/tests/md/code_groups.md @@ -58,4 +58,3 @@ print("code-group-4") ```python >>> print() # repl-7 >>> print() # repl-8 - diff --git a/tests/md/directive1.md b/tests/md/directive1.md index 0798afa..bd8635f 100644 --- a/tests/md/directive1.md +++ b/tests/md/directive1.md @@ -4,20 +4,24 @@ Directives are HTML comments and are not rendered. To see the directives press Edit on GitHub and then the Raw button. -## skip directive. No test case gets generated. +## skip directive. No test case gets generated + It is OK to put a directive above pre-existing HTML comments. The HTML comments are not visible in the rendered Markdown. + ```python assert False ``` -## skip directive on an expected output block. +## skip directive on an expected output block + Generates a test case that runs the code block but does not check the expected output. + ```python from datetime import date @@ -25,13 +29,15 @@ date.today() ``` + ``` datetime.date(2021, 4, 18) ``` -## skip directive on Python session. +## skip directive on Python session + ```py >>> print("Hello World!") incorrect expected output should fail @@ -48,12 +54,13 @@ print("testing bogus output.") incorrect expected output ``` -## skipif directive. +## skipif directive Use skipif on Python code blocks. This test case will only run when Python is version 3.8 or higher. + ```python import sys b = 10 @@ -64,8 +71,10 @@ assert sys.version_info >= (3, 8) 10 ``` -## label directive on a session. +## label directive on a session + + ```py >>> print("coffee") coffee diff --git a/tests/md/directive2.md b/tests/md/directive2.md index 9aaec0c..039e0b8 100644 --- a/tests/md/directive2.md +++ b/tests/md/directive2.md @@ -8,13 +8,16 @@ The setup/teardown directive can be placed on more than 1 block. The setup/teardown blocks may also have an expected output block. The setup/teardown blocks count towards the metrics. -## This run as the setup code. +## This run as the setup code + The setup logic makes the names assigned here appear as globals to the test cases from this file. The code assigns the **names** math, mylist, a, b, and the function doubler(). Setup code does not have an output block. Note the `` directive in the Markdown file. + + ```python import math @@ -25,14 +28,17 @@ def doubler(x): return x * 2 ``` -## This test case shows the setup names are visible. +## This test case shows the setup names are visible + ```python print("math.pi=", round(math.pi, 3)) print(mylist) print(a, b) print("doubler(16)=", doubler(16)) ``` + expected output: + ``` math.pi= 3.142 [1, 2, 3] @@ -40,30 +46,40 @@ math.pi= 3.142 doubler(16)= 32 ``` -## This test case modifies mylist. +## This test case modifies mylist + The objects created by the --setup code can be modified and blocks run afterward will see the changes. + ```python mylist.append(4) print(mylist) ``` + expected output: + ``` [1, 2, 3, 4] ``` -## This test case sees the modified mylist. +## This test case sees the modified mylist + ```python print(mylist == [1, 2, 3, 4]) ``` + expected output: + ``` True ``` -## This will be marked as the teardown code. +## This will be marked as the teardown code + Note `` directive in the Markdown file. + + ```python mylist.clear() assert not mylist, "mylist was not emptied" diff --git a/tests/md/directives.md b/tests/md/directives.md index e2faee0..7503032 100644 --- a/tests/md/directives.md +++ b/tests/md/directives.md @@ -1,6 +1,7 @@ -# Test input file for direct.py for phmutest directives. +# Test input file for direct.py for phmutest directives ## skip directive + @@ -8,14 +9,17 @@ assert False ``` -## skip directive on an expected output block. +## skip directive on an expected output block + Generates a test case that runs the code block but does not check the expected output. + ```python from datetime import date date.today() ``` + @@ -23,7 +27,7 @@ date.today() datetime.date(2021, 4, 18) ``` -## skip directive on Python session. +## skip directive on Python session @@ -34,7 +38,7 @@ incorrect expected output should fail if test case is generated ``` -## skipif directive. +## skipif directive @@ -47,28 +51,34 @@ print(b.as_integer_ratio()) ``` -## test group directive. +## test group directive + ```python b = 10 print(b.as_integer_ratio()) ``` + ``` (10, 1) ``` -## label directive on a session. + +## label directive on a session + This will generate a test case called **doctest_print_coffee()**. It does not start with test_ to avoid collection as a test item. + ```py >>> print("coffee") coffee ``` -## This will be marked as the setup code. +## This will be marked as the setup code + ```python import math @@ -79,16 +89,20 @@ def doubler(x): return x * 2 ``` -## This will be marked as the teardown code. +## This will be marked as the teardown code + Teardown code does not have an output block. Note `` directive in the Markdown file. + + ```python mylist.clear() assert not mylist, "mylist was not emptied" ``` -## Does not qualify as a directive due to 2 blank lines before FCB. +## Does not qualify as a directive due to 2 blank lines before FCB + diff --git a/tests/md/example2.md b/tests/md/example2.md index f8a9e89..8da447f 100644 --- a/tests/md/example2.md +++ b/tests/md/example2.md @@ -1,48 +1,63 @@ # This is Markdown file example2.md -## Fenced code block expected output block pair. + +## Fenced code block expected output block pair + In order for phmdoctest to work with Python source code and terminal output add print statements to the source code to produce the expected output. Example code adapted from the Python Tutorial: + ```python squares = [1, 4, 9, 16, 25] print(squares) ``` + expected output: + ``` [1, 4, 9, 16, 25] ``` -## Another fenced code block expected output block pair. +## Another fenced code block expected output block pair + Example code adapted from What's new in Python: + ```python # Formatted string literals require Python 3.7 name = "Fred" print(f"He said his name is {name}.") ``` + expected output: + ``` He said his name is Fred. ``` -## Here is a second fenced code block with no info string. +## Here is a second fenced code block with no info string + ``` doesn't have an info string ``` -## Here are two Python code blocks in a row and one output block at the end. +## Here are two Python code blocks in a row and one output block at the end + The first one: + ```python a, b = 0, 1 print("There is no output block so this is not checked.") ``` + The second one. This means the preceding code block has no output block. + ```python words = ["cat", "window", "defenestrate"] for w in words: print(w, len(w)) ``` + The expected output block for the second code block: ``` @@ -51,7 +66,7 @@ window 6 defenestrate 12 ``` -## A fenced code block with yaml info string. +## A fenced code block with yaml info string ```yaml dist: xenial @@ -66,9 +81,11 @@ some text ``` ## A doctest session + Here is a Python interactive session. It is described by the Python Standard Library module doctest. Note there is no need for an empty line at the end of the session. + ```py >>> a = "Greetings Planet!" >>> a @@ -78,9 +95,10 @@ no need for an empty line at the end of the session. 12 ``` -## One more code plus expected output pair. +## One more code plus expected output pair Example borrowed from Python Standard Library datetime documentation. + ```python from datetime import date @@ -88,7 +106,7 @@ d = date.fromordinal(730920) # 730920th day after 1. 1. 0001 print(d) ``` -``` +```expected-output 2002-03-11 ``` @@ -96,6 +114,7 @@ print(d) Example borrowed from Python Standard Library fractions documentation. + ```py >>> from fractions import Fraction >>> Fraction(16, -10) diff --git a/tests/md/extra_line_in_output.md b/tests/md/extra_line_in_output.md index 75b2b17..2094c26 100644 --- a/tests/md/extra_line_in_output.md +++ b/tests/md/extra_line_in_output.md @@ -1,5 +1,7 @@ -### Additional output beyond what was expected. +# Additional output beyond what was expected + The code + ```python3 from enum import Enum @@ -12,7 +14,9 @@ class Floats(Enum): for floater in Floats: print(floater) ``` + produces + ``` Floats.APPLES Floats.CIDER @@ -20,4 +24,5 @@ Floats.CHERRIES Floats.ADUCK Floats.VERY_SMALL_ROCKS ``` -Incorrect sample output is missing the Floats.ADUCK line. + +Output has extra line: Floats.VERY_SMALL_ROCKS. diff --git a/tests/md/legacy_directives.md b/tests/md/legacy_directives.md index 4ec7cb8..b58460f 100644 --- a/tests/md/legacy_directives.md +++ b/tests/md/legacy_directives.md @@ -1,6 +1,7 @@ -# Test input file for direct.py for phmdoctest directives. +# Test input file for direct.py for phmdoctest directives ## skip directive + @@ -8,14 +9,17 @@ assert False ``` -## skip directive on an expected output block. +## skip directive on an expected output block + Generates a test case that runs the code block but does not check the expected output. + ```python from datetime import date date.today() ``` + @@ -23,9 +27,10 @@ date.today() datetime.date(2021, 4, 18) ``` -## skip directive on Python session. +## skip directive on Python session No test case gets generated. + ```py @@ -34,10 +39,12 @@ incorrect expected output should fail if test case is generated ``` -## mark.skip directive with label directive. +## mark.skip directive with label directive + - Use `mark.skip` on Python code blocks. + ```python print("testing testing bogus output.") ``` @@ -45,45 +52,54 @@ print("testing testing bogus output.") incorrect expected output ``` -## mark.skipif directive. +## mark.skipif directive Use mark.skipif on Python code blocks. This test case will only run when Python is version 3.8 or higher. int.as_integer_ratio() is new in Python 3.8. + + ```python b = 10 print(b.as_integer_ratio()) ``` + ``` (10, 1) ``` -## mark.skip group name directive. +## mark.skip group name directive + ```python b = 10 print(b.as_integer_ratio()) ``` + ``` (10, 1) ``` -## label directive on a session. +## label directive on a session + This will generate a test case called **doctest_print_coffee()**. It does not start with test_ to avoid collection as a test item. + + ```py >>> print("coffee") coffee ``` -## This will be marked as the setup code. +## This will be marked as the setup code + ```python import math @@ -94,15 +110,19 @@ def doubler(x): return x * 2 ``` -## This will be marked as the teardown code. +## This will be marked as the teardown code + Note `` directive in the Markdown file. + + ```python mylist.clear() assert not mylist, "mylist was not emptied" ``` -## Does not qualify as a directive due to 2 blank lines before FCB. +## Does not qualify as a directive due to 2 blank lines before FCB + @@ -111,4 +131,3 @@ assert not mylist, "mylist was not emptied" mylist.clear() assert not mylist, "mylist was not emptied" ``` - diff --git a/tests/md/missing_some_output.md b/tests/md/missing_some_output.md index c8e9590..99607b2 100644 --- a/tests/md/missing_some_output.md +++ b/tests/md/missing_some_output.md @@ -1,5 +1,7 @@ -### incomplete expected output. +# incomplete expected output + The code + ```python3 from enum import Enum @@ -12,10 +14,13 @@ class Floats(Enum): for floater in Floats: print(floater) ``` + produces + ``` Floats.APPLES Floats.CIDER Floats.CHERRIES ``` + Incorrect sample output is missing the Floats.ADUCK line. diff --git a/tests/md/more_directives.md b/tests/md/more_directives.md index cd40151..1b17d44 100644 --- a/tests/md/more_directives.md +++ b/tests/md/more_directives.md @@ -1,25 +1,32 @@ -# Test input file for direct.py for phmutest directives. +# Test input file for direct.py for phmutest directives + +## directive and FCB indented 3 spaces -## directive and FCB indented 3 spaces. + ```python assert False ``` -## directive indented 4 spaces (one too many) and FCB indented 3 spaces. +## directive indented 4 spaces (one too many) and FCB indented 3 spaces + ```python assert False ``` -## directive is not allowed below the FCB. +## directive is not allowed below the FCB + ```python assert False ``` +A Markdown element is needed here to avoid the skip directive +above from associating with the Python block below. +Two blank lines would suffice, but gets busted by the linter. ```python assert False @@ -27,27 +34,32 @@ assert False Note- the 2 blank lines above the above FCB prevent the phmutest-skip directive from associating with the FCB. -## there is some text between the comment and the fenced code block. +## there is some text between the comment and the fenced code block + + some text prevents directive from associating with the FCB ```python assert False ``` ## skip directive on output block + ```python b = 10 print(b.as_integer_ratio()) ``` + ``` (10, 1) ``` ## skip directive with extra spacing + + ```python assert False ``` - diff --git a/tests/md/multi_label.md b/tests/md/multi_label.md index 246ea05..8490dbb 100644 --- a/tests/md/multi_label.md +++ b/tests/md/multi_label.md @@ -1,13 +1,16 @@ -# File with label directives. Some labels are on more than one block. +# File with label directives. Some labels are on more than one block -## A block and its fences indented 4 spaces. Phmutest does not see it. +## A block and its fences indented 4 spaces. Phmutest does not see it + ```python assert False ``` ## First of 2 blocks with the same label + + ``` first of many ``` @@ -16,42 +19,53 @@ first of many ```python assert False ``` + Note- the 2 blank lines above the above FCB prevent the phmutest-skip directive from associating with the FCB. -## there is some text between the comment and the fenced code block. +## there is some text between the comment and the fenced code block + + some text prevents directive from associating with the FCB + ```python assert False ``` ## skip directive on output block + ```python b = 10 print(b.as_integer_ratio()) ``` + + ``` (10, 1) ``` ## Third of 2 blocks with the same label + + ``` third of many ``` + + ```bash ls -l ``` + ```bash ls -r ``` - diff --git a/tests/md/no_code_blocks.md b/tests/md/no_code_blocks.md index 27d5257..b8ec519 100644 --- a/tests/md/no_code_blocks.md +++ b/tests/md/no_code_blocks.md @@ -1,4 +1,4 @@ -### No Python code blocks +# No Python code blocks Example Markdown file with no Python fenced code blocks. @@ -10,4 +10,4 @@ print('Hello World- this is a non-empty code block') ``` Greetings World -``` \ No newline at end of file +``` diff --git a/tests/md/no_fenced_code_blocks.md b/tests/md/no_fenced_code_blocks.md index 1b41d78..43e52de 100644 --- a/tests/md/no_fenced_code_blocks.md +++ b/tests/md/no_fenced_code_blocks.md @@ -1,4 +1,4 @@ -### Markdown file that has no fenced code blocks. +# Markdown file that has no fenced code blocks This file covers the use case where phmdoctest is given a Markdown file with no fenced code blocks with diff --git a/tests/md/noreadernodes.md b/tests/md/noreadernodes.md index e543932..1eccfee 100644 --- a/tests/md/noreadernodes.md +++ b/tests/md/noreadernodes.md @@ -1,7 +1,8 @@ -# This file does not have any elements that produce reader.DocNode. -This file has no... +# This file does not have any elements that produce reader.DocNode + +This file has no. + 1. HTML comments. 1. Blank lines. 1. Fenced code blocks So that reader.read_markdown() should return an empty list. - diff --git a/tests/md/one_mark_skip.md b/tests/md/one_mark_skip.md index 63d0a3f..00af819 100644 --- a/tests/md/one_mark_skip.md +++ b/tests/md/one_mark_skip.md @@ -1,8 +1,12 @@ +# heading + + ```python print("Hello World!") ``` + ``` incorrect expected output ``` diff --git a/tests/md/optionflags.md b/tests/md/optionflags.md index bd44828..704289e 100644 --- a/tests/md/optionflags.md +++ b/tests/md/optionflags.md @@ -1,18 +1,21 @@ -# Examples to test patching doctest optionflags in --replmode. +# Examples to test patching doctest optionflags in --replmode + +## Passes -## Passes. ```py >>> print("Hello World!") Hello World! ``` -## Passes with NORMALIZE_WHITESPACE. +## Passes with NORMALIZE_WHITESPACE + ```py >>> print("Hello World!") Hello World! ``` -## Passes with ELLIPSIS. +## Passes with ELLIPSIS + ```py >>> print("Hello World!") Hello...World! diff --git a/tests/md/output_has_blank_lines.md b/tests/md/output_has_blank_lines.md index 2490135..036b63d 100644 --- a/tests/md/output_has_blank_lines.md +++ b/tests/md/output_has_blank_lines.md @@ -1,5 +1,7 @@ -### Expected output has blank lines. +# Expected output has blank lines + The code + ```python3 print() print('Hello') @@ -7,7 +9,9 @@ print() print('World!') print() ``` + produces + ``` Hello diff --git a/tests/md/patching1.md b/tests/md/patching1.md index 8ac2481..cbae676 100644 --- a/tests/md/patching1.md +++ b/tests/md/patching1.md @@ -1,8 +1,11 @@ +# heading + ```python print("hello world") ``` -``` + +```expected-output hello world ``` @@ -11,7 +14,8 @@ hello world print("coffee") print("coding") ``` -``` + +```expected-output coffee coding ``` @@ -19,7 +23,7 @@ coding ```ladenpython print("ladenpython is a made up FCB info string.") ``` -``` + +```expected-output ladenpython is a made up FCB info string. ``` - diff --git a/tests/md/printer.md b/tests/md/printer.md index 8d2067f..9749185 100644 --- a/tests/md/printer.md +++ b/tests/md/printer.md @@ -1,8 +1,9 @@ -# Test input file for printer.py:_print(). +# Test input file for printer.py:_print() -## Python FCB that has no output block. +## Python FCB that has no output block The block prints to both stdout and stderr and then raises an AssertionError. + ```python import sys print("asserting False...") @@ -10,7 +11,7 @@ print("asserting False...", file=sys.stderr) assert False, "fail here to show captured stdout and stderr." ``` -## Output blocks can only check stdout. +## Output blocks can only check stdout This block passes. @@ -23,7 +24,7 @@ print("printing to stderr", file=sys.stderr) printing to stdout ``` -## Should capture stdout here since there is a skip directive on the output block. +## Should capture stdout here since there is a skip directive on the output block This block fails too. @@ -32,13 +33,16 @@ b = 10 print(b.as_integer_ratio()) assert False, "fail here to show captured stdout and stderr." ``` + + ``` (10, 1) ``` + ```python h = "hello" w = "world" @@ -48,4 +52,3 @@ print(h, w) ``` hello world ``` - diff --git a/tests/md/project.md b/tests/md/project.md index b5dad16..747ed19 100644 --- a/tests/md/project.md +++ b/tests/md/project.md @@ -1,11 +1,11 @@ -# Examples of mixed code and session blocks. +# Examples of mixed code and session blocks This file (project.md) has some example code and session blocks including a Python doctest directive example. ## An example with a blank line in the output -Note no directive in the output block of a Python +Note no `` directive in the output block of a Python code block output block pair. ```python @@ -17,27 +17,27 @@ print(greeting("World!")) ``` Here is the output it produces. -``` + +```expected-output Hello World! ``` A second FCB that calls the function greeting() defined in the first FCB. + ```python text = greeting("Planet!") text = text.replace("\n\n", " ") assert text == "Hello Planet!" # this assert is in the Markdown example. ``` - ## Interactive Python session requires `` in the expected output Blank lines in the expected output must be replaced with ``. To see un-rendered Markdown navigate to tests/md/project.md on GitHub and press the `Code` button. - ```py >>> print("Hello\n\nWorld!") Hello @@ -52,7 +52,6 @@ expected exception and use of the doctest directive `IGNORE_EXCEPTION_DETAIL`. To see the doctest directive navigate to [project.md unrendered][1]. - ```py >>> int("def") #doctest:+IGNORE_EXCEPTION_DETAIL Traceback (most recent call last): @@ -71,4 +70,3 @@ ValueError: >>> coffee + coding == enjoyment True ``` - diff --git a/tests/md/pythonmatch.md b/tests/md/pythonmatch.md index 29930eb..44c4df3 100644 --- a/tests/md/pythonmatch.md +++ b/tests/md/pythonmatch.md @@ -1,49 +1,50 @@ -# Try some laden info strings. All should match. +# Try some laden info strings. All should match ```python extraneous info string extension from hashlib import sha256 m = sha256() ``` + ```python extraneous info string extension >>> print('Hello World!') Hello World! ``` - ```python3 extraneous info string extension from hashlib import sha256 m = sha256() ``` + ```python3 extraneous info string extension >>> print('Hello World!') Hello World! ``` - ```py extraneous info string extension from hashlib import sha256 m = sha256() ``` + ```py extraneous info string extension >>> print('Hello World!') Hello World! ``` - ```py extraneous info string extension from hashlib import sha256 m = sha256() ``` + ```py extraneous info string extension >>> print('Hello World!') Hello World! ``` - ```py3 extraneous info string extension from hashlib import sha256 m = sha256() ``` + ```py3 extraneous info string extension >>> print('Hello World!') Hello World! @@ -54,22 +55,24 @@ Note- phmutest will identify this as a code block despite the pycon info string. from hashlib import sha256 m = sha256() ``` + ```pycon extraneous info string extension >>> print('Hello World!') Hello World! ``` - ```{ .python } pycon extraneous info string extension from hashlib import sha256 m = sha256() ``` + ```{ .python } extraneous info string extension >>> print('Hello World!') Hello World! ``` -# one more REPL block than code block +## one more REPL block than code block + ```py3 extraneous info string extension >>> print('Hello World!') Hello World! diff --git a/tests/md/reader.md b/tests/md/reader.md index d02ea26..c59574b 100644 --- a/tests/md/reader.md +++ b/tests/md/reader.md @@ -38,4 +38,3 @@ assert False this is not a NodeType.HTML_COMMENT - diff --git a/tests/md/readerfcb.md b/tests/md/readerfcb.md index 0ff9e43..1c8dfa0 100644 --- a/tests/md/readerfcb.md +++ b/tests/md/readerfcb.md @@ -1,4 +1,4 @@ -# Test reader.py FCB DocNodes. +# Test reader.py FCB DocNodes 4 tilde start fence, 4 tilde closing fence ~~~~python @@ -69,4 +69,3 @@ incorrect expected output ``` - diff --git a/tests/md/replerror.md b/tests/md/replerror.md index 6bc7d62..2903c97 100644 --- a/tests/md/replerror.md +++ b/tests/md/replerror.md @@ -20,12 +20,13 @@ Create a MyBumper instance with an str value. ``` Show that command line --skip skips the test. + + ```python >>> assert False, "--skip identified block" ``` - The bump() call below raises a TypeError since += 1 is not allowed on type str. @@ -34,26 +35,26 @@ allowed on type str. aa ``` +Show that --skip MYSKIPPATTERN skips this block. -Show that skip directive skips the test. ```python >>> # MYSKIPPATTERN ->>> assert False, "directive identified block" +>>> assert False, "--skip identified block" ``` - Show that Python version conditional skip skips the test. + + ```python ->>> assert False, "--skip identified block" +>>> assert False, "directive identified block" ``` - - Show that Python version conditional skip runs the test. + + ```python >>> print("run this test if Python version >= 3.3") run this test if Python version >= 3.3 ``` - diff --git a/tests/md/report.md b/tests/md/report.md index 2317cbf..aba33b9 100644 --- a/tests/md/report.md +++ b/tests/md/report.md @@ -1,6 +1,4 @@ -# - -## Example +# Example The example starts by creating the object m. ```python @@ -21,20 +19,22 @@ have an info string. db406 ``` -The example continues here. It will continue for the entire file. This is -the last Python FCB in the file. +The example continues here. It will continue for the entire file. + ```python m.update(b"more bytes") print(m.hexdigest()[0:5]) ``` Note the expected output is different. -``` + +```expected-output 4c6ea ``` + ```python from enum import Enum @@ -56,6 +56,7 @@ Floats.ADUCK ``` + ```yml on: push: diff --git a/tests/md/setupnoteardown.md b/tests/md/setupnoteardown.md index 6acb8b0..6efeccb 100644 --- a/tests/md/setupnoteardown.md +++ b/tests/md/setupnoteardown.md @@ -1,16 +1,19 @@ -# Test Markdown for setup but no teardown directives. +# Test Markdown for setup but no teardown directives -## These are setup code blocks. +## These are setup code blocks Setup code blocks render to SetupClass. Alternatively, they render to setUpModule when file is specified in the command line option --setup-across-files. + ```python import math ``` + + ```python def my_function(x): return x + 1 @@ -19,7 +22,7 @@ def my_function(x): myglobs_list = [1, 2, 3, 4, "A"] ``` -## Python fenced code block and expected output under test. +## Python fenced code block and expected output under test This is the Python example we want to check. @@ -30,15 +33,18 @@ print(my_function(2)) ``` Expected output: -``` + +```expected-output 10 [1, 2, 3, 4, 'A'] 3 ``` + Note that Python prints the string value with single quotes. This block assigns a name to be shared across files. Note that all names including temporary variables are shared. + ```python shared_int = 9999 ``` @@ -48,8 +54,8 @@ shared_string = "coffee" print(shared_string) ``` -``` +```expected-output coffee ``` -## There are no teardown blocks. +## There are no teardown blocks diff --git a/tests/md/setupto.md b/tests/md/setupto.md index 26435c8..1b6471d 100644 --- a/tests/md/setupto.md +++ b/tests/md/setupto.md @@ -1,4 +1,4 @@ -# Test sharing of setup-across-files. +# Test sharing of setup-across-files Names math, myglobs, and my_function shared from setup blocks by using setup directives and sharing their file with @@ -11,9 +11,9 @@ print(my_function(2)) ``` Expected output: -``` + +```expected-output 10 [1, 2, 3, 4, 'A'] 3 ``` - diff --git a/tests/md/sharedto.md b/tests/md/sharedto.md index 0981fb3..7325c37 100644 --- a/tests/md/sharedto.md +++ b/tests/md/sharedto.md @@ -1,4 +1,4 @@ -# Test sharing of setup-across-files and share-across-files. +# Test sharing of setup-across-files and share-across-files Names math, myglobs, and my_function shared from setup blocks by using setup directives and sharing their file with @@ -11,25 +11,26 @@ print(my_function(2)) ``` Expected output: -``` + +```expected-output 10 [1, 2, 3, 4, 'A'] 3 ``` -Note that Python prints the string value with single quotes. +Note that Python prints the string value with single quotes. Names shared_var and shared_string are shared from the non-setup blocks in the file specified by --share-across-files. + ```python print(shared_int) print(shared_string) ``` Expected output: -``` + +```expected-output 9999 coffee ``` - - diff --git a/tests/md/sharedto2.md b/tests/md/sharedto2.md index a0cb4df..b8ee248 100644 --- a/tests/md/sharedto2.md +++ b/tests/md/sharedto2.md @@ -1,4 +1,4 @@ -# Test share-across-files of file with setup directive. +# Test share-across-files of file with setup directive Names shared_int and shared_string are shared from non-setup blocks in the file setupnoteardown.md. That file is specified by the @@ -7,17 +7,17 @@ tests/toml/acrossfiles2.toml. Note that Python prints the string value with single quotes. - Names shared_int and shared_string are shared from the non-setup blocks in the file specified by --share-across-files. + ```python print(shared_int) print(shared_string) ``` Expected output: -``` + +```expected-output 9999 coffee ``` - diff --git a/tests/md/shareimports1.md b/tests/md/shareimports1.md index e7330d2..5631e9b 100644 --- a/tests/md/shareimports1.md +++ b/tests/md/shareimports1.md @@ -1,5 +1,6 @@ -# Share same imports as the top level of the generated testfile. +# Share unittest import at the top level of the generated testfile +## Note contextlib, io, and sys are only imported by FCBs under test ```python import contextlib @@ -13,5 +14,3 @@ with contextlib.redirect_stdout(io.StringIO()) as fout: print("Hello World!") assert fout.getvalue() == "Hello World!\n" ``` - - diff --git a/tests/md/shareimports2.md b/tests/md/shareimports2.md index 32bdc3c..b110a6c 100644 --- a/tests/md/shareimports2.md +++ b/tests/md/shareimports2.md @@ -1,11 +1,13 @@ +# Use some of the shared imports + +## Note umittest is imported at the top level of the generated testfile + ```python with contextlib.redirect_stdout(io.StringIO()) as fout: print("Greetings Planet!") assert fout.getvalue() == "Greetings Planet!\n" ``` - ```python import unittest.mock ``` - diff --git a/tests/md/unexpected_output.md b/tests/md/unexpected_output.md index 89c3410..23c0d07 100644 --- a/tests/md/unexpected_output.md +++ b/tests/md/unexpected_output.md @@ -1,5 +1,7 @@ -# This file's generated test fails due to a bug. +# This file's generated test fails due to a bug + The code + ```python3 from enum import Enum @@ -12,12 +14,15 @@ class Floats(Enum): for floater in Floats: print(floater) ``` + produces -``` + +```expected-output Floats.APPLES Floats.VERY_SMALL_ROCKS Floats.CHERRIES Floats.ADUCK ``` + Incorrect sample output replaces **Floats.CIDER** with **Floats.VERY_SMALL_ROCKS**. diff --git a/tests/py/generated_project.py b/tests/py/generated_project.py index 371d116..106060a 100644 --- a/tests/py/generated_project.py +++ b/tests/py/generated_project.py @@ -23,7 +23,7 @@ def greeting(name: str) -> str: return "Hello" + "\n\n" + name print(greeting("World!")) - # line 20 + # line 21 _phm_expected_str = """\ Hello @@ -32,9 +32,9 @@ def greeting(name: str) -> str: _phm_printer.cancel_print_capture_on_error() _phm_testcase.assertEqual(_phm_expected_str, _phm_printer.stdout()) - # ------ tests/md/project.md:27 ------ - with self.subTest(msg="tests/md/project.md:27"): - with _phmPrinter(_phm_log, "tests/md/project.md:27", False): + # ------ tests/md/project.md:29 ------ + with self.subTest(msg="tests/md/project.md:29"): + with _phmPrinter(_phm_log, "tests/md/project.md:29", False): text = greeting("Planet!") text = text.replace("\n\n", " ") assert text == "Hello Planet!" # this assert is in the Markdown example. diff --git a/tests/py/generated_sharedemo.py b/tests/py/generated_sharedemo.py index 05b6d1b..92e1480 100644 --- a/tests/py/generated_sharedemo.py +++ b/tests/py/generated_sharedemo.py @@ -33,9 +33,9 @@ def tests(self): with _phmPrinter(_phm_log, "docs/share/file1.md:5", False): from dataclasses import dataclass - # ------ docs/share/file1.md:10 ------ - with self.subTest(msg="docs/share/file1.md:10"): - with _phmPrinter(_phm_log, "docs/share/file1.md:10", False): + # ------ docs/share/file1.md:9 ------ + with self.subTest(msg="docs/share/file1.md:9"): + with _phmPrinter(_phm_log, "docs/share/file1.md:9", False): @dataclass class BeverageActivity: beverage: str @@ -59,9 +59,9 @@ def combine(self) -> str: _phm_printer.cancel_print_capture_on_error() _phm_testcase.assertEqual(_phm_expected_str, _phm_printer.stdout()) - # ------ docs/share/file1.md:34 ------ - with self.subTest(msg="docs/share/file1.md:34"): - with _phmPrinter(_phm_log, "docs/share/file1.md:34", False): + # ------ docs/share/file1.md:35 ------ + with self.subTest(msg="docs/share/file1.md:35"): + with _phmPrinter(_phm_log, "docs/share/file1.md:35", False): we = BeverageActivity("water", "exercise") _phm_globals.update(additions=locals(), built_from="docs/share/file1.md", existing_names=None) @@ -103,11 +103,11 @@ class Test003(unittest.TestCase): def tests(self): - # ------ docs/share/file3.md:8 ------ - with self.subTest(msg="docs/share/file3.md:8"): - with _phmPrinter(_phm_log, "docs/share/file3.md:8", False) as _phm_printer: + # ------ docs/share/file3.md:7 ------ + with self.subTest(msg="docs/share/file3.md:7"): + with _phmPrinter(_phm_log, "docs/share/file3.md:7", False) as _phm_printer: print(bp.combine()) - # line 12 + # line 11 _phm_expected_str = """\ beer-partying """ diff --git a/tests/test_docs.py b/tests/test_docs.py index 71f9928..4f75c58 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -29,7 +29,7 @@ def get_command_and_output(markdown_filename: str) -> List[str]: - """Get the first FCB that starts with 'phmutest' and same for 'summary'.""" + """Get the last FCB that starts with 'phmutest' and same for 'summary'.""" command = "" output = "" content_strings = phmutest.tool.fenced_code_blocks(markdown_filename) @@ -43,7 +43,7 @@ def get_command_and_output(markdown_filename: str) -> List[str]: def get_command_and_log(markdown_filename: str) -> List[str]: - """Get the first FCB that starts with 'phmutest' and same for 'log'.""" + """Get the last FCB that starts with 'phmutest' and same for 'log'.""" command = "" output = "" content_strings = phmutest.tool.fenced_code_blocks(markdown_filename) @@ -82,9 +82,13 @@ def test_setup_example(capsys): assert output == capsys.readouterr().out.lstrip() -def test_readme_metrics(): - """Test the blocks status when running on README.md.""" - command, output = get_command_and_log("README.md") +readme_chooser = phmutest.tool.FCBChooser("README.md") +"""Helper to select lists of FCB contents from the Markdown file.""" + + +def test_readme_code_metrics(): + """Test the metrics when running on README.md.""" + command = readme_chooser.select(info_string="shell")[0] # 1st of selected FCBs args = arg_list(command) phmresult = phmutest.main.main(args) want = phmutest.summary.Metrics( @@ -100,9 +104,37 @@ def test_readme_metrics(): assert want == phmresult.metrics -def test_readme_output(capsys): - """Test the expected output block when running on README.md.""" - command, output = get_command_and_log("README.md") +def test_readme_code_output(capsys): + """Test the shell expected output block when running on README.md.""" + command = readme_chooser.select(info_string="shell")[0] # 1st of selected FCBs + output = readme_chooser.select(contains="args.files: 'README.md'")[0] + args = arg_list(command) + _ = phmutest.main.main(args) + assert output == capsys.readouterr().out.lstrip() + + +def test_readme_repl_metrics(): + """Test the metrics when running on README.md.""" + command = readme_chooser.select(info_string="shell")[1] # 1st of selected FCBs + args = arg_list(command) + phmresult = phmutest.main.main(args) + want = phmutest.summary.Metrics( + number_blocks_run=4, + passed=4, + failed=0, + skipped=0, + suite_errors=0, + number_of_files=1, + files_with_no_blocks=0, + number_of_deselected_blocks=0, + ) + assert want == phmresult.metrics + + +def test_readme_repl_output(capsys): + """Test the shell expected output block when running on README.md.""" + command = readme_chooser.select(info_string="shell")[1] # 1st of selected FCBs + output = readme_chooser.select(contains="args.files: 'README.md'")[1] args = arg_list(command) _ = phmutest.main.main(args) assert output == capsys.readouterr().out.lstrip() @@ -188,8 +220,8 @@ def test_replmode_example(capsys): args = arg_list(command) phmresult = phmutest.main.main(args) want = phmutest.summary.Metrics( - number_blocks_run=3, - passed=3, + number_blocks_run=4, + passed=4, failed=0, skipped=0, suite_errors=0, @@ -300,7 +332,6 @@ def test_deselect_example(capsys): def test_skip_directive_example(capsys): """Test skip directive example.""" - # args = ["docs/advanced/skip.md"] command, output = get_command_and_log("docs/advanced/skip.md") args = arg_list(command) phmresult = phmutest.main.main(args) @@ -408,10 +439,17 @@ def test_unittest_stderr_printing(): def extract_usage_options(text): """From --help output get a set of the usage option names.""" - finder = re.finditer( - pattern=r"[-][-][a-z][\-A-Z_a-z]*", string=text, flags=re.MULTILINE | re.DOTALL + + option_name_finder = ( + # 2 dash + r"[-]{2}" + # mix of dash | letter | number | underscore + r"[\-A-Za-z0-9_]*" + ) + """This won't find options with the other punctuation characters.""" + option_mentions = re.findall( + pattern=option_name_finder, string=text, flags=re.MULTILINE | re.DOTALL ) - option_mentions = [m.group() for m in finder] return set(option_mentions) @@ -440,7 +478,7 @@ def test_usage_options(): def test_quick_links(): """Make sure the README.md quick links are up to date.""" filename = "README.md" - readme = Path("README.md").read_text(encoding="utf-8") + readme = Path(filename).read_text(encoding="utf-8") github_links = make_quick_links(filename) # There must be at least one blank line after the last link. assert github_links + "\n\n" in readme diff --git a/tests/test_fenced.py b/tests/test_fenced.py index db8ed75..ce0979e 100644 --- a/tests/test_fenced.py +++ b/tests/test_fenced.py @@ -4,7 +4,7 @@ def test_python_code_matcher(): - """Test Python block identification.""" + """Test Python code block identification.""" line = "tests/md/pythonmatch.md" phmresult = phmutest.main.command(line) want = phmutest.summary.Metrics( @@ -21,7 +21,7 @@ def test_python_code_matcher(): def test_python_repl_matcher(): - """Test Python block identification.""" + """Test Python session block identification.""" line = "tests/md/pythonmatch.md --replmode" phmresult = phmutest.main.command(line) want = phmutest.summary.Metrics( @@ -49,30 +49,30 @@ def test_report(capsys, checker): Fenced blocks from tests/md/report.md: FencedBlock: info_string= python - lines= 6-9 + lines= 4-7 role= Role.CODE output block= no directives= [] FencedBlock: info_string= python - lines= 12-15 + lines= 10-13 role= Role.CODE - output block= lines 20-22 + output block= lines 18-20 directives= [] FencedBlock: info_string= - lines= 20-22 + lines= 18-20 role= Role.OUTPUT directives= [] FencedBlock: info_string= python - lines= 26-29 + lines= 24-27 role= Role.CODE - output block= lines 32-34 + output block= lines 31-33 directives= [] FencedBlock: - info_string= - lines= 32-34 + info_string= expected-output + lines= 31-33 role= Role.OUTPUT directives= [] FencedBlock: @@ -82,8 +82,8 @@ def test_report(capsys, checker): output block= lines 51-56 skip patterns= 'CHERRIES' directives: (line, type, HTML): - 36, LABEL, - 37, SKIP, + 35, LABEL, + 36, SKIP, FencedBlock: info_string= lines= 51-56 @@ -91,7 +91,7 @@ def test_report(capsys, checker): directives= [] FencedBlock: info_string= yml - lines= 59-64 + lines= 60-65 role= Role.NOROLE output block= no directives: (line, type, HTML): diff --git a/tests/test_tool.py b/tests/test_tool.py index 802db0f..9ed229d 100644 --- a/tests/test_tool.py +++ b/tests/test_tool.py @@ -23,10 +23,10 @@ def test_labeled_fcbs(): labeled = phmutest.tool.labeled_fenced_code_blocks(markdown_filename) assert len(labeled) == 2 assert labeled[0].label == "my-label" - assert labeled[0].line == "43" + assert labeled[0].line == "53" assert labeled[0].contents == "(10, 1)\n" assert labeled[1].label == " EXTRA_SPACES " - assert labeled[1].line == "49" + assert labeled[1].line == "61" assert labeled[1].contents == "assert False\n" @@ -58,7 +58,7 @@ def setup_method(self): self.chooser = phmutest.tool.FCBChooser("tests/md/multi_label.md") def test_not_found(self): - """Select multiple blocks with the same label.""" + """No blocks match.""" selected = self.chooser.select(label="no-chance") assert selected == [] From 80bbe06c06e50b62cd10725bb6c159380db2beef Mon Sep 17 00:00:00 2001 From: Mark Taylor <24257134+tmarktaylor@users.noreply.github.com> Date: Wed, 13 Sep 2023 16:25:36 -0400 Subject: [PATCH 12/27] Add phmutest.main.command() --- docs/callfrompython.md | 7 +++---- src/phmutest/summary.py | 2 +- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/docs/callfrompython.md b/docs/callfrompython.md index b0bafa4..3e53cba 100644 --- a/docs/callfrompython.md +++ b/docs/callfrompython.md @@ -18,15 +18,17 @@ assert phmresult.metrics.number_of_files == 1 ``` ## PhmResult + phmutest.main.command() returns a value of type PhmResult defined in src/phmutest/summary.py. + ```python @dataclass class PhmResult: - """phmutest.main.main() return type. Markdown Python example test results.""" + """phmutest.main.command() return type. Markdown Python example test results.""" test_program: Optional[unittest.TestProgram] is_success: bool @@ -49,7 +51,6 @@ assert phmresult.metrics.passed == 3 assert phmresult.metrics.number_of_files == 1 ``` - ## limitation The limitation described here applies to call from Python when @@ -85,5 +86,3 @@ Tests checking this example: - tests/test_subprocess.py:test_callfrompython() - tests/test_docs.py:test_call_from_python() - tests/test_docs.py:test_phmresult() - - diff --git a/src/phmutest/summary.py b/src/phmutest/summary.py index 88cf532..0c07540 100644 --- a/src/phmutest/summary.py +++ b/src/phmutest/summary.py @@ -82,7 +82,7 @@ class Metrics: @dataclass class PhmResult: - """phmutest.main.main() return type. Markdown Python example test results.""" + """phmutest.main.command() return type. Markdown Python example test results.""" test_program: Optional[unittest.TestProgram] is_success: bool From 7282d06dbc25ef15a46ff9f8575a2510c17cf842 Mon Sep 17 00:00:00 2001 From: Mark Taylor <24257134+tmarktaylor@users.noreply.github.com> Date: Wed, 13 Sep 2023 16:26:25 -0400 Subject: [PATCH 13/27] Move entry_points to setup.cfg, --- setup.cfg | 8 ++++---- setup.py | 18 ------------------ 2 files changed, 4 insertions(+), 22 deletions(-) delete mode 100644 setup.py diff --git a/setup.cfg b/setup.cfg index 6fbcd86..6946c3d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -54,10 +54,10 @@ where = src [options.package_data] phmutest = py.typed -# see setup.py -# [options.entry_points] -# console_scripts = -# phmutest = phmutest.main:main +# https://setuptools.readthedocs.io/en/latest/userguide/declarative_config.html +[options.entry_points] +console_scripts = + phmutest = phmutest.main:entry_point [bdist_wheel] # This flag says to generate wheels that support both Python 2 and Python diff --git a/setup.py b/setup.py deleted file mode 100644 index 6f6e10c..0000000 --- a/setup.py +++ /dev/null @@ -1,18 +0,0 @@ -"""A setuptools based setup module. -""" - -# See setup.cfg. -# Note: -# Entry points done here since setuptools minimum version -# for this section in setup.cfg is 51.0.0 per -# https://setuptools.readthedocs.io/en/latest/userguide/declarative_config.html -# Always prefer setuptools over distutils -from setuptools import setup - -setup( - entry_points={ - "console_scripts": [ - "phmutest=phmutest.main:entry_point", - ], - }, -) From 3e4d3ec396a0e38c879ab407863fba7e0f53db76 Mon Sep 17 00:00:00 2001 From: Mark Taylor <24257134+tmarktaylor@users.noreply.github.com> Date: Wed, 13 Sep 2023 16:28:23 -0400 Subject: [PATCH 14/27] Docs updates. --- docs/api.md | 5 +++-- docs/codemode.md | 11 +++++++++-- docs/demos.md | 7 ++++--- docs/howitworks.md | 21 --------------------- tests/doctest/generated_replmode.txt | 19 +++++++++++++++++++ 5 files changed, 35 insertions(+), 28 deletions(-) diff --git a/docs/api.md b/docs/api.md index d8c5c52..5c43dd6 100644 --- a/docs/api.md +++ b/docs/api.md @@ -1,6 +1,8 @@ # API version 0.0.2 -## API - phmutest.tool. +## API - phmutest.tool + +Look for example uses of FCBChooser in tests/test_docs.py. ```python class FCBChooser: @@ -112,4 +114,3 @@ def fenced_code_blocks(markdown_filename: str) -> List[str]: fenced code block. """ ``` - diff --git a/docs/codemode.md b/docs/codemode.md index 32e0dfc..5e0b4bb 100644 --- a/docs/codemode.md +++ b/docs/codemode.md @@ -84,6 +84,7 @@ the unittest run. Refer to the implementation in src/phmutest/globs.py class Globals. ## Setup and teardown + Setup and teardown applies to examples in a single Markdown file. - Blocks with the phmutest-setup directive render into **setUpClass(cls)** @@ -98,6 +99,14 @@ The lines printed by --sharing for setup and teardown directives start with `sharing-class`. See test_share_across_with_setup_sharing() in tests/test_sharing.py. +Setup and teardown block errors do not show the Markdown FCB line number. +If a setup block raises an exception, the Markdown line number does not +get printed in the exception traceback. +Run phmutest with the --summary or --progress options. +A section in the --summary output shows setup and teardown errors. +In the --log and --progress outputs the suffix "setup" is added +to the Markdown location. + ## Setup across files Setup and teardown blocks in a single Markdown file are applied to all FILEs @@ -114,6 +123,4 @@ to copy the names assigned by the entire function to module level globals. This option turns on per block verbose printing. The printing is directed to the standard error stream shared with unittest's verbose printing. - [1]: https://docs.python.org/3/library/unittest.html - diff --git a/docs/demos.md b/docs/demos.md index fc36cb9..b942f4f 100644 --- a/docs/demos.md +++ b/docs/demos.md @@ -1,4 +1,6 @@ -# Sections +# Sections and Demos + +## Sections - [Advanced feature details](advanced.md) - [How it works](howitworks.md) @@ -7,7 +9,7 @@ - [Call from Python example](callfrompython.md) - [Api](api.md) -# Demos +## Demos - [REPL Mode](replmode.md) - [fixture change workdir](fix/code/chdir.md) @@ -19,4 +21,3 @@ - [deselect groups](group/deselect.md) - [setup/teardown](setup/setup.md) - [setup across files](setup/across1.md) - diff --git a/docs/howitworks.md b/docs/howitworks.md index 44a9788..4ad87df 100644 --- a/docs/howitworks.md +++ b/docs/howitworks.md @@ -59,28 +59,7 @@ Test session blocks with --replmode. Test code and output blocks otherwise. [Code mode](codemode.md) | [Session mode](sessionmode.md) -## Hints - -- Since phmutest generates code, the input files should be from a trusted - source. -- The phmutest Markdown parser finds fenced code blocks enclosed by - html `
` and `
` tags. - The tags may require a preceding and trailing blank line - to render correctly. See example at the bottom tests/md/readerfcb.md. -- Markdown indented code blocks ([Spec][4] section 4.4) are ignored. -- A malformed HTML comment ending is bad. Make sure - it ends with both dashes like `-->`. -- A misspelled directive will be missing from the --report output. -- If the generated test file has a compile error phmutest will raise an - ImportError when importing it. -- Blocks skipped with --skip and the phmutest-skip directive - are not rendered. This is useful to avoid above import error. -- In repl mode **no** skipped blocks are rendered. -- Try redirecting `--generate -` standard output into PYPI Pygments to - colorize the generated test file. - [1]: https://github.github.com/gfm/#fenced-code-blocks [2]: https://github.github.com/gfm/#info-string [3]: https://docs.python.org/3/library/unittest.html [4]: https://spec.commonmark.org - diff --git a/tests/doctest/generated_replmode.txt b/tests/doctest/generated_replmode.txt index d255b75..b6e8e64 100644 --- a/tests/doctest/generated_replmode.txt +++ b/tests/doctest/generated_replmode.txt @@ -84,6 +84,23 @@ Hello World! + + + + + + + + + + + + + + + + + @@ -126,6 +143,8 @@ Hello World! + + From 36cae251e8872ebb69f49f2488e755208dcbadeb Mon Sep 17 00:00:00 2001 From: Mark Taylor <24257134+tmarktaylor@users.noreply.github.com> Date: Wed, 13 Sep 2023 16:35:08 -0400 Subject: [PATCH 15/27] Code review fixes --- src/phmutest/fenced.py | 6 +++--- src/phmutest/subtest.py | 3 +-- tests/check_classifiers.py | 2 +- tests/test_api.py | 2 -- tests/test_generate.py | 2 +- tests/test_globs.py | 17 ++++++----------- tests/test_rebind.py | 3 +++ 7 files changed, 15 insertions(+), 20 deletions(-) diff --git a/src/phmutest/fenced.py b/src/phmutest/fenced.py index b8c3ef3..926d644 100644 --- a/src/phmutest/fenced.py +++ b/src/phmutest/fenced.py @@ -60,14 +60,14 @@ def __init__(self, node: phmutest.reader.DocNode) -> None: self.line = node.line self.end_line = node.end_line self.role = Role.NOROLE - self.contents = node.payload # type: str + self.contents: str = node.payload if python_matcher.re.match(self.info_string): if self.contents.startswith(">>> "): self.role = Role.SESSION else: self.role = Role.CODE - self.output = None # type: Optional["FencedBlock"] - self.skip_patterns = [] # type: List[str] + self.output: Optional["FencedBlock"] = None + self.skip_patterns: List[str] = [] self.directives = phmutest.direct.get_directives(node) self._directive_markers = set(d.type for d in self.directives) diff --git a/src/phmutest/subtest.py b/src/phmutest/subtest.py index eceb9ea..25ae652 100644 --- a/src/phmutest/subtest.py +++ b/src/phmutest/subtest.py @@ -63,10 +63,9 @@ def fill_in(template: str, replacements: Mapping[str, str]) -> str: """ for k in replacements: assert not k.startswith("$"), "easy to make mistake, requires no leading $" - finder = re.finditer( + standalone_keys = re.findall( pattern=r"^\s*([$]\w+)$", string=template, flags=re.MULTILINE | re.DOTALL ) - standalone_keys = [m.group(1) for m in finder] for key in standalone_keys: replacement_key = key[1:] if replacement_key not in replacements: diff --git a/tests/check_classifiers.py b/tests/check_classifiers.py index ea44ecd..9a8541c 100644 --- a/tests/check_classifiers.py +++ b/tests/check_classifiers.py @@ -20,7 +20,7 @@ text = config.get("metadata", "classifiers") lines = text.splitlines() # remove blank lines -lines = [line for line in lines if len(line) > 0] +lines = [line for line in lines if line] unique_lines = set(lines) messages = [] # Check for duplicates. diff --git a/tests/test_api.py b/tests/test_api.py index af5843a..b078e30 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -13,8 +13,6 @@ def test_api(checker): codefile = Path("src/phmutest/tool.py") codetext = codefile.read_text(encoding="utf-8") for num, f in enumerate(fcbs, start=1): - # if f not in codetext: - # checker(f, codetext) assert f in codetext, f"{apifile} FCB number {num} is not in {codefile}." # Check that tool.py was not modified since the last diff --git a/tests/test_generate.py b/tests/test_generate.py index e72d76f..8ecc39b 100644 --- a/tests/test_generate.py +++ b/tests/test_generate.py @@ -1,4 +1,4 @@ -"""Check handling of --generate command line options""" +"""Check handling of --generate command line option.""" import os import unittest import unittest.main diff --git a/tests/test_globs.py b/tests/test_globs.py index 65f1643..70ce93c 100644 --- a/tests/test_globs.py +++ b/tests/test_globs.py @@ -90,7 +90,7 @@ def test_update_name_pops(self): assert self.globs.copy() == dict() def test_update_name_ifpops(self): - """Try an update with names that get conditionally popped before the update.""" + """Update with name that gets conditionally popped before the update.""" # Exercises the if XXX in additions.pop('XXX', None) lines items = { "unittest": None, @@ -152,7 +152,7 @@ def test_check_integrity_error(self): def test_no_originals_error(self): """Simulate no pre-existing globals/module attributes can be managed error.""" - # Note that check_integrity() is believed to redundant. + # Note that check_integrity() is believed to be redundant. # It is called near the end of update(). # This test case simulates a failure by calling check_integrity() directly. assert self.globs.get_names() == set() @@ -243,17 +243,12 @@ def test_extractor(capsys): def test_share_testfile_imports(): - """Use same imports in an example that are at the top level of the testfile. + """Use import unittest in an example. + It is imported at the top level of the testfile. Logic in globs.py prevents the copy to the module globals. The example works - because the imports already exist at the top level of the testfile. - Also note that --sharing will not show these imports. - - Generated testfile top level imports: - import contextlib - import io - import sys - import unittest + because the import already exists at the top level of the testfile. + Also note that --sharing will not show the import. """ command = ( "tests/md/shareimports1.md tests/md/shareimports2.md " diff --git a/tests/test_rebind.py b/tests/test_rebind.py index a14c818..e384e55 100644 --- a/tests/test_rebind.py +++ b/tests/test_rebind.py @@ -25,6 +25,8 @@ by the fixture. file2.md sees the value of 'we' assigned by file1.md. The fixture cleanup function sees the original value of 'we'. +Just shows that the rebind in fo we in file1.md did not affect the original value. +The rebind affects the shallow copy. To see the effect of rebind in REPL mode see test_globs.py::test_extractor(). """ @@ -36,6 +38,7 @@ def cleanup(globs): + """Handles clean up of the objects created by the fixture.""" print("cleanup-") assert globs == {"we": 3, "no_conflict": 9999} From c1feb756b433fa542e0a00fc9552c5ce98d8ea51 Mon Sep 17 00:00:00 2001 From: Mark Taylor <24257134+tmarktaylor@users.noreply.github.com> Date: Wed, 13 Sep 2023 16:37:36 -0400 Subject: [PATCH 16/27] Markdown linting changes. --- docs/replmode.md | 21 ++++++++++------ tests/test_cases.py | 59 ++++++++++++++++++++++---------------------- tests/test_direct.py | 12 ++++----- tests/test_reader.py | 6 +++++ 4 files changed, 55 insertions(+), 43 deletions(-) diff --git a/docs/replmode.md b/docs/replmode.md index 276e32d..0dfca49 100644 --- a/docs/replmode.md +++ b/docs/replmode.md @@ -19,6 +19,7 @@ All fenced code blocks in the file are joined into one long docstring. Example borrowed from Python Standard Library fractions documentation. + ```py >>> from fractions import Fraction >>> Fraction(16, -10) @@ -32,20 +33,26 @@ Fraction(3, 7) ``` Here we show name 'b' assigned in the first FCB is still visible. + ```py >>> b 12 ``` -# phmutest command line. - +```py +>>> Fraction('3/7') +Fraction(3, 7) ``` + +## phmutest command line + +```shell phmutest docs/replmode.md --replmode --log ``` -## phmutest output. +## phmutest output -``` +```txt log: args.files: 'docs/replmode.md' args.replmode: 'True' @@ -54,8 +61,8 @@ args.log: 'True' location|label result ------------------- ------ docs/replmode.md:11 pass -docs/replmode.md:22 pass -docs/replmode.md:35 pass +docs/replmode.md:23 pass +docs/replmode.md:37 pass +docs/replmode.md:42 pass ------------------- ------ ``` - diff --git a/tests/test_cases.py b/tests/test_cases.py index 476429f..2e2cfa3 100644 --- a/tests/test_cases.py +++ b/tests/test_cases.py @@ -61,7 +61,6 @@ def test_no_files(): def test_skipif_no_output(): """Source Code block with no ouput and skipif directive. Empty Python block.""" - # This covers the cases.py line near the end: test_classes += "\n" line = "tests/md/cases.md --log" phmresult = phmutest.main.command(line) want = phmutest.summary.Metrics( @@ -136,19 +135,19 @@ def test_print_captured_output(startswith_checker): assert isinstance(phmresult.test_program, unittest.TestProgram) output = ferr.getvalue() - expected = """tests/md/printer.md:6 ... failed + expected = """tests/md/printer.md:7 ... failed === phmutest: captured stdout === asserting False... === end === === phmutest: captured stderr === asserting False... === end === - tests/md/printer.md:17 ... pass - tests/md/printer.md:30 ... failed + tests/md/printer.md:18 ... pass + tests/md/printer.md:31 ... failed === phmutest: captured stdout === (10, 1) === end === - tests/md/printer.md:42 ... skip phmutest-skip + tests/md/printer.md:46 ... skip phmutest-skip """ startswith_checker(expected, output) ferr.close() @@ -213,15 +212,15 @@ def test_setup_no_teardown(capsys): assert want == phmresult.metrics assert phmresult.is_success is True assert isinstance(phmresult.test_program, unittest.TestProgram) - assert "tests/md/setupnoteardown.md:10 setup" in phmresult.log[0][0] - assert "tests/md/setupnoteardown.md:14 setup" in phmresult.log[1][0] - assert "tests/md/setupnoteardown.md:26" in phmresult.log[2][0] - assert "tests/md/setupnoteardown.md:42" in phmresult.log[3][0] - assert "tests/md/setupnoteardown.md:46" in phmresult.log[4][0] + assert "tests/md/setupnoteardown.md:11 setup" in phmresult.log[0][0] + assert "tests/md/setupnoteardown.md:17 setup" in phmresult.log[1][0] + assert "tests/md/setupnoteardown.md:29" in phmresult.log[2][0] + assert "tests/md/setupnoteardown.md:48" in phmresult.log[3][0] + assert "tests/md/setupnoteardown.md:52" in phmresult.log[4][0] def test_setup_across_no_teardown(capsys): - """Run the setup across files example and check the log.""" + """Run the setup across files.""" line = ( "tests/md/setupnoteardown.md tests/md/setupto.md --log " "--setup-across-files tests/md/setupnoteardown.md" @@ -241,17 +240,17 @@ def test_setup_across_no_teardown(capsys): assert phmresult.is_success is True assert isinstance(phmresult.test_program, unittest.TestProgram) assert "setUpModule" in phmresult.log[0][0] - assert "tests/md/setupnoteardown.md:10" in phmresult.log[1][0] - assert "tests/md/setupnoteardown.md:14" in phmresult.log[2][0] - assert "tests/md/setupnoteardown.md:26" in phmresult.log[3][0] - assert "tests/md/setupnoteardown.md:42" in phmresult.log[4][0] - assert "tests/md/setupnoteardown.md:46" in phmresult.log[5][0] + assert "tests/md/setupnoteardown.md:11" in phmresult.log[1][0] + assert "tests/md/setupnoteardown.md:17" in phmresult.log[2][0] + assert "tests/md/setupnoteardown.md:29" in phmresult.log[3][0] + assert "tests/md/setupnoteardown.md:48" in phmresult.log[4][0] + assert "tests/md/setupnoteardown.md:52" in phmresult.log[5][0] assert "tests/md/setupto.md:7" in phmresult.log[6][0] assert "tearDownModule" in phmresult.log[7][0] def test_setup_across_share_across(capsys): - """Run the setup across files example and check the log.""" + """Run the setup+share across files.""" line = "--log --config tests/toml/acrossfiles.toml" phmresult = phmutest.main.command(line) want = phmutest.summary.Metrics( @@ -268,13 +267,13 @@ def test_setup_across_share_across(capsys): assert phmresult.is_success is True assert isinstance(phmresult.test_program, unittest.TestProgram) assert "setUpModule" in phmresult.log[0][0] - assert "tests/md/setupnoteardown.md:10 setup" in phmresult.log[1][0] - assert "tests/md/setupnoteardown.md:14 setup" in phmresult.log[2][0] - assert "tests/md/setupnoteardown.md:26" in phmresult.log[3][0] - assert "tests/md/setupnoteardown.md:42" in phmresult.log[4][0] - assert "tests/md/setupnoteardown.md:46" in phmresult.log[5][0] + assert "tests/md/setupnoteardown.md:11 setup" in phmresult.log[1][0] + assert "tests/md/setupnoteardown.md:17 setup" in phmresult.log[2][0] + assert "tests/md/setupnoteardown.md:29" in phmresult.log[3][0] + assert "tests/md/setupnoteardown.md:48" in phmresult.log[4][0] + assert "tests/md/setupnoteardown.md:52" in phmresult.log[5][0] assert "tests/md/sharedto.md:7" in phmresult.log[6][0] - assert "tests/md/sharedto.md:24" in phmresult.log[7][0] + assert "tests/md/sharedto.md:26" in phmresult.log[7][0] assert "tearDownModule" in phmresult.log[8][0] @@ -296,11 +295,11 @@ def test_share_across_with_setup(capsys): assert phmresult.is_success is True assert isinstance(phmresult.test_program, unittest.TestProgram) assert "setUpModule" in phmresult.log[0][0] - assert "tests/md/setupnoteardown.md:10 setup" in phmresult.log[1][0] - assert "tests/md/setupnoteardown.md:14 setup" in phmresult.log[2][0] - assert "tests/md/setupnoteardown.md:26" in phmresult.log[3][0] - assert "tests/md/setupnoteardown.md:42" in phmresult.log[4][0] - assert "tests/md/setupnoteardown.md:46" in phmresult.log[5][0] + assert "tests/md/setupnoteardown.md:11 setup" in phmresult.log[1][0] + assert "tests/md/setupnoteardown.md:17 setup" in phmresult.log[2][0] + assert "tests/md/setupnoteardown.md:29" in phmresult.log[3][0] + assert "tests/md/setupnoteardown.md:48" in phmresult.log[4][0] + assert "tests/md/setupnoteardown.md:52" in phmresult.log[5][0] assert "tests/md/sharedto2.md:13" in phmresult.log[6][0] assert "tearDownModule" in phmresult.log[7][0] @@ -333,8 +332,8 @@ def test_progress_option(): lines = err.getvalue().splitlines() assert "setUpModule()..." in lines[0] assert "leaving setUpModule." in lines[1] - assert "docs/fix/code/chdir.md:25 ... pass" in lines[2] - assert "docs/fix/code/chdir.md:29 ... pass" in lines[3] + assert "docs/fix/code/chdir.md:24 ... pass" in lines[2] + assert "docs/fix/code/chdir.md:28 ... pass" in lines[3] assert "tearDownModule()..." in lines[4] assert "leaving tearDownModule." in lines[5] err.close() diff --git a/tests/test_direct.py b/tests/test_direct.py index 70abe98..09b9051 100644 --- a/tests/test_direct.py +++ b/tests/test_direct.py @@ -12,7 +12,7 @@ def get_all_fcbs(filename): class TestPhmdoctest: - """Check directives that start with <~--phmdoctest.""" + """Check directives that start with " class TestPhutest: - """Check directives that start with <~--phmutest.""" + """Check directives that start with " def test_comments_blanks(self): directive = self.blocks[12].directives[0] assert directive.type == Marker.SKIP assert directive.value == "" - assert directive.line == 102 + assert directive.line == 116 assert directive.literal == "" @@ -146,7 +146,7 @@ def test_indented(self): assert self.blocks[0].has_directive(Marker.SKIP) assert self.blocks[0].role == Role.CODE directive = self.blocks[0].directives[0] - assert directive.line == 5 + assert directive.line == 6 # over indented HTML comment assert self.blocks[1].directives == [] diff --git a/tests/test_reader.py b/tests/test_reader.py index f802225..1d3d74d 100644 --- a/tests/test_reader.py +++ b/tests/test_reader.py @@ -132,3 +132,9 @@ def test_info_string(self): assert self.nodes[10].info_string == "python extra stuff" assert self.nodes[10].line == 58 assert self.nodes[10].end_line == 60 + + +def test_getline_position_0(): + """Position 0 should be line number 1.""" + line_getter = phmutest.reader.PositionToLineNumber("hello world") + assert line_getter.get_line(position=0) == 1 From 04fc5bdaae1b6fbe3c068fcb904a8a9810cba247 Mon Sep 17 00:00:00 2001 From: Mark Taylor <24257134+tmarktaylor@users.noreply.github.com> Date: Wed, 13 Sep 2023 16:39:29 -0400 Subject: [PATCH 17/27] Add expected-output info string. --- src/phmutest/printer.py | 6 ++++ tests/md/does_not_print.md | 4 +-- tests/md/example1.md | 2 +- tests/md/output_info_string.md | 40 +++++++++++++++++++++ tests/test_patching.py | 64 ++++++++++++++++++++++++++-------- 5 files changed, 99 insertions(+), 17 deletions(-) create mode 100644 tests/md/output_info_string.md diff --git a/src/phmutest/printer.py b/src/phmutest/printer.py index a950a04..36d8eb2 100644 --- a/src/phmutest/printer.py +++ b/src/phmutest/printer.py @@ -76,8 +76,14 @@ def cancel_print_capture_on_error(self) -> None: to assertEqual. It is in the exception message. Thus there is no need to print captured stdout here. Note that any captured stderr is discarded. + + Append the suffix " o" to the Markdown file location to indicate that + captured standard output was compared to an expected output block. + The call here implies that the statements in the code block + ran without raising an exception or assertion. """ self.is_print_capture_on_error = False + self.location += " o" def stdout(self) -> str: """Return captured stdout.""" diff --git a/tests/md/does_not_print.md b/tests/md/does_not_print.md index 8488e6a..eac14c2 100644 --- a/tests/md/does_not_print.md +++ b/tests/md/does_not_print.md @@ -1,4 +1,4 @@ -### Code doesn't call print(). +# Code doesn't call print() Python fenced code block has no print statements. @@ -9,6 +9,6 @@ a += 5 Expected output- -``` +```expected-output a= 8 ``` diff --git a/tests/md/example1.md b/tests/md/example1.md index 9f4a41f..8f407d6 100644 --- a/tests/md/example1.md +++ b/tests/md/example1.md @@ -24,7 +24,7 @@ for floater in Floats: ``` sample output: -``` +```expected_output Floats.APPLES Floats.CIDER Floats.CHERRIES diff --git a/tests/md/output_info_string.md b/tests/md/output_info_string.md new file mode 100644 index 0000000..664b16f --- /dev/null +++ b/tests/md/output_info_string.md @@ -0,0 +1,40 @@ +# Test output FCB info string patch point + +Markdown file to show the expected output FCB info string patch detects the +expected output FCB below. + +## Interactive Python session (doctest) + +```py +>>> print("Hello World!") +Hello World! +``` + +## Source Code and terminal output + +Code: + +```python +from enum import Enum + +class Floats(Enum): + APPLES = 1 + CIDER = 2 + CHERRIES = 3 + ADUCK = 4 + +for floater in Floats: + print(floater) +``` + +sample output with the info string "captured-stdout". +A mock.patch() is required. The --log output will have +a " o" at the end of the location line for the +above python FCB. + +```captured-stdout +Floats.APPLES +Floats.CIDER +Floats.CHERRIES +Floats.ADUCK +``` diff --git a/tests/test_patching.py b/tests/test_patching.py index 5b323c3..6044b44 100644 --- a/tests/test_patching.py +++ b/tests/test_patching.py @@ -9,27 +9,28 @@ import phmutest.fixture import phmutest.main import phmutest.reader +import phmutest.select import phmutest.session import phmutest.summary from phmutest.direct import MarkerPattern # Note that the last block is included because it has the # info string ladenpython in the Markdown input file. -infostring_log = """\ +python_infostring_log = """\ log: args.files: 'tests/md/patching1.md' args.log: 'True' -location|label result -------------------------------------- ------ -tests/md/patching1.md:2 testing-1-2-3 pass -tests/md/patching1.md:10............. pass -tests/md/patching1.md:19............. pass -------------------------------------- ------ +location|label result +--------------------------------------- ------ +tests/md/patching1.md:4 testing-1-2-3 o pass +tests/md/patching1.md:13 o............. pass +tests/md/patching1.md:23 o............. pass +--------------------------------------- ------ """ -def test_infostring_patch(capsys): +def test_python_infostring_patch(capsys): """Patching fenced code block info_string python language matching.""" matcher = phmutest.fenced.PythonMatcher() matcher.python_patterns.append("ladenpython") # Also match info string ladenpython. @@ -48,11 +49,12 @@ def test_infostring_patch(capsys): number_of_deselected_blocks=0, ) assert want == phmresult.metrics - assert infostring_log == capsys.readouterr().out.lstrip() + assert python_infostring_log == capsys.readouterr().out.lstrip() def make_finder(old_name, new_name): """Make a directive MarkerPattern to match new_name that behaves like old_name.""" + assert old_name != new_name finders = [f for f in phmutest.direct.directive_finders if old_name in f.pattern] assert finders, f"no finder pattern containing {old_name}." assert len(finders) == 1, f"more than 1 finder pattern containing {old_name}." @@ -70,11 +72,11 @@ def make_finder(old_name, new_name): args.files: 'tests/md/patching1.md' args.log: 'True' -location|label result -------------------------------------- ------ -tests/md/patching1.md:2 testing-1-2-3 pass -tests/md/patching1.md:10 abc......... pass -------------------------------------- ------ +location|label result +--------------------------------------- ------ +tests/md/patching1.md:4 testing-1-2-3 o pass +tests/md/patching1.md:13 abc o......... pass +--------------------------------------- ------ """ @@ -207,3 +209,37 @@ def test_doctest_optionflags_patch(): ) assert want == phmresult.metrics assert phmresult.is_success is True + + +output_infostring_log = """\ +log: +args.files: 'tests/md/output_info_string.md' +args.log: 'True' + +location|label result +----------------------------------- ------ +tests/md/output_info_string.md:17 o pass +----------------------------------- ------ +""" + + +def test_output_infostring_patch(capsys, endswith_checker): + """Add a new FCB info string that identifies the expected output block.""" + info_strings = phmutest.select.OUTPUT_INFO_STRINGS + info_strings.append("captured-stdout") + line = "tests/md/output_info_string.md --log" + with mock.patch("phmutest.select.OUTPUT_INFO_STRINGS", info_strings): + phmresult = phmutest.main.command(line) + want2 = phmutest.summary.Metrics( + number_blocks_run=1, + passed=1, + failed=0, + skipped=0, + suite_errors=0, + number_of_files=1, + files_with_no_blocks=0, + number_of_deselected_blocks=0, + ) + assert want2 == phmresult.metrics + assert phmresult.is_success is True + assert output_infostring_log == capsys.readouterr().out.lstrip() From 3b224eea3b3ade9210ab2c6b297809106669ead3 Mon Sep 17 00:00:00 2001 From: Mark Taylor <24257134+tmarktaylor@users.noreply.github.com> Date: Wed, 13 Sep 2023 16:41:13 -0400 Subject: [PATCH 18/27] Handle --replmode fixture exception. --- src/phmutest/session.py | 124 ++++++++++++++++++++++++---------------- tests/fail/bumper.py | 6 +- tests/test_errors.py | 43 ++++++++------ tests/test_session.py | 49 ++++++++++++++-- 4 files changed, 150 insertions(+), 72 deletions(-) diff --git a/src/phmutest/session.py b/src/phmutest/session.py index 8c12cbd..d309061 100644 --- a/src/phmutest/session.py +++ b/src/phmutest/session.py @@ -1,13 +1,13 @@ """Generate and run docstests for Python interactive session FCBs.""" import argparse -import contextlib import doctest import importlib import itertools import sys +import traceback from dataclasses import dataclass from pathlib import Path -from typing import Any, Dict, List, Optional, Tuple, cast +from typing import Any, Callable, Dict, List, Optional, Tuple import phmutest.cases import phmutest.globs @@ -186,45 +186,66 @@ def generate(args: argparse.Namespace, block_store: phmutest.select.BlockStore) args.generate.close() +def null_cleanup() -> None: + """Do nothing cleanup function for case where no fixture cleanup specified.""" + pass + + +UserFixtureInfo = Tuple[Dict[str, Any], Callable[[], None], bool] +"""Function return type [globs, cleanup function, success].""" + + def process_user_fixture( args: argparse.Namespace, log: List[List[str]] -) -> Tuple[contextlib.ExitStack, Optional[Dict[str, Any]]]: - """Check for --fixture. Create a context manager and get globs from the fixture. - - Return a null context manager if there is no fixture. - Return an empty dict if there are no fixture globs. - The globs become visible as global variables to code under test. - """ - # Create nullcontext cm for with statement below for case where no fixture cleanup. - # Tell mypy nullcontext looks like an ExitStack. - globs = None - cm = cast(contextlib.ExitStack, contextlib.nullcontext()) - if args.fixture: - modulepackage, function_name = phmutest.cases.get_fixture_parts(args.fixture) - m = importlib.import_module(modulepackage) - f = getattr(m, function_name) - if user_fixture := f(log=log, is_replmode=True): +) -> UserFixtureInfo: + globs = {} + cleanup_function = null_cleanup + modulepackage, function_name = phmutest.cases.get_fixture_parts(args.fixture) + m = importlib.import_module(modulepackage) + f = getattr(m, function_name) + try: + user_fixture = f(log=log, is_replmode=True) + except Exception: + print("-" * 60) + print(f"Caught an exception in --fixture {args.fixture}...") + # Print the traceback to stdout + # since the doctest failure printing is to stdout. + traceback.print_exc(file=sys.stdout) + # Expecting caller to ignore the first two items of the returned tuple. + return {}, null_cleanup, False + + if user_fixture: + if user_fixture.globs is not None: globs = user_fixture.globs - if user_fixture.repl_cleanup: - # Note- DocTestRunner catches exceptions raised by the Example under - # test. The ExitStack with callback here assures the fixture cleanup - # code is run in the event any of this package's code - # wrapped around DocTestRunner raises an exception. - cm = contextlib.ExitStack() - cm.callback(user_fixture.repl_cleanup) - - # The user_fixture.globs default value is None. - # In the event of sharing across files, the calling code needs globs as an - # empty mapping to store names that are shared across files. - if globs is None: - globs = {} - else: - # If --sharing ".", show the fixture globs. - if phmutest.cases.is_verbose_sharing(args, Path("placeholder")): + + if user_fixture.repl_cleanup is not None: + cleanup_function = user_fixture.repl_cleanup + + # When --sharing is ".", and the fixture returned some globs, show them. + if globs and phmutest.cases.is_verbose_sharing(args, Path("placeholder")): glob_names = ", ".join(globs.keys()) print(f"{args.fixture} is sharing: {glob_names}") + return globs, cleanup_function, True - return cm, globs + +def update_globs_show_sharing( + args: argparse.Namespace, + globs: Optional[Dict[str, Any]], + fileblocks: phmutest.select.FileBlocks, + extractor: Optional[phmutest.globs.AssignmentExtractor], +) -> None: + """If sharing across files copy assigned names to globs, Do --sharing.""" + # This is a refactoring to get run_repl() complexity down from 11. + # Modifies globs in place. Clears the extractor. + if extractor is not None and globs is not None: + for name in extractor.assignments: + globs[name] = extractor.assignments[name] + if phmutest.cases.is_verbose_sharing(args, fileblocks.path): + shared_names = ", ".join(extractor.assignments.keys()) + print(f"{fileblocks.built_from} is sharing: {shared_names}") + + if extractor is not None: + extractor.assignments.clear() def run_repl( @@ -239,10 +260,20 @@ def run_repl( return phmutest.summary.EMPTY_PHMRESULT optionflags = doctest.FAIL_FAST if "-f" in extra_args else 0 + globs: Optional[Dict[str, Any]] = {} + cleanup_function = null_cleanup log: List[List[str]] = [] number_of_errors = 0 - cm, globs = process_user_fixture(args, log) - with cm: + if args.fixture: + globs, cleanup_function, success = process_user_fixture(args, log) + if not success: + phm_result = phmutest.summary.EMPTY_PHMRESULT + phm_result.is_success = False + phm_result.metrics.suite_errors = 1 + phm_result.log = [[str(args.fixture), "error", ""]] + return phm_result + + try: for path in args.files: fileblocks = block_store.get_blocks(path) @@ -257,18 +288,7 @@ def run_repl( if args.progress: phmutest.summary.show_log(result.log) - # If sharing across files names assigned by the file, udate globs - # with the shared names. - # Implement command line --sharing for the file. - if extractor is not None and globs is not None: - for name in extractor.assignments: - globs[name] = extractor.assignments[name] - if phmutest.cases.is_verbose_sharing(args, fileblocks.path): - shared_names = ", ".join(extractor.assignments.keys()) - print(f"{fileblocks.built_from} is sharing: {shared_names}") - - if extractor is not None: - extractor.assignments.clear() + update_globs_show_sharing(args, globs, fileblocks, extractor) log.extend(result.log) number_of_errors += result.number_of_errors @@ -277,6 +297,12 @@ def run_repl( ): break + except Exception as e: + cleanup_function() + raise e + + cleanup_function() + metrics = phmutest.summary.compute_metrics( len(args.files), number_of_errors, diff --git a/tests/fail/bumper.py b/tests/fail/bumper.py index 99a122e..f9086fb 100644 --- a/tests/fail/bumper.py +++ b/tests/fail/bumper.py @@ -3,5 +3,9 @@ def __init__(self, value): self.value = value def bump(self): - self.value += 1 + self.value = helper(self.value) return self.value + + +def helper(value): + return value + 1 diff --git a/tests/test_errors.py b/tests/test_errors.py index 08c9351..13a831b 100644 --- a/tests/test_errors.py +++ b/tests/test_errors.py @@ -2,8 +2,6 @@ import contextlib import io -import pytest - import phmutest.main import phmutest.summary @@ -87,17 +85,30 @@ def test_replmode_errors(): def badfixture(**kwargs): """phmutest fixture raises an exception.""" - raise (ValueError("badfixture- having a bad day")) + raise ValueError("badfixture- having a bad day") return None def test_repl_fixture_raises(capsys): - """In --replmode a fixture raises an exception. It just kills phmutest.""" + """In --replmode a fixture raises an exception.""" command = "tests/md/replerror.md --fixture tests.test_errors.badfixture --replmode" args = command.split() - with pytest.raises(ValueError) as exc_info: - _ = phmutest.main.main(args) - assert "badfixture- having a bad day" in str(exc_info.value) + phmresult = phmutest.main.main(args) + want = phmutest.summary.Metrics( + number_blocks_run=0, + passed=0, + failed=0, + skipped=0, + suite_errors=1, + number_of_files=0, + files_with_no_blocks=0, + number_of_deselected_blocks=0, + ) + assert want == phmresult.metrics + assert phmresult.is_success is False + output = capsys.readouterr().out.strip() + assert "Caught an exception in --fixture tests.test_errors.badfixture..." in output + assert "ValueError: badfixture- having a bad day" in output def test_summary_option(capsys, checker): @@ -134,14 +145,14 @@ def test_summary_option(capsys, checker): skipped blocks reason ------------------------- ------------- - tests/md/directive1.md:14 phmutest-skip + tests/md/directive1.md:16 phmutest-skip ------------------------- ------------- """ checker(expected, output) def test_setup_raises(capsys, endswith_checker): - """Check handling of an exception in a <~--phmutest-setup--> block. + """Check handling of an exception in a block. Cover 'setup and teardown errors' statements. The setup block generates into the unittest setUpClass() function. @@ -168,7 +179,7 @@ def test_setup_raises(capsys, endswith_checker): setup and teardown errors ------------------------- - tests/md/badsetup.md:6 setup + tests/md/badsetup.md:7 setup metric -------------------- - @@ -190,7 +201,7 @@ def test_setup_raises(capsys, endswith_checker): def test_setup_across_raises(capsys, endswith_checker): - """Check handling of an exception in a <~--phmutest-setup--> block. + """Check handling of an exception in a block. The setup block is rendered in setUpModule(). The error in setUpModule() cancels the rest of the unittest. @@ -222,7 +233,7 @@ def test_setup_across_raises(capsys, endswith_checker): setup and teardown errors ------------------------- - tests/md/badsetup.md:6 setup + tests/md/badsetup.md:7 setup metric -------------------- - @@ -244,7 +255,7 @@ def test_setup_across_raises(capsys, endswith_checker): def test_teardown_raises(capsys, endswith_checker): - """Check handling of an exception in a <~--phmutest-teardown--> block. + """Check handling of an exception in a block. Cover 'setup and teardown errors' statements. """ @@ -270,7 +281,7 @@ def test_teardown_raises(capsys, endswith_checker): setup and teardown errors ------------------------- - tests/md/badteardown.md:55 teardown + tests/md/badteardown.md:68 teardown metric -------------------- - @@ -292,7 +303,7 @@ def test_teardown_raises(capsys, endswith_checker): def test_teardown_across_raises(capsys, endswith_checker): - """Check handling of an exception in a <~--phmutest-teardown--> block. + """Check handling of an exception in a block. Cover 'setup and teardown errors' statements. """ @@ -321,7 +332,7 @@ def test_teardown_across_raises(capsys, endswith_checker): setup and teardown errors ------------------------- - tests/md/badteardown.md:55 teardown + tests/md/badteardown.md:68 teardown metric -------------------- - diff --git a/tests/test_session.py b/tests/test_session.py index bf80f4b..b17e694 100644 --- a/tests/test_session.py +++ b/tests/test_session.py @@ -1,4 +1,8 @@ """Test cases for session.py.""" +from unittest import mock + +import pytest + import phmutest.main import phmutest.summary from phmutest.fixture import Fixture @@ -61,7 +65,7 @@ def nonecleanupfixture(**kwargs): def test_none_cleanup(capsys): - """Use case where --replmode fixture returns Fixture with replmode=None.""" + """Use case where --replmode fixture returns Fixture with repl_cleanup=None.""" command = ( "tests/md/replerror.md " "--fixture tests.test_session.nonecleanupfixture --replmode --log" @@ -107,11 +111,11 @@ def test_progress(capsys, endswith_checker): tests/md/replerror.md:7. pass tests/md/replerror.md:11 pass tests/md/replerror.md:18 pass - tests/md/replerror.md:24 skip phmutest-skip - tests/md/replerror.md:32 error - tests/md/replerror.md:39 error - tests/md/replerror.md:47 skip requires >=py3.9999 - tests/md/replerror.md:55 pass + tests/md/replerror.md:26 skip phmutest-skip + tests/md/replerror.md:33 error + tests/md/replerror.md:40 error + tests/md/replerror.md:49 skip requires >=py3.9999 + tests/md/replerror.md:57 pass ------------------------ ------ ------------------- location|label result ---------------------- ------ @@ -120,3 +124,36 @@ def test_progress(capsys, endswith_checker): """ output = capsys.readouterr().out.rstrip() endswith_checker(expected, output) + + +def verbose_cleanup(): + """fixture cleanup function that prints.""" + print("Most definitely cleaning up here!") + + +def printing_cleanup_fixture(**kwargs): + """phmutest fixture function with a cleanup function that prints.""" + return Fixture(globs=None, repl_cleanup=verbose_cleanup) + + +def cause_exception(args, globs, fileblocks, extractor): + """Replacement function that raises an exception.""" + raise ValueError("Bad phmutest REPL logic.") + + +def test_repl_still_cleans_up(capsys): + """Show repl fixture cleanup is called for an internal error during session.py.""" + # Get coverage for code that handles an unexpected exception in phmutest code that + # runs in --replmode. The code assures that a fixture cleanup function + # gets called before propagating the exception. + # Use patching to inject the exception. + with mock.patch("phmutest.session.update_globs_show_sharing", cause_exception): + with pytest.raises(ValueError) as exc_info: + command = ( + "tests/md/example1.md tests/md/example2.md " + "--fixture tests.test_session.printing_cleanup_fixture --replmode --log" + ) + args = command.split() + _ = phmutest.main.main(args) + assert "Most definitely cleaning up here!" in capsys.readouterr().out + assert "Bad phmutest REPL logic." in str(exc_info.value) From 00297f54d378fa5d700625592a38db5026d14f77 Mon Sep 17 00:00:00 2001 From: Mark Taylor <24257134+tmarktaylor@users.noreply.github.com> Date: Wed, 13 Sep 2023 16:42:27 -0400 Subject: [PATCH 19/27] Add expected-output info string. --- src/phmutest/select.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/phmutest/select.py b/src/phmutest/select.py index 36cc147..3d4f6c1 100644 --- a/src/phmutest/select.py +++ b/src/phmutest/select.py @@ -9,6 +9,16 @@ from phmutest.direct import Marker from phmutest.fenced import FencedBlock, Role +# This is a designated patch point. Developers: Please treat this as if it were an API. +# Use this add FCB info strings that indicate an expected output FCB. +# To patch: +# - Create a copy of OUTPUT_INFO_STRINGS. +# - Add or remove info string values that indicate the FCB is expected output. +# - with mock.patch("phmutest.select.OUTPUT_INFO_STRINGS", ): +# - See example in tests/test_patching.py. +OUTPUT_INFO_STRINGS = ["", "expected-output"] +"""To be expected output the FCB will have one of these info strings.""" + def identify_output_blocks(blocks: List[FencedBlock]) -> None: """Guess which are blocks are output. @@ -22,7 +32,9 @@ def identify_output_blocks(blocks: List[FencedBlock]) -> None: previous_block = None for block in blocks: if previous_block is not None: - if not block.info_string and previous_block.role == Role.CODE: + if (block.info_string in OUTPUT_INFO_STRINGS) and ( + previous_block.role == Role.CODE + ): block.set(Role.OUTPUT) previous_block.set_link_to_output(block) previous_block = block From 2c96796899ddff83105c395db5678745e7495548 Mon Sep 17 00:00:00 2001 From: Mark Taylor <24257134+tmarktaylor@users.noreply.github.com> Date: Wed, 13 Sep 2023 16:46:22 -0400 Subject: [PATCH 20/27] Update README. --- README.md | 155 ++++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 127 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 18bc22a..3d87d86 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ - Equivalent Python library for calling from test suite. | [Here](#call-from-python) - Python tools to get fenced code block contents from Markdown. | [Here](docs/api.md) -Treats each Markdown file as a single long example which continues +Treats each Markdown file as a single long example, of many FCBs which continues across multiple Markdown [fenced code blocks][3] (FCBs or blocks). - Checks either Python code examples plus output **or** ">>>" REPL examples @@ -24,9 +24,12 @@ Pass objects as global variables to the examples. Cleans up even when fail-fast. | [Suite initialization and cleanup](#suite-initialization-and-cleanup) -### No Markdown edits required for above features +### Some Markdown edits required -Tests Python examples as is, the same way they were written. +No edits required for REPL examples. +Remove or use `expected-output` +as the info string on +expected output FCBs. ### Extendable @@ -48,8 +51,8 @@ by pressing the `Code` button in the banner at the top of the file. - Accepts [phmdoctest][17] directives except share-names and clear-names. - Specify blocks as setup and teardown code for the file or setup across files. - ## main branch status + [![](https://img.shields.io/pypi/l/phmutest.svg)](https://github.com/tmarktaylor/phmutest/blob/main/LICENSE) [![](https://img.shields.io/pypi/v/phmutest.svg)](https://pypi.python.org/pypi/phmutest) [![](https://img.shields.io/pypi/pyversions/phmutest.svg)](https://pypi.python.org/pypi/phmutest) @@ -79,24 +82,27 @@ by pressing the `Code` button in the banner at the top of the file. [Run as a Python module](#run-as-a-python-module) | [Call from Python](#call-from-python) | [Patch points](#patch-points) | -[Related projects](#related-projects) +[Hints](#hints) | +[Related projects](#related-projects) | +[Differences between phmutest and phmdoctest](#differences-between-phmutest-and-phmdoctest) [Sections](docs/demos.md#sections) | [Demos](docs/demos.md#demos) | [Changes](docs/recent_changes.md) | [Contributions](CONTRIBUTING.md) -## Core feature demo +## Markdown code/output demo -### markdown The are no phmutest directives in this file. The example starts by creating the object m. + ```python from hashlib import sha256 m = sha256() ``` The example continues here. + ```python m.update(b"hello World") print(m.hexdigest()[0:5]) @@ -104,24 +110,33 @@ print(m.hexdigest()[0:5]) Expected output here is checked. This fenced code block does not have an info string. + ``` db406 ``` The example continues here. It will continue for the entire file. This is -the last Python fenced code block (FCB) in the file. +the last Python fenced code/output code block (FCB) in the file. + ```python m.update(b"more bytes") print(m.hexdigest()[0:5]) ``` -Note the expected output is different. -``` +Note the expected output below is different. + +This FCB has the info string `expected-output` to avoid +Markdown linting tool nag. +phmutest treats a block with this info string as expected +output if it is the first FCB after a Python code FCB. + +```expected-output 4c6ea ``` ### phmutest command line -``` + +```shell phmutest README.md --log ``` @@ -131,17 +146,83 @@ Here is output from the command line. The output produced by Python standard library unittest module is not shown here. This is printed after the unittest `OK` line. +The lower case "o" after the line number indicates a +subsequent FCB containing expected printed output got checked. +```txt +log: +args.files: 'README.md' +args.log: 'True' + +location|label result +--------------- ------ +README.md:99... pass +README.md:106 o pass +README.md:121 o pass +--------------- ------ ``` + +## Markdown REPL demo + +- Run phmutest with --replmode to test Python interactive session FCBs. +- The REPL FCB's start with ">>>". +- No Markdown edits needed. Tests examples the way they were written. + +```python +>>> a = "Greetings Planet!" +>>> a +'Greetings Planet!' +>>> b = 12 +>>> b +12 +``` + +Example borrowed from Python Standard Library fractions documentation. + +```py +>>> from fractions import Fraction +>>> Fraction(16, -10) +Fraction(-8, 5) +>>> Fraction(123) +Fraction(123, 1) +>>> Fraction() +Fraction(0, 1) +>>> Fraction('3/7') +Fraction(3, 7) +``` + +Here we show names assigned in prior FCBs are still visible. + +```py +>>> b +12 +``` + +```py +>>> Fraction('3/7') +Fraction(3, 7) +``` + +### phmutest --replmode command line + +```shell +phmutest README.md --replmode --log +``` + +### phmutest --replmode output + +```txt log: args.files: 'README.md' +args.replmode: 'True' args.log: 'True' location|label result -------------- ------ -README.md:94.. pass -README.md:100. pass -README.md:113. pass +README.md:171. pass +README.md:182. pass +README.md:196. pass +README.md:201. pass -------------- ------ ``` @@ -150,7 +231,9 @@ See [How it works](docs/howitworks.md) ## Installation - python -m pip install phmutest +```shell +python -m pip install phmutest +``` - No dependencies since Python 3.11. Depends on tomli before Python 3.11. - Pure Python. No binaries. @@ -160,7 +243,7 @@ See [How it works](docs/howitworks.md) `phmutest --help` -``` +```txt usage: phmutest [-h] [--version] [--skip [TEXT ...]] [--fixture DOTTED_PATH.FUNCTION] [--share-across-files [FILE ...]] [--setup-across-files [FILE ...]] [--select [GROUP ...] | --deselect [GROUP ...]] @@ -259,7 +342,6 @@ apply doctest optionflags in --replmode. Do patching cleanup when not in --replmode by calling `unittest.addModuleCleanup(stack.pop_all().close)`. - ## Extend an example across files Names assigned by all the blocks in a file can be shared, as global variables, @@ -267,7 +349,6 @@ to files specified later in the command line. Add a markdown file path to the --share-across-files command line option. The 'shared' file(s) must also be specified as a FILE positional command line argument. - - [share demo](docs/share/share_demo.md) | [how it works](docs/codemode.md#share-across-files) - [--replmode share demo](docs/repl/replshare_demo.md) | @@ -280,9 +361,11 @@ prevents testing of any Python code or REPL block that contains the substring TE The block is logged as skip with `--skip TEXT` as the reason. ## summary option + The example [here](docs/share/share_demo.md) shows --summary output. ## TOML configuration + Command line options can be augmented with values from a `[tool.phmutest]` section in a .toml configuration file. It can be in a new file or added to an existing .toml file like pyproject.toml. @@ -290,11 +373,10 @@ The configuration file is specified by the `--config FILE` command line option. Zero or more of these TOML keys may be present in the `[tool.phmutest]` section. - | TOML key | Usage option | TOML value - double quoted strings | :------------------| :-----------------: | :---------: -| include-globs | FILE | list of filename glob to select files -| exclude-globs | FILE | list of filename glob to deselect files +| include-globs | positional arg FILE | list of filename glob to select files +| exclude-globs | positional arg FILE | list of filename glob to deselect files | share-across-files | --share-across-files | list of path | setup-across-files | --setup-across-files | list of path | fixture | --fixture | dotted path @@ -330,7 +412,6 @@ command line less the phmutest, like this: [Example](docs/callfrompython.md) | [Limitation](docs/callfrompython.md#limitation) - ## Patch points Feel free to **unittest.mock.patch()** at these places in the code and not worry about @@ -342,11 +423,32 @@ breakage in future versions. Look for examples in tests/test_patching.py. | :--------------------------------: | :----------: | phmutest.direct.directive_finders() | Add directive aliases | phmutest.fenced.python_matcher() | Add detect Python from FCB info string +| phmutest.select.OUTPUT_INFO_STRINGS | Change detect expected output FCB info string | phmutest.session.modify_docstring() | Inspect/modify REPL text before testing | phmutest.reader.post() | Inspect/modify DocNode detected in Markdown +## Hints + +- Since phmutest generates code, the input files should be from a trusted + source. +- The phmutest Markdown parser finds fenced code blocks enclosed by + html `
` and `
` tags. + The tags may require a preceding and trailing blank line + to render correctly. See example at the bottom tests/md/readerfcb.md. +- Markdown indented code blocks ([Spec][4] section 4.4) are ignored. +- A malformed HTML comment ending is bad. Make sure + it ends with both dashes like `-->`. +- A misspelled directive will be missing from the --report output. +- If the generated test file has a compile error phmutest will raise an + ImportError when importing it. +- Blocks skipped with --skip and the phmutest-skip directive + are not rendered. This is useful to avoid above import error. +- In repl mode **no** skipped blocks are rendered. +- Try redirecting `--generate -` standard output into PYPI Pygments to + colorize the generated test file. ## Related projects + - phmdoctest - rundoc - byexample @@ -357,11 +459,10 @@ breakage in future versions. Look for examples in tests/test_patching.py. - pytest-phmdoctest - pytest-codeblocks - -Major differences between phmutest and phmdoctest: +## Differences between phmutest and phmdoctest - phmutest treats each Markdown file as a single long example. phmdoctest - tests each FCB separately. Adding a share-names directive is necessary to + tests each FCB in isolation. Adding a share-names directive is necessary to extend an example across FCBs within a file. - Only phmutest can extend an example across files. - phmutest uses Python standard library unittest and doctest as test runners. @@ -378,9 +479,9 @@ Major differences between phmutest and phmdoctest: section for an explanation. - phmutest does not support inline annotations. - [1]: https://github.com/tmarktaylor/phmutest/blob/master/README.md?plain=1 [3]: https://github.github.com/gfm/#fenced-code-blocks +[4]: https://spec.commonmark.org [11]: https://github.github.com/gfm/#info-string [10]: https://phmutest.readthedocs.io/en/latest/docs/api.html [4]: https://docs.python.org/3/library/doctest.html @@ -389,5 +490,3 @@ Major differences between phmutest and phmdoctest: [15]: https://docs.pytest.org/en/stable [16]: https://tmarktaylor.github.io/pytest-phmdoctest [17]: https://pypi.python.org/pypi/phmdoctest - - From fa7ecd316501b35c904a425e6a4ec60765691aea Mon Sep 17 00:00:00 2001 From: Mark Taylor <24257134+tmarktaylor@users.noreply.github.com> Date: Wed, 13 Sep 2023 16:47:12 -0400 Subject: [PATCH 21/27] Update .gitignore. --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index efe742a..6c521b7 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,9 @@ # PyCharm .idea/ +# IDE +.vscode/ + # Mkdocs site _mkdocsin From 0b8ba7d13bb3d77b51d211c9d4b6caad206d0cbb Mon Sep 17 00:00:00 2001 From: Mark Taylor <24257134+tmarktaylor@users.noreply.github.com> Date: Wed, 13 Sep 2023 16:48:42 -0400 Subject: [PATCH 22/27] Update test_examples.py - add a test. --- tests/test_examples.py | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/tests/test_examples.py b/tests/test_examples.py index 0728660..5f14e1c 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -114,7 +114,7 @@ def test_directive1(): def test_directive1_replmode(): - """Test label, skip, and skipif directives on code blocks.""" + """Test label, skip, and skipif directives on session blocks.""" line = "tests/md/directive1.md --replmode" phmresult = phmutest.main.command(line) want = phmutest.summary.Metrics( @@ -130,7 +130,7 @@ def test_directive1_replmode(): assert want == phmresult.metrics assert phmresult.is_success is True assert "phmutest-skip" in phmresult.log[0][2] - assert "tests/md/directive1.md:69" in phmresult.log[1][0] + assert "tests/md/directive1.md:78" in phmresult.log[1][0] assert "doctest_print_coffee" in phmresult.log[1][0] @@ -282,6 +282,24 @@ def test_excess_printing(): assert phmresult.is_success is False +def test_extra_expected_output(): + """Expected output has an additional line.""" + line = "tests/md/extra_line_in_output.md" + phmresult = phmutest.main.command(line) + want = phmutest.summary.Metrics( + number_blocks_run=1, + passed=0, + failed=1, + skipped=0, + suite_errors=0, + number_of_files=1, + files_with_no_blocks=0, + number_of_deselected_blocks=0, + ) + assert want == phmresult.metrics + assert phmresult.is_success is False + + def test_no_fcbs(): """Test file with no FCBs.""" line = "tests/md/no_fenced_code_blocks.md" From 6f40492e73b797d958db9d58667761feb5b39377 Mon Sep 17 00:00:00 2001 From: Mark Taylor <24257134+tmarktaylor@users.noreply.github.com> Date: Wed, 13 Sep 2023 16:48:58 -0400 Subject: [PATCH 23/27] Update recent_changes.md. --- docs/recent_changes.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/recent_changes.md b/docs/recent_changes.md index ae544b0..5adc0b9 100644 --- a/docs/recent_changes.md +++ b/docs/recent_changes.md @@ -1,11 +1,17 @@ # Recent changes + 0.0.1 - 2023-08-14 - Initial upload to Python Package Index. -0.0.2 - 2023-xx-xx +0.0.2 - 2023-09-13 - Add main.command(). - Remove import io, sys, contextlib from generated testfile. +- Catch exception raised by --replmode --fixture. +- Add FCB info string "expected-output" to indicate expected output. +- Add patch point to change info strings that indicate expected output. +- Indicate expected output was checked with "o" in --log location. +- Markdown linting. +- Moved options.entry_points to setup.cfg and removed setup.py. - Setup classifiers, cleanups, Docs, renames. - From 14d2b2775900c655c61a805d82f67727742938d7 Mon Sep 17 00:00:00 2001 From: Mark Taylor <24257134+tmarktaylor@users.noreply.github.com> Date: Thu, 14 Sep 2023 20:54:05 -0400 Subject: [PATCH 24/27] Move twine check dist/* to build.yml. --- .github/workflows/build.yml | 2 ++ tests/requirements_inspect.txt | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 21d830d..c244913 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -18,8 +18,10 @@ jobs: - name: Build dist run: | python -m pip install build --user + python -m pip install twine --user python -m build python -m pip hash dist/* + twine check dist/* - name: Upload dist uses: actions/upload-artifact@v3 diff --git a/tests/requirements_inspect.txt b/tests/requirements_inspect.txt index 74a3051..91e6602 100644 --- a/tests/requirements_inspect.txt +++ b/tests/requirements_inspect.txt @@ -3,6 +3,5 @@ pep8-naming mkdocs mypy typing -twine black isort From cb09481962196b7d0258d1a98ece904a41ca3467 Mon Sep 17 00:00:00 2001 From: Mark Taylor <24257134+tmarktaylor@users.noreply.github.com> Date: Thu, 14 Sep 2023 21:14:10 -0400 Subject: [PATCH 25/27] Replace fixture function import logic. Use the recipe from importlib describing how to import .py source file directly. Add a new function fixture_function_importer which returns the fixture function callable. --- src/phmutest/cases.py | 25 ++++++++++++------------- src/phmutest/importer.py | 38 ++++++++++++++++++++++++++++++++++++++ src/phmutest/session.py | 14 +++++++------- 3 files changed, 57 insertions(+), 20 deletions(-) create mode 100644 src/phmutest/importer.py diff --git a/src/phmutest/cases.py b/src/phmutest/cases.py index 71ba3c7..3faa203 100644 --- a/src/phmutest/cases.py +++ b/src/phmutest/cases.py @@ -1,7 +1,6 @@ """Generate test cases as a unittest test file.""" import argparse from pathlib import Path -from typing import Tuple import phmutest.select import phmutest.subtest @@ -317,6 +316,7 @@ def markdown_file( from phmutest.skip import sys_tool as _phm_sys $importuserfixture +$calluserfixture _phm_globals = None _phm_testcase = unittest.TestCase() _phm_log = [] @@ -326,23 +326,22 @@ def markdown_file( ''' -def get_fixture_parts(fixture_arg: Path) -> Tuple[str, str]: - """Return X, Y for the fixture function where X and Y are 'from X import Y'.""" - function = fixture_arg.suffix # pathlib Rocks! - assert function.startswith("."), "Expecting .FUNCTION." - function = function[1:] - return fixture_arg.stem, function - - def testfile(args: argparse.Namespace, block_store: phmutest.select.BlockStore) -> str: """Generate the unittest module source as directed by command line args args.""" test_classes = "" replacements = {} if args.fixture: - modulepackage, function_name = get_fixture_parts(args.fixture) - replacements[ - "importuserfixture" - ] = f"from {modulepackage} import {function_name} as _phm_user_setup_function" + # modulepackage, function_name = get_fixture_parts(args.fixture) + import_line = ( + "from phmutest.importer import fixture_function_importer " + "as _phm_fixture_function_importer" + ) + replacements["importuserfixture"] = import_line + call_line = ( + f"_phm_user_setup_function = " + f'_phm_fixture_function_importer("{args.fixture}")' + ) + replacements["calluserfixture"] = call_line if args.setup_across_files or args.share_across_files or args.fixture: setupcode = render_setup_module(args, block_store) diff --git a/src/phmutest/importer.py b/src/phmutest/importer.py new file mode 100644 index 0000000..3c2c21c --- /dev/null +++ b/src/phmutest/importer.py @@ -0,0 +1,38 @@ +"""Import user's fixture function given the relative dotted path.""" +import importlib.util +import sys +from pathlib import Path + +from phmutest.fixture import FixtureFunction + + +def python_file_importer(file_path, module_name): # type: ignore + """Import .py source file directly. See importlib Examples.""" + spec = importlib.util.spec_from_file_location(module_name, file_path) + module = importlib.util.module_from_spec(spec) # type: ignore + sys.modules[module_name] = module + spec.loader.exec_module(module) # type: ignore + return module + + +def fixture_function_importer(dotted_path_string: str) -> FixtureFunction: + """Return imported user's fixture function given its relative dotted path. + + The dotted_path has components separated by ".". + The last component is the function name. + The next to last component is the python file name without the .py suffix. + The preceding components identify parent folders. Folders should be + relative to the current working directory which is typically the + project root. + """ + dotted_path = Path(dotted_path_string) + function = dotted_path.suffix # pathlib Rocks! + assert function.startswith("."), "Expecting .FUNCTION" + function_name = function[1:] + dotted_file_name = dotted_path.stem + file_name = dotted_file_name.replace(".", "/") + file_path = Path(file_name).with_suffix(".py") + module_name = dotted_file_name.replace(".", "_") + module = python_file_importer(file_path, module_name) # type: ignore + f = getattr(module, function_name) + return f # type: ignore diff --git a/src/phmutest/session.py b/src/phmutest/session.py index d309061..cd03ef0 100644 --- a/src/phmutest/session.py +++ b/src/phmutest/session.py @@ -1,16 +1,17 @@ """Generate and run docstests for Python interactive session FCBs.""" import argparse import doctest -import importlib import itertools import sys import traceback +import typing from dataclasses import dataclass from pathlib import Path from typing import Any, Callable, Dict, List, Optional, Tuple import phmutest.cases import phmutest.globs +import phmutest.importer import phmutest.select import phmutest.subtest import phmutest.summary @@ -191,18 +192,16 @@ def null_cleanup() -> None: pass -UserFixtureInfo = Tuple[Dict[str, Any], Callable[[], None], bool] +UserFixtureInfo = Tuple[Optional[Dict[str, Any]], Callable[[], None], bool] """Function return type [globs, cleanup function, success].""" def process_user_fixture( args: argparse.Namespace, log: List[List[str]] ) -> UserFixtureInfo: - globs = {} + globs: Optional[Dict[str, Any]] = {} cleanup_function = null_cleanup - modulepackage, function_name = phmutest.cases.get_fixture_parts(args.fixture) - m = importlib.import_module(modulepackage) - f = getattr(m, function_name) + f = phmutest.importer.fixture_function_importer(args.fixture.name) try: user_fixture = f(log=log, is_replmode=True) except Exception: @@ -216,7 +215,8 @@ def process_user_fixture( if user_fixture: if user_fixture.globs is not None: - globs = user_fixture.globs + # Make the fixture's MutableMapping type look like a Dict for doctest. + globs = typing.cast(Optional[Dict[str, Any]], user_fixture.globs) if user_fixture.repl_cleanup is not None: cleanup_function = user_fixture.repl_cleanup From 62b6362b616f80fc0b45455851b33d1c97163ae8 Mon Sep 17 00:00:00 2001 From: Mark Taylor <24257134+tmarktaylor@users.noreply.github.com> Date: Thu, 14 Sep 2023 21:20:15 -0400 Subject: [PATCH 26/27] Update ci.yml, recent_changes.md. - serialize jobs, do coverage job last - move twine check dist to build.yml. - add a 'code mode' command with --fixture to Usage tests. --- .github/workflows/ci.yml | 84 +++++++++++++++++++++------------------- docs/recent_changes.md | 7 +++- 2 files changed, 49 insertions(+), 42 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f210d80..873e065 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -50,7 +50,9 @@ jobs: run: | pytest -vv tests + versions: + needs: os runs-on: ubuntu-latest strategy: matrix: @@ -79,48 +81,12 @@ jobs: phmutest docs/advanced/label.md --log phmutest docs/advanced/labelanyfcb.md --log phmutest tests/md/optionflags.md --log --replmode --fixture tests.test_patching.setflags + phmutest docs/fix/code/globdemo.md --fixture docs.fix.code.globdemo.init_globals --log phmutest tests/md/project.md --report - coverage: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Set up Python 3.x - id: setuppython - uses: actions/setup-python@v4 - with: - python-version: 3.x - - name: Install phmutest - run: | - python -m pip install --upgrade pip - pip install coverage - pip install -r tests/requirements.txt - pip freeze - - name: Tests, coverage report - run: | - coverage erase - coverage run --branch --source=src -m phmutest --version - coverage run --branch --source=src --append -m phmutest README.md --report - coverage run --branch --source=src --append -m phmutest README.md - coverage run --branch --source=src --append -m pytest -vv tests - coverage report --show-missing - coverage xml - env: - PYTHONPATH: ${{ github.workspace }}/src - continue-on-error: true - - - name: Upload coverage reports to Codecov - uses: codecov/codecov-action@v3 - with: - token: ${{ secrets.CODECOV_TOKEN }} - files: ./coverage.xml # optional - flags: pytest,python-${{ steps.setuppython.outputs.python-version }},ubuntu-latest # optional - name: codecov-umbrella # optional - fail_ci_if_error: false # optional (default = false) - verbose: true # optional (default = false) - inspect: + needs: versions runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 @@ -159,8 +125,6 @@ jobs: - name: Deployable run: | python tests/check_classifiers.py - python setup.py sdist - twine check dist/* - name: Docs run: | python docs/premkdocs.py @@ -171,3 +135,43 @@ jobs: name: site path: site retention-days: 5 + + + coverage: + needs: inspect + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Set up Python 3.x + id: setuppython + uses: actions/setup-python@v4 + with: + python-version: 3.x + - name: Install phmutest + run: | + python -m pip install --upgrade pip + pip install coverage + pip install -r tests/requirements.txt + pip freeze + - name: Tests, coverage report + run: | + coverage erase + coverage run --branch --source=src -m phmutest --version + coverage run --branch --source=src --append -m phmutest README.md --report + coverage run --branch --source=src --append -m phmutest README.md + coverage run --branch --source=src --append -m pytest -vv tests + coverage report --show-missing + coverage xml + env: + PYTHONPATH: ${{ github.workspace }}/src + continue-on-error: true + + - name: Upload coverage reports to Codecov + uses: codecov/codecov-action@v3 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: ./coverage.xml # optional + flags: pytest,python-${{ steps.setuppython.outputs.python-version }},ubuntu-latest # optional + name: codecov-umbrella # optional + fail_ci_if_error: false # optional (default = false) + verbose: true # optional (default = false) diff --git a/docs/recent_changes.md b/docs/recent_changes.md index 5adc0b9..55ba187 100644 --- a/docs/recent_changes.md +++ b/docs/recent_changes.md @@ -4,14 +4,17 @@ - Initial upload to Python Package Index. -0.0.2 - 2023-09-13 +0.0.2 - 2023-09-15 +- Bugfix- Use new recipe in importer.py to import the fixture function. - Add main.command(). - Remove import io, sys, contextlib from generated testfile. - Catch exception raised by --replmode --fixture. - Add FCB info string "expected-output" to indicate expected output. - Add patch point to change info strings that indicate expected output. - Indicate expected output was checked with "o" in --log location. -- Markdown linting. - Moved options.entry_points to setup.cfg and removed setup.py. +- Markdown linting. - Setup classifiers, cleanups, Docs, renames. +- Move twine check to build.yml. +- serialize ci.yml jobs From 472ff3d7babea1a31d3daba48393389796482c98 Mon Sep 17 00:00:00 2001 From: Mark Taylor <24257134+tmarktaylor@users.noreply.github.com> Date: Fri, 15 Sep 2023 07:27:57 -0400 Subject: [PATCH 27/27] README.md update, cases.py reformat. Add dotted path details and run with pytest to README.md Removed commented out code from cases.py. --- README.md | 13 +++++++++++++ src/phmutest/cases.py | 7 ++----- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 3d87d86..c8ae326 100644 --- a/README.md +++ b/README.md @@ -342,6 +342,17 @@ apply doctest optionflags in --replmode. Do patching cleanup when not in --replmode by calling `unittest.addModuleCleanup(stack.pop_all().close)`. +### Dotted path details + +The fixture function must be at the top level of a .py file. + +- The dotted_path has components separated by ".". +- The last component is the function name. +- The next to last component is the python file name without the .py suffix. +- The preceding components identify parent folders. Folders should be + relative to the current working directory which is typically the + project root. + ## Extend an example across files Names assigned by all the blocks in a file can be shared, as global variables, @@ -446,6 +457,8 @@ breakage in future versions. Look for examples in tests/test_patching.py. - In repl mode **no** skipped blocks are rendered. - Try redirecting `--generate -` standard output into PYPI Pygments to colorize the generated test file. +- pytest will run a generated test file (--generate TESTFILE). pytest won't + run functions added by unittest.addModuleCleanup(). ## Related projects diff --git a/src/phmutest/cases.py b/src/phmutest/cases.py index 3faa203..36c573d 100644 --- a/src/phmutest/cases.py +++ b/src/phmutest/cases.py @@ -331,17 +331,14 @@ def testfile(args: argparse.Namespace, block_store: phmutest.select.BlockStore) test_classes = "" replacements = {} if args.fixture: - # modulepackage, function_name = get_fixture_parts(args.fixture) - import_line = ( + replacements["importuserfixture"] = ( "from phmutest.importer import fixture_function_importer " "as _phm_fixture_function_importer" ) - replacements["importuserfixture"] = import_line - call_line = ( + replacements["calluserfixture"] = ( f"_phm_user_setup_function = " f'_phm_fixture_function_importer("{args.fixture}")' ) - replacements["calluserfixture"] = call_line if args.setup_across_files or args.share_across_files or args.fixture: setupcode = render_setup_module(args, block_store)