Merge pull request #793 from Textualize/snapshot

Snapshot testing

.github/workflows/pythonpackage.yml

@@ -43,3 +43,9 @@ jobs:
         run: |
           source $VENV
           python e2e_tests/sandbox_basic_test.py basic 2.0
+      - name: Upload snapshot report
+        if: always()
+        uses: actions/upload-artifact@v3
+        with:
+          name: snapshot-report-textual
+          path: tests/snapshot_tests/output/snapshot_report.html

.pre-commit-config.yaml

@@ -13,3 +13,4 @@ repos:
     hooks:
     -   id: black
         exclude: ^tests/
+exclude: ^tests/snapshot_tests

Makefile

@@ -2,6 +2,8 @@ test:
 	pytest --cov-report term-missing --cov=textual tests/ -vv
 unit-test:
 	pytest --cov-report term-missing --cov=textual tests/ -vv -m "not integration_test"
+test-snapshot-update:
+	pytest --cov-report term-missing --cov=textual tests/ -vv --snapshot-update
 typecheck:
 	mypy src/textual
 format:
@@ -14,4 +16,3 @@ docs-build:
 	mkdocs build
 docs-deploy:
 	mkdocs gh-deploy
-

notes/snapshot_testing.md

@@ -0,0 +1,43 @@
+# Snapshot Testing
+
+
+## What is snapshot testing?
+
+Some tests that run for Textual are snapshot tests.
+When you first run a snapshot test, a screenshot of an app is taken and saved to disk.
+Next time you run it, another screenshot is taken and compared with the original one.
+
+If the screenshots don't match, it means something has changed.
+It's up to you to tell the test system whether that change is expected or not.
+
+This allows us to easily catch regressions in how Textual outputs to the terminal.
+
+Snapshot tests run alongside normal unit tests.
+
+## How do I write a snapshot test?
+
+1. Inject the `snap_compare` fixture into your test.
+2. Pass in the path to the file which contains the Textual app.
+
+```python
+def test_grid_layout_basic_overflow(snap_compare):
+    assert snap_compare("docs/examples/guide/layout/grid_layout2.py")
+```
+
+`snap_compare` can take additional arguments such as `press`, which allows
+you to simulate key presses etc.
+See the signature of `snap_compare` for more info.
+
+## A snapshot test failed, what do I do?
+
+When a snapshot test fails, a report will be created on your machine, and you
+can use this report to visually compare the output from your test with the historical output for that test.
+
+This report will be visible at the bottom of the terminal after the `pytest` session completes,
+or, if running in CI, it will be available as an artifact attached to the GitHub Actions summary.
+
+If you're happy that the new output of the app is correct, you can run `pytest` with the
+`--snapshot-update` flag. This flag will update the snapshots for any test that is executed in the run,
+so to update a snapshot for a single test, run only that test.
+
+With your snapshot on disk updated to match the new output, running the test again should result in a pass.

pyproject.toml

@@ -37,7 +37,7 @@ nanoid = "^2.0.0"
 dev = ["aiohttp", "click", "msgpack"]
 
 [tool.poetry.dev-dependencies]
-pytest = "^6.2.3"
+pytest = "^7.1.3"
 black = "^22.3.0"
 mypy = "^0.950"
 pytest-cov = "^2.12.1"
@@ -48,13 +48,15 @@ pre-commit = "^2.13.0"
 pytest-aiohttp = "^1.0.4"
 time-machine = "^2.6.0"
 Jinja2 = "<3.1.0"
+syrupy = "^3.0.0"
 
 [tool.black]
 includes = "src"
 
 [tool.pytest.ini_options]
 asyncio_mode = "auto"
 testpaths = ["tests"]
+addopts = "--strict-markers"
 markers = [
     "integration_test: marks tests as slow integration tests (deselect with '-m \"not integration_test\"')",
 ]

src/textual/_doc.py

@@ -1,19 +1,17 @@
 from __future__ import annotations
 
 import os
-import runpy
 import shlex
-from typing import TYPE_CHECKING, cast
+from typing import Iterable
 
-from textual._import_app import AppFail, import_app
+from textual.app import App
+from textual._import_app import import_app
 
-if TYPE_CHECKING:
-    from textual.app import App
 
 # This module defines our "Custom Fences", powered by SuperFences
 # @link https://facelessuser.github.io/pymdown-extensions/extensions/superfences/#custom-fences
 def format_svg(source, language, css_class, options, md, attrs, **kwargs) -> str:
-    """A superfences formatter to insert a SVG screenshot."""
+    """A superfences formatter to insert an SVG screenshot."""
 
     try:
         cmd: list[str] = shlex.split(attrs["path"])
@@ -23,22 +21,15 @@ def format_svg(source, language, css_class, options, md, attrs, **kwargs) -> str
         press = [*_press.split(",")] if _press else ["_"]
         title = attrs.get("title")
 
-        os.environ["COLUMNS"] = attrs.get("columns", "80")
-        os.environ["LINES"] = attrs.get("lines", "24")
-
         print(f"screenshotting {path!r}")
 
         cwd = os.getcwd()
         try:
-            app = import_app(path)
-            app.run(
-                quit_after=5,
-                press=press or ["ctrl+c"],
-                headless=True,
-                screenshot=True,
-                screenshot_title=title,
+            rows = int(attrs.get("lines", 24))
+            columns = int(attrs.get("columns", 80))
+            svg = take_svg_screenshot(
+                None, path, press, title, terminal_size=(rows, columns)
             )
-            svg = app._screenshot
         finally:
             os.chdir(cwd)
 
@@ -51,8 +42,51 @@ def format_svg(source, language, css_class, options, md, attrs, **kwargs) -> str
         traceback.print_exception(error)
 
 
+def take_svg_screenshot(
+    app: App | None = None,
+    app_path: str | None = None,
+    press: Iterable[str] = ("_",),
+    title: str | None = None,
+    terminal_size: tuple[int, int] = (24, 80),
+) -> str:
+    """
+
+    Args:
+        app: An app instance. Must be supplied if app_path is not.
+        app_path: A path to an app. Must be supplied if app is not.
+        press: Key presses to run before taking screenshot. "_" is a short pause.
+        title: The terminal title in the output image.
+        terminal_size: A pair of integers (rows, columns), representing terminal size.
+
+    Returns:
+        str: An SVG string, showing the content of the terminal window at the time
+            the screenshot was taken.
+
+    """
+    rows, columns = terminal_size
+
+    os.environ["COLUMNS"] = str(columns)
+    os.environ["LINES"] = str(rows)
+
+    if app is None:
+        app = import_app(app_path)
+
+    if title is None:
+        title = app.title
+
+    app.run(
+        quit_after=5,
+        press=press or ["ctrl+c"],
+        headless=True,
+        screenshot=True,
+        screenshot_title=title,
+    )
+    svg = app._screenshot
+    return svg
+
+
 def rich(source, language, css_class, options, md, attrs, **kwargs) -> str:
-    """A superfences formatter to insert a SVG screenshot."""
+    """A superfences formatter to insert an SVG screenshot."""
 
     import io
 

src/textual/app.py

@@ -176,6 +176,7 @@ def __init__(
             markup=False,
             highlight=False,
             emoji=False,
+            legacy_windows=False,
         )
         self.error_console = Console(markup=False, stderr=True)
         self.driver_class = driver_class or self.get_driver_class()
@@ -557,6 +558,7 @@ def export_screenshot(self, *, title: str | None = None) -> str:
             force_terminal=True,
             color_system="truecolor",
             record=True,
+            legacy_windows=False,
         )
         screen_render = self.screen._compositor.render(full=True)
         console.print(screen_render)