diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index c2b82d33..68cc69bb 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -9,7 +9,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python 3.8 uses: actions/setup-python@v4 diff --git a/.github/workflows/fuzz.yml b/.github/workflows/fuzz.yml index 7df6dc6b..a74869a5 100644 --- a/.github/workflows/fuzz.yml +++ b/.github/workflows/fuzz.yml @@ -6,9 +6,11 @@ name: fuzzing # to guard against code introducing crashes in the Markdown parsing, # which should in principle always run against any text input. # See: https://google.github.io/oss-fuzz/getting-started/continuous-integration/#how-it-works +# Note, to reproduce a crash locally, copy to `testcase` file` and run: `tox -e fuzz` on: pull_request: + paths-ignore: ['docs/**', 'tests/**'] jobs: Fuzzing: diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index a82b59bc..8f1cd70b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -19,7 +19,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python 3.8 uses: actions/setup-python@v4 with: @@ -34,7 +34,7 @@ jobs: python-version: ['pypy-3.7', '3.7', '3.8', '3.9', '3.10', '3.11'] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 with: @@ -63,7 +63,7 @@ jobs: matrix: python-version: ['3.8'] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 with: @@ -83,7 +83,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python 3.8 uses: actions/setup-python@v4 @@ -113,7 +113,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout source - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Set up Python 3.8 uses: actions/setup-python@v4 with: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 50e8132a..139ce525 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -29,7 +29,7 @@ repos: - id: isort - repo: https://github.com/psf/black - rev: 23.1.0 + rev: 23.3.0 hooks: - id: black @@ -40,7 +40,7 @@ repos: additional_dependencies: [flake8-bugbear~=22.7] - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.0.1 + rev: v1.2.0 hooks: - id: mypy additional_dependencies: [mdurl] diff --git a/docs/conf.py b/docs/conf.py index 786eff04..52deea47 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -102,7 +102,7 @@ def run_apidoc(app): this_folder = os.path.abspath(os.path.dirname(os.path.realpath(__file__))) api_folder = os.path.join(this_folder, "api") module_path = os.path.normpath(os.path.join(this_folder, "../")) - ignore_paths = ["../profiler.py", "../conftest.py", "../tests", "../benchmarking"] + ignore_paths = ["../scripts", "../conftest.py", "../tests", "../benchmarking"] ignore_paths = [ os.path.normpath(os.path.join(this_folder, p)) for p in ignore_paths ] diff --git a/scripts/build_fuzzers.py b/scripts/build_fuzzers.py new file mode 100644 index 00000000..3dce8ddf --- /dev/null +++ b/scripts/build_fuzzers.py @@ -0,0 +1,42 @@ +"""Build fuzzers idempotently in a given folder.""" +import argparse +from pathlib import Path +import subprocess + + +def main(): + """Build fuzzers idempotently in a given folder.""" + parser = argparse.ArgumentParser() + parser.add_argument("folder") + args = parser.parse_args() + folder = Path(args.folder) + if not folder.exists(): + print(f"Cloning google/oss-fuzz into: {folder}") + folder.mkdir(parents=True) + subprocess.check_call( + [ + "git", + "clone", + "--single-branch", + "https://github.com/google/oss-fuzz", + str(folder), + ] + ) + else: + print(f"Using google/oss-fuzz in: {folder}") + if not (folder / "build").exists(): + print(f"Building fuzzers in: {folder / 'build'}") + subprocess.check_call( + [ + "python", + str(folder / "infra" / "helper.py"), + "build_fuzzers", + "markdown-it-py", + ] + ) + else: + print(f"Using existing fuzzers in: {folder / 'build'}") + + +if __name__ == "__main__": + main() diff --git a/profiler.py b/scripts/profiler.py similarity index 100% rename from profiler.py rename to scripts/profiler.py diff --git a/tox.ini b/tox.ini index 85a7179b..bf0c8367 100644 --- a/tox.ini +++ b/tox.ini @@ -55,11 +55,18 @@ allowlist_externals = dot commands = mkdir -p "{toxworkdir}/prof" - python -m cProfile -o "{toxworkdir}/prof/output.pstats" profiler.py + python -m cProfile -o "{toxworkdir}/prof/output.pstats" scripts/profiler.py gprof2dot -f pstats -o "{toxworkdir}/prof/output.dot" "{toxworkdir}/prof/output.pstats" dot -Tsvg -o "{toxworkdir}/prof/output.svg" "{toxworkdir}/prof/output.dot" python -c 'import pathlib; print("profiler svg output under file://\{0\}".format(pathlib.Path(r"{toxworkdir}") / "prof" / "output.svg"))' +[testenv:fuzz] +description = run fuzzer on testcase file +; See: https://google.github.io/oss-fuzz/ +deps = atheris +commands_pre = python scripts/build_fuzzers.py {envdir}/oss-fuzz +commands = python {envdir}/oss-fuzz/infra/helper.py reproduce markdown-it-py fuzz_markdown {posargs:testcase} + [flake8] max-line-length = 100 extend-ignore = E203