From 68194a913fa0d9f601a55fcd08ff13b7ac35be75 Mon Sep 17 00:00:00 2001 From: David Lakin Date: Tue, 16 Apr 2024 14:41:36 -0400 Subject: [PATCH 1/9] Remove shebangs from fuzz harnesses Prefer executing these files using the OSS-Fuzz or `python` command methods outlined in the `fuzzing/README`. Based on feedback and discussion on: https://github.com/gitpython-developers/GitPython/pull/1901 --- fuzzing/fuzz-targets/fuzz_config.py | 1 - fuzzing/fuzz-targets/fuzz_tree.py | 1 - 2 files changed, 2 deletions(-) diff --git a/fuzzing/fuzz-targets/fuzz_config.py b/fuzzing/fuzz-targets/fuzz_config.py index fc2f0960a..0a06956c8 100644 --- a/fuzzing/fuzz-targets/fuzz_config.py +++ b/fuzzing/fuzz-targets/fuzz_config.py @@ -1,4 +1,3 @@ -#!/usr/bin/python3 # Copyright 2023 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/fuzzing/fuzz-targets/fuzz_tree.py b/fuzzing/fuzz-targets/fuzz_tree.py index b4e0e6b55..464235098 100644 --- a/fuzzing/fuzz-targets/fuzz_tree.py +++ b/fuzzing/fuzz-targets/fuzz_tree.py @@ -1,4 +1,3 @@ -#!/usr/bin/python3 # Copyright 2023 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); From 8954c7151e098a0b12d4d2dec277fe6c63980579 Mon Sep 17 00:00:00 2001 From: David Lakin Date: Tue, 16 Apr 2024 14:45:49 -0400 Subject: [PATCH 2/9] Replace shebang in `build.sh` with ShellCheck directive This script is meant to be sourced by the OSS-Fuzz file of the same name, rather than executed directly. The shebang may lead to the incorrect assumption that the script is meant for direct execution. Replacing it with this directive instructs ShellCheck to treat the script as a Bash script, regardless of how it is executed. Based @EliahKagan's suggestion and feedback on: https://github.com/gitpython-developers/GitPython/pull/1901 --- fuzzing/oss-fuzz-scripts/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fuzzing/oss-fuzz-scripts/build.sh b/fuzzing/oss-fuzz-scripts/build.sh index aff1c4347..4c7de8799 100644 --- a/fuzzing/oss-fuzz-scripts/build.sh +++ b/fuzzing/oss-fuzz-scripts/build.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +# shellcheck shell=bash set -euo pipefail From b0a5b8e66c4da3d603d8e27a71c70aaad53542b8 Mon Sep 17 00:00:00 2001 From: David Lakin Date: Tue, 16 Apr 2024 14:59:05 -0400 Subject: [PATCH 3/9] Set executable bit on `container-environment-bootstrap.sh` This script is executed directly, not sourced as is the case with `build.sh`, so it should have an executable bit set to avoid ambiguity. Based @EliahKagan's suggestion and feedback on: https://github.com/gitpython-developers/GitPython/pull/1901 --- fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh diff --git a/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh b/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh old mode 100644 new mode 100755 From 25f360090cb6a7fd0f01bc127f2a2280659757a2 Mon Sep 17 00:00:00 2001 From: David Lakin Date: Tue, 16 Apr 2024 15:31:05 -0400 Subject: [PATCH 4/9] Minor clarity improvements in `fuzzing/README.md` - Make the link text for the OSS-Fuzz test status URL more descriptive - Fix capitalization of GitPython repository name Based @EliahKagan's suggestion and feedback on: https://github.com/gitpython-developers/GitPython/pull/1901 --- fuzzing/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fuzzing/README.md b/fuzzing/README.md index 09d6fc003..65e311d4a 100644 --- a/fuzzing/README.md +++ b/fuzzing/README.md @@ -6,8 +6,8 @@ This directory contains files related to GitPython's suite of fuzz tests that ar infrastructure provided by [OSS-Fuzz][oss-fuzz-repo]. This document aims to provide necessary information for working with fuzzing in GitPython. -The latest details regarding OSS-Fuzz test status, including build logs and coverage reports, is made available -at [this link](https://introspector.oss-fuzz.com/project-profile?project=gitpython). +The latest details regarding OSS-Fuzz test status, including build logs and coverage reports, is available +on [the Open Source Fuzzing Introspection website](https://introspector.oss-fuzz.com/project-profile?project=gitpython). ## How to Contribute @@ -152,7 +152,7 @@ python infra/helper.py build_fuzzers --sanitizer $SANITIZER gitpython ``` > [!TIP] -> The `build_fuzzers` command above accepts a local file path pointing to your gitpython repository clone as the last +> The `build_fuzzers` command above accepts a local file path pointing to your GitPython repository clone as the last > argument. > This makes it easy to build fuzz targets you are developing locally in this repository without changing anything in > the OSS-Fuzz repo! From d79c176384f1a5b6cb615f500037dbcecd9ee7d9 Mon Sep 17 00:00:00 2001 From: David Lakin Date: Tue, 16 Apr 2024 16:10:08 -0400 Subject: [PATCH 5/9] Simplify read delimiter to use empty string in fuzz harness loop Replaces the null character delimiter `-d $'\0'` with the simpler empty string `-d ''` in the fuzzing harness build loop. This changes leverages the Bash `read` builtin behavior to avoid unnecessary complexity and improving script readability. Based @EliahKagan's suggestion and feedback on: https://github.com/gitpython-developers/GitPython/pull/1901 --- fuzzing/oss-fuzz-scripts/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fuzzing/oss-fuzz-scripts/build.sh b/fuzzing/oss-fuzz-scripts/build.sh index 4c7de8799..a412a1d15 100644 --- a/fuzzing/oss-fuzz-scripts/build.sh +++ b/fuzzing/oss-fuzz-scripts/build.sh @@ -13,7 +13,7 @@ find "$SEED_DATA_DIR" \( -name '*_seed_corpus.zip' -o -name '*.options' -o -name -exec cp {} "$OUT" \; # Build fuzzers in $OUT. -find "$SRC/gitpython/fuzzing" -name 'fuzz_*.py' -print0 | while IFS= read -r -d $'\0' fuzz_harness; do +find "$SRC/gitpython/fuzzing" -name 'fuzz_*.py' -print0 | while IFS= read -r -d '' fuzz_harness; do compile_python_fuzzer "$fuzz_harness" common_base_dictionary_filename="$SEED_DATA_DIR/__base.dict" From e038526b846f4bc5e75a91c736f3384616800aa1 Mon Sep 17 00:00:00 2001 From: David Lakin Date: Tue, 16 Apr 2024 16:27:43 -0400 Subject: [PATCH 6/9] Remove unnecessary semicolon for consistent script formatting Based @EliahKagan's suggestion and feedback on: https://github.com/gitpython-developers/GitPython/pull/1901 --- .../container-environment-bootstrap.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh b/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh index 881161fae..87f817993 100755 --- a/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh +++ b/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh @@ -38,19 +38,19 @@ fetch_seed_corpra() { # Seed corpus zip files are hosted in a separate repository to avoid additional bloat in this repo. git clone --depth 1 https://github.com/gitpython-developers/qa-assets.git qa-assets && rsync -avc qa-assets/gitpython/corpra/ "$SEED_DATA_DIR/" && - rm -rf qa-assets; # Clean up the cloned repo to keep the Docker image as slim as possible. + rm -rf qa-assets # Clean up the cloned repo to keep the Docker image as slim as possible. } ######################## # Main execution logic # ######################## -fetch_seed_corpra; +fetch_seed_corpra download_and_concatenate_common_dictionaries "$SEED_DATA_DIR/__base.dict" \ "https://raw.githubusercontent.com/google/fuzzing/master/dictionaries/utf8.dict" \ - "https://raw.githubusercontent.com/google/fuzzing/master/dictionaries/url.dict"; + "https://raw.githubusercontent.com/google/fuzzing/master/dictionaries/url.dict" # The OSS-Fuzz base image has outdated dependencies by default so we upgrade them below. -python3 -m pip install --upgrade pip; -python3 -m pip install 'setuptools~=69.0' 'pyinstaller~=6.0'; # Uses the latest versions know to work at the time of this commit. +python3 -m pip install --upgrade pip +python3 -m pip install 'setuptools~=69.0' 'pyinstaller~=6.0' # Uses the latest versions know to work at the time of this commit. From d25ae2def1f995afcb7fad69250b12f5bf07b3bb Mon Sep 17 00:00:00 2001 From: David Lakin Date: Tue, 16 Apr 2024 16:38:21 -0400 Subject: [PATCH 7/9] Fix various misspellings of "corpora" & improve script comments A misspelling in the https://github.com/gitpython-developers/qa-assets repository is still present here. It will need to be fixed in that repository first. "corpora" is a difficult word to spell consistently I guess. This made for a good opportunity to improve the phrasing of two other comments at at least. Based @EliahKagan's suggestion and feedback on: https://github.com/gitpython-developers/GitPython/pull/1901 --- fuzzing/oss-fuzz-scripts/build.sh | 4 ++-- .../oss-fuzz-scripts/container-environment-bootstrap.sh | 7 ++++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/fuzzing/oss-fuzz-scripts/build.sh b/fuzzing/oss-fuzz-scripts/build.sh index a412a1d15..ab46ec7a2 100644 --- a/fuzzing/oss-fuzz-scripts/build.sh +++ b/fuzzing/oss-fuzz-scripts/build.sh @@ -4,7 +4,7 @@ set -euo pipefail python3 -m pip install . -# Directory to look in for dictionaries, options files, and seed corpa: +# Directory to look in for dictionaries, options files, and seed corpora: SEED_DATA_DIR="$SRC/seed_data" find "$SEED_DATA_DIR" \( -name '*_seed_corpus.zip' -o -name '*.options' -o -name '*.dict' \) \ @@ -27,7 +27,7 @@ find "$SRC/gitpython/fuzzing" -name 'fuzz_*.py' -print0 | while IFS= read -r -d # If a dictionary file for this fuzzer already exists and is not empty, # we append a new line to the end of it before appending any new entries. # - # libfuzzer will happily ignore multiple empty lines in a dictionary but crash + # LibFuzzer will happily ignore multiple empty lines in a dictionary but fail with an error # if any single line has incorrect syntax (e.g., if we accidentally add two entries to the same line.) # See docs for valid syntax: https://llvm.org/docs/LibFuzzer.html#id32 echo >>"$output_file" diff --git a/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh b/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh index 87f817993..0be012ccd 100755 --- a/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh +++ b/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh @@ -34,7 +34,7 @@ download_and_concatenate_common_dictionaries() { done } -fetch_seed_corpra() { +fetch_seed_corpora() { # Seed corpus zip files are hosted in a separate repository to avoid additional bloat in this repo. git clone --depth 1 https://github.com/gitpython-developers/qa-assets.git qa-assets && rsync -avc qa-assets/gitpython/corpra/ "$SEED_DATA_DIR/" && @@ -45,7 +45,7 @@ fetch_seed_corpra() { # Main execution logic # ######################## -fetch_seed_corpra +fetch_seed_corpora download_and_concatenate_common_dictionaries "$SEED_DATA_DIR/__base.dict" \ "https://raw.githubusercontent.com/google/fuzzing/master/dictionaries/utf8.dict" \ @@ -53,4 +53,5 @@ download_and_concatenate_common_dictionaries "$SEED_DATA_DIR/__base.dict" \ # The OSS-Fuzz base image has outdated dependencies by default so we upgrade them below. python3 -m pip install --upgrade pip -python3 -m pip install 'setuptools~=69.0' 'pyinstaller~=6.0' # Uses the latest versions know to work at the time of this commit. + # Upgrade to the latest versions known to work at the time the below changes were introduced: +python3 -m pip install 'setuptools~=69.0' 'pyinstaller~=6.0' From 23a505f3ef51c4c26998fed924f4edad2438c757 Mon Sep 17 00:00:00 2001 From: David Lakin Date: Wed, 17 Apr 2024 19:40:44 -0400 Subject: [PATCH 8/9] Remove comment suggesting the `undefined` sanitizer is a valid option Also makes come structural improvements to how the local instructions for running OSS-Fuzz are presented now that only the single `address` sanitizer is a valid option. The `undefined` sanitizer was removed from GitPython's `project.yaml` OSS-Fuzz configuration file at the request of OSS-Fuzz project reviewers in https://github.com/google/oss-fuzz/pull/11803. The `undefined` sanitizer is only useful in Python projects that use native exstensions (such as C, C++, Rust, ect.), which GitPython does not currently do. This commit updates the `fuzzing/README` reference to that sanitizer accoirdingly. See: - https://github.com/google/oss-fuzz/pull/11803/commits/b210fb21427f1f994c91f07e95ca0cc977f61f66 - https://github.com/google/oss-fuzz/pull/11803#discussion_r1569160945 --- fuzzing/README.md | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/fuzzing/README.md b/fuzzing/README.md index 65e311d4a..ab9f6a63f 100644 --- a/fuzzing/README.md +++ b/fuzzing/README.md @@ -129,18 +129,7 @@ This approach uses Docker images provided by OSS-Fuzz for building and running f comprehensive features but requires a local clone of the OSS-Fuzz repository and sufficient disk space for Docker containers. -#### Preparation - -Set environment variables to simplify command usage: - -```shell -# $SANITIZER can be either 'address' or 'undefined': -export SANITIZER=address -# specify the fuzz target without the .py extension: -export FUZZ_TARGET=fuzz_config -``` - -#### Build and Run +#### Build the Execution Environment Clone the OSS-Fuzz repository and prepare the Docker environment: @@ -148,7 +137,7 @@ Clone the OSS-Fuzz repository and prepare the Docker environment: git clone --depth 1 https://github.com/google/oss-fuzz.git oss-fuzz cd oss-fuzz python infra/helper.py build_image gitpython -python infra/helper.py build_fuzzers --sanitizer $SANITIZER gitpython +python infra/helper.py build_fuzzers --sanitizer address gitpython ``` > [!TIP] @@ -160,16 +149,25 @@ python infra/helper.py build_fuzzers --sanitizer $SANITIZER gitpython > Then running this command would build new or modified fuzz targets using the `~/code/GitPython/fuzzing/fuzz-targets` > directory: > ```shell -> python infra/helper.py build_fuzzers --sanitizer $SANITIZER gitpython ~/code/GitPython +> python infra/helper.py build_fuzzers --sanitizer address gitpython ~/code/GitPython > ``` - Verify the build of your fuzzers with the optional `check_build` command: ```shell python infra/helper.py check_build gitpython ``` +#### Run a Fuzz Target + +Setting an environment variable for the fuzz target argument of the execution command makes it easier to quickly select +a different target between runs: + +```shell +# specify the fuzz target without the .py extension: +export FUZZ_TARGET=fuzz_config +``` + Execute the desired fuzz target: ```shell From 1d54d4b0b2bdba60cc742f73a4b8d5a88cce8f64 Mon Sep 17 00:00:00 2001 From: David Lakin Date: Wed, 17 Apr 2024 22:11:00 -0400 Subject: [PATCH 9/9] Remove unintentional leading space from comment --- fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh b/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh index 0be012ccd..662808e27 100755 --- a/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh +++ b/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh @@ -53,5 +53,5 @@ download_and_concatenate_common_dictionaries "$SEED_DATA_DIR/__base.dict" \ # The OSS-Fuzz base image has outdated dependencies by default so we upgrade them below. python3 -m pip install --upgrade pip - # Upgrade to the latest versions known to work at the time the below changes were introduced: +# Upgrade to the latest versions known to work at the time the below changes were introduced: python3 -m pip install 'setuptools~=69.0' 'pyinstaller~=6.0'