diff --git a/.pylintrc b/.pylintrc index 3f29aa580..155fc8026 100644 --- a/.pylintrc +++ b/.pylintrc @@ -3,6 +3,17 @@ # Use multiple processes to speed up Pylint. jobs=2 +# List of plugins (as comma separated values of python modules names) to load, +# usually to register additional checkers. +# +# Whole list retrieved on 2017-07-28 from: https://pylint.readthedocs.io/en/latest/technical_reference/extensions.html +# Remaining disabled: pylint.extensions.comparetozero,pylint.extensions.emptystring +load-plugins=pylint.extensions.bad_builtin,pylint.extensions.check_elif,pylint.extensions.docparams,pylint.extensions.docstyle,pylint.extensions.mccabe,pylint.extensions.overlapping_exceptions,pylint.extensions.redefined_variable_type +accept-no-param-doc=no +accept-no-raise-doc=no +accept-no-return-doc=no +accept-no-yields-doc=no + [MESSAGES CONTROL] # Disable the message, report, category or checker with the given id(s). You @@ -14,9 +25,13 @@ jobs=2 # --enable=similarities". If you want to run only the classes checker, but have # no Warning level messages displayed, use"--disable=all --enable=classes # --disable=W" -# -# Ignore here until autobisect-js is no longer around -disable=invalid-name +disable=locally-disabled,no-absolute-import,suppressed-message,useless-suppression + +# Enable the message, report, category or checker with the given id(s). You can +# either give multiple identifier separated by comma (,) or put this option +# multiple time (only on the command line, not in the configuration file where +# it should appear only once). See also the "--disable" option for examples. +enable=apply-builtin,backtick,bad-inline-option,bad-python3-import,basestring-builtin,buffer-builtin,cmp-builtin,cmp-method,coerce-builtin,coerce-method,delslice-method,deprecated-pragma,deprecated-str-translate-call,deprecated-string-function,dict-iter-method,dict-view-method,div-method,eq-without-hash,exception-message-attribute,execfile-builtin,file-builtin,file-ignored,filter-builtin-not-iterating,getslice-method,hex-method,idiv-method,import-star-module-level,indexing-exception,input-builtin,intern-builtin,invalid-str-codec,locally-enabled,long-builtin,long-suffix,map-builtin-not-iterating,metaclass-assignment,next-method-called,nonzero-method,oct-method,old-division,old-ne-operator,old-octal-literal,old-raise-syntax,parameter-unpacking,print-statement,raising-string,range-builtin-not-iterating,raw-checker-failed,raw_input-builtin,rdiv-method,reduce-builtin,reload-builtin,round-builtin,setslice-method,standarderror-builtin,sys-max-int,unichr-builtin,unicode-builtin,unpacking-in-except,using-cmp-argument,xrange-builtin,zip-builtin-not-iterating [BASIC] diff --git a/README.md b/README.md index 6ff1d33a1..b62201e0e 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ [![Build Status](https://travis-ci.org/MozillaSecurity/funfuzz.svg?branch=master)](https://travis-ci.org/MozillaSecurity/funfuzz) [![Build status](https://ci.appveyor.com/api/projects/status/m8gw5echa7f2f26r/branch/master?svg=true)](https://ci.appveyor.com/project/MozillaSecurity/funfuzz/branch/master) -This repository contains several JavaScript-based fuzzers. [jsfunfuzz](js/jsfunfuzz) tests JavaScript engines and can run in a JavaScript shell, compareJIT compares output from SpiderMonkey using different flags, while randorderfuzz throws in random tests from the mozilla-central directory into generated jsfunfuzz output. +This repository contains several JavaScript-based fuzzers. [jsfunfuzz](js/jsfunfuzz) tests JavaScript engines and can run in a JavaScript shell, compare_jit compares output from SpiderMonkey using different flags, while randorderfuzz throws in random tests from the mozilla-central directory into generated jsfunfuzz output. -Most of the code other than testcase generation is written in Python: restarting the program when it exits or crashes, noticing evidence of new bugs from the program's output, [reducing testcases](https://github.com/MozillaSecurity/lithium/), and [identifying when regressions were introduced](autobisect-js/README.md). +Most of the code other than testcase generation is written in Python: restarting the program when it exits or crashes, noticing evidence of new bugs from the program's output, [reducing testcases](https://github.com/MozillaSecurity/lithium/), and [identifying when regressions were introduced](src/funfuzz/autobisectjs/README.md). ## Setup @@ -25,30 +25,23 @@ Here's a guide to [pip and virtualenv](https://www.dabapps.com/blog/introduction ### Windows (only 64-bit supported) -1. Install [MozillaBuild](https://wiki.mozilla.org/MozillaBuild) (Using compileShell for SpiderMonkey requires at least version 2.2.0) to get an msys shell. +1. Install [MozillaBuild](https://wiki.mozilla.org/MozillaBuild) (Using compile_shell for SpiderMonkey requires at least version 3.0). 2. Install [Git for Windows](https://msysgit.github.io/) to get Git for Windows in order to clone these funfuzz repositories. (32-bit works best for now) 3. Install [Debugging Tools for Windows](https://msdn.microsoft.com/en-us/windows/hardware/hh852365.aspx) to get cdb.exe and thus stacks from crashes. -4. Make sure you install at least Microsoft Visual Studio 2015 (Community Edition is recommended) as per the build instructions above in the Setup section. -5. Run `start-shell-msvc2015.bat` to get a MSYS shell. Do not use the MSYS shell that comes with Git for Windows. You can use Git by calling its absolute path, e.g. `/c/Program\ Files\ \(x86\)/Git/bin/git.exe`. +4. Make sure you install at least Microsoft Visual Studio 2015 (Community Edition is recommended) as per the build instructions above in the Setup section. Visual Studio 2017 is preferred. +5. Run `start-shell.bat` to get a MSYS shell. Do not use the MSYS shell that comes with Git for Windows. You can use Git by calling its absolute path, e.g. `/c/Program\ Files/Git/bin/git.exe`. 1. Run the batch file with administrator privileges to get gflags analysis working correctly. ### Mac -1. On Mac OS X 10.9, you must first install a newer version of unzip than the one that comes with the OS. (Old versions [hit an error](https://bugzilla.mozilla.org/show_bug.cgi?id=1032391) on large zip files, such as the "mac64.tests.zip" file that [downloadBuild.py](util/downloadBuild.py) grabs.) - - ``` - brew install homebrew/dupes/unzip - brew link --force unzip - ``` - -2. If you encounter problems accessing the compiler, try re-running this command: +1. If you encounter problems accessing the compiler, try re-running this command: ```xcode-select --install``` especially after updating major/minor OS versions. This sometimes manifests on Mac OS X Combo updates. -3. Install LLVM via Homebrew, to get llvm-symbolizer needed for symbolizing ASan crash stacks. +2. Install LLVM via Homebrew, to get llvm-symbolizer needed for symbolizing ASan crash stacks. ``` brew install llvm @@ -77,23 +70,23 @@ especially after updating major/minor OS versions. This sometimes manifests on M To run **only the js fuzzers** which compiles shells with random configurations every 8 hours and tests them: -`python -u funfuzz/loopBot.py -b "--random" -t "js" --target-time 28800 | tee ~/log-loopBotPy.txt` +`python -u funfuzz.loop_bot -b "--random" -t "js" --target-time 28800 | tee ~/log-loop_botPy.txt` -To test **a patch** (assuming patch is in ~/patch.diff) against a specific branch (assuming **Mercurial** mozilla-inbound is in ~/trees/mozilla-inbound), using a debug 64-bit deterministic shell configuration, every 8 hours: +To test **a patch** (assuming patch is in `~/patch.diff`) against a specific branch (assuming **Mercurial** mozilla-inbound is in `~/trees/mozilla-inbound`), using a debug 64-bit deterministic shell configuration, every 8 hours: -`python -u funfuzz/loopBot.py -b "--enable-debug --enable-more-deterministic -R ~/trees/mozilla-inbound -P ~/patch.diff" -t "js" --target-time 28800 | tee ~/log-loopBotPy.txt` +`python -u funfuzz.loop_bot -b "--enable-debug --enable-more-deterministic -R ~/trees/mozilla-inbound -P ~/patch.diff" -t "js" --target-time 28800 | tee ~/log-loop_botPy.txt` -In js mode, loopBot.py makes use of: +In js mode, loop_bot makes use of: -* [compileShell](js/compileShell.py) -* [jsfunfuzz](js/jsfunfuzz) -* [compareJIT](js/compareJIT.py) (if testing deterministic builds) -* randorderfuzz (included in jsfunfuzz, if tests are present in the mozilla repository) -* [autoBisect](autobisect-js/README.md) (if the mozilla repository is present). +* [compile_shell](js/compile_shell.py) +* [jsfunfuzz](src/funfuzz/js/jsfunfuzz) +* [compare_jit](src/funfuzz/js/compare_jit.py) (if testing deterministic builds) +* randorderfuzz (included in funfuzz, if tests are present in the mozilla repository) +* [autoBisect](src/funfuzz/autobisectjs/README.md) (if the mozilla repository is present). -The parameters in `-b` get passed into [compileShell](js/compileShell.py) and [autoBisect](autobisect-js/README.md). +The parameters in `-b` get passed into [compile_shell](js/compile_shell.py) and [autoBisect](src/funfuzz/autobisectjs/README.md). -FuzzManager support got landed, so you will also need to create a ~/.fuzzmanagerconf file, similar to: +You will also need to need a `~/.fuzzmanagerconf` file, similar to: ``` [Main] @@ -105,25 +98,26 @@ sigdir = /Users//sigcache/ tool = jsfunfuzz ``` -Replace anything between "<" and ">" with your desired parameters. +Replace anything between `<` and `>` with your desired parameters. ## FAQ: **Q: What platforms does funfuzz run on?** -**A:** compileShell has been tested on: +**A:** compile_shell has been tested on: -* Windows 10, 7 and Windows Server 2012 R2, with [MozillaBuild 2.2.0](https://wiki.mozilla.org/MozillaBuild). It should also work with MozillaBuild 3.0. -* Mac OS X 10.12 +* Windows 10 and 7, with [MozillaBuild 3.0](https://wiki.mozilla.org/MozillaBuild). It should also work with Windows Server 2012 R2. +* Mac OS X 10.13 * Ubuntu 16.04 LTS and later + * Note: We may try to make this work on Ubuntu 14.04 LTS (via Travis) Fedora Linux has not been tested extensively and there may be a few bugs along the way. -The following operating systems are old/less common and while they may still work, be prepared to **expect issues** along the way: +The following operating systems are old or less common and while they may still work, be prepared to **expect issues** along the way: * Windows Vista / Windows 8 / Windows 8.1 -* Mac OS X 10.10 / 10.11 -* Ubuntu Linux 14.04 LTS, 15.10 and prior +* Mac OS X 10.10 through 10.12 +* Ubuntu Linux 15.10 and prior (see note above about 14.04 LTS) * Ubuntu (and variants) on [ARM ODROID boards](http://www.hardkernel.com/main/main.php) Support for the following operating systems **have been removed**: diff --git a/__init__.py b/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/after-fix.txt b/after-fix.txt index 1688eb7ce..6b3b82ad5 100644 --- a/after-fix.txt +++ b/after-fix.txt @@ -1,8 +1,7 @@ # https://github.com/jruderman/after-fix -contents bot.py -contents util/*.py -contents detect/*.py -contents js/*.py -contents js/shared/*.js -contents js/jsfunfuzz/*.js +contents src/funfuzz/*.py +contents src/funfuzz/util/*.py +contents src/funfuzz/js/*.py +contents src/funfuzz/js/shared/*.js +contents src/funfuzz/js/jsfunfuzz/*.js diff --git a/autobisect-js/README.md b/autobisect-js/README.md deleted file mode 100644 index 2b56605aa..000000000 --- a/autobisect-js/README.md +++ /dev/null @@ -1,80 +0,0 @@ -autoBisect will help you to find out when a changeset introduced problems. It can also point at a changeset that may have exposed the issue. - -It helps with work allocation: - -* The engineer that most recently worked on the code is the one most likely to know how to fix the bug. -* If not, the engineer may be able to forward to someone more knowledgeable. - -## Find changeset that introduced problems using autoBisect - -For SpiderMonkey, use the following while compiling locally: - -`funfuzz/autobisect-js/autoBisect.py -p "--fuzzing-safe --no-threads --ion-eager testcase.js" -b "--enable-debug --enable-more-deterministic"` - -assuming the testcase requires "--fuzzing-safe --no-threads --ion-eager" as runtime flags. - -This will take about: - -* **45 - 60 minutes** on a relatively recent powerful computer on Linux / Mac - * assuming each compilation takes about 3 minutes - * we should be able to find the problem within 16+ tests. -* **2 hours** on Windows - * where each compilation is assumed to take 6 minutes. - -If you have an internet connection, and the testcase causes problems with: - -* a [downloaded js shell](https://archive.mozilla.org/pub/mozilla.org/firefox/tinderbox-builds/mozilla-central-macosx64-debug/latest/jsshell-mac64.zip) -* these problems started happening within the last month - -you can try bisecting using downloaded builds: - -`funfuzz/autobisect-js/autoBisect.py -p "--fuzzing-safe --no-threads --ion-eager testcase.js" -b "--enable-debug" -T` - -This should take < 5 minutes total assuming a fast internet connection, since it does not need to compile shells. - -Refer to [compileShell.py documentation](../js/README.md) for parameters to be passed into "-b". - -``` -Usage: autoBisect.py [options] - -Options: - -h, --help show this help message and exit - -b BUILDOPTIONS, --build=BUILDOPTIONS - Specify js shell build options, e.g. -b "--enable- - debug --32" (python buildOptions.py --help) - --resetToTipFirst First reset to default tip overwriting all local - changes. Equivalent to first executing `hg update -C - default`. Defaults to "False". - -s STARTREPO, --startRev=STARTREPO - Earliest changeset/build numeric ID to consider - (usually a "good" cset). Defaults to the earliest - revision known to work at all/available. - -e ENDREPO, --endRev=ENDREPO - Latest changeset/build numeric ID to consider (usually - a "bad" cset). Defaults to the head of the main - branch, "default", or latest available build. - -k, --skipInitialRevs - Skip testing the -s and -e revisions and automatically - trust them as -g and -b. - -o OUTPUT, --output=OUTPUT - Stdout or stderr output to be observed. Defaults to - "". For assertions, set to "ssertion fail" - -w WATCHEXITCODE, --watchExitCode=WATCHEXITCODE - Look out for a specific exit code. Only this exit code - will be considered "bad". - -i, --useInterestingnessTests - Interpret the final arguments as an interestingness - test. - -p PARAMETERS, --parameters=PARAMETERS - Specify parameters for the js shell, e.g. -p "-a - --ion-eager testcase.js". - -l COMPILATIONFAILEDLABEL, --compilationFailedLabel=COMPILATIONFAILEDLABEL - Specify how to treat revisions that fail to compile. - (bad, good, or skip) Defaults to "skip" - -T, --useTreeherderBinaries - Use treeherder binaries for quick bisection, assuming - a fast internet connection. Defaults to "False" - -N NAMEOFTREEHERDERBRANCH, --nameOfTreeherderBranch=NAMEOFTREEHERDERBRANCH - Name of the branch to download. Defaults to "mozilla- - inbound" -``` diff --git a/autobisect-js/__init__.py b/autobisect-js/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/autobisect-js/examples-autoBisect.md b/autobisect-js/examples-autoBisect.md deleted file mode 100644 index 448daf15a..000000000 --- a/autobisect-js/examples-autoBisect.md +++ /dev/null @@ -1,47 +0,0 @@ -## Examples - -To try this yourself, run the following commands with the testcases from the bug numbers pasted into the file, e.g. "1188586.js" contains the testcase from [bug 1188586](https://bugzilla.mozilla.org/show_bug.cgi?id=1188586). - -* To test when a bug was introduced by **downloading mozilla-inbound builds** from Mozilla: - -```funfuzz/autobisect-js/autoBisect.py -p "--fuzzing-safe --no-threads --ion-eager 1188586.js" -b "--enable-debug" -T``` - -However, this only works effectively if the bug was recent, because builds are only stored per-push within the past month. - -* The equivalent command using **local compiled builds** is: - -```funfuzz/autobisect-js/autoBisect.py -p "--fuzzing-safe --no-threads --ion-eager 1188586.js" -b "--enable-debug --enable-more-deterministic"``` - -* To **test branches**, e.g. on mozilla-inbound instead (or any other release branch including ESR), assuming the *Mercurial* repository is cloned to "~/trees/mozilla-inbound": - -```funfuzz/autobisect-js/autoBisect.py -p "--fuzzing-safe --no-threads --ion-eager 1188586.js" -b "--enable-debug --enable-more-deterministic -R ~/trees/mozilla-inbound"``` - -* During bisection, perhaps the testcase used to crash in the past; however we are only interested in the assertion failure. You can make autoBisect look out for the **assertion failure message**: - -```funfuzz/autobisect-js/autoBisect.py -p "--fuzzing-safe --no-threads --ion-eager 1188586.js" -b "--enable-debug --enable-more-deterministic" -o "Assertion failure"``` - -* To look out for a particular **exit code**, use "-w": - -```funfuzz/autobisect-js/autoBisect.py -p "--fuzzing-safe --no-threads --ion-eager 1189137.js" -b "--enable-debug --enable-more-deterministic" -w 3``` - -* To specify **starting and ending revisions**, use "-s" and "-e": - -```funfuzz/autobisect-js/autoBisect.py -s 7820fd141998 -e 'parents(322487136b28)' -p "--no-threads --ion-eager --unboxed-objects 1189137.js" -b "--enable-debug --enable-more-deterministic" -o "Assertion failed"``` - -This method can be used to find when a regression was introduced as well as when a bug got fixed. - -* Or, the testcase is **intermittent** and only reproduces once every 5 tries. autoBisect can be set to use the "range" interestingness test to retest 50 times before concluding if it is interesting or not: - -```funfuzz/autobisect-js/autoBisect.py -p "--fuzzing-safe --no-threads --ion-eager 1188586.js" -b "--enable-debug --enable-more-deterministic" -i range 1 50 crashes --timeout=3``` - -Note that this requires the [lithium repository](https://github.com/MozillaSecurity/lithium) to be cloned adjacent to the funfuzz repository. - -You could specify the assertion message this way too: - -```funfuzz/autobisect-js/autoBisect.py -p "--fuzzing-safe --no-threads --ion-eager 1188586.js" -b "--enable-debug --enable-more-deterministic" -i range 1 50 outputs --timeout=3 'Assertion failure'``` - -"-i" should be the last argument on the command line. - -* To bisect **bugs found by compareJIT**: - -```funfuzz/autobisect-js/autoBisect.py -s 6ec4eb9786d8 -p 1183423.js -b "--enable-debug --enable-more-deterministic -R ~/trees/mozilla-central" -i ~/funfuzz/js/compareJIT.py --minlevel=6 mozilla-central``` diff --git a/autobisect-js/findCsetsIntersection.py b/autobisect-js/findCsetsIntersection.py deleted file mode 100644 index f6446aad2..000000000 --- a/autobisect-js/findCsetsIntersection.py +++ /dev/null @@ -1,66 +0,0 @@ -#!/usr/bin/env python -# coding=utf-8 -# pylint: disable=import-error,invalid-name,missing-docstring,wrong-import-position -# -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at https://mozilla.org/MPL/2.0/. - -# This file scans the revsets in ignoreAndEarliestWorkingLists and looks for overlaps. -# -# Usage: python findCsetsIntersection.py -R ~/trees/mozilla-central/ -# -# (first go to knownBrokenEarliestWorking.py and comment out configuration-specific ignore ranges, -# this file does not yet support those.) - -from __future__ import absolute_import, print_function - -import os -import sys -from optparse import OptionParser # pylint: disable=deprecated-module - -import knownBrokenEarliestWorking as kbew -path0 = os.path.dirname(os.path.abspath(__file__)) -path1 = os.path.abspath(os.path.join(path0, os.pardir, 'util')) -sys.path.append(path1) -import subprocesses as sps - - -def parseOptions(): - parser = OptionParser() - parser.add_option('-R', '--repo', dest='rDir', - help='Sets the repository to analyze..') - options, _args = parser.parse_args() - assert options.rDir is not None - assert os.path.isdir(sps.normExpUserPath(options.rDir)) - return options - - -def countCsets(revset, rdir): - """Count the number of changesets in the revsets by outputting ones and counting them.""" - listCmd = ['hg', 'log', '-r', revset, '--template=1'] - rangeIntersectionOnes = sps.captureStdout(listCmd, currWorkingDir=rdir) - assert rangeIntersectionOnes[1] == 0 - return len(rangeIntersectionOnes[0]) - - -def main(): - options = parseOptions() - repoDir = options.rDir - brokenRanges = kbew.knownBrokenRanges(options) - - cnt = 0 - for i in range(0, len(brokenRanges)): - print("Analyzing revset: %s which matches %s changesets" % ( - brokenRanges[i], countCsets(brokenRanges[i], repoDir))) - for j in range(i + 1, len(brokenRanges)): - cnt += 1 - print("Number %s: Compared against revset: %s" % (cnt, brokenRanges[j])) - overlap = countCsets(brokenRanges[i] + ' and ' + brokenRanges[j], repoDir) - if overlap: - print("Number of overlapping changesets: %s" % (overlap,)) - cnt = 0 - - -if __name__ == '__main__': - main() diff --git a/detect/__init__.py b/detect/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/detect/detect_malloc_errors.py b/detect/detect_malloc_errors.py deleted file mode 100755 index c62f6acc2..000000000 --- a/detect/detect_malloc_errors.py +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/env python -# coding=utf-8 -# pylint: disable=global-statement,invalid-name,missing-docstring -# -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at https://mozilla.org/MPL/2.0/. - -from __future__ import absolute_import, print_function - -# Look for "szone_error" (Tiger), "malloc_error_break" (Leopard), "MallocHelp" (?) -# which are signs of malloc being unhappy (double free, out-of-memory, etc). - -pline = "" -ppline = "" - - -def amiss(logPrefix): - foundSomething = False - global pline, ppline - - pline = "" - ppline = "" - - with open(logPrefix + "-err.txt") as f: - for line in f: - if scanLine(line): - foundSomething = True - break # Don't flood the log with repeated malloc failures - - return foundSomething - - -def scanLine(line): - global ppline, pline - - line = line.strip("\x07").rstrip("\n") - - if (line.find("szone_error") != -1 or - line.find("malloc_error_break") != -1 or - line.find("MallocHelp") != -1): - if pline.find("can't allocate region") == -1: - print() - print(ppline) - print(pline) - print(line) - return True - - ppline = pline - pline = line diff --git a/js/README.md b/js/README.md deleted file mode 100644 index d29477742..000000000 --- a/js/README.md +++ /dev/null @@ -1,75 +0,0 @@ -## Compile SpiderMonkey using compileShell - -To compile a SpiderMonkey shell, run: - -`funfuzz/js/compileShell.py -b "--enable-debug --enable-more-deterministic -R ~/trees/mozilla-central"` - -in order to get a debug 64-bit deterministic shell, off the **Mercurial** repository located at ~/trees/mozilla-central. - -Clone the repository to that location using: - -`hg clone https://hg.mozilla.org/mozilla-central/ ~/trees/mozilla-central` - -assuming the ~/trees folder is created and present. - -The options accepted by -b are also available via funfuzz/js/buildOptions.py: - -``` - --random Chooses sensible random build options. Defaults to - "False". - -R REPODIR, --repoDir REPODIR - Sets the source repository. - -P PATCHFILE, --patch PATCHFILE - Define the path to a single JS patch. Ensure mq is - installed. - --32 Build 32-bit shells, but if not enabled, 64-bit shells - are built. - --enable-debug Build shells with --enable-debug. Defaults to "False". - Currently defaults to True in configure.in on mozilla- - central. - --disable-debug Build shells with --disable-debug. Defaults to - "False". Currently defaults to True in configure.in on - mozilla-central. - --enable-optimize Build shells with --enable-optimize. Defaults to - "False". - --disable-optimize Build shells with --disable-optimize. Defaults to - "False". - --enable-profiling Build shells with --enable-profiling. Defaults to - "False". Currently defaults to True in configure.in on - mozilla-central. - --disable-profiling Build with profiling off. Defaults to "True" on Linux, - else "False". - --build-with-clang Build with clang. Defaults to "True" on Macs, "False" - otherwise. - --build-with-asan Build with clang AddressSanitizer support. Defaults to - "False". - --build-with-valgrind - Build with valgrind.h bits. Defaults to "False". - Requires --enable-hardfp for ARM platforms. - --run-with-valgrind Run the shell under Valgrind. Requires --build-with- - valgrind. - --enable-more-deterministic - Build shells with --enable-more-deterministic. - Defaults to "False". - --enable-oom-breakpoint - Build shells with --enable-oom-breakpoint. Defaults to - "False". - --without-intl-api Build shells using --without-intl-api. Defaults to - "False". - --enable-simulator=arm - Build shells with --enable-simulator=arm, only - applicable to 32-bit shells. Defaults to "False". - --enable-simulator=arm64 - Build shells with --enable-simulator=arm64, only - applicable to 64-bit shells. Defaults to "False". - --enable-arm-simulator - Build the shell using --enable-arm-simulator for - legacy purposes. This flag is obsolete and is the - equivalent of --enable-simulator=arm, use --enable- - simulator=[arm|arm64] instead. Defaults to "False". -``` - -## Additional information -* compileShell - * [More examples](examples-compileShell.md) - * [FAQ](faq-compileShell.md) diff --git a/js/__init__.py b/js/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/js/compileShell.py b/js/compileShell.py deleted file mode 100755 index 4e64c9991..000000000 --- a/js/compileShell.py +++ /dev/null @@ -1,754 +0,0 @@ -#!/usr/bin/env python -# coding=utf-8 -# pylint: disable=broad-except,fixme,import-error,invalid-name,missing-docstring -# pylint: disable=too-many-branches,too-many-instance-attributes,too-many-public-methods,too-many-statements -# pylint: disable=wrong-import-position -# -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at https://mozilla.org/MPL/2.0/. - -from __future__ import absolute_import, print_function, unicode_literals - -import copy -import ctypes -import io -import multiprocessing -import os -import shutil -import subprocess -import sys -import tarfile -import traceback - -from optparse import OptionParser # pylint: disable=deprecated-module - -import buildOptions -import inspectShell - -path0 = os.path.dirname(os.path.abspath(__file__)) -path1 = os.path.abspath(os.path.join(path0, os.pardir, 'util')) -sys.path.append(path1) -import hgCmds -import s3cache -import subprocesses as sps -from LockDir import LockDir - -S3_SHELL_CACHE_DIRNAME = 'shell-cache' # Used by autoBisect - -if sps.isWin: - MAKE_BINARY = b"mozmake" - CLANG_PARAMS = b"-fallback" - # CLANG_ASAN_PARAMS = b"-fsanitize=address -Dxmalloc=myxmalloc" - # Note that Windows ASan builds are still a work-in-progress - CLANG_ASAN_PARAMS = b"" -else: - MAKE_BINARY = b"make" - CLANG_PARAMS = b"-Qunused-arguments" - # See https://bugzilla.mozilla.org/show_bug.cgi?id=935795#c3 for some of the following flags: - # CLANG_ASAN_PARAMS = b"-fsanitize=address -Dxmalloc=myxmalloc -mllvm -asan-stack=0" - # The flags above seem to fix a problem not on the js shell. - CLANG_ASAN_PARAMS = b"-fsanitize=address -Dxmalloc=myxmalloc" - SSE2_FLAGS = b"-msse2 -mfpmath=sse" # See bug 948321 - CLANG_X86_FLAG = b"-arch i386" - -if multiprocessing.cpu_count() > 2: - COMPILATION_JOBS = ((multiprocessing.cpu_count() * 5) // 4) -elif sps.isARMv7l: - COMPILATION_JOBS = 3 # An ARM board -else: - COMPILATION_JOBS = 3 # Other single/dual core computers - - -class CompiledShell(object): - def __init__(self, buildOpts, hgHash): - self.shellNameWithoutExt = buildOptions.computeShellName(buildOpts, hgHash) - self.shellNameWithExt = self.shellNameWithoutExt + (b".exe" if sps.isWin else b"") - self.hgHash = hgHash - self.buildOptions = buildOpts - - self.jsObjdir = '' - - self.cfg = '' - self.destDir = '' - self.addedEnv = b"" - self.fullEnv = b"" - self.jsCfgFile = '' - - self.jsMajorVersion = '' - self.jsVersion = '' - - def getCfgCmdExclEnv(self): - return self.cfg - - def setCfgCmdExclEnv(self, cfg): - self.cfg = cfg - - def setEnvAdded(self, addedEnv): - self.addedEnv = addedEnv - - def getEnvAdded(self): - return self.addedEnv - - def setEnvFull(self, fullEnv): - self.fullEnv = fullEnv - - def getEnvFull(self): - return self.fullEnv - - def getHgHash(self): - return self.hgHash - - def getJsCfgPath(self): - self.jsCfgFile = sps.normExpUserPath(os.path.join(self.getRepoDirJsSrc(), 'configure')) - assert os.path.isfile(self.jsCfgFile) - return self.jsCfgFile - - def getJsObjdir(self): - return self.jsObjdir - - def setJsObjdir(self, oDir): - self.jsObjdir = oDir - - def getRepoDir(self): - return self.buildOptions.repoDir - - def getRepoDirJsSrc(self): - return sps.normExpUserPath(os.path.join(self.getRepoDir(), 'js', 'src')) - - def getRepoName(self): - return hgCmds.getRepoNameFromHgrc(self.buildOptions.repoDir) - - def getS3TarballWithExt(self): - return self.getShellNameWithoutExt() + '.tar.bz2' - - def getS3TarballWithExtFullPath(self): - return sps.normExpUserPath(os.path.join(ensureCacheDir(), self.getS3TarballWithExt())) - - def getShellCacheDir(self): - return sps.normExpUserPath(os.path.join(ensureCacheDir(), self.getShellNameWithoutExt())) - - def getShellCacheFullPath(self): - return sps.normExpUserPath(os.path.join(self.getShellCacheDir(), self.getShellNameWithExt())) - - def getShellCompiledPath(self): - return sps.normExpUserPath( - os.path.join(self.getJsObjdir(), 'dist', 'bin', 'js' + ('.exe' if sps.isWin else ''))) - - def getShellCompiledRunLibsPath(self): - lDir = self.getJsObjdir() - libsList = [ - sps.normExpUserPath(os.path.join(lDir, 'dist', 'bin', runLib)) - for runLib in inspectShell.ALL_RUN_LIBS - ] - return libsList - - def getShellNameWithExt(self): - return self.shellNameWithExt - - def getShellNameWithoutExt(self): - return self.shellNameWithoutExt - - # Version numbers - def getMajorVersion(self): - return self.jsMajorVersion - - def setMajorVersion(self, jsMajorVersion): - self.jsMajorVersion = jsMajorVersion - - def getVersion(self): - return self.jsVersion - - def setVersion(self, jsVersion): - self.jsVersion = jsVersion - - -def ensureCacheDir(): - """Return a cache directory for compiled shells to live in, and create one if needed.""" - cacheDir = os.path.join(sps.normExpUserPath('~'), 'shell-cache') - ensureDir(cacheDir) - - # Expand long Windows paths (overcome legacy MS-DOS 8.3 stuff) - # This has to occur after the shell-cache directory is created - if sps.isWin: # adapted from http://stackoverflow.com/a/3931799 - if sys.version_info.major == 2: - utext = unicode # noqa pylint: disable=redefined-builtin,undefined-variable - else: - utext = str - winTmpDir = utext(cacheDir) - GetLongPathName = ctypes.windll.kernel32.GetLongPathNameW - unicodeBuffer = ctypes.create_unicode_buffer(GetLongPathName(winTmpDir, 0, 0)) - GetLongPathName(winTmpDir, unicodeBuffer, len(unicodeBuffer)) - cacheDir = sps.normExpUserPath(str(unicodeBuffer.value)) # convert back to a str - - return cacheDir - - -def ensureDir(directory): - """Create a directory, if it does not already exist.""" - if not os.path.exists(directory): - os.mkdir(directory) - assert os.path.isdir(directory) - - -def autoconfRun(cwDir): - """Run autoconf binaries corresponding to the platform.""" - if sps.isMac: - autoconf213MacBin = '/usr/local/Cellar/autoconf213/2.13/bin/autoconf213' \ - if sps.isProgramInstalled('brew') else 'autoconf213' - # Total hack to support new and old Homebrew configs, we can probably just call autoconf213 - if not os.path.isfile(sps.normExpUserPath(autoconf213MacBin)): - autoconf213MacBin = 'autoconf213' - subprocess.check_call([autoconf213MacBin], cwd=cwDir) - elif sps.isLinux: - # FIXME: We should use a method that is similar to the client.mk one, as per - # https://github.com/MozillaSecurity/funfuzz/issues/9 - try: - # Ubuntu - subprocess.check_call(['autoconf2.13'], cwd=cwDir) - except OSError: - # Fedora has a different name - subprocess.check_call(['autoconf-2.13'], cwd=cwDir) - elif sps.isWin: - # Windows needs to call sh to be able to find autoconf. - subprocess.check_call(['sh', 'autoconf-2.13'], cwd=cwDir) - - -def cfgJsCompile(shell): - """Configures, compiles and copies a js shell according to required parameters.""" - print("Compiling...") # Print *with* a trailing newline to avoid breaking other stuff - os.mkdir(sps.normExpUserPath(os.path.join(shell.getShellCacheDir(), 'objdir-js'))) - shell.setJsObjdir(sps.normExpUserPath(os.path.join(shell.getShellCacheDir(), 'objdir-js'))) - - autoconfRun(shell.getRepoDirJsSrc()) - configureTryCount = 0 - while True: - try: - cfgBin(shell) - break - except Exception as e: - configureTryCount += 1 - if configureTryCount > 3: - print("Configuration of the js binary failed 3 times.") - raise - # This exception message is returned from sps.captureStdout via cfgBin. - # No idea why this is sps.isLinux as well.. - if sps.isLinux or (sps.isWin and 'Windows conftest.exe configuration permission' in repr(e)): - print("Trying once more...") - continue - compileJs(shell) - inspectShell.verifyBinary(shell) - - compileLog = sps.normExpUserPath(os.path.join(shell.getShellCacheDir(), - shell.getShellNameWithoutExt() + '.fuzzmanagerconf')) - if not os.path.isfile(compileLog): - envDump(shell, compileLog) - - -def cfgBin(shell): - """Configure a binary according to required parameters.""" - cfgCmdList = [] - cfgEnvDt = copy.deepcopy(os.environ) - origCfgEnvDt = copy.deepcopy(os.environ) - cfgEnvDt[b"AR"] = b"ar" - if sps.isARMv7l: - # 32-bit shell on ARM boards, e.g. odroid boards. - # This is tested on Ubuntu 14.04 with necessary armel libraries (force)-installed. - assert shell.buildOptions.enable32, 'arm7vl boards are only 32-bit, armv8 boards will be 64-bit.' - if not shell.buildOptions.enableHardFp: - cfgEnvDt[b"CC"] = b"gcc-4.7 -mfloat-abi=softfp -B/usr/lib/gcc/arm-linux-gnueabi/4.7" - cfgEnvDt[b"CXX"] = b"g++-4.7 -mfloat-abi=softfp -B/usr/lib/gcc/arm-linux-gnueabi/4.7" - cfgCmdList.append('sh') - cfgCmdList.append(os.path.normpath(shell.getJsCfgPath())) - # From mjrosenb: things might go wrong if these three lines are not present for - # compiling ARM on a 64-bit host machine. Not needed if compiling on the board itself. - # cfgCmdList.append('--target=arm-linux-gnueabi') - # cfgCmdList.append('--with-arch=armv7-a') - # cfgCmdList.append('--with-thumb') - if not shell.buildOptions.enableHardFp: - cfgCmdList.append('--target=arm-linux-gnueabi') - elif shell.buildOptions.enable32 and os.name == 'posix': - # 32-bit shell on Mac OS X 10.10 Yosemite and greater - if sps.isMac: - assert sps.macVer() >= [10, 10] # We no longer support 10.9 Mavericks and prior. - # Uses system clang - cfgEnvDt[b"CC"] = cfgEnvDt[b"HOST_CC"] = b"clang %s %s" % (CLANG_PARAMS, SSE2_FLAGS) - cfgEnvDt[b"CXX"] = cfgEnvDt[b"HOST_CXX"] = b"clang++ %s %s" % (CLANG_PARAMS, SSE2_FLAGS) - if shell.buildOptions.buildWithAsan: - cfgEnvDt[b"CC"] += b" " + CLANG_ASAN_PARAMS - cfgEnvDt[b"CXX"] += b" " + CLANG_ASAN_PARAMS - cfgEnvDt[b"CC"] += b" " + CLANG_X86_FLAG # only needed for CC, not HOST_CC - cfgEnvDt[b"CXX"] += b" " + CLANG_X86_FLAG # only needed for CXX, not HOST_CXX - cfgEnvDt[b"RANLIB"] = b"ranlib" - cfgEnvDt[b"AS"] = b"$CC" - cfgEnvDt[b"LD"] = b"ld" - cfgEnvDt[b"STRIP"] = b"strip -x -S" - cfgEnvDt[b"CROSS_COMPILE"] = b"1" - if sps.isProgramInstalled('brew'): - cfgEnvDt[b"AUTOCONF"] = b"/usr/local/Cellar/autoconf213/2.13/bin/autoconf213" - # Hacked up for new and old Homebrew configs, we can probably just call autoconf213 - if not os.path.isfile(sps.normExpUserPath(cfgEnvDt[b"AUTOCONF"])): - cfgEnvDt[b"AUTOCONF"] = b"autoconf213" - cfgCmdList.append('sh') - cfgCmdList.append(os.path.normpath(shell.getJsCfgPath())) - cfgCmdList.append('--target=i386-apple-darwin14.5.0') # Yosemite 10.10.5 - if shell.buildOptions.buildWithAsan: - cfgCmdList.append('--enable-address-sanitizer') - if shell.buildOptions.enableSimulatorArm32: - # --enable-arm-simulator became --enable-simulator=arm in rev 25e99bc12482 - # but unknown flags are ignored, so we compile using both till Fx38 ESR is deprecated - # Newer configure.in changes mean that things blow up if unknown/removed configure - # options are entered, so specify it only if it's requested. - if shell.buildOptions.enableArmSimulatorObsolete: - cfgCmdList.append('--enable-arm-simulator') - cfgCmdList.append('--enable-simulator=arm') - # 32-bit shell on 32/64-bit x86 Linux - elif sps.isLinux and not sps.isARMv7l: - cfgEnvDt[b"PKG_CONFIG_LIBDIR"] = b"/usr/lib/pkgconfig" - if shell.buildOptions.buildWithClang: - cfgEnvDt[b"CC"] = cfgEnvDt[b"HOST_CC"] = str( - "clang %s %s %s" % (CLANG_PARAMS, SSE2_FLAGS, CLANG_X86_FLAG)) - cfgEnvDt[b"CXX"] = cfgEnvDt[b"HOST_CXX"] = str( - "clang++ %s %s %s" % (CLANG_PARAMS, SSE2_FLAGS, CLANG_X86_FLAG)) - else: - # apt-get `lib32z1 gcc-multilib g++-multilib` first, if on 64-bit Linux. - cfgEnvDt[b"CC"] = b"gcc -m32 %s" % SSE2_FLAGS - cfgEnvDt[b"CXX"] = b"g++ -m32 %s" % SSE2_FLAGS - if shell.buildOptions.buildWithAsan: - cfgEnvDt[b"CC"] += b" " + CLANG_ASAN_PARAMS - cfgEnvDt[b"CXX"] += b" " + CLANG_ASAN_PARAMS - cfgCmdList.append('sh') - cfgCmdList.append(os.path.normpath(shell.getJsCfgPath())) - cfgCmdList.append('--target=i686-pc-linux') - if shell.buildOptions.buildWithAsan: - cfgCmdList.append('--enable-address-sanitizer') - if shell.buildOptions.enableSimulatorArm32: - # --enable-arm-simulator became --enable-simulator=arm in rev 25e99bc12482 - # but unknown flags are ignored, so we compile using both till Fx38 ESR is deprecated - # Newer configure.in changes mean that things blow up if unknown/removed configure - # options are entered, so specify it only if it's requested. - if shell.buildOptions.enableArmSimulatorObsolete: - cfgCmdList.append('--enable-arm-simulator') - cfgCmdList.append('--enable-simulator=arm') - else: - cfgCmdList.append('sh') - cfgCmdList.append(os.path.normpath(shell.getJsCfgPath())) - # 64-bit shell on Mac OS X 10.10 Yosemite and greater - elif sps.isMac and sps.macVer() >= [10, 10] and not shell.buildOptions.enable32: - cfgEnvDt[b"CC"] = b"clang " + CLANG_PARAMS - cfgEnvDt[b"CXX"] = b"clang++ " + CLANG_PARAMS - if shell.buildOptions.buildWithAsan: - cfgEnvDt[b"CC"] += b" " + CLANG_ASAN_PARAMS - cfgEnvDt[b"CXX"] += b" " + CLANG_ASAN_PARAMS - if sps.isProgramInstalled('brew'): - cfgEnvDt[b"AUTOCONF"] = b"/usr/local/Cellar/autoconf213/2.13/bin/autoconf213" - cfgCmdList.append('sh') - cfgCmdList.append(os.path.normpath(shell.getJsCfgPath())) - cfgCmdList.append('--target=x86_64-apple-darwin14.5.0') # Yosemite 10.10.5 - if shell.buildOptions.buildWithAsan: - cfgCmdList.append('--enable-address-sanitizer') - if shell.buildOptions.enableSimulatorArm64: - cfgCmdList.append('--enable-simulator=arm64') - - elif sps.isWin: - cfgEnvDt[b"MAKE"] = b"mozmake" # Workaround for bug 948534 - if shell.buildOptions.buildWithClang: - cfgEnvDt[b"CC"] = b"clang-cl.exe " + CLANG_PARAMS - cfgEnvDt[b"CXX"] = b"clang-cl.exe " + CLANG_PARAMS - if shell.buildOptions.buildWithAsan: - cfgEnvDt[b"CFLAGS"] = CLANG_ASAN_PARAMS - cfgEnvDt[b"CXXFLAGS"] = CLANG_ASAN_PARAMS - cfgEnvDt[b"LDFLAGS"] = (b"clang_rt.asan_dynamic-x86_64.lib " - b"clang_rt.asan_dynamic_runtime_thunk-x86_64.lib " - b"clang_rt.asan_dynamic-x86_64.dll") - cfgEnvDt[b"HOST_CFLAGS"] = b" " - cfgEnvDt[b"HOST_CXXFLAGS"] = b" " - cfgEnvDt[b"HOST_LDFLAGS"] = b" " - cfgEnvDt[b"LIB"] += br"C:\Program Files\LLVM\lib\clang\4.0.0\lib\windows" - cfgCmdList.append('sh') - cfgCmdList.append(os.path.normpath(shell.getJsCfgPath())) - if shell.buildOptions.enable32: - if shell.buildOptions.enableSimulatorArm32: - # --enable-arm-simulator became --enable-simulator=arm in rev 25e99bc12482 - # but unknown flags are ignored, so we compile using both till Fx38 ESR is deprecated - # Newer configure.in changes mean that things blow up if unknown/removed configure - # options are entered, so specify it only if it's requested. - if shell.buildOptions.enableArmSimulatorObsolete: - cfgCmdList.append('--enable-arm-simulator') - cfgCmdList.append('--enable-simulator=arm') - else: - cfgCmdList.append('--host=x86_64-pc-mingw32') - cfgCmdList.append('--target=x86_64-pc-mingw32') - if shell.buildOptions.enableSimulatorArm64: - cfgCmdList.append('--enable-simulator=arm64') - if shell.buildOptions.buildWithAsan: - cfgCmdList.append('--enable-address-sanitizer') - else: - # We might still be using GCC on Linux 64-bit, so do not use clang unless Asan is specified - if shell.buildOptions.buildWithClang: - cfgEnvDt[b"CC"] = b"clang " + CLANG_PARAMS - cfgEnvDt[b"CXX"] = b"clang++ " + CLANG_PARAMS - if shell.buildOptions.buildWithAsan: - cfgEnvDt[b"CC"] += b" " + CLANG_ASAN_PARAMS - cfgEnvDt[b"CXX"] += b" " + CLANG_ASAN_PARAMS - cfgCmdList.append('sh') - cfgCmdList.append(os.path.normpath(shell.getJsCfgPath())) - if shell.buildOptions.buildWithAsan: - cfgCmdList.append('--enable-address-sanitizer') - - if shell.buildOptions.buildWithClang: - if sps.isWin: - assert b"clang-cl" in cfgEnvDt[b"CC"] - assert b"clang-cl" in cfgEnvDt[b"CXX"] - else: - assert b"clang" in cfgEnvDt[b"CC"] - assert b"clang++" in cfgEnvDt[b"CXX"] - cfgCmdList.append('--disable-jemalloc') # See bug 1146895 - - if shell.buildOptions.enableDbg: - cfgCmdList.append('--enable-debug') - elif shell.buildOptions.disableDbg: - cfgCmdList.append('--disable-debug') - - if shell.buildOptions.enableOpt: - cfgCmdList.append('--enable-optimize' + ('=-O1' if shell.buildOptions.buildWithVg else '')) - elif shell.buildOptions.disableOpt: - cfgCmdList.append('--disable-optimize') - if shell.buildOptions.enableProfiling: # Now obsolete, retained for backward compatibility - cfgCmdList.append('--enable-profiling') - if shell.buildOptions.disableProfiling: - cfgCmdList.append('--disable-profiling') - - if shell.buildOptions.enableMoreDeterministic: - # Fuzzing tweaks for more useful output, implemented in bug 706433 - cfgCmdList.append('--enable-more-deterministic') - if shell.buildOptions.enableOomBreakpoint: # Extra debugging help for OOM assertions - cfgCmdList.append('--enable-oom-breakpoint') - if shell.buildOptions.enableWithoutIntlApi: # Speeds up compilation but is non-default - cfgCmdList.append('--without-intl-api') - - if shell.buildOptions.buildWithVg: - cfgCmdList.append('--enable-valgrind') - cfgCmdList.append('--disable-jemalloc') - - # We add the following flags by default. - if os.name == 'posix': - cfgCmdList.append('--with-ccache') - cfgCmdList.append('--enable-gczeal') - cfgCmdList.append('--enable-debug-symbols') # gets debug symbols on opt shells - cfgCmdList.append('--disable-tests') - - if os.name == 'nt': - # FIXME: Replace this with sps.shellify. - counter = 0 - for entry in cfgCmdList: - if os.sep in entry: - assert sps.isWin # MozillaBuild on Windows sometimes confuses "/" and "\". - cfgCmdList[counter] = cfgCmdList[counter].replace(os.sep, '//') - counter = counter + 1 - - # Print whatever we added to the environment - envVarList = [] - for envVar in set(cfgEnvDt.keys()) - set(origCfgEnvDt.keys()): - strToBeAppended = str(envVar + '="' + cfgEnvDt[str(envVar)] + - '"' if " " in cfgEnvDt[str(envVar)] else envVar + - "=" + cfgEnvDt[str(envVar)]) - envVarList.append(strToBeAppended) - sps.vdump('Command to be run is: ' + sps.shellify(envVarList) + ' ' + sps.shellify(cfgCmdList)) - - wDir = shell.getJsObjdir() - assert os.path.isdir(wDir) - - if sps.isWin: - changedCfgCmdList = [] - for entry in cfgCmdList: - # For JS, quoted from :glandium: "the way icu subconfigure is called is what changed. - # but really, the whole thing likes forward slashes way better" - # See bug 1038590 comment 9. - if '\\' in entry: - entry = entry.replace('\\', '/') - changedCfgCmdList.append(entry) - sps.captureStdout(changedCfgCmdList, ignoreStderr=True, currWorkingDir=wDir, env=cfgEnvDt) - else: - sps.captureStdout(cfgCmdList, ignoreStderr=True, currWorkingDir=wDir, env=cfgEnvDt) - - shell.setEnvAdded(envVarList) - shell.setEnvFull(cfgEnvDt) - shell.setCfgCmdExclEnv(cfgCmdList) - - -def compileJs(shell): - """Compile and copy a binary.""" - try: - cmdList = [MAKE_BINARY, '-C', shell.getJsObjdir(), '-j' + str(COMPILATION_JOBS), '-s'] - out = sps.captureStdout(cmdList, combineStderr=True, ignoreExitCode=True, - currWorkingDir=shell.getJsObjdir(), env=shell.getEnvFull())[0] - except Exception as e: - # This exception message is returned from sps.captureStdout via cmdList. - if (sps.isLinux or sps.isMac) and \ - ('GCC running out of memory' in repr(e) or 'Clang running out of memory' in repr(e)): - # FIXME: Absolute hack to retry after hitting OOM. - print("Trying once more due to the compiler running out of memory...") - out = sps.captureStdout(cmdList, combineStderr=True, ignoreExitCode=True, - currWorkingDir=shell.getJsObjdir(), env=shell.getEnvFull())[0] - # A non-zero error can be returned during make, but eventually a shell still gets compiled. - if os.path.exists(shell.getShellCompiledPath()): - print("A shell was compiled even though there was a non-zero exit code. Continuing...") - else: - print("%s did not result in a js shell:" % MAKE_BINARY.decode("utf-8", errors="replace")) - raise - - if os.path.exists(shell.getShellCompiledPath()): - shutil.copy2(shell.getShellCompiledPath(), shell.getShellCacheFullPath()) - for runLib in shell.getShellCompiledRunLibsPath(): - if os.path.isfile(runLib): - shutil.copy2(runLib, shell.getShellCacheDir()) - - version = extractVersions(shell.getJsObjdir()) - shell.setMajorVersion(version.split('.')[0]) - shell.setVersion(version) - - if sps.isLinux: - # Restrict this to only Linux for now. At least Mac OS X needs some (possibly *.a) - # files in the objdir or else the stacks from failing testcases will lack symbols. - shutil.rmtree(sps.normExpUserPath(os.path.join(shell.getShellCacheDir(), 'objdir-js'))) - else: - print(out.decode("utf-8", errors="replace")) - raise Exception(MAKE_BINARY + " did not result in a js shell, no exception thrown.") - - -def createBustedFile(filename, e): - """Create a .busted file with the exception message and backtrace included.""" - with open(filename, 'wb') as f: - f.write("Caught exception %s (%s)\n" % (repr(e), str(e))) - f.write("Backtrace:\n") - f.write(traceback.format_exc() + "\n") - print("Compilation failed (%s) (details in %s)" % (e.decode("utf-8", errors="replace"), - filename.decode("utf-8", errors="replace"))) - - -def envDump(shell, log): - """Dump environment to a .fuzzmanagerconf file.""" - # Platform and OS detection for the spec, part of which is in: - # https://wiki.mozilla.org/Security/CrashSignatures - if sps.isARMv7l: - fmconfPlatform = 'ARM' - elif sps.isARMv7l and not shell.buildOptions.enable32: - print("ARM64 is not supported in .fuzzmanagerconf yet.") - fmconfPlatform = 'ARM64' - elif shell.buildOptions.enable32: - fmconfPlatform = 'x86' - else: - fmconfPlatform = 'x86-64' - - if sps.isLinux: - fmconfOS = 'linux' - elif sps.isMac: - fmconfOS = 'macosx' - elif sps.isWin: - fmconfOS = 'windows' - - with open(log, 'ab') as f: - f.write('# Information about shell:\n# \n') - - f.write('# Create another shell in shell-cache like this one:\n') - f.write('# python -u %s -b "%s" -r %s\n# \n' % ('~/funfuzz/js/compileShell.py', - shell.buildOptions.buildOptionsStr, shell.getHgHash())) - - f.write('# Full environment is:\n') - f.write('# %s\n# \n' % str(shell.getEnvFull())) - - f.write('# Full configuration command with needed environment variables is:\n') - f.write('# %s %s\n# \n' % (sps.shellify(shell.getEnvAdded()), - sps.shellify(shell.getCfgCmdExclEnv()))) - - # .fuzzmanagerconf details - f.write('\n') - f.write('[Main]\n') - f.write('platform = %s\n' % fmconfPlatform) - f.write('product = %s\n' % shell.getRepoName()) - f.write('product_version = %s\n' % shell.getHgHash()) - f.write('os = %s\n' % fmconfOS) - - f.write('\n') - f.write('[Metadata]\n') - f.write('buildFlags = %s\n' % shell.buildOptions.buildOptionsStr) - f.write('majorVersion = %s\n' % shell.getMajorVersion()) - f.write('pathPrefix = %s%s\n' % (shell.getRepoDir(), - '/' if not shell.getRepoDir().endswith('/') else '')) - f.write('version = %s\n' % shell.getVersion()) - - -def extractVersions(objdir): - """Extract the version from js.pc and put it into *.fuzzmanagerconf.""" - jspcDir = sps.normExpUserPath(os.path.join(objdir, 'js', 'src')) - jspcFilename = os.path.join(jspcDir, 'js.pc') - # Moved to /js/src/build/, see bug 1262241, Fx55 rev 2159959522f4 - jspcNewDir = os.path.join(jspcDir, 'build') - jspcNewFilename = os.path.join(jspcNewDir, 'js.pc') - - def fixateVer(pcfile): - """Returns the current version number (47.0a2).""" - with io.open(pcfile, mode='r', encoding="utf-8", errors="replace") as f: - for line in f: - if line.startswith('Version: '): - # Sample line: 'Version: 47.0a2' - return line.split(': ')[1].rstrip() - - if os.path.isfile(jspcFilename): - return fixateVer(jspcFilename) - elif os.path.isfile(jspcNewFilename): - return fixateVer(jspcNewFilename) - - -def getLockDirPath(repoDir, tboxIdentifier=''): - """Return the name of the lock directory, which is in the cache directory by default.""" - lockDirNameList = ['shell', os.path.basename(repoDir), 'lock'] - if tboxIdentifier: - lockDirNameList.append(tboxIdentifier) - return os.path.join(ensureCacheDir(), '-'.join(lockDirNameList)) - - -def makeTestRev(options): - def testRev(rev): - shell = CompiledShell(options.buildOptions, rev) - print("Rev %s:" % rev.decode("utf-8", errors="replace"), end=" ") - - try: - obtainShell(shell, updateToRev=rev) - except Exception: - return (options.compilationFailedLabel, 'compilation failed') - - print("Testing...", end=" ") - return options.testAndLabel(shell.getShellCacheFullPath(), rev) - return testRev - - -def obtainShell(shell, updateToRev=None, updateLatestTxt=False): - """Obtain a js shell. Keep the objdir for now, especially .a files, for symbols.""" - assert os.path.isdir(getLockDirPath(shell.buildOptions.repoDir)) - cachedNoShell = shell.getShellCacheFullPath() + ".busted" - - if os.path.isfile(shell.getShellCacheFullPath()): - # Don't remove the comma at the end of this line, and thus remove the newline printed. - # We would break JSBugMon. - print("Found cached shell...") - # Assuming that since the binary is present, everything else (e.g. symbols) is also present - verifyFullWinPageHeap(shell.getShellCacheFullPath()) - return - elif os.path.isfile(cachedNoShell): - raise Exception("Found a cached shell that failed compilation...") - elif os.path.isdir(shell.getShellCacheDir()): - print("Found a cache dir without a successful/failed shell...") - sps.rmTreeIncludingReadOnly(shell.getShellCacheDir()) - - os.mkdir(shell.getShellCacheDir()) - hgCmds.destroyPyc(shell.buildOptions.repoDir) - - s3CacheObj = s3cache.S3Cache(S3_SHELL_CACHE_DIRNAME) - useS3Cache = s3CacheObj.connect() - - if useS3Cache: - if s3CacheObj.downloadFile(shell.getShellNameWithoutExt() + '.busted', - shell.getShellCacheFullPath() + '.busted'): - raise Exception('Found a .busted file for rev ' + shell.getHgHash()) - - if s3CacheObj.downloadFile(shell.getShellNameWithoutExt() + '.tar.bz2', - shell.getS3TarballWithExtFullPath()): - print("Extracting shell...") - with tarfile.open(shell.getS3TarballWithExtFullPath(), 'r') as z: - z.extractall(shell.getShellCacheDir()) - # Delete tarball after downloading from S3 - os.remove(shell.getS3TarballWithExtFullPath()) - verifyFullWinPageHeap(shell.getShellCacheFullPath()) - return - - try: - if updateToRev: - updateRepo(shell.buildOptions.repoDir, updateToRev) - if shell.buildOptions.patchFile: - hgCmds.patchHgRepoUsingMq(shell.buildOptions.patchFile, shell.getRepoDir()) - - cfgJsCompile(shell) - verifyFullWinPageHeap(shell.getShellCacheFullPath()) - except KeyboardInterrupt: - sps.rmTreeIncludingReadOnly(shell.getShellCacheDir()) - raise - except Exception as e: - # Remove the cache dir, but recreate it with only the .busted file. - sps.rmTreeIncludingReadOnly(shell.getShellCacheDir()) - os.mkdir(shell.getShellCacheDir()) - createBustedFile(cachedNoShell, e) - if useS3Cache: - s3CacheObj.uploadFileToS3(shell.getShellCacheFullPath() + '.busted') - raise - finally: - if shell.buildOptions.patchFile: - hgCmds.hgQpopQrmAppliedPatch(shell.buildOptions.patchFile, shell.getRepoDir()) - - if useS3Cache: - s3CacheObj.compressAndUploadDirTarball(shell.getShellCacheDir(), shell.getS3TarballWithExtFullPath()) - if updateLatestTxt: - # So js-dbg-64-dm-darwin-cdcd33fd6e39 becomes js-dbg-64-dm-darwin-latest.txt with - # js-dbg-64-dm-darwin-cdcd33fd6e39 as its contents. - txtInfo = '-'.join(shell.getS3TarballWithExt().split('-')[:-1] + ['latest']) + '.txt' - s3CacheObj.uploadStrToS3('', txtInfo, shell.getS3TarballWithExt()) - os.remove(shell.getS3TarballWithExtFullPath()) - - -def updateRepo(repo, rev): - """Update repository to the specific revision.""" - # Print *with* a trailing newline to avoid breaking other stuff - print("Updating to rev %s in the %s repository..." % (rev.decode("utf-8", errors="replace"), - repo.decode("utf-8", errors="replace"))) - sps.captureStdout(["hg", "-R", repo, 'update', '-C', '-r', rev], ignoreStderr=True) - - -def verifyFullWinPageHeap(shellPath): - """Turn on full page heap verification on Windows.""" - # More info: https://msdn.microsoft.com/en-us/library/windows/hardware/ff543097(v=vs.85).aspx - # or https://blogs.msdn.microsoft.com/webdav_101/2010/06/22/detecting-heap-corruption-using-gflags-and-dumps/ - if sps.isWin: - gflagsBin = os.path.join(os.getenv('PROGRAMW6432'), 'Debugging Tools for Windows (x64)', 'gflags.exe') - if os.path.isfile(gflagsBin) and os.path.isfile(shellPath): - print(subprocess.check_output([gflagsBin.decode("utf-8", errors="replace"), - "-p", "/enable", shellPath.decode("utf-8", errors="replace"), "/full"])) - - -def main(): - """Build a shell and place it in the autoBisect cache.""" - usage = 'Usage: %prog [options]' - parser = OptionParser(usage) - parser.disable_interspersed_args() - - parser.set_defaults( - buildOptions="", - ) - - # Specify how the shell will be built. - # See buildOptions.py for details. - parser.add_option('-b', '--build', - dest='buildOptions', - help="Specify build options, e.g. -b '--disable-debug --enable-optimize' " - "(python buildOptions.py --help)") - - parser.add_option('-r', '--rev', - dest='revision', - help='Specify revision to build') - - options = parser.parse_args()[0] - options.buildOptions = buildOptions.parseShellOptions(options.buildOptions) - - with LockDir(getLockDirPath(options.buildOptions.repoDir)): - if options.revision: - shell = CompiledShell(options.buildOptions, options.revision) - else: - localOrigHgHash = hgCmds.getRepoHashAndId(options.buildOptions.repoDir)[0] - shell = CompiledShell(options.buildOptions, localOrigHgHash) - - obtainShell(shell, updateToRev=options.revision) - print(shell.getShellCacheFullPath()) - - -if __name__ == '__main__': - main() diff --git a/js/faq-compileShell.md b/js/faq-compileShell.md deleted file mode 100644 index c0c087911..000000000 --- a/js/faq-compileShell.md +++ /dev/null @@ -1,50 +0,0 @@ -### FAQ: - -**Q: Why is "--enable-more-deterministic" recommended?** - -Fuzzing with this mode on allows us to run compareJIT, which runs testcases generated by jsfunfuzz using different flags and compares the output. In order to compare successfully, the shell should generate consistent output everytime with a fixed input. Since we do not ship deterministic shells by default, if testing deterministic shells, we do not run with compareJIT. - -**Q: What are average build times for SpiderMonkey?** - -On a decent Linux machine or a powerful Mac, both with 4 or more cores, 3-4 minutes on average. On Windows, probably 5-10 minutes. On an ARM ODROID board, up to an hour. - -**Q: How do I get a shell with a patch to be compiled together?** - -Use the -P notation, e.g.: - -`funfuzz/js/compileShell.py -b "--enable-debug --enable-more-deterministic -R ~/trees/mozilla-inbound -P ~/patch.diff"` - -assuming: -* mq is activated in ~/.hgrc -* There are no other patches in the mq stack -* Patch is in ~/patch.diff and can be applied cleanly - * Test by first doing `patch -p1 --dry-run < ~/patch.diff` in the base directory of the repository listed by -R, or the default. -* There is only one patch needed. If more than one patch is needed, first do a roll-up patch. - -**Q: Do these build configure flags get passed into the js configure scripts?** - -No, they are independent. We only implemented the flags that are most useful for fuzzing in the harness. - -**Q: Will the gecko-dev Git mirror of mozilla-central be supported?** - -The "-R" flag assumes a Mercurial clone of mozilla-central is passed in as an argument. Git repositories are not yet supported fully, and especially not for autoBisect. See issue #2. - -**Q: Can I run multiple instances of compileShell?** - -This is not recommended as it will slow down your computer. Running one instance of compileShell will use the maximum number of cores as found by cpu_count() from the Python multiprocessing module, with the exception of ARM boards that tend to have slower cores. - -**Q: What kind of build does compileShell do, and what files do it store on my machine?** - -It creates a clobber build, compiling in the ~/shell-cache directory by default. There may also be a bunch of tempfiles created in the system temporary directory. - -**Q: How do I check if the SpiderMonkey build created is the one I specified?** - -Post-compilation, the harness does a bunch of verification tests to ensure that the desired build is created. To double check, run `getBuildConfiguration();` within the SpiderMonkey shell. If compileShell isn't compiling as desired, file an issue! - -**Q: What happens if the build I want to compile causes a compilation error?** - -A .busted file is created in the shell-cache directory. This will notify the harness not to retry in the future since it is busted. However, if there is an error in the harness, file an issue detailing the build configurations and steps to reproduce, and once it is fixed, remove the corresponding file/directory in ~/shell-cache and retry again. - -**Q: After compiling many shells, I'm now running out of disk space! What should I do?** - -Oops! You can remove the ~/shell-cache directory to reclaim space, and reboot to clear system temporary directories. diff --git a/loopBot.py b/loopBot.py deleted file mode 100755 index 5d306eb15..000000000 --- a/loopBot.py +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env python -# coding=utf-8 -# pylint: disable=invalid-name,missing-docstring -# -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at https://mozilla.org/MPL/2.0/. - -# Loop of { update repos, call bot.py } to allow things to run unattended -# All command-line options are passed through to bot.py - -# Since this script updates the fuzzing repo, it should be very simple, and use subprocess.call() rather than import - -from __future__ import absolute_import, print_function - -import os -import sys -import subprocess -import time - -path0 = os.path.dirname(os.path.abspath(__file__)) - -# Config-ish bits should move to bot.py, OR move into a config file, -# OR this file should subprocess-call ITSELF rather than using a while loop. - - -def loopSequence(cmdSequence, waitTime): - """Call a sequence of commands in a loop. - If any fails, sleep(waitTime) and go back to the beginning of the sequence.""" - i = 0 - while True: - i += 1 - print("localLoop #%d!" % i) - for cmd in cmdSequence: - try: - subprocess.check_call(cmd) - except subprocess.CalledProcessError as e: - print("Something went wrong when calling: %r" % (cmd,)) - print("%r" % (e,)) - import traceback - print(traceback.format_exc()) - print("Waiting %d seconds..." % waitTime) - time.sleep(waitTime) - break - - -def main(): - loopSequence([ - [sys.executable, "-u", os.path.join(path0, 'util', 'reposUpdate.py')], - [sys.executable, "-u", os.path.join(path0, 'bot.py')] + sys.argv[1:] - ], 60) - - -if __name__ == "__main__": - main() diff --git a/requirements.txt b/requirements.txt index 012a4499d..da2d2992c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,2 @@ -git+https://github.com/MozillaSecurity/FuzzManager.git -git+https://github.com/MozillaSecurity/lithium.git +FuzzManager>=0.1.1 +lithium-reducer>=0.2.0 diff --git a/setup.py b/setup.py new file mode 100644 index 000000000..91305744e --- /dev/null +++ b/setup.py @@ -0,0 +1,36 @@ +# coding=utf-8 +# pylint: disable=missing-docstring +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +from setuptools import setup + +if __name__ == "__main__": + setup(name="funfuzz", + version="0.2.0", + entry_points={ + "console_scripts": ["funfuzz = funfuzz.bot:main"] + }, + packages=[ + "funfuzz", + "funfuzz.autobisectjs", + "funfuzz.js", + "funfuzz.util", + "funfuzz.util.tooltool", + ], + package_data={"funfuzz": [ + "autobisectjs/*", + "js/*", + "js/jsfunfuzz/*", + "js/shared/*", + "util/*", + "util/tooltool/*", + ]}, + package_dir={"": "src"}, + install_requires=[ + "FuzzManager>=0.1.1", + "lithium-reducer>=0.2.0", + ], + zip_safe=False) diff --git a/src/funfuzz/__init__.py b/src/funfuzz/__init__.py new file mode 100644 index 000000000..5161d32e0 --- /dev/null +++ b/src/funfuzz/__init__.py @@ -0,0 +1,12 @@ +# coding=utf-8 +# flake8: noqa +# pylint: disable=missing-docstring +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import + +from . import bot +from . import loop_bot diff --git a/src/funfuzz/__main__.py b/src/funfuzz/__main__.py new file mode 100644 index 000000000..43869008a --- /dev/null +++ b/src/funfuzz/__main__.py @@ -0,0 +1,12 @@ +# coding=utf-8 +# pylint: disable=missing-docstring +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import + +from .bot import main + +main() diff --git a/src/funfuzz/autobisectjs/README.md b/src/funfuzz/autobisectjs/README.md new file mode 100644 index 000000000..5fb38c5ff --- /dev/null +++ b/src/funfuzz/autobisectjs/README.md @@ -0,0 +1,35 @@ +autoBisect will help you to find out when a changeset introduced problems. It can also point at a changeset that may have exposed the issue. + +It helps with work allocation: + +* The engineer that most recently worked on the code is the one most likely to know how to fix the bug. +* If not, the engineer may be able to forward to someone more knowledgeable. + +## Find changeset that introduced problems using autoBisect + +For SpiderMonkey, use the following while compiling locally: + +`python -m funfuzz.autobisectjs.autobisectjs -p "--fuzzing-safe --no-threads --ion-eager testcase.js" -b "--enable-debug --enable-more-deterministic"` + +assuming the testcase requires "--fuzzing-safe --no-threads --ion-eager" as runtime flags. + +This will take about: + +* **45 - 60 minutes** on a relatively recent powerful computer on Linux / Mac + * assuming each compilation takes about 3 minutes + * we should be able to find the problem within 16+ tests. +* **2 hours** on Windows + * where each compilation is assumed to take 6 minutes. + +If you have an internet connection, and the testcase causes problems with: + +* a [downloaded js shell](https://archive.mozilla.org/pub/mozilla.org/firefox/tinderbox-builds/mozilla-central-macosx64-debug/latest/jsshell-mac64.zip) +* these problems started happening within the last month + +you can try bisecting using downloaded builds: + +`python -m funfuzz.autobisectjs.autobisectjs -p "--fuzzing-safe --no-threads --ion-eager testcase.js" -b "--enable-debug" -T` + +This should take < 5 minutes total assuming a fast internet connection, since it does not need to compile shells. + +Refer to [compile_shell documentation](../js/README.md) for parameters to be passed into "-b". diff --git a/src/funfuzz/autobisectjs/__init__.py b/src/funfuzz/autobisectjs/__init__.py new file mode 100644 index 000000000..50428ca67 --- /dev/null +++ b/src/funfuzz/autobisectjs/__init__.py @@ -0,0 +1,13 @@ +# coding=utf-8 +# flake8: noqa +# pylint: disable=missing-docstring +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import + +from . import autobisectjs +from . import find_intersecting_changesets +from . import known_broken_earliest_working diff --git a/src/funfuzz/autobisectjs/__main__.py b/src/funfuzz/autobisectjs/__main__.py new file mode 100644 index 000000000..2c635face --- /dev/null +++ b/src/funfuzz/autobisectjs/__main__.py @@ -0,0 +1,18 @@ +# coding=utf-8 +# pylint: disable=missing-docstring +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import + +import sys + +from .autobisectjs import main +from ..util import subprocesses as sps + +if __name__ == '__main__': + # Reopen stdout, unbuffered. This is similar to -u. From http://stackoverflow.com/a/107717 + sys.stdout = sps.Unbuffered(sys.stdout) + main() diff --git a/autobisect-js/autoBisect.py b/src/funfuzz/autobisectjs/autobisectjs.py old mode 100755 new mode 100644 similarity index 79% rename from autobisect-js/autoBisect.py rename to src/funfuzz/autobisectjs/autobisectjs.py index 5ec70c109..7976b1630 --- a/autobisect-js/autoBisect.py +++ b/src/funfuzz/autobisectjs/autobisectjs.py @@ -1,13 +1,14 @@ -#!/usr/bin/env python # coding=utf-8 -# pylint: disable=broad-except,import-error,invalid-name,invalid-unary-operand-type,literal-comparison -# pylint: disable=missing-docstring,too-many-arguments,too-many-boolean-expressions,too-many-branches -# pylint: disable=too-many-locals,too-many-return-statements,too-many-statements,wrong-import-position # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at https://mozilla.org/MPL/2.0/. +"""autobisectjs, for bisecting changeset regression windows. Supports Mercurial repositories and SpiderMonkey only. + +May be replaced in the future, with a better version that supports both Firefox and SpiderMonkey. +""" + from __future__ import absolute_import, print_function import math @@ -15,36 +16,30 @@ import os import re import shutil -import stat +import stat # Fixed after pylint 1.7.2 was released pylint: disable=bad-python3-import import subprocess -import sys import time from optparse import OptionParser # pylint: disable=deprecated-module from lithium.interestingness.utils import rel_or_abs_import -import knownBrokenEarliestWorking as kbew - -path0 = os.path.dirname(os.path.abspath(__file__)) -path2 = os.path.abspath(os.path.join(path0, os.pardir, 'js')) -sys.path.append(path2) -import compileShell -import inspectShell -path4 = os.path.abspath(os.path.join(path0, os.pardir, 'util')) -sys.path.append(path4) -import fileManipulation -import buildOptions -import downloadBuild -import hgCmds -import s3cache -import subprocesses as sps -import LockDir +from . import known_broken_earliest_working as kbew +from ..js import build_options +from ..js import compile_shell +from ..js import inspect_shell +from ..util import file_manipulation +from ..util import download_build +from ..util import hg_helpers +from ..util import s3cache +from ..util import subprocesses as sps +from ..util import LockDir INCOMPLETE_NOTE = 'incompleteBuild.txt' MAX_ITERATIONS = 100 -def parseOpts(): +def parseOpts(): # pylint: disable=invalid-name,missing-docstring,missing-return-doc,missing-return-type-doc + # pylint: disable=too-many-branches,too-complex,too-many-statements usage = 'Usage: %prog [options]' parser = OptionParser(usage) # http://docs.python.org/library/optparse.html#optparse.OptionParser.disable_interspersed_args @@ -60,17 +55,16 @@ def parseOpts(): useInterestingnessTests=False, parameters='-e 42', # http://en.wikipedia.org/wiki/The_Hitchhiker%27s_Guide_to_the_Galaxy compilationFailedLabel='skip', - buildOptions="", + build_options="", useTreeherderBinaries=False, nameOfTreeherderBranch='mozilla-inbound', ) # Specify how the shell will be built. - # See buildOptions.py for details. parser.add_option('-b', '--build', - dest='buildOptions', + dest='build_options', help='Specify js shell build options, e.g. -b "--enable-debug --32"' - "(python buildOptions.py --help)") + "(python -m funfuzz.js.build_options --help)") parser.add_option('-B', '--browser', dest='browserOptions', help='Specify browser build options, e.g. -b "-c mozconfig". Deprecated.') @@ -109,7 +103,7 @@ def parseOpts(): help='Specify parameters for the js shell, e.g. -p "-a --ion-eager testcase.js".') # Specify how to treat revisions that fail to compile. - # (You might want to add these to kbew.knownBrokenRanges in knownBrokenEarliestWorking.py.) + # (You might want to add these to kbew.knownBrokenRanges in known_broken_earliest_working.) parser.add_option('-l', '--compilationFailedLabel', dest='compilationFailedLabel', help="Specify how to treat revisions that fail to compile. " "(bad, good, or skip) Defaults to '%default'") @@ -125,8 +119,8 @@ def parseOpts(): (options, args) = parser.parse_args() if not options.browserOptions: - options.buildOptions = buildOptions.parseShellOptions(options.buildOptions) - options.skipRevs = ' + '.join(kbew.knownBrokenRanges(options.buildOptions)) + options.build_options = build_options.parseShellOptions(options.build_options) + options.skipRevs = ' + '.join(kbew.known_broken_ranges(options.build_options)) options.paramList = [sps.normExpUserPath(x) for x in options.parameters.split(' ') if x] # First check that the testcase is present. @@ -138,16 +132,16 @@ def parseOpts(): assert options.compilationFailedLabel in ('bad', 'good', 'skip') - extraFlags = [] + extraFlags = [] # pylint: disable=invalid-name if options.useInterestingnessTests: if len(args) < 1: print("args are: %s" % args) parser.error('Not enough arguments.') if not options.browserOptions: - for a in args: + for a in args: # pylint: disable=invalid-name if a.startswith("--flags="): - extraFlags = a[8:].split(' ') + extraFlags = a[8:].split(' ') # pylint: disable=invalid-name options.testAndLabel = externalTestAndLabel(options, args) else: assert not options.browserOptions # autoBisect doesn't have a built-in way to run the browser @@ -156,24 +150,25 @@ def parseOpts(): options.testAndLabel = internalTestAndLabel(options) if not options.browserOptions: - earliestKnownQuery = kbew.earliestKnownWorkingRev( - options.buildOptions, options.paramList + extraFlags, options.skipRevs) + earliestKnownQuery = kbew.earliest_known_working_rev( # pylint: disable=invalid-name + options.build_options, options.paramList + extraFlags, options.skipRevs) - earliestKnown = '' + earliestKnown = '' # pylint: disable=invalid-name if not options.useTreeherderBinaries: - earliestKnown = hgCmds.getRepoHashAndId(options.buildOptions.repoDir, repoRev=earliestKnownQuery)[0] + # pylint: disable=invalid-name + earliestKnown = hg_helpers.getRepoHashAndId(options.build_options.repoDir, repoRev=earliestKnownQuery)[0] if options.startRepo is None: if options.useTreeherderBinaries: options.startRepo = 'default' else: options.startRepo = earliestKnown - # elif not (options.useTreeherderBinaries or hgCmds.isAncestor(options.buildOptions.repoDir, + # elif not (options.useTreeherderBinaries or hg_helpers.isAncestor(options.build_options.repoDir, # earliestKnown, options.startRepo)): # raise Exception('startRepo is not a descendant of kbew.earliestKnownWorkingRev for this configuration') # - # if not options.useTreeherderBinaries and not hgCmds.isAncestor(options.buildOptions.repoDir, + # if not options.useTreeherderBinaries and not hg_helpers.isAncestor(options.build_options.repoDir, # earliestKnown, options.endRepo): # raise Exception('endRepo is not a descendant of kbew.earliestKnownWorkingRev for this configuration') @@ -187,14 +182,17 @@ def parseOpts(): return options -def findBlamedCset(options, repoDir, testRev): +def findBlamedCset(options, repoDir, testRev): # pylint: disable=invalid-name,missing-docstring,too-complex + # pylint: disable=too-many-locals,too-many-statements print("%s | Bisecting on: %s" % (time.asctime(), repoDir)) - hgPrefix = ['hg', '-R', repoDir] + hgPrefix = ['hg', '-R', repoDir] # pylint: disable=invalid-name # Resolve names such as "tip", "default", or "52707" to stable hg hash ids, e.g. "9f2641871ce8". - realStartRepo = sRepo = hgCmds.getRepoHashAndId(repoDir, repoRev=options.startRepo)[0] - realEndRepo = eRepo = hgCmds.getRepoHashAndId(repoDir, repoRev=options.endRepo)[0] + # pylint: disable=invalid-name + realStartRepo = sRepo = hg_helpers.getRepoHashAndId(repoDir, repoRev=options.startRepo)[0] + # pylint: disable=invalid-name + realEndRepo = eRepo = hg_helpers.getRepoHashAndId(repoDir, repoRev=options.endRepo)[0] sps.vdump("Bisecting in the range " + sRepo + ":" + eRepo) # Refresh source directory (overwrite all local changes) to default tip if required. @@ -216,7 +214,7 @@ def findBlamedCset(options, repoDir, testRev): labels[sRepo] = ('good', 'assumed start rev is good') labels[eRepo] = ('bad', 'assumed end rev is bad') subprocess.check_call(hgPrefix + ['bisect', '-U', '-g', sRepo]) - currRev = hgCmds.getCsetHashFromBisectMsg(fileManipulation.firstLine( + currRev = hg_helpers.getCsetHashFromBisectMsg(file_manipulation.firstLine( sps.captureStdout(hgPrefix + ['bisect', '-U', '-b', eRepo])[0])) iterNum = 1 @@ -267,16 +265,19 @@ def findBlamedCset(options, repoDir, testRev): sps.vdump("Resetting working directory") sps.captureStdout(hgPrefix + ['update', '-C', '-r', 'default'], ignoreStderr=True) - hgCmds.destroyPyc(repoDir) + hg_helpers.destroyPyc(repoDir) print(time.asctime()) -def internalTestAndLabel(options): +def internalTestAndLabel(options): # pylint: disable=invalid-name,missing-param-doc,missing-return-doc + # pylint: disable=missing-return-type-doc,missing-type-doc,too-complex """Use autoBisectJs without interestingness tests to examine the revision of the js shell.""" - def inner(shellFilename, _hgHash): - (stdoutStderr, exitCode) = inspectShell.testBinary(shellFilename, options.paramList, - options.buildOptions.runWithVg) + def inner(shellFilename, _hgHash): # pylint: disable=invalid-name,missing-docstring,missing-return-doc + # pylint: disable=missing-return-type-doc,too-many-return-statements + # pylint: disable=invalid-name + (stdoutStderr, exitCode) = inspect_shell.testBinary(shellFilename, options.paramList, + options.build_options.runWithVg) if (stdoutStderr.find(options.output) != -1) and (options.output != ''): return ('bad', 'Specified-bad output') @@ -296,11 +297,11 @@ def inner(shellFilename, _hgHash): return ('bad', 'Negative exit code ' + str(exitCode)) elif exitCode == 0: return ('good', 'Exit code 0') - elif (exitCode == 1 or exitCode == 2) and (options.output != '') and \ - (stdoutStderr.find('usage: js [') != -1 or - stdoutStderr.find('Error: Short option followed by junk') != -1 or - stdoutStderr.find('Error: Invalid long option:') != -1 or - stdoutStderr.find('Error: Invalid short option:') != -1): + elif (exitCode == 1 or exitCode == 2) and ( # pylint: disable=too-many-boolean-expressions + options.output != '') and (stdoutStderr.find('usage: js [') != -1 or + stdoutStderr.find('Error: Short option followed by junk') != -1 or + stdoutStderr.find('Error: Invalid long option:') != -1 or + stdoutStderr.find('Error: Invalid short option:') != -1): return ("good", "Exit code 1 or 2 - js shell quits because it does not support a given CLI parameter") elif 3 <= exitCode <= 6: return ('good', 'Acceptable exit code ' + str(exitCode)) @@ -310,28 +311,31 @@ def inner(shellFilename, _hgHash): return inner -def externalTestAndLabel(options, interestingness): +def externalTestAndLabel(options, interestingness): # pylint: disable=invalid-name,missing-param-doc,missing-return-doc + # pylint: disable=missing-return-type-doc,missing-type-doc """Make use of interestingness scripts to decide whether the changeset is good or bad.""" - conditionScript = rel_or_abs_import(interestingness[0]) - conditionArgPrefix = interestingness[1:] - - def inner(shellFilename, hgHash): - conditionArgs = conditionArgPrefix + [shellFilename] + options.paramList - tempDir = tempfile.mkdtemp(prefix="abExtTestAndLabel-" + hgHash) - tempPrefix = os.path.join(tempDir, 't') + conditionScript = rel_or_abs_import(interestingness[0]) # pylint: disable=invalid-name + conditionArgPrefix = interestingness[1:] # pylint: disable=invalid-name + + def inner(shellFilename, hgHash): # pylint: disable=invalid-name,missing-docstring,missing-return-doc + # pylint: disable=missing-return-type-doc + conditionArgs = conditionArgPrefix + [shellFilename] + options.paramList # pylint: disable=invalid-name + tempDir = tempfile.mkdtemp(prefix="abExtTestAndLabel-" + hgHash) # pylint: disable=invalid-name + tempPrefix = os.path.join(tempDir, 't') # pylint: disable=invalid-name if hasattr(conditionScript, "init"): # Since we're changing the js shell name, call init() again! conditionScript.init(conditionArgs) if conditionScript.interesting(conditionArgs, tempPrefix): - innerResult = ('bad', 'interesting') + innerResult = ('bad', 'interesting') # pylint: disable=invalid-name else: - innerResult = ('good', 'not interesting') + innerResult = ('good', 'not interesting') # pylint: disable=invalid-name if os.path.isdir(tempDir): sps.rmTreeIncludingReadOnly(tempDir) return innerResult return inner +# pylint: disable=invalid-name,missing-param-doc,missing-type-doc,too-many-arguments def checkBlameParents(repoDir, blamedRev, blamedGoodOrBad, labels, testRev, startRepo, endRepo): """If bisect blamed a merge, try to figure out why.""" bisectLied = False @@ -348,8 +352,8 @@ def checkBlameParents(repoDir, blamedRev, blamedGoodOrBad, labels, testRev, star if labels.get(p) is None: print() print("Oops! We didn't test rev %s, a parent of the blamed revision! Let's do that now." % p) - if not hgCmds.isAncestor(repoDir, startRepo, p) and \ - not hgCmds.isAncestor(repoDir, endRepo, p): + if not hg_helpers.isAncestor(repoDir, startRepo, p) and \ + not hg_helpers.isAncestor(repoDir, endRepo, p): print("We did not test rev %s because it is not a descendant of either %s or %s." % ( p, startRepo, endRepo)) # Note this in case we later decide the bisect result is wrong. @@ -371,7 +375,7 @@ def checkBlameParents(repoDir, blamedRev, blamedGoodOrBad, labels, testRev, star # Explain why bisect blamed the merge. if bisectLied: if missedCommonAncestor: - ca = hgCmds.findCommonAncestor(repoDir, parents[0], parents[1]) + ca = hg_helpers.findCommonAncestor(repoDir, parents[0], parents[1]) print() print("Bisect blamed the merge because our initial range did not include one") print("of the parents.") @@ -391,7 +395,8 @@ def checkBlameParents(repoDir, blamedRev, blamedGoodOrBad, labels, testRev, star print("I don't know which patches from each side of the merge contributed to the bug. Sorry.") -def sanitizeCsetMsg(msg, repo): +def sanitizeCsetMsg(msg, repo): # pylint: disable=missing-param-doc,missing-return-doc + # pylint: disable=missing-return-type-doc,missing-type-doc """Sanitize changeset messages, removing email addresses.""" msgList = msg.split('\n') sanitizedMsgList = [] @@ -404,14 +409,16 @@ def sanitizeCsetMsg(msg, repo): return '\n'.join(sanitizedMsgList) -def bisectLabel(hgPrefix, options, hgLabel, currRev, startRepo, endRepo): +def bisectLabel(hgPrefix, options, hgLabel, currRev, startRepo, endRepo): # pylint: disable=invalid-name + # pylint: disable=missing-param-doc,missing-raises-doc,missing-return-doc,missing-return-type-doc,missing-type-doc + # pylint: disable=too-many-arguments """Tell hg what we learned about the revision.""" assert hgLabel in ("good", "bad", "skip") outputResult = sps.captureStdout(hgPrefix + ['bisect', '-U', '--' + hgLabel, currRev])[0] outputLines = outputResult.split("\n") - if options.buildOptions: - repoDir = options.buildOptions.repoDir + if options.build_options: + repoDir = options.build_options.repoDir if re.compile("Due to skipped revisions, the first (good|bad) revision could be any of:").match(outputLines[0]): print() @@ -429,7 +436,7 @@ def bisectLabel(hgPrefix, options, hgLabel, currRev, startRepo, endRepo): print(sanitizeCsetMsg(outputResult, repoDir)) print() blamedGoodOrBad = m.group(1) - blamedRev = hgCmds.getCsetHashFromBisectMsg(outputLines[1]) + blamedRev = hg_helpers.getCsetHashFromBisectMsg(outputLines[1]) return blamedGoodOrBad, blamedRev, None, startRepo, endRepo if options.testInitialRevs: @@ -438,11 +445,11 @@ def bisectLabel(hgPrefix, options, hgLabel, currRev, startRepo, endRepo): # e.g. "Testing changeset 52121:573c5fa45cc4 (440 changesets remaining, ~8 tests)" sps.vdump(outputLines[0]) - currRev = hgCmds.getCsetHashFromBisectMsg(outputLines[0]) + currRev = hg_helpers.getCsetHashFromBisectMsg(outputLines[0]) if currRev is None: print("Resetting to default revision...") subprocess.check_call(hgPrefix + ['update', '-C', 'default']) - hgCmds.destroyPyc(repoDir) + hg_helpers.destroyPyc(repoDir) raise Exception("hg did not suggest a changeset to test!") # Update the startRepo/endRepo values. @@ -463,7 +470,8 @@ def bisectLabel(hgPrefix, options, hgLabel, currRev, startRepo, endRepo): ############################################# -def assertSaneJsBinary(cacheF): +def assertSaneJsBinary(cacheF): # pylint: disable=missing-param-doc,missing-raises-doc,missing-return-doc + # pylint: disable=missing-return-type-doc,missing-type-doc """If the cache folder is present, check that the js binary is working properly.""" if os.path.isdir(cacheF): fList = os.listdir(cacheF) @@ -484,7 +492,7 @@ def assertSaneJsBinary(cacheF): # tbpl binaries are always: # * run without Valgrind (they are not compiled with --enable-valgrind) - retCode = inspectShell.testBinary(shellPath, ['-e', '42'], False)[1] + retCode = inspect_shell.testBinary(shellPath, ['-e', '42'], False)[1] # Exit code -1073741515 on Windows shows up when a required DLL is not present. # This was testable at the time of writing, see bug 953314. isDllNotPresentWinStartupError = (sps.isWin and retCode == -1073741515) @@ -495,7 +503,7 @@ def assertSaneJsBinary(cacheF): elif retCode != 0: raise Exception('Non-zero return code: ' + str(retCode)) return True # Binary is working correctly - except (OSError, IOError): + except (OSError, IOError): # pylint: disable=overlapping-except raise Exception("Cache folder %s is corrupt, please delete it and try again." % cacheF) elif INCOMPLETE_NOTE in fList: return True @@ -505,15 +513,16 @@ def assertSaneJsBinary(cacheF): raise Exception('Cache folder ' + cacheF + ' is not found.') -def bisectUsingTboxBins(options): +def bisectUsingTboxBins(options): # pylint: disable=invalid-name,missing-param-doc,missing-raises-doc,missing-type-doc + # pylint: disable=too-complex,too-many-locals,too-many-statements """Download treeherder binaries and bisect them.""" testedIDs = {} - desiredArch = '32' if options.buildOptions.enable32 else '64' - buildType = downloadBuild.defaultBuildType( - options.nameOfTreeherderBranch, desiredArch, options.buildOptions.enableDbg) + desiredArch = '32' if options.build_options.enable32 else '64' + buildType = download_build.defaultBuildType( + options.nameOfTreeherderBranch, desiredArch, options.build_options.enableDbg) # Get list of treeherder IDs - urlsTbox = downloadBuild.getBuildList(buildType, earliestBuild=options.startRepo, latestBuild=options.endRepo) + urlsTbox = download_build.getBuildList(buildType, earliestBuild=options.startRepo, latestBuild=options.endRepo) # Download and test starting point. print() @@ -588,7 +597,7 @@ def bisectUsingTboxBins(options): outputTboxBisectionResults(options, urlsTbox, testedIDs) -def createTboxCacheFolder(cacheFolder): +def createTboxCacheFolder(cacheFolder): # pylint: disable=missing-param-doc,missing-type-doc """Attempt to create the treeherder js shell's cache folder if it does not exist. If it does, check that its binaries are working properly. @@ -600,12 +609,12 @@ def createTboxCacheFolder(cacheFolder): try: ensureCacheDirHasCorrectIdNum(cacheFolder) - except (KeyboardInterrupt, Exception) as e: + except (KeyboardInterrupt, Exception) as e: # pylint: disable=broad-except if 'Folder name numeric ID not equal to source URL numeric ID.' in repr(e): sps.rmTreeIncludingReadOnly(sps.normExpUserPath(os.path.join(cacheFolder, 'build'))) -def ensureCacheDirHasCorrectIdNum(cacheFolder): +def ensureCacheDirHasCorrectIdNum(cacheFolder): # pylint: disable=missing-param-doc,missing-raises-doc,missing-type-doc """Ensure that the cache folder is named with the correct numeric ID.""" srcUrlPath = sps.normExpUserPath(os.path.join(cacheFolder, 'build', 'download', 'source-url.txt')) if os.path.isfile(srcUrlPath): @@ -623,7 +632,8 @@ def ensureCacheDirHasCorrectIdNum(cacheFolder): raise Exception('Folder name numeric ID not equal to source URL numeric ID.') -def getBuildOrNeighbour(isJsShell, preferredIndex, urls, buildType): +def getBuildOrNeighbour(isJsShell, preferredIndex, urls, buildType): # pylint: disable=invalid-name,missing-param-doc + # pylint: disable=missing-return-doc,missing-return-type-doc,missing-type-doc,too-many-branches,too-complex """Download a build. If the build is incomplete, find a working neighbour, then return results.""" offset = None skippedChangesetNum = 0 @@ -639,7 +649,7 @@ def getBuildOrNeighbour(isJsShell, preferredIndex, urls, buildType): if (preferredIndex + offset >= len(urls)) and (preferredIndex - offset < 0): print("Stop looping because everything within the range was tested.") return None, None, None, None - offset = -offset + offset = -offset # pylint: disable=invalid-unary-operand-type else: offset = -offset + 1 # Alternate between positive and negative offsets @@ -655,7 +665,7 @@ def getBuildOrNeighbour(isJsShell, preferredIndex, urls, buildType): if isWorking: try: assertSaneJsBinary(tboxCacheFolder) - except (KeyboardInterrupt, Exception) as e: + except (KeyboardInterrupt, Exception) as e: # pylint: disable=broad-except if 'Shell startup error' in repr(e): writeIncompleteBuildTxtFile(urls[newIndex], tboxCacheFolder, sps.normExpUserPath(os.path.join(tboxCacheFolder, @@ -675,7 +685,8 @@ def getBuildOrNeighbour(isJsShell, preferredIndex, urls, buildType): preferredIndex = 0 -def getHgwebMozillaOrg(branchName): +def getHgwebMozillaOrg(branchName): # pylint: disable=missing-param-doc,missing-return-doc + # pylint: disable=missing-return-type-doc,missing-type-doc """Return the hgweb link of the repository, given a treeherder branch name.""" hgWebAddrList = ['hg.mozilla.org'] if branchName == 'mozilla-central': @@ -690,15 +701,17 @@ def getHgwebMozillaOrg(branchName): return 'https://' + '/'.join(hgWebAddrList) -def getIdFromTboxUrl(url): +def getIdFromTboxUrl(url): # pylint: disable=missing-param-doc,missing-return-doc + # pylint: disable=missing-return-type-doc,missing-type-doc """Return numeric ID from treeherder at https://archive.mozilla.org/pub/firefox/tinderbox-builds/ .""" return [i for i in url.split("/") if i][-1] -def getOneBuild(isJsShell, url, buildType): +def getOneBuild(isJsShell, url, buildType): # pylint: disable=missing-param-doc,missing-return-doc + # pylint: disable=missing-return-type-doc,missing-type-doc """Try to get a complete working build.""" idNum = getIdFromTboxUrl(url) - tboxCacheFolder = sps.normExpUserPath(os.path.join(compileShell.ensureCacheDir(), + tboxCacheFolder = sps.normExpUserPath(os.path.join(compile_shell.ensureCacheDir(), 'tboxjs-' + buildType + '-' + idNum)) createTboxCacheFolder(tboxCacheFolder) @@ -713,7 +726,7 @@ def getOneBuild(isJsShell, url, buildType): readIncompleteBuildTxtFile(incompleteBuildTxtFile, idNum) return False, None, None # Cached, incomplete - if downloadBuild.downloadBuild(url, tboxCacheFolder, jsShell=isJsShell): + if download_build.downloadBuild(url, tboxCacheFolder, jsShell=isJsShell): assert os.listdir(tboxCacheFolder) == ['build'], 'Only ' + \ 'the build subdirectory should be present in ' + tboxCacheFolder return True, idNum, tboxCacheFolder # Downloaded, complete @@ -721,12 +734,14 @@ def getOneBuild(isJsShell, url, buildType): return False, None, None # Downloaded, incomplete -def getTboxJsBinPath(baseDir): +def getTboxJsBinPath(baseDir): # pylint: disable=missing-param-doc,missing-return-doc + # pylint: disable=missing-return-type-doc,missing-type-doc """Return the path to the treeherder js binary from a download folder.""" return sps.normExpUserPath(os.path.join(baseDir, 'build', 'dist', 'js.exe' if sps.isWin else 'js')) -def getTimestampAndHashFromTboxFiles(folder): +def getTimestampAndHashFromTboxFiles(folder): # pylint: disable=missing-param-doc,missing-return-doc + # pylint: disable=missing-return-type-doc,missing-type-doc """Return timestamp and changeset information from the .txt file downloaded from treeherder.""" downloadDir = sps.normExpUserPath(os.path.join(folder, 'build', 'download')) for fn in os.listdir(downloadDir): @@ -738,12 +753,14 @@ def getTimestampAndHashFromTboxFiles(folder): return fContents[0], fContents[1].split('/')[-1] -def isTboxBinInteresting(options, downloadDir, csetHash): +def isTboxBinInteresting(options, downloadDir, csetHash): # pylint: disable=invalid-name,missing-param-doc + # pylint: disable=missing-return-doc,missing-return-type-doc,missing-type-doc """Test the required treeherder binary.""" return options.testAndLabel(getTboxJsBinPath(downloadDir), csetHash) -def outputTboxBisectionResults(options, interestingList, testedBuildsDict): +def outputTboxBisectionResults(options, interestingList, testedBuildsDict): # pylint: disable=missing-param-doc + # pylint: disable=missing-raises-doc,missing-type-doc,too-many-locals """Return formatted bisection results from using treeherder builds.""" sTimestamp, sHash, sResult, _sReason = testedBuildsDict[getIdFromTboxUrl(interestingList[0])] eTimestamp, eHash, eResult, _eReason = testedBuildsDict[getIdFromTboxUrl(interestingList[-1])] @@ -751,7 +768,7 @@ def outputTboxBisectionResults(options, interestingList, testedBuildsDict): print() print("Parameters for compilation bisection:") pOutput = '-p "' + options.parameters + '"' if options.parameters != '-e 42' else '' - oOutput = '-o "' + options.output + '"' if options.output is not '' else '' + oOutput = '-o "' + options.output + '"' if options.output is not '' else '' # pylint: disable=literal-comparison params = [i for i in ["-s " + sHash, "-e " + eHash, pOutput, oOutput, "-b "] if i] print(" ".join(params)) @@ -778,7 +795,7 @@ def outputTboxBisectionResults(options, interestingList, testedBuildsDict): print() -def readIncompleteBuildTxtFile(txtFile, idNum): +def readIncompleteBuildTxtFile(txtFile, idNum): # pylint: disable=missing-raises-doc,missing-param-doc,missing-type-doc """Read the INCOMPLETE_NOTE text file indicating that this particular build is incomplete.""" with open(txtFile, 'rb') as f: contentsF = f.read() @@ -789,12 +806,12 @@ def readIncompleteBuildTxtFile(txtFile, idNum): print("Examined build with numeric ID %s to be incomplete. Trying another build..." % idNum) -def rmOldLocalCachedDirs(cacheDir): +def rmOldLocalCachedDirs(cacheDir): # pylint: disable=missing-param-doc,missing-type-doc """Remove old local cached directories, which were created four weeks ago.""" # This is in autoBisect because it has a lock so we do not race while removing directories # Adapted from http://stackoverflow.com/a/11337407 SECONDS_IN_A_DAY = 24 * 60 * 60 - s3CacheObj = s3cache.S3Cache(compileShell.S3_SHELL_CACHE_DIRNAME) + s3CacheObj = s3cache.S3Cache(compile_shell.S3_SHELL_CACHE_DIRNAME) if s3CacheObj.connect(): NUMBER_OF_DAYS = 1 # EC2 VMs generally have less disk space for local shell caches elif sps.isARMv7l: @@ -812,7 +829,8 @@ def rmOldLocalCachedDirs(cacheDir): shutil.rmtree(name) -def showRemainingNumOfTests(reqList): +def showRemainingNumOfTests(reqList): # pylint: disable=missing-param-doc,missing-return-doc + # pylint: disable=missing-return-type-doc,missing-type-doc """Display the approximate number of tests remaining.""" remainingTests = int(math.ceil(math.log(len(reqList), 2))) - 1 wordTest = 'tests' @@ -821,7 +839,8 @@ def showRemainingNumOfTests(reqList): return '~' + str(remainingTests) + ' ' + wordTest + ' remaining...\n' -def testBuildOrNeighbour(options, preferredIndex, urls, buildType, testedIDs): +def testBuildOrNeighbour(options, preferredIndex, urls, buildType, testedIDs): # pylint: disable=invalid-name + # pylint: disable=missing-param-doc,missing-return-doc,missing-return-type-doc,missing-type-doc """Test the build. If the build is incomplete, find a working neighbour, then return results.""" finalIndex, idNum, tboxCacheFolder, skippedNum = getBuildOrNeighbour( (not options.browserOptions), preferredIndex, urls, buildType @@ -845,7 +864,8 @@ def testBuildOrNeighbour(options, preferredIndex, urls, buildType, testedIDs): return idNum, result, reason, finalIndex, urls, testedIDs, skippedNum -def writeIncompleteBuildTxtFile(url, cacheFolder, txtFile, num): +def writeIncompleteBuildTxtFile(url, cacheFolder, txtFile, num): # pylint: disable=invalid-name,missing-param-doc + # pylint: disable=missing-return-doc,missing-return-type-doc,missing-type-doc """Write a text file indicating that this particular build is incomplete.""" if os.path.isdir(sps.normExpUserPath(os.path.join(cacheFolder, 'build', 'dist'))) or \ os.path.isdir(sps.normExpUserPath(os.path.join(cacheFolder, 'build', 'download'))): @@ -863,21 +883,15 @@ def main(): """Prevent running two instances of autoBisectJs concurrently - we don't want to confuse hg.""" options = parseOpts() - repoDir = options.buildOptions.repoDir if options.buildOptions else options.browserOptions.repoDir + repoDir = options.build_options.repoDir if options.build_options else options.browserOptions.repoDir - with LockDir.LockDir(compileShell.getLockDirPath(options.nameOfTreeherderBranch, tboxIdentifier='Tbox') - if options.useTreeherderBinaries else compileShell.getLockDirPath(repoDir)): + with LockDir.LockDir(compile_shell.getLockDirPath(options.nameOfTreeherderBranch, tboxIdentifier='Tbox') + if options.useTreeherderBinaries else compile_shell.getLockDirPath(repoDir)): if options.useTreeherderBinaries: bisectUsingTboxBins(options) elif not options.browserOptions: # Bisect using local builds - findBlamedCset(options, repoDir, compileShell.makeTestRev(options)) + findBlamedCset(options, repoDir, compile_shell.makeTestRev(options)) # Last thing we do while we have a lock. # Note that this only clears old *local* cached directories, not remote ones. - rmOldLocalCachedDirs(compileShell.ensureCacheDir()) - - -if __name__ == '__main__': - # Reopen stdout, unbuffered. This is similar to -u. From http://stackoverflow.com/a/107717 - sys.stdout = sps.Unbuffered(sys.stdout) - main() + rmOldLocalCachedDirs(compile_shell.ensureCacheDir()) diff --git a/src/funfuzz/autobisectjs/examples.md b/src/funfuzz/autobisectjs/examples.md new file mode 100644 index 000000000..f378d92ae --- /dev/null +++ b/src/funfuzz/autobisectjs/examples.md @@ -0,0 +1,47 @@ +## Examples + +To try this yourself, run the following commands with the testcases from the bug numbers pasted into the file, e.g. "1188586.js" contains the testcase from [bug 1188586](https://bugzilla.mozilla.org/show_bug.cgi?id=1188586). + +* To test when a bug was introduced by **downloading mozilla-inbound builds** from Mozilla: + +```python -m funfuzz.autobisectjs.autobisectjs -p "--fuzzing-safe --no-threads --ion-eager 1188586.js" -b "--enable-debug" -T``` + +However, this only works effectively if the bug was recent, because builds are only stored per-push within the past month. + +* The equivalent command using **local compiled builds** is: + +```python -m funfuzz.autobisectjs.autobisectjs -p "--fuzzing-safe --no-threads --ion-eager 1188586.js" -b "--enable-debug --enable-more-deterministic"``` + +* To **test branches**, e.g. on mozilla-inbound instead (or any other release branch including ESR), assuming the *Mercurial* repository is cloned to "~/trees/mozilla-inbound": + +```python -m funfuzz.autobisectjs.autobisectjs -p "--fuzzing-safe --no-threads --ion-eager 1188586.js" -b "--enable-debug --enable-more-deterministic -R ~/trees/mozilla-inbound"``` + +* During bisection, perhaps the testcase used to crash in the past; however we are only interested in the assertion failure. You can make autoBisect look out for the **assertion failure message**: + +```python -m funfuzz.autobisectjs.autobisectjs -p "--fuzzing-safe --no-threads --ion-eager 1188586.js" -b "--enable-debug --enable-more-deterministic" -o "Assertion failure"``` + +* To look out for a particular **exit code**, use "-w": + +```python -m funfuzz.autobisectjs.autobisectjs -p "--fuzzing-safe --no-threads --ion-eager 1189137.js" -b "--enable-debug --enable-more-deterministic" -w 3``` + +* To specify **starting and ending revisions**, use "-s" and "-e": + +```python -m funfuzz.autobisectjs.autobisectjs -s 7820fd141998 -e 'parents(322487136b28)' -p "--no-threads --ion-eager --unboxed-objects 1189137.js" -b "--enable-debug --enable-more-deterministic" -o "Assertion failed"``` + +This method can be used to find when a regression was introduced as well as when a bug got fixed. + +* Or, the testcase is **intermittent** and only reproduces once every 5 tries. autoBisect can be set to use the "range" interestingness test to retest 50 times before concluding if it is interesting or not: + +```python -m funfuzz.autobisectjs.autobisectjs -p "--fuzzing-safe --no-threads --ion-eager 1188586.js" -b "--enable-debug --enable-more-deterministic" -i range 1 50 crashes --timeout=3``` + +Note that this requires the [lithium repository](https://github.com/MozillaSecurity/lithium) to be cloned adjacent to the funfuzz repository. + +You could specify the assertion message this way too: + +```python -m funfuzz.autobisectjs.autobisectjs -p "--fuzzing-safe --no-threads --ion-eager 1188586.js" -b "--enable-debug --enable-more-deterministic" -i range 1 50 outputs --timeout=3 'Assertion failure'``` + +"-i" should be the last argument on the command line. + +* To bisect **bugs found by compare_jit**: + +```python -m funfuzz.autobisectjs.autobisectjs -s 6ec4eb9786d8 -p 1183423.js -b "--enable-debug --enable-more-deterministic -R ~/trees/mozilla-central" -i funfuzz.js.compare_jit --minlevel=6 mozilla-central``` diff --git a/autobisect-js/faq-autoBisect.md b/src/funfuzz/autobisectjs/faq.md similarity index 74% rename from autobisect-js/faq-autoBisect.md rename to src/funfuzz/autobisectjs/faq.md index 0a0686697..fcfaad180 100644 --- a/autobisect-js/faq-autoBisect.md +++ b/src/funfuzz/autobisectjs/faq.md @@ -4,7 +4,7 @@ Specify the failing revision hash in the "-e" parameter, along with desired "-b" options to build the desired SpiderMonkey build configuration. This failing revision hash must be from **Mercurial**. -```funfuzz/autobisect-js/autoBisect.py -l bad -e FAILINGREV -b "--enable-debug --enable-more-deterministic"``` +```python -m funfuzz.autobisectjs.autobisectjs -l bad -e FAILINGREV -b "--enable-debug --enable-more-deterministic"``` When done, find the first working revision hash after the breakage, as below. @@ -12,13 +12,13 @@ When done, find the first working revision hash after the breakage, as below. Similar to the above, but use "-s" instead of "-e". -```funfuzz/autobisect-js/autoBisect.py -l bad -s FAILINGREV -b "--enable-debug --enable-more-deterministic``` +```python -m funfuzz.autobisectjs.autobisectjs -l bad -s FAILINGREV -b "--enable-debug --enable-more-deterministic``` **Q: What should I do with the known broken changeset ranges to prevent autoBisect from retesting those revisions?** (This assumes you have the first bad and first good revision hashes as per the 2 questions above.) -You can add them to the known broken range functions in [knownBrokenEarliestWorking.py](knownBrokenEarliestWorking.py). Add the first bad and first good changeset **Mercurial** hashes of the build breakage and its fix, along with a short comment. +You can add them to the known broken range functions in [known_broken_earliest_working](known_broken_earliest_working.py). Add the first bad and first good changeset **Mercurial** hashes of the build breakage and its fix, along with a short comment. **Q: The testcase is giving out assorted varied exit codes as it gets executed by older binaries. How can I fixate to a particular interesting exit code?** @@ -26,11 +26,11 @@ Pass in the "-w" argument along with the desired exit code to autoBisect. If it **Q: The testcase is intermittent and giving weird results! What should I do to try and get more reliable results?** -You can try using interestingness tests to look out for the desired symptom, see [the examples](examples-autoBisect.md). +You can try using interestingness tests to look out for the desired symptom, see [the examples](examples.md). **Q: What happens when a new operating system is released, and we now have a new changeset hash that has to be updated as the earliest known working revision?** -You can add the earliest known working **Mercurial** revision to the earliestKnownWorkingRev function in [knownBrokenEarliestWorking.py](knownBrokenEarliestWorking.py). +You can add the earliest known working **Mercurial** revision to the earliestKnownWorkingRev function in [known_broken_earliest_working](known_broken_earliest_working.py). **Q: Does autoBisect work on nightly SpiderMonkey js shells yet?** @@ -44,4 +44,4 @@ autoBisect was [first written](https://bugzilla.mozilla.org/show_bug.cgi?id=4825 mozregression had its [first landing](https://github.com/mozilla/mozregression/commit/d50509b36cb6ba45d7c54917f528bdf482d2c5e6) in February 2010. -autoBisect supports bisections using compiled and downloaded (tinderbox-builds) SpiderMonkey js shells (with rudimentary support for Firefox), while mozregression supports nightly and inbound builds [for various Mozilla products](http://mozilla.github.io/mozregression/). +autoBisect supports bisections using compiled and downloaded (tinderbox-builds) SpiderMonkey js shells, while mozregression supports nightly and inbound builds [for various Mozilla products](http://mozilla.github.io/mozregression/). diff --git a/src/funfuzz/autobisectjs/find_intersecting_changesets.py b/src/funfuzz/autobisectjs/find_intersecting_changesets.py new file mode 100644 index 000000000..5aed13603 --- /dev/null +++ b/src/funfuzz/autobisectjs/find_intersecting_changesets.py @@ -0,0 +1,61 @@ +# coding=utf-8 +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +"""This file scans the revsets in ignoreAndEarliestWorkingLists and looks for overlaps. + +Usage: python -m funfuzz.autobisectjs.find_intersecting_changesets -R ~/trees/mozilla-central/ + +(first go to known_broken_earliest_working and comment out configuration-specific ignore ranges, +this file does not yet support those.)""" + +from __future__ import absolute_import, print_function + +import os +from optparse import OptionParser # pylint: disable=deprecated-module + +from . import known_broken_earliest_working as kbew +from ..util import subprocesses as sps + + +def parse_options(): # pylint: disable=missing-docstring,missing-return-doc,missing-return-type-doc + parser = OptionParser() + parser.add_option('-R', '--repo', dest='rDir', + help='Sets the repository to analyze..') + options, _args = parser.parse_args() + assert options.rDir is not None + assert os.path.isdir(sps.normExpUserPath(options.rDir)) + return options + + +def count_csets(revset, rdir): # pylint: disable=missing-param-doc,missing-return-doc,missing-return-type-doc + # pylint: disable=missing-type-doc + """Count the number of changesets in the revsets by outputting ones and counting them.""" + cmd = ['hg', 'log', '-r', revset, '--template=1'] + range_intersection_ones = sps.captureStdout(cmd, currWorkingDir=rdir) + assert range_intersection_ones[1] == 0 + return len(range_intersection_ones[0]) + + +def main(): # pylint: disable=missing-docstring + options = parse_options() + repo_dir = options.rDir + broken_ranges = kbew.known_broken_ranges(options) + + cnt = 0 + for i in range(0, len(broken_ranges)): + print("Analyzing revset: %s which matches %s changesets" % ( + broken_ranges[i], count_csets(broken_ranges[i], repo_dir))) + for j in range(i + 1, len(broken_ranges)): + cnt += 1 + print("Number %s: Compared against revset: %s" % (cnt, broken_ranges[j])) + overlap = count_csets(broken_ranges[i] + ' and ' + broken_ranges[j], repo_dir) + if overlap: + print("Number of overlapping changesets: %s" % (overlap,)) + cnt = 0 + + +if __name__ == '__main__': + main() diff --git a/autobisect-js/knownBrokenEarliestWorking.py b/src/funfuzz/autobisectjs/known_broken_earliest_working.py similarity index 83% rename from autobisect-js/knownBrokenEarliestWorking.py rename to src/funfuzz/autobisectjs/known_broken_earliest_working.py index 17511a795..c4d96399d 100644 --- a/autobisect-js/knownBrokenEarliestWorking.py +++ b/src/funfuzz/autobisectjs/known_broken_earliest_working.py @@ -1,40 +1,39 @@ -#!/usr/bin/env python # coding=utf-8 -# pylint: disable=import-error,invalid-name,missing-docstring,too-many-branches,wrong-import-position # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at https://mozilla.org/MPL/2.0/. +"""Known broken changeset ranges of SpiderMonkey are specified in this file. +""" + from __future__ import absolute_import, print_function -import os -import sys -from distutils.version import StrictVersion # pylint issue 73 https://git.io/vQAhf pylint: disable=no-name-in-module +# pylint issue 73 https://git.io/vQAhf +from distutils.version import StrictVersion # pylint: disable=import-error,no-name-in-module -path0 = os.path.dirname(os.path.abspath(__file__)) -path1 = os.path.abspath(os.path.join(path0, os.pardir, 'util')) -sys.path.append(path1) -import subprocesses as sps +from ..util import subprocesses as sps -def hgrange(firstBad, firstGood): - """Like "firstBad::firstGood", but includes branches/csets that never got the firstGood fix.""" +def hgrange(first_bad, first_good): # pylint: disable=missing-param-doc,missing-return-doc,missing-return-type-doc + # pylint: disable=missing-type-doc + """Like "first_bad::first_good", but includes branches/csets that never got the first_good fix.""" # NB: mercurial's descendants(x) includes x - # So this revset expression includes firstBad, but does not include firstGood. + # So this revset expression includes first_bad, but does not include first_good. # NB: hg log -r "(descendants(id(badddddd)) - descendants(id(baddddddd)))" happens to return the empty set, # like we want" - return '(descendants(id(' + firstBad + '))-descendants(id(' + firstGood + ')))' + return '(descendants(id(' + first_bad + '))-descendants(id(' + first_good + ')))' -def knownBrokenRanges(options): +def known_broken_ranges(options): # pylint: disable=missing-param-doc,missing-return-doc,missing-return-type-doc + # pylint: disable=missing-type-doc """Return a list of revsets corresponding to known-busted revisions.""" # Paste numbers into: https://hg.mozilla.org/mozilla-central/rev/ to get hgweb link. # To add to the list: # - (1) will tell you when the brokenness started - # - (1) autoBisect.py --compilationFailedLabel=bad -e FAILINGREV + # - (1) python -m funfuzz.autobisectjs.autobisectjs --compilationFailedLabel=bad -e FAILINGREV # - (2) will tell you when the brokenness ended - # - (2) autoBisect.py --compilationFailedLabel=bad -s FAILINGREV + # - (2) python -m funfuzz.autobisectjs.autobisectjs --compilationFailedLabel=bad -s FAILINGREV # ANCIENT FIXME: It might make sense to avoid (or note) these in checkBlameParents. @@ -96,18 +95,19 @@ def knownBrokenRanges(options): return skips -def earliestKnownWorkingRev(options, flags, skipRevs): +def earliest_known_working_rev(options, flags, skip_revs): # pylint: disable=missing-param-doc,missing-return-doc + # pylint: disable=missing-return-type-doc,missing-type-doc,too-many-branches,too-complex """Return a revset which evaluates to the first revision of the shell that compiles with |options| and runs jsfunfuzz successfully with |flags|.""" assert (not sps.isMac) or (sps.macVer() >= [10, 10]) # Only support at least Mac OS X 10.10 # These should be in descending order, or bisection will break at earlier changesets. - gczealValueFlag = False + gczeal_value_flag = False # flags is a list of flags, and the option must exactly match. for entry in flags: # What comes after these flags needs to be a number, so we look for the string instead. if '--gc-zeal=' in entry: - gczealValueFlag = True + gczeal_value_flag = True required = [] @@ -158,12 +158,12 @@ def earliestKnownWorkingRev(options, flags, skipRevs): required.append('bcacb5692ad9') # m-c 222786 Fx37, 1st w/ successful GCC 5.2.x builds on Ubuntu 15.10 onwards if '--ion-sink=on' in flags: required.append('9188c8b7962b') # m-c 217242 Fx36, 1st w/--ion-sink=on, see bug 1093674 - if gczealValueFlag: + if gczeal_value_flag: required.append('03c6a758c9e8') # m-c 216625 Fx36, 1st w/--gc-zeal=14, see bug 1101602 required.append('dc4b163f7db7') # m-c 213475 Fx36, prior builds have issues with Xcode 7.0 and above - return "first((" + commonDescendants(required) + ") - (" + skipRevs + "))" + return "first((" + common_descendants(required) + ") - (" + skip_revs + "))" -def commonDescendants(revs): +def common_descendants(revs): # pylint: disable=missing-docstring,missing-return-doc,missing-return-type-doc return " and ".join("descendants(" + r + ")" for r in revs) diff --git a/bot.py b/src/funfuzz/bot.py old mode 100755 new mode 100644 similarity index 53% rename from bot.py rename to src/funfuzz/bot.py index 400ce5773..a341bcd48 --- a/bot.py +++ b/src/funfuzz/bot.py @@ -1,13 +1,12 @@ -#!/usr/bin/env python # coding=utf-8 -# pylint: disable=broad-except,fixme,import-error,invalid-name,missing-docstring -# pylint: disable=too-few-public-methods,too-many-arguments,wrong-import-position # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at https://mozilla.org/MPL/2.0/. -# bot.py ensures a build is available, then forks a bunch of fuzz-reduce processes +"""Ensures a build is available, then forks a bunch of fuzz-reduce processes. + +""" from __future__ import absolute_import, print_function @@ -21,43 +20,40 @@ from optparse import OptionParser # pylint: disable=deprecated-module -path0 = os.path.dirname(os.path.abspath(__file__)) -path1 = os.path.abspath(os.path.join(path0, 'util')) -sys.path.insert(0, path1) -import downloadBuild -import hgCmds -import subprocesses as sps -import forkJoin -import createCollector -from LockDir import LockDir -path3 = os.path.abspath(os.path.join(path0, 'js')) -sys.path.append(path3) -import buildOptions -import compileShell -import loopjsfunfuzz - -JS_SHELL_DEFAULT_TIMEOUT = 24 # see comments in loopjsfunfuzz.py for tradeoffs - - -class BuildInfo(object): +from .js import build_options +from .js import compile_shell +from .js import loop +from .util import download_build +from .util import hg_helpers +from .util import subprocesses as sps +from .util import fork_join +from .util import create_collector +from .util.LockDir import LockDir + +path0 = os.path.dirname(os.path.abspath(__file__)) # pylint: disable=invalid-name +path3 = os.path.abspath(os.path.join(path0, 'js')) # pylint: disable=invalid-name +JS_SHELL_DEFAULT_TIMEOUT = 24 # see comments in loop for tradeoffs + + +class BuildInfo(object): # pylint: disable=missing-param-doc,missing-type-doc,too-few-public-methods """Store information related to the build, such as its directory, source and type.""" - def __init__(self, bDir, bType, bSrc, bRev, manyTimedRunArgs): - self.buildDir = bDir - self.buildType = bType - self.buildSrc = bSrc - self.buildRev = bRev - self.mtrArgs = manyTimedRunArgs + def __init__(self, bDir, bType, bSrc, bRev, manyTimedRunArgs): # pylint: disable=too-many-arguments + self.buildDir = bDir # pylint: disable=invalid-name + self.buildType = bType # pylint: disable=invalid-name + self.buildSrc = bSrc # pylint: disable=invalid-name + self.buildRev = bRev # pylint: disable=invalid-name + self.mtrArgs = manyTimedRunArgs # pylint: disable=invalid-name -def parseOpts(): +def parseOpts(): # pylint: disable=invalid-name,missing-docstring,missing-return-doc,missing-return-type-doc parser = OptionParser() parser.set_defaults( repoName='mozilla-central', targetTime=15 * 60, # 15 minutes existingBuildDir=None, timeout=0, - buildOptions=None, + build_options=None, useTreeherderBuilds=False, ) @@ -77,67 +73,75 @@ def parseOpts(): help='Download builds from treeherder instead of compiling our own.') # Specify how the shell will be built. - # See js/buildOptions.py for details. parser.add_option('-b', '--build-options', - dest='buildOptions', - help='Specify build options, e.g. -b "-c opt --arch=32" for js (python buildOptions.py --help)') + dest='build_options', + help='Specify build options, e.g. -b "-c opt --arch=32" for js ' + '(python -m funfuzz.js.build_options --help)') parser.add_option('--timeout', type='int', dest='timeout', - help="Sets the timeout for loopjsfunfuzz.py. " + help="Sets the timeout for loop. " "Defaults to taking into account the speed of the computer and debugger (if any).") options, args = parser.parse_args() if args: - print("Warning: bot.py does not use positional arguments") + print("Warning: bot does not use positional arguments") if not options.testType or options.testType == 'dom': raise Exception('options.testType should be set to "js" now that only js engine fuzzing is supported') - if not options.useTreeherderBuilds and not os.path.isdir(buildOptions.DEFAULT_TREES_LOCATION): + if not options.useTreeherderBuilds and not os.path.isdir(build_options.DEFAULT_TREES_LOCATION): # We don't have trees, so we must use treeherder builds. options.useTreeherderBuilds = True - print("Trees were absent from default location: %s" % buildOptions.DEFAULT_TREES_LOCATION) + print() + print("Trees were absent from default location: %s" % build_options.DEFAULT_TREES_LOCATION) print("Using treeherder builds instead...") + print() + sys.exit("Fuzzing downloaded builds is disabled for now, until tooltool is removed. Exiting...") - if options.buildOptions is None: - options.buildOptions = '' - if options.useTreeherderBuilds and options.buildOptions != '': + if options.build_options is None: + options.build_options = '' + if options.useTreeherderBuilds and options.build_options != '': raise Exception('Do not use treeherder builds if one specifies build parameters') return options -def main(): +def main(): # pylint: disable=missing-docstring printMachineInfo() options = parseOpts() - collector = createCollector.createCollector("jsfunfuzz") - refreshSignatures(collector) + collector = create_collector.createCollector("jsfunfuzz") + try: + collector.refresh() + except RuntimeError as ex: + print() + print("Unable to find required entries in .fuzzmanagerconf, exiting...") + sys.exit(ex) options.tempDir = tempfile.mkdtemp("fuzzbot") print(options.tempDir) - buildInfo = ensureBuild(options) - assert os.path.isdir(buildInfo.buildDir) + build_info = ensureBuild(options) + assert os.path.isdir(build_info.buildDir) - numProcesses = multiprocessing.cpu_count() - if "-asan" in buildInfo.buildDir: + number_of_processes = multiprocessing.cpu_count() + if "-asan" in build_info.buildDir: # This should really be based on the amount of RAM available, but I don't know how to compute that in Python. # I could guess 1 GB RAM per core, but that wanders into sketchyville. - numProcesses = max(numProcesses // 2, 1) + number_of_processes = max(number_of_processes // 2, 1) if sps.isARMv7l: # Even though ARM boards generally now have many cores, each core is not as powerful # as x86/64 ones, so restrict fuzzing to only 1 core for now. - numProcesses = 1 + number_of_processes = 1 - forkJoin.forkJoin(options.tempDir, numProcesses, loopFuzzingAndReduction, options, buildInfo, collector) + fork_join.forkJoin(options.tempDir, number_of_processes, loopFuzzingAndReduction, options, build_info, collector) shutil.rmtree(options.tempDir) -def printMachineInfo(): - # Log information about the machine. +def printMachineInfo(): # pylint: disable=invalid-name + """Log information about the machine.""" print("Platform details: %s" % " ".join(platform.uname())) print("hg version: %s" % sps.captureStdout(['hg', '-q', 'version'])[0]) @@ -145,135 +149,127 @@ def printMachineInfo(): try: print("gdb version: %s" % sps.captureStdout(['gdb', '--version'], combineStderr=True, ignoreStderr=True, ignoreExitCode=True)[0]) - except (KeyboardInterrupt, Exception) as e: - print("Error involving gdb is: %r" % (e,)) + except (KeyboardInterrupt, Exception) as ex: # pylint: disable=broad-except + print("Error involving gdb is: %r" % (ex,)) - # FIXME: Should have if os.path.exists(path to git) or something + # FIXME: Should have if os.path.exists(path to git) or something # pylint: disable=fixme # print("git version: %s" % sps.captureStdout(['git', '--version'], combineStderr=True, # ignoreStderr=True, ignoreExitCode=True)[0]) print("Python version: %s" % sys.version.split()[0]) print("Number of cores visible to OS: %d" % multiprocessing.cpu_count()) print("Free space (GB): %.2f" % sps.getFreeSpace("/", 3)) - hgrcLocation = os.path.join(path0, '.hg', 'hgrc') - if os.path.isfile(hgrcLocation): + hgrc_path = os.path.join(path0, '.hg', 'hgrc') + if os.path.isfile(hgrc_path): print("The hgrc of this repository is:") - with open(hgrcLocation, 'rb') as f: - hgrcContentList = f.readlines() - for line in hgrcContentList: + with open(hgrc_path, 'rb') as f: + hgrc_contents = f.readlines() + for line in hgrc_contents: print(line.rstrip()) if os.name == 'posix': # resource library is only applicable to Linux or Mac platforms. - import resource + import resource # pylint: disable=import-error print("Corefile size (soft limit, hard limit) is: %r" % (resource.getrlimit(resource.RLIMIT_CORE),)) -def refreshSignatures(collector): - """Refresh signatures, copying from FuzzManager server to local sigcache.""" - # Btw, you should make sure the server generates the file using - # python manage.py export_signatures files/signatures.zip - # occasionally, e.g. as a cron job. - if collector.serverHost == "127.0.0.1": - # The test server does not serve files - collector.refreshFromZip(os.path.join(path0, "..", "FuzzManager", "server", "files", "signatures.zip")) - else: - # A production server will serve files - collector.refresh() - - -def ensureBuild(options): +def ensureBuild(options): # pylint: disable=invalid-name,missing-docstring,missing-return-doc,missing-return-type-doc if options.existingBuildDir: # Pre-downloaded treeherder builds (browser only for now) - bDir = options.existingBuildDir - bType = 'local-build' - bSrc = bDir - bRev = '' - manyTimedRunArgs = [] + bDir = options.existingBuildDir # pylint: disable=invalid-name + bType = 'local-build' # pylint: disable=invalid-name + bSrc = bDir # pylint: disable=invalid-name + bRev = '' # pylint: disable=invalid-name + manyTimedRunArgs = [] # pylint: disable=invalid-name elif not options.useTreeherderBuilds: if options.testType == "js": # Compiled js shells - options.buildOptions = buildOptions.parseShellOptions(options.buildOptions) + options.build_options = build_options.parseShellOptions(options.build_options) options.timeout = options.timeout or machineTimeoutDefaults(options) - with LockDir(compileShell.getLockDirPath(options.buildOptions.repoDir)): - bRev = hgCmds.getRepoHashAndId(options.buildOptions.repoDir)[0] - cshell = compileShell.CompiledShell(options.buildOptions, bRev) - updateLatestTxt = (options.buildOptions.repoDir == 'mozilla-central') - compileShell.obtainShell(cshell, updateLatestTxt=updateLatestTxt) + with LockDir(compile_shell.getLockDirPath(options.build_options.repoDir)): + bRev = hg_helpers.getRepoHashAndId(options.build_options.repoDir)[0] # pylint: disable=invalid-name + cshell = compile_shell.CompiledShell(options.build_options, bRev) + updateLatestTxt = (options.build_options.repoDir == 'mozilla-central') # pylint: disable=invalid-name + compile_shell.obtainShell(cshell, updateLatestTxt=updateLatestTxt) - bDir = cshell.getShellCacheDir() + bDir = cshell.getShellCacheDir() # pylint: disable=invalid-name # Strip out first 3 chars or else the dir name in fuzzing jobs becomes: # js-js-dbg-opt-64-dm-linux # This is because options.testType gets prepended along with a dash later. - bType = buildOptions.computeShellType(options.buildOptions)[3:] - bSrc = ( + bType = build_options.computeShellType(options.build_options)[3:] # pylint: disable=invalid-name + bSrc = ( # pylint: disable=invalid-name "Create another shell in shell-cache like this one:\n" - 'python -u %s -b "%s -R %s" -r %s\n\n' + 'python -u -m %s -b "%s -R %s" -r %s\n\n' "==============================================\n" "| Fuzzing %s js shell builds\n" "| DATE: %s\n" "==============================================\n\n" % ( - os.path.join(path3, "compileShell.py"), - options.buildOptions.buildOptionsStr, - options.buildOptions.repoDir, + "funfuzz.js.compile_shell", + options.build_options.build_options_str, + options.build_options.repoDir, bRev, cshell.getRepoName(), time.asctime() )) - manyTimedRunArgs = mtrArgsCreation(options, cshell) + manyTimedRunArgs = mtrArgsCreation(options, cshell) # pylint: disable=invalid-name print("buildDir is: %s" % bDir) print("buildSrc is: %s" % bSrc) else: - # FIXME: We can probably remove the testType option + # FIXME: We can probably remove the testType option # pylint: disable=fixme raise Exception('Only testType "js" is supported.') else: # Treeherder js shells and browser # Download from Treeherder and call it 'build' + # pylint: disable=fixme # FIXME: Put 'build' somewhere nicer, like ~/fuzzbuilds/. Don't re-download a build that's up to date. # FIXME: randomize branch selection, get appropriate builds, use appropriate known dirs - bDir = 'build' - bType = downloadBuild.defaultBuildType(options.repoName, None, True) - isJS = options.testType == 'js' - bSrc = downloadBuild.downloadLatestBuild(bType, './', getJsShell=isJS, wantTests=not isJS) - bRev = '' + bDir = 'build' # pylint: disable=invalid-name + bType = download_build.defaultBuildType(options.repoName, None, True) # pylint: disable=invalid-name + isJS = options.testType == 'js' # pylint: disable=invalid-name + # pylint: disable=invalid-name + bSrc = download_build.downloadLatestBuild(bType, './', getJsShell=isJS, wantTests=not isJS) + bRev = '' # pylint: disable=invalid-name # These two lines are only used for treeherder js shells: shell = os.path.join(bDir, "dist", "js.exe" if sps.isWin else "js") + # pylint: disable=invalid-name manyTimedRunArgs = ["--random-flags", str(JS_SHELL_DEFAULT_TIMEOUT), "mozilla-central", shell] return BuildInfo(bDir, bType, bSrc, bRev, manyTimedRunArgs) -def loopFuzzingAndReduction(options, buildInfo, collector, i): - tempDir = tempfile.mkdtemp("loop" + str(i)) +def loopFuzzingAndReduction(options, buildInfo, collector, i): # pylint: disable=invalid-name,missing-docstring + tempDir = tempfile.mkdtemp("loop" + str(i)) # pylint: disable=invalid-name if options.testType == 'js': - loopjsfunfuzz.many_timed_runs(options.targetTime, tempDir, buildInfo.mtrArgs, collector) + loop.many_timed_runs(options.targetTime, tempDir, buildInfo.mtrArgs, collector) else: raise Exception('Only js engine fuzzing is supported') -def machineTimeoutDefaults(options): +def machineTimeoutDefaults(options): # pylint: disable=invalid-name,missing-param-doc,missing-return-doc + # pylint: disable=missing-return-type-doc,missing-type-doc """Set different defaults depending on the machine type or debugger used.""" - if options.buildOptions.runWithVg: + if options.build_options.runWithVg: return 300 elif sps.isARMv7l: return 180 return JS_SHELL_DEFAULT_TIMEOUT -def mtrArgsCreation(options, cshell): +def mtrArgsCreation(options, cshell): # pylint: disable=invalid-name,missing-param-doc,missing-return-doc + # pylint: disable=missing-return-type-doc,missing-type-doc """Create many_timed_run arguments for compiled builds.""" - manyTimedRunArgs = [] - manyTimedRunArgs.append('--repo=' + sps.normExpUserPath(options.buildOptions.repoDir)) - manyTimedRunArgs.append("--build=" + options.buildOptions.buildOptionsStr) - if options.buildOptions.runWithVg: + manyTimedRunArgs = [] # pylint: disable=invalid-name + manyTimedRunArgs.append('--repo=' + sps.normExpUserPath(options.build_options.repoDir)) + manyTimedRunArgs.append("--build=" + options.build_options.build_options_str) + if options.build_options.runWithVg: manyTimedRunArgs.append('--valgrind') - if options.buildOptions.enableMoreDeterministic: - # Treeherder shells not using compareJIT: + if options.build_options.enableMoreDeterministic: + # Treeherder shells not using compare_jit: # They are not built with --enable-more-deterministic - bug 751700 - manyTimedRunArgs.append('--comparejit') + manyTimedRunArgs.append('--compare-jit') manyTimedRunArgs.append('--random-flags') # Ordering of elements in manyTimedRunArgs is important. diff --git a/src/funfuzz/js/README.md b/src/funfuzz/js/README.md new file mode 100644 index 000000000..158d18f52 --- /dev/null +++ b/src/funfuzz/js/README.md @@ -0,0 +1,18 @@ +## Compile SpiderMonkey using compile_shell + +To compile a SpiderMonkey shell, run: + +`python -m funfuzz.js.compile_shell -b "--enable-debug --enable-more-deterministic -R ~/trees/mozilla-central"` + +in order to get a debug 64-bit deterministic shell, off the **Mercurial** repository located at `~/trees/mozilla-central`. + +Clone the repository to that location using: + +`hg clone https://hg.mozilla.org/mozilla-central/ ~/trees/mozilla-central` + +assuming the `~/trees` folder is created and present. + +## Additional information +* compile_shell + * [More examples](examples.md) + * [FAQ](faq.md) diff --git a/src/funfuzz/js/__init__.py b/src/funfuzz/js/__init__.py new file mode 100644 index 000000000..e2599be06 --- /dev/null +++ b/src/funfuzz/js/__init__.py @@ -0,0 +1,18 @@ +# coding=utf-8 +# flake8: noqa +# pylint: disable=missing-docstring +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import + +from . import build_options +from . import compare_jit +from . import compile_shell +from . import inspect_shell +from . import js_interesting +from . import loop +from . import pinpoint +from . import shell_flags diff --git a/js/buildOptions.py b/src/funfuzz/js/build_options.py similarity index 72% rename from js/buildOptions.py rename to src/funfuzz/js/build_options.py index 1a9dc119b..4fa7c2645 100644 --- a/js/buildOptions.py +++ b/src/funfuzz/js/build_options.py @@ -1,12 +1,12 @@ -#!/usr/bin/env python # coding=utf-8 -# pylint: disable=fixme,import-error,invalid-name,missing-docstring -# pylint: disable=too-many-branches,too-many-return-statements,wrong-import-position # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at https://mozilla.org/MPL/2.0/. +"""Allows specification of build configuration parameters. +""" + from __future__ import absolute_import, print_function import argparse @@ -16,44 +16,43 @@ import random import sys -path0 = os.path.dirname(os.path.abspath(__file__)) -path1 = os.path.abspath(os.path.join(path0, os.pardir, 'util')) -sys.path.append(path1) -import hgCmds -import subprocesses as sps +from ..util import hg_helpers +from ..util import subprocesses as sps DEFAULT_TREES_LOCATION = sps.normExpUserPath(os.path.join('~', 'trees')) -deviceIsFast = not sps.isARMv7l +deviceIsFast = not sps.isARMv7l # pylint: disable=invalid-name -def chance(p): +def chance(p): # pylint: disable=invalid-name,missing-docstring,missing-return-doc,missing-return-type-doc return random.random() < p -class Randomizer(object): +class Randomizer(object): # pylint: disable=missing-docstring def __init__(self): self.options = [] - def add(self, name, fastDeviceWeight, slowDeviceWeight): + def add(self, name, fastDeviceWeight, slowDeviceWeight): # pylint: disable=invalid-name,missing-docstring self.options.append({ 'name': name, 'fastDeviceWeight': fastDeviceWeight, 'slowDeviceWeight': slowDeviceWeight }) - def getRandomSubset(self): - def getWeight(o): + def getRandomSubset(self): # pylint: disable=invalid-name,missing-docstring,missing-return-doc + # pylint: disable=missing-return-type-doc + def getWeight(o): # pylint: disable=invalid-name,missing-docstring,missing-return-doc,missing-return-type-doc return o['fastDeviceWeight'] if deviceIsFast else o['slowDeviceWeight'] return [o['name'] for o in self.options if chance(getWeight(o))] -def addParserOptions(): +def addParserOptions(): # pylint: disable=invalid-name,missing-return-doc,missing-return-type-doc """Add parser options.""" # Where to find the source dir and compiler, patching if necessary. parser = argparse.ArgumentParser(description="Usage: Don't use this directly") randomizer = Randomizer() - def randomizeBool(name, fastDeviceWeight, slowDeviceWeight, **kwargs): + def randomizeBool(name, fastDeviceWeight, slowDeviceWeight, **kwargs): # pylint: disable=invalid-name + # pylint: disable=missing-param-doc,missing-type-doc """Add a randomized boolean option that defaults to False. Option also has a [weight] chance of being changed to True when using --random. @@ -159,80 +158,87 @@ def randomizeBool(name, fastDeviceWeight, slowDeviceWeight, **kwargs): return parser, randomizer -def parseShellOptions(inputArgs): - """Return a 'buildOptions' object, which is intended to be immutable.""" +def parseShellOptions(inputArgs): # pylint: disable=invalid-name,missing-param-doc,missing-return-doc + # pylint: disable=missing-return-type-doc,missing-type-doc + """Return a 'build_options' object, which is intended to be immutable.""" parser, randomizer = addParserOptions() - buildOptions = parser.parse_args(inputArgs.split()) + build_options = parser.parse_args(inputArgs.split()) if sps.isMac: - buildOptions.buildWithClang = True # Clang seems to be the only supported compiler + build_options.buildWithClang = True # Clang seems to be the only supported compiler - if buildOptions.enableArmSimulatorObsolete: - buildOptions.enableSimulatorArm32 = True + if build_options.enableArmSimulatorObsolete: + build_options.enableSimulatorArm32 = True - if buildOptions.enableRandom: - buildOptions = generateRandomConfigurations(parser, randomizer) + if build_options.enableRandom: + build_options = generateRandomConfigurations(parser, randomizer) else: - buildOptions.buildOptionsStr = inputArgs - valid = areArgsValid(buildOptions) + build_options.build_options_str = inputArgs + valid = areArgsValid(build_options) if not valid[0]: print("WARNING: This set of build options is not tested well because: %s" % valid[1]) # Ensures releng machines do not enter the if block and assumes mozilla-central always exists if os.path.isdir(DEFAULT_TREES_LOCATION): # Repositories do not get randomized if a repository is specified. - if buildOptions.repoDir is None: + if build_options.repoDir is None: # For patch fuzzing without a specified repo, do not randomize repos, assume m-c instead - if buildOptions.enableRandom and not buildOptions.patchFile: - buildOptions.repoDir = getRandomValidRepo(DEFAULT_TREES_LOCATION) + if build_options.enableRandom and not build_options.patchFile: + build_options.repoDir = getRandomValidRepo(DEFAULT_TREES_LOCATION) else: - buildOptions.repoDir = os.path.realpath(sps.normExpUserPath( + build_options.repoDir = os.path.realpath(sps.normExpUserPath( os.path.join(DEFAULT_TREES_LOCATION, 'mozilla-central'))) - assert hgCmds.isRepoValid(buildOptions.repoDir) + if not os.path.isdir(build_options.repoDir): + sys.exit("repoDir is not specified, and a default repository location cannot be confirmed. Exiting...") + + assert hg_helpers.isRepoValid(build_options.repoDir) - if buildOptions.patchFile: - hgCmds.ensureMqEnabled() - buildOptions.patchFile = sps.normExpUserPath(buildOptions.patchFile) - assert os.path.isfile(buildOptions.patchFile) + if build_options.patchFile: + hg_helpers.ensureMqEnabled() + build_options.patchFile = sps.normExpUserPath(build_options.patchFile) + assert os.path.isfile(build_options.patchFile) + else: + sys.exit("DEFAULT_TREES_LOCATION not found at: %s. Exiting..." % DEFAULT_TREES_LOCATION) - return buildOptions + return build_options -def computeShellType(buildOptions): +def computeShellType(build_options): # pylint: disable=invalid-name,missing-param-doc,missing-return-doc + # pylint: disable=missing-return-type-doc,missing-type-doc,too-complex,too-many-branches """Return configuration information of the shell.""" - fileName = ['js'] - if buildOptions.enableDbg: + fileName = ['js'] # pylint: disable=invalid-name + if build_options.enableDbg: fileName.append('dbg') - if buildOptions.disableOpt: + if build_options.disableOpt: fileName.append('optDisabled') - fileName.append('32' if buildOptions.enable32 else '64') - if buildOptions.enableProfiling: + fileName.append('32' if build_options.enable32 else '64') + if build_options.enableProfiling: fileName.append('prof') - if buildOptions.disableProfiling: + if build_options.disableProfiling: fileName.append('profDisabled') - if buildOptions.enableMoreDeterministic: + if build_options.enableMoreDeterministic: fileName.append('dm') - if buildOptions.buildWithClang: + if build_options.buildWithClang: fileName.append('clang') - if buildOptions.buildWithAsan: + if build_options.buildWithAsan: fileName.append('asan') - if buildOptions.buildWithVg: + if build_options.buildWithVg: fileName.append('vg') - if buildOptions.enableOomBreakpoint: + if build_options.enableOomBreakpoint: fileName.append('oombp') - if buildOptions.enableWithoutIntlApi: + if build_options.enableWithoutIntlApi: fileName.append('intlDisabled') - if buildOptions.enableSimulatorArm32 or buildOptions.enableSimulatorArm64: + if build_options.enableSimulatorArm32 or build_options.enableSimulatorArm64: fileName.append('armSim') if sps.isARMv7l: - fileName.append('armhfp' if buildOptions.enableHardFp else 'armsfp') + fileName.append('armhfp' if build_options.enableHardFp else 'armsfp') fileName.append('windows' if sps.isWin else platform.system().lower()) - if buildOptions.patchFile: + if build_options.patchFile: # We take the name before the first dot, so Windows (hopefully) does not get confused. - fileName.append(os.path.basename(buildOptions.patchFile).split('.')[0]) - with open(os.path.abspath(buildOptions.patchFile), "rb") as f: - readResult = f.read() + fileName.append(os.path.basename(build_options.patchFile).split('.')[0]) + with open(os.path.abspath(build_options.patchFile), "rb") as f: + readResult = f.read() # pylint: disable=invalid-name # Append the patch hash, but this is not equivalent to Mercurial's hash of the patch. fileName.append(hashlib.sha512(readResult).hexdigest()[:12]) @@ -240,12 +246,14 @@ def computeShellType(buildOptions): return '-'.join(fileName) -def computeShellName(buildOptions, buildRev): +def computeShellName(build_options, buildRev): # pylint: disable=invalid-name,missing-param-doc,missing-return-doc + # pylint: disable=missing-return-type-doc,missing-type-doc """Return the shell type together with the build revision.""" - return computeShellType(buildOptions) + '-' + buildRev + return computeShellType(build_options) + '-' + buildRev -def areArgsValid(args): +def areArgsValid(args): # pylint: disable=invalid-name,missing-param-doc,missing-return-doc,missing-return-type-doc + # pylint: disable=missing-type-doc,too-many-branches,too-complex,too-many-return-statements """Check to see if chosen arguments are valid.""" if args.enableDbg and args.disableDbg: return False, 'Making a debug, non-debug build would be contradictory.' @@ -269,7 +277,7 @@ def areArgsValid(args): # if not sps.isProgramInstalled('valgrind'): # return False, 'Valgrind is not installed.' # if not args.enableOpt: - # # FIXME: Isn't this enabled by default?? + # # FIXME: Isn't this enabled by default?? # pylint: disable=fixme # return False, 'Valgrind needs opt builds.' # if args.buildWithAsan: # return False, 'One should not compile with both Valgrind flags and ASan flags.' @@ -320,20 +328,22 @@ def areArgsValid(args): return True, '' -def generateRandomConfigurations(parser, randomizer): +def generateRandomConfigurations(parser, randomizer): # pylint: disable=invalid-name,missing-docstring + # pylint: disable=missing-return-doc,missing-return-type-doc while True: - randomArgs = randomizer.getRandomSubset() + randomArgs = randomizer.getRandomSubset() # pylint: disable=invalid-name if '--build-with-valgrind' in randomArgs and chance(0.95): randomArgs.append('--run-with-valgrind') - buildOptions = parser.parse_args(randomArgs) - if areArgsValid(buildOptions)[0]: - buildOptions.buildOptionsStr = ' '.join(randomArgs) # Used for autoBisect - buildOptions.enableRandom = True # This has to be true since we are randomizing... - return buildOptions + build_options = parser.parse_args(randomArgs) + if areArgsValid(build_options)[0]: + build_options.build_options_str = ' '.join(randomArgs) # Used for autoBisect + build_options.enableRandom = True # This has to be true since we are randomizing... + return build_options -def getRandomValidRepo(treeLocation): - validRepos = [] +def getRandomValidRepo(treeLocation): # pylint: disable=invalid-name,missing-docstring,missing-return-doc + # pylint: disable=missing-return-type-doc + validRepos = [] # pylint: disable=invalid-name for repo in ['mozilla-central', 'mozilla-beta']: if os.path.isfile(sps.normExpUserPath(os.path.join( treeLocation, repo, '.hg', 'hgrc'))): @@ -347,17 +357,17 @@ def getRandomValidRepo(treeLocation): os.path.join(treeLocation, random.choice(validRepos)))) -def main(): +def main(): # pylint: disable=missing-docstring print("Here are some sample random build configurations that can be generated:") parser, randomizer = addParserOptions() - buildOptions = parser.parse_args() + build_options = parser.parse_args() - if buildOptions.enableArmSimulatorObsolete: - buildOptions.enableSimulatorArm32 = True + if build_options.enableArmSimulatorObsolete: + build_options.enableSimulatorArm32 = True for _ in range(30): - buildOptions = generateRandomConfigurations(parser, randomizer) - print(buildOptions.buildOptionsStr) + build_options = generateRandomConfigurations(parser, randomizer) + print(build_options.build_options_str) print() print("Running this file directly doesn't do anything, but here's our subparser help:") diff --git a/js/compareJIT.py b/src/funfuzz/js/compare_jit.py old mode 100755 new mode 100644 similarity index 59% rename from js/compareJIT.py rename to src/funfuzz/js/compare_jit.py index f845936d4..2b8352564 --- a/js/compareJIT.py +++ b/src/funfuzz/js/compare_jit.py @@ -1,46 +1,41 @@ -#!/usr/bin/env python # coding=utf-8 -# pylint: disable=cell-var-from-loop,fixme,global-statement,import-error,invalid-name,missing-docstring -# pylint: disable=no-member,too-many-arguments,too-many-branches,too-many-locals,wrong-import-position # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at https://mozilla.org/MPL/2.0/. +"""Test comparing the output of SpiderMonkey using various flags (usually JIT-related). +""" + from __future__ import absolute_import, print_function import os import sys from optparse import OptionParser # pylint: disable=deprecated-module -import jsInteresting -import pinpoint -import shellFlags - -path0 = os.path.dirname(os.path.abspath(__file__)) -path1 = os.path.abspath(os.path.join(path0, os.pardir, 'util')) -sys.path.append(path1) -import subprocesses as sps -import lithOps -import createCollector - -# no-name-in-module pylint error exists for Python 3 only because FuzzManager is not Python 3-compatible yet -import FTB.Signatures.CrashInfo as CrashInfo # pylint: disable=no-name-in-module -from FTB.ProgramConfiguration import ProgramConfiguration +# These pylint errors exist because FuzzManager is not Python 3-compatible yet +import FTB.Signatures.CrashInfo as CrashInfo # pylint: disable=import-error,no-name-in-module +from FTB.ProgramConfiguration import ProgramConfiguration # pylint: disable=import-error +from . import js_interesting +from . import pinpoint +from . import shell_flags +from ..util import create_collector +from ..util import lithium_helpers +from ..util import subprocesses as sps -gOptions = "" -lengthLimit = 1000000 +gOptions = "" # pylint: disable=invalid-name +lengthLimit = 1000000 # pylint: disable=invalid-name -def lastLine(err): +def lastLine(err): # pylint: disable=invalid-name,missing-docstring,missing-return-doc,missing-return-type-doc lines = err.split("\n") if len(lines) >= 2: return lines[-2] return "" -def ignoreSomeOfStderr(e): +def ignoreSomeOfStderr(e): # pylint: disable=invalid-name,missing-docstring,missing-return-doc,missing-return-type-doc lines = [] for line in e: if line.endswith("malloc: enabling scribbling to detect mods to free blocks"): @@ -55,28 +50,32 @@ def ignoreSomeOfStderr(e): return lines -# For use by loopjsfunfuzz.py +# For use by loop # Returns True if any kind of bug is found -def compareJIT(jsEngine, flags, infilename, logPrefix, repo, buildOptionsStr, targetTime, options): +def compare_jit(jsEngine, flags, infilename, logPrefix, repo, build_options_str, targetTime, options): + # pylint: disable=invalid-name,missing-docstring,missing-return-doc,missing-return-type-doc + # pylint: disable=too-many-arguments,too-many-locals + + # pylint: disable=invalid-name cl = compareLevel(jsEngine, flags, infilename, logPrefix + "-initial", options, False, True) lev = cl[0] - if lev != jsInteresting.JS_FINE: + if lev != js_interesting.JS_FINE: itest = [__file__, "--flags=" + ' '.join(flags), "--minlevel=" + str(lev), "--timeout=" + str(options.timeout), options.knownPath] - (lithResult, _lithDetails, autoBisectLog) = pinpoint.pinpoint( - itest, logPrefix, jsEngine, [], infilename, repo, buildOptionsStr, targetTime, lev) - if lithResult == lithOps.LITH_FINISHED: + (lithResult, _lithDetails, autoBisectLog) = pinpoint.pinpoint( # pylint: disable=invalid-name + itest, logPrefix, jsEngine, [], infilename, repo, build_options_str, targetTime, lev) + if lithResult == lithium_helpers.LITH_FINISHED: print("Retesting %s after running Lithium:" % infilename) retest_cl = compareLevel(jsEngine, flags, infilename, logPrefix + "-final", options, True, False) - if retest_cl[0] != jsInteresting.JS_FINE: + if retest_cl[0] != js_interesting.JS_FINE: cl = retest_cl quality = 0 else: quality = 6 else: quality = 10 - print("compareJIT: Uploading %s with quality %s" % (infilename, quality)) + print("compare_jit: Uploading %s with quality %s" % (infilename, quality)) metadata = {} if autoBisectLog: @@ -88,11 +87,14 @@ def compareJIT(jsEngine, flags, infilename, logPrefix, repo, buildOptionsStr, ta def compareLevel(jsEngine, flags, infilename, logPrefix, options, showDetailedDiffs, quickMode): - # options dict must be one we can pass to jsInteresting.ShellResult + # pylint: disable=invalid-name,missing-docstring,missing-return-doc,missing-return-type-doc,too-complex + # pylint: disable=too-many-branches,too-many-arguments,too-many-locals + + # options dict must be one we can pass to js_interesting.ShellResult # we also use it directly for knownPath, timeout, and collector - # Return: (lev, crashInfo) or (jsInteresting.JS_FINE, None) + # Return: (lev, crashInfo) or (js_interesting.JS_FINE, None) - combos = shellFlags.basicFlagSets(jsEngine) + combos = shell_flags.basicFlagSets(jsEngine) if quickMode: # Only used during initial fuzzing. Allowed to have false negatives. @@ -106,9 +108,9 @@ def compareLevel(jsEngine, flags, infilename, logPrefix, options, showDetailedDi for i in range(0, len(commands)): prefix = logPrefix + "-r" + str(i) command = commands[i] - r = jsInteresting.ShellResult(options, command, prefix, True) + r = js_interesting.ShellResult(options, command, prefix, True) # pylint: disable=invalid-name - oom = jsInteresting.oomed(r.err) + oom = js_interesting.oomed(r.err) r.err = ignoreSomeOfStderr(r.err) if (r.return_code == 1 or r.return_code == 2) and (anyLineContains(r.out, '[[script] scriptArgs*]') or ( @@ -116,52 +118,60 @@ def compareLevel(jsEngine, flags, infilename, logPrefix, options, showDetailedDi print("Got usage error from:") print(" %s" % sps.shellify(command)) assert i - jsInteresting.deleteLogs(prefix) - elif r.lev > jsInteresting.JS_OVERALL_MISMATCH: + js_interesting.deleteLogs(prefix) + elif r.lev > js_interesting.JS_OVERALL_MISMATCH: # would be more efficient to run lithium on one or the other, but meh print("%s | %s" % (infilename, - jsInteresting.summaryString(r.issues + ["compareJIT found a more serious bug"], - r.lev, - r.runinfo.elapsedtime))) + js_interesting.summaryString(r.issues + ["compare_jit found a more serious bug"], + r.lev, + r.runinfo.elapsedtime))) with open(logPrefix + "-summary.txt", 'wb') as f: - f.write('\n'.join(r.issues + [sps.shellify(command), "compareJIT found a more serious bug"]) + '\n') + f.write('\n'.join(r.issues + [sps.shellify(command), "compare_jit found a more serious bug"]) + '\n') print(" %s" % sps.shellify(command)) return (r.lev, r.crashInfo) - elif r.lev != jsInteresting.JS_FINE or r.return_code != 0: - print("%s | %s" % (infilename, jsInteresting.summaryString( - r.issues + ["compareJIT is not comparing output, because the shell exited strangely"], + elif r.lev != js_interesting.JS_FINE or r.return_code != 0: + print("%s | %s" % (infilename, js_interesting.summaryString( + r.issues + ["compare_jit is not comparing output, because the shell exited strangely"], r.lev, r.runinfo.elapsedtime))) print(" %s" % sps.shellify(command)) - jsInteresting.deleteLogs(prefix) + js_interesting.deleteLogs(prefix) if not i: - return (jsInteresting.JS_FINE, None) + return (js_interesting.JS_FINE, None) elif oom: # If the shell or python hit a memory limit, we consider the rest of the computation # "tainted" for the purpose of correctness comparison. - message = "compareJIT is not comparing output: OOM" - print("%s | %s" % (infilename, jsInteresting.summaryString( + message = "compare_jit is not comparing output: OOM" + print("%s | %s" % (infilename, js_interesting.summaryString( r.issues + [message], r.lev, r.runinfo.elapsedtime))) - jsInteresting.deleteLogs(prefix) + js_interesting.deleteLogs(prefix) if not i: - return (jsInteresting.JS_FINE, None) + return (js_interesting.JS_FINE, None) elif not i: # Stash output from this run (the first one), so for subsequent runs, we can compare against it. - (r0, prefix0) = (r, prefix) + (r0, prefix0) = (r, prefix) # pylint: disable=invalid-name else: # Compare the output of this run (r.out) to the output of the first run (r0.out), etc. - def fpuOptionDisabledAsmOnOneSide(fpuAsmMsg): - fpuOptionDisabledAsm = fpuAsmMsg in r0.err or fpuAsmMsg in r.err + def fpuOptionDisabledAsmOnOneSide(fpuAsmMsg): # pylint: disable=invalid-name,missing-docstring + # pylint: disable=missing-return-doc,missing-return-type-doc + # pylint: disable=invalid-name + fpuOptionDisabledAsm = fpuAsmMsg in r0.err or fpuAsmMsg in r.err # pylint: disable=cell-var-from-loop + # pylint: disable=invalid-name + # pylint: disable=cell-var-from-loop fpuOptionDiffers = (("--no-fpu" in commands[0]) != ("--no-fpu" in command)) return fpuOptionDisabledAsm and fpuOptionDiffers - def optionDisabledAsmOnOneSide(): - asmMsg = "asm.js type error: Disabled by javascript.options.asmjs" + def optionDisabledAsmOnOneSide(): # pylint: disable=invalid-name,missing-docstring,missing-return-doc + # pylint: disable=missing-return-type-doc + asmMsg = "asm.js type error: Disabled by javascript.options.asmjs" # pylint: disable=invalid-name + # pylint: disable=invalid-name + # pylint: disable=cell-var-from-loop optionDisabledAsm = anyLineContains(r0.err, asmMsg) or anyLineContains(r.err, asmMsg) + # pylint: disable=invalid-name optionDiffers = (("--no-asmjs" in commands[0]) != ("--no-asmjs" in command)) return optionDisabledAsm and optionDiffers - mismatchErr = (r.err != r0.err and + mismatchErr = (r.err != r0.err and # pylint: disable=invalid-name # --no-fpu (on debug x86_32 only) turns off asm.js compilation, among other things. # This should only affect asm.js diagnostics on stderr. not fpuOptionDisabledAsmOnOneSide("asm.js type error: " @@ -169,38 +179,40 @@ def optionDisabledAsmOnOneSide(): # And also wasm stuff. See bug 1243031. not fpuOptionDisabledAsmOnOneSide("WebAssembly is not supported on the current device") and not optionDisabledAsmOnOneSide()) - mismatchOut = (r.out != r0.out) + mismatchOut = (r.out != r0.out) # pylint: disable=invalid-name if mismatchErr or mismatchOut: # Generate a short summary for stdout and a long summary for a "*-summary.txt" file. - rerunCommand = sps.shellify(['~/funfuzz/js/compareJIT.py', "--flags=" + ' '.join(flags), + # pylint: disable=invalid-name + rerunCommand = sps.shellify(["python -m funfuzz.js.compare_jit", "--flags=" + " ".join(flags), "--timeout=" + str(options.timeout), options.knownPath, jsEngine, os.path.basename(infilename)]) (summary, issues) = summarizeMismatch(mismatchErr, mismatchOut, prefix0, prefix) summary = " " + sps.shellify(commands[0]) + "\n " + sps.shellify(command) + "\n\n" + summary with open(logPrefix + "-summary.txt", 'wb') as f: f.write(rerunCommand + "\n\n" + summary) - print("%s | %s" % (infilename, jsInteresting.summaryString( - issues, jsInteresting.JS_OVERALL_MISMATCH, r.runinfo.elapsedtime))) + print("%s | %s" % (infilename, js_interesting.summaryString( + issues, js_interesting.JS_OVERALL_MISMATCH, r.runinfo.elapsedtime))) if quickMode: print(rerunCommand) if showDetailedDiffs: print(summary) print() # Create a crashInfo object with empty stdout, and stderr showing diffs - pc = ProgramConfiguration.fromBinary(jsEngine) - pc.addProgramArguments(flags) - crashInfo = CrashInfo.CrashInfo.fromRawCrashData([], summary, pc) - return (jsInteresting.JS_OVERALL_MISMATCH, crashInfo) + pc = ProgramConfiguration.fromBinary(jsEngine) # pylint: disable=invalid-name + pc.addProgramArguments(flags) # pylint: disable=invalid-name + crashInfo = CrashInfo.CrashInfo.fromRawCrashData([], summary, pc) # pylint: disable=invalid-name + return (js_interesting.JS_OVERALL_MISMATCH, crashInfo) else: - # print "compareJIT: match" - jsInteresting.deleteLogs(prefix) + # print "compare_jit: match" + js_interesting.deleteLogs(prefix) # All matched :) - jsInteresting.deleteLogs(prefix0) - return (jsInteresting.JS_FINE, None) + js_interesting.deleteLogs(prefix0) + return (js_interesting.JS_FINE, None) +# pylint: disable=invalid-name,missing-docstring,missing-return-doc,missing-return-type-doc def summarizeMismatch(mismatchErr, mismatchOut, prefix0, prefix): issues = [] summary = "" @@ -215,18 +227,19 @@ def summarizeMismatch(mismatchErr, mismatchOut, prefix0, prefix): return (summary, issues) -def diffFiles(f1, f2): +def diffFiles(f1, f2): # pylint: disable=invalid-name,missing-param-doc,missing-type-doc """Return a command to diff two files, along with the diff output (if it's short).""" diffcmd = ["diff", "-u", f1, f2] - s = ' '.join(diffcmd) + "\n\n" + s = ' '.join(diffcmd) + "\n\n" # pylint: disable=invalid-name diff = sps.captureStdout(diffcmd, ignoreExitCode=True)[0] if len(diff) < 10000: - s += diff + "\n\n" + s += diff + "\n\n" # pylint: disable=invalid-name else: - s += diff[:10000] + "\n(truncated after 10000 bytes)... \n\n" + s += diff[:10000] + "\n(truncated after 10000 bytes)... \n\n" # pylint: disable=invalid-name return s +# pylint: disable=invalid-name,missing-docstring,missing-return-doc,missing-return-type-doc def anyLineContains(lines, needle): for line in lines: if needle in line: @@ -235,13 +248,13 @@ def anyLineContains(lines, needle): return False -def parseOptions(args): +def parseOptions(args): # pylint: disable=invalid-name parser = OptionParser() parser.disable_interspersed_args() parser.add_option("--minlevel", type="int", dest="minimumInterestingLevel", - default=jsInteresting.JS_OVERALL_MISMATCH, - help="minimum js/jsInteresting.py level for lithium to consider the testcase interesting") + default=js_interesting.JS_OVERALL_MISMATCH, + help="minimum js_interesting level for lithium to consider the testcase interesting") parser.add_option("--timeout", type="int", dest="timeout", default=10, @@ -260,23 +273,23 @@ def parseOptions(args): if not os.path.exists(options.jsengine): raise Exception("js shell does not exist: " + options.jsengine) - # For jsInteresting: + # For js_interesting: options.valgrind = False - options.shellIsDeterministic = True # We shouldn't be in compareJIT with a non-deterministic build - options.collector = createCollector.createCollector("jsfunfuzz") + options.shellIsDeterministic = True # We shouldn't be in compare_jit with a non-deterministic build + options.collector = create_collector.createCollector("jsfunfuzz") return options # For use by Lithium and autoBisect. (autoBisect calls init multiple times because it changes the js engine name) def init(args): - global gOptions + global gOptions # pylint: disable=invalid-name,global-statement gOptions = parseOptions(args) -# FIXME: _args is unused here, we should check if it can be removed? -def interesting(_args, tempPrefix): - actualLevel = compareLevel( +# FIXME: _args is unused here, we should check if it can be removed? # pylint: disable=fixme +def interesting(_args, tempPrefix): # pylint: disable=invalid-name + actualLevel = compareLevel( # pylint: disable=invalid-name gOptions.jsengine, gOptions.flags, gOptions.infilename, tempPrefix, gOptions, False, False)[0] return actualLevel >= gOptions.minimumInterestingLevel @@ -285,8 +298,8 @@ def main(): import tempfile options = parseOptions(sys.argv[1:]) print(compareLevel( - options.jsengine, options.flags, options.infilename, - tempfile.mkdtemp("compareJITmain"), options, True, False)[0]) + options.jsengine, options.flags, options.infilename, # pylint: disable=no-member + tempfile.mkdtemp("compare_jitmain"), options, True, False)[0]) if __name__ == "__main__": diff --git a/src/funfuzz/js/compile_shell.py b/src/funfuzz/js/compile_shell.py new file mode 100644 index 000000000..e1ae37867 --- /dev/null +++ b/src/funfuzz/js/compile_shell.py @@ -0,0 +1,763 @@ +# coding=utf-8 +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +"""Compiles SpiderMonkey shells on different platforms using various specified configuration parameters. +""" + +from __future__ import absolute_import, print_function, unicode_literals + +import copy +import ctypes +import io +import multiprocessing +import os +import shutil +import subprocess +import sys +import tarfile +import traceback +from optparse import OptionParser # pylint: disable=deprecated-module + +from . import build_options +from . import inspect_shell +from ..util import hg_helpers +from ..util import s3cache +from ..util import subprocesses as sps +from ..util.LockDir import LockDir + +S3_SHELL_CACHE_DIRNAME = 'shell-cache' # Used by autoBisect + +if sps.isWin: + MAKE_BINARY = b"mozmake" + CLANG_PARAMS = b"-fallback" + # CLANG_ASAN_PARAMS = b"-fsanitize=address -Dxmalloc=myxmalloc" + # Note that Windows ASan builds are still a work-in-progress + CLANG_ASAN_PARAMS = b"" +else: + MAKE_BINARY = b"make" + CLANG_PARAMS = b"-Qunused-arguments" + # See https://bugzilla.mozilla.org/show_bug.cgi?id=935795#c3 for some of the following flags: + # CLANG_ASAN_PARAMS = b"-fsanitize=address -Dxmalloc=myxmalloc -mllvm -asan-stack=0" + # The flags above seem to fix a problem not on the js shell. + CLANG_ASAN_PARAMS = b"-fsanitize=address -Dxmalloc=myxmalloc" + SSE2_FLAGS = b"-msse2 -mfpmath=sse" # See bug 948321 + CLANG_X86_FLAG = b"-arch i386" + +if multiprocessing.cpu_count() > 2: + COMPILATION_JOBS = ((multiprocessing.cpu_count() * 5) // 4) +elif sps.isARMv7l: + COMPILATION_JOBS = 3 # An ARM board +else: + COMPILATION_JOBS = 3 # Other single/dual core computers + + +class CompiledShell(object): # pylint: disable=missing-docstring,too-many-instance-attributes,too-many-public-methods + def __init__(self, buildOpts, hgHash): + self.shellNameWithoutExt = build_options.computeShellName(buildOpts, hgHash) # pylint: disable=invalid-name + # pylint: disable=invalid-name + self.shellNameWithExt = self.shellNameWithoutExt + (b".exe" if sps.isWin else b"") + self.hgHash = hgHash # pylint: disable=invalid-name + self.build_options = buildOpts + + self.jsObjdir = '' # pylint: disable=invalid-name + + self.cfg = '' + self.destDir = '' # pylint: disable=invalid-name + self.addedEnv = b"" # pylint: disable=invalid-name + self.fullEnv = b"" # pylint: disable=invalid-name + self.jsCfgFile = '' # pylint: disable=invalid-name + + self.jsMajorVersion = '' # pylint: disable=invalid-name + self.jsVersion = '' # pylint: disable=invalid-name + + def getCfgCmdExclEnv(self): # pylint: disable=invalid-name,missing-docstring,missing-return-doc + # pylint: disable=missing-return-type-doc + return self.cfg + + def setCfgCmdExclEnv(self, cfg): # pylint: disable=invalid-name,missing-docstring + self.cfg = cfg + + def setEnvAdded(self, addedEnv): # pylint: disable=invalid-name,missing-docstring + self.addedEnv = addedEnv + + def getEnvAdded(self): # pylint: disable=invalid-name,missing-docstring,missing-return-doc,missing-return-type-doc + return self.addedEnv + + def setEnvFull(self, fullEnv): # pylint: disable=invalid-name,missing-docstring + self.fullEnv = fullEnv + + def getEnvFull(self): # pylint: disable=invalid-name,missing-docstring,missing-return-doc,missing-return-type-doc + return self.fullEnv + + def getHgHash(self): # pylint: disable=invalid-name,missing-docstring,missing-return-doc,missing-return-type-doc + return self.hgHash + + def getJsCfgPath(self): # pylint: disable=invalid-name,missing-docstring,missing-return-doc,missing-return-type-doc + self.jsCfgFile = sps.normExpUserPath(os.path.join(self.getRepoDirJsSrc(), 'configure')) + assert os.path.isfile(self.jsCfgFile) + return self.jsCfgFile + + def getJsObjdir(self): # pylint: disable=invalid-name,missing-docstring,missing-return-doc,missing-return-type-doc + return self.jsObjdir + + def setJsObjdir(self, oDir): # pylint: disable=invalid-name,missing-docstring + self.jsObjdir = oDir + + def getRepoDir(self): # pylint: disable=invalid-name,missing-docstring,missing-return-doc,missing-return-type-doc + return self.build_options.repoDir + + def getRepoDirJsSrc(self): # pylint: disable=invalid-name,missing-docstring,missing-return-doc + # pylint: disable=missing-return-type-doc + return sps.normExpUserPath(os.path.join(self.getRepoDir(), 'js', 'src')) + + def getRepoName(self): # pylint: disable=invalid-name,missing-docstring,missing-return-doc,missing-return-type-doc + return hg_helpers.getRepoNameFromHgrc(self.build_options.repoDir) + + def getS3TarballWithExt(self): # pylint: disable=invalid-name,missing-docstring,missing-return-doc + # pylint: disable=missing-return-type-doc + return self.getShellNameWithoutExt() + '.tar.bz2' + + def getS3TarballWithExtFullPath(self): # pylint: disable=invalid-name,missing-docstring,missing-return-doc + # pylint: disable=missing-return-type-doc + return sps.normExpUserPath(os.path.join(ensureCacheDir(), self.getS3TarballWithExt())) + + def getShellCacheDir(self): # pylint: disable=invalid-name,missing-docstring,missing-return-doc + # pylint: disable=missing-return-type-doc + return sps.normExpUserPath(os.path.join(ensureCacheDir(), self.getShellNameWithoutExt())) + + def getShellCacheFullPath(self): # pylint: disable=invalid-name,missing-docstring,missing-return-doc + # pylint: disable=missing-return-type-doc + return sps.normExpUserPath(os.path.join(self.getShellCacheDir(), self.getShellNameWithExt())) + + def getShellCompiledPath(self): # pylint: disable=invalid-name,missing-docstring,missing-return-doc + # pylint: disable=missing-return-type-doc + return sps.normExpUserPath( + os.path.join(self.getJsObjdir(), 'dist', 'bin', 'js' + ('.exe' if sps.isWin else ''))) + + def getShellCompiledRunLibsPath(self): # pylint: disable=invalid-name,missing-docstring,missing-return-doc + # pylint: disable=missing-return-type-doc + libs_list = [ + sps.normExpUserPath(os.path.join(self.getJsObjdir(), 'dist', 'bin', runLib)) + for runLib in inspect_shell.ALL_RUN_LIBS + ] + return libs_list + + def getShellNameWithExt(self): # pylint: disable=invalid-name,missing-docstring,missing-return-doc + # pylint: disable=missing-return-type-doc + return self.shellNameWithExt + + def getShellNameWithoutExt(self): # pylint: disable=invalid-name,missing-docstring,missing-return-doc + # pylint: disable=missing-return-type-doc + return self.shellNameWithoutExt + + # Version numbers + def getMajorVersion(self): # pylint: disable=invalid-name,missing-docstring,missing-return-doc + # pylint: disable=missing-return-type-doc + return self.jsMajorVersion + + def setMajorVersion(self, jsMajorVersion): # pylint: disable=invalid-name,missing-docstring + self.jsMajorVersion = jsMajorVersion + + def getVersion(self): # pylint: disable=invalid-name,missing-docstring,missing-return-doc,missing-return-type-doc + return self.jsVersion + + def setVersion(self, jsVersion): # pylint: disable=invalid-name,missing-docstring + self.jsVersion = jsVersion + + +def ensureCacheDir(): # pylint: disable=invalid-name,missing-return-doc,missing-return-type-doc + """Return a cache directory for compiled shells to live in, and create one if needed.""" + cache_dir = os.path.join(sps.normExpUserPath('~'), 'shell-cache') + ensureDir(cache_dir) + + # Expand long Windows paths (overcome legacy MS-DOS 8.3 stuff) + # This has to occur after the shell-cache directory is created + if sps.isWin: # adapted from http://stackoverflow.com/a/3931799 + if sys.version_info.major == 2: + utext = unicode # noqa pylint: disable=redefined-builtin,undefined-variable,unicode-builtin + else: + utext = str + win_temp_dir = utext(cache_dir) + get_long_path_name = ctypes.windll.kernel32.GetLongPathNameW + unicode_buf = ctypes.create_unicode_buffer(get_long_path_name(win_temp_dir, 0, 0)) + get_long_path_name(win_temp_dir, unicode_buf, len(unicode_buf)) + cache_dir = sps.normExpUserPath(str(unicode_buf.value)) # convert back to a str + + return cache_dir + + +def ensureDir(directory): # pylint: disable=invalid-name,missing-param-doc,missing-type-doc + """Create a directory, if it does not already exist.""" + if not os.path.exists(directory): + os.mkdir(directory) + assert os.path.isdir(directory) + + +def autoconfRun(cwDir): # pylint: disable=invalid-name,missing-param-doc,missing-type-doc + """Run autoconf binaries corresponding to the platform.""" + if sps.isMac: + autoconf213_mac_bin = '/usr/local/Cellar/autoconf213/2.13/bin/autoconf213' \ + if sps.isProgramInstalled('brew') else 'autoconf213' + # Total hack to support new and old Homebrew configs, we can probably just call autoconf213 + if not os.path.isfile(sps.normExpUserPath(autoconf213_mac_bin)): + autoconf213_mac_bin = 'autoconf213' + subprocess.check_call([autoconf213_mac_bin], cwd=cwDir) + elif sps.isLinux: + # FIXME: We should use a method that is similar to the client.mk one, as per # pylint: disable=fixme + # https://github.com/MozillaSecurity/funfuzz/issues/9 + try: + # Ubuntu + subprocess.check_call(['autoconf2.13'], cwd=cwDir) + except OSError: + # Fedora has a different name + subprocess.check_call(['autoconf-2.13'], cwd=cwDir) + elif sps.isWin: + # Windows needs to call sh to be able to find autoconf. + subprocess.check_call(['sh', 'autoconf-2.13'], cwd=cwDir) + + +def cfgJsCompile(shell): # pylint: disable=invalid-name,missing-param-doc,missing-raises-doc,missing-type-doc + """Configures, compiles and copies a js shell according to required parameters.""" + print("Compiling...") # Print *with* a trailing newline to avoid breaking other stuff + os.mkdir(sps.normExpUserPath(os.path.join(shell.getShellCacheDir(), 'objdir-js'))) + shell.setJsObjdir(sps.normExpUserPath(os.path.join(shell.getShellCacheDir(), 'objdir-js'))) + + autoconfRun(shell.getRepoDirJsSrc()) + configure_try_count = 0 + while True: + try: + cfgBin(shell) + break + except Exception as ex: # pylint: disable=broad-except + configure_try_count += 1 + if configure_try_count > 3: + print("Configuration of the js binary failed 3 times.") + raise + # This exception message is returned from sps.captureStdout via cfgBin. + # No idea why this is sps.isLinux as well.. + if sps.isLinux or (sps.isWin and 'Windows conftest.exe configuration permission' in repr(ex)): + print("Trying once more...") + continue + compileJs(shell) + inspect_shell.verifyBinary(shell) + + compile_log = sps.normExpUserPath(os.path.join(shell.getShellCacheDir(), + shell.getShellNameWithoutExt() + '.fuzzmanagerconf')) + if not os.path.isfile(compile_log): + envDump(shell, compile_log) + + +def cfgBin(shell): # pylint: disable=invalid-name,missing-param-doc,missing-type-doc,too-complex,too-many-branches + # pylint: disable=too-many-statements + """Configure a binary according to required parameters.""" + cfg_cmds = [] + cfg_env = copy.deepcopy(os.environ) + orig_cfg_env = copy.deepcopy(os.environ) + cfg_env[b"AR"] = b"ar" + if sps.isARMv7l: + # 32-bit shell on ARM boards, e.g. odroid boards. + # This is tested on Ubuntu 14.04 with necessary armel libraries (force)-installed. + assert shell.build_options.enable32, 'arm7vl boards are only 32-bit, armv8 boards will be 64-bit.' + if not shell.build_options.enableHardFp: + cfg_env[b"CC"] = b"gcc-4.7 -mfloat-abi=softfp -B/usr/lib/gcc/arm-linux-gnueabi/4.7" + cfg_env[b"CXX"] = b"g++-4.7 -mfloat-abi=softfp -B/usr/lib/gcc/arm-linux-gnueabi/4.7" + cfg_cmds.append('sh') + cfg_cmds.append(os.path.normpath(shell.getJsCfgPath())) + # From mjrosenb: things might go wrong if these three lines are not present for + # compiling ARM on a 64-bit host machine. Not needed if compiling on the board itself. + # cfg_cmds.append('--target=arm-linux-gnueabi') + # cfg_cmds.append('--with-arch=armv7-a') + # cfg_cmds.append('--with-thumb') + if not shell.build_options.enableHardFp: + cfg_cmds.append('--target=arm-linux-gnueabi') + elif shell.build_options.enable32 and os.name == 'posix': + # 32-bit shell on Mac OS X 10.10 Yosemite and greater + if sps.isMac: + assert sps.macVer() >= [10, 10] # We no longer support 10.9 Mavericks and prior. + # Uses system clang + cfg_env[b"CC"] = cfg_env[b"HOST_CC"] = b"clang %s %s" % (CLANG_PARAMS, SSE2_FLAGS) + cfg_env[b"CXX"] = cfg_env[b"HOST_CXX"] = b"clang++ %s %s" % (CLANG_PARAMS, SSE2_FLAGS) + if shell.build_options.buildWithAsan: + cfg_env[b"CC"] += b" " + CLANG_ASAN_PARAMS + cfg_env[b"CXX"] += b" " + CLANG_ASAN_PARAMS + cfg_env[b"CC"] += b" " + CLANG_X86_FLAG # only needed for CC, not HOST_CC + cfg_env[b"CXX"] += b" " + CLANG_X86_FLAG # only needed for CXX, not HOST_CXX + cfg_env[b"RANLIB"] = b"ranlib" + cfg_env[b"AS"] = b"$CC" + cfg_env[b"LD"] = b"ld" + cfg_env[b"STRIP"] = b"strip -x -S" + cfg_env[b"CROSS_COMPILE"] = b"1" + if sps.isProgramInstalled('brew'): + cfg_env[b"AUTOCONF"] = b"/usr/local/Cellar/autoconf213/2.13/bin/autoconf213" + # Hacked up for new and old Homebrew configs, we can probably just call autoconf213 + if not os.path.isfile(sps.normExpUserPath(cfg_env[b"AUTOCONF"])): + cfg_env[b"AUTOCONF"] = b"autoconf213" + cfg_cmds.append('sh') + cfg_cmds.append(os.path.normpath(shell.getJsCfgPath())) + cfg_cmds.append('--target=i386-apple-darwin14.5.0') # Yosemite 10.10.5 + if shell.build_options.buildWithAsan: + cfg_cmds.append('--enable-address-sanitizer') + if shell.build_options.enableSimulatorArm32: + # --enable-arm-simulator became --enable-simulator=arm in rev 25e99bc12482 + # but unknown flags are ignored, so we compile using both till Fx38 ESR is deprecated + # Newer configure.in changes mean that things blow up if unknown/removed configure + # options are entered, so specify it only if it's requested. + if shell.build_options.enableArmSimulatorObsolete: + cfg_cmds.append('--enable-arm-simulator') + cfg_cmds.append('--enable-simulator=arm') + # 32-bit shell on 32/64-bit x86 Linux + elif sps.isLinux and not sps.isARMv7l: + cfg_env[b"PKG_CONFIG_LIBDIR"] = b"/usr/lib/pkgconfig" + if shell.build_options.buildWithClang: + cfg_env[b"CC"] = cfg_env[b"HOST_CC"] = str( + "clang %s %s %s" % (CLANG_PARAMS, SSE2_FLAGS, CLANG_X86_FLAG)) + cfg_env[b"CXX"] = cfg_env[b"HOST_CXX"] = str( + "clang++ %s %s %s" % (CLANG_PARAMS, SSE2_FLAGS, CLANG_X86_FLAG)) + else: + # apt-get `lib32z1 gcc-multilib g++-multilib` first, if on 64-bit Linux. + cfg_env[b"CC"] = b"gcc -m32 %s" % SSE2_FLAGS + cfg_env[b"CXX"] = b"g++ -m32 %s" % SSE2_FLAGS + if shell.build_options.buildWithAsan: + cfg_env[b"CC"] += b" " + CLANG_ASAN_PARAMS + cfg_env[b"CXX"] += b" " + CLANG_ASAN_PARAMS + cfg_cmds.append('sh') + cfg_cmds.append(os.path.normpath(shell.getJsCfgPath())) + cfg_cmds.append('--target=i686-pc-linux') + if shell.build_options.buildWithAsan: + cfg_cmds.append('--enable-address-sanitizer') + if shell.build_options.enableSimulatorArm32: + # --enable-arm-simulator became --enable-simulator=arm in rev 25e99bc12482 + # but unknown flags are ignored, so we compile using both till Fx38 ESR is deprecated + # Newer configure.in changes mean that things blow up if unknown/removed configure + # options are entered, so specify it only if it's requested. + if shell.build_options.enableArmSimulatorObsolete: + cfg_cmds.append('--enable-arm-simulator') + cfg_cmds.append('--enable-simulator=arm') + else: + cfg_cmds.append('sh') + cfg_cmds.append(os.path.normpath(shell.getJsCfgPath())) + # 64-bit shell on Mac OS X 10.10 Yosemite and greater + elif sps.isMac and sps.macVer() >= [10, 10] and not shell.build_options.enable32: + cfg_env[b"CC"] = b"clang " + CLANG_PARAMS + cfg_env[b"CXX"] = b"clang++ " + CLANG_PARAMS + if shell.build_options.buildWithAsan: + cfg_env[b"CC"] += b" " + CLANG_ASAN_PARAMS + cfg_env[b"CXX"] += b" " + CLANG_ASAN_PARAMS + if sps.isProgramInstalled('brew'): + cfg_env[b"AUTOCONF"] = b"/usr/local/Cellar/autoconf213/2.13/bin/autoconf213" + cfg_cmds.append('sh') + cfg_cmds.append(os.path.normpath(shell.getJsCfgPath())) + cfg_cmds.append('--target=x86_64-apple-darwin14.5.0') # Yosemite 10.10.5 + if shell.build_options.buildWithAsan: + cfg_cmds.append('--enable-address-sanitizer') + if shell.build_options.enableSimulatorArm64: + cfg_cmds.append('--enable-simulator=arm64') + + elif sps.isWin: + cfg_env[b"MAKE"] = b"mozmake" # Workaround for bug 948534 + if shell.build_options.buildWithClang: + cfg_env[b"CC"] = b"clang-cl.exe " + CLANG_PARAMS + cfg_env[b"CXX"] = b"clang-cl.exe " + CLANG_PARAMS + if shell.build_options.buildWithAsan: + cfg_env[b"CFLAGS"] = CLANG_ASAN_PARAMS + cfg_env[b"CXXFLAGS"] = CLANG_ASAN_PARAMS + cfg_env[b"LDFLAGS"] = (b"clang_rt.asan_dynamic-x86_64.lib " + b"clang_rt.asan_dynamic_runtime_thunk-x86_64.lib " + b"clang_rt.asan_dynamic-x86_64.dll") + cfg_env[b"HOST_CFLAGS"] = b" " + cfg_env[b"HOST_CXXFLAGS"] = b" " + cfg_env[b"HOST_LDFLAGS"] = b" " + cfg_env[b"LIB"] += br"C:\Program Files\LLVM\lib\clang\4.0.0\lib\windows" + cfg_cmds.append('sh') + cfg_cmds.append(os.path.normpath(shell.getJsCfgPath())) + if shell.build_options.enable32: + if shell.build_options.enableSimulatorArm32: + # --enable-arm-simulator became --enable-simulator=arm in rev 25e99bc12482 + # but unknown flags are ignored, so we compile using both till Fx38 ESR is deprecated + # Newer configure.in changes mean that things blow up if unknown/removed configure + # options are entered, so specify it only if it's requested. + if shell.build_options.enableArmSimulatorObsolete: + cfg_cmds.append('--enable-arm-simulator') + cfg_cmds.append('--enable-simulator=arm') + else: + cfg_cmds.append('--host=x86_64-pc-mingw32') + cfg_cmds.append('--target=x86_64-pc-mingw32') + if shell.build_options.enableSimulatorArm64: + cfg_cmds.append('--enable-simulator=arm64') + if shell.build_options.buildWithAsan: + cfg_cmds.append('--enable-address-sanitizer') + else: + # We might still be using GCC on Linux 64-bit, so do not use clang unless Asan is specified + if shell.build_options.buildWithClang: + cfg_env[b"CC"] = b"clang " + CLANG_PARAMS + cfg_env[b"CXX"] = b"clang++ " + CLANG_PARAMS + if shell.build_options.buildWithAsan: + cfg_env[b"CC"] += b" " + CLANG_ASAN_PARAMS + cfg_env[b"CXX"] += b" " + CLANG_ASAN_PARAMS + cfg_cmds.append('sh') + cfg_cmds.append(os.path.normpath(shell.getJsCfgPath())) + if shell.build_options.buildWithAsan: + cfg_cmds.append('--enable-address-sanitizer') + + if shell.build_options.buildWithClang: + if sps.isWin: + assert b"clang-cl" in cfg_env[b"CC"] + assert b"clang-cl" in cfg_env[b"CXX"] + else: + assert b"clang" in cfg_env[b"CC"] + assert b"clang++" in cfg_env[b"CXX"] + cfg_cmds.append('--disable-jemalloc') # See bug 1146895 + + if shell.build_options.enableDbg: + cfg_cmds.append('--enable-debug') + elif shell.build_options.disableDbg: + cfg_cmds.append('--disable-debug') + + if shell.build_options.enableOpt: + cfg_cmds.append('--enable-optimize' + ('=-O1' if shell.build_options.buildWithVg else '')) + elif shell.build_options.disableOpt: + cfg_cmds.append('--disable-optimize') + if shell.build_options.enableProfiling: # Now obsolete, retained for backward compatibility + cfg_cmds.append('--enable-profiling') + if shell.build_options.disableProfiling: + cfg_cmds.append('--disable-profiling') + + if shell.build_options.enableMoreDeterministic: + # Fuzzing tweaks for more useful output, implemented in bug 706433 + cfg_cmds.append('--enable-more-deterministic') + if shell.build_options.enableOomBreakpoint: # Extra debugging help for OOM assertions + cfg_cmds.append('--enable-oom-breakpoint') + if shell.build_options.enableWithoutIntlApi: # Speeds up compilation but is non-default + cfg_cmds.append('--without-intl-api') + + if shell.build_options.buildWithVg: + cfg_cmds.append('--enable-valgrind') + cfg_cmds.append('--disable-jemalloc') + + # We add the following flags by default. + if os.name == 'posix': + cfg_cmds.append('--with-ccache') + cfg_cmds.append('--enable-gczeal') + cfg_cmds.append('--enable-debug-symbols') # gets debug symbols on opt shells + cfg_cmds.append('--disable-tests') + + if os.name == 'nt': + # FIXME: Replace this with sps.shellify. # pylint: disable=fixme + counter = 0 + for entry in cfg_cmds: + if os.sep in entry: + assert sps.isWin # MozillaBuild on Windows sometimes confuses "/" and "\". + cfg_cmds[counter] = cfg_cmds[counter].replace(os.sep, '//') + counter = counter + 1 + + # Print whatever we added to the environment + env_vars = [] + for env_var in set(cfg_env.keys()) - set(orig_cfg_env.keys()): + str_to_be_appended = str(env_var + '="' + cfg_env[str(env_var)] + + '"' if " " in cfg_env[str(env_var)] else env_var + + "=" + cfg_env[str(env_var)]) + env_vars.append(str_to_be_appended) + sps.vdump('Command to be run is: ' + sps.shellify(env_vars) + ' ' + sps.shellify(cfg_cmds)) + + js_objdir = shell.getJsObjdir() + assert os.path.isdir(js_objdir) + + if sps.isWin: + changed_cfg_cmds = [] + for entry in cfg_cmds: + # For JS, quoted from :glandium: "the way icu subconfigure is called is what changed. + # but really, the whole thing likes forward slashes way better" + # See bug 1038590 comment 9. + if '\\' in entry: + entry = entry.replace('\\', '/') + changed_cfg_cmds.append(entry) + sps.captureStdout(changed_cfg_cmds, ignoreStderr=True, currWorkingDir=js_objdir, env=cfg_env) + else: + sps.captureStdout(cfg_cmds, ignoreStderr=True, currWorkingDir=js_objdir, env=cfg_env) + + shell.setEnvAdded(env_vars) + shell.setEnvFull(cfg_env) + shell.setCfgCmdExclEnv(cfg_cmds) + + +def compileJs(shell): # pylint: disable=invalid-name,missing-param-doc,missing-raises-doc,missing-type-doc + """Compile and copy a binary.""" + try: + cmd_list = [MAKE_BINARY, '-C', shell.getJsObjdir(), '-j' + str(COMPILATION_JOBS), '-s'] + out = sps.captureStdout(cmd_list, combineStderr=True, ignoreExitCode=True, + currWorkingDir=shell.getJsObjdir(), env=shell.getEnvFull())[0] + except Exception as ex: # pylint: disable=broad-except + # This exception message is returned from sps.captureStdout via cmd_list. + if (sps.isLinux or sps.isMac) and \ + ('GCC running out of memory' in repr(ex) or 'Clang running out of memory' in repr(ex)): + # FIXME: Absolute hack to retry after hitting OOM. # pylint: disable=fixme + print("Trying once more due to the compiler running out of memory...") + out = sps.captureStdout(cmd_list, combineStderr=True, ignoreExitCode=True, + currWorkingDir=shell.getJsObjdir(), env=shell.getEnvFull())[0] + # A non-zero error can be returned during make, but eventually a shell still gets compiled. + if os.path.exists(shell.getShellCompiledPath()): + print("A shell was compiled even though there was a non-zero exit code. Continuing...") + else: + print("%s did not result in a js shell:" % MAKE_BINARY.decode("utf-8", errors="replace")) + raise + + if os.path.exists(shell.getShellCompiledPath()): + shutil.copy2(shell.getShellCompiledPath(), shell.getShellCacheFullPath()) + for run_lib in shell.getShellCompiledRunLibsPath(): + if os.path.isfile(run_lib): + shutil.copy2(run_lib, shell.getShellCacheDir()) + + version = extractVersions(shell.getJsObjdir()) + shell.setMajorVersion(version.split('.')[0]) + shell.setVersion(version) + + if sps.isLinux: + # Restrict this to only Linux for now. At least Mac OS X needs some (possibly *.a) + # files in the objdir or else the stacks from failing testcases will lack symbols. + shutil.rmtree(sps.normExpUserPath(os.path.join(shell.getShellCacheDir(), 'objdir-js'))) + else: + print(out.decode("utf-8", errors="replace")) + raise Exception(MAKE_BINARY + " did not result in a js shell, no exception thrown.") + + +def createBustedFile(filename, e): # pylint: disable=invalid-name,missing-param-doc,missing-type-doc + """Create a .busted file with the exception message and backtrace included.""" + with open(filename, 'wb') as f: + f.write("Caught exception %s (%s)\n" % (repr(e), str(e))) + f.write("Backtrace:\n") + f.write(traceback.format_exc() + "\n") + print("Compilation failed (%s) (details in %s)" % (e.decode("utf-8", errors="replace"), + filename.decode("utf-8", errors="replace"))) + + +def envDump(shell, log): # pylint: disable=invalid-name,missing-param-doc,missing-type-doc + """Dump environment to a .fuzzmanagerconf file.""" + # Platform and OS detection for the spec, part of which is in: + # https://wiki.mozilla.org/Security/CrashSignatures + if sps.isARMv7l: + fmconf_platform = 'ARM' + elif sps.isARMv7l and not shell.build_options.enable32: + print("ARM64 is not supported in .fuzzmanagerconf yet.") + fmconf_platform = 'ARM64' + elif shell.build_options.enable32: + fmconf_platform = 'x86' + else: + fmconf_platform = 'x86-64' + + if sps.isLinux: + fmconf_os = 'linux' + elif sps.isMac: + fmconf_os = 'macosx' + elif sps.isWin: + fmconf_os = 'windows' + + with open(log, 'ab') as f: + f.write('# Information about shell:\n# \n') + + f.write('# Create another shell in shell-cache like this one:\n') + f.write('# python -u -m %s -b "%s" -r %s\n# \n' % ('funfuzz.js.compile_shell', + shell.build_options.build_options_str, shell.getHgHash())) + + f.write('# Full environment is:\n') + f.write('# %s\n# \n' % str(shell.getEnvFull())) + + f.write('# Full configuration command with needed environment variables is:\n') + f.write('# %s %s\n# \n' % (sps.shellify(shell.getEnvAdded()), + sps.shellify(shell.getCfgCmdExclEnv()))) + + # .fuzzmanagerconf details + f.write('\n') + f.write('[Main]\n') + f.write('platform = %s\n' % fmconf_platform) + f.write('product = %s\n' % shell.getRepoName()) + f.write('product_version = %s\n' % shell.getHgHash()) + f.write('os = %s\n' % fmconf_os) + + f.write('\n') + f.write('[Metadata]\n') + f.write('buildFlags = %s\n' % shell.build_options.build_options_str) + f.write('majorVersion = %s\n' % shell.getMajorVersion()) + f.write('pathPrefix = %s%s\n' % (shell.getRepoDir(), + '/' if not shell.getRepoDir().endswith('/') else '')) + f.write('version = %s\n' % shell.getVersion()) + + +def extractVersions(objdir): # pylint: disable=invalid-name,missing-param-doc,missing-return-doc + # pylint: disable=missing-return-type-doc,missing-type-doc + """Extract the version from js.pc and put it into *.fuzzmanagerconf.""" + jspc_dir = sps.normExpUserPath(os.path.join(objdir, 'js', 'src')) + jspc_name = os.path.join(jspc_dir, 'js.pc') + # Moved to /js/src/build/, see bug 1262241, Fx55 rev 2159959522f4 + jspc_new_dir = os.path.join(jspc_dir, 'build') + jspc_new_name = os.path.join(jspc_new_dir, 'js.pc') + + def fixateVer(pcfile): # pylint: disable=invalid-name,missing-param-doc,missing-return-doc,missing-return-type-doc + # pylint: disable=missing-type-doc + """Returns the current version number (47.0a2).""" + with io.open(pcfile, mode='r', encoding="utf-8", errors="replace") as f: + for line in f: + if line.startswith('Version: '): + # Sample line: 'Version: 47.0a2' + return line.split(': ')[1].rstrip() + + if os.path.isfile(jspc_name): + return fixateVer(jspc_name) + elif os.path.isfile(jspc_new_name): + return fixateVer(jspc_new_name) + + +def getLockDirPath(repoDir, tboxIdentifier=''): # pylint: disable=invalid-name,missing-param-doc,missing-return-doc + # pylint: disable=missing-return-type-doc,missing-type-doc + """Return the name of the lock directory, which is in the cache directory by default.""" + lockdir_name = ['shell', os.path.basename(repoDir), 'lock'] + if tboxIdentifier: + lockdir_name.append(tboxIdentifier) + return os.path.join(ensureCacheDir(), '-'.join(lockdir_name)) + + +def makeTestRev(options): # pylint: disable=invalid-name,missing-docstring,missing-return-doc,missing-return-type-doc + def testRev(rev): # pylint: disable=invalid-name,missing-docstring,missing-return-doc,missing-return-type-doc + shell = CompiledShell(options.build_options, rev) + print("Rev %s:" % rev.decode("utf-8", errors="replace"), end=" ") + + try: + obtainShell(shell, updateToRev=rev) + except Exception: # pylint: disable=broad-except + return (options.compilationFailedLabel, 'compilation failed') + + print("Testing...", end=" ") + return options.testAndLabel(shell.getShellCacheFullPath(), rev) + return testRev + + +def obtainShell(shell, updateToRev=None, updateLatestTxt=False): # pylint: disable=invalid-name,missing-param-doc + # pylint: disable=missing-raises-doc,missing-type-doc,too-many-branches,too-complex,too-many-statements + """Obtain a js shell. Keep the objdir for now, especially .a files, for symbols.""" + assert os.path.isdir(getLockDirPath(shell.build_options.repoDir)) + cached_no_shell = shell.getShellCacheFullPath() + ".busted" + + if os.path.isfile(shell.getShellCacheFullPath()): + # Don't remove the comma at the end of this line, and thus remove the newline printed. + # We would break JSBugMon. + print("Found cached shell...") + # Assuming that since the binary is present, everything else (e.g. symbols) is also present + verifyFullWinPageHeap(shell.getShellCacheFullPath()) + return + elif os.path.isfile(cached_no_shell): + raise Exception("Found a cached shell that failed compilation...") + elif os.path.isdir(shell.getShellCacheDir()): + print("Found a cache dir without a successful/failed shell...") + sps.rmTreeIncludingReadOnly(shell.getShellCacheDir()) + + os.mkdir(shell.getShellCacheDir()) + hg_helpers.destroyPyc(shell.build_options.repoDir) + + s3cache_obj = s3cache.S3Cache(S3_SHELL_CACHE_DIRNAME) + use_s3cache = s3cache_obj.connect() + + if use_s3cache: + if s3cache_obj.downloadFile(shell.getShellNameWithoutExt() + '.busted', + shell.getShellCacheFullPath() + '.busted'): + raise Exception('Found a .busted file for rev ' + shell.getHgHash()) + + if s3cache_obj.downloadFile(shell.getShellNameWithoutExt() + '.tar.bz2', + shell.getS3TarballWithExtFullPath()): + print("Extracting shell...") + with tarfile.open(shell.getS3TarballWithExtFullPath(), 'r') as f: + f.extractall(shell.getShellCacheDir()) + # Delete tarball after downloading from S3 + os.remove(shell.getS3TarballWithExtFullPath()) + verifyFullWinPageHeap(shell.getShellCacheFullPath()) + return + + try: + if updateToRev: + updateRepo(shell.build_options.repoDir, updateToRev) + if shell.build_options.patchFile: + hg_helpers.patchHgRepoUsingMq(shell.build_options.patchFile, shell.getRepoDir()) + + cfgJsCompile(shell) + verifyFullWinPageHeap(shell.getShellCacheFullPath()) + except KeyboardInterrupt: + sps.rmTreeIncludingReadOnly(shell.getShellCacheDir()) + raise + except Exception as ex: + # Remove the cache dir, but recreate it with only the .busted file. + sps.rmTreeIncludingReadOnly(shell.getShellCacheDir()) + os.mkdir(shell.getShellCacheDir()) + createBustedFile(cached_no_shell, ex) + if use_s3cache: + s3cache_obj.uploadFileToS3(shell.getShellCacheFullPath() + '.busted') + raise + finally: + if shell.build_options.patchFile: + hg_helpers.hgQpopQrmAppliedPatch(shell.build_options.patchFile, shell.getRepoDir()) + + if use_s3cache: + s3cache_obj.compressAndUploadDirTarball(shell.getShellCacheDir(), shell.getS3TarballWithExtFullPath()) + if updateLatestTxt: + # So js-dbg-64-dm-darwin-cdcd33fd6e39 becomes js-dbg-64-dm-darwin-latest.txt with + # js-dbg-64-dm-darwin-cdcd33fd6e39 as its contents. + txt_info = '-'.join(shell.getS3TarballWithExt().split('-')[:-1] + ['latest']) + '.txt' + s3cache_obj.uploadStrToS3('', txt_info, shell.getS3TarballWithExt()) + os.remove(shell.getS3TarballWithExtFullPath()) + + +def updateRepo(repo, rev): # pylint: disable=invalid-name,missing-param-doc,missing-type-doc + """Update repository to the specific revision.""" + # Print *with* a trailing newline to avoid breaking other stuff + print("Updating to rev %s in the %s repository..." % (rev.decode("utf-8", errors="replace"), + repo.decode("utf-8", errors="replace"))) + sps.captureStdout(["hg", "-R", repo, 'update', '-C', '-r', rev], ignoreStderr=True) + + +def verifyFullWinPageHeap(shellPath): # pylint: disable=invalid-name,missing-param-doc,missing-type-doc + """Turn on full page heap verification on Windows.""" + # More info: https://msdn.microsoft.com/en-us/library/windows/hardware/ff543097(v=vs.85).aspx + # or https://blogs.msdn.microsoft.com/webdav_101/2010/06/22/detecting-heap-corruption-using-gflags-and-dumps/ + if sps.isWin: + gflags_bin_path = os.path.join(os.getenv('PROGRAMW6432'), 'Debugging Tools for Windows (x64)', 'gflags.exe') + if os.path.isfile(gflags_bin_path) and os.path.isfile(shellPath): + print(subprocess.check_output([gflags_bin_path.decode("utf-8", errors="replace"), + "-p", "/enable", shellPath.decode("utf-8", errors="replace"), "/full"])) + + +def main(): + """Build a shell and place it in the autoBisect cache.""" + usage = 'Usage: %prog [options]' + parser = OptionParser(usage) + parser.disable_interspersed_args() + + parser.set_defaults( + build_options="", + ) + + # Specify how the shell will be built. + parser.add_option('-b', '--build', + dest='build_options', + help="Specify build options, e.g. -b '--disable-debug --enable-optimize' " + "(python -m funfuzz.js.build_options --help)") + + parser.add_option('-r', '--rev', + dest='revision', + help='Specify revision to build') + + options = parser.parse_args()[0] + options.build_options = build_options.parseShellOptions(options.build_options) + + with LockDir(getLockDirPath(options.build_options.repoDir)): + if options.revision: + shell = CompiledShell(options.build_options, options.revision) + else: + local_orig_hg_hash = hg_helpers.getRepoHashAndId(options.build_options.repoDir)[0] + shell = CompiledShell(options.build_options, local_orig_hg_hash) + + obtainShell(shell, updateToRev=options.revision) + print(shell.getShellCacheFullPath()) + + +if __name__ == '__main__': + main() diff --git a/js/examples-compileShell.md b/src/funfuzz/js/examples.md similarity index 52% rename from js/examples-compileShell.md rename to src/funfuzz/js/examples.md index 26c49b511..7089a5fd7 100644 --- a/js/examples-compileShell.md +++ b/src/funfuzz/js/examples.md @@ -2,35 +2,35 @@ * To compile a debug 64-bit deterministic shell, do: -`funfuzz/js/compileShell.py -b "--enable-debug --enable-more-deterministic -R ~/trees/mozilla-central"` +`python -m funfuzz.js.compile_shell -b "--enable-debug --enable-more-deterministic -R ~/trees/mozilla-central"` * To compile an optimized 32-bit shell, do: -`funfuzz/js/compileShell.py -b "--32 --enable-optimize -R ~/trees/mozilla-central"` +`python -m funfuzz.js.compile_shell -b "--32 --enable-optimize -R ~/trees/mozilla-central"` By default, js should compile an optimized shell even without --enable-optimize explicitly specified. * To compile a debug 32-bit ARM-simulator shell, do: -`funfuzz/js/compileShell.py -b "--32 --enable-debug --enable-simulator=arm -R ~/trees/mozilla-central"` +`python -m funfuzz.js.compile_shell -b "--32 --enable-debug --enable-simulator=arm -R ~/trees/mozilla-central"` * To compile a debug 64-bit shell with AddressSanitizer (ASan) support, do: -`funfuzz/js/compileShell.py -b "--enable-debug --build-with-asan -R ~/trees/mozilla-central"` +`python -m funfuzz.js.compile_shell -b "--enable-debug --build-with-asan -R ~/trees/mozilla-central"` Note that this uses git to clone a specific known working revision of LLVM into `~/llvm`, compiles it, then uses this specific revision to compile SpiderMonkey. * To compile an optimized 64-bit shell with Valgrind support, do: -`funfuzz/js/compileShell.py -b "--enable-optimize --build-with-valgrind -R ~/trees/mozilla-central"` +`python -m funfuzz.js.compile_shell -b "--enable-optimize --build-with-valgrind -R ~/trees/mozilla-central"` * To test a patch with a debug 64-bit deterministic shell, do: -`funfuzz/js/compileShell.py -b "--enable-debug --enable-more-deterministic -R ~/trees/mozilla-central -P "` +`python -m funfuzz.js.compile_shell -b "--enable-debug --enable-more-deterministic -R ~/trees/mozilla-central -P "` Note that this **requires mq to be activated** in Mercurial and assumes that there are **no patches** in the patch queue. * To compile a debug 64-bit deterministic shell from a specific mozilla-central revision, do: -`funfuzz/js/compileShell.py -b "--enable-debug --enable-more-deterministic -R ~/trees/mozilla-central" -r ` +`python -m funfuzz.js.compile_shell -b "--enable-debug --enable-more-deterministic -R ~/trees/mozilla-central" -r ` diff --git a/src/funfuzz/js/faq.md b/src/funfuzz/js/faq.md new file mode 100644 index 000000000..c6f3bf4a3 --- /dev/null +++ b/src/funfuzz/js/faq.md @@ -0,0 +1,50 @@ +### FAQ: + +**Q: Why is "--enable-more-deterministic" recommended?** + +Fuzzing with this mode on allows us to run compare_jit, which runs testcases generated by jsfunfuzz using different flags and compares the output. In order to compare successfully, the shell should generate consistent output everytime with a fixed input. Since we do not ship deterministic shells by default, if testing deterministic shells, we do not run with compare_jit. + +**Q: What are average build times for SpiderMonkey?** + +On a decent Linux machine or a powerful Mac, both with 4 or more cores, 3-4 minutes on average. On Windows, probably 5-10 minutes. On an ARM ODROID board, up to an hour. + +**Q: How do I get a shell with a patch to be compiled together?** + +Use the -P notation, e.g.: + +`python -m funfuzz.js.compile_shell -b "--enable-debug --enable-more-deterministic -R ~/trees/mozilla-inbound -P ~/patch.diff"` + +assuming: +* mq is activated in `~/.hgrc` +* There are no other patches in the mq stack +* Patch is in `~/patch.diff` and can be applied cleanly + * Test by first doing `patch -p1 --dry-run < ~/patch.diff` in the base directory of the repository listed by -R, or the default. +* There is only one patch needed. If more than one patch is needed, first do a roll-up patch. + +**Q: Do these build configure flags get passed into the js configure scripts?** + +No, they are independent. We only implemented the flags that are most useful for fuzzing in the harness. + +**Q: Will the gecko-dev Git mirror of mozilla-central be supported?** + +The "-R" flag assumes a Mercurial clone of mozilla-central is passed in as an argument. Git repositories are not yet supported fully, and especially not for autoBisect. See [issue #2](https://github.com/MozillaSecurity/funfuzz/issues/2). + +**Q: Can I run multiple instances of compile_shell?** + +This is not recommended as it will slow down your computer. Running one instance of compile_shell will use the maximum number of cores as found by cpu_count() from the Python multiprocessing module, with the exception of ARM boards that tend to have slower cores. + +**Q: What kind of build does compile_shell do, and what files do it store on my machine?** + +It creates a clobber build, compiling in the `~/shell-cache` directory by default. There may also be a bunch of tempfiles created in the system temporary directory. + +**Q: How do I check if the SpiderMonkey build created is the one I specified?** + +Post-compilation, the harness does a bunch of verification tests to ensure that the desired build is created. To double check, run `getBuildConfiguration();` within the SpiderMonkey shell. If compile_shell isn't compiling as desired, file an issue! + +**Q: What happens if the build I want to compile causes a compilation error?** + +A `.busted` file is created in the `~/shell-cache` directory. This will notify the harness not to retry in the future since it is busted. However, if there is an error in the harness, file an issue detailing the build configurations and steps to reproduce, and once it is fixed, remove the corresponding file/directory in `~/shell-cache` and retry again. + +**Q: After compiling many shells, I'm now running out of disk space! What should I do?** + +Oops! You can remove the `~/shell-cache` directory to reclaim space, and reboot to clear system temporary directories. diff --git a/js/files-to-link.txt b/src/funfuzz/js/files_to_link.txt similarity index 100% rename from js/files-to-link.txt rename to src/funfuzz/js/files_to_link.txt diff --git a/js/inspectShell.py b/src/funfuzz/js/inspect_shell.py similarity index 59% rename from js/inspectShell.py rename to src/funfuzz/js/inspect_shell.py index 9e8739516..420a6cf87 100644 --- a/js/inspectShell.py +++ b/src/funfuzz/js/inspect_shell.py @@ -1,23 +1,21 @@ -#!/usr/bin/env python # coding=utf-8 -# pylint: disable=import-error,invalid-name,missing-docstring,wrong-import-position # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at https://mozilla.org/MPL/2.0/. +"""Allows inspection of the SpiderMonkey shell to ensure that it is compiled as intended with specified configurations. +""" + from __future__ import absolute_import, print_function import os import platform -import sys -path0 = os.path.dirname(os.path.abspath(__file__)) -path1 = os.path.abspath(os.path.join(path0, os.pardir, 'util')) -sys.path.append(path1) -import subprocesses as sps from lithium.interestingness.utils import env_with_path +from ..util import subprocesses as sps + RUN_NSPR_LIB = '' RUN_PLDS_LIB = '' RUN_PLC_LIB = '' @@ -70,10 +68,11 @@ ALL_RUN_LIBS.append(RUN_ICUTUD_LIB_EXCL_EXT + str(icu_ver) + '.dll') -def archOfBinary(binary): +def archOfBinary(binary): # pylint: disable=invalid-name,missing-param-doc,missing-raises-doc,missing-return-doc + # pylint: disable=missing-return-type-doc,missing-type-doc """Test if a binary is 32-bit or 64-bit.""" - unsplitFiletype = sps.captureStdout(['file', binary])[0] - filetype = unsplitFiletype.split(':', 1)[1] + unsplit_file_type = sps.captureStdout(['file', binary])[0] + filetype = unsplit_file_type.split(':', 1)[1] if sps.isWin: assert 'MS Windows' in filetype return '32' if 'Intel 80386 32-bit' in filetype else '64' @@ -88,96 +87,102 @@ def archOfBinary(binary): return '64' -def constructVgCmdList(errorCode=77): +def constructVgCmdList(errorCode=77): # pylint: disable=invalid-name,missing-param-doc,missing-return-doc + # pylint: disable=missing-return-type-doc,missing-type-doc """Construct default parameters needed to run valgrind with.""" - vgCmdList = [] - vgCmdList.append('valgrind') + valgrind_cmds = [] + valgrind_cmds.append('valgrind') if sps.isMac: - vgCmdList.append('--dsymutil=yes') - vgCmdList.append('--error-exitcode=' + str(errorCode)) + valgrind_cmds.append('--dsymutil=yes') + valgrind_cmds.append('--error-exitcode=' + str(errorCode)) if not sps.isARMv7l: # jseward mentioned that ARM does not need --smc-check= - vgCmdList.append('--smc-check=all-non-file') + valgrind_cmds.append('--smc-check=all-non-file') # See bug 913876 comment 18: - vgCmdList.append('--vex-iropt-register-updates=allregs-at-mem-access') - vgCmdList.append('--gen-suppressions=all') - vgCmdList.append('--leak-check=full') - vgCmdList.append('--errors-for-leak-kinds=definite') - vgCmdList.append('--show-leak-kinds=definite') - vgCmdList.append('--show-possibly-lost=no') - vgCmdList.append('--num-callers=50') - return vgCmdList - - -def shellSupports(shellPath, args): + valgrind_cmds.append('--vex-iropt-register-updates=allregs-at-mem-access') + valgrind_cmds.append('--gen-suppressions=all') + valgrind_cmds.append('--leak-check=full') + valgrind_cmds.append('--errors-for-leak-kinds=definite') + valgrind_cmds.append('--show-leak-kinds=definite') + valgrind_cmds.append('--show-possibly-lost=no') + valgrind_cmds.append('--num-callers=50') + return valgrind_cmds + + +def shellSupports(shellPath, args): # pylint: disable=invalid-name,missing-param-doc,missing-raises-doc + # pylint: disable=missing-return-doc,missing-return-type-doc,missing-type-doc """Return True if the shell likes the args. You can add support for a function, e.g. ['-e', 'foo()'], or a flag, e.g. ['-j', '-e', '42']. """ - retCode = testBinary(shellPath, args, False)[1] - if retCode == 0: + return_code = testBinary(shellPath, args, False)[1] + if return_code == 0: return True - elif 1 <= retCode <= 3: + elif 1 <= return_code <= 3: # Exit codes 1 through 3 are all plausible "non-support": # * "Usage error" is 1 in new js shell, 2 in old js shell, 2 in xpcshell. # * "Script threw an error" is 3 in most shells, but 1 in some versions (see bug 751425). # Since we want autoBisect to support all shell versions, allow all these exit codes. return False else: - raise Exception('Unexpected exit code in shellSupports ' + str(retCode)) + raise Exception('Unexpected exit code in shellSupports ' + str(return_code)) -def testBinary(shellPath, args, useValgrind): +def testBinary(shellPath, args, useValgrind): # pylint: disable=invalid-name,missing-param-doc,missing-return-doc + # pylint: disable=missing-return-type-doc,missing-type-doc """Test the given shell with the given args.""" - testCmd = (constructVgCmdList() if useValgrind else []) + [shellPath] + args - sps.vdump('The testing command is: ' + sps.shellify(testCmd)) - out, rCode = sps.captureStdout(testCmd, combineStderr=True, ignoreStderr=True, - ignoreExitCode=True, env=env_with_path( - os.path.dirname(os.path.abspath(shellPath)))) - sps.vdump('The exit code is: ' + str(rCode)) - return out, rCode + test_cmd = (constructVgCmdList() if useValgrind else []) + [shellPath] + args + sps.vdump('The testing command is: ' + sps.shellify(test_cmd)) + out, return_code = sps.captureStdout(test_cmd, combineStderr=True, ignoreStderr=True, + ignoreExitCode=True, env=env_with_path( + os.path.dirname(os.path.abspath(shellPath)))) + sps.vdump('The exit code is: ' + str(return_code)) + return out, return_code -def testJsShellOrXpcshell(s): +def testJsShellOrXpcshell(s): # pylint: disable=invalid-name,missing-param-doc,missing-return-doc + # pylint: disable=missing-return-type-doc,missing-type-doc """Test if a binary is a js shell or xpcshell.""" return 'xpcshell' if shellSupports(s, ['-e', 'Components']) else 'jsShell' -def queryBuildConfiguration(s, parameter): +def queryBuildConfiguration(s, parameter): # pylint: disable=invalid-name,missing-param-doc,missing-return-doc + # pylint: disable=missing-return-type-doc,missing-type-doc """Test if a binary is compiled with specified parameters, in getBuildConfiguration().""" ans = testBinary(s, ['-e', 'print(getBuildConfiguration()["' + parameter + '"])'], False)[0] return ans.find('true') != -1 -def testIsHardFpShellARM(s): +def testIsHardFpShellARM(s): # pylint: disable=invalid-name,missing-param-doc,missing-raises-doc,missing-return-doc + # pylint: disable=missing-return-type-doc,missing-type-doc """Test if the ARM shell is compiled with hardfp support.""" - readelfBin = '/usr/bin/readelf' - if os.path.exists(readelfBin): - newEnv = env_with_path(os.path.dirname(os.path.abspath(s))) - readelfOutput = sps.captureStdout([readelfBin, '-A', s], env=newEnv)[0] - return 'Tag_ABI_VFP_args: VFP registers' in readelfOutput + readelf_bin_path = '/usr/bin/readelf' + if os.path.exists(readelf_bin_path): + new_env = env_with_path(os.path.dirname(os.path.abspath(s))) + readelf_output = sps.captureStdout([readelf_bin_path, '-A', s], env=new_env)[0] + return 'Tag_ABI_VFP_args: VFP registers' in readelf_output else: raise Exception('readelf is not found.') -def verifyBinary(sh): +def verifyBinary(sh): # pylint: disable=invalid-name,missing-param-doc,missing-type-doc """Verify that the binary is compiled as intended.""" binary = sh.getShellCacheFullPath() - assert archOfBinary(binary) == ('32' if sh.buildOptions.enable32 else '64') + assert archOfBinary(binary) == ('32' if sh.build_options.enable32 else '64') # Testing for debug or opt builds are different because there can be hybrid debug-opt builds. - assert queryBuildConfiguration(binary, 'debug') == sh.buildOptions.enableDbg + assert queryBuildConfiguration(binary, 'debug') == sh.build_options.enableDbg if sps.isARMv7l: - assert testIsHardFpShellARM(binary) == sh.buildOptions.enableHardFp + assert testIsHardFpShellARM(binary) == sh.build_options.enableHardFp - assert queryBuildConfiguration(binary, 'more-deterministic') == sh.buildOptions.enableMoreDeterministic - assert queryBuildConfiguration(binary, 'asan') == sh.buildOptions.buildWithAsan + assert queryBuildConfiguration(binary, 'more-deterministic') == sh.build_options.enableMoreDeterministic + assert queryBuildConfiguration(binary, 'asan') == sh.build_options.buildWithAsan assert (queryBuildConfiguration(binary, 'arm-simulator') and - sh.buildOptions.enable32) == sh.buildOptions.enableSimulatorArm32 + sh.build_options.enable32) == sh.build_options.enableSimulatorArm32 assert (queryBuildConfiguration(binary, 'arm-simulator') and not - sh.buildOptions.enable32) == sh.buildOptions.enableSimulatorArm64 + sh.build_options.enable32) == sh.build_options.enableSimulatorArm64 # Note that we should test whether a shell has profiling turned on or not. # m-c rev 324836:800a887c705e turned profiling on by default, so once this is beyond the # earliest known working revision, we can probably test it here. diff --git a/js/jsInteresting.py b/src/funfuzz/js/js_interesting.py old mode 100755 new mode 100644 similarity index 70% rename from js/jsInteresting.py rename to src/funfuzz/js/js_interesting.py index 522fbfac2..9da8d2c67 --- a/js/jsInteresting.py +++ b/src/funfuzz/js/js_interesting.py @@ -1,35 +1,30 @@ -#!/usr/bin/env python # coding=utf-8 -# pylint: disable=fixme,global-statement,import-error,invalid-name,missing-docstring,no-member -# pylint: disable=too-few-public-methods,too-many-branches,too-many-instance-attributes,too-many-locals -# pylint: disable=too-many-statements,wrong-import-position # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at https://mozilla.org/MPL/2.0/. +"""Check whether a testcase causes an interesting result in a shell. +""" + from __future__ import absolute_import, print_function import os import sys from optparse import OptionParser # pylint: disable=deprecated-module -import inspectShell -p0 = os.path.dirname(os.path.abspath(__file__)) -p2 = os.path.abspath(os.path.join(p0, os.pardir, "detect")) -sys.path.append(p2) -import detect_malloc_errors -import findIgnoreLists -p3 = os.path.abspath(os.path.join(p0, os.pardir, 'util')) -sys.path.append(p3) -import subprocesses as sps -import createCollector -import fileManipulation import lithium.interestingness.timed_run as timed_run -# no-name-in-module pylint error exists for Python 3 only because FuzzManager is not Python 3-compatible yet -import FTB.Signatures.CrashInfo as CrashInfo # pylint: disable=no-name-in-module -from FTB.ProgramConfiguration import ProgramConfiguration +# These pylint errors exist because FuzzManager is not Python 3-compatible yet +import FTB.Signatures.CrashInfo as CrashInfo # pylint: disable=import-error,no-name-in-module +from FTB.ProgramConfiguration import ProgramConfiguration # pylint: disable=import-error + +from . import inspect_shell +from ..util import create_collector +from ..util import detect_malloc_errors +from ..util import file_manipulation +from ..util import subprocesses as sps +from ..util.find_ignore_lists import find_ignore_lists # Levels of unhappiness. @@ -50,29 +45,31 @@ JS_FINE, JS_DID_NOT_FINISH, # correctness (only jsfunfuzzLevel) JS_DECIDED_TO_EXIT, # correctness (only jsfunfuzzLevel) - JS_OVERALL_MISMATCH, # correctness (only compareJIT) + JS_OVERALL_MISMATCH, # correctness (only compare_jit) JS_VG_AMISS, # memory safety JS_NEW_ASSERT_OR_CRASH # memory safety or other issue that is definitely a bug ) = range(JS_LEVELS) -gOptions = "" +gOptions = "" # pylint: disable=invalid-name VALGRIND_ERROR_EXIT_CODE = 77 -class ShellResult(object): +class ShellResult(object): # pylint: disable=missing-docstring,too-many-instance-attributes,too-few-public-methods # options dict should include: timeout, knownPath, collector, valgrind, shellIsDeterministic - def __init__(self, options, runthis, logPrefix, inCompareJIT): - pathToBinary = runthis[0] - # This relies on the shell being a local one from compileShell.py: + def __init__(self, options, runthis, logPrefix, in_compare_jit): # pylint: disable=too-complex,too-many-branches + # pylint: disable=too-many-locals,too-many-statements + pathToBinary = runthis[0] # pylint: disable=invalid-name + # This relies on the shell being a local one from compile_shell: # Ignore trailing ".exe" in Win, also abspath makes it work w/relative paths like './js' + # pylint: disable=invalid-name pc = ProgramConfiguration.fromBinary(os.path.abspath(pathToBinary).split('.')[0]) pc.addProgramArguments(runthis[1:-1]) if options.valgrind: runthis = ( - inspectShell.constructVgCmdList(errorCode=VALGRIND_ERROR_EXIT_CODE) + + inspect_shell.constructVgCmdList(errorCode=VALGRIND_ERROR_EXIT_CODE) + valgrindSuppressions(options.knownPath) + runthis) @@ -81,7 +78,7 @@ def __init__(self, options, runthis, logPrefix, inCompareJIT): lev = JS_FINE issues = [] - auxCrashData = [] + auxCrashData = [] # pylint: disable=invalid-name # FuzzManager expects a list of strings rather than an iterable, so bite the # bullet and 'readlines' everything into memory. @@ -104,7 +101,7 @@ def __init__(self, options, runthis, logPrefix, inCompareJIT): elif detect_malloc_errors.amiss(logPrefix): issues.append("malloc error") lev = max(lev, JS_NEW_ASSERT_OR_CRASH) - elif runinfo.return_code == 0 and not inCompareJIT: + elif runinfo.return_code == 0 and not in_compare_jit: # We might have(??) run jsfunfuzz directly, so check for special kinds of bugs for line in out: if line.startswith("Found a bug: ") and not ("NestTest" in line and oomed(err)): @@ -114,7 +111,7 @@ def __init__(self, options, runthis, logPrefix, inCompareJIT): issues.append("jsfunfuzz didn't finish") lev = JS_DID_NOT_FINISH - # Copy non-crash issues to where FuzzManager's "AssertionHelper.py" can see it. + # Copy non-crash issues to where FuzzManager's "AssertionHelper" can see it. if lev != JS_FINE: for issue in issues: err.append("[Non-crash bug] " + issue) @@ -122,7 +119,7 @@ def __init__(self, options, runthis, logPrefix, inCompareJIT): # Finally, make a CrashInfo object and parse stack traces for asan/crash/assertion bugs crashInfo = CrashInfo.CrashInfo.fromRawCrashData(out, err, pc, auxCrashData=auxCrashData) - createCollector.printCrashInfo(crashInfo) + create_collector.printCrashInfo(crashInfo) # We only care about crashes and assertion failures on shells with no symbols # Note that looking out for the Assertion failure message is highly SpiderMonkey-specific if not isinstance(crashInfo, CrashInfo.NoCrashInfo) or \ @@ -133,13 +130,13 @@ def __init__(self, options, runthis, logPrefix, inCompareJIT): match = options.collector.search(crashInfo) if match[0] is not None: - createCollector.printMatchingSignature(match) + create_collector.printMatchingSignature(match) lev = JS_FINE print("%s | %s" % (logPrefix, summaryString(issues, lev, runinfo.elapsedtime))) if lev != JS_FINE: - fileManipulation.writeLinesToFile( + file_manipulation.writeLinesToFile( ['Number: ' + logPrefix + '\n', 'Command: ' + sps.shellify(runthis) + '\n'] + ['Status: ' + i + "\n" for i in issues], @@ -149,13 +146,14 @@ def __init__(self, options, runthis, logPrefix, inCompareJIT): self.out = out self.err = err self.issues = issues - self.crashInfo = crashInfo + self.crashInfo = crashInfo # pylint: disable=invalid-name self.match = match self.runinfo = runinfo self.return_code = runinfo.return_code -def understoodJsfunfuzzExit(out, err): +def understoodJsfunfuzzExit(out, err): # pylint: disable=invalid-name,missing-docstring,missing-return-doc + # pylint: disable=missing-return-type-doc for line in err: if "terminate called" in line or "quit called" in line: return True @@ -171,7 +169,8 @@ def understoodJsfunfuzzExit(out, err): return False -def hitMemoryLimit(err): +def hitMemoryLimit(err): # pylint: disable=invalid-name,missing-param-doc,missing-return-doc,missing-return-type-doc + # pylint: disable=missing-type-doc """Return True iff stderr text indicates that the shell hit a memory limit.""" if "ReportOverRecursed called" in err: # --enable-more-deterministic @@ -189,7 +188,7 @@ def hitMemoryLimit(err): return None -def oomed(err): +def oomed(err): # pylint: disable=invalid-name,missing-docstring,missing-return-doc,missing-return-type-doc # spidermonkey shells compiled with --enable-more-deterministic will tell us on stderr if they run out of memory for line in err: if hitMemoryLimit(line): @@ -197,22 +196,24 @@ def oomed(err): return False -def summaryString(issues, level, elapsedtime): - amissDetails = ("") if (not issues) else (" | " + repr(issues[:5]) + " ") +def summaryString(issues, level, elapsedtime): # pylint: disable=invalid-name,missing-docstring,missing-return-doc + # pylint: disable=missing-return-type-doc + amissDetails = ("") if (not issues) else (" | " + repr(issues[:5]) + " ") # pylint: disable=invalid-name return "%5.1fs | %d | %s%s" % (elapsedtime, level, JS_LEVEL_NAMES[level], amissDetails) -def truncateFile(fn, maxSize): +def truncateFile(fn, maxSize): # pylint: disable=invalid-name,missing-docstring if os.path.exists(fn) and os.path.getsize(fn) > maxSize: with open(fn, "r+") as f: f.truncate(maxSize) -def valgrindSuppressions(knownPath): - return ["--suppressions=" + filename for filename in findIgnoreLists.findIgnoreLists(knownPath, "valgrind.txt")] +def valgrindSuppressions(knownPath): # pylint: disable=invalid-name,missing-docstring,missing-return-doc + # pylint: disable=missing-return-type-doc + return ["--suppressions=" + filename for filename in find_ignore_lists(knownPath, "valgrind.txt")] -def deleteLogs(logPrefix): +def deleteLogs(logPrefix): # pylint: disable=invalid-name,missing-param-doc,missing-type-doc """Whoever might call baseLevel should eventually call this function (unless a bug was found).""" # If this turns up a WindowsError on Windows, remember to have excluded fuzzing locations in # the search indexer, anti-virus realtime protection and backup applications. @@ -222,28 +223,30 @@ def deleteLogs(logPrefix): os.remove(logPrefix + "-crash.txt") if os.path.exists(logPrefix + "-vg.xml"): os.remove(logPrefix + "-vg.xml") - # FIXME: in some cases, subprocesses.py gzips a core file only for us to delete it immediately. + # pylint: disable=fixme + # FIXME: in some cases, subprocesses gzips a core file only for us to delete it immediately. if os.path.exists(logPrefix + "-core.gz"): os.remove(logPrefix + "-core.gz") -def ulimitSet(): +def ulimitSet(): # pylint: disable=invalid-name """When called as a preexec_fn, sets appropriate resource limits for the JS shell. Must only be called on POSIX.""" - import resource # module only available on POSIX + # module only available on POSIX + import resource # pylint: disable=import-error # Limit address space to 2GB (or 1GB on ARM boards such as ODROID). - GB = 2**30 + GB = 2**30 # pylint: disable=invalid-name if sps.isARMv7l: resource.setrlimit(resource.RLIMIT_AS, (1 * GB, 1 * GB)) else: resource.setrlimit(resource.RLIMIT_AS, (2 * GB, 2 * GB)) # Limit corefiles to 0.5 GB. - halfGB = int(GB / 2) + halfGB = int(GB / 2) # pylint: disable=invalid-name,old-division resource.setrlimit(resource.RLIMIT_CORE, (halfGB, halfGB)) -def parseOptions(args): +def parseOptions(args): # pylint: disable=invalid-name,missing-docstring,missing-return-doc,missing-return-type-doc parser = OptionParser() parser.disable_interspersed_args() parser.add_option("--valgrind", @@ -257,7 +260,7 @@ def parseOptions(args): parser.add_option("--minlevel", type="int", dest="minimumInterestingLevel", default=JS_FINE + 1, - help="minimum js/jsInteresting.py level for lithium to consider the testcase interesting") + help="minimum js/js_interesting level for lithium to consider the testcase interesting") parser.add_option("--timeout", type="int", dest="timeout", default=120, @@ -267,28 +270,29 @@ def parseOptions(args): raise Exception("Not enough positional arguments") options.knownPath = args[0] options.jsengineWithArgs = args[1:] - options.collector = createCollector.createCollector("jsfunfuzz") + options.collector = create_collector.createCollector("jsfunfuzz") if not os.path.exists(options.jsengineWithArgs[0]): raise Exception("js shell does not exist: " + options.jsengineWithArgs[0]) - options.shellIsDeterministic = inspectShell.queryBuildConfiguration( + options.shellIsDeterministic = inspect_shell.queryBuildConfiguration( options.jsengineWithArgs[0], 'more-deterministic') return options -# loopjsfunfuzz.py uses parseOptions and ShellResult [with inCompareJIT = False] -# compareJIT.py uses ShellResult [with inCompareJIT = True] +# loop uses parseOptions and ShellResult [with in_compare_jit = False] +# compare_jit uses ShellResult [with in_compare_jit = True] # For use by Lithium and autoBisect. (autoBisect calls init multiple times because it changes the js engine name) -def init(args): - global gOptions +def init(args): # pylint: disable=missing-docstring + global gOptions # pylint: disable=global-statement,invalid-name gOptions = parseOptions(args) -# FIXME: _args is unused here, we should check if it can be removed? -def interesting(_args, tempPrefix): +# FIXME: _args is unused here, we should check if it can be removed? # pylint: disable=fixme +def interesting(_args, tempPrefix): # pylint: disable=invalid-name,missing-docstring,missing-return-doc + # pylint: disable=missing-return-type-doc options = gOptions - # options, runthis, logPrefix, inCompareJIT + # options, runthis, logPrefix, in_compare_jit res = ShellResult(options, options.jsengineWithArgs, tempPrefix, False) truncateFile(tempPrefix + "-out.txt", 1000000) truncateFile(tempPrefix + "-err.txt", 1000000) @@ -296,17 +300,17 @@ def interesting(_args, tempPrefix): # For direct, manual use -def main(): +def main(): # pylint: disable=missing-docstring options = parseOptions(sys.argv[1:]) - tempPrefix = "m" - res = ShellResult(options, options.jsengineWithArgs, tempPrefix, False) + tempPrefix = "m" # pylint: disable=invalid-name + res = ShellResult(options, options.jsengineWithArgs, tempPrefix, False) # pylint: disable=no-member print(res.lev) - if options.submit: - if res.lev >= options.minimumInterestingLevel: - testcaseFilename = options.jsengineWithArgs[-1] + if options.submit: # pylint: disable=no-member + if res.lev >= options.minimumInterestingLevel: # pylint: disable=no-member + testcaseFilename = options.jsengineWithArgs[-1] # pylint: disable=invalid-name,no-member print("Submitting %s" % testcaseFilename) quality = 0 - options.collector.submit(res.crashInfo, testcaseFilename, quality) + options.collector.submit(res.crashInfo, testcaseFilename, quality) # pylint: disable=no-member else: print("Not submitting (not interesting)") diff --git a/js/jsfunfuzz/README.md b/src/funfuzz/js/jsfunfuzz/README.md similarity index 71% rename from js/jsfunfuzz/README.md rename to src/funfuzz/js/jsfunfuzz/README.md index 2b6916447..55d3251e8 100644 --- a/js/jsfunfuzz/README.md +++ b/src/funfuzz/js/jsfunfuzz/README.md @@ -13,16 +13,16 @@ Once it creates a function body, it does the following things with it: To test an existing SpiderMonkey shell called `./js`, run: -`funfuzz/js/loopjsfunfuzz.py --random-flags --comparejit 20 mozilla-central ./js` +`python -m funfuzz.js.loop --random-flags --compare-jit 20 mozilla-central ./js` -* `--random-flags` tells it to use [shellFlags.py](../shellFlags.py) to -* `--comparejit` tells it to run [compareJIT.py](../compareJIT.py) on most of the generated code, detecting bugs where adding optimization flags like --ion-eager changes the output. +* `--random-flags` tells it to use [shell_flags](../shell_flags.py) to +* `--compare-jit` tells it to run [compare_jit](../compare_jit.py) on most of the generated code, detecting bugs where adding optimization flags like --ion-eager changes the output. * `20` tells it to kill any instance that runs for more than 20 seconds * `mozilla-central` or any other string is no longer used, and this argument will be removed in the future. -If loopjsfunfuzz detects a new bug, it will run [Lithium](https://github.com/MozillaSecurity/lithium/) to reduce the testcase. It will call Lithium with either [jsInteresting.py](../jsInteresting.py) or [compareJIT.py](../compareJIT.py) as the "interestingness test". +If loop detects a new bug, it will run [Lithium](https://github.com/MozillaSecurity/lithium/) to reduce the testcase. It will call Lithium with either [js_interesting](../js_interesting.py) or [compare_jit](../compare_jit.py) as the "interestingness test". -Using [bot.py](../../bot.py) --test-type=js, you can automate downloading or building new versions of the SpiderMonkey shell, and running several instances of loopjsfunfuzz.py for parallelism. +Using [funfuzz.bot](../../bot.py) --test-type=js, you can automate downloading or building new versions of the SpiderMonkey shell, and running several instances of loop for parallelism. Through randorderfuzz, if the harness detects tests in the mozilla-central tree, it may load or incorporate tests into its fuzzing input in a random order. @@ -34,4 +34,5 @@ Through randorderfuzz, if the harness detects tests in the mozilla-central tree, * [Christian Holler](https://twitter.com/mozdeco) improved the compilation scripts * [Jan de Mooij](https://twitter.com/jandemooij) prototyped [stress-testing objects and PICs](https://bugzilla.mozilla.org/show_bug.cgi?id=6309960) * [David Keeler](https://twitter.com/mozkeeler) modified the regular expression generator to also generate (almost-)matching strings, based on an idea from [Oliver Hunt](https://twitter.com/ohunt). +* [Jesse Schwartzentruber](https://github.com/jschwartzentruber/) reviewed a lot of the Python harness improvements * [The SpiderMonkey team](https://twitter.com/SpiderMonkeyJS) fixed over 2000 of our bugs, so we could keep fuzzing! diff --git a/js/jsfunfuzz/avoid-known-bugs.js b/src/funfuzz/js/jsfunfuzz/avoid-known-bugs.js similarity index 99% rename from js/jsfunfuzz/avoid-known-bugs.js rename to src/funfuzz/js/jsfunfuzz/avoid-known-bugs.js index 41aed7285..51c2e2cfd 100644 --- a/js/jsfunfuzz/avoid-known-bugs.js +++ b/src/funfuzz/js/jsfunfuzz/avoid-known-bugs.js @@ -20,7 +20,7 @@ function whatToTestSpidermonkeyTrunk(code) allowIter: true, // Ideally we'd detect whether the shell was compiled with --enable-more-deterministic - // Ignore both within-process & across-process, e.g. nestTest mismatch & compareJIT + // Ignore both within-process & across-process, e.g. nestTest mismatch & compare_jit expectConsistentOutput: true && (gcIsQuiet || code.indexOf("gc") == -1) && code.indexOf("/*NODIFF*/") == -1 // Ignore diff testing on these labels @@ -59,7 +59,7 @@ function whatToTestSpidermonkeyTrunk(code) , expectConsistentOutputAcrossJITs: true - // across-process (e.g. running js shell with different run-time options) e.g. compareJIT + // across-process (e.g. running js shell with different run-time options) e.g. compare_jit && code.indexOf("isAsmJSCompilationAvailable") == -1 // Causes false positives with --no-asmjs && code.indexOf("'strict") == -1 // see bug 743425 && code.indexOf("disassemble") == -1 // see bug 1237403 (related to asm.js) diff --git a/js/jsfunfuzz/built-in-constructors.js b/src/funfuzz/js/jsfunfuzz/built-in-constructors.js similarity index 100% rename from js/jsfunfuzz/built-in-constructors.js rename to src/funfuzz/js/jsfunfuzz/built-in-constructors.js diff --git a/js/jsfunfuzz/detect-engine.js b/src/funfuzz/js/jsfunfuzz/detect-engine.js similarity index 100% rename from js/jsfunfuzz/detect-engine.js rename to src/funfuzz/js/jsfunfuzz/detect-engine.js diff --git a/js/jsfunfuzz/driver.js b/src/funfuzz/js/jsfunfuzz/driver.js similarity index 100% rename from js/jsfunfuzz/driver.js rename to src/funfuzz/js/jsfunfuzz/driver.js diff --git a/js/jsfunfuzz/error-reporting.js b/src/funfuzz/js/jsfunfuzz/error-reporting.js similarity index 88% rename from js/jsfunfuzz/error-reporting.js rename to src/funfuzz/js/jsfunfuzz/error-reporting.js index 4a2c3b2c8..b56b7acb9 100644 --- a/js/jsfunfuzz/error-reporting.js +++ b/src/funfuzz/js/jsfunfuzz/error-reporting.js @@ -6,7 +6,7 @@ function confused(s) { if (jsshell) { - // Magic string that jsInteresting.py looks for + // Magic string that js_interesting looks for print("jsfunfuzz broke its own scripting environment: " + s); quit(); } @@ -14,7 +14,7 @@ function confused(s) function foundABug(summary, details) { - // Magic pair of strings that jsInteresting.py looks for + // Magic pair of strings that js_interesting looks for // Break up the following string so internal js functions do not print it deliberately printImportant("Found" + " a bug: " + summary); if (details) { diff --git a/js/jsfunfuzz/gen-asm.js b/src/funfuzz/js/jsfunfuzz/gen-asm.js similarity index 100% rename from js/jsfunfuzz/gen-asm.js rename to src/funfuzz/js/jsfunfuzz/gen-asm.js diff --git a/js/jsfunfuzz/gen-grammar.js b/src/funfuzz/js/jsfunfuzz/gen-grammar.js similarity index 99% rename from js/jsfunfuzz/gen-grammar.js rename to src/funfuzz/js/jsfunfuzz/gen-grammar.js index 77bf25049..cbdfc14c8 100644 --- a/js/jsfunfuzz/gen-grammar.js +++ b/src/funfuzz/js/jsfunfuzz/gen-grammar.js @@ -330,7 +330,7 @@ function regressionTestIsEvil(contents) return true; } if (contents.indexOf("print = ") != -1) { - // A testcase that clobbers the |print| function would confuse jsInteresting.py + // A testcase that clobbers the |print| function would confuse js_interesting return true; } return false; @@ -930,7 +930,7 @@ var fuzzTestingFunctions = fuzzTestingFunctionsCtor(!jsshell, fuzzTestingFunctio // Ensure that even if makeExpr returns "" or "1, 2", we only pass one argument to functions like schedulegc // (null || (" + makeExpr(d - 2, b) + ")) -// Darn, only |this| and local variables are safe: an expression with side effects breaks the statement-level compareJIT hack +// Darn, only |this| and local variables are safe: an expression with side effects breaks the statement-level compare_jit hack function fuzzTestingFunctionArg(d, b) { return "this"; } function makeTestingFunctionCall(d, b) @@ -945,12 +945,12 @@ function makeTestingFunctionCall(d, b) if (jsshell && rnd(5) === 0) { // Differential testing hack! - // The idea here: make compareJIT tell us when functions like gc() surprise + // The idea here: make compare_jit tell us when functions like gc() surprise // us with visible side effects. // * Functions in testing-functions.js are chosen to be ones with no visible // side effects except for return values (voided) or throwing (caught). - // * This condition is controlled by --no-asmjs, which compareJIT.py flips. - // (A more principled approach would be to have compareJIT set an environment + // * This condition is controlled by --no-asmjs, which compare_jit flips. + // (A more principled approach would be to have compare_jit set an environment // variable and read it here using os.getenv(), but os is not available // when running with --fuzzing-safe...) // * The extra braces prevent a stray "else" from being associated with this "if". diff --git a/js/jsfunfuzz/gen-math.js b/src/funfuzz/js/jsfunfuzz/gen-math.js similarity index 100% rename from js/jsfunfuzz/gen-math.js rename to src/funfuzz/js/jsfunfuzz/gen-math.js diff --git a/js/jsfunfuzz/gen-proxy.js b/src/funfuzz/js/jsfunfuzz/gen-proxy.js similarity index 100% rename from js/jsfunfuzz/gen-proxy.js rename to src/funfuzz/js/jsfunfuzz/gen-proxy.js diff --git a/js/jsfunfuzz/gen-recursion.js b/src/funfuzz/js/jsfunfuzz/gen-recursion.js similarity index 100% rename from js/jsfunfuzz/gen-recursion.js rename to src/funfuzz/js/jsfunfuzz/gen-recursion.js diff --git a/js/jsfunfuzz/gen-regex.js b/src/funfuzz/js/jsfunfuzz/gen-regex.js similarity index 100% rename from js/jsfunfuzz/gen-regex.js rename to src/funfuzz/js/jsfunfuzz/gen-regex.js diff --git a/js/jsfunfuzz/gen-stomp-on-registers.js b/src/funfuzz/js/jsfunfuzz/gen-stomp-on-registers.js similarity index 100% rename from js/jsfunfuzz/gen-stomp-on-registers.js rename to src/funfuzz/js/jsfunfuzz/gen-stomp-on-registers.js diff --git a/js/jsfunfuzz/gen-type-aware-code.js b/src/funfuzz/js/jsfunfuzz/gen-type-aware-code.js similarity index 100% rename from js/jsfunfuzz/gen-type-aware-code.js rename to src/funfuzz/js/jsfunfuzz/gen-type-aware-code.js diff --git a/js/jsfunfuzz/mess-grammar.js b/src/funfuzz/js/jsfunfuzz/mess-grammar.js similarity index 100% rename from js/jsfunfuzz/mess-grammar.js rename to src/funfuzz/js/jsfunfuzz/mess-grammar.js diff --git a/js/jsfunfuzz/mess-tokens.js b/src/funfuzz/js/jsfunfuzz/mess-tokens.js similarity index 100% rename from js/jsfunfuzz/mess-tokens.js rename to src/funfuzz/js/jsfunfuzz/mess-tokens.js diff --git a/js/jsfunfuzz/preamble.js b/src/funfuzz/js/jsfunfuzz/preamble.js similarity index 100% rename from js/jsfunfuzz/preamble.js rename to src/funfuzz/js/jsfunfuzz/preamble.js diff --git a/js/jsfunfuzz/run-in-sandbox.js b/src/funfuzz/js/jsfunfuzz/run-in-sandbox.js similarity index 100% rename from js/jsfunfuzz/run-in-sandbox.js rename to src/funfuzz/js/jsfunfuzz/run-in-sandbox.js diff --git a/js/jsfunfuzz/run-reduction-marker.js b/src/funfuzz/js/jsfunfuzz/run-reduction-marker.js similarity index 100% rename from js/jsfunfuzz/run-reduction-marker.js rename to src/funfuzz/js/jsfunfuzz/run-reduction-marker.js diff --git a/js/jsfunfuzz/run.js b/src/funfuzz/js/jsfunfuzz/run.js similarity index 98% rename from js/jsfunfuzz/run.js rename to src/funfuzz/js/jsfunfuzz/run.js index 9c4bb9e5e..4f47d1943 100644 --- a/js/jsfunfuzz/run.js +++ b/src/funfuzz/js/jsfunfuzz/run.js @@ -181,14 +181,14 @@ function tryItOut(code) if (code.indexOf("\n") == -1 && code.indexOf("\r") == -1 && code.indexOf("\f") == -1 && code.indexOf("\0") == -1 && code.indexOf("\u2028") == -1 && code.indexOf("\u2029") == -1 && code.indexOf("<--") == -1 && code.indexOf("-->") == -1 && code.indexOf("//") == -1) { - // FCM cookie, lines with this cookie are used for compareJIT + // FCM cookie, lines with this cookie are used for compare_jit var cookie1 = "/*F"; var cookie2 = "CM*/"; var nCode = code; // Avoid compile-time errors because those are no fun. // But leave some things out of function(){} because some bugs are only detectable at top-level, and // pure jsfunfuzz doesn't test top-level at all. - // (This is a good reason to use compareJIT even if I'm not interested in finding JIT bugs!) + // (This is a good reason to use compare_jit even if I'm not interested in finding JIT bugs!) if (nCode.indexOf("return") != -1 || nCode.indexOf("yield") != -1 || nCode.indexOf("const") != -1 || failsToCompileInTry(nCode)) nCode = "(function(){" + nCode + "})()"; dumpln(cookie1 + cookie2 + " try { " + nCode + " } catch(e) { }"); diff --git a/js/jsfunfuzz/tail.js b/src/funfuzz/js/jsfunfuzz/tail.js similarity index 89% rename from js/jsfunfuzz/tail.js rename to src/funfuzz/js/jsfunfuzz/tail.js index aac3645b8..c8c6a59f0 100644 --- a/js/jsfunfuzz/tail.js +++ b/src/funfuzz/js/jsfunfuzz/tail.js @@ -19,7 +19,7 @@ start(this); // SPLICE DDEND if (jsshell) - print("It's looking good!"); // Magic string that jsInteresting.py looks for + print("It's looking good!"); // Magic string that js_interesting looks for // 3. Run it. diff --git a/js/jsfunfuzz/test-asm.js b/src/funfuzz/js/jsfunfuzz/test-asm.js similarity index 100% rename from js/jsfunfuzz/test-asm.js rename to src/funfuzz/js/jsfunfuzz/test-asm.js diff --git a/js/jsfunfuzz/test-consistency.js b/src/funfuzz/js/jsfunfuzz/test-consistency.js similarity index 100% rename from js/jsfunfuzz/test-consistency.js rename to src/funfuzz/js/jsfunfuzz/test-consistency.js diff --git a/js/jsfunfuzz/test-math.js b/src/funfuzz/js/jsfunfuzz/test-math.js similarity index 95% rename from js/jsfunfuzz/test-math.js rename to src/funfuzz/js/jsfunfuzz/test-math.js index 89ee86e36..1920c4b94 100644 --- a/js/jsfunfuzz/test-math.js +++ b/src/funfuzz/js/jsfunfuzz/test-math.js @@ -79,13 +79,13 @@ function testMathyFunction(f, inputs) } } /* Use uneval to distinguish -0, 0, "0", etc. */ - /* Use hashStr to shorten the output and keep compareJIT files small. */ + /* Use hashStr to shorten the output and keep compare_jit files small. */ print(hashStr(uneval(results))); } function mathInitFCM() { - // FCM cookie, lines with this cookie are used for compareJIT + // FCM cookie, lines with this cookie are used for compare_jit var cookie = "/*F" + "CM*/"; print(cookie + hashStr.toString().replace(/\n/g, " ")); diff --git a/js/jsfunfuzz/test-misc.js b/src/funfuzz/js/jsfunfuzz/test-misc.js similarity index 100% rename from js/jsfunfuzz/test-misc.js rename to src/funfuzz/js/jsfunfuzz/test-misc.js diff --git a/js/jsfunfuzz/test-regex.js b/src/funfuzz/js/jsfunfuzz/test-regex.js similarity index 100% rename from js/jsfunfuzz/test-regex.js rename to src/funfuzz/js/jsfunfuzz/test-regex.js diff --git a/js/loopjsfunfuzz.py b/src/funfuzz/js/loop.py old mode 100755 new mode 100644 similarity index 54% rename from js/loopjsfunfuzz.py rename to src/funfuzz/js/loop.py index 2879ffba1..719c8dc10 --- a/js/loopjsfunfuzz.py +++ b/src/funfuzz/js/loop.py @@ -1,12 +1,12 @@ -#!/usr/bin/env python # coding=utf-8 -# pylint: disable=fixme,import-error,invalid-name,missing-docstring,no-member,too-many-branches -# pylint: disable=too-many-locals,too-many-statements,wrong-import-position # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at https://mozilla.org/MPL/2.0/. +"""Allows the funfuzz harness to run continuously. +""" + from __future__ import absolute_import, print_function import json @@ -16,27 +16,22 @@ import time from optparse import OptionParser # pylint: disable=deprecated-module -import compareJIT -import jsInteresting -import pinpoint -import shellFlags - -p0 = os.path.dirname(os.path.abspath(__file__)) -interestingpy = os.path.abspath(os.path.join(p0, 'jsInteresting.py')) -p1 = os.path.abspath(os.path.join(p0, os.pardir, 'util')) -sys.path.append(p1) -import createCollector -import fileManipulation -import lithOps -import linkJS -import subprocesses as sps +from . import compare_jit +from . import js_interesting +from . import pinpoint +from . import shell_flags +from ..util import create_collector +from ..util import file_manipulation +from ..util import lithium_helpers +from ..util import link_js +from ..util import subprocesses as sps -def parseOpts(args): +def parseOpts(args): # pylint: disable=invalid-name,missing-docstring,missing-return-doc,missing-return-type-doc parser = OptionParser() parser.disable_interspersed_args() - parser.add_option("--comparejit", - action="store_true", dest="useCompareJIT", + parser.add_option("--compare-jit", + action="store_true", dest="use_compare_jit", default=False, help="After running the fuzzer, run the FCM lines against the engine " "in two configurations and compare the output.") @@ -49,25 +44,25 @@ def parseOpts(args): default=os.path.expanduser("~/trees/mozilla-central/"), help="The hg repository (e.g. ~/trees/mozilla-central/), for bisection") parser.add_option("--build", - action="store", dest="buildOptionsStr", + action="store", dest="build_options_str", help="The build options, for bisection", - default=None) # if you run loopjsfunfuzz.py directly without --build, pinpoint will try to guess + default=None) # if you run loop directly without --build, pinpoint will try to guess parser.add_option("--valgrind", action="store_true", dest="valgrind", default=False, help="use valgrind with a reasonable set of options") options, args = parser.parse_args(args) - if options.valgrind and options.useCompareJIT: - print("Note: When running comparejit, the --valgrind option will be ignored") + if options.valgrind and options.use_compare_jit: + print("Note: When running compare_jit, the --valgrind option will be ignored") # kill js shell if it runs this long. # jsfunfuzz will quit after half this time if it's not ilooping. # higher = more complex mixing, especially with regression tests. - # lower = less time wasted in timeouts and in compareJIT testcases that are thrown away due to OOMs. + # lower = less time wasted in timeouts and in compare_jit testcases that are thrown away due to OOMs. options.timeout = int(args[0]) - # FIXME: We can probably remove args[1] + # FIXME: We can probably remove args[1] # pylint: disable=fixme options.knownPath = 'mozilla-central' options.jsEngine = args[2] options.engineFlags = args[3:] @@ -75,7 +70,8 @@ def parseOpts(args): return options -def showtail(filename): +def showtail(filename): # pylint: disable=missing-docstring + # pylint: disable=fixme # FIXME: Get jsfunfuzz to output start & end of interesting result boundaries instead of this. cmd = [] cmd.extend(['tail', '-n', '20']) @@ -87,13 +83,14 @@ def showtail(filename): print() -def linkFuzzer(target_fn, prologue): - source_base = p0 - file_list_fn = sps.normExpUserPath(os.path.join(p0, "files-to-link.txt")) - linkJS.linkJS(target_fn, file_list_fn, source_base, prologue) +def linkFuzzer(target_fn, prologue): # pylint: disable=invalid-name,missing-docstring + source_base = os.path.dirname(os.path.abspath(__file__)) + file_list_fn = sps.normExpUserPath(os.path.join(source_base, "files_to_link.txt")) + link_js.link_js(target_fn, file_list_fn, source_base, prologue) -def makeRegressionTestPrologue(repo): +def makeRegressionTestPrologue(repo): # pylint: disable=invalid-name,missing-docstring,missing-param-doc + # pylint: disable=missing-return-doc,missing-return-type-doc,missing-type-doc """Generate a JS string to tell jsfunfuzz where to find SpiderMonkey's regression tests.""" repo = sps.normExpUserPath(repo) + os.sep @@ -106,28 +103,32 @@ def makeRegressionTestPrologue(repo): json.dumps(inTreeRegressionTests(repo))) -def inTreeRegressionTests(repo): - jitTests = jsFilesIn(len(repo), os.path.join(repo, 'js', 'src', 'jit-test', 'tests')) - jsTests = jsFilesIn(len(repo), os.path.join(repo, 'js', 'src', 'tests')) - return jitTests + jsTests +def inTreeRegressionTests(repo): # pylint: disable=invalid-name,missing-docstring,missing-return-doc + # pylint: disable=missing-return-type-doc + jit_tests = jsFilesIn(len(repo), os.path.join(repo, 'js', 'src', 'jit-test', 'tests')) + js_tests = jsFilesIn(len(repo), os.path.join(repo, 'js', 'src', 'tests')) + return jit_tests + js_tests -def jsFilesIn(repoPathLength, root): +def jsFilesIn(repoPathLength, root): # pylint: disable=invalid-name,missing-docstring,missing-return-doc + # pylint: disable=missing-return-type-doc return [os.path.join(path, filename)[repoPathLength:] for path, _dirs, files in os.walk(sps.normExpUserPath(root)) for filename in files if filename.endswith('.js')] -def many_timed_runs(targetTime, wtmpDir, args, collector): +def many_timed_runs(targetTime, wtmpDir, args, collector): # pylint: disable=invalid-name,missing-docstring,too-complex + # pylint: disable=too-many-branches,too-many-locals,too-many-statements options = parseOpts(args) - engineFlags = options.engineFlags # engineFlags is overwritten later if --random-flags is set. - startTime = time.time() + # engineFlags is overwritten later if --random-flags is set. + engineFlags = options.engineFlags # pylint: disable=invalid-name + startTime = time.time() # pylint: disable=invalid-name if os.path.isdir(sps.normExpUserPath(options.repo)): - regressionTestPrologue = makeRegressionTestPrologue(options.repo) + regressionTestPrologue = makeRegressionTestPrologue(options.repo) # pylint: disable=invalid-name else: - regressionTestPrologue = "" + regressionTestPrologue = "" # pylint: disable=invalid-name fuzzjs = sps.normExpUserPath(os.path.join(wtmpDir, "jsfunfuzz.js")) linkFuzzer(fuzzjs, regressionTestPrologue) @@ -142,54 +143,60 @@ def many_timed_runs(targetTime, wtmpDir, args, collector): break # Construct command needed to loop jsfunfuzz fuzzing. - jsInterestingArgs = [] - jsInterestingArgs.append('--timeout=' + str(options.timeout)) + js_interesting_args = [] + js_interesting_args.append('--timeout=' + str(options.timeout)) if options.valgrind: - jsInterestingArgs.append('--valgrind') - jsInterestingArgs.append(options.knownPath) - jsInterestingArgs.append(options.jsEngine) + js_interesting_args.append('--valgrind') + js_interesting_args.append(options.knownPath) + js_interesting_args.append(options.jsEngine) if options.randomFlags: - engineFlags = shellFlags.randomFlagSet(options.jsEngine) - jsInterestingArgs.extend(engineFlags) - jsInterestingArgs.extend(['-e', 'maxRunTime=' + str(options.timeout * (1000 / 2))]) - jsInterestingArgs.extend(['-f', fuzzjs]) - jsInterestingOptions = jsInteresting.parseOptions(jsInterestingArgs) + engineFlags = shell_flags.randomFlagSet(options.jsEngine) # pylint: disable=invalid-name + js_interesting_args.extend(engineFlags) + # pylint: disable=old-division + js_interesting_args.extend(['-e', 'maxRunTime=' + str(options.timeout * (1000 / 2))]) + js_interesting_args.extend(['-f', fuzzjs]) + js_interesting_options = js_interesting.parseOptions(js_interesting_args) iteration += 1 - logPrefix = sps.normExpUserPath(os.path.join(wtmpDir, "w" + str(iteration))) + logPrefix = sps.normExpUserPath(os.path.join(wtmpDir, "w" + str(iteration))) # pylint: disable=invalid-name - res = jsInteresting.ShellResult(jsInterestingOptions, jsInterestingOptions.jsengineWithArgs, logPrefix, False) + res = js_interesting.ShellResult(js_interesting_options, + # pylint: disable=no-member + js_interesting_options.jsengineWithArgs, logPrefix, False) - if res.lev != jsInteresting.JS_FINE: + if res.lev != js_interesting.JS_FINE: showtail(logPrefix + "-out.txt") showtail(logPrefix + "-err.txt") # splice jsfunfuzz.js with `grep "/*FRC-" wN-out` - filenameToReduce = logPrefix + "-reduced.js" - [before, after] = fileManipulation.fuzzSplice(fuzzjs) + filenameToReduce = logPrefix + "-reduced.js" # pylint: disable=invalid-name + [before, after] = file_manipulation.fuzzSplice(fuzzjs) with open(logPrefix + '-out.txt', 'rb') as f: - newfileLines = before + [ - l.replace('/*FRC-', '/*') for l in fileManipulation.linesStartingWith(f, "/*FRC-")] + after - fileManipulation.writeLinesToFile(newfileLines, logPrefix + "-orig.js") - fileManipulation.writeLinesToFile(newfileLines, filenameToReduce) + newfileLines = before + [ # pylint: disable=invalid-name + l.replace('/*FRC-', '/*') for l in file_manipulation.linesStartingWith(f, "/*FRC-")] + after + file_manipulation.writeLinesToFile(newfileLines, logPrefix + "-orig.js") + file_manipulation.writeLinesToFile(newfileLines, filenameToReduce) # Run Lithium and autobisect (make a reduced testcase and find a regression window) + interestingpy = "funfuzz.js.js_interesting" # pylint: disable=invalid-name itest = [interestingpy] if options.valgrind: itest.append("--valgrind") itest.append("--minlevel=" + str(res.lev)) itest.append("--timeout=" + str(options.timeout)) itest.append(options.knownPath) - (lithResult, _lithDetails, autoBisectLog) = pinpoint.pinpoint( + (lithResult, _lithDetails, autoBisectLog) = pinpoint.pinpoint( # pylint: disable=invalid-name itest, logPrefix, options.jsEngine, engineFlags, filenameToReduce, options.repo, - options.buildOptionsStr, targetTime, res.lev) + options.build_options_str, targetTime, res.lev) # Upload with final output - if lithResult == lithOps.LITH_FINISHED: - fargs = jsInterestingOptions.jsengineWithArgs[:-1] + [filenameToReduce] - retestResult = jsInteresting.ShellResult(jsInterestingOptions, fargs, logPrefix + "-final", False) - if retestResult.lev > jsInteresting.JS_FINE: + if lithResult == lithium_helpers.LITH_FINISHED: + # pylint: disable=no-member + fargs = js_interesting_options.jsengineWithArgs[:-1] + [filenameToReduce] + # pylint: disable=invalid-name + retestResult = js_interesting.ShellResult(js_interesting_options, fargs, logPrefix + "-final", False) + if retestResult.lev > js_interesting.JS_FINE: res = retestResult quality = 0 else: @@ -197,7 +204,7 @@ def many_timed_runs(targetTime, wtmpDir, args, collector): else: quality = 10 - # ddsize = lithOps.ddsize(filenameToReduce) + # ddsize = lithium_helpers.ddsize(filenameToReduce) print("Submitting %s (quality=%s) at %s" % (filenameToReduce, quality, time.asctime())) metadata = {} @@ -207,23 +214,26 @@ def many_timed_runs(targetTime, wtmpDir, args, collector): print("Submitted %s" % filenameToReduce) else: - flagsAreDeterministic = "--dump-bytecode" not in engineFlags and '-D' not in engineFlags - if options.useCompareJIT and res.lev == jsInteresting.JS_FINE and \ - jsInterestingOptions.shellIsDeterministic and flagsAreDeterministic: - linesToCompare = jitCompareLines(logPrefix + '-out.txt', "/*FCM*/") + are_flags_deterministic = "--dump-bytecode" not in engineFlags and '-D' not in engineFlags + # pylint: disable=no-member + if options.use_compare_jit and res.lev == js_interesting.JS_FINE and \ + js_interesting_options.shellIsDeterministic and are_flags_deterministic: + linesToCompare = jitCompareLines(logPrefix + '-out.txt', "/*FCM*/") # pylint: disable=invalid-name jitcomparefilename = logPrefix + "-cj-in.js" - fileManipulation.writeLinesToFile(linesToCompare, jitcomparefilename) - anyBug = compareJIT.compareJIT(options.jsEngine, engineFlags, jitcomparefilename, - logPrefix + "-cj", options.repo, - options.buildOptionsStr, targetTime, jsInterestingOptions) + file_manipulation.writeLinesToFile(linesToCompare, jitcomparefilename) + # pylint: disable=invalid-name + anyBug = compare_jit.compare_jit(options.jsEngine, engineFlags, jitcomparefilename, + logPrefix + "-cj", options.repo, + options.build_options_str, targetTime, js_interesting_options) if not anyBug: os.remove(jitcomparefilename) - jsInteresting.deleteLogs(logPrefix) + js_interesting.deleteLogs(logPrefix) -def jitCompareLines(jsfunfuzzOutputFilename, marker): - """Create a compareJIT file, using the lines marked by jsfunfuzz as valid for comparison.""" +def jitCompareLines(jsfunfuzzOutputFilename, marker): # pylint: disable=invalid-name,missing-param-doc + # pylint: disable=missing-return-doc,missing-return-type-doc,missing-type-doc + """Create a compare_jit file, using the lines marked by jsfunfuzz as valid for comparison.""" lines = [ "backtrace = function() { };\n", "dumpHeap = function() { };\n", @@ -243,8 +253,8 @@ def jitCompareLines(jsfunfuzzOutputFilename, marker): for line in f: if line.startswith(marker): sline = line[len(marker):] - divisionIsInconsistent = sps.isWin # Really 'if MSVC' -- revisit if we add clang builds on Windows - if divisionIsInconsistent and mightUseDivision(sline): + is_division_consistent = sps.isWin # Really 'if MSVC' -- revisit if we add clang builds on Windows + if is_division_consistent and mightUseDivision(sline): pass elif "newGlobal" in sline and "wasmIsSupported" in sline: # We only override wasmIsSupported above for the main global. @@ -260,7 +270,7 @@ def jitCompareLines(jsfunfuzzOutputFilename, marker): return lines -def mightUseDivision(code): +def mightUseDivision(code): # pylint: disable=invalid-name,missing-docstring,missing-return-doc,missing-return-type-doc # Work around MSVC division inconsistencies (bug 948321) # by leaving division out of *-cj-in.js files on Windows. # (Unfortunately, this will also match regexps and a bunch @@ -290,4 +300,6 @@ def mightUseDivision(code): if __name__ == "__main__": - many_timed_runs(None, sps.createWtmpDir(os.getcwdu()), sys.argv[1:], createCollector.createCollector("jsfunfuzz")) + # FIXME: Replace os.getcwdu() prior to moving to Python 3 # pylint: disable=fixme + # pylint: disable=no-member + many_timed_runs(None, sps.createWtmpDir(os.getcwdu()), sys.argv[1:], create_collector.createCollector("jsfunfuzz")) diff --git a/js/pinpoint.py b/src/funfuzz/js/pinpoint.py old mode 100755 new mode 100644 similarity index 65% rename from js/pinpoint.py rename to src/funfuzz/js/pinpoint.py index 4d5596afb..178475ebb --- a/js/pinpoint.py +++ b/src/funfuzz/js/pinpoint.py @@ -1,12 +1,12 @@ -#!/usr/bin/env python # coding=utf-8 -# pylint: disable=import-error,invalid-name,literal-comparison,missing-docstring -# pylint: disable=too-many-arguments,too-many-branches,too-many-locals,too-many-statements,wrong-import-position # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at https://mozilla.org/MPL/2.0/. +"""Enables the funfuzz harness to use autobisectjs to discover the regressing changeset. +""" + from __future__ import absolute_import, print_function import os @@ -15,75 +15,78 @@ import shutil import subprocess import sys -from jsInteresting import JS_OVERALL_MISMATCH, JS_VG_AMISS -from inspectShell import testJsShellOrXpcshell from lithium.interestingness.utils import file_contains_str -p0 = os.path.dirname(os.path.abspath(__file__)) -autobisectpy = os.path.abspath(os.path.join(p0, os.pardir, 'autobisect-js', 'autoBisect.py')) - -p1 = os.path.abspath(os.path.join(p0, os.pardir, 'util')) -sys.path.append(p1) -import fileManipulation -from lithOps import LITH_FINISHED, LITH_PLEASE_CONTINUE, runLithium -import subprocesses as sps +from .js_interesting import JS_OVERALL_MISMATCH, JS_VG_AMISS +from .inspect_shell import testJsShellOrXpcshell +from ..util import file_manipulation +from ..util.lithium_helpers import LITH_FINISHED, LITH_PLEASE_CONTINUE, runLithium +from ..util import subprocesses as sps -def pinpoint(itest, logPrefix, jsEngine, engineFlags, infilename, - bisectRepo, buildOptionsStr, targetTime, suspiciousLevel): +def pinpoint(itest, logPrefix, jsEngine, engineFlags, infilename, # pylint: disable=invalid-name,missing-param-doc + bisectRepo, build_options_str, targetTime, suspiciousLevel): + # pylint: disable=missing-return-doc,missing-return-type-doc,missing-type-doc,too-many-arguments,too-many-locals """Run Lithium and autobisect. itest must be an array of the form [module, ...] where module is an interestingness module. The module's "interesting" function must accept [...] + [jsEngine] + engineFlags + infilename (If it's not prepared to accept engineFlags, engineFlags must be empty.) """ - lithArgs = itest + [jsEngine] + engineFlags + [infilename] + lithArgs = itest + [jsEngine] + engineFlags + [infilename] # pylint: disable=invalid-name - (lithResult, lithDetails) = strategicReduction(logPrefix, infilename, lithArgs, targetTime, suspiciousLevel) + (lithResult, lithDetails) = strategicReduction( # pylint: disable=invalid-name + logPrefix, infilename, lithArgs, targetTime, suspiciousLevel) print() print("Done running Lithium on the part in between DDBEGIN and DDEND. To reproduce, run:") print(sps.shellify([sys.executable, "-u", "-m", "lithium", "--strategy=check-only"] + lithArgs)) print() - if bisectRepo is not "none" and targetTime >= 3 * 60 * 60 and buildOptionsStr is not None: + # pylint: disable=literal-comparison + if bisectRepo is not "none" and targetTime >= 3 * 60 * 60 and build_options_str is not None: if platform.uname()[2] == 'XP': print("Not pinpointing to exact changeset since autoBisect does not work well in WinXP.") elif testJsShellOrXpcshell(jsEngine) != "xpcshell": - autobisectCmd = ( - [sys.executable, autobisectpy] + - ["-b", buildOptionsStr] + + autobisectCmd = ( # pylint: disable=invalid-name + [sys.executable, "-u", "-m", "funfuzz.autobisectjs"] + + ["-b", build_options_str] + ["-p", ' '.join(engineFlags + [infilename])] + ["-i"] + itest ) print(sps.shellify(autobisectCmd)) - autoBisectLogFilename = logPrefix + "-autobisect.txt" + autoBisectLogFilename = logPrefix + "-autobisect.txt" # pylint: disable=invalid-name subprocess.call(autobisectCmd, stdout=open(autoBisectLogFilename, "w"), stderr=subprocess.STDOUT) print("Done running autobisect. Log: %s" % autoBisectLogFilename) with open(autoBisectLogFilename, 'rb') as f: lines = f.readlines() - autoBisectLog = fileManipulation.truncateMid(lines, 50, ["..."]) + autoBisectLog = file_manipulation.truncateMid(lines, 50, ["..."]) # pylint: disable=invalid-name else: - autoBisectLog = [] + autoBisectLog = [] # pylint: disable=invalid-name return (lithResult, lithDetails, autoBisectLog) -def strategicReduction(logPrefix, infilename, lithArgs, targetTime, lev): +def strategicReduction(logPrefix, infilename, lithArgs, targetTime, lev): # pylint: disable=invalid-name + # pylint: disable=missing-param-doc,missing-return-doc,missing-return-type-doc,missing-type-doc,too-complex + # pylint: disable=too-many-branches,too-many-locals,too-many-statements """Reduce jsfunfuzz output files using Lithium by using various strategies.""" - reductionCount = [0] # This is an array because Python does not like assigning to upvars. - backupFilename = infilename + '-backup' + # This is an array because Python does not like assigning to upvars. + reductionCount = [0] # pylint: disable=invalid-name + backupFilename = infilename + '-backup' # pylint: disable=invalid-name - def lithReduceCmd(strategy): + def lithReduceCmd(strategy): # pylint: disable=invalid-name,missing-param-doc,missing-return-doc + # pylint: disable=missing-return-type-doc,missing-type-doc """Lithium reduction commands accepting various strategies.""" reductionCount[0] += 1 - fullLithArgs = [x for x in (strategy + lithArgs) if x] # Remove empty elements + # Remove empty elements + fullLithArgs = [x for x in (strategy + lithArgs) if x] # pylint: disable=invalid-name print(sps.shellify([sys.executable, "-u", "-m", "lithium"] + fullLithArgs)) desc = '-chars' if strategy == '--char' else '-lines' - (lithResult, lithDetails) = runLithium( + (lithResult, lithDetails) = runLithium( # pylint: disable=invalid-name fullLithArgs, "%s-%s%s" % (logPrefix, reductionCount[0], desc), targetTime) if lithResult == LITH_FINISHED: shutil.copy2(infilename, backupFilename) @@ -94,30 +97,32 @@ def lithReduceCmd(strategy): print("Running the first line reduction...") print() # Step 1: Run the first instance of line reduction. - lithResult, lithDetails = lithReduceCmd([]) + lithResult, lithDetails = lithReduceCmd([]) # pylint: disable=invalid-name if lithDetails is not None: # lithDetails can be None if testcase no longer becomes interesting - origNumOfLines = int(lithDetails.split()[0]) + origNumOfLines = int(lithDetails.split()[0]) # pylint: disable=invalid-name - hasTryItOut = False - hasTryItOutRegex = re.compile(r'count=[0-9]+; tryItOut\("') + hasTryItOut = False # pylint: disable=invalid-name + hasTryItOutRegex = re.compile(r'count=[0-9]+; tryItOut\("') # pylint: disable=invalid-name with open(infilename, 'rb') as f: - for line in fileManipulation.linesWith(f, '; tryItOut("'): - # Checks if testcase came from jsfunfuzz or compareJIT. + for line in file_manipulation.linesWith(f, '; tryItOut("'): + # Checks if testcase came from jsfunfuzz or compare_jit. # Do not use .match here, it only matches from the start of the line: # https://docs.python.org/2/library/re.html#search-vs-match - hasTryItOut = hasTryItOutRegex.search(line) + hasTryItOut = hasTryItOutRegex.search(line) # pylint: disable=invalid-name if hasTryItOut: # Stop searching after finding the first tryItOut line. break # Step 2: Run 1 instance of 1-line reduction after moving tryItOut and count=X around. if lithResult == LITH_FINISHED and origNumOfLines <= 50 and hasTryItOut and lev >= JS_VG_AMISS: - tryItOutAndCountRegex = re.compile(r'"\);\ncount=([0-9]+); tryItOut\("', re.MULTILINE) + tryItOutAndCountRegex = re.compile(r'"\);\ncount=([0-9]+); tryItOut\("', # pylint: disable=invalid-name + re.MULTILINE) with open(infilename, 'rb') as f: - infileContents = f.read() - infileContents = re.sub(tryItOutAndCountRegex, ';\\\n"); count=\\1; tryItOut("\\\n', + infileContents = f.read() # pylint: disable=invalid-name + infileContents = re.sub(tryItOutAndCountRegex, # pylint: disable=invalid-name + ';\\\n"); count=\\1; tryItOut("\\\n', infileContents) with open(infilename, 'wb') as f: f.write(infileContents) @@ -126,12 +131,12 @@ def lithReduceCmd(strategy): print("Running 1 instance of 1-line reduction after moving tryItOut and count=X...") print() # --chunksize=1: Reduce only individual lines, for only 1 round. - lithResult, lithDetails = lithReduceCmd(['--chunksize=1']) + lithResult, lithDetails = lithReduceCmd(['--chunksize=1']) # pylint: disable=invalid-name # Step 3: Run 1 instance of 2-line reduction after moving count=X to its own line and add a # 1-line offset. if lithResult == LITH_FINISHED and origNumOfLines <= 50 and hasTryItOut and lev >= JS_VG_AMISS: - intendedLines = [] + intendedLines = [] # pylint: disable=invalid-name with open(infilename, 'rb') as f: for line in f: # The testcase is likely to already be partially reduced. if 'dumpln(cookie' not in line: # jsfunfuzz-specific line ignore @@ -141,32 +146,32 @@ def lithReduceCmd(strategy): # The 1-line offset is added here. .replace('SPLICE DDBEGIN', 'SPLICE DDBEGIN\n')) - fileManipulation.writeLinesToFile(intendedLines, infilename) + file_manipulation.writeLinesToFile(intendedLines, infilename) print() print("Running 1 instance of 2-line reduction after moving count=X to its own line...") print() - lithResult, lithDetails = lithReduceCmd(['--chunksize=2']) + lithResult, lithDetails = lithReduceCmd(['--chunksize=2']) # pylint: disable=invalid-name # Step 4: Run 1 instance of 2-line reduction again, e.g. to remove pairs of STRICT_MODE lines. if lithResult == LITH_FINISHED and origNumOfLines <= 50 and hasTryItOut and lev >= JS_VG_AMISS: print() print("Running 1 instance of 2-line reduction again...") print() - lithResult, lithDetails = lithReduceCmd(['--chunksize=2']) + lithResult, lithDetails = lithReduceCmd(['--chunksize=2']) # pylint: disable=invalid-name - isLevOverallMismatchAsmJsAvailable = (lev == JS_OVERALL_MISMATCH) and \ - file_contains_str(infilename, 'isAsmJSCompilationAvailable') + isLevOverallMismatchAsmJsAvailable = (lev == JS_OVERALL_MISMATCH and # pylint: disable=invalid-name + file_contains_str(infilename, 'isAsmJSCompilationAvailable')) # Step 5 (not always run): Run character reduction within interesting lines. if lithResult == LITH_FINISHED and origNumOfLines <= 50 and targetTime is None and \ lev >= JS_OVERALL_MISMATCH and not isLevOverallMismatchAsmJsAvailable: print() print("Running character reduction...") print() - lithResult, lithDetails = lithReduceCmd(['--char']) + lithResult, lithDetails = lithReduceCmd(['--char']) # pylint: disable=invalid-name # Step 6: Run line reduction after activating SECOND DDBEGIN with a 1-line offset. if lithResult == LITH_FINISHED and origNumOfLines <= 50 and hasTryItOut and lev >= JS_VG_AMISS: - infileContents = [] + infileContents = [] # pylint: disable=invalid-name with open(infilename, 'rb') as f: for line in f: if 'NIGEBDD' in line: @@ -180,14 +185,14 @@ def lithReduceCmd(strategy): print() print("Running line reduction with a 1-line offset...") print() - lithResult, lithDetails = lithReduceCmd([]) + lithResult, lithDetails = lithReduceCmd([]) # pylint: disable=invalid-name # Step 7: Run line reduction for a final time. if lithResult == LITH_FINISHED and origNumOfLines <= 50 and hasTryItOut and lev >= JS_VG_AMISS: print() print("Running the final line reduction...") print() - lithResult, lithDetails = lithReduceCmd([]) + lithResult, lithDetails = lithReduceCmd([]) # pylint: disable=invalid-name # Restore from backup if testcase can no longer be reproduced halfway through reduction. if lithResult != LITH_FINISHED and lithResult != LITH_PLEASE_CONTINUE: diff --git a/js/shared/mersenne-twister.js b/src/funfuzz/js/shared/mersenne-twister.js similarity index 100% rename from js/shared/mersenne-twister.js rename to src/funfuzz/js/shared/mersenne-twister.js diff --git a/js/shared/random.js b/src/funfuzz/js/shared/random.js similarity index 100% rename from js/shared/random.js rename to src/funfuzz/js/shared/random.js diff --git a/js/shared/testing-functions.js b/src/funfuzz/js/shared/testing-functions.js similarity index 99% rename from js/shared/testing-functions.js rename to src/funfuzz/js/shared/testing-functions.js index c49afd327..c7c0370ec 100644 --- a/js/shared/testing-functions.js +++ b/src/funfuzz/js/shared/testing-functions.js @@ -5,7 +5,7 @@ // Generate calls to SpiderMonkey "testing functions" for: // * testing that they do not cause assertions/crashes -// * testing that they do not alter visible results (compareJIT with and without the call) +// * testing that they do not alter visible results (compare_jit with and without the call) function fuzzTestingFunctionsCtor(browser, fGlobal, fObject) { diff --git a/js/shellFlags.py b/src/funfuzz/js/shell_flags.py similarity index 77% rename from js/shellFlags.py rename to src/funfuzz/js/shell_flags.py index 45bf3286a..77d4f9a03 100644 --- a/js/shellFlags.py +++ b/src/funfuzz/js/shell_flags.py @@ -1,33 +1,29 @@ -#!/usr/bin/env python # coding=utf-8 -# pylint: disable=import-error,invalid-name,missing-docstring -# pylint: disable=too-many-branches,too-many-statements,wrong-import-position # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at https://mozilla.org/MPL/2.0/. +"""Allows detection of support for various command-line flags. +""" + from __future__ import absolute_import, print_function import multiprocessing -import os import random import sys -import inspectShell - -path0 = os.path.dirname(os.path.abspath(__file__)) -path1 = os.path.abspath(os.path.join(path0, os.pardir, 'util')) -sys.path.append(path1) -import subprocesses as sps +from . import inspect_shell +from ..util import subprocesses as sps -def memoize(f, cache=None): +def memoize(f, cache=None): # pylint: disable=missing-param-doc,missing-return-doc,missing-return-type-doc + # pylint: disable=missing-type-doc """Function decorator that caches function results.""" cache = cache or {} # From http://code.activestate.com/recipes/325205-cache-decorator-in-python-24/#c9 - def g(*args, **kwargs): + def g(*args, **kwargs): # pylint: disable=missing-docstring,missing-return-doc,missing-return-type-doc key = (f, tuple(args), frozenset(kwargs.items())) if key not in cache: cache[key] = f(*args, **kwargs) @@ -36,15 +32,17 @@ def g(*args, **kwargs): @memoize -def shellSupportsFlag(shellPath, flag): - return inspectShell.shellSupports(shellPath, [flag, '-e', '42']) +def shellSupportsFlag(shellPath, flag): # pylint: disable=invalid-name,missing-docstring,missing-return-doc + # pylint: disable=missing-return-type-doc + return inspect_shell.shellSupports(shellPath, [flag, '-e', '42']) -def chance(p): +def chance(p): # pylint: disable=invalid-name,missing-docstring,missing-return-doc,missing-return-type-doc return random.random() < p -def randomFlagSet(shellPath): +def randomFlagSet(shellPath): # pylint: disable=invalid-name,missing-param-doc,missing-return-doc + # pylint: disable=missing-return-type-doc,missing-type-doc,too-complex,too-many-branches,too-many-statements """Return a random list of CLI flags appropriate for the given shell. Only works for spidermonkey js shell. Does not work for xpcshell. @@ -79,8 +77,8 @@ def randomFlagSet(shellPath): args.append("--ion-pgo=on") # --ion-pgo=on landed in bug 1209515 if shellSupportsFlag(shellPath, '--ion-sincos=on') and chance(.5): - sincosValue = "on" if chance(0.5) else "off" - args.append("--ion-sincos=" + sincosValue) # --ion-sincos=[on|off] landed in bug 984018 + sincos_switch = "on" if chance(0.5) else "off" + args.append("--ion-sincos=" + sincos_switch) # --ion-sincos=[on|off] landed in bug 984018 if shellSupportsFlag(shellPath, '--ion-instruction-reordering=on') and chance(.2): args.append("--ion-instruction-reordering=on") # --ion-instruction-reordering=on landed in bug 1195545 @@ -112,8 +110,8 @@ def randomFlagSet(shellPath): if shellSupportsFlag(shellPath, '--gc-zeal=0') and chance(.9): # Focus testing on CheckNursery (16), see: # https://hg.mozilla.org/mozilla-central/rev/bdbb5822afe1 - gczealValue = 16 if chance(0.5) else random.randint(0, 16) - args.append("--gc-zeal=" + str(gczealValue)) # --gc-zeal= landed in bug 1101602 + gczeal_value = 16 if chance(0.5) else random.randint(0, 16) + args.append("--gc-zeal=" + str(gczeal_value)) # --gc-zeal= landed in bug 1101602 if shellSupportsFlag(shellPath, '--enable-small-chunk-size') and chance(.1): args.append("--enable-small-chunk-size") # --enable-small-chunk-size landed in bug 941804 @@ -132,19 +130,19 @@ def randomFlagSet(shellPath): shellSupportsFlag(shellPath, '--arm-asm-nop-fill=0') and chance(0.3): # It was suggested to focus more on the range between 0 and 1. # Reduced the upper limit to 8, see bug 1053996 comment 8. - asmNopFill = random.randint(1, 8) if chance(0.3) else random.randint(0, 1) - args.append("--arm-asm-nop-fill=" + str(asmNopFill)) # Landed in bug 1020834 + asm_nop_fill = random.randint(1, 8) if chance(0.3) else random.randint(0, 1) + args.append("--arm-asm-nop-fill=" + str(asm_nop_fill)) # Landed in bug 1020834 # See bug 1026919 comment 60: if sps.isARMv7l and \ shellSupportsFlag(shellPath, '--asm-pool-max-offset=1024') and chance(0.3): - asmPoolMaxOffset = random.randint(5, 1024) - args.append("--asm-pool-max-offset=" + str(asmPoolMaxOffset)) # Landed in bug 1026919 + asm_pool_max_offset = random.randint(5, 1024) + args.append("--asm-pool-max-offset=" + str(asm_pool_max_offset)) # Landed in bug 1026919 if shellSupportsFlag(shellPath, '--no-native-regexp') and chance(.1): args.append("--no-native-regexp") # See bug 976446 - if inspectShell.queryBuildConfiguration(shellPath, 'arm-simulator') and chance(.4): + if inspect_shell.queryBuildConfiguration(shellPath, 'arm-simulator') and chance(.4): args.append('--arm-sim-icache-checks') if (shellSupportsFlag(shellPath, '--no-sse3') and shellSupportsFlag(shellPath, '--no-sse4')) and chance(.2): @@ -176,8 +174,8 @@ def randomFlagSet(shellPath): elif chance(.5) and multiprocessing.cpu_count() > 1 and \ shellSupportsFlag(shellPath, '--thread-count=1'): # Adjusts default number of threads for parallel compilation (turned on by default) - totalThreads = random.randint(2, (multiprocessing.cpu_count() * 2)) - args.append('--thread-count=' + str(totalThreads)) + total_threads = random.randint(2, (multiprocessing.cpu_count() * 2)) + args.append('--thread-count=' + str(total_threads)) # else: # Default is to have --ion-offthread-compile=on and --thread-count= elif shellSupportsFlag(shellPath, '--ion-parallel-compile=off'): @@ -188,8 +186,8 @@ def randomFlagSet(shellPath): elif chance(.5) and multiprocessing.cpu_count() > 1 and \ shellSupportsFlag(shellPath, '--thread-count=1'): # Adjusts default number of threads for parallel compilation (turned on by default) - totalThreads = random.randint(2, (multiprocessing.cpu_count() * 2)) - args.append('--thread-count=' + str(totalThreads)) + total_threads = random.randint(2, (multiprocessing.cpu_count() * 2)) + args.append('--thread-count=' + str(total_threads)) # else: # The default is to have --ion-parallel-compile=on and --thread-count= @@ -232,13 +230,14 @@ def randomFlagSet(shellPath): return args -def basicFlagSets(shellPath): +def basicFlagSets(shellPath): # pylint: disable=invalid-name,missing-param-doc,missing-return-doc + # pylint: disable=missing-return-type-doc,missing-type-doc """These flag combos are used w/the original flag sets when run through Lithium & autoBisect.""" if shellSupportsFlag(shellPath, "--no-threads"): - basicFlagList = [ + basic_flags = [ # Parts of this flag permutation come from: # https://hg.mozilla.org/mozilla-central/file/c91249f41e37/js/src/tests/lib/tests.py#l13 - # compareJIT uses the following first flag set as the sole baseline when fuzzing + # compare_jit uses the following first flag set as the sole baseline when fuzzing ['--fuzzing-safe', '--no-threads', '--ion-eager'], ['--fuzzing-safe', '--ion-offthread-compile=off', '--ion-eager'], ['--fuzzing-safe', '--ion-offthread-compile=off'], @@ -247,22 +246,22 @@ def basicFlagSets(shellPath): ['--fuzzing-safe', '--no-baseline', '--no-ion'], ] if shellSupportsFlag(shellPath, "--non-writable-jitcode"): - basicFlagList.append(['--fuzzing-safe', '--no-threads', '--ion-eager', - '--non-writable-jitcode', '--ion-check-range-analysis', - '--ion-extra-checks', '--no-sse3']) + basic_flags.append(['--fuzzing-safe', '--no-threads', '--ion-eager', + '--non-writable-jitcode', '--ion-check-range-analysis', + '--ion-extra-checks', '--no-sse3']) if shellSupportsFlag(shellPath, "--no-wasm"): - basicFlagList.append(['--fuzzing-safe', '--no-baseline', '--no-asmjs', - '--no-wasm', '--no-native-regexp']) + basic_flags.append(['--fuzzing-safe', '--no-baseline', '--no-asmjs', + '--no-wasm', '--no-native-regexp']) if shellSupportsFlag(shellPath, "--wasm-always-baseline"): - basicFlagList.append(['--fuzzing-safe', '--no-threads', '--ion-eager', - '--wasm-always-baseline']) - return basicFlagList + basic_flags.append(['--fuzzing-safe', '--no-threads', '--ion-eager', + '--wasm-always-baseline']) + return basic_flags elif shellSupportsFlag(shellPath, "--ion-offthread-compile=off"): - basicFlagList = [ + basic_flags = [ # Parts of this flag permutation come from: # https://hg.mozilla.org/mozilla-central/file/84bd8d9f4256/js/src/tests/lib/tests.py#l12 # as well as other interesting flag combinations that have found / may find new bugs. - # compareJIT uses the following first flag set as the sole baseline when fuzzing + # compare_jit uses the following first flag set as the sole baseline when fuzzing ['--fuzzing-safe', '--ion-offthread-compile=off'], ['--fuzzing-safe', '--ion-offthread-compile=off', '--no-baseline'], # Not in jit_test.py though... ['--fuzzing-safe', '--ion-offthread-compile=off', '--no-baseline', '--no-ion'], @@ -273,18 +272,18 @@ def basicFlagSets(shellPath): # ['--fuzzing-safe', '--ion-offthread-compile=off', '--no-fpu'], # --no-fpu seems to be deprecated now ] if shellSupportsFlag(shellPath, "--thread-count=1"): - basicFlagList.append(['--fuzzing-safe', '--ion-offthread-compile=off', '--ion-eager']) + basic_flags.append(['--fuzzing-safe', '--ion-offthread-compile=off', '--ion-eager']) # Range analysis had only started to stabilize around the time when --no-sse3 landed. if shellSupportsFlag(shellPath, '--no-sse3'): - basicFlagList.append(['--fuzzing-safe', '--ion-offthread-compile=off', - '--ion-eager', '--ion-check-range-analysis', '--no-sse3']) - return basicFlagList + basic_flags.append(['--fuzzing-safe', '--ion-offthread-compile=off', + '--ion-eager', '--ion-check-range-analysis', '--no-sse3']) + return basic_flags else: - basicFlagList = [ + basic_flags = [ # Parts of this flag permutation come from: # https://hg.mozilla.org/mozilla-central/file/10932f3a0ba0/js/src/tests/lib/tests.py#l12 # as well as other interesting flag combinations that have found / may find new bugs. - # compareJIT uses the following first flag set as the sole baseline when fuzzing + # compare_jit uses the following first flag set as the sole baseline when fuzzing ['--fuzzing-safe', '--ion-parallel-compile=off'], ['--fuzzing-safe', '--ion-parallel-compile=off', '--no-baseline'], # Not in jit_test.py though... ['--fuzzing-safe', '--ion-parallel-compile=off', '--no-baseline', '--no-ion'], @@ -297,19 +296,19 @@ def basicFlagSets(shellPath): # ['--fuzzing-safe', '--ion-parallel-compile=off', '--baseline-eager', '--no-fpu'], ] if shellSupportsFlag(shellPath, "--thread-count=1"): - basicFlagList.append(['--fuzzing-safe', '--ion-eager', '--ion-parallel-compile=off']) + basic_flags.append(['--fuzzing-safe', '--ion-eager', '--ion-parallel-compile=off']) # Range analysis had only started to stabilize around the time when --no-sse3 landed. if shellSupportsFlag(shellPath, '--no-sse3'): - basicFlagList.append(['--fuzzing-safe', '--ion-parallel-compile=off', - '--ion-eager', '--ion-check-range-analysis', '--no-sse3']) - return basicFlagList + basic_flags.append(['--fuzzing-safe', '--ion-parallel-compile=off', + '--ion-eager', '--ion-check-range-analysis', '--no-sse3']) + return basic_flags -# Consider adding a function (for compareJIT reduction) that takes a flag set +# Consider adding a function (for compare_jit reduction) that takes a flag set # and returns all its (meaningful) subsets. -def testRandomFlags(): +def testRandomFlags(): # pylint: disable=invalid-name,missing-docstring for _ in range(100): print(" ".join(randomFlagSet(sys.argv[1]))) diff --git a/src/funfuzz/loop_bot.py b/src/funfuzz/loop_bot.py new file mode 100644 index 000000000..b973ff2d8 --- /dev/null +++ b/src/funfuzz/loop_bot.py @@ -0,0 +1,51 @@ +# coding=utf-8 +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +"""Loop of { update repos, call bot } to allow things to run unattended +All command-line options are passed through to bot + +Since this script updates the fuzzing repo, it should be very simple, and use subprocess.call() rather than import + +Config-ish bits should move to bot, OR move into a config file, +OR this file should subprocess-call ITSELF rather than using a while loop. +""" + +from __future__ import absolute_import, print_function + +import sys +import subprocess +import time + + +def loop_seq(cmd_seq, wait_time): # pylint: disable=missing-param-doc,missing-type-doc + """Call a sequence of commands in a loop. + If any fails, sleep(wait_time) and go back to the beginning of the sequence.""" + i = 0 + while True: + i += 1 + print("localLoop #%d!" % i) + for cmd in cmd_seq: + try: + subprocess.check_call(cmd) + except subprocess.CalledProcessError as ex: + print("Something went wrong when calling: %r" % (cmd,)) + print("%r" % (ex,)) + import traceback + print(traceback.format_exc()) + print("Waiting %d seconds..." % wait_time) + time.sleep(wait_time) + break + + +def main(): # pylint: disable=missing-docstring + loop_seq([ + [sys.executable, "-u", "-m", "funfuzz.util.repos_update"], + [sys.executable, "-u", "-m", "funfuzz.bot"] + sys.argv[1:] + ], 60) + + +if __name__ == "__main__": + main() diff --git a/util/LockDir.py b/src/funfuzz/util/LockDir.py similarity index 52% rename from util/LockDir.py rename to src/funfuzz/util/LockDir.py index 3c11e92b1..bb2bd3d7a 100644 --- a/util/LockDir.py +++ b/src/funfuzz/util/LockDir.py @@ -1,34 +1,35 @@ -#!/usr/bin/env python # coding=utf-8 -# pylint: disable=invalid-name,missing-docstring,too-few-public-methods # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at https://mozilla.org/MPL/2.0/. +"""A class to create a filesystem-based lock while in scope. The lock directory will be deleted after the lock is +released. +""" + from __future__ import absolute_import, print_function import os -class LockDir(object): - """ - Create a filesystem-based lock while in scope. +class LockDir(object): # pylint: disable=missing-param-doc,missing-type-doc,too-few-public-methods + """Create a filesystem-based lock while in scope. Use: with LockDir(path): # No other code is concurrently using LockDir(path) """ - def __init__(self, d): - self.d = d + def __init__(self, directory): + self.directory = directory def __enter__(self): try: - os.mkdir(self.d) + os.mkdir(self.directory) except OSError: - print("Lock file exists: %s" % self.d) + print("Lock file exists: %s" % self.directory) raise def __exit__(self, exc_type, exc_val, exc_tb): - os.rmdir(self.d) + os.rmdir(self.directory) diff --git a/src/funfuzz/util/__init__.py b/src/funfuzz/util/__init__.py new file mode 100644 index 000000000..832d3086e --- /dev/null +++ b/src/funfuzz/util/__init__.py @@ -0,0 +1,24 @@ +# coding=utf-8 +# flake8: noqa +# pylint: disable=missing-docstring +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import + +from . import crashesat +from . import create_collector +from . import detect_malloc_errors +from . import download_build +from . import file_manipulation +from . import find_ignore_lists +from . import fork_join +from . import hg_helpers +from . import link_js +from . import lithium_helpers +from . import LockDir +from . import repos_update +from . import s3cache +from . import subprocesses diff --git a/util/cdbCmds.txt b/src/funfuzz/util/cdb_cmds.txt similarity index 100% rename from util/cdbCmds.txt rename to src/funfuzz/util/cdb_cmds.txt diff --git a/util/crashesat.py b/src/funfuzz/util/crashesat.py old mode 100755 new mode 100644 similarity index 61% rename from util/crashesat.py rename to src/funfuzz/util/crashesat.py index 176be6992..bf1152d9b --- a/util/crashesat.py +++ b/src/funfuzz/util/crashesat.py @@ -1,22 +1,26 @@ -#!/usr/bin/env python # coding=utf-8 -# pylint: disable=import-error,invalid-name,missing-docstring,wrong-import-position # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at https://mozilla.org/MPL/2.0/. +"""Lithium's "crashesat" interestingness test to assess whether a binary crashes at a desired location. + +Not merged into Lithium, unsure if this still works for now. Still relies on grabCrashLog. +""" + from __future__ import absolute_import, print_function import os from optparse import OptionParser # pylint: disable=deprecated-module -import subprocesses as sps import lithium.interestingness.timed_run as timed_run from lithium.interestingness.utils import file_contains +from . import subprocesses as sps + -def parseOptions(arguments): +def parse_options(arguments): # pylint: disable=missing-docstring,missing-return-doc,missing-return-type-doc parser = OptionParser() parser.disable_interspersed_args() parser.add_option('-r', '--regex', action='store_true', dest='useRegex', @@ -34,28 +38,28 @@ def parseOptions(arguments): return options.useRegex, options.sig, options.condTimeout, args -def interesting(cliArgs, tempPrefix): - (regexEnabled, crashSig, timeout, args) = parseOptions(cliArgs) +def interesting(cli_args, temp_prefix): # pylint: disable=missing-docstring,missing-return-doc,missing-return-type-doc + (regex_enabled, crash_sig, timeout, args) = parse_options(cli_args) - # Examine stack for crash signature, this is needed if crashSig is specified. - runinfo = timed_run.timed_run(args, timeout, tempPrefix) + # Examine stack for crash signature, this is needed if crash_sig is specified. + runinfo = timed_run.timed_run(args, timeout, temp_prefix) if runinfo.sta == timed_run.CRASHED: - sps.grabCrashLog(args[0], runinfo.pid, tempPrefix, True) + sps.grabCrashLog(args[0], runinfo.pid, temp_prefix, True) - timeString = " (%.3f seconds)" % runinfo.elapsedtime + time_str = " (%.3f seconds)" % runinfo.elapsedtime - crashLogName = tempPrefix + "-crash.txt" + crash_log = temp_prefix + "-crash.txt" if runinfo.sta == timed_run.CRASHED: - if os.path.exists(crashLogName): + if os.path.exists(crash_log): # When using this script, remember to escape characters, e.g. "\(" instead of "(" ! - found, _foundSig = file_contains(crashLogName, crashSig, regexEnabled) + found, _found_sig = file_contains(crash_log, crash_sig, regex_enabled) if found: - print("Exit status: %s%s" % (runinfo.msg, timeString)) + print("Exit status: %s%s" % (runinfo.msg, time_str)) return True - print("[Uninteresting] It crashed somewhere else!" + timeString) + print("[Uninteresting] It crashed somewhere else!" + time_str) return False - print("[Uninteresting] It appeared to crash, but no crash log was found?" + timeString) + print("[Uninteresting] It appeared to crash, but no crash log was found?" + time_str) return False - print("[Uninteresting] It didn't crash." + timeString) + print("[Uninteresting] It didn't crash." + time_str) return False diff --git a/util/createCollector.py b/src/funfuzz/util/create_collector.py similarity index 62% rename from util/createCollector.py rename to src/funfuzz/util/create_collector.py index e8f687660..10cbe014f 100644 --- a/util/createCollector.py +++ b/src/funfuzz/util/create_collector.py @@ -1,11 +1,12 @@ -#!/usr/bin/env python # coding=utf-8 -# pylint: disable=import-error,invalid-name,missing-docstring,wrong-import-position # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at https://mozilla.org/MPL/2.0/. +"""Functions here make use of a Collector created from FuzzManager. +""" + from __future__ import absolute_import, print_function import os @@ -13,18 +14,18 @@ from Collector.Collector import Collector -def createCollector(tool): +def createCollector(tool): # pylint: disable=invalid-name,missing-docstring,missing-return-doc,missing-return-type-doc assert tool == "jsfunfuzz" - cacheDir = os.path.normpath(os.path.expanduser(os.path.join("~", "sigcache"))) + cache_dir = os.path.normpath(os.path.expanduser(os.path.join("~", "sigcache"))) try: - os.mkdir(cacheDir) + os.mkdir(cache_dir) except OSError: - pass # cacheDir already exists - collector = Collector(sigCacheDir=cacheDir, tool=tool) + pass # cache_dir already exists + collector = Collector(sigCacheDir=cache_dir, tool=tool) return collector -def printCrashInfo(crashInfo): +def printCrashInfo(crashInfo): # pylint: disable=invalid-name,missing-docstring if crashInfo.createShortSignature() != "No crash detected": print() print("crashInfo:") @@ -34,7 +35,7 @@ def printCrashInfo(crashInfo): print() -def printMatchingSignature(match): +def printMatchingSignature(match): # pylint: disable=invalid-name,missing-docstring print("Matches signature in FuzzManager:") print(" Signature description: %s" % match[1].get('shortDescription')) print(" Signature file: %s" % match[0]) diff --git a/src/funfuzz/util/detect_malloc_errors.py b/src/funfuzz/util/detect_malloc_errors.py new file mode 100644 index 000000000..2ca3919c4 --- /dev/null +++ b/src/funfuzz/util/detect_malloc_errors.py @@ -0,0 +1,49 @@ +# coding=utf-8 +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +"""Look for "szone_error" (Tiger), "malloc_error_break" (Leopard), "MallocHelp" (?) +which are signs of malloc being unhappy (double free, out-of-memory, etc). +""" + +from __future__ import absolute_import, print_function + +PLINE = "" +PPLINE = "" + + +def amiss(log_prefix): # pylint: disable=missing-docstring,missing-return-doc,missing-return-type-doc + found_something = False + global PLINE, PPLINE # pylint: disable=global-statement + + PLINE = "" + PPLINE = "" + + with open(log_prefix + "-err.txt") as f: + for line in f: + if scanLine(line): + found_something = True + break # Don't flood the log with repeated malloc failures + + return found_something + + +def scanLine(line): # pylint: disable=invalid-name,missing-docstring,missing-return-doc,missing-return-type-doc + global PPLINE, PLINE # pylint: disable=global-statement + + line = line.strip("\x07").rstrip("\n") + + if (line.find("szone_error") != -1 or + line.find("malloc_error_break") != -1 or + line.find("MallocHelp") != -1): + if PLINE.find("can't allocate region") == -1: + print() + print(PPLINE) + print(PLINE) + print(line) + return True + + PPLINE = PLINE + PLINE = line diff --git a/util/downloadBuild.py b/src/funfuzz/util/download_build.py old mode 100755 new mode 100644 similarity index 69% rename from util/downloadBuild.py rename to src/funfuzz/util/download_build.py index 29a4ba17d..faaf22f9f --- a/util/downloadBuild.py +++ b/src/funfuzz/util/download_build.py @@ -1,42 +1,44 @@ -#!/usr/bin/env python # coding=utf-8 -# pylint: disable=attribute-defined-outside-init,fixme,import-error,invalid-name,missing-docstring -# pylint: disable=too-many-branches,too-many-locals,too-many-statements # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at https://mozilla.org/MPL/2.0/. +"""Allow downloading of builds. +""" + from __future__ import absolute_import, print_function import argparse -import ConfigParser # pylint: disable=import-error +import ConfigParser # pylint: disable=bad-python3-import,import-error import os import platform import re import shutil -import stat +import stat # Fixed after pylint 1.7.2 was released pylint: disable=bad-python3-import import subprocess import sys import urllib -from HTMLParser import HTMLParser # pylint: disable=import-error +from HTMLParser import HTMLParser # pylint: disable=bad-python3-import,import-error -import subprocesses as sps +from . import subprocesses as sps -def readFromURL(url): +def readFromURL(url): # pylint: disable=invalid-name,missing-param-doc,missing-return-doc,missing-return-type-doc + # pylint: disable=missing-type-doc """Read in a URL and returns its contents as a list.""" return urllib.urlopen(url).read() # pylint: disable=no-member -def dlReport(count, bs, size): +def dlReport(count, bs, size): # pylint: disable=invalid-name,missing-docstring transferred = (100 * count * bs) // size if transferred < 100: sys.stdout.write('\x08\x08\x08%2d%%' % transferred) sys.stdout.flush() -def downloadURL(url, dest, quiet=False): +def downloadURL(url, dest, quiet=False): # pylint: disable=invalid-name,missing-param-doc,missing-return-doc + # pylint: disable=missing-return-type-doc,missing-type-doc """Read in a URL and downloads it to a destination.""" quiet = quiet or not sys.stdout.isatty() if not quiet: @@ -48,7 +50,7 @@ def downloadURL(url, dest, quiet=False): return dest -def parseOptions(): +def parseOptions(): # pylint: disable=invalid-name,missing-docstring,missing-return-doc,missing-return-type-doc usage = 'Usage: %(prog)s [options]' parser = argparse.ArgumentParser(usage) @@ -84,24 +86,26 @@ def parseOptions(): return parser.parse_args() -class MyHTMLParser(HTMLParser): +class MyHTMLParser(HTMLParser): # pylint: disable=missing-docstring - def getHrefLinks(self, html, baseURI): + def getHrefLinks(self, html, baseURI): # pylint: disable=invalid-name,missing-docstring,missing-return-doc + # pylint: disable=missing-return-type-doc thirdslash = find_nth(baseURI, "/", 0, 3) + # pylint: disable=attribute-defined-outside-init self.basepath = baseURI[thirdslash:] # e.g. "/pub/firefox/tinderbox-builds/" - self.hrefLinksList = [] + self.hrefLinksList = [] # pylint: disable=invalid-name,attribute-defined-outside-init self.feed(html) return self.hrefLinksList - def handle_starttag(self, tag, attrs): - aTagFound = False + def handle_starttag(self, tag, attrs): # pylint: disable=missing-docstring + a_tag_found = False if tag == 'a': - aTagFound = True + a_tag_found = True for attr in attrs: - if not aTagFound: + if not a_tag_found: break - if aTagFound and attr[0] == 'href': + if a_tag_found and attr[0] == 'href': if attr[1][0] == '/': # Convert site-relative URI to fully-relative URI if attr[1].startswith(self.basepath): @@ -111,7 +115,8 @@ def handle_starttag(self, tag, attrs): self.hrefLinksList.append(attr[1]) -def find_nth(haystack, needle, start, n): +def find_nth(haystack, needle, start, n): # pylint: disable=invalid-name,missing-docstring,missing-return-doc + # pylint: disable=missing-return-type-doc for _ in range(n): start = haystack.find(needle, start + 1) if start == -1: @@ -119,30 +124,30 @@ def find_nth(haystack, needle, start, n): return start -def httpDirList(directory): +def httpDirList(directory): # pylint: disable=invalid-name,missing-param-doc,missing-return-doc,missing-return-type-doc + # pylint: disable=missing-type-doc """Read an Apache-style directory listing and returns a list of its contents, as relative URLs.""" print("Looking in %s ..." % directory) page = readFromURL(directory) sps.vdump('Finished reading from: ' + directory) parser = MyHTMLParser() - fileList = parser.getHrefLinks(page, directory) - return fileList + return parser.getHrefLinks(page, directory) -def unzip(fn, dest): +def unzip(fn, dest): # pylint: disable=invalid-name,missing-param-doc,missing-type-doc """Extract .zip files to their destination.""" subprocess.check_output(['unzip', fn, '-d', dest]) -def untarbz2(fn, dest): +def untarbz2(fn, dest): # pylint: disable=invalid-name,missing-param-doc,missing-type-doc """Extract .tar.bz2 files to their destination.""" if not os.path.exists(dest): os.mkdir(dest) subprocess.check_output(['tar', '-C', dest, '-xjf', os.path.abspath(fn)]) -def undmg(fn, dest, mountpoint): +def undmg(fn, dest, mountpoint): # pylint: disable=invalid-name,missing-param-doc,missing-type-doc """Extract .dmg files to their destination via a mount point.""" if os.path.exists(mountpoint): # If the mount point already exists, detach it first. @@ -156,22 +161,24 @@ def undmg(fn, dest, mountpoint): subprocess.check_output(['hdiutil', 'detach', mountpoint]) -def downloadBuild(httpDir, targetDir, jsShell=False, wantSymbols=True, wantTests=True): +def downloadBuild(httpDir, targetDir, jsShell=False, wantSymbols=True, wantTests=True): # pylint: disable=invalid-name + # pylint: disable=missing-param-doc,missing-return-doc,missing-return-type-doc,missing-type-doc,too-complex + # pylint: disable=too-many-locals,too-many-statements """Download the build specified, along with symbols and tests. Returns True when all are obtained.""" wantSymbols = wantSymbols and not jsShell # Bug 715365, js shell currently lacks native symbols wantSymbols = wantSymbols and '-asan' not in httpDir # Doesn't make sense for asan wantTests = wantTests and not jsShell - gotApp = False - gotTests = False - gotTxtFile = False - gotSyms = False + gotApp = False # pylint: disable=invalid-name + gotTests = False # pylint: disable=invalid-name + gotTxtFile = False # pylint: disable=invalid-name + gotSyms = False # pylint: disable=invalid-name # Create build folder and a download subfolder. - buildDir = os.path.abspath(sps.normExpUserPath(os.path.join(targetDir, 'build'))) + buildDir = os.path.abspath(sps.normExpUserPath(os.path.join(targetDir, 'build'))) # pylint: disable=invalid-name if os.path.exists(buildDir): print("Deleting old build...") shutil.rmtree(buildDir) os.mkdir(buildDir) - downloadFolder = os.path.join(buildDir, 'download') + downloadFolder = os.path.join(buildDir, 'download') # pylint: disable=invalid-name os.mkdir(downloadFolder) with open(os.path.join(downloadFolder, "source-url.txt"), "w") as f: @@ -179,11 +186,12 @@ def downloadBuild(httpDir, targetDir, jsShell=False, wantSymbols=True, wantTests # Hack #1 for making os.path.join(reftestScriptDir, automation.DEFAULT_APP) work is to: # Call this directory "dist". - appDir = os.path.join(buildDir, 'dist') + os.sep - testsDir = os.path.join(buildDir, 'tests') + os.sep - symbolsDir = os.path.join(buildDir, 'symbols') + os.sep - fileHttpRawList = httpDirList(httpDir) + appDir = os.path.join(buildDir, 'dist') + os.sep # pylint: disable=invalid-name + testsDir = os.path.join(buildDir, 'tests') + os.sep # pylint: disable=invalid-name + symbolsDir = os.path.join(buildDir, 'symbols') + os.sep # pylint: disable=invalid-name + fileHttpRawList = httpDirList(httpDir) # pylint: disable=invalid-name # We only want files, those with file extensions, not folders. + # pylint: disable=invalid-name fileHttpList = [httpDir + x for x in fileHttpRawList if '.' in x and 'mozilla.org' not in x] for remotefn in fileHttpList: @@ -221,7 +229,7 @@ def downloadBuild(httpDir, targetDir, jsShell=False, wantSymbols=True, wantTests print("completed!") gotApp = True # Bug 715365 - note that js shell currently lacks native symbols writeDownloadedShellFMConf(remotefn, buildDir) - else: + else: # pylint: disable=else-if-used if re.search(r'(\.linux-(x86_64|i686)(-asan)?|^target)\.tar\.bz2$', fn): print("Downloading application...", end=" ") dlAction = downloadURL(remotefn, localfn) @@ -277,56 +285,56 @@ def downloadBuild(httpDir, targetDir, jsShell=False, wantSymbols=True, wantTests return gotApp and gotTxtFile and (gotTests or not wantTests) and (gotSyms or not wantSymbols) -def downloadMDSW(buildDir, manifestPlatform): +def downloadMDSW(buildDir, manifestPlatform): # pylint: disable=invalid-name,missing-param-doc,missing-type-doc """Download the minidump_stackwalk[.exe] binary for this platform.""" - THIS_SCRIPT_DIRECTORY = os.path.dirname(os.path.abspath(__file__)) - TOOLTOOL_PY = os.path.join(THIS_SCRIPT_DIRECTORY, "tooltool", "tooltool.py") - - # Find the tooltool manifest for this platform - manifestFilename = os.path.join(THIS_SCRIPT_DIRECTORY, "tooltool", manifestPlatform + ".manifest") - # Download the binary (using tooltool) - subprocess.check_call([sys.executable, TOOLTOOL_PY, "-m", manifestFilename, "fetch"], cwd=buildDir) + subprocess.check_call([sys.executable, + os.path.join(os.path.dirname(os.path.abspath(__file__)), "tooltool", "tooltool.py"), + "-m", + # Find the tooltool manifest for this platform + os.path.join(os.path.dirname(os.path.abspath(__file__)), + "tooltool", manifestPlatform + ".manifest"), + "fetch"], cwd=buildDir) # Mark the binary as executable if platform.system() != 'Windows': - stackwalkBin = os.path.join(buildDir, "minidump_stackwalk") - os.chmod(stackwalkBin, stat.S_IRWXU) + os.chmod(os.path.join(buildDir, "minidump_stackwalk"), stat.S_IRWXU) -def moveCrashInjector(tests): - # Hackaround for crashinject.exe not being a reliable way to kill firefox.exe (see bug 888748) - testsBin = os.path.join(tests, "bin") - crashinject = os.path.join(testsBin, "crashinject.exe") +def moveCrashInjector(tests): # pylint: disable=invalid-name,missing-param-doc,missing-type-doc + """Hackaround for crashinject.exe not being a reliable way to kill firefox.exe (see bug 888748)""" + tests_bin = os.path.join(tests, "bin") + crashinject = os.path.join(tests_bin, "crashinject.exe") if os.path.exists(crashinject): - shutil.move(crashinject, os.path.join(testsBin, "crashinject-disabled.exe")) - - -def mIfyMozcrash(testsDir): - # Terrible hack to pass "-m" to breakpad through mozcrash - mozcrashDir = os.path.join(testsDir, "mozbase", "mozcrash", "mozcrash") - mozcrashPy = os.path.join(mozcrashDir, "mozcrash.py") - # print mozcrashPy - mozcrashPyBak = os.path.join(mozcrashDir, "mozcrash.py.bak") - shutil.copyfile(mozcrashPy, mozcrashPyBak) - with open(mozcrashPy, "w") as outfile: - with open(mozcrashPyBak) as infile: + shutil.move(crashinject, os.path.join(tests_bin, "crashinject-disabled.exe")) + + +def mIfyMozcrash(testsDir): # pylint: disable=invalid-name,missing-param-doc,missing-type-doc + """Terrible hack to pass "-m" to breakpad through mozcrash""" + mozcrash_dir = os.path.join(testsDir, "mozbase", "mozcrash", "mozcrash") + mozcrash_py = os.path.join(mozcrash_dir, "mozcrash.py") + mozcrash_py_bak = os.path.join(mozcrash_dir, "mozcrash.py.bak") + shutil.copyfile(mozcrash_py, mozcrash_py_bak) + with open(mozcrash_py, "w") as outfile: + with open(mozcrash_py_bak) as infile: for line in infile: outfile.write(line) if line.strip() == "self.stackwalk_binary,": outfile.write("\"-m\",\n") -def isNumericSubDir(n): +def isNumericSubDir(n): # pylint: disable=invalid-name,missing-param-doc,missing-return-doc,missing-return-type-doc + # pylint: disable=missing-type-doc """Return True if input is a numeric directory, False if not. e.g. 1234/ returns True.""" return re.match(r'^\d+$', n.split('/')[0]) -def getBuildList(buildType, earliestBuild='default', latestBuild='default'): +def getBuildList(buildType, earliestBuild='default', latestBuild='default'): # pylint: disable=invalid-name + # pylint: disable=missing-param-doc,missing-raises-doc,missing-return-doc,missing-return-type-doc,missing-type-doc """Return the list of URLs of builds (e.g. 1386614507) that are present in tinderbox-builds/.""" - buildsHttpDir = 'https://archive.mozilla.org/pub/firefox/tinderbox-builds/' + \ - buildType + '/' - dirNames = httpDirList(buildsHttpDir) + buildsHttpDir = ("https://archive.mozilla.org/pub/firefox/tinderbox-builds/%s/" # pylint: disable=invalid-name + % buildType) + dirNames = httpDirList(buildsHttpDir) # pylint: disable=invalid-name if earliestBuild != 'default': earliestBuild = earliestBuild + '/' @@ -338,17 +346,19 @@ def getBuildList(buildType, earliestBuild='default', latestBuild='default'): # Earlier downloaded builds fail to start properly on macOS Sierra 10.12 # First known working build is in: # https://archive.mozilla.org/pub/firefox/tinderbox-builds/mozilla-inbound-macosx64-debug/1468314445/ - # Note: if this gets more populated, we should move it to knownBrokenEarliestWorking + # Note: if this gets more populated, we should move it to known_broken_earliest_working if sps.isMac and int(earliestBuild[:-1]) < 1468314445: earliestBuild = '1468314445/' try: - earliestBuildIndex = dirNames.index(earliestBuild) # Set the start boundary + # Set the start boundary + earliestBuildIndex = dirNames.index(earliestBuild) # pylint: disable=invalid-name except ValueError: # Sometimes 1468314445 is not found if sps.isMac and int(earliestBuild[:-1]) < 1468333601: earliestBuild = '1468333601/' - earliestBuildIndex = dirNames.index(earliestBuild) # Set the start boundary + # Set the start boundary + earliestBuildIndex = dirNames.index(earliestBuild) # pylint: disable=invalid-name if latestBuild != 'default': latestBuild = latestBuild + '/' @@ -356,39 +366,41 @@ def getBuildList(buildType, earliestBuild='default', latestBuild='default'): raise Exception('Latest build is not found in list of IDs.') else: latestBuild = dirNames[-1] - latestBuildIndex = dirNames.index(latestBuild) # Set the end boundary + # Set the end boundary + latestBuildIndex = dirNames.index(latestBuild) # pylint: disable=invalid-name - dirNames = dirNames[earliestBuildIndex:latestBuildIndex + 1] + dirNames = dirNames[earliestBuildIndex:latestBuildIndex + 1] # pylint: disable=invalid-name - buildDirs = [(buildsHttpDir + d) for d in dirNames if isNumericSubDir(d)] + buildDirs = [(buildsHttpDir + d) for d in dirNames if isNumericSubDir(d)] # pylint: disable=invalid-name if len(buildDirs) < 1: print("Warning: No builds in %s!" % buildsHttpDir) return buildDirs -def downloadLatestBuild(buildType, workingDir, getJsShell=False, wantTests=False): +def downloadLatestBuild(buildType, workingDir, getJsShell=False, wantTests=False): # pylint: disable=invalid-name + # pylint: disable=missing-param-doc,missing-raises-doc,missing-return-doc,missing-return-type-doc,missing-type-doc """Download the latest build based on machine type, e.g. mozilla-central-macosx64-debug.""" # Try downloading the latest build first. - for buildURL in reversed(getBuildList(buildType)): - if downloadBuild(buildURL, workingDir, jsShell=getJsShell, wantTests=wantTests): - return buildURL + for build_url in reversed(getBuildList(buildType)): + if downloadBuild(build_url, workingDir, jsShell=getJsShell, wantTests=wantTests): + return build_url raise Exception("No complete builds found.") -def mozPlatformDetails(): +def mozPlatformDetails(): # pylint: disable=invalid-name,missing-raises-doc,missing-return-doc,missing-return-type-doc """Determine the platform of the system and returns the RelEng-specific build type.""" - s = platform.system() - if s == "Darwin": + if platform.system() == "Darwin": return ("macosx", "macosx64", platform.architecture()[0] == "64bit") - elif s == "Linux": + elif platform.system() == "Linux": return ("linux", "linux64", platform.machine() == "x86_64") - elif s == 'Windows': + elif platform.system() == 'Windows': return ("win32", "win64", False) else: - raise Exception("Unknown platform.system(): " + s) + raise Exception("Unknown platform.system(): " + platform.system()) -def mozPlatform(arch): +def mozPlatform(arch): # pylint: disable=invalid-name,missing-param-doc,missing-raises-doc,missing-return-doc + # pylint: disable=missing-return-type-doc,missing-type-doc """Return the native build type of the current machine.""" (name32, name64, native64) = mozPlatformDetails() if arch == "64": @@ -396,6 +408,7 @@ def mozPlatform(arch): elif arch == "32": return name32 elif arch is None: + # pylint: disable=fixme # FIXME: Eventually, we should set 64-bit as native for Win64. We should also aim to test # both 32-bit and 64-bit Firefox builds on any platform that supports both. Let us make # sure Python detects 32-bit Windows vs 64-bit Windows correctly before changing this. @@ -404,14 +417,16 @@ def mozPlatform(arch): raise Exception("The arch passed to mozPlatform must be '64', '32', or None") -def defaultBuildType(repoName, arch, debug, asan=False): +def defaultBuildType(repoName, arch, debug, asan=False): # pylint: disable=invalid-name,missing-param-doc + # pylint: disable=missing-return-doc,missing-return-type-doc,missing-type-doc """Return the default build type as per RelEng, e.g. mozilla-central-macosx-debug.""" return repoName + '-' + mozPlatform(arch) + ('-asan' if asan else '') + ('-debug' if debug else '') -def writeDownloadedShellFMConf(urlLink, bDir): +def writeDownloadedShellFMConf(urlLink, bDir): # pylint: disable=invalid-name,missing-param-doc,missing-type-doc + # pylint: disable=too-complex,too-many-branches """Writes an arbitrary .fuzzmanagerconf file for downloaded js shells.""" - downloadedShellCfg = ConfigParser.SafeConfigParser() + downloadedShellCfg = ConfigParser.SafeConfigParser() # pylint: disable=invalid-name downloadedShellCfg.add_section('Main') # Note that this does not differentiate between debug/asan/optimized builds @@ -448,7 +463,7 @@ def writeDownloadedShellFMConf(urlLink, bDir): downloadedShellCfg.set('Main', 'product', 'mozilla-esr52') downloadedShellCfg.set('Main', 'os', osname) - downloadedShellFMConfPath = os.path.join(bDir, 'dist', 'js.fuzzmanagerconf') + downloadedShellFMConfPath = os.path.join(bDir, 'dist', 'js.fuzzmanagerconf') # pylint: disable=invalid-name if not os.path.isfile(downloadedShellFMConfPath): with open(downloadedShellFMConfPath, 'wb') as cfgfile: downloadedShellCfg.write(cfgfile) @@ -461,7 +476,7 @@ def writeDownloadedShellFMConf(urlLink, bDir): assert cfg.get('Main', 'os') -def main(): +def main(): # pylint: disable=missing-docstring options = parseOptions() # On Windows, if a path surrounded by quotes ends with '\', the last quote is considered escaped and will be # part of the option. This is not what the user expects, so remove any trailing quotes from paths: @@ -471,9 +486,9 @@ def main(): print(downloadBuild( options.remoteDir, options.downloadFolder, jsShell=options.enableJsShell, wantTests=options.wantTests)) else: - buildType = defaultBuildType(options.repoName, options.arch, (options.compileType == 'dbg'), - asan=options.useAsan) - downloadLatestBuild(buildType, options.downloadFolder, + build_type = defaultBuildType(options.repoName, options.arch, (options.compileType == 'dbg'), + asan=options.useAsan) + downloadLatestBuild(build_type, options.downloadFolder, getJsShell=options.enableJsShell, wantTests=options.wantTests) diff --git a/util/fileManipulation.py b/src/funfuzz/util/file_manipulation.py similarity index 50% rename from util/fileManipulation.py rename to src/funfuzz/util/file_manipulation.py index 1cf50e675..3dd7b1495 100644 --- a/util/fileManipulation.py +++ b/src/funfuzz/util/file_manipulation.py @@ -1,20 +1,23 @@ -#!/usr/bin/env python # coding=utf-8 -# pylint: disable=invalid-name,missing-docstring # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at https://mozilla.org/MPL/2.0/. +"""Functions dealing with files and their contents. +""" + from __future__ import absolute_import, print_function -def firstLine(s): +def firstLine(s): # pylint: disable=invalid-name,missing-param-doc,missing-return-doc,missing-return-type-doc + # pylint: disable=missing-type-doc """Return the first line of any series of text with / without line breaks.""" return s.split('\n')[0] -def fuzzDice(filename): +def fuzzDice(filename): # pylint: disable=invalid-name,missing-param-doc,missing-return-doc,missing-return-type-doc + # pylint: disable=missing-type-doc """Return the lines of the file, except for the one line containing DICE.""" before = [] after = [] @@ -28,7 +31,8 @@ def fuzzDice(filename): return [before, after] -def fuzzSplice(filename): +def fuzzSplice(filename): # pylint: disable=invalid-name,missing-param-doc,missing-return-doc,missing-return-type-doc + # pylint: disable=missing-type-doc """Return the lines of a file, minus the ones between the two lines containing SPLICE.""" before = [] after = [] @@ -46,32 +50,35 @@ def fuzzSplice(filename): return [before, after] -def linesWith(lines, searchFor): +def linesWith(lines, search_for): # pylint: disable=invalid-name,missing-param-doc,missing-return-doc + # pylint: disable=missing-return-type-doc,missing-type-doc """Return the lines from an array that contain a given string.""" - matchingLines = [] + matched = [] for line in lines: - if line.find(searchFor) != -1: - matchingLines.append(line) - return matchingLines + if line.find(search_for) != -1: + matched.append(line) + return matched -def linesStartingWith(lines, searchFor): +def linesStartingWith(lines, search_for): # pylint: disable=invalid-name,missing-param-doc,missing-return-doc + # pylint: disable=missing-return-type-doc,missing-type-doc """Return the lines from an array that start with a given string.""" - matchingLines = [] + matched = [] for line in lines: - if line.startswith(searchFor): - matchingLines.append(line) - return matchingLines + if line.startswith(search_for): + matched.append(line) + return matched -def truncateMid(a, limitEachSide, insertIfTruncated): - """Return a list with the middle portion removed, if it has more than limitEachSide*2 items.""" - if len(a) <= limitEachSide + limitEachSide: +def truncateMid(a, limit_each_side, insert_if_truncated): # pylint: disable=invalid-name,missing-param-doc + # pylint: disable=missing-return-doc,missing-return-type-doc,missing-type-doc + """Return a list with the middle portion removed, if it has more than limit_each_side*2 items.""" + if len(a) <= limit_each_side + limit_each_side: return a - return a[0:limitEachSide] + insertIfTruncated + a[-limitEachSide:] + return a[0:limit_each_side] + insert_if_truncated + a[-limit_each_side:] -def writeLinesToFile(lines, filename): +def writeLinesToFile(lines, filename): # pylint: disable=invalid-name,missing-param-doc,missing-type-doc """Write lines to a given filename.""" with open(filename, 'wb') as f: f.writelines(lines) diff --git a/detect/findIgnoreLists.py b/src/funfuzz/util/find_ignore_lists.py similarity index 52% rename from detect/findIgnoreLists.py rename to src/funfuzz/util/find_ignore_lists.py index aac6bbfec..b813022ce 100644 --- a/detect/findIgnoreLists.py +++ b/src/funfuzz/util/find_ignore_lists.py @@ -1,11 +1,12 @@ -#!/usr/bin/env python # coding=utf-8 -# pylint: disable=invalid-name,missing-docstring # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at https://mozilla.org/MPL/2.0/. +"""Functions here attempt to find lists of bugs to ignore, e.g. via suppression files. +""" + from __future__ import absolute_import, print_function import os @@ -18,22 +19,23 @@ # from private&public fuzzing repos # for project branches and for their base branches (e.g. mozilla-central) # -# Given a targetRepo "mozilla-central/ionmonkey" and a name "crashes.txt", returns a list of 2N absolute paths like: +# Given a target_repo "mozilla-central/ionmonkey" and a name "crashes.txt", returns a list of 2N absolute paths like: # ???/funfuzz*/known/mozilla-central/ionmonkey/crashes.txt # ???/funfuzz*/known/mozilla-central/crashes.txt -def findIgnoreLists(targetRepo, needle): - r = [] - assert not targetRepo.startswith("/") +def find_ignore_lists(target_repo, needle): # pylint: disable=missing-docstring,missing-return-doc + # pylint: disable=missing-return-type-doc + suppressions = [] + assert not target_repo.startswith("/") for name in sorted(os.listdir(REPO_PARENT_PATH)): if name.startswith("funfuzz"): - knownPath = os.path.join(REPO_PARENT_PATH, name, "known", targetRepo) - if os.path.isdir(knownPath): - while os.path.basename(knownPath) != "known": - filename = os.path.join(knownPath, needle) + known_path = os.path.join(REPO_PARENT_PATH, name, "known", target_repo) + if os.path.isdir(known_path): + while os.path.basename(known_path) != "known": + filename = os.path.join(known_path, needle) if os.path.exists(filename): - r.append(filename) - knownPath = os.path.dirname(knownPath) - assert r - return r + suppressions.append(filename) + known_path = os.path.dirname(known_path) + assert suppressions + return suppressions diff --git a/util/forkJoin.py b/src/funfuzz/util/fork_join.py similarity index 71% rename from util/forkJoin.py rename to src/funfuzz/util/fork_join.py index e3e01c1ae..f0500f25e 100644 --- a/util/forkJoin.py +++ b/src/funfuzz/util/fork_join.py @@ -1,11 +1,12 @@ -#!/usr/bin/env python # coding=utf-8 -# pylint: disable=invalid-name,missing-docstring # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at https://mozilla.org/MPL/2.0/. +"""Functions dealing with multiple processes. +""" + from __future__ import absolute_import, print_function import multiprocessing @@ -16,8 +17,9 @@ # Call |fun| in a bunch of separate processes, then wait for them all to finish. # fun is called with someArgs, plus an additional argument with a numeric ID. # |fun| must be a top-level function (not a closure) so it can be pickled on Windows. -def forkJoin(logDir, numProcesses, fun, *someArgs): - def showFile(fn): +def forkJoin(logDir, numProcesses, fun, *someArgs): # pylint: disable=invalid-name,missing-docstring,missing-return-doc + # pylint: disable=missing-return-type-doc + def showFile(fn): # pylint: disable=invalid-name,missing-docstring print("==== %s ====" % fn) print() with open(fn) as f: @@ -27,16 +29,16 @@ def showFile(fn): # Fork a bunch of processes print("Forking %d children..." % numProcesses) - ps = [] + ps = [] # pylint: disable=invalid-name for i in range(numProcesses): - p = multiprocessing.Process( + p = multiprocessing.Process( # pylint: disable=invalid-name target=redirectOutputAndCallFun, args=[logDir, i, fun, someArgs], name="Parallel process " + str(i)) p.start() ps.append(p) # Wait for them all to finish, and splat their outputs for i in range(numProcesses): - p = ps[i] + p = ps[i] # pylint: disable=invalid-name print("=== Waiting for child #%d (%d) to finish... ===" % (i, p.pid)) p.join() print("=== Child process #%d exited with code %d ===" % (i, p.exitcode)) @@ -47,11 +49,12 @@ def showFile(fn): # Functions used by forkJoin are top-level so they can be "pickled" (required on Windows) -def logFileName(logDir, i, t): +def logFileName(logDir, i, t): # pylint: disable=invalid-name,missing-docstring,missing-return-doc + # pylint: disable=missing-return-type-doc return os.path.join(logDir, "forkjoin-" + str(i) + "-" + t + ".txt") -def redirectOutputAndCallFun(logDir, i, fun, someArgs): +def redirectOutputAndCallFun(logDir, i, fun, someArgs): # pylint: disable=invalid-name,missing-docstring sys.stdout = open(logFileName(logDir, i, "out"), 'wb', buffering=0) sys.stderr = open(logFileName(logDir, i, "err"), 'wb', buffering=0) fun(*(someArgs + (i,))) @@ -61,11 +64,11 @@ def redirectOutputAndCallFun(logDir, i, fun, someArgs): # * "Green Chairs" from the first few processes # * A pause and error (with stack trace) from process 5 # * "Green Chairs" again from the rest. -def test_forkJoin(): +def test_forkJoin(): # pylint: disable=invalid-name,missing-docstring forkJoin(".", 8, test_forkJoin_inner, "Green", "Chairs") -def test_forkJoin_inner(adj, noun, forkjoin_id): +def test_forkJoin_inner(adj, noun, forkjoin_id): # pylint: disable=invalid-name,missing-docstring import time print("%s %s" % (adj, noun)) print(forkjoin_id) diff --git a/util/gdb-quick.txt b/src/funfuzz/util/gdb_cmds.txt similarity index 100% rename from util/gdb-quick.txt rename to src/funfuzz/util/gdb_cmds.txt diff --git a/src/funfuzz/util/hg_helpers.py b/src/funfuzz/util/hg_helpers.py new file mode 100644 index 000000000..93dd75c67 --- /dev/null +++ b/src/funfuzz/util/hg_helpers.py @@ -0,0 +1,184 @@ +# coding=utf-8 +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +"""Helper functions involving Mercurial (hg). +""" + +from __future__ import absolute_import, print_function + +import ConfigParser # pylint: disable=bad-python3-import,import-error +import os +import re +import sys +import subprocess + +from . import subprocesses as sps + + +try: + input = raw_input # pylint: disable=invalid-name,raw_input-builtin,redefined-builtin +except NameError: + pass + + +def destroyPyc(repoDir): # pylint: disable=invalid-name,missing-docstring + # This is roughly equivalent to ['hg', 'purge', '--all', '--include=**.pyc']) + # but doesn't run into purge's issues (incompatbility with -R, requiring an hg extension) + for root, dirs, files in os.walk(repoDir): + for fn in files: # pylint: disable=invalid-name + if fn.endswith(".pyc"): + os.remove(os.path.join(root, fn)) + if '.hg' in dirs: + # Don't visit .hg dir + dirs.remove('.hg') + + +def ensureMqEnabled(): # pylint: disable=invalid-name,missing-raises-doc + """Ensure that mq is enabled in the ~/.hgrc file.""" + user_hgrc = os.path.join(os.path.expanduser('~'), '.hgrc') + assert os.path.isfile(user_hgrc) + + user_hgrc_cfg = ConfigParser.SafeConfigParser() + user_hgrc_cfg.read(user_hgrc) + + try: + user_hgrc_cfg.get('extensions', 'mq') + except ConfigParser.NoOptionError: + raise Exception('Please first enable mq in ~/.hgrc by having "mq =" in [extensions].') + + +def findCommonAncestor(repoDir, a, b): # pylint: disable=invalid-name,missing-docstring,missing-return-doc + # pylint: disable=missing-return-type-doc + return sps.captureStdout(['hg', '-R', repoDir, 'log', '-r', 'ancestor(' + a + ',' + b + ')', + '--template={node|short}'])[0] + + +def isAncestor(repoDir, a, b): # pylint: disable=invalid-name,missing-param-doc,missing-return-doc + # pylint: disable=missing-return-type-doc,missing-type-doc + """Return true iff |a| is an ancestor of |b|. Throw if |a| or |b| does not exist.""" + return sps.captureStdout(['hg', '-R', repoDir, 'log', '-r', a + ' and ancestor(' + a + ',' + b + ')', + '--template={node|short}'])[0] != "" + + +def existsAndIsAncestor(repoDir, a, b): # pylint: disable=invalid-name,missing-param-doc,missing-return-doc + # pylint: disable=missing-return-type-doc,missing-type-doc + """Return true iff |a| exists and is an ancestor of |b|.""" + # Takes advantage of "id(badhash)" being the empty set, in contrast to just "badhash", which is an error + out = sps.captureStdout(['hg', '-R', repoDir, 'log', '-r', a + ' and ancestor(' + a + ',' + b + ')', + '--template={node|short}'], combineStderr=True, ignoreExitCode=True)[0] + return out != "" and out.find("abort: unknown revision") < 0 + + +def getCsetHashFromBisectMsg(msg): # pylint: disable=invalid-name,missing-docstring,missing-return-doc + # pylint: disable=missing-return-type-doc + # Example bisect msg: "Testing changeset 41831:4f4c01fb42c3 (2 changesets remaining, ~1 tests)" + rgx = re.compile(r"(^|.* )(\d+):(\w{12}).*") + matched = rgx.match(msg) + if matched: + return matched.group(3) + + +assert getCsetHashFromBisectMsg("x 12345:abababababab") == "abababababab" +assert getCsetHashFromBisectMsg("x 12345:123412341234") == "123412341234" +assert getCsetHashFromBisectMsg("12345:abababababab y") == "abababababab" + + +def getRepoHashAndId(repoDir, repoRev='parents() and default'): # pylint: disable=invalid-name,missing-param-doc + # pylint: disable=missing-raises-doc,missing-return-doc,missing-return-type-doc,missing-type-doc + """Return the repository hash and id, and whether it is on default. + + It will also ask what the user would like to do, should the repository not be on default. + """ + # This returns null if the repository is not on default. + hg_log_template_cmds = ['hg', '-R', repoDir, 'log', '-r', repoRev, + '--template', '{node|short} {rev}'] + hg_id_full = sps.captureStdout(hg_log_template_cmds)[0] + is_on_default = bool(hg_id_full) + if not is_on_default: + update_default = input("Not on default tip! " + "Would you like to (a)bort, update to (d)efault, or (u)se this rev: ") + update_default = update_default.strip() + if update_default == 'a': + print("Aborting...") + sys.exit(0) + elif update_default == 'd': + subprocess.check_call(['hg', '-R', repoDir, 'update', 'default']) + is_on_default = True + elif update_default == 'u': + hg_log_template_cmds = ['hg', '-R', repoDir, 'log', '-r', 'parents()', '--template', + '{node|short} {rev}'] + else: + raise Exception('Invalid choice.') + hg_id_full = sps.captureStdout(hg_log_template_cmds)[0] + assert hg_id_full != '' + (hg_id_hash, hg_id_local_num) = hg_id_full.split(' ') + sps.vdump('Finished getting the hash and local id number of the repository.') + return hg_id_hash, hg_id_local_num, is_on_default + + +def getRepoNameFromHgrc(repoDir): # pylint: disable=invalid-name,missing-param-doc,missing-return-doc + # pylint: disable=missing-return-type-doc,missing-type-doc + """Look in the hgrc file in the .hg directory of the repository and return the name.""" + assert isRepoValid(repoDir) + hgrc_cfg = ConfigParser.SafeConfigParser() + hgrc_cfg.read(sps.normExpUserPath(os.path.join(repoDir, '.hg', 'hgrc'))) + # Not all default entries in [paths] end with "/". + return [i for i in hgrc_cfg.get('paths', 'default').split('/') if i][-1] + + +def isRepoValid(repo): # pylint: disable=invalid-name,missing-param-doc,missing-return-doc + # pylint: disable=missing-return-type-doc,missing-type-doc + """Check that a repository is valid by ensuring that the hgrc file is around.""" + return os.path.isfile(sps.normExpUserPath(os.path.join(repo, '.hg', 'hgrc'))) + + +def patchHgRepoUsingMq(patchFile, workingDir=None): # pylint: disable=invalid-name,missing-docstring,missing-return-doc + # pylint: disable=missing-return-type-doc + workingDir = workingDir or ( + os.getcwdu() if sys.version_info.major == 2 else os.getcwd()) # pylint: disable=no-member + # We may have passed in the patch with or without the full directory. + patch_abs_path = os.path.abspath(sps.normExpUserPath(patchFile)) + pname = os.path.basename(patch_abs_path) + assert pname != '' + qimport_output, qimport_return_code = sps.captureStdout(['hg', '-R', workingDir, 'qimport', patch_abs_path], + combineStderr=True, ignoreStderr=True, + ignoreExitCode=True) + if qimport_return_code != 0: + if 'already exists' in qimport_output: + print("A patch with the same name has already been qpush'ed. Please qremove it first.") + raise Exception('Return code from `hg qimport` is: ' + str(qimport_return_code)) + + print("Patch qimport'ed...", end=" ") + + qpush_output, qpush_return_code = sps.captureStdout(['hg', '-R', workingDir, 'qpush', pname], + combineStderr=True, ignoreStderr=True) + assert ' is empty' not in qpush_output, "Patch to be qpush'ed should not be empty." + + if qpush_return_code != 0: + hgQpopQrmAppliedPatch(patchFile, workingDir) + print("You may have untracked .rej or .orig files in the repository.") + print("`hg status` output of the repository of interesting files in %s :" % workingDir) + subprocess.check_call(['hg', '-R', workingDir, 'status', '--modified', '--added', + '--removed', '--deleted']) + raise Exception('Return code from `hg qpush` is: ' + str(qpush_return_code)) + + print("Patch qpush'ed. Continuing...", end=" ") + return pname + + +def hgQpopQrmAppliedPatch(patchFile, repoDir): # pylint: disable=invalid-name,missing-param-doc,missing-raises-doc + # pylint: disable=missing-type-doc + """Remove applied patch using `hg qpop` and `hg qdelete`.""" + qpop_output, qpop_return_code = sps.captureStdout(['hg', '-R', repoDir, 'qpop'], + combineStderr=True, ignoreStderr=True, + ignoreExitCode=True) + if qpop_return_code != 0: + print("`hg qpop` output is: " % qpop_output) + raise Exception('Return code from `hg qpop` is: ' + str(qpop_return_code)) + + print("Patch qpop'ed...", end=" ") + subprocess.check_call(['hg', '-R', repoDir, 'qdelete', os.path.basename(patchFile)]) + print("Patch qdelete'd.") diff --git a/util/linkJS.py b/src/funfuzz/util/link_js.py similarity index 69% rename from util/linkJS.py rename to src/funfuzz/util/link_js.py index aaeaa803a..821aa5812 100644 --- a/util/linkJS.py +++ b/src/funfuzz/util/link_js.py @@ -1,17 +1,19 @@ -#!/usr/bin/env python # coding=utf-8 -# pylint: disable=invalid-name,missing-docstring # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at https://mozilla.org/MPL/2.0/. +"""Functions to concatenate files, with one specially for js files. +""" + from __future__ import absolute_import, print_function import os -def linkJS(target_fn, file_list_fn, source_base, prologue="", module_dirs=None): +def link_js(target_fn, file_list_fn, source_base, prologue="", module_dirs=None): + # pylint: disable=missing-docstring module_dirs = module_dirs or [] with open(target_fn, "wb") as target: target.write(prologue) @@ -21,16 +23,16 @@ def linkJS(target_fn, file_list_fn, source_base, prologue="", module_dirs=None): for source_fn in file_list: source_fn = source_fn.replace("/", os.path.sep).strip() if source_fn and source_fn[0] != "#": - addContents(os.path.join(source_base, source_fn), target) + add_contents(os.path.join(source_base, source_fn), target) # Add all *.js files in module_dirs for module_base in module_dirs: for module_fn in os.listdir(module_base): if module_fn.endswith(".js"): - addContents(os.path.join(module_base, module_fn), target) + add_contents(os.path.join(module_base, module_fn), target) -def addContents(source_fn, target): +def add_contents(source_fn, target): # pylint: disable=missing-docstring target.write("\n\n// " + source_fn + "\n\n") with open(source_fn) as source: for line in source: diff --git a/util/lithOps.py b/src/funfuzz/util/lithium_helpers.py similarity index 71% rename from util/lithOps.py rename to src/funfuzz/util/lithium_helpers.py index 134cc8ce8..1c68c88db 100644 --- a/util/lithOps.py +++ b/src/funfuzz/util/lithium_helpers.py @@ -1,11 +1,12 @@ -#!/usr/bin/env python # coding=utf-8 -# pylint: disable=fixme,import-error,invalid-name,missing-docstring # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at https://mozilla.org/MPL/2.0/. +"""Helper functions to use the Lithium reducer. +""" + from __future__ import absolute_import, print_function import os @@ -14,28 +15,29 @@ import subprocess import tempfile -import subprocesses as sps +from . import subprocesses as sps -runlithiumpy = [sys.executable, "-u", "-m", "lithium"] +runlithiumpy = [sys.executable, "-u", "-m", "lithium"] # pylint: disable=invalid-name # Status returns for runLithium and many_timed_runs (HAPPY, NO_REPRO_AT_ALL, NO_REPRO_EXCEPT_BY_URL, LITH_NO_REPRO, LITH_FINISHED, LITH_RETESTED_STILL_INTERESTING, LITH_PLEASE_CONTINUE, LITH_BUSTED) = range(8) -def runLithium(lithArgs, logPrefix, targetTime): +def runLithium(lithArgs, logPrefix, targetTime): # pylint: disable=invalid-name,missing-param-doc,missing-return-doc + # pylint: disable=missing-return-type-doc,missing-type-doc """Run Lithium as a subprocess: reduce to the smallest file that has at least the same unhappiness level. Returns a tuple of (lithlogfn, LITH_*, details). """ - deletableLithTemp = None + deletableLithTemp = None # pylint: disable=invalid-name if targetTime: - # FIXME: this could be based on whether bot.py has a remoteHost - # loopjsfunfuzz.py is being used by bot.py - deletableLithTemp = tempfile.mkdtemp(prefix="fuzzbot-lithium") + # FIXME: this could be based on whether bot has a remoteHost # pylint: disable=fixme + # loop is being used by bot + deletableLithTemp = tempfile.mkdtemp(prefix="fuzzbot-lithium") # pylint: disable=invalid-name lithArgs = ["--maxruntime=" + str(targetTime), "--tempdir=" + deletableLithTemp] + lithArgs else: - # loopjsfunfuzz.py is being run standalone + # loop is being run standalone lithtmp = logPrefix + "-lith-tmp" os.mkdir(lithtmp) lithArgs = ["--tempdir=" + lithtmp] + lithArgs @@ -46,12 +48,13 @@ def runLithium(lithArgs, logPrefix, targetTime): print("Done running Lithium") if deletableLithTemp: shutil.rmtree(deletableLithTemp) - r = readLithiumResult(lithlogfn) + r = readLithiumResult(lithlogfn) # pylint: disable=invalid-name subprocess.call(["gzip", "-f", lithlogfn]) return r -def readLithiumResult(lithlogfn): +def readLithiumResult(lithlogfn): # pylint: disable=invalid-name,missing-docstring,missing-return-doc + # pylint: disable=missing-return-type-doc with open(lithlogfn) as f: for line in f: if line.startswith("Lithium result"): @@ -59,18 +62,21 @@ def readLithiumResult(lithlogfn): if line.startswith("Lithium result: interesting"): return (LITH_RETESTED_STILL_INTERESTING, None) elif line.startswith("Lithium result: succeeded, reduced to: "): + # pylint: disable=invalid-name reducedTo = line[len("Lithium result: succeeded, reduced to: "):].rstrip() # e.g. "4 lines" return (LITH_FINISHED, reducedTo) elif (line.startswith("Lithium result: not interesting") or line.startswith("Lithium result: the original testcase is not")): return (LITH_NO_REPRO, None) elif line.startswith("Lithium result: please continue using: "): + # pylint: disable=invalid-name lithiumHint = line[len("Lithium result: please continue using: "):].rstrip() return (LITH_PLEASE_CONTINUE, lithiumHint) return (LITH_BUSTED, None) -def ddsize(fn): +def ddsize(fn): # pylint: disable=invalid-name,missing-param-doc,missing-return-doc + # pylint: disable=missing-return-type-doc,missing-type-doc """Count the number of chars between DDBEGIN and DDEND in a file.""" count = 0 with open(fn) as f: diff --git a/util/reposUpdate.py b/src/funfuzz/util/repos_update.py old mode 100755 new mode 100644 similarity index 55% rename from util/reposUpdate.py rename to src/funfuzz/util/repos_update.py index aa62df359..71721c263 --- a/util/reposUpdate.py +++ b/src/funfuzz/util/repos_update.py @@ -1,37 +1,36 @@ -#!/usr/bin/env python # coding=utf-8 -# pylint: disable=import-error,invalid-name,missing-docstring # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at https://mozilla.org/MPL/2.0/. -# To update specified repositories to default tip and provide a short list of latest checkins. -# Only supports hg (Mercurial) for now. -# -# Assumes that the repositories are located in ../../trees/*. +"""To update specified repositories to default tip and provide a short list of latest checkins. +Only supports hg (Mercurial) for now. + +Assumes that the repositories are located in ../../trees/*. +""" from __future__ import absolute_import, print_function from copy import deepcopy import logging import os +import platform import time -import subprocesses as sps +import pip +from . import subprocesses as sps logging.basicConfig(level=logging.INFO) -logger = logging.getLogger(__name__) - -THIS_SCRIPT_DIRECTORY = os.path.dirname(os.path.abspath(__file__)) -REPO_PARENT_PATH = os.path.abspath(os.path.join(THIS_SCRIPT_DIRECTORY, os.pardir, os.pardir)) +logger = logging.getLogger(__name__) # pylint: disable=invalid-name # Add your repository here. Note that Valgrind does not have a hg repository. REPOS = ['gecko-dev', 'octo'] + \ ['mozilla-' + x for x in ['inbound', 'central', 'beta', 'release']] if sps.isWin: + # pylint: disable=invalid-name git_64bit_path = os.path.normpath(os.path.join(os.getenv('PROGRAMFILES'), 'Git', 'bin', 'git.exe')) git_32bit_path = os.path.normpath(os.path.join(os.getenv('PROGRAMFILES(X86)'), 'Git', 'bin', 'git.exe')) if os.path.isfile(git_64bit_path): @@ -44,27 +43,40 @@ GITBINARY = 'git' -def typeOfRepo(r): +def typeOfRepo(r): # pylint: disable=invalid-name,missing-param-doc,missing-raises-doc,missing-return-doc + # pylint: disable=missing-return-type-doc,missing-type-doc """Return the type of repository.""" - repoList = [] - repoList.append('.hg') - repoList.append('.git') - for rtype in repoList: + repo_types = [] + repo_types.append('.hg') + repo_types.append('.git') + for rtype in repo_types: if os.path.isdir(os.path.join(r, rtype)): return rtype[1:] raise Exception('Type of repository located at ' + r + ' cannot be determined.') -def updateRepo(repo): +def update_funfuzz(): + """Updates the funfuzz repository.""" + # funfuzz repository assumed to be located at ~/funfuzz + funfuzz_dir = sps.normExpUserPath(os.path.join("~", "funfuzz")) + assert os.path.isdir(funfuzz_dir) + assert typeOfRepo(funfuzz_dir) == "git" + if pip.main(["install", "--upgrade", funfuzz_dir]) and platform.system() == "Linux": + logger.info('\npip errored out, retrying with "--user"\n') + pip.main(["install", "--user", "--upgrade", funfuzz_dir]) + + +def updateRepo(repo): # pylint: disable=invalid-name,missing-param-doc,missing-raises-doc,missing-return-doc + # pylint: disable=missing-return-type-doc,missing-type-doc """Update a repository. Return False if missing; return True if successful; raise an exception if updating fails.""" assert os.path.isdir(repo) - repoType = typeOfRepo(repo) + repo_type = typeOfRepo(repo) - if repoType == 'hg': + if repo_type == 'hg': sps.timeSubprocess(['hg', 'pull', '-u'], ignoreStderr=True, combineStderr=True, cwd=repo, vb=True) sps.timeSubprocess(['hg', 'log', '-r', 'default'], cwd=repo, vb=True) - elif repoType == 'git': + elif repo_type == 'git': # Ignore exit codes so the loop can continue retrying up to number of counts. gitenv = deepcopy(os.environ) if sps.isWin: @@ -72,32 +84,34 @@ def updateRepo(repo): sps.timeSubprocess([GITBINARY, 'pull', '--rebase'], env=gitenv, ignoreStderr=True, combineStderr=True, ignoreExitCode=True, cwd=repo, vb=True) else: - raise Exception('Unknown repository type: ' + repoType) + raise Exception('Unknown repository type: ' + repo_type) return True -def updateRepos(): +def updateRepos(): # pylint: disable=invalid-name """Update Mercurial and Git repositories located in ~ and ~/trees .""" + home_dir = sps.normExpUserPath("~") trees = [ - os.path.normpath(os.path.join(REPO_PARENT_PATH)), - os.path.normpath(os.path.join(REPO_PARENT_PATH, 'trees')) + os.path.normpath(os.path.join(home_dir)), + os.path.normpath(os.path.join(home_dir, 'trees')) ] for tree in trees: for name in sorted(os.listdir(tree)): - namePath = os.path.join(tree, name) - if os.path.isdir(namePath) and (name in REPOS or name.startswith("funfuzz")): + name_path = os.path.join(tree, name) + if os.path.isdir(name_path) and (name in REPOS or (name.startswith("funfuzz") and "-" in name)): print("Updating %s ..." % name) - updateRepo(namePath) + updateRepo(name_path) -def main(): +def main(): # pylint: disable=missing-docstring logger.info(time.asctime()) try: + update_funfuzz() updateRepos() - except OSError as e: + except OSError as ex: print("WARNING: OSError hit:") - print(e) + print(ex) logger.info(time.asctime()) diff --git a/util/s3cache.py b/src/funfuzz/util/s3cache.py similarity index 68% rename from util/s3cache.py rename to src/funfuzz/util/s3cache.py index 5d17960f8..a9514fec4 100644 --- a/util/s3cache.py +++ b/src/funfuzz/util/s3cache.py @@ -1,35 +1,32 @@ -#!/usr/bin/env python # coding=utf-8 -# pylint: disable=import-error,invalid-name,missing-docstring,wrong-import-position # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at https://mozilla.org/MPL/2.0/. +"""Functions here interact with Amazon EC2 using boto. +""" + from __future__ import absolute_import, print_function import os import shutil -import sys -path0 = os.path.dirname(os.path.abspath(__file__)) -path1 = os.path.abspath(os.path.join(path0, os.pardir, 'util')) -sys.path.append(path1) -import subprocesses as sps +from . import subprocesses as sps -isBoto = False +isBoto = False # pylint: disable=invalid-name # We need to first install boto into MozillaBuild via psbootstrap on Windows if not sps.isMac: try: from boto.s3.connection import S3Connection, Key import boto.exception import boto.utils # Cannot find this if only boto is imported - isBoto = True + isBoto = True # pylint: disable=invalid-name except ImportError: - isBoto = False + isBoto = False # pylint: disable=invalid-name -def isEC2VM(): +def isEC2VM(): # pylint: disable=invalid-name,missing-return-doc,missing-return-type-doc """Test to see if the specified S3 cache is available.""" if sps.isMac or not isBoto: return False @@ -40,17 +37,17 @@ def isEC2VM(): return False -class S3Cache(object): +class S3Cache(object): # pylint: disable=missing-docstring def __init__(self, bucket_name): self.bucket = None self.bucket_name = bucket_name - def connect(self): + def connect(self): # pylint: disable=missing-return-doc,missing-return-type-doc """Connect to the S3 bucket.""" if not isBoto: return False - EC2_PROFILE = None if isEC2VM() else 'laniakea' + EC2_PROFILE = None if isEC2VM() else 'laniakea' # pylint: disable=invalid-name try: conn = S3Connection(profile_name=EC2_PROFILE) self.bucket = conn.get_bucket(self.bucket_name) @@ -62,7 +59,8 @@ def connect(self): print('Unable to connect to the following bucket "%s", please check your credentials.' % self.bucket_name) return False - def downloadFile(self, origin, dest): + def downloadFile(self, origin, dest): # pylint: disable=invalid-name,missing-param-doc,missing-return-doc + # pylint: disable=missing-return-type-doc,missing-type-doc """Download files from S3.""" key = self.bucket.get_key(origin) if key is not None: @@ -71,15 +69,17 @@ def downloadFile(self, origin, dest): return True return False - def compressAndUploadDirTarball(self, directory, tarball_path): + def compressAndUploadDirTarball(self, directory, tarball_path): # pylint: disable=invalid-name,missing-param-doc + # pylint: disable=missing-type-doc """Compress a directory into a bz2 tarball and upload it to S3.""" print("Creating archive...") shutil.make_archive(directory, 'bztar', directory) self.uploadFileToS3(tarball_path) - def uploadFileToS3(self, filename): + def uploadFileToS3(self, filename): # pylint: disable=invalid-name,missing-param-doc,missing-type-doc """Upload file to S3.""" - destDir = '' # Root folder of the S3 bucket + # Root folder of the S3 bucket + destDir = '' # pylint: disable=invalid-name destpath = os.path.join(destDir, os.path.basename(filename)) print("Uploading %s to Amazon S3 bucket %s" % (filename, self.bucket_name)) @@ -87,11 +87,12 @@ def uploadFileToS3(self, filename): k.key = destpath k.set_contents_from_filename(filename, reduced_redundancy=True) - def uploadStrToS3(self, destDir, filename, contents): + def uploadStrToS3(self, destDir, filename, contents): # pylint: disable=invalid-name,missing-param-doc + # pylint: disable=missing-type-doc """Upload a string to an S3 file.""" print("Uploading %s to Amazon S3 bucket %s" % (filename, self.bucket_name)) - k2 = Key(self.bucket) + k2 = Key(self.bucket) # pylint: disable=invalid-name k2.key = os.path.join(destDir, filename) k2.set_contents_from_string(contents, reduced_redundancy=True) print() # This newline is needed to get the path of the compiled binary printed on a newline. diff --git a/util/subprocesses.py b/src/funfuzz/util/subprocesses.py similarity index 78% rename from util/subprocesses.py rename to src/funfuzz/util/subprocesses.py index b25b59e48..5ad1586c5 100644 --- a/util/subprocesses.py +++ b/src/funfuzz/util/subprocesses.py @@ -1,13 +1,12 @@ -#!/usr/bin/env python # coding=utf-8 -# pylint: disable=consider-using-enumerate,invalid-name,missing-docstring -# pylint: disable=too-few-public-methods,too-many-arguments,too-many-branches -# pylint: disable=too-many-statements # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at https://mozilla.org/MPL/2.0/. +"""Miscellaneous helper functions. +""" + from __future__ import absolute_import, print_function import ctypes @@ -16,30 +15,34 @@ import platform import re import shutil -import stat +import stat # Fixed after pylint 1.7.2 was released pylint: disable=bad-python3-import import subprocess import sys import time -verbose = False +verbose = False # pylint: disable=invalid-name -isARMv7l = (platform.uname()[4] == 'armv7l') -isLinux = (platform.system() == 'Linux') -isMac = (platform.system() == 'Darwin') -isWin = (platform.system() == 'Windows') -isWin10 = isWin and (platform.uname()[2] == '10') -isWin64 = ('PROGRAMFILES(X86)' in os.environ) +isARMv7l = (platform.uname()[4] == 'armv7l') # pylint: disable=invalid-name +isLinux = (platform.system() == 'Linux') # pylint: disable=invalid-name +isMac = (platform.system() == 'Darwin') # pylint: disable=invalid-name +isWin = (platform.system() == 'Windows') # pylint: disable=invalid-name +isWin10 = isWin and (platform.uname()[2] == '10') # pylint: disable=invalid-name +isWin64 = ('PROGRAMFILES(X86)' in os.environ) # pylint: disable=invalid-name # Note that sys.getwindowsversion will be inaccurate from Win8+ onwards: http://stackoverflow.com/q/19128219 -isWinVistaOrHigher = isWin and (sys.getwindowsversion()[0] >= 6) # pylint: disable=no-member -isMozBuild64 = False +isWinVistaOrHigher = isWin and (sys.getwindowsversion()[0] >= 6) # pylint: disable=invalid-name,no-member +isMozBuild64 = False # pylint: disable=invalid-name # This refers to the Win-specific "MozillaBuild" environment in which Python is running, which is # spawned from the MozillaBuild script for 64-bit compilers, e.g. start-msvc10-x64.bat if os.environ.get('MOZ_MSVCBITS'): - isMozBuild64 = isWin and '64' in os.environ['MOZ_MSVCBITS'] # For MozillaBuild 2.0.0 onwards + # For MozillaBuild 2.0.0 onwards + isMozBuild64 = isWin and '64' in os.environ['MOZ_MSVCBITS'] # pylint: disable=invalid-name elif os.environ.get('MOZ_TOOLS'): - isMozBuild64 = (os.name == 'nt') and ('x64' in os.environ['MOZ_TOOLS'].split(os.sep)[-1]) # For MozillaBuild 1.x + # For MozillaBuild 1.x + # pylint: disable=invalid-name + isMozBuild64 = (os.name == 'nt') and ('x64' in os.environ['MOZ_TOOLS'].split(os.sep)[-1]) # else do not set; the script is running stand-alone and the isMozBuild64 variable should not be needed. +# pylint: disable=invalid-name noMinidumpMsg = r""" WARNING: Minidumps are not being generated, so all crashes will be uninteresting. WARNING: Make sure the following key value exists in this key: @@ -53,33 +56,35 @@ ######################## -def macVer(): +def macVer(): # pylint: disable=invalid-name,missing-return-doc,missing-return-type-doc """If system is a Mac, return the mac type.""" assert platform.system() == 'Darwin' return [int(x) for x in platform.mac_ver()[0].split('.')] -def getFreeSpace(folder, mulVar): +def getFreeSpace(folder, mulVar): # pylint: disable=invalid-name,missing-param-doc,missing-return-doc + # pylint: disable=missing-return-type-doc,missing-type-doc """Return folder/drive free space in bytes if mulVar is 0. Adapted from http://stackoverflow.com/a/2372171 .""" assert mulVar >= 0 if platform.system() == 'Windows': free_bytes = ctypes.c_ulonglong(0) ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(folder), None, None, ctypes.pointer(free_bytes)) - retVal = float(free_bytes.value) + return_value = float(free_bytes.value) else: # os.statvfs is Unix-only - retVal = float(os.statvfs(folder).f_bfree * os.statvfs(folder).f_frsize) # pylint: disable=no-member + return_value = float(os.statvfs(folder).f_bfree * os.statvfs(folder).f_frsize) # pylint: disable=no-member - return retVal // (1024 ** mulVar) + return return_value // (1024 ** mulVar) ##################### # Shell Functions # ##################### - -def captureStdout(inputCmd, ignoreStderr=False, combineStderr=False, ignoreExitCode=False, - currWorkingDir=None, env='NOTSET', verbosity=False): +# pylint: disable=invalid-name,missing-param-doc,missing-raises-doc,missing-return-doc,missing-return-type-doc +# pylint: disable=missing-type-doc,too-complex,too-many-arguments,too-many-branches,too-many-statements +def captureStdout(inputCmd, ignoreStderr=False, combineStderr=False, ignoreExitCode=False, currWorkingDir=None, + env='NOTSET', verbosity=False): """Capture standard output, return the output as a string, along with the return value.""" currWorkingDir = currWorkingDir or ( os.getcwdu() if sys.version_info.major == 2 else os.getcwd()) # pylint: disable=no-member @@ -157,7 +162,8 @@ def captureStdout(inputCmd, ignoreStderr=False, combineStderr=False, ignoreExitC return stdout.rstrip(), p.returncode -def createWtmpDir(tmpDirBase): +def createWtmpDir(tmpDirBase): # pylint: disable=invalid-name,missing-param-doc,missing-return-doc + # pylint: disable=missing-return-type-doc,missing-type-doc """Create wtmp directory, incrementing the number if one is already found.""" i = 1 while True: @@ -178,12 +184,13 @@ def disableCorefile(): resource.setrlimit(resource.RLIMIT_CORE, (0, 0)) -def getCoreLimit(): +def getCoreLimit(): # pylint: disable=invalid-name,missing-docstring,missing-return-doc,missing-return-type-doc import resource # module only available on POSIX pylint: disable=import-error return resource.getrlimit(resource.RLIMIT_CORE) -def grabMacCrashLog(progname, crashedPID, logPrefix, useLogFiles): +def grabMacCrashLog(progname, crashedPID, logPrefix, useLogFiles): # pylint: disable=invalid-name,missing-param-doc + # pylint: disable=missing-return-doc,missing-return-type-doc,missing-type-doc """Find the required crash log in the given crash reporter directory.""" assert platform.system() == 'Darwin' and macVer() >= [10, 6] reportDirList = [os.path.expanduser('~'), '/'] @@ -222,7 +229,7 @@ def grabMacCrashLog(progname, crashedPID, logPrefix, useLogFiles): return fullfn # return open(fullfn).read() - except (OSError, IOError): + except (OSError, IOError): # pylint: disable=overlapping-except # Maybe the log was rotated out between when we got the list # of files and when we tried to open this file. If so, it's # clearly not The One. @@ -230,7 +237,8 @@ def grabMacCrashLog(progname, crashedPID, logPrefix, useLogFiles): return None -def grabCrashLog(progfullname, crashedPID, logPrefix, wantStack): +def grabCrashLog(progfullname, crashedPID, logPrefix, wantStack): # pylint: disable=invalid-name,missing-param-doc + # pylint: disable=missing-return-doc,missing-return-type-doc,missing-type-doc,too-complex,too-many-branches """Return the crash log if found.""" progname = os.path.basename(progfullname) @@ -303,7 +311,8 @@ def grabCrashLog(progfullname, crashedPID, logPrefix, wantStack): "You can increase it with 'ulimit -c' in bash." % getCoreLimit()[0]) -def constructCdbCommand(progfullname, crashedPID): +def constructCdbCommand(progfullname, crashedPID): # pylint: disable=invalid-name,missing-param-doc,missing-return-doc + # pylint: disable=missing-return-type-doc,missing-type-doc """Construct a command that uses the Windows debugger (cdb.exe) to turn a minidump file into a stack trace.""" # On Windows Vista and above, look for a minidump. dumpFilename = normExpUserPath(os.path.join( @@ -325,7 +334,7 @@ def constructCdbCommand(progfullname, crashedPID): maxLoops = 300 while True: if os.path.exists(dumpFilename): - debuggerCmdPath = getAbsPathForAdjacentFile('cdbCmds.txt') + debuggerCmdPath = getAbsPathForAdjacentFile('cdb_cmds.txt') assert os.path.exists(debuggerCmdPath) cdbCmdList = [] @@ -344,7 +353,8 @@ def constructCdbCommand(progfullname, crashedPID): return None -def isWinDumpingToDefaultLocation(): +def isWinDumpingToDefaultLocation(): # pylint: disable=invalid-name,missing-return-doc,missing-return-type-doc + # pylint: disable=too-complex,too-many-branches """Check whether Windows minidumps are enabled and set to go to Windows' default location.""" if sys.version_info.major == 2: import _winreg as winreg # pylint: disable=import-error @@ -401,80 +411,84 @@ def isWinDumpingToDefaultLocation(): raise -def constructGdbCommand(progfullname, crashedPID): +def constructGdbCommand(progfullname, crashedPID): # pylint: disable=invalid-name,missing-param-doc,missing-return-doc + # pylint: disable=missing-return-type-doc,missing-type-doc """Construct a command that uses the POSIX debugger (gdb) to turn a minidump file into a stack trace.""" # On Mac and Linux, look for a core file. - coreFilename = None + core_name = None if isMac: # Core files will be generated if you do: # mkdir -p /cores/ # ulimit -c 2147483648 (or call resource.setrlimit from a preexec_fn hook) - coreFilename = "/cores/core." + str(crashedPID) + core_name = "/cores/core." + str(crashedPID) elif isLinux: - isPidUsed = False + is_pid_used = False if os.path.exists('/proc/sys/kernel/core_uses_pid'): with open('/proc/sys/kernel/core_uses_pid') as f: - isPidUsed = bool(int(f.read()[0])) # Setting [0] turns the input to a str. - coreFilename = 'core.' + str(crashedPID) if isPidUsed else 'core' # relative path - if not os.path.isfile(coreFilename): - coreFilename = normExpUserPath(os.path.join('~', coreFilename)) # try the home dir + is_pid_used = bool(int(f.read()[0])) # Setting [0] turns the input to a str. + core_name = 'core.' + str(crashedPID) if is_pid_used else 'core' # relative path + if not os.path.isfile(core_name): + core_name = normExpUserPath(os.path.join('~', core_name)) # try the home dir - if coreFilename and os.path.exists(coreFilename): - debuggerCmdPath = getAbsPathForAdjacentFile('gdb-quick.txt') + if core_name and os.path.exists(core_name): + debuggerCmdPath = getAbsPathForAdjacentFile('gdb_cmds.txt') # pylint: disable=invalid-name assert os.path.exists(debuggerCmdPath) # Run gdb and move the core file. Tip: gdb gives more info for: # (debug with intact build dir > debug > opt with frame pointers > opt) - return ["gdb", "-n", "-batch", "-x", debuggerCmdPath, progfullname, coreFilename] + return ["gdb", "-n", "-batch", "-x", debuggerCmdPath, progfullname, core_name] return None -def getAbsPathForAdjacentFile(filename): +def getAbsPathForAdjacentFile(filename): # pylint: disable=invalid-name,missing-param-doc,missing-return-doc + # pylint: disable=missing-return-type-doc,missing-type-doc """Get the absolute path of a particular file, given its base directory and filename.""" return os.path.join(os.path.dirname(os.path.abspath(__file__)), filename) -def isProgramInstalled(program): +def isProgramInstalled(program): # pylint: disable=invalid-name,missing-param-doc,missing-return-doc + # pylint: disable=missing-return-type-doc,missing-type-doc """Check if the specified program is installed.""" - whichExit = captureStdout(['which', program], ignoreStderr=True, combineStderr=True, ignoreExitCode=True)[1] - return whichExit == 0 + which_exit = captureStdout(['which', program], ignoreStderr=True, combineStderr=True, ignoreExitCode=True)[1] + return which_exit == 0 -def rmDirIfEmpty(eDir): +def rmDirIfEmpty(eDir): # pylint: disable=invalid-name,missing-param-doc,missing-type-doc """Remove directory if empty.""" assert os.path.isdir(eDir) if not os.listdir(eDir): os.rmdir(eDir) -def rmTreeIfExists(dirTree): +def rmTreeIfExists(dirTree): # pylint: disable=invalid-name,missing-param-doc,missing-type-doc """Remove a directory with all sub-directories and files if the directory exists.""" if os.path.isdir(dirTree): rmTreeIncludingReadOnly(dirTree) assert not os.path.isdir(dirTree) -def rmTreeIncludingReadOnly(dirTree): +def rmTreeIncludingReadOnly(dirTree): # pylint: disable=invalid-name,missing-docstring shutil.rmtree(dirTree, onerror=handleRemoveReadOnly) -def test_rmTreeIncludingReadOnly(): - """Run this function in the same directory as subprocesses.py to test.""" - testDir = 'test_rmTreeIncludingReadOnly' - os.mkdir(testDir) - readOnlyDir = os.path.join(testDir, 'nestedReadOnlyDir') - os.mkdir(readOnlyDir) - filename = os.path.join(readOnlyDir, 'test.txt') +def test_rmTreeIncludingReadOnly(): # pylint: disable=invalid-name + """Run this function in the same directory as subprocesses to test.""" + test_dir = 'test_rmTreeIncludingReadOnly' + os.mkdir(test_dir) + read_only_dir = os.path.join(test_dir, 'nestedReadOnlyDir') + os.mkdir(read_only_dir) + filename = os.path.join(read_only_dir, 'test.txt') with open(filename, 'wb') as f: f.write('testing\n') os.chmod(filename, stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH) - os.chmod(readOnlyDir, stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH) + os.chmod(read_only_dir, stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH) - rmTreeIncludingReadOnly(testDir) # Should pass here + rmTreeIncludingReadOnly(test_dir) # Should pass here -def handleRemoveReadOnly(func, path, exc): +def handleRemoveReadOnly(func, path, exc): # pylint: disable=invalid-name,missing-param-doc,missing-raises-doc + # pylint: disable=missing-type-doc """Handle read-only files. Adapted from http://stackoverflow.com/q/1213706 .""" if func in (os.rmdir, os.remove) and exc[1].errno == errno.EACCES: if os.name == 'posix': @@ -489,16 +503,16 @@ def handleRemoveReadOnly(func, path, exc): raise OSError("Unable to handle read-only files.") -def normExpUserPath(p): +def normExpUserPath(p): # pylint: disable=invalid-name,missing-docstring,missing-return-doc,missing-return-type-doc return os.path.normpath(os.path.expanduser(p)) -def shellify(cmd): +def shellify(cmd): # pylint: disable=missing-param-doc,missing-return-doc,missing-return-type-doc,missing-type-doc """Try to convert an arguments array to an equivalent string that can be pasted into a shell.""" - okUnquotedRE = re.compile(r"""^[a-zA-Z0-9\-\_\.\,\/\=\~@\+]*$""") - okQuotedRE = re.compile(r"""^[a-zA-Z0-9\-\_\.\,\/\=\~@\{\}\|\(\)\+ ]*$""") + okUnquotedRE = re.compile(r"""^[a-zA-Z0-9\-\_\.\,\/\=\~@\+]*$""") # pylint: disable=invalid-name + okQuotedRE = re.compile(r"""^[a-zA-Z0-9\-\_\.\,\/\=\~@\{\}\|\(\)\+ ]*$""") # pylint: disable=invalid-name ssc = [] - for i in range(len(cmd)): + for i in range(len(cmd)): # pylint: disable=consider-using-enumerate item = cmd[i] if okUnquotedRE.match(item): ssc.append(item) @@ -512,6 +526,8 @@ def shellify(cmd): def timeSubprocess(command, ignoreStderr=False, combineStderr=False, ignoreExitCode=False, cwd=None, env=None, vb=False): + # pylint: disable=invalid-name,missing-param-doc,missing-return-doc,missing-return-type-doc + # pylint: disable=missing-type-doc,too-many-arguments """Calculate how long a captureStdout command takes and prints it. Return the stdout and return value that captureStdout passes on. @@ -530,27 +546,28 @@ def timeSubprocess(command, ignoreStderr=False, combineStderr=False, ignoreExitC return stdOutput, retVal -class Unbuffered(object): +class Unbuffered(object): # pylint: disable=missing-param-doc,missing-type-doc,too-few-public-methods """From http://stackoverflow.com/a/107717 - Unbuffered stdout by default, similar to -u.""" def __init__(self, stream): self.stream = stream - def write(self, data): + def write(self, data): # pylint: disable=missing-docstring self.stream.write(data) self.stream.flush() - def __getattr__(self, attr): + def __getattr__(self, attr): # pylint: disable=missing-return-doc,missing-return-type-doc return getattr(self.stream, attr) -def vdump(inp): +def vdump(inp): # pylint: disable=missing-param-doc,missing-type-doc """Append the word 'DEBUG' to any verbose output.""" if verbose: print("DEBUG - %s" % inp) -def verCheck(prog): +def verCheck(prog): # pylint: disable=invalid-name,missing-param-doc,missing-return-doc,missing-return-type-doc + # pylint: disable=missing-type-doc """Runs the program with --version and returns the result.""" return subprocess.check_output([prog, '--version']) diff --git a/util/tooltool/README b/src/funfuzz/util/tooltool/README similarity index 76% rename from util/tooltool/README rename to src/funfuzz/util/tooltool/README index 463470fd9..c09030517 100644 --- a/util/tooltool/README +++ b/src/funfuzz/util/tooltool/README @@ -1,4 +1,4 @@ -This stuff helps downloadBuild.py find minidump_stackwalk binaries. It's similar to what Mozilla's regression test automation uses, but with slightly less automatic updating. +This stuff helps download_build.py find minidump_stackwalk binaries. It's similar to what Mozilla's regression test automation uses, but with slightly less automatic updating. tooltool.py is from: diff --git a/src/funfuzz/util/tooltool/__init__.py b/src/funfuzz/util/tooltool/__init__.py new file mode 100644 index 000000000..5267cdead --- /dev/null +++ b/src/funfuzz/util/tooltool/__init__.py @@ -0,0 +1,11 @@ +# coding=utf-8 +# flake8: noqa +# pylint: disable=missing-docstring +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import + +from . import tooltool diff --git a/util/tooltool/linux32.manifest b/src/funfuzz/util/tooltool/linux32.manifest similarity index 100% rename from util/tooltool/linux32.manifest rename to src/funfuzz/util/tooltool/linux32.manifest diff --git a/util/tooltool/linux64.manifest b/src/funfuzz/util/tooltool/linux64.manifest similarity index 100% rename from util/tooltool/linux64.manifest rename to src/funfuzz/util/tooltool/linux64.manifest diff --git a/util/tooltool/macosx64.manifest b/src/funfuzz/util/tooltool/macosx64.manifest similarity index 100% rename from util/tooltool/macosx64.manifest rename to src/funfuzz/util/tooltool/macosx64.manifest diff --git a/util/tooltool/tooltool.py b/src/funfuzz/util/tooltool/tooltool.py old mode 100755 new mode 100644 similarity index 98% rename from util/tooltool/tooltool.py rename to src/funfuzz/util/tooltool/tooltool.py index 13393ed4c..1a71f74d0 --- a/util/tooltool/tooltool.py +++ b/src/funfuzz/util/tooltool/tooltool.py @@ -1,6 +1,8 @@ #!/usr/bin/env python # coding=utf-8 -# pylint: disable=assignment-from-no-return,broad-except,fixme,invalid-name,logging-too-few-args,missing-docstring +# pylint: disable=assignment-from-no-return,bad-python3-import,broad-except,fixme,invalid-name,logging-too-few-args +# pylint: disable=missing-docstring,missing-param-doc,missing-raises-doc,missing-return-doc,missing-return-type-doc +# pylint: disable=missing-type-doc,overlapping-except,too-complex # pylint: disable=too-many-arguments,too-many-branches,too-many-locals,too-many-lines,too-many-return-statements # pylint: disable=too-many-statements @@ -84,7 +86,7 @@ class MissingFileException(ExceptionWithFilename): pass -class FileRecord(object): +class FileRecord(object): # pylint: disable=eq-without-hash def __init__(self, filename, size, digest, algorithm, unpack=False, visibility=None, setup=None): @@ -245,7 +247,7 @@ def decode(self, s): # pylint:disable=arguments-differ return rv -class Manifest(object): +class Manifest(object): # pylint: disable=eq-without-hash valid_formats = ('json',) diff --git a/util/tooltool/win32.manifest b/src/funfuzz/util/tooltool/win32.manifest similarity index 100% rename from util/tooltool/win32.manifest rename to src/funfuzz/util/tooltool/win32.manifest diff --git a/util/__init__.py b/util/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/util/hgCmds.py b/util/hgCmds.py deleted file mode 100644 index 9c7679aa1..000000000 --- a/util/hgCmds.py +++ /dev/null @@ -1,174 +0,0 @@ -#!/usr/bin/env python -# coding=utf-8 -# pylint: disable=import-error,invalid-name,missing-docstring -# -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at https://mozilla.org/MPL/2.0/. - -from __future__ import absolute_import, print_function - -import ConfigParser -import os -import re -import sys -import subprocess - -import subprocesses as sps - - -try: - input = raw_input # pylint: disable=redefined-builtin -except NameError: - pass - - -def destroyPyc(repoDir): - # This is roughly equivalent to ['hg', 'purge', '--all', '--include=**.pyc']) - # but doesn't run into purge's issues (incompatbility with -R, requiring an hg extension) - for root, dirs, files in os.walk(repoDir): - for fn in files: - if fn.endswith(".pyc"): - os.remove(os.path.join(root, fn)) - if '.hg' in dirs: - # Don't visit .hg dir - dirs.remove('.hg') - - -def ensureMqEnabled(): - """Ensure that mq is enabled in the ~/.hgrc file.""" - usrHgrc = os.path.join(os.path.expanduser('~'), '.hgrc') - assert os.path.isfile(usrHgrc) - - usrHgrcCfg = ConfigParser.SafeConfigParser() - usrHgrcCfg.read(usrHgrc) - - try: - usrHgrcCfg.get('extensions', 'mq') - except ConfigParser.NoOptionError: - raise Exception('Please first enable mq in ~/.hgrc by having "mq =" in [extensions].') - - -def findCommonAncestor(repoDir, a, b): - return sps.captureStdout(['hg', '-R', repoDir, 'log', '-r', 'ancestor(' + a + ',' + b + ')', - '--template={node|short}'])[0] - - -def isAncestor(repoDir, a, b): - """Return true iff |a| is an ancestor of |b|. Throw if |a| or |b| does not exist.""" - return sps.captureStdout(['hg', '-R', repoDir, 'log', '-r', a + ' and ancestor(' + a + ',' + b + ')', - '--template={node|short}'])[0] != "" - - -def existsAndIsAncestor(repoDir, a, b): - """Return true iff |a| exists and is an ancestor of |b|.""" - # Takes advantage of "id(badhash)" being the empty set, in contrast to just "badhash", which is an error - out = sps.captureStdout(['hg', '-R', repoDir, 'log', '-r', a + ' and ancestor(' + a + ',' + b + ')', - '--template={node|short}'], combineStderr=True, ignoreExitCode=True)[0] - return out != "" and out.find("abort: unknown revision") < 0 - - -def getCsetHashFromBisectMsg(msg): - # Example bisect msg: "Testing changeset 41831:4f4c01fb42c3 (2 changesets remaining, ~1 tests)" - r = re.compile(r"(^|.* )(\d+):(\w{12}).*") - m = r.match(msg) - if m: - return m.group(3) - - -assert getCsetHashFromBisectMsg("x 12345:abababababab") == "abababababab" -assert getCsetHashFromBisectMsg("x 12345:123412341234") == "123412341234" -assert getCsetHashFromBisectMsg("12345:abababababab y") == "abababababab" - - -def getRepoHashAndId(repoDir, repoRev='parents() and default'): - """Return the repository hash and id, and whether it is on default. - - It will also ask what the user would like to do, should the repository not be on default. - """ - # This returns null if the repository is not on default. - hgLogTmplList = ['hg', '-R', repoDir, 'log', '-r', repoRev, - '--template', '{node|short} {rev}'] - hgIdFull = sps.captureStdout(hgLogTmplList)[0] - onDefault = bool(hgIdFull) - if not onDefault: - updateDefault = input("Not on default tip! " - "Would you like to (a)bort, update to (d)efault, or (u)se this rev: ") - updateDefault = updateDefault.strip() - if updateDefault == 'a': - print("Aborting...") - sys.exit(0) - elif updateDefault == 'd': - subprocess.check_call(['hg', '-R', repoDir, 'update', 'default']) - onDefault = True - elif updateDefault == 'u': - hgLogTmplList = ['hg', '-R', repoDir, 'log', '-r', 'parents()', '--template', - '{node|short} {rev}'] - else: - raise Exception('Invalid choice.') - hgIdFull = sps.captureStdout(hgLogTmplList)[0] - assert hgIdFull != '' - (hgIdChangesetHash, hgIdLocalNum) = hgIdFull.split(' ') - sps.vdump('Finished getting the hash and local id number of the repository.') - return hgIdChangesetHash, hgIdLocalNum, onDefault - - -def getRepoNameFromHgrc(repoDir): - """Look in the hgrc file in the .hg directory of the repository and return the name.""" - assert isRepoValid(repoDir) - hgCfg = ConfigParser.SafeConfigParser() - hgCfg.read(sps.normExpUserPath(os.path.join(repoDir, '.hg', 'hgrc'))) - # Not all default entries in [paths] end with "/". - return [i for i in hgCfg.get('paths', 'default').split('/') if i][-1] - - -def isRepoValid(repo): - """Check that a repository is valid by ensuring that the hgrc file is around.""" - return os.path.isfile(sps.normExpUserPath(os.path.join(repo, '.hg', 'hgrc'))) - - -def patchHgRepoUsingMq(patchFile, workingDir=None): - workingDir = workingDir or ( - os.getcwdu() if sys.version_info.major == 2 else os.getcwd()) # pylint: disable=no-member - # We may have passed in the patch with or without the full directory. - patchAbsPath = os.path.abspath(sps.normExpUserPath(patchFile)) - pname = os.path.basename(patchAbsPath) - assert pname != '' - qimportOutput, qimportRetCode = sps.captureStdout(['hg', '-R', workingDir, 'qimport', patchAbsPath], - combineStderr=True, ignoreStderr=True, - ignoreExitCode=True) - if qimportRetCode != 0: - if 'already exists' in qimportOutput: - print("A patch with the same name has already been qpush'ed. Please qremove it first.") - raise Exception('Return code from `hg qimport` is: ' + str(qimportRetCode)) - - print("Patch qimport'ed...", end=" ") - - qpushOutput, qpushRetCode = sps.captureStdout(['hg', '-R', workingDir, 'qpush', pname], - combineStderr=True, ignoreStderr=True) - assert ' is empty' not in qpushOutput, "Patch to be qpush'ed should not be empty." - - if qpushRetCode != 0: - hgQpopQrmAppliedPatch(patchFile, workingDir) - print("You may have untracked .rej or .orig files in the repository.") - print("`hg status` output of the repository of interesting files in %s :" % workingDir) - subprocess.check_call(['hg', '-R', workingDir, 'status', '--modified', '--added', - '--removed', '--deleted']) - raise Exception('Return code from `hg qpush` is: ' + str(qpushRetCode)) - - print("Patch qpush'ed. Continuing...", end=" ") - return pname - - -def hgQpopQrmAppliedPatch(patchFile, repoDir): - """Remove applied patch using `hg qpop` and `hg qdelete`.""" - qpopOutput, qpopRetCode = sps.captureStdout(['hg', '-R', repoDir, 'qpop'], - combineStderr=True, ignoreStderr=True, - ignoreExitCode=True) - if qpopRetCode != 0: - print("`hg qpop` output is: " % qpopOutput) - raise Exception('Return code from `hg qpop` is: ' + str(qpopRetCode)) - - print("Patch qpop'ed...", end=" ") - subprocess.check_call(['hg', '-R', repoDir, 'qdelete', os.path.basename(patchFile)]) - print("Patch qdelete'd.") diff --git a/util/tooltool/__init__.py b/util/tooltool/__init__.py deleted file mode 100644 index e69de29bb..000000000