diff --git a/Lib/test/test_tools/test_fixcid.py b/Lib/test/test_tools/test_fixcid.py deleted file mode 100644 index a72f74b8b15b34..00000000000000 --- a/Lib/test/test_tools/test_fixcid.py +++ /dev/null @@ -1,94 +0,0 @@ -'''Test Tools/scripts/fixcid.py.''' - -from io import StringIO -import os, os.path -import runpy -import sys -from test import support -from test.support import os_helper -from test.test_tools import skip_if_missing, scriptsdir -import unittest - -skip_if_missing() - -class Test(unittest.TestCase): - def test_parse_strings(self): - old1 = 'int xx = "xx\\"xx"[xx];\n' - old2 = "int xx = 'x\\'xx' + xx;\n" - output = self.run_script(old1 + old2) - new1 = 'int yy = "xx\\"xx"[yy];\n' - new2 = "int yy = 'x\\'xx' + yy;\n" - self.assertMultiLineEqual(output, - "1\n" - "< {old1}" - "> {new1}" - "{new1}" - "2\n" - "< {old2}" - "> {new2}" - "{new2}".format(old1=old1, old2=old2, new1=new1, new2=new2) - ) - - def test_alter_comments(self): - output = self.run_script( - substfile= - "xx yy\n" - "*aa bb\n", - args=("-c", "-",), - input= - "/* xx altered */\n" - "int xx;\n" - "/* aa unaltered */\n" - "int aa;\n", - ) - self.assertMultiLineEqual(output, - "1\n" - "< /* xx altered */\n" - "> /* yy altered */\n" - "/* yy altered */\n" - "2\n" - "< int xx;\n" - "> int yy;\n" - "int yy;\n" - "/* aa unaltered */\n" - "4\n" - "< int aa;\n" - "> int bb;\n" - "int bb;\n" - ) - - def test_directory(self): - os.mkdir(os_helper.TESTFN) - self.addCleanup(os_helper.rmtree, os_helper.TESTFN) - c_filename = os.path.join(os_helper.TESTFN, "file.c") - with open(c_filename, "w", encoding="utf-8") as file: - file.write("int xx;\n") - with open(os.path.join(os_helper.TESTFN, "file.py"), "w", - encoding="utf-8") as file: - file.write("xx = 'unaltered'\n") - script = os.path.join(scriptsdir, "fixcid.py") - output = self.run_script(args=(os_helper.TESTFN,)) - self.assertMultiLineEqual(output, - "{}:\n" - "1\n" - '< int xx;\n' - '> int yy;\n'.format(c_filename) - ) - - def run_script(self, input="", *, args=("-",), substfile="xx yy\n"): - substfilename = os_helper.TESTFN + ".subst" - with open(substfilename, "w", encoding="utf-8") as file: - file.write(substfile) - self.addCleanup(os_helper.unlink, substfilename) - - argv = ["fixcid.py", "-s", substfilename] + list(args) - script = os.path.join(scriptsdir, "fixcid.py") - with support.swap_attr(sys, "argv", argv), \ - support.swap_attr(sys, "stdin", StringIO(input)), \ - support.captured_stdout() as output, \ - support.captured_stderr(): - try: - runpy.run_path(script, run_name="__main__") - except SystemExit as exit: - self.assertEqual(exit.code, 0) - return output.getvalue() diff --git a/Lib/test/test_tools/test_pdeps.py b/Lib/test/test_tools/test_pdeps.py deleted file mode 100644 index a986d10e499d9b..00000000000000 --- a/Lib/test/test_tools/test_pdeps.py +++ /dev/null @@ -1,32 +0,0 @@ -"""Tests for the pdeps script in the Tools directory.""" - -import os -import unittest -import tempfile - -from test.test_tools import skip_if_missing, import_tool - -skip_if_missing() - - -class PdepsTests(unittest.TestCase): - - @classmethod - def setUpClass(self): - self.pdeps = import_tool('pdeps') - - def test_process_errors(self): - # Issue #14492: m_import.match(line) can be None. - with tempfile.TemporaryDirectory() as tmpdir: - fn = os.path.join(tmpdir, 'foo') - with open(fn, 'w', encoding='utf-8') as stream: - stream.write("#!/this/will/fail") - self.pdeps.process(fn, {}) - - def test_inverse_attribute_error(self): - # Issue #14492: this used to fail with an AttributeError. - self.pdeps.inverse({'a': []}) - - -if __name__ == '__main__': - unittest.main() diff --git a/Lib/test/test_tools/test_pindent.py b/Lib/test/test_tools/test_pindent.py deleted file mode 100644 index 61e97fea48cfba..00000000000000 --- a/Lib/test/test_tools/test_pindent.py +++ /dev/null @@ -1,339 +0,0 @@ -"""Tests for the pindent script in the Tools directory.""" - -import os -import sys -import unittest -import subprocess -import textwrap -from test.support import os_helper -from test.support.script_helper import assert_python_ok - -from test.test_tools import scriptsdir, skip_if_missing - -skip_if_missing() - - -class PindentTests(unittest.TestCase): - script = os.path.join(scriptsdir, 'pindent.py') - - def assertFileEqual(self, fn1, fn2): - with open(fn1) as f1, open(fn2) as f2: - self.assertEqual(f1.readlines(), f2.readlines()) - - def pindent(self, source, *args): - with subprocess.Popen( - (sys.executable, self.script) + args, - stdin=subprocess.PIPE, stdout=subprocess.PIPE, - universal_newlines=True) as proc: - out, err = proc.communicate(source) - self.assertIsNone(err) - return out - - def lstriplines(self, data): - return '\n'.join(line.lstrip() for line in data.splitlines()) + '\n' - - def test_selftest(self): - self.maxDiff = None - with os_helper.temp_dir() as directory: - data_path = os.path.join(directory, '_test.py') - with open(self.script, encoding='utf-8') as f: - closed = f.read() - with open(data_path, 'w', encoding='utf-8') as f: - f.write(closed) - - rc, out, err = assert_python_ok(self.script, '-d', data_path) - self.assertEqual(out, b'') - self.assertEqual(err, b'') - backup = data_path + '~' - self.assertTrue(os.path.exists(backup)) - with open(backup, encoding='utf-8') as f: - self.assertEqual(f.read(), closed) - with open(data_path, encoding='utf-8') as f: - clean = f.read() - compile(clean, '_test.py', 'exec') - self.assertEqual(self.pindent(clean, '-c'), closed) - self.assertEqual(self.pindent(closed, '-d'), clean) - - rc, out, err = assert_python_ok(self.script, '-c', data_path) - self.assertEqual(out, b'') - self.assertEqual(err, b'') - with open(backup, encoding='utf-8') as f: - self.assertEqual(f.read(), clean) - with open(data_path, encoding='utf-8') as f: - self.assertEqual(f.read(), closed) - - broken = self.lstriplines(closed) - with open(data_path, 'w', encoding='utf-8') as f: - f.write(broken) - rc, out, err = assert_python_ok(self.script, '-r', data_path) - self.assertEqual(out, b'') - self.assertEqual(err, b'') - with open(backup, encoding='utf-8') as f: - self.assertEqual(f.read(), broken) - with open(data_path, encoding='utf-8') as f: - indented = f.read() - compile(indented, '_test.py', 'exec') - self.assertEqual(self.pindent(broken, '-r'), indented) - - def pindent_test(self, clean, closed): - self.assertEqual(self.pindent(clean, '-c'), closed) - self.assertEqual(self.pindent(closed, '-d'), clean) - broken = self.lstriplines(closed) - self.assertEqual(self.pindent(broken, '-r', '-e', '-s', '4'), closed) - - def test_statements(self): - clean = textwrap.dedent("""\ - if a: - pass - - if a: - pass - else: - pass - - if a: - pass - elif: - pass - else: - pass - - while a: - break - - while a: - break - else: - pass - - for i in a: - break - - for i in a: - break - else: - pass - - try: - pass - finally: - pass - - try: - pass - except TypeError: - pass - except ValueError: - pass - else: - pass - - try: - pass - except TypeError: - pass - except ValueError: - pass - finally: - pass - - with a: - pass - - class A: - pass - - def f(): - pass - """) - - closed = textwrap.dedent("""\ - if a: - pass - # end if - - if a: - pass - else: - pass - # end if - - if a: - pass - elif: - pass - else: - pass - # end if - - while a: - break - # end while - - while a: - break - else: - pass - # end while - - for i in a: - break - # end for - - for i in a: - break - else: - pass - # end for - - try: - pass - finally: - pass - # end try - - try: - pass - except TypeError: - pass - except ValueError: - pass - else: - pass - # end try - - try: - pass - except TypeError: - pass - except ValueError: - pass - finally: - pass - # end try - - with a: - pass - # end with - - class A: - pass - # end class A - - def f(): - pass - # end def f - """) - self.pindent_test(clean, closed) - - def test_multilevel(self): - clean = textwrap.dedent("""\ - def foobar(a, b): - if a == b: - a = a+1 - elif a < b: - b = b-1 - if b > a: a = a-1 - else: - print 'oops!' - """) - closed = textwrap.dedent("""\ - def foobar(a, b): - if a == b: - a = a+1 - elif a < b: - b = b-1 - if b > a: a = a-1 - # end if - else: - print 'oops!' - # end if - # end def foobar - """) - self.pindent_test(clean, closed) - - def test_preserve_indents(self): - clean = textwrap.dedent("""\ - if a: - if b: - pass - """) - closed = textwrap.dedent("""\ - if a: - if b: - pass - # end if - # end if - """) - self.assertEqual(self.pindent(clean, '-c'), closed) - self.assertEqual(self.pindent(closed, '-d'), clean) - broken = self.lstriplines(closed) - self.assertEqual(self.pindent(broken, '-r', '-e', '-s', '9'), closed) - clean = textwrap.dedent("""\ - if a: - \tif b: - \t\tpass - """) - closed = textwrap.dedent("""\ - if a: - \tif b: - \t\tpass - \t# end if - # end if - """) - self.assertEqual(self.pindent(clean, '-c'), closed) - self.assertEqual(self.pindent(closed, '-d'), clean) - broken = self.lstriplines(closed) - self.assertEqual(self.pindent(broken, '-r'), closed) - - def test_escaped_newline(self): - clean = textwrap.dedent("""\ - class\\ - \\ - A: - def\ - \\ - f: - pass - """) - closed = textwrap.dedent("""\ - class\\ - \\ - A: - def\ - \\ - f: - pass - # end def f - # end class A - """) - self.assertEqual(self.pindent(clean, '-c'), closed) - self.assertEqual(self.pindent(closed, '-d'), clean) - - def test_empty_line(self): - clean = textwrap.dedent("""\ - if a: - - pass - """) - closed = textwrap.dedent("""\ - if a: - - pass - # end if - """) - self.pindent_test(clean, closed) - - def test_oneline(self): - clean = textwrap.dedent("""\ - if a: pass - """) - closed = textwrap.dedent("""\ - if a: pass - # end if - """) - self.pindent_test(clean, closed) - - -if __name__ == '__main__': - unittest.main() diff --git a/Misc/NEWS.d/next/Security/2022-09-28-12-10-57.gh-issue-97612.y6NvOQ.rst b/Misc/NEWS.d/next/Security/2022-09-28-12-10-57.gh-issue-97612.y6NvOQ.rst deleted file mode 100644 index 2f113492d42dde..00000000000000 --- a/Misc/NEWS.d/next/Security/2022-09-28-12-10-57.gh-issue-97612.y6NvOQ.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix a shell code injection vulnerability in the ``get-remote-certificate.py`` -example script. The script no longer uses a shell to run ``openssl`` commands. -Issue reported and initial fix by Caleb Shortt. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Tools-Demos/2022-09-30-14-30-12.gh-issue-97669.gvbgcg.rst b/Misc/NEWS.d/next/Tools-Demos/2022-09-30-14-30-12.gh-issue-97669.gvbgcg.rst new file mode 100644 index 00000000000000..ddb7938732fa65 --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2022-09-30-14-30-12.gh-issue-97669.gvbgcg.rst @@ -0,0 +1,2 @@ +Remove outdated example scripts. Copy them from Python 3.11 if you still +need them. Patch by Victor Stinner. diff --git a/PCbuild/lib.pyproj b/PCbuild/lib.pyproj index e8c99f6b246c82..9934dc577bf36a 100644 --- a/PCbuild/lib.pyproj +++ b/PCbuild/lib.pyproj @@ -1334,12 +1334,9 @@ - - - diff --git a/Tools/scripts/README b/Tools/scripts/README index c1d66731ba6495..0637142ce13c69 100644 --- a/Tools/scripts/README +++ b/Tools/scripts/README @@ -1,65 +1,24 @@ This directory contains a collection of executable Python scripts that are -useful while building, extending or managing Python. Some (e.g., dutree or lll) -are also generally useful UNIX tools. +useful while building, extending or managing Python. 2to3 Main script for running the 2to3 conversion tool abitype.py Converts a C file to use the PEP 384 type definition API analyze_dxp.py Analyzes the result of sys.getdxp() -byext.py Print lines/words/chars stats of files by extension -byteyears.py Print product of a file's size and age -cleanfuture.py Fix redundant Python __future__ statements combinerefs.py A helper for analyzing PYTHONDUMPREFS output -copytime.py Copy one file's atime and mtime to another -crlf.py Change CRLF line endings to LF (Windows to Unix) -db2pickle.py Dump a database file to a pickle diff.py Print file diffs in context, unified, or ndiff formats -dutree.py Format du(1) output as a tree sorted by size eptags.py Create Emacs TAGS file for Python modules -finddiv.py A grep-like tool that looks for division operators -findlinksto.py Recursively find symbolic links to a given path prefix -findnocoding.py Find source files which need an encoding declaration -find_recursionlimit.py Find the maximum recursion limit on this machine -find-uname.py Look for the given arguments in the sets of all Unicode names -fixcid.py Massive identifier substitution on C source files -fixdiv.py Tool to fix division operators. -fixheader.py Add some cpp magic to a C include file -fixnotice.py Fix the copyright notice in source files -fixps.py Fix Python scripts' first line (if #!) -ftpmirror.py FTP mirror script -get-remote-certificate.py Fetch the certificate that the server(s) are providing in PEM form -google.py Open a webbrowser with Google gprof2html.py Transform gprof(1) output into useful HTML -highlight.py Python syntax highlighting with HTML output idle3 Main program to start IDLE -ifdef.py Remove #if(n)def groups from C sources -import_diagnostics.py Miscellaneous diagnostics for the import system -lfcr.py Change LF line endings to CRLF (Unix to Windows) -linktree.py Make a copy of a tree with links to original files -lll.py Find and list symbolic links in current directory -mailerdaemon.py Parse error messages from mailer daemons (Sjoerd&Jack) -make_ctype.py Generate ctype.h replacement in stringobject.c md5sum.py Print MD5 checksums of argument files -mkreal.py Turn a symbolic link into a real file or directory ndiff.py Intelligent diff between text files (Tim Peters) nm2def.py Create a template for PC/python_nt.def (Marc Lemburg) -objgraph.py Print object graph from nm output on a library parseentities.py Utility for parsing HTML entity definitions parse_html5_entities.py Utility for parsing HTML5 entity definitions patchcheck.py Perform common checks and cleanup before committing pathfix.py Change #!/usr/local/bin/python into something else -pdeps.py Print dependencies between Python modules -pickle2db.py Load a pickle generated by db2pickle.py to a database -pindent.py Indent Python code, giving block-closing comments ptags.py Create vi tags file for Python modules pydoc3 Python documentation browser -pysource.py Find Python source files reindent.py Change .py files to use 4-space indents -reindent-rst.py Fix-up reStructuredText file whitespace -rgrep.py Reverse grep through a file (useful for big logfiles) run_tests.py Run the test suite with more sensible default options stable_abi.py Stable ABI checks and file generators. -suff.py Sort a list of files by suffix -texi2html.py Convert GNU texinfo files into HTML -untabify.py Replace tabs with spaces in argument files -which.py Find a program in $PATH win_add2path.py Add Python to the search path on Windows diff --git a/Tools/scripts/byext.py b/Tools/scripts/byext.py deleted file mode 100755 index a4b2f7ff6d828b..00000000000000 --- a/Tools/scripts/byext.py +++ /dev/null @@ -1,132 +0,0 @@ -#! /usr/bin/env python3 - -"""Show file statistics by extension.""" - -import os -import sys - - -class Stats: - - def __init__(self): - self.stats = {} - - def statargs(self, args): - for arg in args: - if os.path.isdir(arg): - self.statdir(arg) - elif os.path.isfile(arg): - self.statfile(arg) - else: - sys.stderr.write("Can't find %s\n" % arg) - self.addstats("", "unknown", 1) - - def statdir(self, dir): - self.addstats("", "dirs", 1) - try: - names = os.listdir(dir) - except OSError as err: - sys.stderr.write("Can't list %s: %s\n" % (dir, err)) - self.addstats("", "unlistable", 1) - return - for name in sorted(names): - if name.startswith(".#"): - continue # Skip CVS temp files - if name.endswith("~"): - continue # Skip Emacs backup files - full = os.path.join(dir, name) - if os.path.islink(full): - self.addstats("", "links", 1) - elif os.path.isdir(full): - self.statdir(full) - else: - self.statfile(full) - - def statfile(self, filename): - head, ext = os.path.splitext(filename) - head, base = os.path.split(filename) - if ext == base: - ext = "" # E.g. .cvsignore is deemed not to have an extension - ext = os.path.normcase(ext) - if not ext: - ext = "" - self.addstats(ext, "files", 1) - try: - with open(filename, "rb") as f: - data = f.read() - except IOError as err: - sys.stderr.write("Can't open %s: %s\n" % (filename, err)) - self.addstats(ext, "unopenable", 1) - return - self.addstats(ext, "bytes", len(data)) - if b'\0' in data: - self.addstats(ext, "binary", 1) - return - if not data: - self.addstats(ext, "empty", 1) - # self.addstats(ext, "chars", len(data)) - lines = str(data, "latin-1").splitlines() - self.addstats(ext, "lines", len(lines)) - del lines - words = data.split() - self.addstats(ext, "words", len(words)) - - def addstats(self, ext, key, n): - d = self.stats.setdefault(ext, {}) - d[key] = d.get(key, 0) + n - - def report(self): - exts = sorted(self.stats) - # Get the column keys - columns = {} - for ext in exts: - columns.update(self.stats[ext]) - cols = sorted(columns) - colwidth = {} - colwidth["ext"] = max(map(len, exts)) - minwidth = 6 - self.stats["TOTAL"] = {} - for col in cols: - total = 0 - cw = max(minwidth, len(col)) - for ext in exts: - value = self.stats[ext].get(col) - if value is None: - w = 0 - else: - w = len("%d" % value) - total += value - cw = max(cw, w) - cw = max(cw, len(str(total))) - colwidth[col] = cw - self.stats["TOTAL"][col] = total - exts.append("TOTAL") - for ext in exts: - self.stats[ext]["ext"] = ext - cols.insert(0, "ext") - - def printheader(): - for col in cols: - print("%*s" % (colwidth[col], col), end=' ') - print() - - printheader() - for ext in exts: - for col in cols: - value = self.stats[ext].get(col, "") - print("%*s" % (colwidth[col], value), end=' ') - print() - printheader() # Another header at the bottom - - -def main(): - args = sys.argv[1:] - if not args: - args = [os.curdir] - s = Stats() - s.statargs(args) - s.report() - - -if __name__ == "__main__": - main() diff --git a/Tools/scripts/byteyears.py b/Tools/scripts/byteyears.py deleted file mode 100755 index f58c34608faaf4..00000000000000 --- a/Tools/scripts/byteyears.py +++ /dev/null @@ -1,61 +0,0 @@ -#! /usr/bin/env python3 - -# Print the product of age and size of each file, in suitable units. -# -# Usage: byteyears [ -a | -m | -c ] file ... -# -# Options -[amc] select atime, mtime (default) or ctime as age. - -import sys, os, time -from stat import * - -def main(): - - # Use lstat() to stat files if it exists, else stat() - try: - statfunc = os.lstat - except AttributeError: - statfunc = os.stat - - # Parse options - if sys.argv[1] == '-m': - itime = ST_MTIME - del sys.argv[1] - elif sys.argv[1] == '-c': - itime = ST_CTIME - del sys.argv[1] - elif sys.argv[1] == '-a': - itime = ST_CTIME - del sys.argv[1] - else: - itime = ST_MTIME - - secs_per_year = 365.0 * 24.0 * 3600.0 # Scale factor - now = time.time() # Current time, for age computations - status = 0 # Exit status, set to 1 on errors - - # Compute max file name length - maxlen = 1 - for filename in sys.argv[1:]: - maxlen = max(maxlen, len(filename)) - - # Process each argument in turn - for filename in sys.argv[1:]: - try: - st = statfunc(filename) - except OSError as msg: - sys.stderr.write("can't stat %r: %r\n" % (filename, msg)) - status = 1 - st = () - if st: - anytime = st[itime] - size = st[ST_SIZE] - age = now - anytime - byteyears = float(size) * float(age) / secs_per_year - print(filename.ljust(maxlen), end=' ') - print(repr(int(byteyears)).rjust(8)) - - sys.exit(status) - -if __name__ == '__main__': - main() diff --git a/Tools/scripts/cleanfuture.py b/Tools/scripts/cleanfuture.py deleted file mode 100755 index 94f69126321550..00000000000000 --- a/Tools/scripts/cleanfuture.py +++ /dev/null @@ -1,275 +0,0 @@ -#! /usr/bin/env python3 - -"""cleanfuture [-d][-r][-v] path ... - --d Dry run. Analyze, but don't make any changes to, files. --r Recurse. Search for all .py files in subdirectories too. --v Verbose. Print informative msgs. - -Search Python (.py) files for future statements, and remove the features -from such statements that are already mandatory in the version of Python -you're using. - -Pass one or more file and/or directory paths. When a directory path, all -.py files within the directory will be examined, and, if the -r option is -given, likewise recursively for subdirectories. - -Overwrites files in place, renaming the originals with a .bak extension. If -cleanfuture finds nothing to change, the file is left alone. If cleanfuture -does change a file, the changed file is a fixed-point (i.e., running -cleanfuture on the resulting .py file won't change it again, at least not -until you try it again with a later Python release). - -Limitations: You can do these things, but this tool won't help you then: - -+ A future statement cannot be mixed with any other statement on the same - physical line (separated by semicolon). - -+ A future statement cannot contain an "as" clause. - -Example: Assuming you're using Python 2.2, if a file containing - -from __future__ import nested_scopes, generators - -is analyzed by cleanfuture, the line is rewritten to - -from __future__ import generators - -because nested_scopes is no longer optional in 2.2 but generators is. -""" - -import __future__ -import tokenize -import os -import sys - -dryrun = 0 -recurse = 0 -verbose = 0 - -def errprint(*args): - strings = map(str, args) - msg = ' '.join(strings) - if msg[-1:] != '\n': - msg += '\n' - sys.stderr.write(msg) - -def main(): - import getopt - global verbose, recurse, dryrun - try: - opts, args = getopt.getopt(sys.argv[1:], "drv") - except getopt.error as msg: - errprint(msg) - return - for o, a in opts: - if o == '-d': - dryrun += 1 - elif o == '-r': - recurse += 1 - elif o == '-v': - verbose += 1 - if not args: - errprint("Usage:", __doc__) - return - for arg in args: - check(arg) - -def check(file): - if os.path.isdir(file) and not os.path.islink(file): - if verbose: - print("listing directory", file) - names = os.listdir(file) - for name in names: - fullname = os.path.join(file, name) - if ((recurse and os.path.isdir(fullname) and - not os.path.islink(fullname)) - or name.lower().endswith(".py")): - check(fullname) - return - - if verbose: - print("checking", file, "...", end=' ') - try: - f = open(file) - except IOError as msg: - errprint("%r: I/O Error: %s" % (file, str(msg))) - return - - with f: - ff = FutureFinder(f, file) - changed = ff.run() - if changed: - ff.gettherest() - if changed: - if verbose: - print("changed.") - if dryrun: - print("But this is a dry run, so leaving it alone.") - for s, e, line in changed: - print("%r lines %d-%d" % (file, s+1, e+1)) - for i in range(s, e+1): - print(ff.lines[i], end=' ') - if line is None: - print("-- deleted") - else: - print("-- change to:") - print(line, end=' ') - if not dryrun: - bak = file + ".bak" - if os.path.exists(bak): - os.remove(bak) - os.rename(file, bak) - if verbose: - print("renamed", file, "to", bak) - with open(file, "w") as g: - ff.write(g) - if verbose: - print("wrote new", file) - else: - if verbose: - print("unchanged.") - -class FutureFinder: - - def __init__(self, f, fname): - self.f = f - self.fname = fname - self.ateof = 0 - self.lines = [] # raw file lines - - # List of (start_index, end_index, new_line) triples. - self.changed = [] - - # Line-getter for tokenize. - def getline(self): - if self.ateof: - return "" - line = self.f.readline() - if line == "": - self.ateof = 1 - else: - self.lines.append(line) - return line - - def run(self): - STRING = tokenize.STRING - NL = tokenize.NL - NEWLINE = tokenize.NEWLINE - COMMENT = tokenize.COMMENT - NAME = tokenize.NAME - OP = tokenize.OP - - changed = self.changed - get = tokenize.generate_tokens(self.getline).__next__ - type, token, (srow, scol), (erow, ecol), line = get() - - # Chew up initial comments and blank lines (if any). - while type in (COMMENT, NL, NEWLINE): - type, token, (srow, scol), (erow, ecol), line = get() - - # Chew up docstring (if any -- and it may be implicitly catenated!). - while type is STRING: - type, token, (srow, scol), (erow, ecol), line = get() - - # Analyze the future stmts. - while 1: - # Chew up comments and blank lines (if any). - while type in (COMMENT, NL, NEWLINE): - type, token, (srow, scol), (erow, ecol), line = get() - - if not (type is NAME and token == "from"): - break - startline = srow - 1 # tokenize is one-based - type, token, (srow, scol), (erow, ecol), line = get() - - if not (type is NAME and token == "__future__"): - break - type, token, (srow, scol), (erow, ecol), line = get() - - if not (type is NAME and token == "import"): - break - type, token, (srow, scol), (erow, ecol), line = get() - - # Get the list of features. - features = [] - while type is NAME: - features.append(token) - type, token, (srow, scol), (erow, ecol), line = get() - - if not (type is OP and token == ','): - break - type, token, (srow, scol), (erow, ecol), line = get() - - # A trailing comment? - comment = None - if type is COMMENT: - comment = token - type, token, (srow, scol), (erow, ecol), line = get() - - if type is not NEWLINE: - errprint("Skipping file %r; can't parse line %d:\n%s" % - (self.fname, srow, line)) - return [] - - endline = srow - 1 - - # Check for obsolete features. - okfeatures = [] - for f in features: - object = getattr(__future__, f, None) - if object is None: - # A feature we don't know about yet -- leave it in. - # They'll get a compile-time error when they compile - # this program, but that's not our job to sort out. - okfeatures.append(f) - else: - released = object.getMandatoryRelease() - if released is None or released <= sys.version_info: - # Withdrawn or obsolete. - pass - else: - okfeatures.append(f) - - # Rewrite the line if at least one future-feature is obsolete. - if len(okfeatures) < len(features): - if len(okfeatures) == 0: - line = None - else: - line = "from __future__ import " - line += ', '.join(okfeatures) - if comment is not None: - line += ' ' + comment - line += '\n' - changed.append((startline, endline, line)) - - # Loop back for more future statements. - - return changed - - def gettherest(self): - if self.ateof: - self.therest = '' - else: - self.therest = self.f.read() - - def write(self, f): - changed = self.changed - assert changed - # Prevent calling this again. - self.changed = [] - # Apply changes in reverse order. - changed.reverse() - for s, e, line in changed: - if line is None: - # pure deletion - del self.lines[s:e+1] - else: - self.lines[s:e+1] = [line] - f.writelines(self.lines) - # Copy over the remainder of the file. - if self.therest: - f.write(self.therest) - -if __name__ == '__main__': - main() diff --git a/Tools/scripts/copytime.py b/Tools/scripts/copytime.py deleted file mode 100755 index 715683f12557ca..00000000000000 --- a/Tools/scripts/copytime.py +++ /dev/null @@ -1,26 +0,0 @@ -#! /usr/bin/env python3 - -# Copy one file's atime and mtime to another - -import sys -import os -from stat import ST_ATIME, ST_MTIME # Really constants 7 and 8 - -def main(): - if len(sys.argv) != 3: - sys.stderr.write('usage: copytime source destination\n') - sys.exit(2) - file1, file2 = sys.argv[1], sys.argv[2] - try: - stat1 = os.stat(file1) - except OSError: - sys.stderr.write(file1 + ': cannot stat\n') - sys.exit(1) - try: - os.utime(file2, (stat1[ST_ATIME], stat1[ST_MTIME])) - except OSError: - sys.stderr.write(file2 + ': cannot change time\n') - sys.exit(2) - -if __name__ == '__main__': - main() diff --git a/Tools/scripts/crlf.py b/Tools/scripts/crlf.py deleted file mode 100755 index f231d292cebecd..00000000000000 --- a/Tools/scripts/crlf.py +++ /dev/null @@ -1,23 +0,0 @@ -#! /usr/bin/env python3 -"Replace CRLF with LF in argument files. Print names of changed files." - -import sys, os - -def main(): - for filename in sys.argv[1:]: - if os.path.isdir(filename): - print(filename, "Directory!") - continue - with open(filename, "rb") as f: - data = f.read() - if b'\0' in data: - print(filename, "Binary!") - continue - newdata = data.replace(b"\r\n", b"\n") - if newdata != data: - print(filename) - with open(filename, "wb") as f: - f.write(newdata) - -if __name__ == '__main__': - main() diff --git a/Tools/scripts/db2pickle.py b/Tools/scripts/db2pickle.py deleted file mode 100755 index a5532a8f3a412e..00000000000000 --- a/Tools/scripts/db2pickle.py +++ /dev/null @@ -1,135 +0,0 @@ -#!/usr/bin/env python3 - -""" -Synopsis: %(prog)s [-h|-g|-b|-r|-a] dbfile [ picklefile ] - -Convert the database file given on the command line to a pickle -representation. The optional flags indicate the type of the database: - - -a - open using dbm (any supported format) - -b - open as bsddb btree file - -d - open as dbm file - -g - open as gdbm file - -h - open as bsddb hash file - -r - open as bsddb recno file - -The default is hash. If a pickle file is named it is opened for write -access (deleting any existing data). If no pickle file is named, the pickle -output is written to standard output. - -""" - -import getopt -try: - import bsddb -except ImportError: - bsddb = None -try: - import dbm.ndbm as dbm -except ImportError: - dbm = None -try: - import dbm.gnu as gdbm -except ImportError: - gdbm = None -try: - import dbm.ndbm as anydbm -except ImportError: - anydbm = None -import sys -try: - import pickle as pickle -except ImportError: - import pickle - -prog = sys.argv[0] - -def usage(): - sys.stderr.write(__doc__ % globals()) - -def main(args): - try: - opts, args = getopt.getopt(args, "hbrdag", - ["hash", "btree", "recno", "dbm", - "gdbm", "anydbm"]) - except getopt.error: - usage() - return 1 - - if len(args) == 0 or len(args) > 2: - usage() - return 1 - elif len(args) == 1: - dbfile = args[0] - pfile = sys.stdout - else: - dbfile = args[0] - try: - pfile = open(args[1], 'wb') - except IOError: - sys.stderr.write("Unable to open %s\n" % args[1]) - return 1 - - dbopen = None - for opt, arg in opts: - if opt in ("-h", "--hash"): - try: - dbopen = bsddb.hashopen - except AttributeError: - sys.stderr.write("bsddb module unavailable.\n") - return 1 - elif opt in ("-b", "--btree"): - try: - dbopen = bsddb.btopen - except AttributeError: - sys.stderr.write("bsddb module unavailable.\n") - return 1 - elif opt in ("-r", "--recno"): - try: - dbopen = bsddb.rnopen - except AttributeError: - sys.stderr.write("bsddb module unavailable.\n") - return 1 - elif opt in ("-a", "--anydbm"): - try: - dbopen = anydbm.open - except AttributeError: - sys.stderr.write("dbm module unavailable.\n") - return 1 - elif opt in ("-g", "--gdbm"): - try: - dbopen = gdbm.open - except AttributeError: - sys.stderr.write("dbm.gnu module unavailable.\n") - return 1 - elif opt in ("-d", "--dbm"): - try: - dbopen = dbm.open - except AttributeError: - sys.stderr.write("dbm.ndbm module unavailable.\n") - return 1 - if dbopen is None: - if bsddb is None: - sys.stderr.write("bsddb module unavailable - ") - sys.stderr.write("must specify dbtype.\n") - return 1 - else: - dbopen = bsddb.hashopen - - try: - db = dbopen(dbfile, 'r') - except bsddb.error: - sys.stderr.write("Unable to open %s. " % dbfile) - sys.stderr.write("Check for format or version mismatch.\n") - return 1 - - for k in db.keys(): - pickle.dump((k, db[k]), pfile, 1==1) - - db.close() - pfile.close() - - return 0 - -if __name__ == "__main__": - sys.exit(main(sys.argv[1:])) diff --git a/Tools/scripts/dutree.doc b/Tools/scripts/dutree.doc deleted file mode 100644 index 490126b0182d1c..00000000000000 --- a/Tools/scripts/dutree.doc +++ /dev/null @@ -1,54 +0,0 @@ -Path: cwi.nl!sun4nl!mcsun!uunet!cs.utexas.edu!convex!usenet -From: tchrist@convex.COM (Tom Christiansen) -Newsgroups: comp.lang.perl -Subject: Re: The problems of Perl (Re: Question (silly?)) -Message-ID: <1992Jan17.053115.4220@convex.com> -Date: 17 Jan 92 05:31:15 GMT -References: <17458@ector.cs.purdue.edu> <1992Jan16.165347.25583@cherokee.uswest.com> <=#Hues+4@cs.psu.edu> -Sender: usenet@convex.com (news access account) -Reply-To: tchrist@convex.COM (Tom Christiansen) -Organization: CONVEX Realtime Development, Colorado Springs, CO -Lines: 83 -Nntp-Posting-Host: pixel.convex.com - -From the keyboard of flee@cs.psu.edu (Felix Lee): -:And Perl is definitely awkward with data types. I haven't yet found a -:pleasant way of shoving non-trivial data types into Perl's grammar. - -Yes, it's pretty awful at that, alright. Sometimes I write perl programs -that need them, and sometimes it just takes a little creativity. But -sometimes it's not worth it. I actually wrote a C program the other day -(gasp) because I didn't want to deal with a game matrix with six links per node. - -:Here's a very simple problem that's tricky to express in Perl: process -:the output of "du" to produce output that's indented to reflect the -:tree structure, and with each subtree sorted by size. Something like: -: 434 /etc -: | 344 . -: | 50 install -: | 35 uucp -: | 3 nserve -: | | 2 . -: | | 1 auth.info -: | 1 sm -: | 1 sm.bak - -At first I thought I could just keep one local list around -at once, but this seems inherently recursive. Which means -I need an real recursive data structure. Maybe you could -do it with one of the %assoc arrays Larry uses in the begat -programs, but I broke down and got dirty. I think the hardest -part was matching Felix's desired output exactly. It's not -blazingly fast: I should probably inline the &childof routine, -but it *was* faster to write than I could have written the -equivalent C program. - - ---tom - --- -"GUIs normally make it simple to accomplish simple actions and impossible -to accomplish complex actions." --Doug Gwyn (22/Jun/91 in comp.unix.wizards) - - Tom Christiansen tchrist@convex.com convex!tchrist - diff --git a/Tools/scripts/dutree.py b/Tools/scripts/dutree.py deleted file mode 100755 index d25cf72b707ecf..00000000000000 --- a/Tools/scripts/dutree.py +++ /dev/null @@ -1,60 +0,0 @@ -#! /usr/bin/env python3 -# Format du output in a tree shape - -import os, sys, errno - -def main(): - total, d = None, {} - with os.popen('du ' + ' '.join(sys.argv[1:])) as p: - for line in p: - i = 0 - while line[i] in '0123456789': i = i+1 - size = eval(line[:i]) - while line[i] in ' \t': i = i+1 - filename = line[i:-1] - comps = filename.split('/') - if comps[0] == '': comps[0] = '/' - if comps[len(comps)-1] == '': del comps[len(comps)-1] - total, d = store(size, comps, total, d) - try: - display(total, d) - except IOError as e: - if e.errno != errno.EPIPE: - raise - -def store(size, comps, total, d): - if comps == []: - return size, d - if comps[0] not in d: - d[comps[0]] = None, {} - t1, d1 = d[comps[0]] - d[comps[0]] = store(size, comps[1:], t1, d1) - return total, d - -def display(total, d): - show(total, d, '') - -def show(total, d, prefix): - if not d: return - list = [] - sum = 0 - for key in d.keys(): - tsub, dsub = d[key] - list.append((tsub, key)) - if tsub is not None: sum = sum + tsub -## if sum < total: -## list.append((total - sum, os.curdir)) - list.sort() - list.reverse() - width = len(repr(list[0][0])) - for tsub, key in list: - if tsub is None: - psub = prefix - else: - print(prefix + repr(tsub).rjust(width) + ' ' + key) - psub = prefix + ' '*(width-1) + '|' + ' '*(len(key)+1) - if key in d: - show(tsub, d[key][1], psub) - -if __name__ == '__main__': - main() diff --git a/Tools/scripts/find-uname.py b/Tools/scripts/find-uname.py deleted file mode 100755 index b6ec1b6d79060c..00000000000000 --- a/Tools/scripts/find-uname.py +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env python3 - -""" -For each argument on the command line, look for it in the set of all Unicode -names. Arguments are treated as case-insensitive regular expressions, e.g.: - - % find-uname 'small letter a$' 'horizontal line' - *** small letter a$ matches *** - LATIN SMALL LETTER A (97) - COMBINING LATIN SMALL LETTER A (867) - CYRILLIC SMALL LETTER A (1072) - PARENTHESIZED LATIN SMALL LETTER A (9372) - CIRCLED LATIN SMALL LETTER A (9424) - FULLWIDTH LATIN SMALL LETTER A (65345) - *** horizontal line matches *** - HORIZONTAL LINE EXTENSION (9135) -""" - -import unicodedata -import sys -import re - -def main(args): - unicode_names = [] - for ix in range(sys.maxunicode+1): - try: - unicode_names.append((ix, unicodedata.name(chr(ix)))) - except ValueError: # no name for the character - pass - for arg in args: - pat = re.compile(arg, re.I) - matches = [(y,x) for (x,y) in unicode_names - if pat.search(y) is not None] - if matches: - print("***", arg, "matches", "***") - for match in matches: - print("%s (%d)" % match) - -if __name__ == "__main__": - main(sys.argv[1:]) diff --git a/Tools/scripts/find_recursionlimit.py b/Tools/scripts/find_recursionlimit.py deleted file mode 100755 index b2842a62efdbd6..00000000000000 --- a/Tools/scripts/find_recursionlimit.py +++ /dev/null @@ -1,128 +0,0 @@ -#! /usr/bin/env python3 -"""Find the maximum recursion limit that prevents interpreter termination. - -This script finds the maximum safe recursion limit on a particular -platform. If you need to change the recursion limit on your system, -this script will tell you a safe upper bound. To use the new limit, -call sys.setrecursionlimit(). - -This module implements several ways to create infinite recursion in -Python. Different implementations end up pushing different numbers of -C stack frames, depending on how many calls through Python's abstract -C API occur. - -After each round of tests, it prints a message: -"Limit of NNNN is fine". - -The highest printed value of "NNNN" is therefore the highest potentially -safe limit for your system (which depends on the OS, architecture, but also -the compilation flags). Please note that it is practically impossible to -test all possible recursion paths in the interpreter, so the results of -this test should not be trusted blindly -- although they give a good hint -of which values are reasonable. - -NOTE: When the C stack space allocated by your system is exceeded due -to excessive recursion, exact behaviour depends on the platform, although -the interpreter will always fail in a likely brutal way: either a -segmentation fault, a MemoryError, or just a silent abort. - -NB: A program that does not use __methods__ can set a higher limit. -""" - -import sys -import itertools - -class RecursiveBlowup1: - def __init__(self): - self.__init__() - -def test_init(): - return RecursiveBlowup1() - -class RecursiveBlowup2: - def __repr__(self): - return repr(self) - -def test_repr(): - return repr(RecursiveBlowup2()) - -class RecursiveBlowup4: - def __add__(self, x): - return x + self - -def test_add(): - return RecursiveBlowup4() + RecursiveBlowup4() - -class RecursiveBlowup5: - def __getattr__(self, attr): - return getattr(self, attr) - -def test_getattr(): - return RecursiveBlowup5().attr - -class RecursiveBlowup6: - def __getitem__(self, item): - return self[item - 2] + self[item - 1] - -def test_getitem(): - return RecursiveBlowup6()[5] - -def test_recurse(): - return test_recurse() - -def test_cpickle(_cache={}): - import io - try: - import _pickle - except ImportError: - print("cannot import _pickle, skipped!") - return - k, l = None, None - for n in itertools.count(): - try: - l = _cache[n] - continue # Already tried and it works, let's save some time - except KeyError: - for i in range(100): - l = [k, l] - k = {i: l} - _pickle.Pickler(io.BytesIO(), protocol=-1).dump(l) - _cache[n] = l - -def test_compiler_recursion(): - # The compiler uses a scaling factor to support additional levels - # of recursion. This is a sanity check of that scaling to ensure - # it still raises RecursionError even at higher recursion limits - compile("()" * (10 * sys.getrecursionlimit()), "", "single") - -def check_limit(n, test_func_name): - sys.setrecursionlimit(n) - if test_func_name.startswith("test_"): - print(test_func_name[5:]) - else: - print(test_func_name) - test_func = globals()[test_func_name] - try: - test_func() - # AttributeError can be raised because of the way e.g. PyDict_GetItem() - # silences all exceptions and returns NULL, which is usually interpreted - # as "missing attribute". - except (RecursionError, AttributeError): - pass - else: - print("Yikes!") - -if __name__ == '__main__': - - limit = 1000 - while 1: - check_limit(limit, "test_recurse") - check_limit(limit, "test_add") - check_limit(limit, "test_repr") - check_limit(limit, "test_init") - check_limit(limit, "test_getattr") - check_limit(limit, "test_getitem") - check_limit(limit, "test_cpickle") - check_limit(limit, "test_compiler_recursion") - print("Limit of %d is fine" % limit) - limit = limit + 100 diff --git a/Tools/scripts/finddiv.py b/Tools/scripts/finddiv.py deleted file mode 100755 index d21253cf1c8a14..00000000000000 --- a/Tools/scripts/finddiv.py +++ /dev/null @@ -1,89 +0,0 @@ -#! /usr/bin/env python3 - -"""finddiv - a grep-like tool that looks for division operators. - -Usage: finddiv [-l] file_or_directory ... - -For directory arguments, all files in the directory whose name ends in -.py are processed, and subdirectories are processed recursively. - -This actually tokenizes the files to avoid false hits in comments or -strings literals. - -By default, this prints all lines containing a / or /= operator, in -grep -n style. With the -l option specified, it prints the filename -of files that contain at least one / or /= operator. -""" - -import os -import sys -import getopt -import tokenize - -def main(): - try: - opts, args = getopt.getopt(sys.argv[1:], "lh") - except getopt.error as msg: - usage(msg) - return 2 - if not args: - usage("at least one file argument is required") - return 2 - listnames = 0 - for o, a in opts: - if o == "-h": - print(__doc__) - return - if o == "-l": - listnames = 1 - exit = None - for filename in args: - x = process(filename, listnames) - exit = exit or x - return exit - -def usage(msg): - sys.stderr.write("%s: %s\n" % (sys.argv[0], msg)) - sys.stderr.write("Usage: %s [-l] file ...\n" % sys.argv[0]) - sys.stderr.write("Try `%s -h' for more information.\n" % sys.argv[0]) - -def process(filename, listnames): - if os.path.isdir(filename): - return processdir(filename, listnames) - try: - fp = open(filename) - except IOError as msg: - sys.stderr.write("Can't open: %s\n" % msg) - return 1 - with fp: - g = tokenize.generate_tokens(fp.readline) - lastrow = None - for type, token, (row, col), end, line in g: - if token in ("/", "/="): - if listnames: - print(filename) - break - if row != lastrow: - lastrow = row - print("%s:%d:%s" % (filename, row, line), end=' ') - -def processdir(dir, listnames): - try: - names = os.listdir(dir) - except OSError as msg: - sys.stderr.write("Can't list directory: %s\n" % dir) - return 1 - files = [] - for name in names: - fn = os.path.join(dir, name) - if os.path.normcase(fn).endswith(".py") or os.path.isdir(fn): - files.append(fn) - files.sort(key=os.path.normcase) - exit = None - for fn in files: - x = process(fn, listnames) - exit = exit or x - return exit - -if __name__ == "__main__": - sys.exit(main()) diff --git a/Tools/scripts/findlinksto.py b/Tools/scripts/findlinksto.py deleted file mode 100755 index b924f27b095eb0..00000000000000 --- a/Tools/scripts/findlinksto.py +++ /dev/null @@ -1,43 +0,0 @@ -#! /usr/bin/env python3 - -# findlinksto -# -# find symbolic links to a path matching a regular expression - -import os -import sys -import re -import getopt - -def main(): - try: - opts, args = getopt.getopt(sys.argv[1:], '') - if len(args) < 2: - raise getopt.GetoptError('not enough arguments', None) - except getopt.GetoptError as msg: - sys.stdout = sys.stderr - print(msg) - print('usage: findlinksto pattern directory ...') - sys.exit(2) - pat, dirs = args[0], args[1:] - prog = re.compile(pat) - for dirname in dirs: - os.walk(dirname, visit, prog) - -def visit(prog, dirname, names): - if os.path.islink(dirname): - names[:] = [] - return - if os.path.ismount(dirname): - print('descend into', dirname) - for name in names: - name = os.path.join(dirname, name) - try: - linkto = os.readlink(name) - if prog.search(linkto) is not None: - print(name, '->', linkto) - except OSError: - pass - -if __name__ == '__main__': - main() diff --git a/Tools/scripts/findnocoding.py b/Tools/scripts/findnocoding.py deleted file mode 100755 index 6c16b1ce151852..00000000000000 --- a/Tools/scripts/findnocoding.py +++ /dev/null @@ -1,107 +0,0 @@ -#!/usr/bin/env python3 - -"""List all those Python files that require a coding directive - -Usage: findnocoding.py dir1 [dir2...] -""" - -__author__ = "Oleg Broytmann, Georg Brandl" - -import sys, os, re, getopt - -# our pysource module finds Python source files -try: - import pysource -except ImportError: - # emulate the module with a simple os.walk - class pysource: - has_python_ext = looks_like_python = can_be_compiled = None - def walk_python_files(self, paths, *args, **kwargs): - for path in paths: - if os.path.isfile(path): - yield path.endswith(".py") - elif os.path.isdir(path): - for root, dirs, files in os.walk(path): - for filename in files: - if filename.endswith(".py"): - yield os.path.join(root, filename) - pysource = pysource() - - - print("The pysource module is not available; " - "no sophisticated Python source file search will be done.", file=sys.stderr) - - -decl_re = re.compile(rb'^[ \t\f]*#.*?coding[:=][ \t]*([-\w.]+)') -blank_re = re.compile(rb'^[ \t\f]*(?:[#\r\n]|$)') - -def get_declaration(line): - match = decl_re.match(line) - if match: - return match.group(1) - return b'' - -def has_correct_encoding(text, codec): - try: - str(text, codec) - except UnicodeDecodeError: - return False - else: - return True - -def needs_declaration(fullpath): - try: - infile = open(fullpath, 'rb') - except IOError: # Oops, the file was removed - ignore it - return None - - with infile: - line1 = infile.readline() - line2 = infile.readline() - - if (get_declaration(line1) or - blank_re.match(line1) and get_declaration(line2)): - # the file does have an encoding declaration, so trust it - return False - - # check the whole file for non utf-8 characters - rest = infile.read() - - if has_correct_encoding(line1+line2+rest, "utf-8"): - return False - - return True - - -usage = """Usage: %s [-cd] paths... - -c: recognize Python source files trying to compile them - -d: debug output""" % sys.argv[0] - -if __name__ == '__main__': - - try: - opts, args = getopt.getopt(sys.argv[1:], 'cd') - except getopt.error as msg: - print(msg, file=sys.stderr) - print(usage, file=sys.stderr) - sys.exit(1) - - is_python = pysource.looks_like_python - debug = False - - for o, a in opts: - if o == '-c': - is_python = pysource.can_be_compiled - elif o == '-d': - debug = True - - if not args: - print(usage, file=sys.stderr) - sys.exit(1) - - for fullpath in pysource.walk_python_files(args, is_python): - if debug: - print("Testing for coding: %s" % fullpath) - result = needs_declaration(fullpath) - if result: - print(fullpath) diff --git a/Tools/scripts/fixcid.py b/Tools/scripts/fixcid.py deleted file mode 100755 index 8f35eaeeb4f61f..00000000000000 --- a/Tools/scripts/fixcid.py +++ /dev/null @@ -1,316 +0,0 @@ -#! /usr/bin/env python3 - -# Perform massive identifier substitution on C source files. -# This actually tokenizes the files (to some extent) so it can -# avoid making substitutions inside strings or comments. -# Inside strings, substitutions are never made; inside comments, -# it is a user option (off by default). -# -# The substitutions are read from one or more files whose lines, -# when not empty, after stripping comments starting with #, -# must contain exactly two words separated by whitespace: the -# old identifier and its replacement. -# -# The option -r reverses the sense of the substitutions (this may be -# useful to undo a particular substitution). -# -# If the old identifier is prefixed with a '*' (with no intervening -# whitespace), then it will not be substituted inside comments. -# -# Command line arguments are files or directories to be processed. -# Directories are searched recursively for files whose name looks -# like a C file (ends in .h or .c). The special filename '-' means -# operate in filter mode: read stdin, write stdout. -# -# Symbolic links are always ignored (except as explicit directory -# arguments). -# -# The original files are kept as back-up with a "~" suffix. -# -# Changes made are reported to stdout in a diff-like format. -# -# NB: by changing only the function fixline() you can turn this -# into a program for different changes to C source files; by -# changing the function wanted() you can make a different selection of -# files. - -import sys -import re -import os -from stat import * -import getopt - -err = sys.stderr.write -dbg = err -rep = sys.stdout.write - -def usage(): - progname = sys.argv[0] - err('Usage: ' + progname + - ' [-c] [-r] [-s file] ... file-or-directory ...\n') - err('\n') - err('-c : substitute inside comments\n') - err('-r : reverse direction for following -s options\n') - err('-s substfile : add a file of substitutions\n') - err('\n') - err('Each non-empty non-comment line in a substitution file must\n') - err('contain exactly two words: an identifier and its replacement.\n') - err('Comments start with a # character and end at end of line.\n') - err('If an identifier is preceded with a *, it is not substituted\n') - err('inside a comment even when -c is specified.\n') - -def main(): - try: - opts, args = getopt.getopt(sys.argv[1:], 'crs:') - except getopt.error as msg: - err('Options error: ' + str(msg) + '\n') - usage() - sys.exit(2) - bad = 0 - if not args: # No arguments - usage() - sys.exit(2) - for opt, arg in opts: - if opt == '-c': - setdocomments() - if opt == '-r': - setreverse() - if opt == '-s': - addsubst(arg) - for arg in args: - if os.path.isdir(arg): - if recursedown(arg): bad = 1 - elif os.path.islink(arg): - err(arg + ': will not process symbolic links\n') - bad = 1 - else: - if fix(arg): bad = 1 - sys.exit(bad) - -# Change this regular expression to select a different set of files -Wanted = r'^[a-zA-Z0-9_]+\.[ch]$' -def wanted(name): - return re.match(Wanted, name) - -def recursedown(dirname): - dbg('recursedown(%r)\n' % (dirname,)) - bad = 0 - try: - names = os.listdir(dirname) - except OSError as msg: - err(dirname + ': cannot list directory: ' + str(msg) + '\n') - return 1 - names.sort() - subdirs = [] - for name in names: - if name in (os.curdir, os.pardir): continue - fullname = os.path.join(dirname, name) - if os.path.islink(fullname): pass - elif os.path.isdir(fullname): - subdirs.append(fullname) - elif wanted(name): - if fix(fullname): bad = 1 - for fullname in subdirs: - if recursedown(fullname): bad = 1 - return bad - -def fix(filename): -## dbg('fix(%r)\n' % (filename,)) - if filename == '-': - # Filter mode - f = sys.stdin - g = sys.stdout - else: - # File replacement mode - try: - f = open(filename, 'r') - except IOError as msg: - err(filename + ': cannot open: ' + str(msg) + '\n') - return 1 - head, tail = os.path.split(filename) - tempname = os.path.join(head, '@' + tail) - g = None - # If we find a match, we rewind the file and start over but - # now copy everything to a temp file. - lineno = 0 - initfixline() - while 1: - line = f.readline() - if not line: break - lineno = lineno + 1 - while line[-2:] == '\\\n': - nextline = f.readline() - if not nextline: break - line = line + nextline - lineno = lineno + 1 - newline = fixline(line) - if newline != line: - if g is None: - try: - g = open(tempname, 'w') - except IOError as msg: - f.close() - err(tempname+': cannot create: '+ - str(msg)+'\n') - return 1 - f.seek(0) - lineno = 0 - initfixline() - rep(filename + ':\n') - continue # restart from the beginning - rep(repr(lineno) + '\n') - rep('< ' + line) - rep('> ' + newline) - if g is not None: - g.write(newline) - - # End of file - if filename == '-': return 0 # Done in filter mode - f.close() - if not g: return 0 # No changes - g.close() - - # Finishing touch -- move files - - # First copy the file's mode to the temp file - try: - statbuf = os.stat(filename) - os.chmod(tempname, statbuf[ST_MODE] & 0o7777) - except OSError as msg: - err(tempname + ': warning: chmod failed (' + str(msg) + ')\n') - # Then make a backup of the original file as filename~ - try: - os.rename(filename, filename + '~') - except OSError as msg: - err(filename + ': warning: backup failed (' + str(msg) + ')\n') - # Now move the temp file to the original file - try: - os.rename(tempname, filename) - except OSError as msg: - err(filename + ': rename failed (' + str(msg) + ')\n') - return 1 - # Return success - return 0 - -# Tokenizing ANSI C (partly) - -Identifier = '(struct )?[a-zA-Z_][a-zA-Z0-9_]+' -String = r'"([^\n\\"]|\\.)*"' -Char = r"'([^\n\\']|\\.)*'" -CommentStart = r'/\*' -CommentEnd = r'\*/' - -Hexnumber = '0[xX][0-9a-fA-F]*[uUlL]*' -Octnumber = '0[0-7]*[uUlL]*' -Decnumber = '[1-9][0-9]*[uUlL]*' -Intnumber = Hexnumber + '|' + Octnumber + '|' + Decnumber -Exponent = '[eE][-+]?[0-9]+' -Pointfloat = r'([0-9]+\.[0-9]*|\.[0-9]+)(' + Exponent + r')?' -Expfloat = '[0-9]+' + Exponent -Floatnumber = Pointfloat + '|' + Expfloat -Number = Floatnumber + '|' + Intnumber - -# Anything else is an operator -- don't list this explicitly because of '/*' - -OutsideComment = (Identifier, Number, String, Char, CommentStart) -OutsideCommentPattern = '(' + '|'.join(OutsideComment) + ')' -OutsideCommentProgram = re.compile(OutsideCommentPattern) - -InsideComment = (Identifier, Number, CommentEnd) -InsideCommentPattern = '(' + '|'.join(InsideComment) + ')' -InsideCommentProgram = re.compile(InsideCommentPattern) - -def initfixline(): - global Program - Program = OutsideCommentProgram - -def fixline(line): - global Program -## print('-->', repr(line)) - i = 0 - while i < len(line): - match = Program.search(line, i) - if match is None: break - i = match.start() - found = match.group(0) -## if Program is InsideCommentProgram: print(end='... ') -## else: print(end=' ') -## print(found) - if len(found) == 2: - if found == '/*': - Program = InsideCommentProgram - elif found == '*/': - Program = OutsideCommentProgram - n = len(found) - if found in Dict: - subst = Dict[found] - if Program is InsideCommentProgram: - if not Docomments: - print('Found in comment:', found) - i = i + n - continue - if found in NotInComment: -## print(end='Ignored in comment: ') -## print(found, '-->', subst) -## print('Line:', line, end='') - subst = found -## else: -## print(end='Substituting in comment: ') -## print(found, '-->', subst) -## print('Line:', line, end='') - line = line[:i] + subst + line[i+n:] - n = len(subst) - i = i + n - return line - -Docomments = 0 -def setdocomments(): - global Docomments - Docomments = 1 - -Reverse = 0 -def setreverse(): - global Reverse - Reverse = (not Reverse) - -Dict = {} -NotInComment = {} -def addsubst(substfile): - try: - fp = open(substfile, 'r') - except IOError as msg: - err(substfile + ': cannot read substfile: ' + str(msg) + '\n') - sys.exit(1) - with fp: - lineno = 0 - while 1: - line = fp.readline() - if not line: break - lineno = lineno + 1 - try: - i = line.index('#') - except ValueError: - i = -1 # Happens to delete trailing \n - words = line[:i].split() - if not words: continue - if len(words) == 3 and words[0] == 'struct': - words[:2] = [words[0] + ' ' + words[1]] - elif len(words) != 2: - err(substfile + '%s:%r: warning: bad line: %r' % (substfile, lineno, line)) - continue - if Reverse: - [value, key] = words - else: - [key, value] = words - if value[0] == '*': - value = value[1:] - if key[0] == '*': - key = key[1:] - NotInComment[key] = value - if key in Dict: - err('%s:%r: warning: overriding: %r %r\n' % (substfile, lineno, key, value)) - err('%s:%r: warning: previous: %r\n' % (substfile, lineno, Dict[key])) - Dict[key] = value - -if __name__ == '__main__': - main() diff --git a/Tools/scripts/fixdiv.py b/Tools/scripts/fixdiv.py deleted file mode 100755 index df7c481aa22808..00000000000000 --- a/Tools/scripts/fixdiv.py +++ /dev/null @@ -1,378 +0,0 @@ -#! /usr/bin/env python3 - -"""fixdiv - tool to fix division operators. - -To use this tool, first run `python -Qwarnall yourscript.py 2>warnings'. -This runs the script `yourscript.py' while writing warning messages -about all uses of the classic division operator to the file -`warnings'. The warnings look like this: - - :: DeprecationWarning: classic division - -The warnings are written to stderr, so you must use `2>' for the I/O -redirect. I know of no way to redirect stderr on Windows in a DOS -box, so you will have to modify the script to set sys.stderr to some -kind of log file if you want to do this on Windows. - -The warnings are not limited to the script; modules imported by the -script may also trigger warnings. In fact a useful technique is to -write a test script specifically intended to exercise all code in a -particular module or set of modules. - -Then run `python fixdiv.py warnings'. This first reads the warnings, -looking for classic division warnings, and sorts them by file name and -line number. Then, for each file that received at least one warning, -it parses the file and tries to match the warnings up to the division -operators found in the source code. If it is successful, it writes -its findings to stdout, preceded by a line of dashes and a line of the -form: - - Index: - -If the only findings found are suggestions to change a / operator into -a // operator, the output is acceptable input for the Unix 'patch' -program. - -Here are the possible messages on stdout (N stands for a line number): - -- A plain-diff-style change ('NcN', a line marked by '<', a line - containing '---', and a line marked by '>'): - - A / operator was found that should be changed to //. This is the - recommendation when only int and/or long arguments were seen. - -- 'True division / operator at line N' and a line marked by '=': - - A / operator was found that can remain unchanged. This is the - recommendation when only float and/or complex arguments were seen. - -- 'Ambiguous / operator (..., ...) at line N', line marked by '?': - - A / operator was found for which int or long as well as float or - complex arguments were seen. This is highly unlikely; if it occurs, - you may have to restructure the code to keep the classic semantics, - or maybe you don't care about the classic semantics. - -- 'No conclusive evidence on line N', line marked by '*': - - A / operator was found for which no warnings were seen. This could - be code that was never executed, or code that was only executed - with user-defined objects as arguments. You will have to - investigate further. Note that // can be overloaded separately from - /, using __floordiv__. True division can also be separately - overloaded, using __truediv__. Classic division should be the same - as either of those. (XXX should I add a warning for division on - user-defined objects, to disambiguate this case from code that was - never executed?) - -- 'Phantom ... warnings for line N', line marked by '*': - - A warning was seen for a line not containing a / operator. The most - likely cause is a warning about code executed by 'exec' or eval() - (see note below), or an indirect invocation of the / operator, for - example via the div() function in the operator module. It could - also be caused by a change to the file between the time the test - script was run to collect warnings and the time fixdiv was run. - -- 'More than one / operator in line N'; or - 'More than one / operator per statement in lines N-N': - - The scanner found more than one / operator on a single line, or in a - statement split across multiple lines. Because the warnings - framework doesn't (and can't) show the offset within the line, and - the code generator doesn't always give the correct line number for - operations in a multi-line statement, we can't be sure whether all - operators in the statement were executed. To be on the safe side, - by default a warning is issued about this case. In practice, these - cases are usually safe, and the -m option suppresses these warning. - -- 'Can't find the / operator in line N', line marked by '*': - - This really shouldn't happen. It means that the tokenize module - reported a '/' operator but the line it returns didn't contain a '/' - character at the indicated position. - -- 'Bad warning for line N: XYZ', line marked by '*': - - This really shouldn't happen. It means that a 'classic XYZ - division' warning was read with XYZ being something other than - 'int', 'long', 'float', or 'complex'. - -Notes: - -- The augmented assignment operator /= is handled the same way as the - / operator. - -- This tool never looks at the // operator; no warnings are ever - generated for use of this operator. - -- This tool never looks at the / operator when a future division - statement is in effect; no warnings are generated in this case, and - because the tool only looks at files for which at least one classic - division warning was seen, it will never look at files containing a - future division statement. - -- Warnings may be issued for code not read from a file, but executed - using the exec() or eval() functions. These may have - in the filename position, in which case the fixdiv script - will attempt and fail to open a file named '' and issue a - warning about this failure; or these may be reported as 'Phantom' - warnings (see above). You're on your own to deal with these. You - could make all recommended changes and add a future division - statement to all affected files, and then re-run the test script; it - should not issue any warnings. If there are any, and you have a - hard time tracking down where they are generated, you can use the - -Werror option to force an error instead of a first warning, - generating a traceback. - -- The tool should be run from the same directory as that from which - the original script was run, otherwise it won't be able to open - files given by relative pathnames. -""" - -import sys -import getopt -import re -import tokenize - -multi_ok = 0 - -def main(): - try: - opts, args = getopt.getopt(sys.argv[1:], "hm") - except getopt.error as msg: - usage(msg) - return 2 - for o, a in opts: - if o == "-h": - print(__doc__) - return - if o == "-m": - global multi_ok - multi_ok = 1 - if not args: - usage("at least one file argument is required") - return 2 - if args[1:]: - sys.stderr.write("%s: extra file arguments ignored\n", sys.argv[0]) - warnings = readwarnings(args[0]) - if warnings is None: - return 1 - files = list(warnings.keys()) - if not files: - print("No classic division warnings read from", args[0]) - return - files.sort() - exit = None - for filename in files: - x = process(filename, warnings[filename]) - exit = exit or x - return exit - -def usage(msg): - sys.stderr.write("%s: %s\n" % (sys.argv[0], msg)) - sys.stderr.write("Usage: %s [-m] warnings\n" % sys.argv[0]) - sys.stderr.write("Try `%s -h' for more information.\n" % sys.argv[0]) - -PATTERN = (r"^(.+?):(\d+): DeprecationWarning: " - r"classic (int|long|float|complex) division$") - -def readwarnings(warningsfile): - prog = re.compile(PATTERN) - warnings = {} - try: - f = open(warningsfile) - except IOError as msg: - sys.stderr.write("can't open: %s\n" % msg) - return - with f: - while 1: - line = f.readline() - if not line: - break - m = prog.match(line) - if not m: - if line.find("division") >= 0: - sys.stderr.write("Warning: ignored input " + line) - continue - filename, lineno, what = m.groups() - list = warnings.get(filename) - if list is None: - warnings[filename] = list = [] - list.append((int(lineno), sys.intern(what))) - return warnings - -def process(filename, list): - print("-"*70) - assert list # if this fails, readwarnings() is broken - try: - fp = open(filename) - except IOError as msg: - sys.stderr.write("can't open: %s\n" % msg) - return 1 - with fp: - print("Index:", filename) - f = FileContext(fp) - list.sort() - index = 0 # list[:index] has been processed, list[index:] is still to do - g = tokenize.generate_tokens(f.readline) - while 1: - startlineno, endlineno, slashes = lineinfo = scanline(g) - if startlineno is None: - break - assert startlineno <= endlineno is not None - orphans = [] - while index < len(list) and list[index][0] < startlineno: - orphans.append(list[index]) - index += 1 - if orphans: - reportphantomwarnings(orphans, f) - warnings = [] - while index < len(list) and list[index][0] <= endlineno: - warnings.append(list[index]) - index += 1 - if not slashes and not warnings: - pass - elif slashes and not warnings: - report(slashes, "No conclusive evidence") - elif warnings and not slashes: - reportphantomwarnings(warnings, f) - else: - if len(slashes) > 1: - if not multi_ok: - rows = [] - lastrow = None - for (row, col), line in slashes: - if row == lastrow: - continue - rows.append(row) - lastrow = row - assert rows - if len(rows) == 1: - print("*** More than one / operator in line", rows[0]) - else: - print("*** More than one / operator per statement", end=' ') - print("in lines %d-%d" % (rows[0], rows[-1])) - intlong = [] - floatcomplex = [] - bad = [] - for lineno, what in warnings: - if what in ("int", "long"): - intlong.append(what) - elif what in ("float", "complex"): - floatcomplex.append(what) - else: - bad.append(what) - lastrow = None - for (row, col), line in slashes: - if row == lastrow: - continue - lastrow = row - line = chop(line) - if line[col:col+1] != "/": - print("*** Can't find the / operator in line %d:" % row) - print("*", line) - continue - if bad: - print("*** Bad warning for line %d:" % row, bad) - print("*", line) - elif intlong and not floatcomplex: - print("%dc%d" % (row, row)) - print("<", line) - print("---") - print(">", line[:col] + "/" + line[col:]) - elif floatcomplex and not intlong: - print("True division / operator at line %d:" % row) - print("=", line) - elif intlong and floatcomplex: - print("*** Ambiguous / operator (%s, %s) at line %d:" % - ("|".join(intlong), "|".join(floatcomplex), row)) - print("?", line) - -def reportphantomwarnings(warnings, f): - blocks = [] - lastrow = None - lastblock = None - for row, what in warnings: - if row != lastrow: - lastblock = [row] - blocks.append(lastblock) - lastblock.append(what) - for block in blocks: - row = block[0] - whats = "/".join(block[1:]) - print("*** Phantom %s warnings for line %d:" % (whats, row)) - f.report(row, mark="*") - -def report(slashes, message): - lastrow = None - for (row, col), line in slashes: - if row != lastrow: - print("*** %s on line %d:" % (message, row)) - print("*", chop(line)) - lastrow = row - -class FileContext: - def __init__(self, fp, window=5, lineno=1): - self.fp = fp - self.window = 5 - self.lineno = 1 - self.eoflookahead = 0 - self.lookahead = [] - self.buffer = [] - def fill(self): - while len(self.lookahead) < self.window and not self.eoflookahead: - line = self.fp.readline() - if not line: - self.eoflookahead = 1 - break - self.lookahead.append(line) - def readline(self): - self.fill() - if not self.lookahead: - return "" - line = self.lookahead.pop(0) - self.buffer.append(line) - self.lineno += 1 - return line - def __getitem__(self, index): - self.fill() - bufstart = self.lineno - len(self.buffer) - lookend = self.lineno + len(self.lookahead) - if bufstart <= index < self.lineno: - return self.buffer[index - bufstart] - if self.lineno <= index < lookend: - return self.lookahead[index - self.lineno] - raise KeyError - def report(self, first, last=None, mark="*"): - if last is None: - last = first - for i in range(first, last+1): - try: - line = self[first] - except KeyError: - line = "" - print(mark, chop(line)) - -def scanline(g): - slashes = [] - startlineno = None - endlineno = None - for type, token, start, end, line in g: - endlineno = end[0] - if startlineno is None: - startlineno = endlineno - if token in ("/", "/="): - slashes.append((start, line)) - if type == tokenize.NEWLINE: - break - return startlineno, endlineno, slashes - -def chop(line): - if line.endswith("\n"): - return line[:-1] - else: - return line - -if __name__ == "__main__": - sys.exit(main()) diff --git a/Tools/scripts/fixheader.py b/Tools/scripts/fixheader.py deleted file mode 100755 index c834eec1e20e60..00000000000000 --- a/Tools/scripts/fixheader.py +++ /dev/null @@ -1,49 +0,0 @@ -#! /usr/bin/env python3 - -# Add some standard cpp magic to a header file - -import sys - -def main(): - args = sys.argv[1:] - for filename in args: - process(filename) - -def process(filename): - try: - f = open(filename, 'r') - except IOError as msg: - sys.stderr.write('%s: can\'t open: %s\n' % (filename, str(msg))) - return - with f: - data = f.read() - if data[:2] != '/*': - sys.stderr.write('%s does not begin with C comment\n' % filename) - return - try: - f = open(filename, 'w') - except IOError as msg: - sys.stderr.write('%s: can\'t write: %s\n' % (filename, str(msg))) - return - with f: - sys.stderr.write('Processing %s ...\n' % filename) - magic = 'Py_' - for c in filename: - if ord(c)<=0x80 and c.isalnum(): - magic = magic + c.upper() - else: magic = magic + '_' - print('#ifndef', magic, file=f) - print('#define', magic, file=f) - print('#ifdef __cplusplus', file=f) - print('extern "C" {', file=f) - print('#endif', file=f) - print(file=f) - f.write(data) - print(file=f) - print('#ifdef __cplusplus', file=f) - print('}', file=f) - print('#endif', file=f) - print('#endif /*', '!'+magic, '*/', file=f) - -if __name__ == '__main__': - main() diff --git a/Tools/scripts/fixnotice.py b/Tools/scripts/fixnotice.py deleted file mode 100755 index 317051dd82f3e9..00000000000000 --- a/Tools/scripts/fixnotice.py +++ /dev/null @@ -1,109 +0,0 @@ -#! /usr/bin/env python3 - -"""(Ostensibly) fix copyright notices in files. - -Actually, this script will simply replace a block of text in a file from one -string to another. It will only do this once though, i.e. not globally -throughout the file. It writes a backup file and then does an os.rename() -dance for atomicity. - -Usage: fixnotices.py [options] [filenames] -Options: - -h / --help - Print this message and exit - - --oldnotice=file - Use the notice in the file as the old (to be replaced) string, instead - of the hard coded value in the script. - - --newnotice=file - Use the notice in the file as the new (replacement) string, instead of - the hard coded value in the script. - - --dry-run - Don't actually make the changes, but print out the list of files that - would change. When used with -v, a status will be printed for every - file. - - -v / --verbose - Print a message for every file looked at, indicating whether the file - is changed or not. -""" - -OLD_NOTICE = """/*********************************************************** -Copyright (c) 2000, BeOpen.com. -Copyright (c) 1995-2000, Corporation for National Research Initiatives. -Copyright (c) 1990-1995, Stichting Mathematisch Centrum. -All rights reserved. - -See the file "Misc/COPYRIGHT" for information on usage and -redistribution of this file, and for a DISCLAIMER OF ALL WARRANTIES. -******************************************************************/ -""" -import os -import sys -import getopt - -NEW_NOTICE = "" -DRYRUN = 0 -VERBOSE = 0 - - -def usage(code, msg=''): - print(__doc__ % globals()) - if msg: - print(msg) - sys.exit(code) - - -def main(): - global DRYRUN, OLD_NOTICE, NEW_NOTICE, VERBOSE - try: - opts, args = getopt.getopt(sys.argv[1:], 'hv', - ['help', 'oldnotice=', 'newnotice=', - 'dry-run', 'verbose']) - except getopt.error as msg: - usage(1, msg) - - for opt, arg in opts: - if opt in ('-h', '--help'): - usage(0) - elif opt in ('-v', '--verbose'): - VERBOSE = 1 - elif opt == '--dry-run': - DRYRUN = 1 - elif opt == '--oldnotice': - with open(arg) as fp: - OLD_NOTICE = fp.read() - elif opt == '--newnotice': - with open(arg) as fp: - NEW_NOTICE = fp.read() - - for arg in args: - process(arg) - - -def process(file): - with open(file) as f: - data = f.read() - i = data.find(OLD_NOTICE) - if i < 0: - if VERBOSE: - print('no change:', file) - return - elif DRYRUN or VERBOSE: - print(' change:', file) - if DRYRUN: - # Don't actually change the file - return - data = data[:i] + NEW_NOTICE + data[i+len(OLD_NOTICE):] - new = file + ".new" - backup = file + ".bak" - with open(new, "w") as f: - f.write(data) - os.rename(file, backup) - os.rename(new, file) - - -if __name__ == '__main__': - main() diff --git a/Tools/scripts/fixps.py b/Tools/scripts/fixps.py deleted file mode 100755 index 725300e56a27db..00000000000000 --- a/Tools/scripts/fixps.py +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env python3 - -# Fix Python script(s) to reference the interpreter via /usr/bin/env python. -# Warning: this overwrites the file without making a backup. - -import sys -import re - - -def main(): - for filename in sys.argv[1:]: - try: - f = open(filename, 'r') - except IOError as msg: - print(filename, ': can\'t open :', msg) - continue - with f: - line = f.readline() - if not re.match('^#! */usr/local/bin/python', line): - print(filename, ': not a /usr/local/bin/python script') - continue - rest = f.read() - line = re.sub('/usr/local/bin/python', - '/usr/bin/env python', line) - print(filename, ':', repr(line)) - with open(filename, "w") as f: - f.write(line) - f.write(rest) - -if __name__ == '__main__': - main() diff --git a/Tools/scripts/get-remote-certificate.py b/Tools/scripts/get-remote-certificate.py deleted file mode 100755 index 68272fca83fe27..00000000000000 --- a/Tools/scripts/get-remote-certificate.py +++ /dev/null @@ -1,70 +0,0 @@ -#!/usr/bin/env python3 -# -# fetch the certificate that the server(s) are providing in PEM form -# -# args are HOST:PORT [, HOST:PORT...] -# -# By Bill Janssen. - -import re -import os -import sys -import tempfile - - -def fetch_server_certificate (host, port): - - def subproc(cmd): - from subprocess import Popen, PIPE, STDOUT, DEVNULL - proc = Popen(cmd, stdout=PIPE, stderr=STDOUT, stdin=DEVNULL) - status = proc.wait() - output = proc.stdout.read() - return status, output - - def strip_to_x509_cert(certfile_contents, outfile=None): - m = re.search(br"^([-]+BEGIN CERTIFICATE[-]+[\r]*\n" - br".*[\r]*^[-]+END CERTIFICATE[-]+)$", - certfile_contents, re.MULTILINE | re.DOTALL) - if not m: - return None - else: - tn = tempfile.mktemp() - with open(tn, "wb") as fp: - fp.write(m.group(1) + b"\n") - try: - tn2 = (outfile or tempfile.mktemp()) - cmd = ['openssl', 'x509', '-in', tn, '-out', tn2] - status, output = subproc(cmd) - if status != 0: - raise RuntimeError('OpenSSL x509 failed with status %s and ' - 'output: %r' % (status, output)) - with open(tn2, 'rb') as fp: - data = fp.read() - os.unlink(tn2) - return data - finally: - os.unlink(tn) - - cmd = ['openssl', 's_client', '-connect', '%s:%s' % (host, port), '-showcerts'] - status, output = subproc(cmd) - - if status != 0: - raise RuntimeError('OpenSSL connect failed with status %s and ' - 'output: %r' % (status, output)) - certtext = strip_to_x509_cert(output) - if not certtext: - raise ValueError("Invalid response received from server at %s:%s" % - (host, port)) - return certtext - - -if __name__ == "__main__": - if len(sys.argv) < 2: - sys.stderr.write( - "Usage: %s HOSTNAME:PORTNUMBER [, HOSTNAME:PORTNUMBER...]\n" % - sys.argv[0]) - sys.exit(1) - for arg in sys.argv[1:]: - host, port = arg.split(":") - sys.stdout.buffer.write(fetch_server_certificate(host, int(port))) - sys.exit(0) diff --git a/Tools/scripts/google.py b/Tools/scripts/google.py deleted file mode 100755 index 82fb2871885d03..00000000000000 --- a/Tools/scripts/google.py +++ /dev/null @@ -1,25 +0,0 @@ -#! /usr/bin/env python3 - -"""Script to search with Google - -Usage: - python3 google.py [search terms] -""" - -import sys -import urllib.parse -import webbrowser - - -def main(args): - def quote(arg): - if ' ' in arg: - arg = '"%s"' % arg - return urllib.parse.quote_plus(arg) - - qstring = '+'.join(quote(arg) for arg in args) - url = urllib.parse.urljoin('https://www.google.com/search', '?q=' + qstring) - webbrowser.open(url) - -if __name__ == '__main__': - main(sys.argv[1:]) diff --git a/Tools/scripts/highlight.py b/Tools/scripts/highlight.py deleted file mode 100755 index 9272fee4ee3b9d..00000000000000 --- a/Tools/scripts/highlight.py +++ /dev/null @@ -1,265 +0,0 @@ -#!/usr/bin/env python3 -'''Add syntax highlighting to Python source code''' - -__author__ = 'Raymond Hettinger' - -import builtins -import functools -import html as html_module -import keyword -import re -import tokenize - -#### Analyze Python Source ################################# - -def is_builtin(s): - 'Return True if s is the name of a builtin' - return hasattr(builtins, s) - -def combine_range(lines, start, end): - 'Join content from a range of lines between start and end' - (srow, scol), (erow, ecol) = start, end - if srow == erow: - return lines[srow-1][scol:ecol], end - rows = [lines[srow-1][scol:]] + lines[srow: erow-1] + [lines[erow-1][:ecol]] - return ''.join(rows), end - -def analyze_python(source): - '''Generate and classify chunks of Python for syntax highlighting. - Yields tuples in the form: (category, categorized_text). - ''' - lines = source.splitlines(True) - lines.append('') - readline = functools.partial(next, iter(lines), '') - kind = tok_str = '' - tok_type = tokenize.COMMENT - written = (1, 0) - for tok in tokenize.generate_tokens(readline): - prev_tok_type, prev_tok_str = tok_type, tok_str - tok_type, tok_str, (srow, scol), (erow, ecol), logical_lineno = tok - kind = '' - if tok_type == tokenize.COMMENT: - kind = 'comment' - elif tok_type == tokenize.OP and tok_str[:1] not in '{}[](),.:;@': - kind = 'operator' - elif tok_type == tokenize.STRING: - kind = 'string' - if prev_tok_type == tokenize.INDENT or scol==0: - kind = 'docstring' - elif tok_type == tokenize.NAME: - if tok_str in ('def', 'class', 'import', 'from'): - kind = 'definition' - elif prev_tok_str in ('def', 'class'): - kind = 'defname' - elif keyword.iskeyword(tok_str): - kind = 'keyword' - elif is_builtin(tok_str) and prev_tok_str != '.': - kind = 'builtin' - if kind: - text, written = combine_range(lines, written, (srow, scol)) - yield '', text - text, written = tok_str, (erow, ecol) - yield kind, text - line_upto_token, written = combine_range(lines, written, (erow, ecol)) - yield '', line_upto_token - -#### Raw Output ########################################### - -def raw_highlight(classified_text): - 'Straight text display of text classifications' - result = [] - for kind, text in classified_text: - result.append('%15s: %r\n' % (kind or 'plain', text)) - return ''.join(result) - -#### ANSI Output ########################################### - -default_ansi = { - 'comment': ('\033[0;31m', '\033[0m'), - 'string': ('\033[0;32m', '\033[0m'), - 'docstring': ('\033[0;32m', '\033[0m'), - 'keyword': ('\033[0;33m', '\033[0m'), - 'builtin': ('\033[0;35m', '\033[0m'), - 'definition': ('\033[0;33m', '\033[0m'), - 'defname': ('\033[0;34m', '\033[0m'), - 'operator': ('\033[0;33m', '\033[0m'), -} - -def ansi_highlight(classified_text, colors=default_ansi): - 'Add syntax highlighting to source code using ANSI escape sequences' - # http://en.wikipedia.org/wiki/ANSI_escape_code - result = [] - for kind, text in classified_text: - opener, closer = colors.get(kind, ('', '')) - result += [opener, text, closer] - return ''.join(result) - -#### HTML Output ########################################### - -def html_highlight(classified_text,opener='
\n', closer='
\n'): - 'Convert classified text to an HTML fragment' - result = [opener] - for kind, text in classified_text: - if kind: - result.append('' % kind) - result.append(html_module.escape(text)) - if kind: - result.append('') - result.append(closer) - return ''.join(result) - -default_css = { - '.comment': '{color: crimson;}', - '.string': '{color: forestgreen;}', - '.docstring': '{color: forestgreen; font-style:italic;}', - '.keyword': '{color: darkorange;}', - '.builtin': '{color: purple;}', - '.definition': '{color: darkorange; font-weight:bold;}', - '.defname': '{color: blue;}', - '.operator': '{color: brown;}', -} - -default_html = '''\ - - - - - {title} - - - -{body} - - -''' - -def build_html_page(classified_text, title='python', - css=default_css, html=default_html): - 'Create a complete HTML page with colorized source code' - css_str = '\n'.join(['%s %s' % item for item in css.items()]) - result = html_highlight(classified_text) - title = html_module.escape(title) - return html.format(title=title, css=css_str, body=result) - -#### LaTeX Output ########################################## - -default_latex_commands = { - 'comment': r'{\color{red}#1}', - 'string': r'{\color{ForestGreen}#1}', - 'docstring': r'{\emph{\color{ForestGreen}#1}}', - 'keyword': r'{\color{orange}#1}', - 'builtin': r'{\color{purple}#1}', - 'definition': r'{\color{orange}#1}', - 'defname': r'{\color{blue}#1}', - 'operator': r'{\color{brown}#1}', -} - -default_latex_document = r''' -\documentclass{article} -\usepackage{alltt} -\usepackage{upquote} -\usepackage{color} -\usepackage[usenames,dvipsnames]{xcolor} -\usepackage[cm]{fullpage} -%(macros)s -\begin{document} -\center{\LARGE{%(title)s}} -\begin{alltt} -%(body)s -\end{alltt} -\end{document} -''' - -def alltt_escape(s): - 'Replace backslash and braces with their escaped equivalents' - xlat = {'{': r'\{', '}': r'\}', '\\': r'\textbackslash{}'} - return re.sub(r'[\\{}]', lambda mo: xlat[mo.group()], s) - -def latex_highlight(classified_text, title = 'python', - commands = default_latex_commands, - document = default_latex_document): - 'Create a complete LaTeX document with colorized source code' - macros = '\n'.join(r'\newcommand{\py%s}[1]{%s}' % c for c in commands.items()) - result = [] - for kind, text in classified_text: - if kind: - result.append(r'\py%s{' % kind) - result.append(alltt_escape(text)) - if kind: - result.append('}') - return default_latex_document % dict(title=title, macros=macros, body=''.join(result)) - - -if __name__ == '__main__': - import argparse - import os.path - import sys - import textwrap - import webbrowser - - parser = argparse.ArgumentParser( - description = 'Add syntax highlighting to Python source code', - formatter_class=argparse.RawDescriptionHelpFormatter, - epilog = textwrap.dedent(''' - examples: - - # Show syntax highlighted code in the terminal window - $ ./highlight.py myfile.py - - # Colorize myfile.py and display in a browser - $ ./highlight.py -b myfile.py - - # Create an HTML section to embed in an existing webpage - ./highlight.py -s myfile.py - - # Create a complete HTML file - $ ./highlight.py -c myfile.py > myfile.html - - # Create a PDF using LaTeX - $ ./highlight.py -l myfile.py | pdflatex - - ''')) - parser.add_argument('sourcefile', metavar = 'SOURCEFILE', - help = 'file containing Python sourcecode') - parser.add_argument('-b', '--browser', action = 'store_true', - help = 'launch a browser to show results') - parser.add_argument('-c', '--complete', action = 'store_true', - help = 'build a complete html webpage') - parser.add_argument('-l', '--latex', action = 'store_true', - help = 'build a LaTeX document') - parser.add_argument('-r', '--raw', action = 'store_true', - help = 'raw parse of categorized text') - parser.add_argument('-s', '--section', action = 'store_true', - help = 'show an HTML section rather than a complete webpage') - args = parser.parse_args() - - if args.section and (args.browser or args.complete): - parser.error('The -s/--section option is incompatible with ' - 'the -b/--browser or -c/--complete options') - - sourcefile = args.sourcefile - with open(sourcefile) as f: - source = f.read() - classified_text = analyze_python(source) - - if args.raw: - encoded = raw_highlight(classified_text) - elif args.complete or args.browser: - encoded = build_html_page(classified_text, title=sourcefile) - elif args.section: - encoded = html_highlight(classified_text) - elif args.latex: - encoded = latex_highlight(classified_text, title=sourcefile) - else: - encoded = ansi_highlight(classified_text) - - if args.browser: - htmlfile = os.path.splitext(os.path.basename(sourcefile))[0] + '.html' - with open(htmlfile, 'w') as f: - f.write(encoded) - webbrowser.open('file://' + os.path.abspath(htmlfile)) - else: - sys.stdout.write(encoded) diff --git a/Tools/scripts/ifdef.py b/Tools/scripts/ifdef.py deleted file mode 100755 index 22249b2d0af5dc..00000000000000 --- a/Tools/scripts/ifdef.py +++ /dev/null @@ -1,111 +0,0 @@ -#! /usr/bin/env python3 - -# Selectively preprocess #ifdef / #ifndef statements. -# Usage: -# ifdef [-Dname] ... [-Uname] ... [file] ... -# -# This scans the file(s), looking for #ifdef and #ifndef preprocessor -# commands that test for one of the names mentioned in the -D and -U -# options. On standard output it writes a copy of the input file(s) -# minus those code sections that are suppressed by the selected -# combination of defined/undefined symbols. The #if(n)def/#else/#else -# lines themselves (if the #if(n)def tests for one of the mentioned -# names) are removed as well. - -# Features: Arbitrary nesting of recognized and unrecognized -# preprocessor statements works correctly. Unrecognized #if* commands -# are left in place, so it will never remove too much, only too -# little. It does accept whitespace around the '#' character. - -# Restrictions: There should be no comments or other symbols on the -# #if(n)def lines. The effect of #define/#undef commands in the input -# file or in included files is not taken into account. Tests using -# #if and the defined() pseudo function are not recognized. The #elif -# command is not recognized. Improperly nesting is not detected. -# Lines that look like preprocessor commands but which are actually -# part of comments or string literals will be mistaken for -# preprocessor commands. - -import sys -import getopt - -defs = [] -undefs = [] - -def main(): - opts, args = getopt.getopt(sys.argv[1:], 'D:U:') - for o, a in opts: - if o == '-D': - defs.append(a) - if o == '-U': - undefs.append(a) - if not args: - args = ['-'] - for filename in args: - if filename == '-': - process(sys.stdin, sys.stdout) - else: - with open(filename) as f: - process(f, sys.stdout) - -def process(fpi, fpo): - keywords = ('if', 'ifdef', 'ifndef', 'else', 'endif') - ok = 1 - stack = [] - while 1: - line = fpi.readline() - if not line: break - while line[-2:] == '\\\n': - nextline = fpi.readline() - if not nextline: break - line = line + nextline - tmp = line.strip() - if tmp[:1] != '#': - if ok: fpo.write(line) - continue - tmp = tmp[1:].strip() - words = tmp.split() - keyword = words[0] - if keyword not in keywords: - if ok: fpo.write(line) - continue - if keyword in ('ifdef', 'ifndef') and len(words) == 2: - if keyword == 'ifdef': - ko = 1 - else: - ko = 0 - word = words[1] - if word in defs: - stack.append((ok, ko, word)) - if not ko: ok = 0 - elif word in undefs: - stack.append((ok, not ko, word)) - if ko: ok = 0 - else: - stack.append((ok, -1, word)) - if ok: fpo.write(line) - elif keyword == 'if': - stack.append((ok, -1, '')) - if ok: fpo.write(line) - elif keyword == 'else' and stack: - s_ok, s_ko, s_word = stack[-1] - if s_ko < 0: - if ok: fpo.write(line) - else: - s_ko = not s_ko - ok = s_ok - if not s_ko: ok = 0 - stack[-1] = s_ok, s_ko, s_word - elif keyword == 'endif' and stack: - s_ok, s_ko, s_word = stack[-1] - if s_ko < 0: - if ok: fpo.write(line) - del stack[-1] - ok = s_ok - else: - sys.stderr.write('Unknown keyword %s\n' % keyword) - if stack: - sys.stderr.write('stack: %s\n' % stack) - -if __name__ == '__main__': - main() diff --git a/Tools/scripts/import_diagnostics.py b/Tools/scripts/import_diagnostics.py deleted file mode 100755 index c907221d049c52..00000000000000 --- a/Tools/scripts/import_diagnostics.py +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env python3 -"""Miscellaneous diagnostics for the import system""" - -import sys -import argparse -from pprint import pprint - -def _dump_state(args): - print(sys.version) - for name in args.attributes: - print("sys.{}:".format(name)) - pprint(getattr(sys, name)) - -def _add_dump_args(cmd): - cmd.add_argument("attributes", metavar="ATTR", nargs="+", - help="sys module attribute to display") - -COMMANDS = ( - ("dump", "Dump import state", _dump_state, _add_dump_args), -) - -def _make_parser(): - parser = argparse.ArgumentParser() - sub = parser.add_subparsers(title="Commands") - for name, description, implementation, add_args in COMMANDS: - cmd = sub.add_parser(name, help=description) - cmd.set_defaults(command=implementation) - add_args(cmd) - return parser - -def main(args): - parser = _make_parser() - args = parser.parse_args(args) - return args.command(args) - -if __name__ == "__main__": - sys.exit(main(sys.argv[1:])) diff --git a/Tools/scripts/lfcr.py b/Tools/scripts/lfcr.py deleted file mode 100755 index bf8fe1c245ef4c..00000000000000 --- a/Tools/scripts/lfcr.py +++ /dev/null @@ -1,24 +0,0 @@ -#! /usr/bin/env python3 - -"Replace LF with CRLF in argument files. Print names of changed files." - -import sys, re, os - -def main(): - for filename in sys.argv[1:]: - if os.path.isdir(filename): - print(filename, "Directory!") - continue - with open(filename, "rb") as f: - data = f.read() - if b'\0' in data: - print(filename, "Binary!") - continue - newdata = re.sub(b"\r?\n", b"\r\n", data) - if newdata != data: - print(filename) - with open(filename, "wb") as f: - f.write(newdata) - -if __name__ == '__main__': - main() diff --git a/Tools/scripts/linktree.py b/Tools/scripts/linktree.py deleted file mode 100755 index e83f198593ade1..00000000000000 --- a/Tools/scripts/linktree.py +++ /dev/null @@ -1,80 +0,0 @@ -#! /usr/bin/env python3 - -# linktree -# -# Make a copy of a directory tree with symbolic links to all files in the -# original tree. -# All symbolic links go to a special symbolic link at the top, so you -# can easily fix things if the original source tree moves. -# See also "mkreal". -# -# usage: mklinks oldtree newtree - -import sys, os - -LINK = '.LINK' # Name of special symlink at the top. - -debug = 0 - -def main(): - if not 3 <= len(sys.argv) <= 4: - print('usage:', sys.argv[0], 'oldtree newtree [linkto]') - return 2 - oldtree, newtree = sys.argv[1], sys.argv[2] - if len(sys.argv) > 3: - link = sys.argv[3] - link_may_fail = 1 - else: - link = LINK - link_may_fail = 0 - if not os.path.isdir(oldtree): - print(oldtree + ': not a directory') - return 1 - try: - os.mkdir(newtree, 0o777) - except OSError as msg: - print(newtree + ': cannot mkdir:', msg) - return 1 - linkname = os.path.join(newtree, link) - try: - os.symlink(os.path.join(os.pardir, oldtree), linkname) - except OSError as msg: - if not link_may_fail: - print(linkname + ': cannot symlink:', msg) - return 1 - else: - print(linkname + ': warning: cannot symlink:', msg) - linknames(oldtree, newtree, link) - return 0 - -def linknames(old, new, link): - if debug: print('linknames', (old, new, link)) - try: - names = os.listdir(old) - except OSError as msg: - print(old + ': warning: cannot listdir:', msg) - return - for name in names: - if name not in (os.curdir, os.pardir): - oldname = os.path.join(old, name) - linkname = os.path.join(link, name) - newname = os.path.join(new, name) - if debug > 1: print(oldname, newname, linkname) - if os.path.isdir(oldname) and \ - not os.path.islink(oldname): - try: - os.mkdir(newname, 0o777) - ok = 1 - except: - print(newname + \ - ': warning: cannot mkdir:', msg) - ok = 0 - if ok: - linkname = os.path.join(os.pardir, - linkname) - linknames(oldname, newname, linkname) - else: - os.symlink(linkname, newname) - -if __name__ == '__main__': - sys.exit(main()) diff --git a/Tools/scripts/lll.py b/Tools/scripts/lll.py deleted file mode 100755 index 1b48eac8aad879..00000000000000 --- a/Tools/scripts/lll.py +++ /dev/null @@ -1,27 +0,0 @@ -#! /usr/bin/env python3 - -# Find symbolic links and show where they point to. -# Arguments are directories to search; default is current directory. -# No recursion. -# (This is a totally different program from "findsymlinks.py"!) - -import sys, os - -def lll(dirname): - for name in os.listdir(dirname): - if name not in (os.curdir, os.pardir): - full = os.path.join(dirname, name) - if os.path.islink(full): - print(name, '->', os.readlink(full)) -def main(args): - if not args: args = [os.curdir] - first = 1 - for arg in args: - if len(args) > 1: - if not first: print() - first = 0 - print(arg + ':') - lll(arg) - -if __name__ == '__main__': - main(sys.argv[1:]) diff --git a/Tools/scripts/mailerdaemon.py b/Tools/scripts/mailerdaemon.py deleted file mode 100755 index 9595ee4a01533e..00000000000000 --- a/Tools/scripts/mailerdaemon.py +++ /dev/null @@ -1,246 +0,0 @@ -#!/usr/bin/env python3 -"""Classes to parse mailer-daemon messages.""" - -import calendar -import email.message -import re -import os -import sys - - -class Unparseable(Exception): - pass - - -class ErrorMessage(email.message.Message): - def __init__(self): - email.message.Message.__init__(self) - self.sub = '' - - def is_warning(self): - sub = self.get('Subject') - if not sub: - return 0 - sub = sub.lower() - if sub.startswith('waiting mail'): - return 1 - if 'warning' in sub: - return 1 - self.sub = sub - return 0 - - def get_errors(self): - for p in EMPARSERS: - self.rewindbody() - try: - return p(self.fp, self.sub) - except Unparseable: - pass - raise Unparseable - -# List of re's or tuples of re's. -# If a re, it should contain at least a group (?P...) which -# should refer to the email address. The re can also contain a group -# (?P...) which should refer to the reason (error message). -# If no reason is present, the emparse_list_reason list is used to -# find a reason. -# If a tuple, the tuple should contain 2 re's. The first re finds a -# location, the second re is repeated one or more times to find -# multiple email addresses. The second re is matched (not searched) -# where the previous match ended. -# The re's are compiled using the re module. -emparse_list_list = [ - 'error: (?Punresolvable): (?P.+)', - ('----- The following addresses had permanent fatal errors -----\n', - '(?P[^ \n].*)\n( .*\n)?'), - 'remote execution.*\n.*rmail (?P.+)', - ('The following recipients did not receive your message:\n\n', - ' +(?P.*)\n(The following recipients did not receive your message:\n\n)?'), - '------- Failure Reasons --------\n\n(?P.*)\n(?P.*)', - '^<(?P.*)>:\n(?P.*)', - '^(?PUser mailbox exceeds allowed size): (?P.+)', - '^5\\d{2} <(?P[^\n>]+)>\\.\\.\\. (?P.+)', - '^Original-Recipient: rfc822;(?P.*)', - '^did not reach the following recipient\\(s\\):\n\n(?P.*) on .*\n +(?P.*)', - '^ <(?P[^\n>]+)> \\.\\.\\. (?P.*)', - '^Report on your message to: (?P.*)\nReason: (?P.*)', - '^Your message was not delivered to +(?P.*)\n +for the following reason:\n +(?P.*)', - '^ was not +(?P[^ \n].*?) *\n.*\n.*\n.*\n because:.*\n +(?P[^ \n].*?) *\n', - ] -# compile the re's in the list and store them in-place. -for i in range(len(emparse_list_list)): - x = emparse_list_list[i] - if isinstance(x, str): - x = re.compile(x, re.MULTILINE) - else: - xl = [] - for x in x: - xl.append(re.compile(x, re.MULTILINE)) - x = tuple(xl) - del xl - emparse_list_list[i] = x - del x -del i - -# list of re's used to find reasons (error messages). -# if a string, "<>" is replaced by a copy of the email address. -# The expressions are searched for in order. After the first match, -# no more expressions are searched for. So, order is important. -emparse_list_reason = [ - r'^5\d{2} <>\.\.\. (?P.*)', - r'<>\.\.\. (?P.*)', - re.compile(r'^<<< 5\d{2} (?P.*)', re.MULTILINE), - re.compile('===== stderr was =====\nrmail: (?P.*)'), - re.compile('^Diagnostic-Code: (?P.*)', re.MULTILINE), - ] -emparse_list_from = re.compile('^From:', re.IGNORECASE|re.MULTILINE) -def emparse_list(fp, sub): - data = fp.read() - res = emparse_list_from.search(data) - if res is None: - from_index = len(data) - else: - from_index = res.start(0) - errors = [] - emails = [] - reason = None - for regexp in emparse_list_list: - if isinstance(regexp, tuple): - res = regexp[0].search(data, 0, from_index) - if res is not None: - try: - reason = res.group('reason') - except IndexError: - pass - while 1: - res = regexp[1].match(data, res.end(0), from_index) - if res is None: - break - emails.append(res.group('email')) - break - else: - res = regexp.search(data, 0, from_index) - if res is not None: - emails.append(res.group('email')) - try: - reason = res.group('reason') - except IndexError: - pass - break - if not emails: - raise Unparseable - if not reason: - reason = sub - if reason[:15] == 'returned mail: ': - reason = reason[15:] - for regexp in emparse_list_reason: - if isinstance(regexp, str): - for i in range(len(emails)-1,-1,-1): - email = emails[i] - exp = re.compile(re.escape(email).join(regexp.split('<>')), re.MULTILINE) - res = exp.search(data) - if res is not None: - errors.append(' '.join((email.strip()+': '+res.group('reason')).split())) - del emails[i] - continue - res = regexp.search(data) - if res is not None: - reason = res.group('reason') - break - for email in emails: - errors.append(' '.join((email.strip()+': '+reason).split())) - return errors - -EMPARSERS = [emparse_list] - -def sort_numeric(a, b): - a = int(a) - b = int(b) - if a < b: - return -1 - elif a > b: - return 1 - else: - return 0 - -def parsedir(dir, modify): - os.chdir(dir) - pat = re.compile('^[0-9]*$') - errordict = {} - errorfirst = {} - errorlast = {} - nok = nwarn = nbad = 0 - - # find all numeric file names and sort them - files = list(filter(lambda fn, pat=pat: pat.match(fn) is not None, os.listdir('.'))) - files.sort(sort_numeric) - - for fn in files: - # Lets try to parse the file. - fp = open(fn) - m = email.message_from_file(fp, _class=ErrorMessage) - sender = m.getaddr('From') - print('%s\t%-40s\t'%(fn, sender[1]), end=' ') - - if m.is_warning(): - fp.close() - print('warning only') - nwarn = nwarn + 1 - if modify: - os.rename(fn, ','+fn) -## os.unlink(fn) - continue - - try: - errors = m.get_errors() - except Unparseable: - print('** Not parseable') - nbad = nbad + 1 - fp.close() - continue - print(len(errors), 'errors') - - # Remember them - for e in errors: - try: - mm, dd = m.getdate('date')[1:1+2] - date = '%s %02d' % (calendar.month_abbr[mm], dd) - except: - date = '??????' - if e not in errordict: - errordict[e] = 1 - errorfirst[e] = '%s (%s)' % (fn, date) - else: - errordict[e] = errordict[e] + 1 - errorlast[e] = '%s (%s)' % (fn, date) - - fp.close() - nok = nok + 1 - if modify: - os.rename(fn, ','+fn) -## os.unlink(fn) - - print('--------------') - print(nok, 'files parsed,',nwarn,'files warning-only,', end=' ') - print(nbad,'files unparseable') - print('--------------') - list = [] - for e in errordict.keys(): - list.append((errordict[e], errorfirst[e], errorlast[e], e)) - list.sort() - for num, first, last, e in list: - print('%d %s - %s\t%s' % (num, first, last, e)) - -def main(): - modify = 0 - if len(sys.argv) > 1 and sys.argv[1] == '-d': - modify = 1 - del sys.argv[1] - if len(sys.argv) > 1: - for folder in sys.argv[1:]: - parsedir(folder, modify) - else: - parsedir('/ufs/jack/Mail/errorsinbox', modify) - -if __name__ == '__main__' or sys.argv[0] == __name__: - main() diff --git a/Tools/scripts/make_ctype.py b/Tools/scripts/make_ctype.py deleted file mode 100755 index afee1c583330a5..00000000000000 --- a/Tools/scripts/make_ctype.py +++ /dev/null @@ -1,94 +0,0 @@ -#!/usr/bin/env python3 -"""Script that generates the ctype.h-replacement in stringobject.c.""" - -NAMES = ("LOWER", "UPPER", "ALPHA", "DIGIT", "XDIGIT", "ALNUM", "SPACE") - -print(""" -#define FLAG_LOWER 0x01 -#define FLAG_UPPER 0x02 -#define FLAG_ALPHA (FLAG_LOWER|FLAG_UPPER) -#define FLAG_DIGIT 0x04 -#define FLAG_ALNUM (FLAG_ALPHA|FLAG_DIGIT) -#define FLAG_SPACE 0x08 -#define FLAG_XDIGIT 0x10 - -static unsigned int ctype_table[256] = {""") - -for i in range(128): - c = chr(i) - flags = [] - for name in NAMES: - if name in ("ALPHA", "ALNUM"): - continue - if name == "XDIGIT": - method = lambda: c.isdigit() or c.upper() in "ABCDEF" - else: - method = getattr(c, "is" + name.lower()) - if method(): - flags.append("FLAG_" + name) - rc = repr(c) - if c == '\v': - rc = "'\\v'" - elif c == '\f': - rc = "'\\f'" - if not flags: - print(" 0, /* 0x%x %s */" % (i, rc)) - else: - print(" %s, /* 0x%x %s */" % ("|".join(flags), i, rc)) - -for i in range(128, 256, 16): - print(" %s," % ", ".join(16*["0"])) - -print("};") -print("") - -for name in NAMES: - print("#define IS%s(c) (ctype_table[Py_CHARMASK(c)] & FLAG_%s)" % - (name, name)) - -print("") - -for name in NAMES: - name = "is" + name.lower() - print("#undef %s" % name) - print("#define %s(c) undefined_%s(c)" % (name, name)) - -print(""" -static unsigned char ctype_tolower[256] = {""") - -for i in range(0, 256, 8): - values = [] - for i in range(i, i+8): - if i < 128: - c = chr(i) - if c.isupper(): - i = ord(c.lower()) - values.append("0x%02x" % i) - print(" %s," % ", ".join(values)) - -print("};") - -print(""" -static unsigned char ctype_toupper[256] = {""") - -for i in range(0, 256, 8): - values = [] - for i in range(i, i+8): - if i < 128: - c = chr(i) - if c.islower(): - i = ord(c.upper()) - values.append("0x%02x" % i) - print(" %s," % ", ".join(values)) - -print("};") - -print(""" -#define TOLOWER(c) (ctype_tolower[Py_CHARMASK(c)]) -#define TOUPPER(c) (ctype_toupper[Py_CHARMASK(c)]) - -#undef tolower -#define tolower(c) undefined_tolower(c) -#undef toupper -#define toupper(c) undefined_toupper(c) -""") diff --git a/Tools/scripts/md5sum.py b/Tools/scripts/md5sum.py deleted file mode 100755 index f910576377aa3d..00000000000000 --- a/Tools/scripts/md5sum.py +++ /dev/null @@ -1,93 +0,0 @@ -#! /usr/bin/env python3 - -"""Python utility to print MD5 checksums of argument files. -""" - - -bufsize = 8096 -fnfilter = None -rmode = 'rb' - -usage = """ -usage: md5sum.py [-b] [-t] [-l] [-s bufsize] [file ...] --b : read files in binary mode (default) --t : read files in text mode (you almost certainly don't want this!) --l : print last pathname component only --s bufsize: read buffer size (default %d) -file ... : files to sum; '-' or no files means stdin -""" % bufsize - -import io -import sys -import os -import getopt -from hashlib import md5 - -def sum(*files): - sts = 0 - if files and isinstance(files[-1], io.IOBase): - out, files = files[-1], files[:-1] - else: - out = sys.stdout - if len(files) == 1 and not isinstance(files[0], str): - files = files[0] - for f in files: - if isinstance(f, str): - if f == '-': - sts = printsumfp(sys.stdin, '', out) or sts - else: - sts = printsum(f, out) or sts - else: - sts = sum(f, out) or sts - return sts - -def printsum(filename, out=sys.stdout): - try: - fp = open(filename, rmode) - except IOError as msg: - sys.stderr.write('%s: Can\'t open: %s\n' % (filename, msg)) - return 1 - with fp: - if fnfilter: - filename = fnfilter(filename) - sts = printsumfp(fp, filename, out) - return sts - -def printsumfp(fp, filename, out=sys.stdout): - m = md5() - try: - while 1: - data = fp.read(bufsize) - if not data: - break - if isinstance(data, str): - data = data.encode(fp.encoding) - m.update(data) - except IOError as msg: - sys.stderr.write('%s: I/O error: %s\n' % (filename, msg)) - return 1 - out.write('%s %s\n' % (m.hexdigest(), filename)) - return 0 - -def main(args = sys.argv[1:], out=sys.stdout): - global fnfilter, rmode, bufsize - try: - opts, args = getopt.getopt(args, 'blts:') - except getopt.error as msg: - sys.stderr.write('%s: %s\n%s' % (sys.argv[0], msg, usage)) - return 2 - for o, a in opts: - if o == '-l': - fnfilter = os.path.basename - elif o == '-b': - rmode = 'rb' - elif o == '-t': - rmode = 'r' - elif o == '-s': - bufsize = int(a) - if not args: - args = ['-'] - return sum(args, out) - -if __name__ == '__main__' or __name__ == sys.argv[0]: - sys.exit(main(sys.argv[1:], sys.stdout)) diff --git a/Tools/scripts/mkreal.py b/Tools/scripts/mkreal.py deleted file mode 100755 index f169da43fe116c..00000000000000 --- a/Tools/scripts/mkreal.py +++ /dev/null @@ -1,65 +0,0 @@ -#! /usr/bin/env python3 - -# mkreal -# -# turn a symlink to a directory into a real directory - -import sys -import os -from stat import * - -join = os.path.join - -error = 'mkreal error' - -BUFSIZE = 32*1024 - -def mkrealfile(name): - st = os.stat(name) # Get the mode - mode = S_IMODE(st[ST_MODE]) - linkto = os.readlink(name) # Make sure again it's a symlink - with open(name, 'rb') as f_in: # This ensures it's a file - os.unlink(name) - with open(name, 'wb') as f_out: - while 1: - buf = f_in.read(BUFSIZE) - if not buf: break - f_out.write(buf) - os.chmod(name, mode) - -def mkrealdir(name): - st = os.stat(name) # Get the mode - mode = S_IMODE(st[ST_MODE]) - linkto = os.readlink(name) - files = os.listdir(name) - os.unlink(name) - os.mkdir(name, mode) - os.chmod(name, mode) - linkto = join(os.pardir, linkto) - # - for filename in files: - if filename not in (os.curdir, os.pardir): - os.symlink(join(linkto, filename), join(name, filename)) - -def main(): - sys.stdout = sys.stderr - progname = os.path.basename(sys.argv[0]) - if progname == '-c': progname = 'mkreal' - args = sys.argv[1:] - if not args: - print('usage:', progname, 'path ...') - sys.exit(2) - status = 0 - for name in args: - if not os.path.islink(name): - print(progname+':', name+':', 'not a symlink') - status = 1 - else: - if os.path.isdir(name): - mkrealdir(name) - else: - mkrealfile(name) - sys.exit(status) - -if __name__ == '__main__': - main() diff --git a/Tools/scripts/objgraph.py b/Tools/scripts/objgraph.py deleted file mode 100755 index add41e692c0338..00000000000000 --- a/Tools/scripts/objgraph.py +++ /dev/null @@ -1,211 +0,0 @@ -#! /usr/bin/env python3 - -# objgraph -# -# Read "nm -o" input of a set of libraries or modules and print various -# interesting listings, such as: -# -# - which names are used but not defined in the set (and used where), -# - which names are defined in the set (and where), -# - which modules use which other modules, -# - which modules are used by which other modules. -# -# Usage: objgraph [-cdu] [file] ... -# -c: print callers per objectfile -# -d: print callees per objectfile -# -u: print usage of undefined symbols -# If none of -cdu is specified, all are assumed. -# Use "nm -o" to generate the input -# e.g.: nm -o /lib/libc.a | objgraph - - -import sys -import os -import getopt -import re - -# Types of symbols. -# -definitions = 'TRGDSBAEC' -externals = 'UV' -ignore = 'Nntrgdsbavuc' - -# Regular expression to parse "nm -o" output. -# -matcher = re.compile('(.*):\t?........ (.) (.*)$') - -# Store "item" in "dict" under "key". -# The dictionary maps keys to lists of items. -# If there is no list for the key yet, it is created. -# -def store(dict, key, item): - if key in dict: - dict[key].append(item) - else: - dict[key] = [item] - -# Return a flattened version of a list of strings: the concatenation -# of its elements with intervening spaces. -# -def flat(list): - s = '' - for item in list: - s = s + ' ' + item - return s[1:] - -# Global variables mapping defined/undefined names to files and back. -# -file2undef = {} -def2file = {} -file2def = {} -undef2file = {} - -# Read one input file and merge the data into the tables. -# Argument is an open file. -# -def readinput(fp): - while 1: - s = fp.readline() - if not s: - break - # If you get any output from this line, - # it is probably caused by an unexpected input line: - if matcher.search(s) < 0: s; continue # Shouldn't happen - (ra, rb), (r1a, r1b), (r2a, r2b), (r3a, r3b) = matcher.regs[:4] - fn, name, type = s[r1a:r1b], s[r3a:r3b], s[r2a:r2b] - if type in definitions: - store(def2file, name, fn) - store(file2def, fn, name) - elif type in externals: - store(file2undef, fn, name) - store(undef2file, name, fn) - elif not type in ignore: - print(fn + ':' + name + ': unknown type ' + type) - -# Print all names that were undefined in some module and where they are -# defined. -# -def printcallee(): - flist = sorted(file2undef.keys()) - for filename in flist: - print(filename + ':') - elist = file2undef[filename] - elist.sort() - for ext in elist: - if len(ext) >= 8: - tabs = '\t' - else: - tabs = '\t\t' - if ext not in def2file: - print('\t' + ext + tabs + ' *undefined') - else: - print('\t' + ext + tabs + flat(def2file[ext])) - -# Print for each module the names of the other modules that use it. -# -def printcaller(): - files = sorted(file2def.keys()) - for filename in files: - callers = [] - for label in file2def[filename]: - if label in undef2file: - callers = callers + undef2file[label] - if callers: - callers.sort() - print(filename + ':') - lastfn = '' - for fn in callers: - if fn != lastfn: - print('\t' + fn) - lastfn = fn - else: - print(filename + ': unused') - -# Print undefined names and where they are used. -# -def printundef(): - undefs = {} - for filename in list(file2undef.keys()): - for ext in file2undef[filename]: - if ext not in def2file: - store(undefs, ext, filename) - elist = sorted(undefs.keys()) - for ext in elist: - print(ext + ':') - flist = sorted(undefs[ext]) - for filename in flist: - print('\t' + filename) - -# Print warning messages about names defined in more than one file. -# -def warndups(): - savestdout = sys.stdout - sys.stdout = sys.stderr - names = sorted(def2file.keys()) - for name in names: - if len(def2file[name]) > 1: - print('warning:', name, 'multiply defined:', end=' ') - print(flat(def2file[name])) - sys.stdout = savestdout - -# Main program -# -def main(): - try: - optlist, args = getopt.getopt(sys.argv[1:], 'cdu') - except getopt.error: - sys.stdout = sys.stderr - print('Usage:', os.path.basename(sys.argv[0]), end=' ') - print('[-cdu] [file] ...') - print('-c: print callers per objectfile') - print('-d: print callees per objectfile') - print('-u: print usage of undefined symbols') - print('If none of -cdu is specified, all are assumed.') - print('Use "nm -o" to generate the input') - print('e.g.: nm -o /lib/libc.a | objgraph') - return 1 - optu = optc = optd = 0 - for opt, void in optlist: - if opt == '-u': - optu = 1 - elif opt == '-c': - optc = 1 - elif opt == '-d': - optd = 1 - if optu == optc == optd == 0: - optu = optc = optd = 1 - if not args: - args = ['-'] - for filename in args: - if filename == '-': - readinput(sys.stdin) - else: - with open(filename) as f: - readinput(f) - # - warndups() - # - more = (optu + optc + optd > 1) - if optd: - if more: - print('---------------All callees------------------') - printcallee() - if optu: - if more: - print('---------------Undefined callees------------') - printundef() - if optc: - if more: - print('---------------All Callers------------------') - printcaller() - return 0 - -# Call the main program. -# Use its return value as exit status. -# Catch interrupts to avoid stack trace. -# -if __name__ == '__main__': - try: - sys.exit(main()) - except KeyboardInterrupt: - sys.exit(1) diff --git a/Tools/scripts/parse_html5_entities.py b/Tools/scripts/parse_html5_entities.py deleted file mode 100755 index 1e5bdad2165548..00000000000000 --- a/Tools/scripts/parse_html5_entities.py +++ /dev/null @@ -1,114 +0,0 @@ -#!/usr/bin/env python3 -""" -Utility for parsing HTML5 entity definitions available from: - - https://html.spec.whatwg.org/entities.json - https://html.spec.whatwg.org/multipage/named-characters.html - -The page now contains the following note: - - "This list is static and will not be expanded or changed in the future." - -Written by Ezio Melotti and Iuliia Proskurnia. -""" - -import os -import sys -import json -from urllib.request import urlopen -from html.entities import html5 - -PAGE_URL = 'https://html.spec.whatwg.org/multipage/named-characters.html' -ENTITIES_URL = 'https://html.spec.whatwg.org/entities.json' -HTML5_SECTION_START = '# HTML5 named character references' - -def get_json(url): - """Download the json file from the url and returns a decoded object.""" - with urlopen(url) as f: - data = f.read().decode('utf-8') - return json.loads(data) - -def create_dict(entities): - """Create the html5 dict from the decoded json object.""" - new_html5 = {} - for name, value in entities.items(): - new_html5[name.lstrip('&')] = value['characters'] - return new_html5 - -def compare_dicts(old, new): - """Compare the old and new dicts and print the differences.""" - added = new.keys() - old.keys() - if added: - print('{} entitie(s) have been added:'.format(len(added))) - for name in sorted(added): - print(' {!r}: {!r}'.format(name, new[name])) - removed = old.keys() - new.keys() - if removed: - print('{} entitie(s) have been removed:'.format(len(removed))) - for name in sorted(removed): - print(' {!r}: {!r}'.format(name, old[name])) - changed = set() - for name in (old.keys() & new.keys()): - if old[name] != new[name]: - changed.add((name, old[name], new[name])) - if changed: - print('{} entitie(s) have been modified:'.format(len(changed))) - for item in sorted(changed): - print(' {!r}: {!r} -> {!r}'.format(*item)) - -def write_items(entities, file=sys.stdout): - """Write the items of the dictionary in the specified file.""" - # The keys in the generated dictionary should be sorted - # in a case-insensitive way, however, when two keys are equal, - # the uppercase version should come first so that the result - # looks like: ['Aacute', 'aacute', 'Aacute;', 'aacute;', ...] - # To do this we first sort in a case-sensitive way (so all the - # uppercase chars come first) and then sort with key=str.lower. - # Since the sorting is stable the uppercase keys will eventually - # be before their equivalent lowercase version. - keys = sorted(entities.keys()) - keys = sorted(keys, key=str.lower) - print(HTML5_SECTION_START, file=file) - print(f'# Generated by {sys.argv[0]!r}\n' - f'# from {ENTITIES_URL} and\n' - f'# {PAGE_URL}.\n' - f'# Map HTML5 named character references to the ' - f'equivalent Unicode character(s).', file=file) - print('html5 = {', file=file) - for name in keys: - print(f' {name!r}: {entities[name]!a},', file=file) - print('}', file=file) - - -if __name__ == '__main__': - # without args print a diff between html.entities.html5 and new_html5 - # with --create print the new html5 dict - # with --patch patch the Lib/html/entities.py file - new_html5 = create_dict(get_json(ENTITIES_URL)) - if '--create' in sys.argv: - write_items(new_html5) - elif '--patch' in sys.argv: - fname = 'Lib/html/entities.py' - temp_fname = fname + '.temp' - with open(fname) as f1, open(temp_fname, 'w') as f2: - skip = False - for line in f1: - if line.startswith(HTML5_SECTION_START): - write_items(new_html5, file=f2) - skip = True - continue - if skip: - # skip the old items until the } - if line.startswith('}'): - skip = False - continue - f2.write(line) - os.remove(fname) - os.rename(temp_fname, fname) - else: - if html5 == new_html5: - print('The current dictionary is updated.') - else: - compare_dicts(html5, new_html5) - print('Run "./python {0} --patch" to update Lib/html/entities.html ' - 'or "./python {0} --create" to see the generated ' 'dictionary.'.format(__file__)) diff --git a/Tools/scripts/pdeps.py b/Tools/scripts/pdeps.py deleted file mode 100755 index ab0040f48e2e79..00000000000000 --- a/Tools/scripts/pdeps.py +++ /dev/null @@ -1,164 +0,0 @@ -#! /usr/bin/env python3 - -# pdeps -# -# Find dependencies between a bunch of Python modules. -# -# Usage: -# pdeps file1.py file2.py ... -# -# Output: -# Four tables separated by lines like '--- Closure ---': -# 1) Direct dependencies, listing which module imports which other modules -# 2) The inverse of (1) -# 3) Indirect dependencies, or the closure of the above -# 4) The inverse of (3) -# -# To do: -# - command line options to select output type -# - option to automatically scan the Python library for referenced modules -# - option to limit output to particular modules - - -import sys -import re -import os - - -# Main program -# -def main(): - args = sys.argv[1:] - if not args: - print('usage: pdeps file.py file.py ...') - return 2 - # - table = {} - for arg in args: - process(arg, table) - # - print('--- Uses ---') - printresults(table) - # - print('--- Used By ---') - inv = inverse(table) - printresults(inv) - # - print('--- Closure of Uses ---') - reach = closure(table) - printresults(reach) - # - print('--- Closure of Used By ---') - invreach = inverse(reach) - printresults(invreach) - # - return 0 - - -# Compiled regular expressions to search for import statements -# -m_import = re.compile('^[ \t]*from[ \t]+([^ \t]+)[ \t]+') -m_from = re.compile('^[ \t]*import[ \t]+([^#]+)') - - -# Collect data from one file -# -def process(filename, table): - with open(filename, encoding='utf-8') as fp: - mod = os.path.basename(filename) - if mod[-3:] == '.py': - mod = mod[:-3] - table[mod] = list = [] - while 1: - line = fp.readline() - if not line: break - while line[-1:] == '\\': - nextline = fp.readline() - if not nextline: break - line = line[:-1] + nextline - m_found = m_import.match(line) or m_from.match(line) - if m_found: - (a, b), (a1, b1) = m_found.regs[:2] - else: continue - words = line[a1:b1].split(',') - # print '#', line, words - for word in words: - word = word.strip() - if word not in list: - list.append(word) - - -# Compute closure (this is in fact totally general) -# -def closure(table): - modules = list(table.keys()) - # - # Initialize reach with a copy of table - # - reach = {} - for mod in modules: - reach[mod] = table[mod][:] - # - # Iterate until no more change - # - change = 1 - while change: - change = 0 - for mod in modules: - for mo in reach[mod]: - if mo in modules: - for m in reach[mo]: - if m not in reach[mod]: - reach[mod].append(m) - change = 1 - # - return reach - - -# Invert a table (this is again totally general). -# All keys of the original table are made keys of the inverse, -# so there may be empty lists in the inverse. -# -def inverse(table): - inv = {} - for key in table.keys(): - if key not in inv: - inv[key] = [] - for item in table[key]: - store(inv, item, key) - return inv - - -# Store "item" in "dict" under "key". -# The dictionary maps keys to lists of items. -# If there is no list for the key yet, it is created. -# -def store(dict, key, item): - if key in dict: - dict[key].append(item) - else: - dict[key] = [item] - - -# Tabulate results neatly -# -def printresults(table): - modules = sorted(table.keys()) - maxlen = 0 - for mod in modules: maxlen = max(maxlen, len(mod)) - for mod in modules: - list = sorted(table[mod]) - print(mod.ljust(maxlen), ':', end=' ') - if mod in list: - print('(*)', end=' ') - for ref in list: - print(ref, end=' ') - print() - - -# Call main and honor exit status -if __name__ == '__main__': - try: - sys.exit(main()) - except KeyboardInterrupt: - sys.exit(1) diff --git a/Tools/scripts/pickle2db.py b/Tools/scripts/pickle2db.py deleted file mode 100755 index b5b6571863282f..00000000000000 --- a/Tools/scripts/pickle2db.py +++ /dev/null @@ -1,147 +0,0 @@ -#!/usr/bin/env python3 - -""" -Synopsis: %(prog)s [-h|-b|-g|-r|-a|-d] [ picklefile ] dbfile - -Read the given picklefile as a series of key/value pairs and write to a new -database. If the database already exists, any contents are deleted. The -optional flags indicate the type of the output database: - - -a - open using dbm (open any supported format) - -b - open as bsddb btree file - -d - open as dbm.ndbm file - -g - open as dbm.gnu file - -h - open as bsddb hash file - -r - open as bsddb recno file - -The default is hash. If a pickle file is named it is opened for read -access. If no pickle file is named, the pickle input is read from standard -input. - -Note that recno databases can only contain integer keys, so you can't dump a -hash or btree database using db2pickle.py and reconstitute it to a recno -database with %(prog)s unless your keys are integers. - -""" - -import getopt -try: - import bsddb -except ImportError: - bsddb = None -try: - import dbm.ndbm as dbm -except ImportError: - dbm = None -try: - import dbm.gnu as gdbm -except ImportError: - gdbm = None -try: - import dbm.ndbm as anydbm -except ImportError: - anydbm = None -import sys -try: - import pickle as pickle -except ImportError: - import pickle - -prog = sys.argv[0] - -def usage(): - sys.stderr.write(__doc__ % globals()) - -def main(args): - try: - opts, args = getopt.getopt(args, "hbrdag", - ["hash", "btree", "recno", "dbm", "anydbm", - "gdbm"]) - except getopt.error: - usage() - return 1 - - if len(args) == 0 or len(args) > 2: - usage() - return 1 - elif len(args) == 1: - pfile = sys.stdin - dbfile = args[0] - else: - try: - pfile = open(args[0], 'rb') - except IOError: - sys.stderr.write("Unable to open %s\n" % args[0]) - return 1 - dbfile = args[1] - - dbopen = None - for opt, arg in opts: - if opt in ("-h", "--hash"): - try: - dbopen = bsddb.hashopen - except AttributeError: - sys.stderr.write("bsddb module unavailable.\n") - return 1 - elif opt in ("-b", "--btree"): - try: - dbopen = bsddb.btopen - except AttributeError: - sys.stderr.write("bsddb module unavailable.\n") - return 1 - elif opt in ("-r", "--recno"): - try: - dbopen = bsddb.rnopen - except AttributeError: - sys.stderr.write("bsddb module unavailable.\n") - return 1 - elif opt in ("-a", "--anydbm"): - try: - dbopen = anydbm.open - except AttributeError: - sys.stderr.write("dbm module unavailable.\n") - return 1 - elif opt in ("-g", "--gdbm"): - try: - dbopen = gdbm.open - except AttributeError: - sys.stderr.write("dbm.gnu module unavailable.\n") - return 1 - elif opt in ("-d", "--dbm"): - try: - dbopen = dbm.open - except AttributeError: - sys.stderr.write("dbm.ndbm module unavailable.\n") - return 1 - if dbopen is None: - if bsddb is None: - sys.stderr.write("bsddb module unavailable - ") - sys.stderr.write("must specify dbtype.\n") - return 1 - else: - dbopen = bsddb.hashopen - - try: - db = dbopen(dbfile, 'c') - except bsddb.error: - sys.stderr.write("Unable to open %s. " % dbfile) - sys.stderr.write("Check for format or version mismatch.\n") - return 1 - else: - for k in list(db.keys()): - del db[k] - - while 1: - try: - (key, val) = pickle.load(pfile) - except EOFError: - break - db[key] = val - - db.close() - pfile.close() - - return 0 - -if __name__ == "__main__": - sys.exit(main(sys.argv[1:])) diff --git a/Tools/scripts/pindent.py b/Tools/scripts/pindent.py deleted file mode 100755 index 33334204a4d455..00000000000000 --- a/Tools/scripts/pindent.py +++ /dev/null @@ -1,506 +0,0 @@ -#! /usr/bin/env python3 - -# This file contains a class and a main program that perform three -# related (though complimentary) formatting operations on Python -# programs. When called as "pindent -c", it takes a valid Python -# program as input and outputs a version augmented with block-closing -# comments. When called as "pindent -d", it assumes its input is a -# Python program with block-closing comments and outputs a commentless -# version. When called as "pindent -r" it assumes its input is a -# Python program with block-closing comments but with its indentation -# messed up, and outputs a properly indented version. - -# A "block-closing comment" is a comment of the form '# end ' -# where is the keyword that opened the block. If the -# opening keyword is 'def' or 'class', the function or class name may -# be repeated in the block-closing comment as well. Here is an -# example of a program fully augmented with block-closing comments: - -# def foobar(a, b): -# if a == b: -# a = a+1 -# elif a < b: -# b = b-1 -# if b > a: a = a-1 -# # end if -# else: -# print 'oops!' -# # end if -# # end def foobar - -# Note that only the last part of an if...elif...else... block needs a -# block-closing comment; the same is true for other compound -# statements (e.g. try...except). Also note that "short-form" blocks -# like the second 'if' in the example must be closed as well; -# otherwise the 'else' in the example would be ambiguous (remember -# that indentation is not significant when interpreting block-closing -# comments). - -# The operations are idempotent (i.e. applied to their own output -# they yield an identical result). Running first "pindent -c" and -# then "pindent -r" on a valid Python program produces a program that -# is semantically identical to the input (though its indentation may -# be different). Running "pindent -e" on that output produces a -# program that only differs from the original in indentation. - -# Other options: -# -s stepsize: set the indentation step size (default 8) -# -t tabsize : set the number of spaces a tab character is worth (default 8) -# -e : expand TABs into spaces -# file ... : input file(s) (default standard input) -# The results always go to standard output - -# Caveats: -# - comments ending in a backslash will be mistaken for continued lines -# - continuations using backslash are always left unchanged -# - continuations inside parentheses are not extra indented by -r -# but must be indented for -c to work correctly (this breaks -# idempotency!) -# - continued lines inside triple-quoted strings are totally garbled - -# Secret feature: -# - On input, a block may also be closed with an "end statement" -- -# this is a block-closing comment without the '#' sign. - -# Possible improvements: -# - check syntax based on transitions in 'next' table -# - better error reporting -# - better error recovery -# - check identifier after class/def - -# The following wishes need a more complete tokenization of the source: -# - Don't get fooled by comments ending in backslash -# - reindent continuation lines indicated by backslash -# - handle continuation lines inside parentheses/braces/brackets -# - handle triple quoted strings spanning lines -# - realign comments -# - optionally do much more thorough reformatting, a la C indent - -# Defaults -STEPSIZE = 8 -TABSIZE = 8 -EXPANDTABS = False - -import io -import re -import sys - -next = {} -next['if'] = next['elif'] = 'elif', 'else', 'end' -next['while'] = next['for'] = 'else', 'end' -next['try'] = 'except', 'finally' -next['except'] = 'except', 'else', 'finally', 'end' -next['else'] = next['finally'] = next['with'] = \ - next['def'] = next['class'] = 'end' -next['end'] = () -start = 'if', 'while', 'for', 'try', 'with', 'def', 'class' - -class PythonIndenter: - - def __init__(self, fpi = sys.stdin, fpo = sys.stdout, - indentsize = STEPSIZE, tabsize = TABSIZE, expandtabs = EXPANDTABS): - self.fpi = fpi - self.fpo = fpo - self.indentsize = indentsize - self.tabsize = tabsize - self.lineno = 0 - self.expandtabs = expandtabs - self._write = fpo.write - self.kwprog = re.compile( - r'^(?:\s|\\\n)*(?P[a-z]+)' - r'((?:\s|\\\n)+(?P[a-zA-Z_]\w*))?' - r'[^\w]') - self.endprog = re.compile( - r'^(?:\s|\\\n)*#?\s*end\s+(?P[a-z]+)' - r'(\s+(?P[a-zA-Z_]\w*))?' - r'[^\w]') - self.wsprog = re.compile(r'^[ \t]*') - # end def __init__ - - def write(self, line): - if self.expandtabs: - self._write(line.expandtabs(self.tabsize)) - else: - self._write(line) - # end if - # end def write - - def readline(self): - line = self.fpi.readline() - if line: self.lineno += 1 - # end if - return line - # end def readline - - def error(self, fmt, *args): - if args: fmt = fmt % args - # end if - sys.stderr.write('Error at line %d: %s\n' % (self.lineno, fmt)) - self.write('### %s ###\n' % fmt) - # end def error - - def getline(self): - line = self.readline() - while line[-2:] == '\\\n': - line2 = self.readline() - if not line2: break - # end if - line += line2 - # end while - return line - # end def getline - - def putline(self, line, indent): - tabs, spaces = divmod(indent*self.indentsize, self.tabsize) - i = self.wsprog.match(line).end() - line = line[i:] - if line[:1] not in ('\n', '\r', ''): - line = '\t'*tabs + ' '*spaces + line - # end if - self.write(line) - # end def putline - - def reformat(self): - stack = [] - while True: - line = self.getline() - if not line: break # EOF - # end if - m = self.endprog.match(line) - if m: - kw = 'end' - kw2 = m.group('kw') - if not stack: - self.error('unexpected end') - elif stack.pop()[0] != kw2: - self.error('unmatched end') - # end if - self.putline(line, len(stack)) - continue - # end if - m = self.kwprog.match(line) - if m: - kw = m.group('kw') - if kw in start: - self.putline(line, len(stack)) - stack.append((kw, kw)) - continue - # end if - if kw in next and stack: - self.putline(line, len(stack)-1) - kwa, kwb = stack[-1] - stack[-1] = kwa, kw - continue - # end if - # end if - self.putline(line, len(stack)) - # end while - if stack: - self.error('unterminated keywords') - for kwa, kwb in stack: - self.write('\t%s\n' % kwa) - # end for - # end if - # end def reformat - - def delete(self): - begin_counter = 0 - end_counter = 0 - while True: - line = self.getline() - if not line: break # EOF - # end if - m = self.endprog.match(line) - if m: - end_counter += 1 - continue - # end if - m = self.kwprog.match(line) - if m: - kw = m.group('kw') - if kw in start: - begin_counter += 1 - # end if - # end if - self.write(line) - # end while - if begin_counter - end_counter < 0: - sys.stderr.write('Warning: input contained more end tags than expected\n') - elif begin_counter - end_counter > 0: - sys.stderr.write('Warning: input contained less end tags than expected\n') - # end if - # end def delete - - def complete(self): - stack = [] - todo = [] - currentws = thisid = firstkw = lastkw = topid = '' - while True: - line = self.getline() - i = self.wsprog.match(line).end() - m = self.endprog.match(line) - if m: - thiskw = 'end' - endkw = m.group('kw') - thisid = m.group('id') - else: - m = self.kwprog.match(line) - if m: - thiskw = m.group('kw') - if thiskw not in next: - thiskw = '' - # end if - if thiskw in ('def', 'class'): - thisid = m.group('id') - else: - thisid = '' - # end if - elif line[i:i+1] in ('\n', '#'): - todo.append(line) - continue - else: - thiskw = '' - # end if - # end if - indentws = line[:i] - indent = len(indentws.expandtabs(self.tabsize)) - current = len(currentws.expandtabs(self.tabsize)) - while indent < current: - if firstkw: - if topid: - s = '# end %s %s\n' % ( - firstkw, topid) - else: - s = '# end %s\n' % firstkw - # end if - self.write(currentws + s) - firstkw = lastkw = '' - # end if - currentws, firstkw, lastkw, topid = stack.pop() - current = len(currentws.expandtabs(self.tabsize)) - # end while - if indent == current and firstkw: - if thiskw == 'end': - if endkw != firstkw: - self.error('mismatched end') - # end if - firstkw = lastkw = '' - elif not thiskw or thiskw in start: - if topid: - s = '# end %s %s\n' % ( - firstkw, topid) - else: - s = '# end %s\n' % firstkw - # end if - self.write(currentws + s) - firstkw = lastkw = topid = '' - # end if - # end if - if indent > current: - stack.append((currentws, firstkw, lastkw, topid)) - if thiskw and thiskw not in start: - # error - thiskw = '' - # end if - currentws, firstkw, lastkw, topid = \ - indentws, thiskw, thiskw, thisid - # end if - if thiskw: - if thiskw in start: - firstkw = lastkw = thiskw - topid = thisid - else: - lastkw = thiskw - # end if - # end if - for l in todo: self.write(l) - # end for - todo = [] - if not line: break - # end if - self.write(line) - # end while - # end def complete -# end class PythonIndenter - -# Simplified user interface -# - xxx_filter(input, output): read and write file objects -# - xxx_string(s): take and return string object -# - xxx_file(filename): process file in place, return true iff changed - -def complete_filter(input = sys.stdin, output = sys.stdout, - stepsize = STEPSIZE, tabsize = TABSIZE, expandtabs = EXPANDTABS): - pi = PythonIndenter(input, output, stepsize, tabsize, expandtabs) - pi.complete() -# end def complete_filter - -def delete_filter(input= sys.stdin, output = sys.stdout, - stepsize = STEPSIZE, tabsize = TABSIZE, expandtabs = EXPANDTABS): - pi = PythonIndenter(input, output, stepsize, tabsize, expandtabs) - pi.delete() -# end def delete_filter - -def reformat_filter(input = sys.stdin, output = sys.stdout, - stepsize = STEPSIZE, tabsize = TABSIZE, expandtabs = EXPANDTABS): - pi = PythonIndenter(input, output, stepsize, tabsize, expandtabs) - pi.reformat() -# end def reformat_filter - -def complete_string(source, stepsize = STEPSIZE, tabsize = TABSIZE, expandtabs = EXPANDTABS): - input = io.StringIO(source) - output = io.StringIO() - pi = PythonIndenter(input, output, stepsize, tabsize, expandtabs) - pi.complete() - return output.getvalue() -# end def complete_string - -def delete_string(source, stepsize = STEPSIZE, tabsize = TABSIZE, expandtabs = EXPANDTABS): - input = io.StringIO(source) - output = io.StringIO() - pi = PythonIndenter(input, output, stepsize, tabsize, expandtabs) - pi.delete() - return output.getvalue() -# end def delete_string - -def reformat_string(source, stepsize = STEPSIZE, tabsize = TABSIZE, expandtabs = EXPANDTABS): - input = io.StringIO(source) - output = io.StringIO() - pi = PythonIndenter(input, output, stepsize, tabsize, expandtabs) - pi.reformat() - return output.getvalue() -# end def reformat_string - -def make_backup(filename): - import os, os.path - backup = filename + '~' - if os.path.lexists(backup): - try: - os.remove(backup) - except OSError: - print("Can't remove backup %r" % (backup,), file=sys.stderr) - # end try - # end if - try: - os.rename(filename, backup) - except OSError: - print("Can't rename %r to %r" % (filename, backup), file=sys.stderr) - # end try -# end def make_backup - -def complete_file(filename, stepsize = STEPSIZE, tabsize = TABSIZE, expandtabs = EXPANDTABS): - with open(filename, 'r') as f: - source = f.read() - # end with - result = complete_string(source, stepsize, tabsize, expandtabs) - if source == result: return 0 - # end if - make_backup(filename) - with open(filename, 'w') as f: - f.write(result) - # end with - return 1 -# end def complete_file - -def delete_file(filename, stepsize = STEPSIZE, tabsize = TABSIZE, expandtabs = EXPANDTABS): - with open(filename, 'r') as f: - source = f.read() - # end with - result = delete_string(source, stepsize, tabsize, expandtabs) - if source == result: return 0 - # end if - make_backup(filename) - with open(filename, 'w') as f: - f.write(result) - # end with - return 1 -# end def delete_file - -def reformat_file(filename, stepsize = STEPSIZE, tabsize = TABSIZE, expandtabs = EXPANDTABS): - with open(filename, 'r') as f: - source = f.read() - # end with - result = reformat_string(source, stepsize, tabsize, expandtabs) - if source == result: return 0 - # end if - make_backup(filename) - with open(filename, 'w') as f: - f.write(result) - # end with - return 1 -# end def reformat_file - -# Test program when called as a script - -usage = """ -usage: pindent (-c|-d|-r) [-s stepsize] [-t tabsize] [-e] [file] ... --c : complete a correctly indented program (add #end directives) --d : delete #end directives --r : reformat a completed program (use #end directives) --s stepsize: indentation step (default %(STEPSIZE)d) --t tabsize : the worth in spaces of a tab (default %(TABSIZE)d) --e : expand TABs into spaces (default OFF) -[file] ... : files are changed in place, with backups in file~ -If no files are specified or a single - is given, -the program acts as a filter (reads stdin, writes stdout). -""" % vars() - -def error_both(op1, op2): - sys.stderr.write('Error: You can not specify both '+op1+' and -'+op2[0]+' at the same time\n') - sys.stderr.write(usage) - sys.exit(2) -# end def error_both - -def test(): - import getopt - try: - opts, args = getopt.getopt(sys.argv[1:], 'cdrs:t:e') - except getopt.error as msg: - sys.stderr.write('Error: %s\n' % msg) - sys.stderr.write(usage) - sys.exit(2) - # end try - action = None - stepsize = STEPSIZE - tabsize = TABSIZE - expandtabs = EXPANDTABS - for o, a in opts: - if o == '-c': - if action: error_both(o, action) - # end if - action = 'complete' - elif o == '-d': - if action: error_both(o, action) - # end if - action = 'delete' - elif o == '-r': - if action: error_both(o, action) - # end if - action = 'reformat' - elif o == '-s': - stepsize = int(a) - elif o == '-t': - tabsize = int(a) - elif o == '-e': - expandtabs = True - # end if - # end for - if not action: - sys.stderr.write( - 'You must specify -c(omplete), -d(elete) or -r(eformat)\n') - sys.stderr.write(usage) - sys.exit(2) - # end if - if not args or args == ['-']: - action = eval(action + '_filter') - action(sys.stdin, sys.stdout, stepsize, tabsize, expandtabs) - else: - action = eval(action + '_file') - for filename in args: - action(filename, stepsize, tabsize, expandtabs) - # end for - # end if -# end def test - -if __name__ == '__main__': - test() -# end if diff --git a/Tools/scripts/pysource.py b/Tools/scripts/pysource.py deleted file mode 100755 index 69e8e0df4ff080..00000000000000 --- a/Tools/scripts/pysource.py +++ /dev/null @@ -1,130 +0,0 @@ -#!/usr/bin/env python3 - -"""\ -List python source files. - -There are three functions to check whether a file is a Python source, listed -here with increasing complexity: - -- has_python_ext() checks whether a file name ends in '.py[w]'. -- look_like_python() checks whether the file is not binary and either has - the '.py[w]' extension or the first line contains the word 'python'. -- can_be_compiled() checks whether the file can be compiled by compile(). - -The file also must be of appropriate size - not bigger than a megabyte. - -walk_python_files() recursively lists all Python files under the given directories. -""" -__author__ = "Oleg Broytmann, Georg Brandl" - -__all__ = ["has_python_ext", "looks_like_python", "can_be_compiled", "walk_python_files"] - - -import os, re - -binary_re = re.compile(br'[\x00-\x08\x0E-\x1F\x7F]') - -debug = False - -def print_debug(msg): - if debug: print(msg) - - -def _open(fullpath): - try: - size = os.stat(fullpath).st_size - except OSError as err: # Permission denied - ignore the file - print_debug("%s: permission denied: %s" % (fullpath, err)) - return None - - if size > 1024*1024: # too big - print_debug("%s: the file is too big: %d bytes" % (fullpath, size)) - return None - - try: - return open(fullpath, "rb") - except IOError as err: # Access denied, or a special file - ignore it - print_debug("%s: access denied: %s" % (fullpath, err)) - return None - -def has_python_ext(fullpath): - return fullpath.endswith(".py") or fullpath.endswith(".pyw") - -def looks_like_python(fullpath): - infile = _open(fullpath) - if infile is None: - return False - - with infile: - line = infile.readline() - - if binary_re.search(line): - # file appears to be binary - print_debug("%s: appears to be binary" % fullpath) - return False - - if fullpath.endswith(".py") or fullpath.endswith(".pyw"): - return True - elif b"python" in line: - # disguised Python script (e.g. CGI) - return True - - return False - -def can_be_compiled(fullpath): - infile = _open(fullpath) - if infile is None: - return False - - with infile: - code = infile.read() - - try: - compile(code, fullpath, "exec") - except Exception as err: - print_debug("%s: cannot compile: %s" % (fullpath, err)) - return False - - return True - - -def walk_python_files(paths, is_python=looks_like_python, exclude_dirs=None): - """\ - Recursively yield all Python source files below the given paths. - - paths: a list of files and/or directories to be checked. - is_python: a function that takes a file name and checks whether it is a - Python source file - exclude_dirs: a list of directory base names that should be excluded in - the search - """ - if exclude_dirs is None: - exclude_dirs=[] - - for path in paths: - print_debug("testing: %s" % path) - if os.path.isfile(path): - if is_python(path): - yield path - elif os.path.isdir(path): - print_debug(" it is a directory") - for dirpath, dirnames, filenames in os.walk(path): - for exclude in exclude_dirs: - if exclude in dirnames: - dirnames.remove(exclude) - for filename in filenames: - fullpath = os.path.join(dirpath, filename) - print_debug("testing: %s" % fullpath) - if is_python(fullpath): - yield fullpath - else: - print_debug(" unknown type") - - -if __name__ == "__main__": - # Two simple examples/tests - for fullpath in walk_python_files(['.']): - print(fullpath) - print("----------") - for fullpath in walk_python_files(['.'], is_python=can_be_compiled): - print(fullpath) diff --git a/Tools/scripts/reindent-rst.py b/Tools/scripts/reindent-rst.py deleted file mode 100755 index 25608af66ac2fc..00000000000000 --- a/Tools/scripts/reindent-rst.py +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env python3 - -# Make a reST file compliant to our pre-commit hook. -# Currently just remove trailing whitespace. - -import sys - -import patchcheck - -def main(argv=sys.argv): - patchcheck.normalize_docs_whitespace(argv[1:]) - -if __name__ == '__main__': - sys.exit(main()) diff --git a/Tools/scripts/rgrep.py b/Tools/scripts/rgrep.py deleted file mode 100755 index c39bf93aad3bb2..00000000000000 --- a/Tools/scripts/rgrep.py +++ /dev/null @@ -1,67 +0,0 @@ -#! /usr/bin/env python3 - -"""Reverse grep. - -Usage: rgrep [-i] pattern file -""" - -import sys -import re -import getopt - - -def main(): - bufsize = 64 * 1024 - reflags = 0 - opts, args = getopt.getopt(sys.argv[1:], "i") - for o, a in opts: - if o == '-i': - reflags = reflags | re.IGNORECASE - if len(args) < 2: - usage("not enough arguments") - if len(args) > 2: - usage("exactly one file argument required") - pattern, filename = args - try: - prog = re.compile(pattern, reflags) - except re.error as msg: - usage("error in regular expression: %s" % msg) - try: - f = open(filename) - except IOError as msg: - usage("can't open %r: %s" % (filename, msg), 1) - with f: - f.seek(0, 2) - pos = f.tell() - leftover = None - while pos > 0: - size = min(pos, bufsize) - pos = pos - size - f.seek(pos) - buffer = f.read(size) - lines = buffer.split("\n") - del buffer - if leftover is None: - if not lines[-1]: - del lines[-1] - else: - lines[-1] = lines[-1] + leftover - if pos > 0: - leftover = lines[0] - del lines[0] - else: - leftover = None - for line in reversed(lines): - if prog.search(line): - print(line) - - -def usage(msg, code=2): - sys.stdout = sys.stderr - print(msg) - print(__doc__) - sys.exit(code) - - -if __name__ == '__main__': - main() diff --git a/Tools/scripts/suff.py b/Tools/scripts/suff.py deleted file mode 100755 index 0eea0d7548611c..00000000000000 --- a/Tools/scripts/suff.py +++ /dev/null @@ -1,26 +0,0 @@ -#! /usr/bin/env python3 - -# suff -# -# show different suffixes amongst arguments - -import sys - - -def main(): - files = sys.argv[1:] - suffixes = {} - for filename in files: - suff = getsuffix(filename) - suffixes.setdefault(suff, []).append(filename) - for suff, filenames in sorted(suffixes.items()): - print(repr(suff), len(filenames)) - - -def getsuffix(filename): - name, sep, suff = filename.rpartition('.') - return sep + suff if sep else '' - - -if __name__ == '__main__': - main() diff --git a/Tools/scripts/texi2html.py b/Tools/scripts/texi2html.py deleted file mode 100755 index c06d812ab3fbc0..00000000000000 --- a/Tools/scripts/texi2html.py +++ /dev/null @@ -1,2071 +0,0 @@ -#! /usr/bin/env python3 - -# Convert GNU texinfo files into HTML, one file per node. -# Based on Texinfo 2.14. -# Usage: texi2html [-d] [-d] [-c] inputfile outputdirectory -# The input file must be a complete texinfo file, e.g. emacs.texi. -# This creates many files (one per info node) in the output directory, -# overwriting existing files of the same name. All files created have -# ".html" as their extension. - - -# XXX To do: -# - handle @comment*** correctly -# - handle @xref {some words} correctly -# - handle @ftable correctly (items aren't indexed?) -# - handle @itemx properly -# - handle @exdent properly -# - add links directly to the proper line from indices -# - check against the definitive list of @-cmds; we still miss (among others): -# - @defindex (hard) -# - @c(omment) in the middle of a line (rarely used) -# - @this* (not really needed, only used in headers anyway) -# - @today{} (ever used outside title page?) - -# More consistent handling of chapters/sections/etc. -# Lots of documentation -# Many more options: -# -top designate top node -# -links customize which types of links are included -# -split split at chapters or sections instead of nodes -# -name Allow different types of filename handling. Non unix systems -# will have problems with long node names -# ... -# Support the most recent texinfo version and take a good look at HTML 3.0 -# More debugging output (customizable) and more flexible error handling -# How about icons ? - -# rpyron 2002-05-07 -# Robert Pyron -# 1. BUGFIX: In function makefile(), strip blanks from the nodename. -# This is necessary to match the behavior of parser.makeref() and -# parser.do_node(). -# 2. BUGFIX fixed KeyError in end_ifset (well, I may have just made -# it go away, rather than fix it) -# 3. BUGFIX allow @menu and menu items inside @ifset or @ifclear -# 4. Support added for: -# @uref URL reference -# @image image file reference (see note below) -# @multitable output an HTML table -# @vtable -# 5. Partial support for accents, to match MAKEINFO output -# 6. I added a new command-line option, '-H basename', to specify -# HTML Help output. This will cause three files to be created -# in the current directory: -# `basename`.hhp HTML Help Workshop project file -# `basename`.hhc Contents file for the project -# `basename`.hhk Index file for the project -# When fed into HTML Help Workshop, the resulting file will be -# named `basename`.chm. -# 7. A new class, HTMLHelp, to accomplish item 6. -# 8. Various calls to HTMLHelp functions. -# A NOTE ON IMAGES: Just as 'outputdirectory' must exist before -# running this program, all referenced images must already exist -# in outputdirectory. - -import os -import sys -import string -import re - -MAGIC = '\\input texinfo' - -cmprog = re.compile('^@([a-z]+)([ \t]|$)') # Command (line-oriented) -blprog = re.compile('^[ \t]*$') # Blank line -kwprog = re.compile('@[a-z]+') # Keyword (embedded, usually - # with {} args) -spprog = re.compile('[\n@{}&<>]') # Special characters in - # running text - # - # menu item (Yuck!) -miprog = re.compile(r'^\* ([^:]*):(:|[ \t]*([^\t,\n.]+)([^ \t\n]*))[ \t\n]*') -# 0 1 1 2 3 34 42 0 -# ----- ---------- --------- -# -|----------------------------- -# ----------------------------------------------------- - - - - -class HTMLNode: - """Some of the parser's functionality is separated into this class. - - A Node accumulates its contents, takes care of links to other Nodes - and saves itself when it is finished and all links are resolved. - """ - - DOCTYPE = '' - - type = 0 - cont = '' - epilogue = '\n' - - def __init__(self, dir, name, topname, title, next, prev, up): - self.dirname = dir - self.name = name - if topname: - self.topname = topname - else: - self.topname = name - self.title = title - self.next = next - self.prev = prev - self.up = up - self.lines = [] - - def write(self, *lines): - for line in lines: - self.lines.append(line) - - def flush(self): - with open(self.dirname + '/' + makefile(self.name), 'w') as fp: - fp.write(self.prologue) - fp.write(self.text) - fp.write(self.epilogue) - - def link(self, label, nodename, rel=None, rev=None): - if nodename: - if nodename.lower() == '(dir)': - addr = '../dir.html' - title = '' - else: - addr = makefile(nodename) - title = ' TITLE="%s"' % nodename - self.write(label, ': ', nodename, ' \n') - - def finalize(self): - length = len(self.lines) - self.text = ''.join(self.lines) - self.lines = [] - self.open_links() - self.output_links() - self.close_links() - links = ''.join(self.lines) - self.lines = [] - self.prologue = ( - self.DOCTYPE + - '\n\n' - ' \n' - ' ' + self.title + '\n' - ' \n' - ' \n' - ' \n' - '\n' + - links) - if length > 20: - self.epilogue = '

\n%s\n' % links - - def open_links(self): - self.write('

\n') - - def close_links(self): - self.write('
\n') - - def output_links(self): - if self.cont != self.next: - self.link(' Cont', self.cont) - self.link(' Next', self.next, rel='Next') - self.link(' Prev', self.prev, rel='Previous') - self.link(' Up', self.up, rel='Up') - if self.name != self.topname: - self.link(' Top', self.topname) - - -class HTML3Node(HTMLNode): - - DOCTYPE = '' - - def open_links(self): - self.write('
\n
\n') - - def close_links(self): - self.write('
\n
\n') - - -class TexinfoParser: - - COPYRIGHT_SYMBOL = "©" - FN_ID_PATTERN = "(%(id)s)" - FN_SOURCE_PATTERN = '' \ - + FN_ID_PATTERN + '' - FN_TARGET_PATTERN = '' \ - + FN_ID_PATTERN + '\n%(text)s

\n' - FN_HEADER = '\n

\n

\n' \ - 'Footnotes\n

' - - - Node = HTMLNode - - # Initialize an instance - def __init__(self): - self.unknown = {} # statistics about unknown @-commands - self.filenames = {} # Check for identical filenames - self.debugging = 0 # larger values produce more output - self.print_headers = 0 # always print headers? - self.nodefp = None # open file we're writing to - self.nodelineno = 0 # Linenumber relative to node - self.links = None # Links from current node - self.savetext = None # If not None, save text head instead - self.savestack = [] # If not None, save text head instead - self.htmlhelp = None # html help data - self.dirname = 'tmp' # directory where files are created - self.includedir = '.' # directory to search @include files - self.nodename = '' # name of current node - self.topname = '' # name of top node (first node seen) - self.title = '' # title of this whole Texinfo tree - self.resetindex() # Reset all indices - self.contents = [] # Reset table of contents - self.numbering = [] # Reset section numbering counters - self.nofill = 0 # Normal operation: fill paragraphs - self.values={'html': 1} # Names that should be parsed in ifset - self.stackinfo={} # Keep track of state in the stack - # XXX The following should be reset per node?! - self.footnotes = [] # Reset list of footnotes - self.itemarg = None # Reset command used by @item - self.itemnumber = None # Reset number for @item in @enumerate - self.itemindex = None # Reset item index name - self.node = None - self.nodestack = [] - self.cont = 0 - self.includedepth = 0 - - # Set htmlhelp helper class - def sethtmlhelp(self, htmlhelp): - self.htmlhelp = htmlhelp - - # Set (output) directory name - def setdirname(self, dirname): - self.dirname = dirname - - # Set include directory name - def setincludedir(self, includedir): - self.includedir = includedir - - # Parse the contents of an entire file - def parse(self, fp): - line = fp.readline() - lineno = 1 - while line and (line[0] == '%' or blprog.match(line)): - line = fp.readline() - lineno = lineno + 1 - if line[:len(MAGIC)] != MAGIC: - raise SyntaxError('file does not begin with %r' % (MAGIC,)) - self.parserest(fp, lineno) - - # Parse the contents of a file, not expecting a MAGIC header - def parserest(self, fp, initial_lineno): - lineno = initial_lineno - self.done = 0 - self.skip = 0 - self.stack = [] - accu = [] - while not self.done: - line = fp.readline() - self.nodelineno = self.nodelineno + 1 - if not line: - if accu: - if not self.skip: self.process(accu) - accu = [] - if initial_lineno > 0: - print('*** EOF before @bye') - break - lineno = lineno + 1 - mo = cmprog.match(line) - if mo: - a, b = mo.span(1) - cmd = line[a:b] - if cmd in ('noindent', 'refill'): - accu.append(line) - else: - if accu: - if not self.skip: - self.process(accu) - accu = [] - self.command(line, mo) - elif blprog.match(line) and \ - 'format' not in self.stack and \ - 'example' not in self.stack: - if accu: - if not self.skip: - self.process(accu) - if self.nofill: - self.write('\n') - else: - self.write('

\n') - accu = [] - else: - # Append the line including trailing \n! - accu.append(line) - # - if self.skip: - print('*** Still skipping at the end') - if self.stack: - print('*** Stack not empty at the end') - print('***', self.stack) - if self.includedepth == 0: - while self.nodestack: - self.nodestack[-1].finalize() - self.nodestack[-1].flush() - del self.nodestack[-1] - - # Start saving text in a buffer instead of writing it to a file - def startsaving(self): - if self.savetext is not None: - self.savestack.append(self.savetext) - # print '*** Recursively saving text, expect trouble' - self.savetext = '' - - # Return the text saved so far and start writing to file again - def collectsavings(self): - savetext = self.savetext - if len(self.savestack) > 0: - self.savetext = self.savestack[-1] - del self.savestack[-1] - else: - self.savetext = None - return savetext or '' - - # Write text to file, or save it in a buffer, or ignore it - def write(self, *args): - try: - text = ''.join(args) - except: - print(args) - raise TypeError - if self.savetext is not None: - self.savetext = self.savetext + text - elif self.nodefp: - self.nodefp.write(text) - elif self.node: - self.node.write(text) - - # Complete the current node -- write footnotes and close file - def endnode(self): - if self.savetext is not None: - print('*** Still saving text at end of node') - dummy = self.collectsavings() - if self.footnotes: - self.writefootnotes() - if self.nodefp: - if self.nodelineno > 20: - self.write('

\n') - [name, next, prev, up] = self.nodelinks[:4] - self.link('Next', next) - self.link('Prev', prev) - self.link('Up', up) - if self.nodename != self.topname: - self.link('Top', self.topname) - self.write('
\n') - self.write('\n') - self.nodefp.close() - self.nodefp = None - elif self.node: - if not self.cont and \ - (not self.node.type or \ - (self.node.next and self.node.prev and self.node.up)): - self.node.finalize() - self.node.flush() - else: - self.nodestack.append(self.node) - self.node = None - self.nodename = '' - - # Process a list of lines, expanding embedded @-commands - # This mostly distinguishes between menus and normal text - def process(self, accu): - if self.debugging > 1: - print('!'*self.debugging, 'process:', self.skip, self.stack, end=' ') - if accu: print(accu[0][:30], end=' ') - if accu[0][30:] or accu[1:]: print('...', end=' ') - print() - if self.inmenu(): - # XXX should be done differently - for line in accu: - mo = miprog.match(line) - if not mo: - line = line.strip() + '\n' - self.expand(line) - continue - bgn, end = mo.span(0) - a, b = mo.span(1) - c, d = mo.span(2) - e, f = mo.span(3) - g, h = mo.span(4) - label = line[a:b] - nodename = line[c:d] - if nodename[0] == ':': nodename = label - else: nodename = line[e:f] - punct = line[g:h] - self.write('
  • ', nodename, - '', punct, '\n') - self.htmlhelp.menuitem(nodename) - self.expand(line[end:]) - else: - text = ''.join(accu) - self.expand(text) - - # find 'menu' (we might be inside 'ifset' or 'ifclear') - def inmenu(self): - #if 'menu' in self.stack: - # print 'inmenu :', self.skip, self.stack, self.stackinfo - stack = self.stack - while stack and stack[-1] in ('ifset','ifclear'): - try: - if self.stackinfo[len(stack)]: - return 0 - except KeyError: - pass - stack = stack[:-1] - return (stack and stack[-1] == 'menu') - - # Write a string, expanding embedded @-commands - def expand(self, text): - stack = [] - i = 0 - n = len(text) - while i < n: - start = i - mo = spprog.search(text, i) - if mo: - i = mo.start() - else: - self.write(text[start:]) - break - self.write(text[start:i]) - c = text[i] - i = i+1 - if c == '\n': - self.write('\n') - continue - if c == '<': - self.write('<') - continue - if c == '>': - self.write('>') - continue - if c == '&': - self.write('&') - continue - if c == '{': - stack.append('') - continue - if c == '}': - if not stack: - print('*** Unmatched }') - self.write('}') - continue - cmd = stack[-1] - del stack[-1] - try: - method = getattr(self, 'close_' + cmd) - except AttributeError: - self.unknown_close(cmd) - continue - method() - continue - if c != '@': - # Cannot happen unless spprog is changed - raise RuntimeError('unexpected funny %r' % c) - start = i - while i < n and text[i] in string.ascii_letters: i = i+1 - if i == start: - # @ plus non-letter: literal next character - i = i+1 - c = text[start:i] - if c == ':': - # `@:' means no extra space after - # preceding `.', `?', `!' or `:' - pass - else: - # `@.' means a sentence-ending period; - # `@@', `@{', `@}' quote `@', `{', `}' - self.write(c) - continue - cmd = text[start:i] - if i < n and text[i] == '{': - i = i+1 - stack.append(cmd) - try: - method = getattr(self, 'open_' + cmd) - except AttributeError: - self.unknown_open(cmd) - continue - method() - continue - try: - method = getattr(self, 'handle_' + cmd) - except AttributeError: - self.unknown_handle(cmd) - continue - method() - if stack: - print('*** Stack not empty at para:', stack) - - # --- Handle unknown embedded @-commands --- - - def unknown_open(self, cmd): - print('*** No open func for @' + cmd + '{...}') - cmd = cmd + '{' - self.write('@', cmd) - if cmd not in self.unknown: - self.unknown[cmd] = 1 - else: - self.unknown[cmd] = self.unknown[cmd] + 1 - - def unknown_close(self, cmd): - print('*** No close func for @' + cmd + '{...}') - cmd = '}' + cmd - self.write('}') - if cmd not in self.unknown: - self.unknown[cmd] = 1 - else: - self.unknown[cmd] = self.unknown[cmd] + 1 - - def unknown_handle(self, cmd): - print('*** No handler for @' + cmd) - self.write('@', cmd) - if cmd not in self.unknown: - self.unknown[cmd] = 1 - else: - self.unknown[cmd] = self.unknown[cmd] + 1 - - # XXX The following sections should be ordered as the texinfo docs - - # --- Embedded @-commands without {} argument list -- - - def handle_noindent(self): pass - - def handle_refill(self): pass - - # --- Include file handling --- - - def do_include(self, args): - file = args - file = os.path.join(self.includedir, file) - try: - fp = open(file, 'r') - except IOError as msg: - print('*** Can\'t open include file', repr(file)) - return - with fp: - print('!'*self.debugging, '--> file', repr(file)) - save_done = self.done - save_skip = self.skip - save_stack = self.stack - self.includedepth = self.includedepth + 1 - self.parserest(fp, 0) - self.includedepth = self.includedepth - 1 - self.done = save_done - self.skip = save_skip - self.stack = save_stack - print('!'*self.debugging, '<-- file', repr(file)) - - # --- Special Insertions --- - - def open_dmn(self): pass - def close_dmn(self): pass - - def open_dots(self): self.write('...') - def close_dots(self): pass - - def open_bullet(self): pass - def close_bullet(self): pass - - def open_TeX(self): self.write('TeX') - def close_TeX(self): pass - - def handle_copyright(self): self.write(self.COPYRIGHT_SYMBOL) - def open_copyright(self): self.write(self.COPYRIGHT_SYMBOL) - def close_copyright(self): pass - - def open_minus(self): self.write('-') - def close_minus(self): pass - - # --- Accents --- - - # rpyron 2002-05-07 - # I would like to do at least as well as makeinfo when - # it is producing HTML output: - # - # input output - # @"o @"o umlaut accent - # @'o 'o acute accent - # @,{c} @,{c} cedilla accent - # @=o @=o macron/overbar accent - # @^o @^o circumflex accent - # @`o `o grave accent - # @~o @~o tilde accent - # @dotaccent{o} @dotaccent{o} overdot accent - # @H{o} @H{o} long Hungarian umlaut - # @ringaccent{o} @ringaccent{o} ring accent - # @tieaccent{oo} @tieaccent{oo} tie-after accent - # @u{o} @u{o} breve accent - # @ubaraccent{o} @ubaraccent{o} underbar accent - # @udotaccent{o} @udotaccent{o} underdot accent - # @v{o} @v{o} hacek or check accent - # @exclamdown{} ¡ upside-down ! - # @questiondown{} ¿ upside-down ? - # @aa{},@AA{} å,Å a,A with circle - # @ae{},@AE{} æ,Æ ae,AE ligatures - # @dotless{i} @dotless{i} dotless i - # @dotless{j} @dotless{j} dotless j - # @l{},@L{} l/,L/ suppressed-L,l - # @o{},@O{} ø,Ø O,o with slash - # @oe{},@OE{} oe,OE oe,OE ligatures - # @ss{} ß es-zet or sharp S - # - # The following character codes and approximations have been - # copied from makeinfo's HTML output. - - def open_exclamdown(self): self.write('¡') # upside-down ! - def close_exclamdown(self): pass - def open_questiondown(self): self.write('¿') # upside-down ? - def close_questiondown(self): pass - def open_aa(self): self.write('å') # a with circle - def close_aa(self): pass - def open_AA(self): self.write('Å') # A with circle - def close_AA(self): pass - def open_ae(self): self.write('æ') # ae ligatures - def close_ae(self): pass - def open_AE(self): self.write('Æ') # AE ligatures - def close_AE(self): pass - def open_o(self): self.write('ø') # o with slash - def close_o(self): pass - def open_O(self): self.write('Ø') # O with slash - def close_O(self): pass - def open_ss(self): self.write('ß') # es-zet or sharp S - def close_ss(self): pass - def open_oe(self): self.write('oe') # oe ligatures - def close_oe(self): pass - def open_OE(self): self.write('OE') # OE ligatures - def close_OE(self): pass - def open_l(self): self.write('l/') # suppressed-l - def close_l(self): pass - def open_L(self): self.write('L/') # suppressed-L - def close_L(self): pass - - # --- Special Glyphs for Examples --- - - def open_result(self): self.write('=>') - def close_result(self): pass - - def open_expansion(self): self.write('==>') - def close_expansion(self): pass - - def open_print(self): self.write('-|') - def close_print(self): pass - - def open_error(self): self.write('error-->') - def close_error(self): pass - - def open_equiv(self): self.write('==') - def close_equiv(self): pass - - def open_point(self): self.write('-!-') - def close_point(self): pass - - # --- Cross References --- - - def open_pxref(self): - self.write('see ') - self.startsaving() - def close_pxref(self): - self.makeref() - - def open_xref(self): - self.write('See ') - self.startsaving() - def close_xref(self): - self.makeref() - - def open_ref(self): - self.startsaving() - def close_ref(self): - self.makeref() - - def open_inforef(self): - self.write('See info file ') - self.startsaving() - def close_inforef(self): - text = self.collectsavings() - args = [s.strip() for s in text.split(',')] - while len(args) < 3: args.append('') - node = args[0] - file = args[2] - self.write('`', file, '\', node `', node, '\'') - - def makeref(self): - text = self.collectsavings() - args = [s.strip() for s in text.split(',')] - while len(args) < 5: args.append('') - nodename = label = args[0] - if args[2]: label = args[2] - file = args[3] - title = args[4] - href = makefile(nodename) - if file: - href = '../' + file + '/' + href - self.write('', label, '') - - # rpyron 2002-05-07 uref support - def open_uref(self): - self.startsaving() - def close_uref(self): - text = self.collectsavings() - args = [s.strip() for s in text.split(',')] - while len(args) < 2: args.append('') - href = args[0] - label = args[1] - if not label: label = href - self.write('', label, '') - - # rpyron 2002-05-07 image support - # GNU makeinfo producing HTML output tries `filename.png'; if - # that does not exist, it tries `filename.jpg'. If that does - # not exist either, it complains. GNU makeinfo does not handle - # GIF files; however, I include GIF support here because - # MySQL documentation uses GIF files. - - def open_image(self): - self.startsaving() - def close_image(self): - self.makeimage() - def makeimage(self): - text = self.collectsavings() - args = [s.strip() for s in text.split(',')] - while len(args) < 5: args.append('') - filename = args[0] - width = args[1] - height = args[2] - alt = args[3] - ext = args[4] - - # The HTML output will have a reference to the image - # that is relative to the HTML output directory, - # which is what 'filename' gives us. However, we need - # to find it relative to our own current directory, - # so we construct 'imagename'. - imagelocation = self.dirname + '/' + filename - - if os.path.exists(imagelocation+'.png'): - filename += '.png' - elif os.path.exists(imagelocation+'.jpg'): - filename += '.jpg' - elif os.path.exists(imagelocation+'.gif'): # MySQL uses GIF files - filename += '.gif' - else: - print("*** Cannot find image " + imagelocation) - #TODO: what is 'ext'? - self.write('' ) - self.htmlhelp.addimage(imagelocation) - - - # --- Marking Words and Phrases --- - - # --- Other @xxx{...} commands --- - - def open_(self): pass # Used by {text enclosed in braces} - def close_(self): pass - - open_asis = open_ - close_asis = close_ - - def open_cite(self): self.write('') - def close_cite(self): self.write('') - - def open_code(self): self.write('') - def close_code(self): self.write('') - - def open_t(self): self.write('') - def close_t(self): self.write('') - - def open_dfn(self): self.write('') - def close_dfn(self): self.write('') - - def open_emph(self): self.write('') - def close_emph(self): self.write('') - - def open_i(self): self.write('') - def close_i(self): self.write('') - - def open_footnote(self): - # if self.savetext is not None: - # print '*** Recursive footnote -- expect weirdness' - id = len(self.footnotes) + 1 - self.write(self.FN_SOURCE_PATTERN % {'id': repr(id)}) - self.startsaving() - - def close_footnote(self): - id = len(self.footnotes) + 1 - self.footnotes.append((id, self.collectsavings())) - - def writefootnotes(self): - self.write(self.FN_HEADER) - for id, text in self.footnotes: - self.write(self.FN_TARGET_PATTERN - % {'id': repr(id), 'text': text}) - self.footnotes = [] - - def open_file(self): self.write('') - def close_file(self): self.write('') - - def open_kbd(self): self.write('') - def close_kbd(self): self.write('') - - def open_key(self): self.write('') - def close_key(self): self.write('') - - def open_r(self): self.write('') - def close_r(self): self.write('') - - def open_samp(self): self.write('`') - def close_samp(self): self.write('\'') - - def open_sc(self): self.write('') - def close_sc(self): self.write('') - - def open_strong(self): self.write('') - def close_strong(self): self.write('') - - def open_b(self): self.write('') - def close_b(self): self.write('') - - def open_var(self): self.write('') - def close_var(self): self.write('') - - def open_w(self): self.write('') - def close_w(self): self.write('') - - def open_url(self): self.startsaving() - def close_url(self): - text = self.collectsavings() - self.write('', text, '') - - def open_email(self): self.startsaving() - def close_email(self): - text = self.collectsavings() - self.write('', text, '') - - open_titlefont = open_ - close_titlefont = close_ - - def open_small(self): pass - def close_small(self): pass - - def command(self, line, mo): - a, b = mo.span(1) - cmd = line[a:b] - args = line[b:].strip() - if self.debugging > 1: - print('!'*self.debugging, 'command:', self.skip, self.stack, \ - '@' + cmd, args) - try: - func = getattr(self, 'do_' + cmd) - except AttributeError: - try: - func = getattr(self, 'bgn_' + cmd) - except AttributeError: - # don't complain if we are skipping anyway - if not self.skip: - self.unknown_cmd(cmd, args) - return - self.stack.append(cmd) - func(args) - return - if not self.skip or cmd == 'end': - func(args) - - def unknown_cmd(self, cmd, args): - print('*** unknown', '@' + cmd, args) - if cmd not in self.unknown: - self.unknown[cmd] = 1 - else: - self.unknown[cmd] = self.unknown[cmd] + 1 - - def do_end(self, args): - words = args.split() - if not words: - print('*** @end w/o args') - else: - cmd = words[0] - if not self.stack or self.stack[-1] != cmd: - print('*** @end', cmd, 'unexpected') - else: - del self.stack[-1] - try: - func = getattr(self, 'end_' + cmd) - except AttributeError: - self.unknown_end(cmd) - return - func() - - def unknown_end(self, cmd): - cmd = 'end ' + cmd - print('*** unknown', '@' + cmd) - if cmd not in self.unknown: - self.unknown[cmd] = 1 - else: - self.unknown[cmd] = self.unknown[cmd] + 1 - - # --- Comments --- - - def do_comment(self, args): pass - do_c = do_comment - - # --- Conditional processing --- - - def bgn_ifinfo(self, args): pass - def end_ifinfo(self): pass - - def bgn_iftex(self, args): self.skip = self.skip + 1 - def end_iftex(self): self.skip = self.skip - 1 - - def bgn_ignore(self, args): self.skip = self.skip + 1 - def end_ignore(self): self.skip = self.skip - 1 - - def bgn_tex(self, args): self.skip = self.skip + 1 - def end_tex(self): self.skip = self.skip - 1 - - def do_set(self, args): - fields = args.split(' ') - key = fields[0] - if len(fields) == 1: - value = 1 - else: - value = ' '.join(fields[1:]) - self.values[key] = value - - def do_clear(self, args): - self.values[args] = None - - def bgn_ifset(self, args): - if args not in self.values or self.values[args] is None: - self.skip = self.skip + 1 - self.stackinfo[len(self.stack)] = 1 - else: - self.stackinfo[len(self.stack)] = 0 - def end_ifset(self): - try: - if self.stackinfo[len(self.stack) + 1]: - self.skip = self.skip - 1 - del self.stackinfo[len(self.stack) + 1] - except KeyError: - print('*** end_ifset: KeyError :', len(self.stack) + 1) - - def bgn_ifclear(self, args): - if args in self.values and self.values[args] is not None: - self.skip = self.skip + 1 - self.stackinfo[len(self.stack)] = 1 - else: - self.stackinfo[len(self.stack)] = 0 - def end_ifclear(self): - try: - if self.stackinfo[len(self.stack) + 1]: - self.skip = self.skip - 1 - del self.stackinfo[len(self.stack) + 1] - except KeyError: - print('*** end_ifclear: KeyError :', len(self.stack) + 1) - - def open_value(self): - self.startsaving() - - def close_value(self): - key = self.collectsavings() - if key in self.values: - self.write(self.values[key]) - else: - print('*** Undefined value: ', key) - - # --- Beginning a file --- - - do_finalout = do_comment - do_setchapternewpage = do_comment - do_setfilename = do_comment - - def do_settitle(self, args): - self.startsaving() - self.expand(args) - self.title = self.collectsavings() - def do_parskip(self, args): pass - - # --- Ending a file --- - - def do_bye(self, args): - self.endnode() - self.done = 1 - - # --- Title page --- - - def bgn_titlepage(self, args): self.skip = self.skip + 1 - def end_titlepage(self): self.skip = self.skip - 1 - def do_shorttitlepage(self, args): pass - - def do_center(self, args): - # Actually not used outside title page... - self.write('

    ') - self.expand(args) - self.write('

    \n') - do_title = do_center - do_subtitle = do_center - do_author = do_center - - do_vskip = do_comment - do_vfill = do_comment - do_smallbook = do_comment - - do_paragraphindent = do_comment - do_setchapternewpage = do_comment - do_headings = do_comment - do_footnotestyle = do_comment - - do_evenheading = do_comment - do_evenfooting = do_comment - do_oddheading = do_comment - do_oddfooting = do_comment - do_everyheading = do_comment - do_everyfooting = do_comment - - # --- Nodes --- - - def do_node(self, args): - self.endnode() - self.nodelineno = 0 - parts = [s.strip() for s in args.split(',')] - while len(parts) < 4: parts.append('') - self.nodelinks = parts - [name, next, prev, up] = parts[:4] - file = self.dirname + '/' + makefile(name) - if file in self.filenames: - print('*** Filename already in use: ', file) - else: - if self.debugging: print('!'*self.debugging, '--- writing', file) - self.filenames[file] = 1 - # self.nodefp = open(file, 'w') - self.nodename = name - if self.cont and self.nodestack: - self.nodestack[-1].cont = self.nodename - if not self.topname: self.topname = name - title = name - if self.title: title = title + ' -- ' + self.title - self.node = self.Node(self.dirname, self.nodename, self.topname, - title, next, prev, up) - self.htmlhelp.addnode(self.nodename,next,prev,up,file) - - def link(self, label, nodename): - if nodename: - if nodename.lower() == '(dir)': - addr = '../dir.html' - else: - addr = makefile(nodename) - self.write(label, ': ', nodename, ' \n') - - # --- Sectioning commands --- - - def popstack(self, type): - if (self.node): - self.node.type = type - while self.nodestack: - if self.nodestack[-1].type > type: - self.nodestack[-1].finalize() - self.nodestack[-1].flush() - del self.nodestack[-1] - elif self.nodestack[-1].type == type: - if not self.nodestack[-1].next: - self.nodestack[-1].next = self.node.name - if not self.node.prev: - self.node.prev = self.nodestack[-1].name - self.nodestack[-1].finalize() - self.nodestack[-1].flush() - del self.nodestack[-1] - else: - if type > 1 and not self.node.up: - self.node.up = self.nodestack[-1].name - break - - def do_chapter(self, args): - self.heading('H1', args, 0) - self.popstack(1) - - def do_unnumbered(self, args): - self.heading('H1', args, -1) - self.popstack(1) - def do_appendix(self, args): - self.heading('H1', args, -1) - self.popstack(1) - def do_top(self, args): - self.heading('H1', args, -1) - def do_chapheading(self, args): - self.heading('H1', args, -1) - def do_majorheading(self, args): - self.heading('H1', args, -1) - - def do_section(self, args): - self.heading('H1', args, 1) - self.popstack(2) - - def do_unnumberedsec(self, args): - self.heading('H1', args, -1) - self.popstack(2) - def do_appendixsec(self, args): - self.heading('H1', args, -1) - self.popstack(2) - do_appendixsection = do_appendixsec - def do_heading(self, args): - self.heading('H1', args, -1) - - def do_subsection(self, args): - self.heading('H2', args, 2) - self.popstack(3) - def do_unnumberedsubsec(self, args): - self.heading('H2', args, -1) - self.popstack(3) - def do_appendixsubsec(self, args): - self.heading('H2', args, -1) - self.popstack(3) - def do_subheading(self, args): - self.heading('H2', args, -1) - - def do_subsubsection(self, args): - self.heading('H3', args, 3) - self.popstack(4) - def do_unnumberedsubsubsec(self, args): - self.heading('H3', args, -1) - self.popstack(4) - def do_appendixsubsubsec(self, args): - self.heading('H3', args, -1) - self.popstack(4) - def do_subsubheading(self, args): - self.heading('H3', args, -1) - - def heading(self, type, args, level): - if level >= 0: - while len(self.numbering) <= level: - self.numbering.append(0) - del self.numbering[level+1:] - self.numbering[level] = self.numbering[level] + 1 - x = '' - for i in self.numbering: - x = x + repr(i) + '.' - args = x + ' ' + args - self.contents.append((level, args, self.nodename)) - self.write('<', type, '>') - self.expand(args) - self.write('\n') - if self.debugging or self.print_headers: - print('---', args) - - def do_contents(self, args): - # pass - self.listcontents('Table of Contents', 999) - - def do_shortcontents(self, args): - pass - # self.listcontents('Short Contents', 0) - do_summarycontents = do_shortcontents - - def listcontents(self, title, maxlevel): - self.write('

    ', title, '

    \n
      \n') - prevlevels = [0] - for level, title, node in self.contents: - if level > maxlevel: - continue - if level > prevlevels[-1]: - # can only advance one level at a time - self.write(' '*prevlevels[-1], '
        \n') - prevlevels.append(level) - elif level < prevlevels[-1]: - # might drop back multiple levels - while level < prevlevels[-1]: - del prevlevels[-1] - self.write(' '*prevlevels[-1], - '
      \n') - self.write(' '*level, '
    • ') - self.expand(title) - self.write('\n') - self.write('
    \n' * len(prevlevels)) - - # --- Page lay-out --- - - # These commands are only meaningful in printed text - - def do_page(self, args): pass - - def do_need(self, args): pass - - def bgn_group(self, args): pass - def end_group(self): pass - - # --- Line lay-out --- - - def do_sp(self, args): - if self.nofill: - self.write('\n') - else: - self.write('

    \n') - - def do_hline(self, args): - self.write('

    ') - - # --- Function and variable definitions --- - - def bgn_deffn(self, args): - self.write('
    ') - self.do_deffnx(args) - - def end_deffn(self): - self.write('
    \n') - - def do_deffnx(self, args): - self.write('
    ') - words = splitwords(args, 2) - [category, name], rest = words[:2], words[2:] - self.expand('@b{%s}' % name) - for word in rest: self.expand(' ' + makevar(word)) - #self.expand(' -- ' + category) - self.write('\n
    ') - self.index('fn', name) - - def bgn_defun(self, args): self.bgn_deffn('Function ' + args) - end_defun = end_deffn - def do_defunx(self, args): self.do_deffnx('Function ' + args) - - def bgn_defmac(self, args): self.bgn_deffn('Macro ' + args) - end_defmac = end_deffn - def do_defmacx(self, args): self.do_deffnx('Macro ' + args) - - def bgn_defspec(self, args): self.bgn_deffn('{Special Form} ' + args) - end_defspec = end_deffn - def do_defspecx(self, args): self.do_deffnx('{Special Form} ' + args) - - def bgn_defvr(self, args): - self.write('
    ') - self.do_defvrx(args) - - end_defvr = end_deffn - - def do_defvrx(self, args): - self.write('
    ') - words = splitwords(args, 2) - [category, name], rest = words[:2], words[2:] - self.expand('@code{%s}' % name) - # If there are too many arguments, show them - for word in rest: self.expand(' ' + word) - #self.expand(' -- ' + category) - self.write('\n
    ') - self.index('vr', name) - - def bgn_defvar(self, args): self.bgn_defvr('Variable ' + args) - end_defvar = end_defvr - def do_defvarx(self, args): self.do_defvrx('Variable ' + args) - - def bgn_defopt(self, args): self.bgn_defvr('{User Option} ' + args) - end_defopt = end_defvr - def do_defoptx(self, args): self.do_defvrx('{User Option} ' + args) - - # --- Ditto for typed languages --- - - def bgn_deftypefn(self, args): - self.write('
    ') - self.do_deftypefnx(args) - - end_deftypefn = end_deffn - - def do_deftypefnx(self, args): - self.write('
    ') - words = splitwords(args, 3) - [category, datatype, name], rest = words[:3], words[3:] - self.expand('@code{%s} @b{%s}' % (datatype, name)) - for word in rest: self.expand(' ' + makevar(word)) - #self.expand(' -- ' + category) - self.write('\n
    ') - self.index('fn', name) - - - def bgn_deftypefun(self, args): self.bgn_deftypefn('Function ' + args) - end_deftypefun = end_deftypefn - def do_deftypefunx(self, args): self.do_deftypefnx('Function ' + args) - - def bgn_deftypevr(self, args): - self.write('
    ') - self.do_deftypevrx(args) - - end_deftypevr = end_deftypefn - - def do_deftypevrx(self, args): - self.write('
    ') - words = splitwords(args, 3) - [category, datatype, name], rest = words[:3], words[3:] - self.expand('@code{%s} @b{%s}' % (datatype, name)) - # If there are too many arguments, show them - for word in rest: self.expand(' ' + word) - #self.expand(' -- ' + category) - self.write('\n
    ') - self.index('fn', name) - - def bgn_deftypevar(self, args): - self.bgn_deftypevr('Variable ' + args) - end_deftypevar = end_deftypevr - def do_deftypevarx(self, args): - self.do_deftypevrx('Variable ' + args) - - # --- Ditto for object-oriented languages --- - - def bgn_defcv(self, args): - self.write('
    ') - self.do_defcvx(args) - - end_defcv = end_deftypevr - - def do_defcvx(self, args): - self.write('
    ') - words = splitwords(args, 3) - [category, classname, name], rest = words[:3], words[3:] - self.expand('@b{%s}' % name) - # If there are too many arguments, show them - for word in rest: self.expand(' ' + word) - #self.expand(' -- %s of @code{%s}' % (category, classname)) - self.write('\n
    ') - self.index('vr', '%s @r{on %s}' % (name, classname)) - - def bgn_defivar(self, args): - self.bgn_defcv('{Instance Variable} ' + args) - end_defivar = end_defcv - def do_defivarx(self, args): - self.do_defcvx('{Instance Variable} ' + args) - - def bgn_defop(self, args): - self.write('
    ') - self.do_defopx(args) - - end_defop = end_defcv - - def do_defopx(self, args): - self.write('
    ') - words = splitwords(args, 3) - [category, classname, name], rest = words[:3], words[3:] - self.expand('@b{%s}' % name) - for word in rest: self.expand(' ' + makevar(word)) - #self.expand(' -- %s of @code{%s}' % (category, classname)) - self.write('\n
    ') - self.index('fn', '%s @r{on %s}' % (name, classname)) - - def bgn_defmethod(self, args): - self.bgn_defop('Method ' + args) - end_defmethod = end_defop - def do_defmethodx(self, args): - self.do_defopx('Method ' + args) - - # --- Ditto for data types --- - - def bgn_deftp(self, args): - self.write('
    ') - self.do_deftpx(args) - - end_deftp = end_defcv - - def do_deftpx(self, args): - self.write('
    ') - words = splitwords(args, 2) - [category, name], rest = words[:2], words[2:] - self.expand('@b{%s}' % name) - for word in rest: self.expand(' ' + word) - #self.expand(' -- ' + category) - self.write('\n
    ') - self.index('tp', name) - - # --- Making Lists and Tables - - def bgn_enumerate(self, args): - if not args: - self.write('
      \n') - self.stackinfo[len(self.stack)] = '
    \n' - else: - self.itemnumber = args - self.write('
      \n') - self.stackinfo[len(self.stack)] = '
    \n' - def end_enumerate(self): - self.itemnumber = None - self.write(self.stackinfo[len(self.stack) + 1]) - del self.stackinfo[len(self.stack) + 1] - - def bgn_itemize(self, args): - self.itemarg = args - self.write('
      \n') - def end_itemize(self): - self.itemarg = None - self.write('
    \n') - - def bgn_table(self, args): - self.itemarg = args - self.write('
    \n') - def end_table(self): - self.itemarg = None - self.write('
    \n') - - def bgn_ftable(self, args): - self.itemindex = 'fn' - self.bgn_table(args) - def end_ftable(self): - self.itemindex = None - self.end_table() - - def bgn_vtable(self, args): - self.itemindex = 'vr' - self.bgn_table(args) - def end_vtable(self): - self.itemindex = None - self.end_table() - - def do_item(self, args): - if self.itemindex: self.index(self.itemindex, args) - if self.itemarg: - if self.itemarg[0] == '@' and self.itemarg[1] and \ - self.itemarg[1] in string.ascii_letters: - args = self.itemarg + '{' + args + '}' - else: - # some other character, e.g. '-' - args = self.itemarg + ' ' + args - if self.itemnumber is not None: - args = self.itemnumber + '. ' + args - self.itemnumber = increment(self.itemnumber) - if self.stack and self.stack[-1] == 'table': - self.write('
    ') - self.expand(args) - self.write('\n
    ') - elif self.stack and self.stack[-1] == 'multitable': - self.write('') - self.expand(args) - self.write('\n\n') - else: - self.write('
  • ') - self.expand(args) - self.write(' ') - do_itemx = do_item # XXX Should suppress leading blank line - - # rpyron 2002-05-07 multitable support - def bgn_multitable(self, args): - self.itemarg = None # should be handled by columnfractions - self.write('\n') - def end_multitable(self): - self.itemarg = None - self.write('
    \n
    \n') - def handle_columnfractions(self): - # It would be better to handle this, but for now it's in the way... - self.itemarg = None - def handle_tab(self): - self.write('\n ') - - # --- Enumerations, displays, quotations --- - # XXX Most of these should increase the indentation somehow - - def bgn_quotation(self, args): self.write('
    ') - def end_quotation(self): self.write('
    \n') - - def bgn_example(self, args): - self.nofill = self.nofill + 1 - self.write('
    ')
-    def end_example(self):
-        self.write('
    \n') - self.nofill = self.nofill - 1 - - bgn_lisp = bgn_example # Synonym when contents are executable lisp code - end_lisp = end_example - - bgn_smallexample = bgn_example # XXX Should use smaller font - end_smallexample = end_example - - bgn_smalllisp = bgn_lisp # Ditto - end_smalllisp = end_lisp - - bgn_display = bgn_example - end_display = end_example - - bgn_format = bgn_display - end_format = end_display - - def do_exdent(self, args): self.expand(args + '\n') - # XXX Should really mess with indentation - - def bgn_flushleft(self, args): - self.nofill = self.nofill + 1 - self.write('
    \n')
-    def end_flushleft(self):
-        self.write('
    \n') - self.nofill = self.nofill - 1 - - def bgn_flushright(self, args): - self.nofill = self.nofill + 1 - self.write('
    \n') - def end_flushright(self): - self.write('
    \n') - self.nofill = self.nofill - 1 - - def bgn_menu(self, args): - self.write('\n') - self.write(' Menu

    \n') - self.htmlhelp.beginmenu() - def end_menu(self): - self.write('

    \n') - self.htmlhelp.endmenu() - - def bgn_cartouche(self, args): pass - def end_cartouche(self): pass - - # --- Indices --- - - def resetindex(self): - self.noncodeindices = ['cp'] - self.indextitle = {} - self.indextitle['cp'] = 'Concept' - self.indextitle['fn'] = 'Function' - self.indextitle['ky'] = 'Keyword' - self.indextitle['pg'] = 'Program' - self.indextitle['tp'] = 'Type' - self.indextitle['vr'] = 'Variable' - # - self.whichindex = {} - for name in self.indextitle: - self.whichindex[name] = [] - - def user_index(self, name, args): - if name in self.whichindex: - self.index(name, args) - else: - print('*** No index named', repr(name)) - - def do_cindex(self, args): self.index('cp', args) - def do_findex(self, args): self.index('fn', args) - def do_kindex(self, args): self.index('ky', args) - def do_pindex(self, args): self.index('pg', args) - def do_tindex(self, args): self.index('tp', args) - def do_vindex(self, args): self.index('vr', args) - - def index(self, name, args): - self.whichindex[name].append((args, self.nodename)) - self.htmlhelp.index(args, self.nodename) - - def do_synindex(self, args): - words = args.split() - if len(words) != 2: - print('*** bad @synindex', args) - return - [old, new] = words - if old not in self.whichindex or \ - new not in self.whichindex: - print('*** bad key(s) in @synindex', args) - return - if old != new and \ - self.whichindex[old] is not self.whichindex[new]: - inew = self.whichindex[new] - inew[len(inew):] = self.whichindex[old] - self.whichindex[old] = inew - do_syncodeindex = do_synindex # XXX Should use code font - - def do_printindex(self, args): - words = args.split() - for name in words: - if name in self.whichindex: - self.prindex(name) - else: - print('*** No index named', repr(name)) - - def prindex(self, name): - iscodeindex = (name not in self.noncodeindices) - index = self.whichindex[name] - if not index: return - if self.debugging: - print('!'*self.debugging, '--- Generating', \ - self.indextitle[name], 'index') - # The node already provides a title - index1 = [] - junkprog = re.compile('^(@[a-z]+)?{') - for key, node in index: - sortkey = key.lower() - # Remove leading `@cmd{' from sort key - # -- don't bother about the matching `}' - oldsortkey = sortkey - while 1: - mo = junkprog.match(sortkey) - if not mo: - break - i = mo.end() - sortkey = sortkey[i:] - index1.append((sortkey, key, node)) - del index[:] - index1.sort() - self.write('
    \n') - prevkey = prevnode = None - for sortkey, key, node in index1: - if (key, node) == (prevkey, prevnode): - continue - if self.debugging > 1: print('!'*self.debugging, key, ':', node) - self.write('
    ') - if iscodeindex: key = '@code{' + key + '}' - if key != prevkey: - self.expand(key) - self.write('\n
    %s\n' % (makefile(node), node)) - prevkey, prevnode = key, node - self.write('
    \n') - - # --- Final error reports --- - - def report(self): - if self.unknown: - print('--- Unrecognized commands ---') - cmds = sorted(self.unknown.keys()) - for cmd in cmds: - print(cmd.ljust(20), self.unknown[cmd]) - - -class TexinfoParserHTML3(TexinfoParser): - - COPYRIGHT_SYMBOL = "©" - FN_ID_PATTERN = "[%(id)s]" - FN_SOURCE_PATTERN = '' + FN_ID_PATTERN + '' - FN_TARGET_PATTERN = '\n' \ - '

    ' + FN_ID_PATTERN \ - + '\n%(text)s

    \n' - FN_HEADER = '
    \n
    \n' \ - ' Footnotes\n

    \n' - - Node = HTML3Node - - def bgn_quotation(self, args): self.write('') - def end_quotation(self): self.write('\n') - - def bgn_example(self, args): - # this use of would not be legal in HTML 2.0, - # but is in more recent DTDs. - self.nofill = self.nofill + 1 - self.write('

    ')
-    def end_example(self):
-        self.write("
    \n") - self.nofill = self.nofill - 1 - - def bgn_flushleft(self, args): - self.nofill = self.nofill + 1 - self.write('
    \n')
-
-    def bgn_flushright(self, args):
-        self.nofill = self.nofill + 1
-        self.write('
    \n')
-    def end_flushright(self):
-        self.write('
    \n')
-        self.nofill = self.nofill - 1
-
-    def bgn_menu(self, args):
-        self.write('
      \n')
-        self.write('  Menu\n')
-    def end_menu(self):
-        self.write('
    \n')
-
-
-# rpyron 2002-05-07
-class HTMLHelp:
-    """
-    This class encapsulates support for HTML Help. Node names,
-    file names, menu items, index items, and image file names are
-    accumulated until a call to finalize(). At that time, three
-    output files are created in the current directory:
-
-        `helpbase`.hhp  is a HTML Help Workshop project file.
-                        It contains various information, some of
-                        which I do not understand; I just copied
-                        the default project info from a fresh
-                        installation.
-        `helpbase`.hhc  is the Contents file for the project.
-        `helpbase`.hhk  is the Index file for the project.
-
-    When these files are used as input to HTML Help Workshop,
-    the resulting file will be named:
-
-        `helpbase`.chm
-
-    If none of the defaults in `helpbase`.hhp are changed,
-    the .CHM file will have Contents, Index, Search, and
-    Favorites tabs.
-    """
-
-    codeprog = re.compile('@code{(.*?)}')
-
-    def __init__(self,helpbase,dirname):
-        self.helpbase    = helpbase
-        self.dirname     = dirname
-        self.projectfile = None
-        self.contentfile = None
-        self.indexfile   = None
-        self.nodelist    = []
-        self.nodenames   = {}         # nodename : index
-        self.nodeindex   = {}
-        self.filenames   = {}         # filename : filename
-        self.indexlist   = []         # (args,nodename) == (key,location)
-        self.current     = ''
-        self.menudict    = {}
-        self.dumped      = {}
-
-
-    def addnode(self,name,next,prev,up,filename):
-        node = (name,next,prev,up,filename)
-        # add this file to dict
-        # retrieve list with self.filenames.values()
-        self.filenames[filename] = filename
-        # add this node to nodelist
-        self.nodeindex[name] = len(self.nodelist)
-        self.nodelist.append(node)
-        # set 'current' for menu items
-        self.current = name
-        self.menudict[self.current] = []
-
-    def menuitem(self,nodename):
-        menu = self.menudict[self.current]
-        menu.append(nodename)
-
-
-    def addimage(self,imagename):
-        self.filenames[imagename] = imagename
-
-    def index(self, args, nodename):
-        self.indexlist.append((args,nodename))
-
-    def beginmenu(self):
-        pass
-
-    def endmenu(self):
-        pass
-
-    def finalize(self):
-        if not self.helpbase:
-            return
-
-        # generate interesting filenames
-        resultfile   = self.helpbase + '.chm'
-        projectfile  = self.helpbase + '.hhp'
-        contentfile  = self.helpbase + '.hhc'
-        indexfile    = self.helpbase + '.hhk'
-
-        # generate a reasonable title
-        title        = self.helpbase
-
-        # get the default topic file
-        (topname,topnext,topprev,topup,topfile) = self.nodelist[0]
-        defaulttopic = topfile
-
-        # PROJECT FILE
-        try:
-            with open(projectfile, 'w') as fp:
-                print('[OPTIONS]', file=fp)
-                print('Auto Index=Yes', file=fp)
-                print('Binary TOC=No', file=fp)
-                print('Binary Index=Yes', file=fp)
-                print('Compatibility=1.1', file=fp)
-                print('Compiled file=' + resultfile + '', file=fp)
-                print('Contents file=' + contentfile + '', file=fp)
-                print('Default topic=' + defaulttopic + '', file=fp)
-                print('Error log file=ErrorLog.log', file=fp)
-                print('Index file=' + indexfile + '', file=fp)
-                print('Title=' + title + '', file=fp)
-                print('Display compile progress=Yes', file=fp)
-                print('Full-text search=Yes', file=fp)
-                print('Default window=main', file=fp)
-                print('', file=fp)
-                print('[WINDOWS]', file=fp)
-                print('main=,"' + contentfile + '","' + indexfile
-                            + '","","",,,,,0x23520,222,0x1046,[10,10,780,560],'
-                            '0xB0000,,,,,,0', file=fp)
-                print('', file=fp)
-                print('[FILES]', file=fp)
-                print('', file=fp)
-                self.dumpfiles(fp)
-        except IOError as msg:
-            print(projectfile, ':', msg)
-            sys.exit(1)
-
-        # CONTENT FILE
-        try:
-            with open(contentfile, 'w') as fp:
-                print('', file=fp)
-                print('', file=fp)
-                print('', file=fp)
-                print('', file=fp)
-                print('', file=fp)
-                print('', file=fp)
-                print('', file=fp)
-                print('', file=fp)
-                print('   ', file=fp)
-                print('     ', file=fp)
-                print('     ', file=fp)
-                print('     ', file=fp)
-                print('   ', file=fp)
-                self.dumpnodes(fp)
-                print('', file=fp)
-                print('', file=fp)
-        except IOError as msg:
-            print(contentfile, ':', msg)
-            sys.exit(1)
-
-        # INDEX FILE
-        try:
-            with open(indexfile, 'w') as fp:
-                print('', file=fp)
-                print('', file=fp)
-                print('', file=fp)
-                print('', file=fp)
-                print('', file=fp)
-                print('', file=fp)
-                print('', file=fp)
-                print('', file=fp)
-                print('', file=fp)
-                print('', file=fp)
-                self.dumpindex(fp)
-                print('', file=fp)
-                print('', file=fp)
-        except IOError as msg:
-            print(indexfile  , ':', msg)
-            sys.exit(1)
-
-    def dumpfiles(self, outfile=sys.stdout):
-        filelist = sorted(self.filenames.values())
-        for filename in filelist:
-            print(filename, file=outfile)
-
-    def dumpnodes(self, outfile=sys.stdout):
-        self.dumped = {}
-        if self.nodelist:
-            nodename, dummy, dummy, dummy, dummy = self.nodelist[0]
-            self.topnode = nodename
-
-        print('
      ', file=outfile)
-        for node in self.nodelist:
-            self.dumpnode(node,0,outfile)
-        print('
    ', file=outfile)
-
-    def dumpnode(self, node, indent=0, outfile=sys.stdout):
-        if node:
-            # Retrieve info for this node
-            (nodename,next,prev,up,filename) = node
-            self.current = nodename
-
-            # Have we been dumped already?
-            if nodename in self.dumped:
-                return
-            self.dumped[nodename] = 1
-
-            # Print info for this node
-            print(' '*indent, end=' ', file=outfile)
-            print('
  • ', end=' ', file=outfile)
-            print('', end=' ', file=outfile)
-            print('', end=' ', file=outfile)
-            print('', file=outfile)
-
-            # Does this node have menu items?
-            try:
-                menu = self.menudict[nodename]
-                self.dumpmenu(menu,indent+2,outfile)
-            except KeyError:
-                pass
-
-    def dumpmenu(self, menu, indent=0, outfile=sys.stdout):
-        if menu:
-            currentnode = self.current
-            if currentnode != self.topnode:    # XXX this is a hack
-                print(' '*indent + '
      ', file=outfile)
-                indent += 2
-            for item in menu:
-                menunode = self.getnode(item)
-                self.dumpnode(menunode,indent,outfile)
-            if currentnode != self.topnode:    # XXX this is a hack
-                print(' '*indent + '
    ', file=outfile)
-                indent -= 2
-
-    def getnode(self, nodename):
-        try:
-            index = self.nodeindex[nodename]
-            return self.nodelist[index]
-        except KeyError:
-            return None
-        except IndexError:
-            return None
-
-    # (args,nodename) == (key,location)
-    def dumpindex(self, outfile=sys.stdout):
-        print('
      ', file=outfile)
-        for (key,location) in self.indexlist:
-            key = self.codeexpand(key)
-            location = makefile(location)
-            location = self.dirname + '/' + location
-            print('
    • ', end=' ', file=outfile)
-            print('', end=' ', file=outfile)
-            print('', end=' ', file=outfile)
-            print('', file=outfile)
-        print('
    ', file=outfile)
-
-    def codeexpand(self, line):
-        co = self.codeprog.match(line)
-        if not co:
-            return line
-        bgn, end = co.span(0)
-        a, b = co.span(1)
-        line = line[:bgn] + line[a:b] + line[end:]
-        return line
-
-
-# Put @var{} around alphabetic substrings
-def makevar(str):
-    return '@var{'+str+'}'
-
-
-# Split a string in "words" according to findwordend
-def splitwords(str, minlength):
-    words = []
-    i = 0
-    n = len(str)
-    while i < n:
-        while i < n and str[i] in ' \t\n': i = i+1
-        if i >= n: break
-        start = i
-        i = findwordend(str, i, n)
-        words.append(str[start:i])
-    while len(words) < minlength: words.append('')
-    return words
-
-
-# Find the end of a "word", matching braces and interpreting @@ @{ @}
-fwprog = re.compile('[@{} ]')
-def findwordend(str, i, n):
-    level = 0
-    while i < n:
-        mo = fwprog.search(str, i)
-        if not mo:
-            break
-        i = mo.start()
-        c = str[i]; i = i+1
-        if c == '@': i = i+1 # Next character is not special
-        elif c == '{': level = level+1
-        elif c == '}': level = level-1
-        elif c == ' ' and level <= 0: return i-1
-    return n
-
-
-# Convert a node name into a file name
-def makefile(nodename):
-    nodename = nodename.strip()
-    return fixfunnychars(nodename) + '.html'
-
-
-# Characters that are perfectly safe in filenames and hyperlinks
-goodchars = string.ascii_letters + string.digits + '!@-=+.'
-
-# Replace characters that aren't perfectly safe by dashes
-# Underscores are bad since Cern HTTPD treats them as delimiters for
-# encoding times, so you get mismatches if you compress your files:
-# a.html.gz will map to a_b.html.gz
-def fixfunnychars(addr):
-    i = 0
-    while i < len(addr):
-        c = addr[i]
-        if c not in goodchars:
-            c = '-'
-            addr = addr[:i] + c + addr[i+1:]
-        i = i + len(c)
-    return addr
-
-
-# Increment a string used as an enumeration
-def increment(s):
-    if not s:
-        return '1'
-    for sequence in string.digits, string.ascii_lowercase, string.ascii_uppercase:
-        lastc = s[-1]
-        if lastc in sequence:
-            i = sequence.index(lastc) + 1
-            if i >= len(sequence):
-                if len(s) == 1:
-                    s = sequence[0]*2
-                    if s == '00':
-                        s = '10'
-                else:
-                    s = increment(s[:-1]) + sequence[0]
-            else:
-                s = s[:-1] + sequence[i]
-            return s
-    return s # Don't increment
-
-
-def test():
-    import sys
-    debugging = 0
-    print_headers = 0
-    cont = 0
-    html3 = 0
-    htmlhelp = ''
-
-    while sys.argv[1] == ['-d']:
-        debugging = debugging + 1
-        del sys.argv[1]
-    if sys.argv[1] == '-p':
-        print_headers = 1
-        del sys.argv[1]
-    if sys.argv[1] == '-c':
-        cont = 1
-        del sys.argv[1]
-    if sys.argv[1] == '-3':
-        html3 = 1
-        del sys.argv[1]
-    if sys.argv[1] == '-H':
-        helpbase = sys.argv[2]
-        del sys.argv[1:3]
-    if len(sys.argv) != 3:
-        print('usage: texi2hh [-d [-d]] [-p] [-c] [-3] [-H htmlhelp]', \
-              'inputfile outputdirectory')
-        sys.exit(2)
-
-    if html3:
-        parser = TexinfoParserHTML3()
-    else:
-        parser = TexinfoParser()
-    parser.cont = cont
-    parser.debugging = debugging
-    parser.print_headers = print_headers
-
-    file = sys.argv[1]
-    dirname  = sys.argv[2]
-    parser.setdirname(dirname)
-    parser.setincludedir(os.path.dirname(file))
-
-    htmlhelp = HTMLHelp(helpbase, dirname)
-    parser.sethtmlhelp(htmlhelp)
-
-    try:
-        fp = open(file, 'r')
-    except IOError as msg:
-        print(file, ':', msg)
-        sys.exit(1)
-
-    with fp:
-        parser.parse(fp)
-    parser.report()
-
-    htmlhelp.finalize()
-
-
-if __name__ == "__main__":
-    test()
diff --git a/Tools/scripts/untabify.py b/Tools/scripts/untabify.py
deleted file mode 100755
index 861c83ced90f24..00000000000000
--- a/Tools/scripts/untabify.py
+++ /dev/null
@@ -1,55 +0,0 @@
-#! /usr/bin/env python3
-
-"Replace tabs with spaces in argument files.  Print names of changed files."
-
-import os
-import sys
-import getopt
-import tokenize
-
-def main():
-    tabsize = 8
-    try:
-        opts, args = getopt.getopt(sys.argv[1:], "t:")
-        if not args:
-            raise getopt.error("At least one file argument required")
-    except getopt.error as msg:
-        print(msg)
-        print("usage:", sys.argv[0], "[-t tabwidth] file ...")
-        return
-    for optname, optvalue in opts:
-        if optname == '-t':
-            tabsize = int(optvalue)
-
-    for filename in args:
-        process(filename, tabsize)
-
-
-def process(filename, tabsize, verbose=True):
-    try:
-        with tokenize.open(filename) as f:
-            text = f.read()
-            encoding = f.encoding
-    except IOError as msg:
-        print("%r: I/O error: %s" % (filename, msg))
-        return
-    newtext = text.expandtabs(tabsize)
-    if newtext == text:
-        return
-    backup = filename + "~"
-    try:
-        os.unlink(backup)
-    except OSError:
-        pass
-    try:
-        os.rename(filename, backup)
-    except OSError:
-        pass
-    with open(filename, "w", encoding=encoding) as f:
-        f.write(newtext)
-    if verbose:
-        print(filename)
-
-
-if __name__ == '__main__':
-    main()
diff --git a/Tools/scripts/which.py b/Tools/scripts/which.py
deleted file mode 100755
index b42e07c74ecac8..00000000000000
--- a/Tools/scripts/which.py
+++ /dev/null
@@ -1,61 +0,0 @@
-#! /usr/bin/env python3
-
-# Variant of "which".
-# On stderr, near and total misses are reported.
-# '-l' argument adds ls -l of each file found.
-
-import sys
-if sys.path[0] in (".", ""): del sys.path[0]
-
-import sys, os
-from stat import *
-
-def msg(str):
-    sys.stderr.write(str + '\n')
-
-def main():
-    pathlist = os.environ['PATH'].split(os.pathsep)
-
-    sts = 0
-    longlist = ''
-
-    if sys.argv[1:] and sys.argv[1][:2] == '-l':
-        longlist = sys.argv[1]
-        del sys.argv[1]
-
-    for prog in sys.argv[1:]:
-        ident = ()
-        for dir in pathlist:
-            filename = os.path.join(dir, prog)
-            try:
-                st = os.stat(filename)
-            except OSError:
-                continue
-            if not S_ISREG(st[ST_MODE]):
-                msg(filename + ': not a disk file')
-            else:
-                mode = S_IMODE(st[ST_MODE])
-                if mode & 0o111:
-                    if not ident:
-                        print(filename)
-                        ident = st[:3]
-                    else:
-                        if st[:3] == ident:
-                            s = 'same as: '
-                        else:
-                            s = 'also: '
-                        msg(s + filename)
-                else:
-                    msg(filename + ': not executable')
-            if longlist:
-                sts = os.system('ls ' + longlist + ' ' + filename)
-                sts = os.waitstatus_to_exitcode(sts)
-                if sts: msg('"ls -l" exit status: ' + repr(sts))
-        if not ident:
-            msg(prog + ': not found')
-            sts = 1
-
-    sys.exit(sts)
-
-if __name__ == '__main__':
-    main()