From 535c079dca7a8343dafcb1f9f5ce7486e2cbf953 Mon Sep 17 00:00:00 2001 From: Ezio Melotti Date: Sun, 8 May 2022 21:12:24 +0200 Subject: [PATCH 1/2] Update files related to HTML entities. --- .github/CODEOWNERS | 8 +++++--- Doc/library/html.entities.rst | 4 ++-- Lib/html/entities.py | 9 ++++++--- Tools/scripts/parse_html5_entities.py | 27 ++++++++++++++++++--------- Tools/scripts/parseentities.py | 2 ++ 5 files changed, 33 insertions(+), 17 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 62ee6f89cda679..8ff2e9ae6fce35 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -50,9 +50,11 @@ Python/pythonrun.c @iritkatriel /PC/launcher.c @vsajip # HTML -/Lib/html/ @ezio-melotti -/Lib/_markupbase.py @ezio-melotti -/Lib/test/test_html*.py @ezio-melotti +/Lib/html/ @ezio-melotti +/Lib/_markupbase.py @ezio-melotti +/Lib/test/test_html*.py @ezio-melotti +/Tools/scripts/parseentities.py @ezio-melotti +/Tools/scripts/parse_html5_entities.py @ezio-melotti # Import (including importlib). # Ignoring importlib.h so as to not get flagged on diff --git a/Doc/library/html.entities.rst b/Doc/library/html.entities.rst index 7d836fe7380245..0719655d35455b 100644 --- a/Doc/library/html.entities.rst +++ b/Doc/library/html.entities.rst @@ -34,12 +34,12 @@ This module defines four dictionaries, :data:`html5`, .. data:: name2codepoint - A dictionary that maps HTML entity names to the Unicode code points. + A dictionary that maps HTML4 entity names to the Unicode code points. .. data:: codepoint2name - A dictionary that maps Unicode code points to HTML entity names. + A dictionary that maps Unicode code points to HTML4 entity names. .. rubric:: Footnotes diff --git a/Lib/html/entities.py b/Lib/html/entities.py index dc508631ac4789..cc59bc314499ad 100644 --- a/Lib/html/entities.py +++ b/Lib/html/entities.py @@ -3,8 +3,7 @@ __all__ = ['html5', 'name2codepoint', 'codepoint2name', 'entitydefs'] -# maps the HTML entity name to the Unicode code point -# from https://html.spec.whatwg.org/multipage/named-characters.html +# maps HTML4 entity name to the Unicode code point name2codepoint = { 'AElig': 0x00c6, # latin capital letter AE = latin capital ligature AE, U+00C6 ISOlat1 'Aacute': 0x00c1, # latin capital letter A with acute, U+00C1 ISOlat1 @@ -261,7 +260,11 @@ } -# maps the HTML5 named character references to the equivalent Unicode character(s) +# HTML5 named character references +# Generated by 'Tools/scripts/parse_html5_entities.py' +# from https://html.spec.whatwg.org/entities.json and +# https://html.spec.whatwg.org/multipage/named-characters.html. +# Map HTML5 named character references to the equivalent Unicode character(s). html5 = { 'Aacute': '\xc1', 'aacute': '\xe1', diff --git a/Tools/scripts/parse_html5_entities.py b/Tools/scripts/parse_html5_entities.py index c011328b0101bf..1e5bdad2165548 100755 --- a/Tools/scripts/parse_html5_entities.py +++ b/Tools/scripts/parse_html5_entities.py @@ -2,10 +2,14 @@ """ Utility for parsing HTML5 entity definitions available from: - http://dev.w3.org/html5/spec/entities.json + https://html.spec.whatwg.org/entities.json + https://html.spec.whatwg.org/multipage/named-characters.html -Written by Ezio Melotti and Iuliia Proskurnia. +The page now contains the following note: + + "This list is static and will not be expanded or changed in the future." +Written by Ezio Melotti and Iuliia Proskurnia. """ import os @@ -14,7 +18,9 @@ from urllib.request import urlopen from html.entities import html5 -entities_url = 'http://dev.w3.org/html5/spec/entities.json' +PAGE_URL = 'https://html.spec.whatwg.org/multipage/named-characters.html' +ENTITIES_URL = 'https://html.spec.whatwg.org/entities.json' +HTML5_SECTION_START = '# HTML5 named character references' def get_json(url): """Download the json file from the url and returns a decoded object.""" @@ -62,9 +68,15 @@ def write_items(entities, file=sys.stdout): # be before their equivalent lowercase version. keys = sorted(entities.keys()) keys = sorted(keys, key=str.lower) + print(HTML5_SECTION_START, file=file) + print(f'# Generated by {sys.argv[0]!r}\n' + f'# from {ENTITIES_URL} and\n' + f'# {PAGE_URL}.\n' + f'# Map HTML5 named character references to the ' + f'equivalent Unicode character(s).', file=file) print('html5 = {', file=file) for name in keys: - print(' {!r}: {!a},'.format(name, entities[name]), file=file) + print(f' {name!r}: {entities[name]!a},', file=file) print('}', file=file) @@ -72,11 +84,8 @@ def write_items(entities, file=sys.stdout): # without args print a diff between html.entities.html5 and new_html5 # with --create print the new html5 dict # with --patch patch the Lib/html/entities.py file - new_html5 = create_dict(get_json(entities_url)) + new_html5 = create_dict(get_json(ENTITIES_URL)) if '--create' in sys.argv: - print('# map the HTML5 named character references to the ' - 'equivalent Unicode character(s)') - print('# Generated by {}. Do not edit manually.'.format(__file__)) write_items(new_html5) elif '--patch' in sys.argv: fname = 'Lib/html/entities.py' @@ -84,7 +93,7 @@ def write_items(entities, file=sys.stdout): with open(fname) as f1, open(temp_fname, 'w') as f2: skip = False for line in f1: - if line.startswith('html5 = {'): + if line.startswith(HTML5_SECTION_START): write_items(new_html5, file=f2) skip = True continue diff --git a/Tools/scripts/parseentities.py b/Tools/scripts/parseentities.py index 0229d3af86ba78..45cc5699010b25 100755 --- a/Tools/scripts/parseentities.py +++ b/Tools/scripts/parseentities.py @@ -11,6 +11,8 @@ Marc-Andre Lemburg, mal@lemburg.com, 1999. Use as you like. NO WARRANTIES. + See parse_html5_entities.py for HTML5 entity definitions. + """ import re,sys From fd9635fb6c27068afd9c986fc6923a5b5aba18cd Mon Sep 17 00:00:00 2001 From: Ezio Melotti Date: Sun, 19 Jun 2022 15:07:13 +0200 Subject: [PATCH 2/2] Remove the now-obsolete `parseentities.py` script. --- .github/CODEOWNERS | 9 ++- ...2-06-19-14-56-33.gh-issue-86087.R8MkRy.rst | 2 + Tools/scripts/parseentities.py | 66 ------------------- 3 files changed, 6 insertions(+), 71 deletions(-) create mode 100644 Misc/NEWS.d/next/Tools-Demos/2022-06-19-14-56-33.gh-issue-86087.R8MkRy.rst delete mode 100755 Tools/scripts/parseentities.py diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 8ff2e9ae6fce35..db7d322c7b714a 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -50,11 +50,10 @@ Python/pythonrun.c @iritkatriel /PC/launcher.c @vsajip # HTML -/Lib/html/ @ezio-melotti -/Lib/_markupbase.py @ezio-melotti -/Lib/test/test_html*.py @ezio-melotti -/Tools/scripts/parseentities.py @ezio-melotti -/Tools/scripts/parse_html5_entities.py @ezio-melotti +/Lib/html/ @ezio-melotti +/Lib/_markupbase.py @ezio-melotti +/Lib/test/test_html*.py @ezio-melotti +/Tools/scripts/*html5* @ezio-melotti # Import (including importlib). # Ignoring importlib.h so as to not get flagged on diff --git a/Misc/NEWS.d/next/Tools-Demos/2022-06-19-14-56-33.gh-issue-86087.R8MkRy.rst b/Misc/NEWS.d/next/Tools-Demos/2022-06-19-14-56-33.gh-issue-86087.R8MkRy.rst new file mode 100644 index 00000000000000..a10d4c76884522 --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2022-06-19-14-56-33.gh-issue-86087.R8MkRy.rst @@ -0,0 +1,2 @@ +The ``Tools/scripts/parseentities.py`` script used to parse HTML4 entities +has been removed. diff --git a/Tools/scripts/parseentities.py b/Tools/scripts/parseentities.py deleted file mode 100755 index 45cc5699010b25..00000000000000 --- a/Tools/scripts/parseentities.py +++ /dev/null @@ -1,66 +0,0 @@ -#!/usr/bin/env python3 -""" Utility for parsing HTML entity definitions available from: - - http://www.w3.org/ as e.g. - http://www.w3.org/TR/REC-html40/HTMLlat1.ent - - Input is read from stdin, output is written to stdout in form of a - Python snippet defining a dictionary "entitydefs" mapping literal - entity name to character or numeric entity. - - Marc-Andre Lemburg, mal@lemburg.com, 1999. - Use as you like. NO WARRANTIES. - - See parse_html5_entities.py for HTML5 entity definitions. - -""" -import re,sys - -entityRE = re.compile(r'') - -def parse(text,pos=0,endpos=None): - - pos = 0 - if endpos is None: - endpos = len(text) - d = {} - while 1: - m = entityRE.search(text,pos,endpos) - if not m: - break - name,charcode,comment = m.groups() - d[name] = charcode,comment - pos = m.end() - return d - -def writefile(f,defs): - - f.write("entitydefs = {\n") - items = sorted(defs.items()) - for name, (charcode,comment) in items: - if charcode[:2] == '&#': - code = int(charcode[2:-1]) - if code < 256: - charcode = r"'\%o'" % code - else: - charcode = repr(charcode) - else: - charcode = repr(charcode) - comment = ' '.join(comment.split()) - f.write(" '%s':\t%s, \t# %s\n" % (name,charcode,comment)) - f.write('\n}\n') - -if __name__ == '__main__': - if len(sys.argv) > 1: - with open(sys.argv[1]) as infile: - text = infile.read() - else: - text = sys.stdin.read() - - defs = parse(text) - - if len(sys.argv) > 2: - with open(sys.argv[2],'w') as outfile: - writefile(outfile, defs) - else: - writefile(sys.stdout, defs)