diff --git a/.gitignore b/.gitignore index 2f5e4a37b8a..e59c1b4349f 100644 --- a/.gitignore +++ b/.gitignore @@ -29,8 +29,8 @@ dlldata.c .sconsign.dblite user_docs/*/*.html user_docs/*/*.css -user_docs/*/keyCommands.t2t -user_docs/build.t2tConf +# TODO remove when migrating from t2t to md as markdown files will be committed. +user_docs/*/*.md extras/controllerClient/x86/nvdaController.h extras/controllerClient/x64 extras/controllerClient/arm64 diff --git a/keyCommandsDoc.py b/keyCommandsDoc.py index 170c3dfd72d..514788165eb 100644 --- a/keyCommandsDoc.py +++ b/keyCommandsDoc.py @@ -1,181 +1,186 @@ # -*- coding: UTF-8 -*- -#keyCommandsDoc.py -#A part of NonVisual Desktop Access (NVDA) -#This file is covered by the GNU General Public License. -#See the file COPYING for more details. -#Copyright (C) 2010-2019 NV Access Limited, Mesar Hameed, Takuya Nishimoto +# A part of NonVisual Desktop Access (NVDA) +# Copyright (C) 2023-2024 NV Access Limited, Mesar Hameed, Takuya Nishimoto +# This file is covered by the GNU General Public License. +# See the file COPYING for more details. -"""Utilities related to NVDA Key Commands documents. """ +TODO: move to site_scons/site_tools -import os -import codecs +Generates the Key Commands document from the User Guide. +Works as a Python Markdown Extension: +https://python-markdown.github.io/extensions/ + +TODO move docs to docs folder + +Generation of the Key Commands document requires certain commands to be included in the user guide. +These commands must begin at the start of the line and take the form:: + %kc:command: arg + +The kc:title command must appear first and specifies the title of the Key Commands document. +For example: + %kc:title: NVDA Key Commands + +The rest of these commands are used to include key commands into the document. +Appropriate headings from the User Guide will be included implicitly. + +The kc:beginInclude command begins a block of text which should be included verbatim. +The block ends at the kc:endInclude command. +For example:: + %kc:beginInclude + || Name | Desktop command | Laptop command | Description | + ... + %kc:endInclude + +The kc:settingsSection command indicates the beginning of a section documenting individual settings. +It specifies the header row for a table summarising the settings indicated by the kc:setting command +(see below). +In order, it must consist of a name column, a column for each keyboard layout and a description column. +For example:: + %kc:settingsSection: || Name | Desktop command | Laptop command | Description | + +The kc:setting command indicates a section for an individual setting. +It must be followed by: + * A heading containing the name of the setting; + * A table row for each keyboard layout, or if the key is common to all layouts, + a single line of text specifying the key after a colon; + * A blank line; and + * A line describing the setting. +For example: + %kc:setting + ==== Braille Tethered To ==== + | Desktop command | NVDA+control+t | + | Laptop Command | NVDA+control+t | + This option allows you to choose whether the braille display will follow the system focus, + or whether it follows the navigator object / review cursor. +""" + +from enum import auto, Enum, IntEnum, StrEnum import re -import txt2tags +from collections.abc import Iterator + +from markdown import Extension, Markdown +from markdown.preprocessors import Preprocessor + + +LINE_END = "\r\n" + + +class Section(IntEnum): + """Sections must be nested in this order.""" + HEADER = auto() + BODY = auto() + + +class Command(StrEnum): + TITLE = "title" + BEGIN_INCLUDE = "beginInclude" + END_INCLUDE = "endInclude" + SETTING = "setting" + SETTINGS_SECTION = "settingsSection" + + def t2tRegex(self) -> re.Pattern: + return re.compile(rf"%kc:({self.value}.*)") + + +class Regex(Enum): + COMMAND = re.compile(r"^$") + HEADING = re.compile(r"^(?P#+)(?P.*)$") + SETTING_SINGLE_KEY = re.compile(r"^[^|]+?[::]\s*(.+?)\s*$") + TABLE_ROW = re.compile(r"^(\|.*\|)$") -LINE_END = u"\r\n" class KeyCommandsError(Exception): """Raised due to an error encountered in the User Guide related to generation of the Key Commands document. """ -class KeyCommandsMaker(object): - """Generates the Key Commands document from the User Guide. - To generate a Key Commands document, create an instance and then call L{make} on it. - - Generation of the Key Commands document requires certain commands to be included in the user guide. - These commands must begin at the start of the line and take the form:: - %kc:command: arg - - The kc:title command must appear first and specifies the title of the Key Commands document. For example:: - %kc:title: NVDA Key Commands - - The kc:includeconf command allows you to insert a txt2tags includeconf command in the Key Commands document. For example:: - %kc:includeconf: ../ar.t2tconf - You may use multiple kc:includeconf commands, but they must appear before any of the other commands below. - - The rest of these commands are used to include key commands into the document. - Appropriate headings from the User Guide will be included implicitly. - - The kc:beginInclude command begins a block of text which should be included verbatim. - The block ends at the kc:endInclude command. - For example:: - %kc:beginInclude - || Name | Desktop command | Laptop command | Description | - ... - %kc:endInclude - - The kc:settingsSection command indicates the beginning of a section documenting individual settings. - It specifies the header row for a table summarising the settings indicated by the kc:setting command (see below). - In order, it must consist of a name column, a column for each keyboard layout and a description column. - For example:: - %kc:settingsSection: || Name | Desktop command | Laptop command | Description | - - The kc:setting command indicates a section for an individual setting. - It must be followed by: - * A heading containing the name of the setting; - * A table row for each keyboard layout, or if the key is common to all layouts, a single line of text specifying the key after a colon; - * A blank line; and - * A line describing the setting. - For example:: - %kc:setting - ==== Braille Tethered To ==== - | Desktop command | NVDA+control+t | - | Laptop Command | NVDA+control+t | - This option allows you to choose whether the braille display will follow the system focus, or whether it follows the navigator object / review cursor. - """ - t2tRe = None - RE_COMMAND = re.compile(r"^%kc:(?P[^:\s]+)(?:: (?P.*))?$") - KCSECT_HEADER = 0 - KCSECT_CONFIG = 1 - KCSECT_BODY = 2 +class KeyCommandsExtension(Extension): + # Magic number, priorities are not well documented. + # It's unclear what the range of priorities are to compare to, but 25 seems to work, 1 doesn't. + # See https://python-markdown.github.io/extensions/api/#registries + PRIORITY = 25 - @classmethod - def _initClass(cls): - if cls.t2tRe: - return - # Only fetch this once. - cls.t2tRe = txt2tags.getRegexes() - - def __init__(self, userGuideFilename,keyCommandsFileName): - """Constructor. - @param userGuideFilename: The file name of the User Guide to be used as input. - @type userGuideFilename: str - @param keyCommandsFilename: The file name of the key commands file to be output. - @type keyCommandsFilename: str - """ - self._initClass() - self.ugFn = userGuideFilename - #: The file name of the Key Commands document that will be generated. - #: This will be in the same directory as the User Guide. - self.kcFn = keyCommandsFileName + def extendMarkdown(self, md: Markdown): + md.preprocessors.register(KeyCommandsPreprocessor(md), 'key_commands', self.PRIORITY) + + +class KeyCommandsPreprocessor(Preprocessor): + def __init__(self, md: Markdown | None): + super().__init__(md) + self.initialize() + + def initialize(self): + self._ugLines: Iterator[str] = iter(()) + self._kcLines: list[str] = [] #: The current section of the key commands file. - self._kcSect = self.KCSECT_HEADER + self._kcSect: Section = Section.HEADER #: The current stack of headings. - self._headings = [] + self._headings: list[re.Match] = [] #: The 0 based level of the last heading in L{_headings} written to the key commands file. - self._kcLastHeadingLevel = -1 + self._kcLastHeadingLevel: int = -1 #: Whether lines which aren't commands should be written to the key commands file as is. - self._kcInclude = False + self._kcInclude: bool = False #: The header row for settings sections. - self._settingsHeaderRow = None + self._settingsHeaderRow: str | None = None #: The number of layouts for settings in a settings section. - self._settingsNumLayouts = 0 + self._settingsNumLayouts: int = 0 #: The current line number being processed, used to present location of syntax errors - self._lineNum = 0 - - def make(self): - """Generate the Key Commands document. - @postcondition: If the User Guide contains appropriate commands, the Key Commands document will be generated and saved as L{kcFn}. - Otherwise, no file will be generated. - @return: C{True} if a document was generated, C{False} otherwise. - @rtype: bool - @raise IOError: - @raise KeyCommandsError: - """ - tKcFn=self.kcFn+'__' - self._ug = codecs.open(self.ugFn, "r", "utf-8-sig") - self._kc = codecs.open(tKcFn, "w", "utf-8-sig") - - success=False - with self._ug, self._kc: - self._make() - success=self._kc.tell() > 0 - if success: - os.rename(tKcFn,self.kcFn) - else: - os.remove(tKcFn) - return success + self._lineNum: int = 0 + # We want to skip the title line to replace it with the KC:TITLE command argument. + self._skippedTitle = False - def _make(self): - for line in self._ug: + def run(self, lines: list[str]) -> list[str]: + # Turn this into an iterator so we can use next() to seek through lines. + self._ugLines = iter(lines) + for line in self._ugLines: + line = line.strip() self._lineNum += 1 - line = line.rstrip() - m = self.RE_COMMAND.match(line) + + # We want to skip the title line to replace it with the KC:TITLE command argument. + if line.startswith("# ") and not self._skippedTitle: + self._skippedTitle = True + continue + + m = Regex.COMMAND.value.match(line) if m: self._command(**m.groupdict()) continue - m = self.t2tRe["numtitle"].match(line) + m = Regex.HEADING.value.match(line) if m: self._heading(m) continue if self._kcInclude: - self._kc.write(line + LINE_END) + self._kcLines.append(line) - def _command(self, cmd=None, arg=None): + return self._kcLines.copy() + + def _command(self, cmd: Command | None = None, arg: str | None = None): # Handle header commands. - if cmd == "title": - if self._kcSect > self.KCSECT_HEADER: - raise KeyCommandsError("%d, title command is not valid here" % self._lineNum) + if cmd == Command.TITLE.value: + if self._kcSect > Section.HEADER: + raise KeyCommandsError(f"{self._lineNum}, title command is not valid here") # Write the title and two blank lines to complete the txt2tags header section. - self._kc.write(arg + LINE_END * 3) - self._kcSect = self.KCSECT_CONFIG - self._kc.write("%%!includeconf: ../global.t2tconf%s" % LINE_END) - return - elif self._kcSect == self.KCSECT_HEADER: - raise KeyCommandsError("%d, title must be the first command" % self._lineNum) - elif cmd == "includeconf": - if self._kcSect > self.KCSECT_CONFIG: - raise KeyCommandsError("%d, includeconf command is not valid here" % self._lineNum) - self._kc.write("%%!includeconf: %s%s" % (arg, LINE_END)) + self._kcLines.append("# " + arg + LINE_END * 2) + self._kcSect = Section.BODY return - elif self._kcSect == self.KCSECT_CONFIG: - self._kc.write(LINE_END) - self._kcSect = self.KCSECT_BODY - if cmd == "beginInclude": + elif self._kcSect == Section.HEADER: + raise KeyCommandsError(f"{self._lineNum}, title must be the first command") + + if cmd == Command.BEGIN_INCLUDE.value: self._writeHeadings() self._kcInclude = True - elif cmd == "endInclude": + elif cmd == Command.END_INCLUDE.value: self._kcInclude = False - self._kc.write(LINE_END) + self._kcLines.append("") - elif cmd == "settingsSection": + elif cmd == Command.SETTINGS_SECTION.value: # The argument is the table header row for the settings section. - self._settingsHeaderRow = arg + # Replace t2t header syntax with markdown syntax. + self._settingsHeaderRow = arg.replace("||", "|") # There are name and description columns. # Each of the remaining columns provides keystrokes for one layout. # There's one less delimiter than there are columns, hence subtracting 1 instead of 2. @@ -185,30 +190,37 @@ def _command(self, cmd=None, arg=None): f"{self._lineNum}, settingsSection command must specify the header row for a table" " summarising the settings" ) - elif cmd == "setting": + + elif cmd == Command.SETTING.value: self._handleSetting() else: - raise KeyCommandsError("%d, Invalid command %s" % (self._lineNum, cmd)) + raise KeyCommandsError(f"{self._lineNum}, Invalid command {cmd}") + + def _seekNonEmptyLine(self) -> str: + """Seeks to the next non-empty line in the user guide. + """ + line = next(self._ugLines).strip() + self._lineNum += 1 + while not line: + try: + line = next(self._ugLines).strip() + except StopIteration: + return line + self._lineNum += 1 + return line - def _areHeadingsPending(self): + def _areHeadingsPending(self) -> bool: return self._kcLastHeadingLevel < len(self._headings) - 1 def _writeHeadings(self): level = self._kcLastHeadingLevel + 1 # Only write headings we haven't yet written. for level, heading in enumerate(self._headings[level:], level): - # We don't want numbered headings in the output. - label=heading.group("label") - headingText = u"{id}{txt}{id}{label}".format( - id="=" * len(heading.group("id")), - txt=heading.group("txt"), - label="[%s]" % label if label else "") - # Write the heading and a blank line. - self._kc.write(headingText + LINE_END * 2) + self._kcLines.append(heading.group(0)) self._kcLastHeadingLevel = level - def _heading(self, m): + def _heading(self, m: re.Match): # We work with 0 based heading levels. level = len(m.group("id")) - 1 try: @@ -218,7 +230,6 @@ def _heading(self, m): self._headings.append(m) self._kcLastHeadingLevel = min(self._kcLastHeadingLevel, level - 1) - RE_SETTING_SINGLE_KEY = re.compile(r"^[^|]+?[::]\s*(.+?)\s*$") def _handleSetting(self): if not self._settingsHeaderRow: raise KeyCommandsError("%d, setting command cannot be used before settingsSection command" % self._lineNum) @@ -226,60 +237,65 @@ def _handleSetting(self): if self._areHeadingsPending(): # There are new headings to write. # If there was a previous settings table, it ends here, so write a blank line. - self._kc.write(LINE_END) + self._kcLines.append("") self._writeHeadings() # New headings were written, so we need to output the header row. - self._kc.write(self._settingsHeaderRow + LINE_END) + self._kcLines.append(self._settingsHeaderRow) + numCols = self._settingsNumLayouts + 2 # name + description + layouts + self._kcLines.append("|" + "---|" * numCols) # The next line should be a heading which is the name of the setting. - line = next(self._ug) - self._lineNum += 1 - m = self.t2tRe["title"].match(line) + line = self._seekNonEmptyLine() + m = Regex.HEADING.value.match(line) if not m: - raise KeyCommandsError("%d, setting command must be followed by heading" % self._lineNum) + raise KeyCommandsError(f"{self._lineNum}, setting command must be followed by heading") name = m.group("txt") # The next few lines should be table rows for each layout. # Alternatively, if the key is common to all layouts, there will be a single line of text specifying the key after a colon. - keys = [] - for layout in range(self._settingsNumLayouts): - line = next(self._ug).strip() - self._lineNum += 1 - m = self.RE_SETTING_SINGLE_KEY.match(line) + keys: list[str] = [] + for _layout in range(self._settingsNumLayouts): + line = self._seekNonEmptyLine() + + m = Regex.SETTING_SINGLE_KEY.value.match(line) if m: keys.append(m.group(1)) break - elif not self.t2tRe["table"].match(line): - raise KeyCommandsError("%d, setting command: There must be one table row for each keyboard layout" % self._lineNum) + elif not Regex.TABLE_ROW.value.match(line): + raise KeyCommandsError( + f"{self._lineNum}, setting command: " + "There must be one table row for each keyboard layout" + ) + # This is a table row. # The key will be the second column. - # TODO: Error checking. - key = line.strip("|").split("|")[1].strip() - keys.append(key) + try: + key = line.strip("|").split("|")[1].strip() + except IndexError: + raise KeyCommandsError(f"{self._lineNum}, setting command: Key entry not found in table row.") + else: + keys.append(key) + if 1 == len(keys) < self._settingsNumLayouts: # The key has only been specified once, so it is the same in all layouts. key = keys[0] - keys[1:] = (key for layout in range(self._settingsNumLayouts - 1)) + keys[1:] = (key for _layout in range(self._settingsNumLayouts - 1)) # There should now be a blank line. - line = next(self._ug).strip() + line = next(self._ugLines).strip() self._lineNum += 1 if line: - raise KeyCommandsError("%d, setting command: The keyboard shortcuts must be followed by a blank line. Multiple keys must be included in a table. Erroneous key: %s" % (self._lineNum, key)) + raise KeyCommandsError( + f"{self._lineNum}, setting command: The keyboard shortcuts must be followed by a blank line. " + "Multiple keys must be included in a table. " + f"Erroneous key: {key}" + ) # Finally, the next line should be the description. - desc = next(self._ug).strip() - self._lineNum += 1 - - self._kc.write(u"| {name} | {keys} | {desc} |{lineEnd}".format( - name=name, - keys=u" | ".join(keys), - desc=desc, lineEnd=LINE_END)) - - def remove(self): - """Remove the generated Key Commands document. - """ - try: - os.remove(self.kcFn) - except OSError: - pass + desc = self._seekNonEmptyLine() + self._kcLines.append(f"| {name} | {' | '.join(keys)} | {desc} |") + if not self._kcLines[-2].startswith("|"): + # The previous line was not a table, so this is a new table. + # Write the header row. + numCols = len(keys) + 2 # name + description + layouts + self._kcLines.append("|" + "---|" * numCols) diff --git a/miscDeps b/miscDeps index 0858efa727e..ec251b62be4 160000 --- a/miscDeps +++ b/miscDeps @@ -1 +1 @@ -Subproject commit 0858efa727e8eedc7e0ce4c159ae6a232234cf10 +Subproject commit ec251b62be456860ddb3b9c0346b1ea0dcc7fd67 diff --git a/projectDocs/dev/developerGuide/developerGuide.t2t b/projectDocs/dev/developerGuide/developerGuide.t2t index 2fbd33829cd..0b9d7ced24f 100644 --- a/projectDocs/dev/developerGuide/developerGuide.t2t +++ b/projectDocs/dev/developerGuide/developerGuide.t2t @@ -3,9 +3,6 @@ NVDA NVDA_VERSION Developer Guide %!includeconf: ../../../user_docs/userGuide.t2tconf -% Remove double spacing from the beginning of each line as txt2tags seems to indent preformatted text by two spaces -%!PostProc(html): '^ ' '' - = Table of Contents =[toc] %%toc diff --git a/projectDocs/dev/developerGuide/sconscript b/projectDocs/dev/developerGuide/sconscript index 46776b3bec9..c1dba7c73be 100644 --- a/projectDocs/dev/developerGuide/sconscript +++ b/projectDocs/dev/developerGuide/sconscript @@ -5,28 +5,36 @@ import sys -Import("env", "outputDir", "sourceDir", "t2tBuildConf") +Import("env", "outputDir", "sourceDir") env = env.Clone() devDocsOutputDir=outputDir.Dir('devDocs') #Build the developer guide and move it to the output directory -htmlFile=env.txt2tags('developerGuide.t2t') -env.Depends(htmlFile, t2tBuildConf) -env.SideEffect('_txt2tags',htmlFile) +mdFile = env.txt2tags('developerGuide.t2t') +env.SideEffect('_txt2tags', mdFile) +htmlFile = env.md2html('developerGuide.md') +env.Depends(htmlFile, mdFile) +devGuideMD = env.Command( + target=devDocsOutputDir.File('developerGuide.md'), + source=mdFile, + action=Move('$TARGET', '$SOURCE') +) +env.Depends(devGuideMD, mdFile) devGuide = env.Command( target=devDocsOutputDir.File('developerGuide.html'), source=htmlFile, - action=Move('$TARGET','$SOURCE') + action=Move('$TARGET', '$SOURCE') ) +env.Depends(devGuide, devGuideMD) env.Alias("developerGuide",devGuide) devDocs_nvdaHelper_temp=env.Doxygen(source='../../../nvdaHelper/doxyfile') devDocs_nvdaHelper = env.Command( target=devDocsOutputDir.Dir('nvdaHelper'), source=devDocs_nvdaHelper_temp, - action=Move('$TARGET','$SOURCE') + action=Move('$TARGET', '$SOURCE') ) env.Alias('devDocs_nvdaHelper', devDocs_nvdaHelper) env.Clean('devDocs_nvdaHelper', devDocs_nvdaHelper) diff --git a/requirements.txt b/requirements.txt index a27a4f916b3..962e26bd7b7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -19,6 +19,12 @@ git+https://github.com/py2exe/py2exe@4e7b2b2c60face592e67cb1bc935172a20fa371d#eg # Creating XML unit test reports unittest-xml-reporting==3.2.0 +# Building user documentation +Markdown==3.5.1 +mdx_truly_sane_lists==1.3 +markdown-link-attr-modifier==0.2.1 +mdx-gh-links==0.4 + # For building developer documentation sphinx==7.2.6 sphinx_rtd_theme==1.3.0 diff --git a/sconstruct b/sconstruct index d5af27f7485..0aefe68a08d 100755 --- a/sconstruct +++ b/sconstruct @@ -54,7 +54,6 @@ sys.path.append(sourceEnvPath) import sourceEnv sys.path.remove(sourceEnvPath) import time -from glob import glob import importlib.util import winreg @@ -84,20 +83,7 @@ makensis = os.path.abspath(os.path.join("include", "nsis", "NSIS", "makensis.exe # Get the path to xgettext. XGETTEXT = os.path.abspath(os.path.join("miscDeps", "tools", "xgettext.exe")) -def keyCommandsDocTool(env): - import keyCommandsDoc - kcdAction=env.Action( - lambda target,source,env: not keyCommandsDoc.KeyCommandsMaker(source[0].path,target[0].path).make(), - lambda target,source,env: 'Generating %s'%target[0], - ) - kcdBuilder=env.Builder( - action=kcdAction, - suffix='.t2t', - src_suffix='.t2t', - ) - env['BUILDERS']['keyCommandsDoc']=kcdBuilder - -keyCommandsLangBlacklist=set([]) +import txt2tags # noqa: F401, import required here to build docs vars = Variables() vars.Add("version", "The version of this build", versionInfo.version) @@ -117,8 +103,8 @@ vars.Add(EnumVariable('nvdaHelperLogLevel','The level of logging you wish to see env = Environment(variables=vars,HOST_ARCH='x86',tools=[ "textfile", "gettextTool", - "t2t" - ,keyCommandsDocTool, + "t2t", + "md2html", "doxygen", "recursiveInstall" ]) @@ -248,19 +234,6 @@ envArm64.SConscript('nvdaHelper/archBuild_sconscript',exports={'env':envArm64,'c for po in env.Glob(sourceDir.path+'/locale/*/lc_messages/*.po'): env.gettextMoFile(po) -#Allow all key command t2t files to be generated from their userGuide t2t sources -for lang in os.listdir(userDocsDir.path): - if lang in keyCommandsLangBlacklist: continue - for ug in glob(os.path.join(userDocsDir.path,lang,'userGuide.t2t')): - env.keyCommandsDoc(File(ug).File('keyCommands.t2t'),ug) - -t2tBuildConf = env.Textfile(userDocsDir.File("build.t2tConf"), [ - # We need to do this one as PostProc so it gets converted for the title. - r"%!PostProc: NVDA_VERSION {}".format(version), - r"%!PreProc: NVDA_URL {}".format(versionInfo.url), - r"%!PreProc: NVDA_COPYRIGHT_YEARS {}".format(versionInfo.copyrightYears), -]) - userGuideConf = os.path.join(userDocsDir.path, 'userGuide.t2tconf') globalConf = os.path.join(userDocsDir.path, 'global.t2tconf') changesConf = os.path.join(userDocsDir.path, 'changes.t2tconf') @@ -269,22 +242,33 @@ styles = os.path.join(userDocsDir.path, 'styles.css') #Allow all t2t files to be converted to html in user_docs #As we use scons Glob this will also include the keyCommands.t2t files for t2tFile in env.Glob(os.path.join(userDocsDir.path,'*','*.t2t')): - htmlFile=env.txt2tags(t2tFile) + mdFile = env.txt2tags(t2tFile) #txt2tags can not be run in parallel so make sure scons knows this - env.SideEffect('_txt2tags',htmlFile) - # All of our t2t files require build.t2tConf. - styleInstallPath = os.path.dirname(t2tFile.abspath) - installedStyle = env.Install(styleInstallPath, styles) + env.SideEffect('_txt2tags', mdFile) env.Depends( - htmlFile, + mdFile, [ - t2tBuildConf, userGuideConf, globalConf, changesConf, + ] + ) + htmlFile = env.md2html(t2tFile.abspath.replace(".t2t", ".md")) + styleInstallPath = os.path.dirname(t2tFile.abspath) + installedStyle = env.Install(styleInstallPath, styles) + env.Depends( + htmlFile, + [ styles, - installedStyle - ]) + installedStyle, + ] + ) + env.Depends(htmlFile, mdFile) + +# Create key commands files +for userGuideFile in env.Glob(os.path.join(userDocsDir.path, '*', 'userGuide.md')): + keyCommandsHtmlFile = env.md2html(userGuideFile.abspath.replace("userGuide.md", "keyCommands.html"), userGuideFile) + env.Depends(keyCommandsHtmlFile, userGuideFile) # Build unicode CLDR dictionaries env.SConscript('cldrDict_sconscript',exports=['env', 'sourceDir']) @@ -430,12 +414,24 @@ env.Alias("client", clientArchive) outputStylesFile=env.Command(outputDir.File("styles.css"),userDocsDir.File('styles.css'),Copy('$TARGET','$SOURCE')) changesFile=env.Command(outputDir.File("%s_changes.html" % outFilePrefix),userDocsDir.File('en/changes.html'),Copy('$TARGET','$SOURCE')) +changesMDFile = env.Command( + outputDir.File("changes.md"), + userDocsDir.File('en/changes.md'), + Copy('$TARGET','$SOURCE') +) +env.Depends(changesFile, changesMDFile) env.Depends(changesFile, outputStylesFile) env.Alias('changes',changesFile) userGuideFile=env.Command(outputDir.File("userGuide.html"),userDocsDir.File('en/userGuide.html'),Copy('$TARGET','$SOURCE')) +userGuideMDFile = env.Command( + outputDir.File("userGuide.md"), + userDocsDir.File('en/userGuide.md'), + Copy('$TARGET', '$SOURCE') +) +env.Depends(userGuideFile, userGuideMDFile) env.Depends(userGuideFile, outputStylesFile) -env.Alias('userGuide',userGuideFile) +env.Alias('userGuide', userGuideFile) keyCommandsFile = env.Command( outputDir.File("keyCommands.html"), @@ -494,7 +490,7 @@ def makePot(target, source, env): os.rename(tmpFn, potFn) -env.SConscript("projectDocs/dev/developerGuide/sconscript", exports=["env", "outputDir", "sourceDir", "t2tBuildConf"]) +env.SConscript("projectDocs/dev/developerGuide/sconscript", exports=["env", "outputDir", "sourceDir"]) def getSubDirs(path): diff --git a/site_scons/site_tools/md2html.py b/site_scons/site_tools/md2html.py new file mode 100644 index 00000000000..9048f760fc8 --- /dev/null +++ b/site_scons/site_tools/md2html.py @@ -0,0 +1,161 @@ +# A part of NonVisual Desktop Access (NVDA) +# Copyright (C) 2023 NV Access Limited +# This file may be used under the terms of the GNU General Public License, version 2 or later. +# For more details see: https://www.gnu.org/licenses/gpl-2.0.html + +from importlib.util import find_spec +import io +import pathlib +import re +import shutil + +import SCons.Node.FS +import SCons.Environment + +DEFAULT_EXTENSIONS = frozenset({ + # Supports tables, HTML mixed with markdown, code blocks and more + "markdown.extensions.extra", + # Allows TOC with [TOC], allows anchors set by "# Title {#foo}" + "markdown.extensions.toc", + # Allows setting attributes with {@id=foo} + "markdown.extensions.legacy_attrs", + # Adds code highlighting to code blocks + "markdown.extensions.codehilite", + # Makes list behaviour better, including 2 space indents by default + "mdx_truly_sane_lists", + # External links will open in a new tab, and title will be set to the link text + "markdown_link_attr_modifier", + # Adds links to GitHub authors, issues and PRs + "mdx_gh_links", +}) + +_extensionConfigs = { + "markdown_link_attr_modifier": { + "new_tab": "external_only", + "auto_title": "on", + }, + "mdx_gh_links": { + "user": "nvaccess", + "repo": "nvda", + }, +} + +_RTLlangCodes = {"ar", "fa", "he"} + + +def _replaceNVDATags(md: str, env: SCons.Environment.Environment) -> str: + import versionInfo + # Replace tags in source file + md = md.replace("NVDA_VERSION", env["version"]) + md = md.replace("NVDA_URL", versionInfo.url) + md = md.replace("NVDA_COPYRIGHT_YEARS", versionInfo.copyrightYears) + return md + + +def _getTitle(mdBuffer: io.BytesIO, isKeyCommands: bool = False) -> str: + if isKeyCommands: + TITLE_RE = re.compile(r"^$") + # Make next read at start of buffer + mdBuffer.seek(0) + for line in mdBuffer.readlines(): + match = TITLE_RE.match(line.decode("utf-8").strip()) + if match: + return match.group(1) + + raise ValueError("No KC:title command found in userGuide.md") + + else: + # Make next read at start of buffer + mdBuffer.seek(0) + # Remove heading hashes and trailing whitespace to get the tab title + title = mdBuffer.readline().decode("utf-8").strip().lstrip("# ") + + return title + + +def md2html_actionFunc( + target: list[SCons.Node.FS.File], + source: list[SCons.Node.FS.File], + env: SCons.Environment.Environment +): + import markdown + isKeyCommands = target[0].path.endswith("keyCommands.html") + + with open(source[0].path, "r", encoding="utf-8") as mdFile: + mdStr = mdFile.read() + + mdStr = _replaceNVDATags(mdStr, env) + + # Write replaced source to buffer. + # md has a bug with StringIO, so BytesIO is required. + mdBuffer = io.BytesIO() + mdBuffer.write(mdStr.encode("utf-8")) + + lang = pathlib.Path(source[0].path).parent.name + title = _getTitle(mdBuffer, isKeyCommands) + # md has a bug with StringIO, so BytesIO is required. + htmlBuffer = io.BytesIO() + htmlBuffer.write( + f""" + + + + +{title} + + + + + """.strip().encode("utf-8") + ) + + # Make next read from start of buffer + mdBuffer.seek(0) + # Make next write append at end of buffer + htmlBuffer.seek(0, io.SEEK_END) + + extensions = set(DEFAULT_EXTENSIONS) + if isKeyCommands: + from keyCommandsDoc import KeyCommandsExtension + extensions.add(KeyCommandsExtension()) + + markdown.markdownFromFile( + input=mdBuffer, + output=htmlBuffer, + # https://python-markdown.github.io/extensions/ + extensions=extensions, + extension_configs=_extensionConfigs, + ) + + # Make next write append at end of buffer + htmlBuffer.seek(0, io.SEEK_END) + htmlBuffer.write(b"\n\n\n") + + with open(target[0].path, "wb") as targetFile: + # Make next read at start of buffer + htmlBuffer.seek(0) + shutil.copyfileobj(htmlBuffer, targetFile) + + mdBuffer.close() + htmlBuffer.close() + + +def exists(env: SCons.Environment.Environment) -> bool: + for ext in [ + "markdown", + "markdown_link_attr_modifier", + "mdx_truly_sane_lists", + "mdx_gh_links", + "keyCommandsDoc", + ]: + if find_spec(ext) is None: + return False + return True + + +def generate(env: SCons.Environment.Environment): + env["BUILDERS"]["md2html"] = env.Builder( + action=env.Action(md2html_actionFunc, lambda t, s, e: f"Converting {s[0].path} to {t[0].path}"), + suffix=".html", + src_suffix=".md" + ) diff --git a/site_scons/site_tools/t2t.py b/site_scons/site_tools/t2t.py index 8f600723822..2eab5a34f30 100644 --- a/site_scons/site_tools/t2t.py +++ b/site_scons/site_tools/t2t.py @@ -1,31 +1,47 @@ -### -#This file is a part of the NVDA project. -#URL: http://www.nvda-project.org/ -#Copyright 2010 James Teh . -#This program is free software: you can redistribute it and/or modify -#it under the terms of the GNU General Public License version 2.0, as published by -#the Free Software Foundation. -#This program is distributed in the hope that it will be useful, -#but WITHOUT ANY WARRANTY; without even the implied warranty of -#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. -#This license can be found at: -#http://www.gnu.org/licenses/old-licenses/gpl-2.0.html -### - -def txt2tags_actionFunc(target,source,env): +# A part of NonVisual Desktop Access (NVDA) +# Copyright (C) 2010-2024 NV Access Limited, James Teh +# This file is covered by the GNU General Public License. +# See the file COPYING for more details. + +import SCons + + +def txt2tags_actionFunc( + target: list[SCons.Node.FS.File], + source: list[SCons.Node.FS.File], + env: SCons.Environment.Environment +): import txt2tags - txt2tags.exec_command_line([str(source[0])]) + from keyCommandsDoc import Command + + with open(source[0].path, "r", encoding="utf-8") as mdFile: + mdOriginal = mdFile.read() + mdStr = str(mdOriginal) + + with open(source[0].path, "w", encoding="utf-8") as mdFile: + # Substitute t2t key commands with markdown comments temporarily + for command in Command: + mdStr = command.t2tRegex().sub(lambda m: f"", mdStr) + mdFile.write(mdStr) -def exists(env): + txt2tags.exec_command_line([source[0].path]) + + with open(source[0].path, "w", encoding="utf-8") as mdFile: + # Restore to original + mdFile.write(mdOriginal) + + +def exists(env: SCons.Environment.Environment) -> bool: try: import txt2tags return True except ImportError: return False -def generate(env): - env['BUILDERS']['txt2tags']=env.Builder( - action=env.Action(txt2tags_actionFunc,lambda t,s,e: 'Converting %s to html'%s[0].path), - suffix='.html', + +def generate(env: SCons.Environment.Environment): + env["BUILDERS"]["txt2tags"] = env.Builder( + action=env.Action(txt2tags_actionFunc, lambda t, s, e: f"Converting {s[0].path} to md"), + suffix='.md', src_suffix='.t2t' ) diff --git a/user_docs/ar/locale.t2tconf b/user_docs/ar/locale.t2tconf index 73483d6fb5c..e69de29bb2d 100644 --- a/user_docs/ar/locale.t2tconf +++ b/user_docs/ar/locale.t2tconf @@ -1,36 +0,0 @@ -%!PostProc(html): ^$ -%!PostProc(html): TEXT="black">$ TEXT='black' STYLE='font-family: Simplified Arabic'> -%!PostProc(html): NVDA_AR NVDA - -% add rtl macro -%!PostProc: %ltr{(.*)} \1 - -% reverse the numbering, to be displayed correctly in arabic. -%PostProc(html): >([0-9]+)\.([0-9]+)\.([0-9]+)\. > ar.\3ar.\2ar.\1 -%PostProc(html): >([0-9]+)\.([0-9]+)\. > ar.\2ar.\1 -%PostProc(html): >([0-9]+)\. > ar.\1 - -% now replace Arabic numerals with indian once. -% 10 11, ... etc should go before signle digits, otherwise they will interfear -% and not produce the correct output. -%PostProc(html): ar\.10 ١٠. -%PostProc(html): ar\.11 ١١. -%PostProc(html): ar\.12 ١٢. -%PostProc(html): ar\.13 ١٣. -%PostProc(html): ar\.14 ١٤. -%PostProc(html): ar\.15 ١٥. -%PostProc(html): ar\.16 ١٦. -%PostProc(html): ar\.17 ١٧. -%PostProc(html): ar\.18 ١٨. -%PostProc(html): ar\.19 ١٩. - -%PostProc(html): ar\.0 ٠. -%PostProc(html): ar\.1 ١. -%PostProc(html): ar\2. ٢. -%PostProc(html): ar\.3 .3 -%PostProc(html): ar\.4 ٤. -%PostProc(html): ar\.5 ٥. -%PostProc(html): ar\.6 ٦. -%PostProc(html): ar\.7 ٧. -%PostProc(html): ar\.8 ٨. -%PostProc(html): ar\.9 ٩. diff --git a/user_docs/ca/locale.t2tconf b/user_docs/ca/locale.t2tconf index a1cb4bb37f2..e7144385b66 100644 --- a/user_docs/ca/locale.t2tconf +++ b/user_docs/ca/locale.t2tconf @@ -1,8 +1,5 @@ % locale.t2tconf for Catalan -% Set document language -%!PostProc(html): '^string -%!postproc(html): '(?i)([a-z-_]+)_' '\1 target="_blank">' diff --git a/user_docs/de/developerGuide.t2t b/user_docs/de/developerGuide.t2t index 1059be63fa9..02e78586f9d 100644 --- a/user_docs/de/developerGuide.t2t +++ b/user_docs/de/developerGuide.t2t @@ -3,9 +3,6 @@ %!includeconf: ../userGuide.t2tconf -% Remove double spacing from the beginning of each line as txt2tags seems to indent preformatted text by two spaces -%!PostProc(html): '^ ' '' - = Inhaltsverzeichnis =[toc] %%toc diff --git a/user_docs/fa/locale.t2tconf b/user_docs/fa/locale.t2tconf index 92284f575de..e69de29bb2d 100644 --- a/user_docs/fa/locale.t2tconf +++ b/user_docs/fa/locale.t2tconf @@ -1,37 +0,0 @@ -%!PostProc(html): ^$ -%!PostProc(html): TEXT="black">$ TEXT='black' STYLE='font-family: Simplified Arabic'> - -% add rtl macro -%!PostProc: %ltr{(.*)} \1 - -% reverse the numbering, to be displayed correctly in Persian. -%!PostProc(html): >([0-9]+)\.([0-9]+)\.([0-9]+)\. > fa.\3fa.\2fa.\1 -%!PostProc(html): >([0-9]+)\.([0-9]+)\. > fa.\2fa.\1 -%!PostProc(html): >([0-9]+)\. > fa.\1 - -% now replace Persian numerals with indian ones. -% 10 11, ... etc should go before signle digits, otherwise they will interfear -% and not produce the correct output. -%!PostProc(html): fa\.10 ۱۰. -%!PostProc(html): fa\.11 ۱۱. -%!PostProc(html): fa\.12 ۱۲. -%!PostProc(html): fa\.13 ۱۳. -%!PostProc(html): fa\.14 ۱۴. -%!PostProc(html): fa\.15 ۱۵. -%!PostProc(html): fa\.16 ۱۶. -%!PostProc(html): fa\.17 ۱۷. -%!PostProc(html): fa\.18 ۱۸. -%!PostProc(html): fa\.19 ۱۹. -%!PostProc(html): fa\.20 ۲۰. -%!PostProc(html): fa\.21 ۲۱. - -%!PostProc(html): fa\.0 ۰. -%!PostProc(html): fa\.1 ۱. -%!PostProc(html): fa\.2 ۲. -%!PostProc(html): fa\.3 ۳. -%!PostProc(html): fa\.4 ۴. -%!PostProc(html): fa\.5 ۵. -%!PostProc(html): fa\.6 ۶. -%!PostProc(html): fa\.7 ۷. -%!PostProc(html): fa\.8 ۸. -%!PostProc(html): fa\.9 ۹. diff --git a/user_docs/global.t2tconf b/user_docs/global.t2tconf index 5757bfe8b8d..7601f7360b0 100644 --- a/user_docs/global.t2tconf +++ b/user_docs/global.t2tconf @@ -1,24 +1,6 @@ -%!Target: html +%!Target: md %!Encoding: UTF-8 % Remove the Table of Contents heading from the toc. -%!PostProc(html): '^.*\\.*\.*$' '' - -% h1 in html should really be the document title only. -% Therefore, change h1 through h5 in the output to h2 through h6. -%!PostProc(html): ^
(.*)
$
\1
-%!PostProc(html): ^

(.*)

$
\1
-%!PostProc(html): ^

(.*)

$

\1

-%!PostProc(html): ^

(.*)

$

\1

-%!PostProc(html): ^

(.*)

$

\1

- -% Some of our files contain the UTF-8 BOM. -% txt2tags doesn't care about encodings internally, -% so it will just include the BOM at the start of the title. -% Therefore, strip the BOM from the title. -%!PostProc(html): \<(TITLE|H1)\>\xef\xbb\xbf <\1> - -%!Options: --style styles.css - -% This provides the macros NVDA_VERSION, NVDA_URL and NVDA_COPYRIGHT_YEARS. -%!includeconf: build.t2tconf +% For some reason t2t to md conversion always forces the TOC at the top of the document +%!PostProc(md): '^#.*\{#toc\}.*$' '' diff --git a/user_docs/pt_BR/developerGuide.t2t b/user_docs/pt_BR/developerGuide.t2t index 040a388f791..4eb6d7f77b0 100644 --- a/user_docs/pt_BR/developerGuide.t2t +++ b/user_docs/pt_BR/developerGuide.t2t @@ -3,9 +3,6 @@ %!includeconf: ../userGuide.t2tconf -% Remove double spacing from the beginning of each line as txt2tags seems to indent preformatted text by two spaces -%!PostProc(html): '^ ' '' - Nota; Alguns termos deverão ser mantidos em Inglês por referirem-se a programação e Jargões Técnicos diff --git a/user_docs/styles.css b/user_docs/styles.css index 76311cfe121..f8e25ac6072 100644 --- a/user_docs/styles.css +++ b/user_docs/styles.css @@ -7,6 +7,10 @@ body { color: #333; } +:lang(ar) body *, :lang(fa) body * { + font-family: "Simplified Arabic", "Traditional Arabic", "Arial", sans-serif !important; +} + h1 { color:white; background-color: #472F5F; /* NVDA purple */ diff --git a/user_docs/userGuide.t2tconf b/user_docs/userGuide.t2tconf index 8885ab8e723..a5b5f11d7e6 100644 --- a/user_docs/userGuide.t2tconf +++ b/user_docs/userGuide.t2tconf @@ -1,3 +1,3 @@ %!includeconf: global.t2tconf -%!Options: --toc +%!Options: --toc --no-enum-title