Allow to adjust collection URLs and installation instructions.

changelogs/fragments/26-sphinx-antsibull-cfg.yml

@@ -1,2 +1,8 @@
 minor_changes:
   - "The ``sphinx-init`` subcommand now also creates an ``antsibull-docs.cfg`` file and moves configuration settings from CLI flags in ``build.sh`` to this configuration file (https://github.com/ansible-community/antsibull-docs/pull/26)."
+  - "There are two new options for explicitly specified configuration files named ``collection_url`` and ``collection_install``.
+     These allow to override the URLs pointing to collections (default link to galaxy.ansible.com),
+     and the commands to install collections (use ``ansible-galaxy collection install`` by default).
+     This can be useful when documenting (internal) collections that are not available on Ansible Galaxy.
+     The default ``antsibull-docs.cfg`` generated by the ``sphinx-init`` subcommand shows how this can be configured
+     (https://github.com/ansible-community/antsibull-docs/issues/15, https://github.com/ansible-community/antsibull-docs/pull/26)."

src/antsibull_docs/cli/doc_commands/lint_collection_docs.py

@@ -11,6 +11,7 @@
 from ...collection_links import lint_collection_links
 from ...lint_extra_docs import lint_collection_extra_docs_files
 from ...lint_plugin_docs import lint_collection_plugin_docs
+from ...utils.collection_name_transformer import CollectionNameTransformer
 
 
 mlog = log.fields(mod=__name__)
@@ -39,7 +40,12 @@ def lint_collection_docs() -> int:
 
     if plugin_docs:
         flog.notice('Linting plugin docs')
-        errors.extend(lint_collection_plugin_docs(collection_root))
+        collection_url = CollectionNameTransformer(
+            app_ctx.collection_url, 'https://galaxy.ansible.com/{namespace}/{name}')
+        collection_install = CollectionNameTransformer(
+            app_ctx.collection_install, 'ansible-galaxy collection install {namespace}.{name}')
+        errors.extend(lint_collection_plugin_docs(
+            collection_root, collection_url=collection_url, collection_install=collection_install))
 
     messages = sorted(set(f'{error[0]}:{error[1]}:{error[2]}: {error[3]}' for error in errors))
 

src/antsibull_docs/cli/doc_commands/plugin.py

@@ -26,6 +26,7 @@
 from ...docs_parsing import AnsibleCollectionMetadata
 from ...docs_parsing.fqcn import get_fqcn_parts, is_fqcn
 from ...jinja2.environment import doc_environment
+from ...utils.collection_name_transformer import CollectionNameTransformer
 from ...write_docs import write_plugin_rst
 
 
@@ -100,7 +101,14 @@ def generate_plugin_docs(plugin_type: str, plugin_name: str,
     }))
 
     # Setup the jinja environment
-    env = doc_environment(('antsibull_docs.data', 'docsite'))
+    collection_url = CollectionNameTransformer(
+        app_ctx.collection_url, 'https://galaxy.ansible.com/{namespace}/{name}')
+    collection_install = CollectionNameTransformer(
+        app_ctx.collection_install, 'ansible-galaxy collection install {namespace}.{name}')
+    env = doc_environment(
+        ('antsibull_docs.data', 'docsite'),
+        collection_url=collection_url,
+        collection_install=collection_install)
     # Get the templates
     plugin_tmpl = env.get_template('plugin.rst.j2')
     error_tmpl = env.get_template('plugin-error.rst.j2')

src/antsibull_docs/cli/doc_commands/sphinx_init.py

@@ -7,6 +7,7 @@
 
 import os
 import os.path
+import typing as t
 
 from antsibull_core.logging import log
 
@@ -43,6 +44,18 @@ def write_file(filename: str, content: str) -> None:
         f.write(content)
 
 
+def toperky(value: t.Any) -> str:
+    if isinstance(value, str):
+        value = value.replace('\\', '\\\\')
+        value = value.replace('\n', '\\n')
+        value = value.replace('\r', '\\r')
+        value = value.replace('"', '\\"')
+        if all(ch not in value for ch in '\\={}[]') and value.strip() == value:
+            return value
+        return f'"{value}"'
+    raise Exception(f'toperky filter cannot handle type {type(value)}')
+
+
 def site_init() -> int:
     """
     Initialize a Sphinx site template for a collection docsite.
@@ -68,14 +81,19 @@ def site_init() -> int:
     use_html_blobs = app_ctx.use_html_blobs
     breadcrumbs = app_ctx.breadcrumbs
     indexes = app_ctx.indexes
+    collection_url = app_ctx.collection_url
+    collection_install = app_ctx.collection_install
 
     sphinx_theme = 'sphinx_ansible_theme'
     sphinx_theme_package = 'sphinx-ansible-theme >= 0.9.0'
     if app_ctx.extra['sphinx_theme'] != 'sphinx-ansible-theme':
         sphinx_theme = app_ctx.extra['sphinx_theme']
         sphinx_theme_package = app_ctx.extra['sphinx_theme']
 
-    env = doc_environment(('antsibull_docs.data', 'sphinx_init'))
+    env = doc_environment(
+        ('antsibull_docs.data', 'sphinx_init'),
+        extra_filters={'toperky': toperky},
+    )
 
     for filename in TEMPLATES:
         source = filename.replace('.', '_').replace('/', '_') + '.j2'
@@ -94,6 +112,8 @@ def site_init() -> int:
             indexes=indexes,
             sphinx_theme=sphinx_theme,
             sphinx_theme_package=sphinx_theme_package,
+            collection_url=collection_url,
+            collection_install=collection_install,
         ) + '\n'
 
         destination = os.path.join(dest_dir, filename)

src/antsibull_docs/cli/doc_commands/stable.py

@@ -38,6 +38,7 @@
     remove_redirect_duplicates,
 )
 from ...schemas.docs import DOCS_SCHEMAS
+from ...utils.collection_name_transformer import CollectionNameTransformer
 from ...write_docs import (
     output_all_plugin_rst,
     output_all_plugin_stub_rst,
@@ -333,6 +334,8 @@ def generate_docs_for_all_collections(venv: t.Union[VenvRunner, FakeVenvRunner],
     flog = mlog.fields(func='generate_docs_for_all_collections')
     flog.notice('Begin')
 
+    app_ctx = app_context.app_ctx.get()
+
     # Get the info from the plugins
     plugin_info, collection_metadata = asyncio_run(get_ansible_plugin_info(
         venv, collection_dir, collection_names=collection_names))
@@ -383,21 +386,34 @@ def generate_docs_for_all_collections(venv: t.Union[VenvRunner, FakeVenvRunner],
 
     collection_namespaces = get_collection_namespaces(collection_to_plugin_info.keys())
 
+    collection_url = CollectionNameTransformer(
+        app_ctx.collection_url, 'https://galaxy.ansible.com/{namespace}/{name}')
+    collection_install = CollectionNameTransformer(
+        app_ctx.collection_install, 'ansible-galaxy collection install {namespace}.{name}')
+
     # Only build top-level index if requested
     if create_indexes:
-        asyncio_run(output_collection_index(
-            collection_to_plugin_info, collection_namespaces, dest_dir, breadcrumbs=breadcrumbs,
-            for_official_docsite=for_official_docsite))
+        asyncio_run(output_collection_index(collection_to_plugin_info, collection_namespaces,
+                                            dest_dir, collection_url=collection_url,
+                                            collection_install=collection_install,
+                                            breadcrumbs=breadcrumbs,
+                                            for_official_docsite=for_official_docsite))
         flog.notice('Finished writing collection index')
         asyncio_run(output_collection_namespace_indexes(collection_namespaces, dest_dir,
+                                                        collection_url=collection_url,
+                                                        collection_install=collection_install,
                                                         breadcrumbs=breadcrumbs,
                                                         for_official_docsite=for_official_docsite))
         flog.notice('Finished writing collection namespace index')
         asyncio_run(output_plugin_indexes(plugin_contents, dest_dir,
+                                          collection_url=collection_url,
+                                          collection_install=collection_install,
                                           for_official_docsite=for_official_docsite))
         flog.notice('Finished writing plugin indexes')
 
     asyncio_run(output_indexes(collection_to_plugin_info, dest_dir,
+                               collection_url=collection_url,
+                               collection_install=collection_install,
                                collection_metadata=collection_metadata,
                                squash_hierarchy=squash_hierarchy,
                                extra_docs_data=extra_docs_data,
@@ -407,6 +423,8 @@ def generate_docs_for_all_collections(venv: t.Union[VenvRunner, FakeVenvRunner],
     flog.notice('Finished writing indexes')
 
     asyncio_run(output_all_plugin_stub_rst(stubs_info, dest_dir,
+                                           collection_url=collection_url,
+                                           collection_install=collection_install,
                                            collection_metadata=collection_metadata,
                                            link_data=link_data,
                                            squash_hierarchy=squash_hierarchy,
@@ -415,6 +433,8 @@ def generate_docs_for_all_collections(venv: t.Union[VenvRunner, FakeVenvRunner],
 
     asyncio_run(output_all_plugin_rst(collection_to_plugin_info, new_plugin_info,
                                       nonfatal_errors, dest_dir,
+                                      collection_url=collection_url,
+                                      collection_install=collection_install,
                                       collection_metadata=collection_metadata,
                                       link_data=link_data,
                                       squash_hierarchy=squash_hierarchy,

src/antsibull_docs/data/docsite/plugin-error.rst.j2

@@ -36,4 +36,4 @@ The errors were:
 
 {% endfor %}
 
-File a bug with the `@{ collection }@ collection <https://galaxy.ansible.com/@{collection | replace(".", "/", 1)}@>`_ in order to have it corrected.
+File a bug with the `@{ collection }@ collection <@{ collection | collection_url }@>`_ in order to have it corrected.

src/antsibull_docs/data/docsite/plugin-redirect.rst.j2

@@ -31,7 +31,7 @@
     the same module name.
 {% else %}
 .. note::
-    This redirect is part of the `@{collection}@ collection <https://galaxy.ansible.com/@{collection | replace('.', '/', 1)}@>`_{% if collection_version %} (version @{ collection_version }@){% endif %}.
+    This redirect is part of the `@{collection}@ collection <@{ collection | collection_url }@>`_{% if collection_version %} (version @{ collection_version }@){% endif %}.
 
 {%   if not deprecation %}
     To use it in a playbook, specify: :code:`@{plugin_name}@`.

src/antsibull_docs/data/docsite/plugin-tombstone.rst.j2

@@ -24,7 +24,7 @@
     This module was part of ``ansible-core`` and was included in all Ansible installations.
 {% else %}
 .. note::
-    This plugin was part of the `@{collection}@ collection <https://galaxy.ansible.com/@{collection | replace('.', '/', 1)}@>`_{% if collection_version %} (version @{ collection_version }@){% endif %}.
+    This plugin was part of the `@{collection}@ collection <@{ collection | collection_url }@>`_{% if collection_version %} (version @{ collection_version }@){% endif %}.
 {% endif %}
 
 This {% if plugin_type == 'module' %}module{% else %}@{ plugin_type }@ plugin{% endif %} has been removed

src/antsibull_docs/data/docsite/plugin.rst.j2

@@ -91,15 +91,15 @@
     the same {% if plugin_type == 'module' %}module{% else %}@{ plugin_type }@ plugin{% endif %} name.
 {% else %}
 .. note::
-    This {% if plugin_type == 'module' %}module{% else %}@{ plugin_type }@ plugin{% endif %} is part of the `@{collection}@ collection <https://galaxy.ansible.com/@{collection | replace('.', '/', 1)}@>`_{% if collection_version %} (version @{ collection_version }@){% endif %}.
+    This {% if plugin_type == 'module' %}module{% else %}@{ plugin_type }@ plugin{% endif %} is part of the `@{collection}@ collection <@{ collection | collection_url }@>`_{% if collection_version %} (version @{ collection_version }@){% endif %}.
 {%   if for_official_docsite %}
 
     You might already have this collection installed if you are using the ``ansible`` package.
     It is not included in ``ansible-core``.
     To check whether it is installed, run :code:`ansible-galaxy collection list`.
 {%   endif %}
 
-    To install it, use: :code:`ansible-galaxy collection install @{collection}@`.
+    To install it, use: @{ collection | collection_install | rst_code }@.
 {%   if doc['requirements'] %}
     You need further requirements to be able to use this {% if plugin_type == 'module' %}module{% else %}@{ plugin_type }@ plugin{% endif %},
     see :ref:`Requirements <ansible_collections.@{plugin_name}@_@{plugin_type}@_requirements>` for details.

src/antsibull_docs/data/docsite/role.rst.j2

@@ -58,15 +58,15 @@
     installations.
 {% else %}
 .. note::
-    This role is part of the `@{collection}@ collection <https://galaxy.ansible.com/@{collection | replace('.', '/', 1)}@>`_{% if collection_version %} (version @{ collection_version }@){% endif %}.
+    This role is part of the `@{collection}@ collection <@{ collection | collection_url }@>`_{% if collection_version %} (version @{ collection_version }@){% endif %}.
 {%   if for_official_docsite %}
 
     You might already have this collection installed if you are using the ``ansible`` package.
     It is not included in ``ansible-core``.
     To check whether it is installed, run :code:`ansible-galaxy collection list`.
 {%   endif %}
 
-    To install it use: :code:`ansible-galaxy collection install @{collection}@`.
+    To install it use: @{ collection | collection_install | rst_code }@.
 
     To use it in a playbook, specify: :code:`@{plugin_name}@`.
 {% endif %}

src/antsibull_docs/data/sphinx_init/antsibull-docs_cfg.j2

@@ -5,3 +5,22 @@
 breadcrumbs = @{ breadcrumbs | tojson }@
 indexes = @{ indexes | tojson }@
 use_html_blobs = @{ use_html_blobs | tojson }@
+
+# You can specify ways to convert a collection name (<namespace>.<name>) to an URL here.
+# You can replace either of <namespace> or <name> by "*" to match all values in that place,
+# or use "*" for the collection name to match all collections. In the URL, you can use
+# {namespace} and {name} for the two components of the collection name. If you want to use
+# "{" or "}" in the URL, write "{{" or "}}" instead. Basically these are Python format
+# strings (https://docs.python.org/3.8/library/string.html#formatstrings).
+collection_url = {
+{% for pattern, value in collection_url | dictsort %}
+  @{ pattern | toperky }@ = @{ value | toperky }@
+{% endfor %}
+}
+
+# The same wildcard rules and formatting rules as for collection_url apply.
+collection_install = {
+{% for pattern, value in collection_install | dictsort %}
+  @{ pattern | toperky }@ = @{ value | toperky }@
+{% endfor %}
+}

src/antsibull_docs/jinja2/environment.py

@@ -6,15 +6,18 @@
 
 import json
 import os.path
+import typing as t
 
-from jinja2 import Environment, FileSystemLoader, PackageLoader
+from jinja2 import Environment, FileSystemLoader, PackageLoader, BaseLoader
+
+from ..utils.collection_name_transformer import CollectionNameTransformer
 
 from .filters import (
     do_max, documented_type, rst_fmt, rst_xline, move_first, massage_author_name,
     extract_options_from_list, remove_options_from_list, to_json,
 )
 from .htmlify import html_ify
-from .rstify import rst_ify, rst_escape
+from .rstify import rst_ify, rst_code, rst_escape
 from .tests import still_relevant, test_list
 
 
@@ -40,7 +43,14 @@ def reference_plugin_rst(plugin_name: str, plugin_type: str) -> str:
     return f"\\ :ref:`{rst_escape(fqcn)} <ansible_collections.{fqcn}_{plugin_type}>`\\ "
 
 
-def doc_environment(template_location):
+def doc_environment(template_location: t.Union[str, t.Tuple[str, str]],
+                    *,
+                    extra_filters: t.Optional[t.Mapping[str, t.Callable]] = None,
+                    extra_tests: t.Optional[t.Mapping[str, t.Callable]] = None,
+                    collection_url: t.Optional[CollectionNameTransformer] = None,
+                    collection_install: t.Optional[CollectionNameTransformer] = None,
+                    ) -> Environment:
+    loader: BaseLoader
     if isinstance(template_location, str) and os.path.exists(template_location):
         loader = FileSystemLoader(template_location)
     else:
@@ -75,6 +85,7 @@ def doc_environment(template_location):
     env.filters['rst_ify'] = rst_ify
     env.filters['html_ify'] = html_ify
     env.filters['fmt'] = rst_fmt
+    env.filters['rst_code'] = rst_code
     env.filters['rst_escape'] = rst_escape
     env.filters['xline'] = rst_xline
     env.filters['documented_type'] = documented_type
@@ -83,7 +94,16 @@ def doc_environment(template_location):
     env.filters['extract_options_from_list'] = extract_options_from_list
     env.filters['remove_options_from_list'] = remove_options_from_list
     env.filters['antsibull_to_json'] = to_json
+    if collection_url is not None:
+        env.filters['collection_url'] = collection_url
+    if collection_install is not None:
+        env.filters['collection_install'] = collection_install
+    if extra_filters:
+        env.filters.update(extra_filters)
+
     env.tests['list'] = test_list
     env.tests['still_relevant'] = still_relevant
+    if extra_tests:
+        env.tests.update(extra_tests)
 
     return env

src/antsibull_docs/jinja2/rstify.py

@@ -43,6 +43,13 @@ def rst_escape(value: t.Any, escape_ending_whitespace=False) -> str:
     return value
 
 
+def rst_code(value: str) -> str:
+    ''' Write value as :code:`...` RST construct. '''
+    if not isinstance(value, str):
+        value = str(value)
+    return f':code:`{rst_escape(value, escape_ending_whitespace=True)}`'
+
+
 def _escape_url(url: str) -> str:
     # We include '<>[]{}' in safe to allow urls such as 'https://<HOST>:[PORT]/v{version}/' to
     # remain unmangled by percent encoding

src/antsibull_docs/jinja2/tests.py

@@ -6,6 +6,7 @@
 
 import warnings
 from functools import partial
+import typing as t
 
 from packaging.version import Version as PypiVer
 from semantic_version import Version as SemVer
@@ -18,7 +19,7 @@
 # than this already.
 TOO_OLD_TO_BE_NOTABLE = '0.0.0'
 
-test_list = partial(is_sequence, include_strings=False)
+test_list: t.Callable[[t.Any], bool] = partial(is_sequence, include_strings=False)
 
 
 def still_relevant(version, cutoff=TOO_OLD_TO_BE_NOTABLE, collection=None):