nixos/*: split docs build

lib/options.nix

@@ -177,7 +177,7 @@ rec {
         docOption = rec {
           loc = opt.loc;
           name = showOption opt.loc;
-          description = opt.description or (lib.warn "Option `${name}' has no description." "This option has no description.");
+          description = opt.description or null;
           declarations = filter (x: x != unknownModule) opt.declarations;
           internal = opt.internal or false;
           visible =

nixos/doc/manual/default.nix

@@ -1,4 +1,13 @@
-{ pkgs, options, config, version, revision, extraSources ? [] }:
+{ pkgs
+, options
+, config
+, version
+, revision
+, extraSources ? []
+, baseOptionsJSON ? null
+, warningsAreErrors ? true
+, prefix ? ../../..
+}:
 
 with pkgs;
 
@@ -11,11 +20,11 @@ let
   #
   # E.g. if some `options` came from modules in ${pkgs.customModules}/nix,
   # you'd need to include `extraSources = [ pkgs.customModules ]`
-  prefixesToStrip = map (p: "${toString p}/") ([ ../../.. ] ++ extraSources);
+  prefixesToStrip = map (p: "${toString p}/") ([ prefix ] ++ extraSources);
   stripAnyPrefixes = lib.flip (lib.foldr lib.removePrefix) prefixesToStrip;
 
   optionsDoc = buildPackages.nixosOptionsDoc {
-    inherit options revision;
+    inherit options revision baseOptionsJSON warningsAreErrors;
     transformOptions = opt: opt // {
       # Clean up declaration sites to not refer to the NixOS source tree.
       declarations = map stripAnyPrefixes opt.declarations;
@@ -161,7 +170,7 @@ let
 in rec {
   inherit generatedSources;
 
-  inherit (optionsDoc) optionsJSON optionsDocBook;
+  inherit (optionsDoc) optionsJSON optionsNix optionsDocBook;
 
   # Generate the NixOS manual.
   manualHTML = runCommand "nixos-manual-html"

nixos/doc/manual/development/meta-attributes.section.md

@@ -5,7 +5,7 @@ extra information. Module meta attributes are defined in the `meta.nix`
 special module.
 
 `meta` is a top level attribute like `options` and `config`. Available
-meta-attributes are `maintainers` and `doc`.
+meta-attributes are `maintainers`, `doc`, and `buildDocsInSandbox`.
 
 Each of the meta-attributes must be defined at most once per module
 file.
@@ -24,6 +24,7 @@ file.
   meta = {
     maintainers = with lib.maintainers; [ ericsagnes ];
     doc = ./default.xml;
+    buildDocsInSandbox = true;
   };
 }
 ```
@@ -38,3 +39,28 @@ file.
     ```ShellSession
     $ nix-build nixos/release.nix -A manual.x86_64-linux
     ```
+
+-  `buildDocsInSandbox` indicates whether the option documentation for the
+   module can be built in a derivation sandbox. This option is currently only
+   honored for modules shipped by nixpkgs. User modules and modules taken from
+   `NIXOS_EXTRA_MODULE_PATH` are always built outside of the sandbox, as has
+   been the case in previous releases.
+
+   Building NixOS option documentation in a sandbox allows caching of the built
+   documentation, which greatly decreases the amount of time needed to evaluate
+   a system configuration that has NixOS documentation enabled. The sandbox also
+   restricts which attributes may be referenced by documentation attributes
+   (such as option descriptions) to the `options` and `lib` module arguments and
+   the `pkgs.formats` attribute of the `pkgs` argument, `config` and the rest of
+   `pkgs` are disallowed and will cause doc build failures when used. This
+   restriction is necessary because we cannot reproduce the full nixpkgs
+   instantiation with configuration and overlays from a system configuration
+   inside the sandbox. The `options` argument only includes options of modules
+   that are also built inside the sandbox, referencing an option of a module
+   that isn't built in the sandbox is also forbidden.
+
+   The default is `true` and should usually not be changed; set it to `false`
+   only if the module requires access to `pkgs` in its documentation (e.g.
+   because it loads information from a linked package to build an option type)
+   or if its documentation depends on other modules that also aren't sandboxed
+   (e.g. by using types defined in the other module).

nixos/doc/manual/from_md/development/meta-attributes.section.xml

@@ -8,8 +8,8 @@
   <para>
     <literal>meta</literal> is a top level attribute like
     <literal>options</literal> and <literal>config</literal>. Available
-    meta-attributes are <literal>maintainers</literal> and
-    <literal>doc</literal>.
+    meta-attributes are <literal>maintainers</literal>,
+    <literal>doc</literal>, and <literal>buildDocsInSandbox</literal>.
   </para>
   <para>
     Each of the meta-attributes must be defined at most once per module
@@ -29,6 +29,7 @@
   meta = {
     maintainers = with lib.maintainers; [ ericsagnes ];
     doc = ./default.xml;
+    buildDocsInSandbox = true;
   };
 }
 </programlisting>
@@ -51,5 +52,44 @@
 $ nix-build nixos/release.nix -A manual.x86_64-linux
 </programlisting>
     </listitem>
+    <listitem>
+      <para>
+        <literal>buildDocsInSandbox</literal> indicates whether the
+        option documentation for the module can be built in a derivation
+        sandbox. This option is currently only honored for modules
+        shipped by nixpkgs. User modules and modules taken from
+        <literal>NIXOS_EXTRA_MODULE_PATH</literal> are always built
+        outside of the sandbox, as has been the case in previous
+        releases.
+      </para>
+      <para>
+        Building NixOS option documentation in a sandbox allows caching
+        of the built documentation, which greatly decreases the amount
+        of time needed to evaluate a system configuration that has NixOS
+        documentation enabled. The sandbox also restricts which
+        attributes may be referenced by documentation attributes (such
+        as option descriptions) to the <literal>options</literal> and
+        <literal>lib</literal> module arguments and the
+        <literal>pkgs.formats</literal> attribute of the
+        <literal>pkgs</literal> argument, <literal>config</literal> and
+        the rest of <literal>pkgs</literal> are disallowed and will
+        cause doc build failures when used. This restriction is
+        necessary because we cannot reproduce the full nixpkgs
+        instantiation with configuration and overlays from a system
+        configuration inside the sandbox. The <literal>options</literal>
+        argument only includes options of modules that are also built
+        inside the sandbox, referencing an option of a module that isn’t
+        built in the sandbox is also forbidden.
+      </para>
+      <para>
+        The default is <literal>true</literal> and should usually not be
+        changed; set it to <literal>false</literal> only if the module
+        requires access to <literal>pkgs</literal> in its documentation
+        (e.g. because it loads information from a linked package to
+        build an option type) or if its documentation depends on other
+        modules that also aren’t sandboxed (e.g. by using types defined
+        in the other module).
+      </para>
+    </listitem>
   </itemizedlist>
 </section>

nixos/lib/eval-cacheable-options.nix

@@ -0,0 +1,53 @@
+{ libPath
+, pkgsLibPath
+, nixosPath
+, modules
+, stateVersion
+, release
+}:
+
+let
+  lib = import libPath;
+  modulesPath = "${nixosPath}/modules";
+  # dummy pkgs set that contains no packages, only `pkgs.lib` from the full set.
+  # not having `pkgs.lib` causes all users of `pkgs.formats` to fail.
+  pkgs = import pkgsLibPath {
+    inherit lib;
+    pkgs = null;
+  };
+  utils = import "${nixosPath}/lib/utils.nix" {
+    inherit config lib;
+    pkgs = null;
+  };
+  # this is used both as a module and as specialArgs.
+  # as a module it sets the _module special values, as specialArgs it makes `config`
+  # unusable. this causes documentation attributes depending on `config` to fail.
+  config = {
+    _module.check = false;
+    _module.args = {};
+    system.stateVersion = stateVersion;
+  };
+  eval = lib.evalModules {
+    modules = (map (m: "${modulesPath}/${m}") modules) ++ [
+      config
+    ];
+    specialArgs = {
+      inherit config pkgs utils;
+    };
+  };
+  docs = import "${nixosPath}/doc/manual" {
+    pkgs = pkgs // {
+      inherit lib;
+      # duplicate of the declaration in all-packages.nix
+      buildPackages.nixosOptionsDoc = attrs:
+        (import "${nixosPath}/lib/make-options-doc")
+          ({ inherit pkgs lib; } // attrs);
+    };
+    config = config.config;
+    options = eval.options;
+    version = release;
+    revision = "release-${release}";
+    prefix = modulesPath;
+  };
+in
+  docs.optionsNix

nixos/lib/make-options-doc/default.nix

@@ -21,6 +21,13 @@
 , options
 , transformOptions ? lib.id  # function for additional tranformations of the options
 , revision ? "" # Specify revision for the options
+# a set of options the docs we are generating will be merged into, as if by recursiveUpdate.
+# used to split the options doc build into a static part (nixos/modules) and a dynamic part
+# (non-nixos modules imported via configuration.nix, other module sources).
+, baseOptionsJSON ? null
+# instead of printing warnings for eg options with missing descriptions (which may be lost
+# by nix build unless -L is given), emit errors instead and fail the build
+, warningsAreErrors ? true
 }:
 
 let
@@ -51,10 +58,15 @@ let
   # ../../../lib/options.nix influences.
   #
   # Each element of `relatedPackages` can be either
-  # - a string:  that will be interpreted as an attribute name from `pkgs`,
-  # - a list:    that will be interpreted as an attribute path from `pkgs`,
-  # - an attrset: that can specify `name`, `path`, `package`, `comment`
+  # - a string:  that will be interpreted as an attribute name from `pkgs` and turned into a link
+  #              to search.nixos.org,
+  # - a list:    that will be interpreted as an attribute path from `pkgs` and turned into a link
+  #              to search.nixos.org,
+  # - an attrset: that can specify `name`, `path`, `comment`
   #   (either of `name`, `path` is required, the rest are optional).
+  #
+  # NOTE: No checks against `pkgs` are made to ensure that the referenced package actually exists.
+  # Such checks are not compatible with option docs caching.
   genRelatedPackages = packages: optName:
     let
       unpack = p: if lib.isString p then { name = p; }
@@ -64,16 +76,16 @@ let
         let
           title = args.title or null;
           name = args.name or (lib.concatStringsSep "." args.path);
-          path = args.path or [ args.name ];
-          package = args.package or (lib.attrByPath path (throw "Invalid package attribute path `${toString path}' found while evaluating `relatedPackages' of option `${optName}'") pkgs);
-        in "<listitem>"
-        + "<para><literal>${lib.optionalString (title != null) "${title} aka "}pkgs.${name} (${package.meta.name})</literal>"
-        + lib.optionalString (!package.meta.available) " <emphasis>[UNAVAILABLE]</emphasis>"
-        + ": ${package.meta.description or "???"}.</para>"
-        + lib.optionalString (args ? comment) "\n<para>${args.comment}</para>"
-        # Lots of `longDescription's break DocBook, so we just wrap them into <programlisting>
-        + lib.optionalString (package.meta ? longDescription) "\n<programlisting>${package.meta.longDescription}</programlisting>"
-        + "</listitem>";
+        in ''
+          <listitem>
+            <para>
+              <link xlink:href="https://search.nixos.org/packages?show=${name}&amp;sort=relevance&amp;query=${name}">
+                <literal>${lib.optionalString (title != null) "${title} aka "}pkgs.${name}</literal>
+              </link>
+            </para>
+            ${lib.optionalString (args ? comment) "<para>${args.comment}</para>"}
+          </listitem>
+        '';
     in "<itemizedlist>${lib.concatStringsSep "\n" (map (p: describe (unpack p)) packages)}</itemizedlist>";
 
   # Remove invisible and internal options.
@@ -99,13 +111,24 @@ in rec {
   optionsJSON = pkgs.runCommand "options.json"
     { meta.description = "List of NixOS options in JSON format";
       buildInputs = [ pkgs.brotli ];
+      options = builtins.toFile "options.json"
+        (builtins.unsafeDiscardStringContext (builtins.toJSON optionsNix));
     }
     ''
       # Export list of options in different format.
       dst=$out/share/doc/nixos
       mkdir -p $dst
 
-      cp ${builtins.toFile "options.json" (builtins.unsafeDiscardStringContext (builtins.toJSON optionsNix))} $dst/options.json
+      ${
+        if baseOptionsJSON == null
+          then "cp $options $dst/options.json"
+          else ''
+            ${pkgs.python3Minimal}/bin/python ${./mergeJSON.py} \
+              ${lib.optionalString warningsAreErrors "--warnings-are-errors"} \
+              ${baseOptionsJSON} $options \
+              > $dst/options.json
+          ''
+      }
 
       brotli -9 < $dst/options.json > $dst/options.json.br
 

nixos/lib/make-options-doc/mergeJSON.py

@@ -0,0 +1,86 @@
+import collections
+import json
+import sys
+from typing import Any, Dict, List
+
+JSON = Dict[str, Any]
+
+class Key:
+    def __init__(self, path: List[str]):
+        self.path = path
+    def __hash__(self):
+        result = 0
+        for id in self.path:
+            result ^= hash(id)
+        return result
+    def __eq__(self, other):
+        return type(self) is type(other) and self.path == other.path
+
+Option = collections.namedtuple('Option', ['name', 'value'])
+
+# pivot a dict of options keyed by their display name to a dict keyed by their path
+def pivot(options: Dict[str, JSON]) -> Dict[Key, Option]:
+    result: Dict[Key, Option] = dict()
+    for (name, opt) in options.items():
+        result[Key(opt['loc'])] = Option(name, opt)
+    return result
+
+# pivot back to indexed-by-full-name
+# like the docbook build we'll just fail if multiple options with differing locs
+# render to the same option name.
+def unpivot(options: Dict[Key, Option]) -> Dict[str, JSON]:
+    result: Dict[str, Dict] = dict()
+    for (key, opt) in options.items():
+        if opt.name in result:
+            raise RuntimeError(
+                'multiple options with colliding ids found',
+                opt.name,
+                result[opt.name]['loc'],
+                opt.value['loc'],
+            )
+        result[opt.name] = opt.value
+    return result
+
+warningsAreErrors = sys.argv[1] == "--warnings-are-errors"
+optOffset = 1 if warningsAreErrors else 0
+options = pivot(json.load(open(sys.argv[1 + optOffset], 'r')))
+overrides = pivot(json.load(open(sys.argv[2 + optOffset], 'r')))
+
+# fix up declaration paths in lazy options, since we don't eval them from a full nixpkgs dir
+for (k, v) in options.items():
+    v.value['declarations'] = list(map(lambda s: f'nixos/modules/{s}', v.value['declarations']))
+
+# merge both descriptions
+for (k, v) in overrides.items():
+    cur = options.setdefault(k, v).value
+    for (ok, ov) in v.value.items():
+        if ok == 'declarations':
+            decls = cur[ok]
+            for d in ov:
+                if d not in decls:
+                    decls += [d]
+        elif ok == "type":
+            # ignore types of placeholder options
+            if ov != "_unspecified" or cur[ok] == "_unspecified":
+                cur[ok] = ov
+        elif ov is not None or cur.get(ok, None) is None:
+            cur[ok] = ov
+
+# check that every option has a description
+hasWarnings = False
+for (k, v) in options.items():
+    if v.value.get('description', None) is None:
+        severity = "error" if warningsAreErrors else "warning"
+        hasWarnings = True
+        print(f"\x1b[1;31m{severity}: option {v.name} has no description\x1b[0m", file=sys.stderr)
+        v.value['description'] = "This option has no description."
+if hasWarnings and warningsAreErrors:
+    print(
+        "\x1b[1;31m" +
+        "Treating warnings as errors. Set documentation.nixos.options.warningsAreErrors " +
+        "to false to ignore these warnings." +
+        "\x1b[0m",
+        file=sys.stderr)
+    sys.exit(1)
+
+json.dump(unpivot(options), fp=sys.stdout)

nixos/modules/i18n/input-method/fcitx.nix

@@ -40,4 +40,7 @@ in
     };
     services.xserver.displayManager.sessionCommands = "${fcitxPackage}/bin/fcitx";
   };
+
+  # uses attributes of the linked package
+  meta.buildDocsInSandbox = false;
 }

nixos/modules/i18n/input-method/ibus.nix

@@ -80,4 +80,7 @@ in
       ibusPackage
     ];
   };
+
+  # uses attributes of the linked package
+  meta.buildDocsInSandbox = false;
 }