Skip to content

Commit

Permalink
Merge pull request #149532 from pennae/split-docs-build
Browse files Browse the repository at this point in the history
nixos/*: split docs build
  • Loading branch information
roberth authored Jan 4, 2022
2 parents d8e5799 + 1511e72 commit 70d2730
Show file tree
Hide file tree
Showing 41 changed files with 464 additions and 55 deletions.
2 changes: 1 addition & 1 deletion lib/options.nix
Original file line number Diff line number Diff line change
Expand Up @@ -177,7 +177,7 @@ rec {
docOption = rec {
loc = opt.loc;
name = showOption opt.loc;
description = opt.description or (lib.warn "Option `${name}' has no description." "This option has no description.");
description = opt.description or null;
declarations = filter (x: x != unknownModule) opt.declarations;
internal = opt.internal or false;
visible =
Expand Down
17 changes: 13 additions & 4 deletions nixos/doc/manual/default.nix
Original file line number Diff line number Diff line change
@@ -1,4 +1,13 @@
{ pkgs, options, config, version, revision, extraSources ? [] }:
{ pkgs
, options
, config
, version
, revision
, extraSources ? []
, baseOptionsJSON ? null
, warningsAreErrors ? true
, prefix ? ../../..
}:

with pkgs;

Expand All @@ -11,11 +20,11 @@ let
#
# E.g. if some `options` came from modules in ${pkgs.customModules}/nix,
# you'd need to include `extraSources = [ pkgs.customModules ]`
prefixesToStrip = map (p: "${toString p}/") ([ ../../.. ] ++ extraSources);
prefixesToStrip = map (p: "${toString p}/") ([ prefix ] ++ extraSources);
stripAnyPrefixes = lib.flip (lib.foldr lib.removePrefix) prefixesToStrip;

optionsDoc = buildPackages.nixosOptionsDoc {
inherit options revision;
inherit options revision baseOptionsJSON warningsAreErrors;
transformOptions = opt: opt // {
# Clean up declaration sites to not refer to the NixOS source tree.
declarations = map stripAnyPrefixes opt.declarations;
Expand Down Expand Up @@ -161,7 +170,7 @@ let
in rec {
inherit generatedSources;

inherit (optionsDoc) optionsJSON optionsDocBook;
inherit (optionsDoc) optionsJSON optionsNix optionsDocBook;

# Generate the NixOS manual.
manualHTML = runCommand "nixos-manual-html"
Expand Down
28 changes: 27 additions & 1 deletion nixos/doc/manual/development/meta-attributes.section.md
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ extra information. Module meta attributes are defined in the `meta.nix`
special module.

`meta` is a top level attribute like `options` and `config`. Available
meta-attributes are `maintainers` and `doc`.
meta-attributes are `maintainers`, `doc`, and `buildDocsInSandbox`.

Each of the meta-attributes must be defined at most once per module
file.
Expand All @@ -24,6 +24,7 @@ file.
meta = {
maintainers = with lib.maintainers; [ ericsagnes ];
doc = ./default.xml;
buildDocsInSandbox = true;
};
}
```
Expand All @@ -38,3 +39,28 @@ file.
```ShellSession
$ nix-build nixos/release.nix -A manual.x86_64-linux
```

- `buildDocsInSandbox` indicates whether the option documentation for the
module can be built in a derivation sandbox. This option is currently only
honored for modules shipped by nixpkgs. User modules and modules taken from
`NIXOS_EXTRA_MODULE_PATH` are always built outside of the sandbox, as has
been the case in previous releases.

Building NixOS option documentation in a sandbox allows caching of the built
documentation, which greatly decreases the amount of time needed to evaluate
a system configuration that has NixOS documentation enabled. The sandbox also
restricts which attributes may be referenced by documentation attributes
(such as option descriptions) to the `options` and `lib` module arguments and
the `pkgs.formats` attribute of the `pkgs` argument, `config` and the rest of
`pkgs` are disallowed and will cause doc build failures when used. This
restriction is necessary because we cannot reproduce the full nixpkgs
instantiation with configuration and overlays from a system configuration
inside the sandbox. The `options` argument only includes options of modules
that are also built inside the sandbox, referencing an option of a module
that isn't built in the sandbox is also forbidden.

The default is `true` and should usually not be changed; set it to `false`
only if the module requires access to `pkgs` in its documentation (e.g.
because it loads information from a linked package to build an option type)
or if its documentation depends on other modules that also aren't sandboxed
(e.g. by using types defined in the other module).
44 changes: 42 additions & 2 deletions nixos/doc/manual/from_md/development/meta-attributes.section.xml
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,8 @@
<para>
<literal>meta</literal> is a top level attribute like
<literal>options</literal> and <literal>config</literal>. Available
meta-attributes are <literal>maintainers</literal> and
<literal>doc</literal>.
meta-attributes are <literal>maintainers</literal>,
<literal>doc</literal>, and <literal>buildDocsInSandbox</literal>.
</para>
<para>
Each of the meta-attributes must be defined at most once per module
Expand All @@ -29,6 +29,7 @@
meta = {
maintainers = with lib.maintainers; [ ericsagnes ];
doc = ./default.xml;
buildDocsInSandbox = true;
};
}
</programlisting>
Expand All @@ -51,5 +52,44 @@
$ nix-build nixos/release.nix -A manual.x86_64-linux
</programlisting>
</listitem>
<listitem>
<para>
<literal>buildDocsInSandbox</literal> indicates whether the
option documentation for the module can be built in a derivation
sandbox. This option is currently only honored for modules
shipped by nixpkgs. User modules and modules taken from
<literal>NIXOS_EXTRA_MODULE_PATH</literal> are always built
outside of the sandbox, as has been the case in previous
releases.
</para>
<para>
Building NixOS option documentation in a sandbox allows caching
of the built documentation, which greatly decreases the amount
of time needed to evaluate a system configuration that has NixOS
documentation enabled. The sandbox also restricts which
attributes may be referenced by documentation attributes (such
as option descriptions) to the <literal>options</literal> and
<literal>lib</literal> module arguments and the
<literal>pkgs.formats</literal> attribute of the
<literal>pkgs</literal> argument, <literal>config</literal> and
the rest of <literal>pkgs</literal> are disallowed and will
cause doc build failures when used. This restriction is
necessary because we cannot reproduce the full nixpkgs
instantiation with configuration and overlays from a system
configuration inside the sandbox. The <literal>options</literal>
argument only includes options of modules that are also built
inside the sandbox, referencing an option of a module that isn’t
built in the sandbox is also forbidden.
</para>
<para>
The default is <literal>true</literal> and should usually not be
changed; set it to <literal>false</literal> only if the module
requires access to <literal>pkgs</literal> in its documentation
(e.g. because it loads information from a linked package to
build an option type) or if its documentation depends on other
modules that also aren’t sandboxed (e.g. by using types defined
in the other module).
</para>
</listitem>
</itemizedlist>
</section>
53 changes: 53 additions & 0 deletions nixos/lib/eval-cacheable-options.nix
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
{ libPath
, pkgsLibPath
, nixosPath
, modules
, stateVersion
, release
}:

let
lib = import libPath;
modulesPath = "${nixosPath}/modules";
# dummy pkgs set that contains no packages, only `pkgs.lib` from the full set.
# not having `pkgs.lib` causes all users of `pkgs.formats` to fail.
pkgs = import pkgsLibPath {
inherit lib;
pkgs = null;
};
utils = import "${nixosPath}/lib/utils.nix" {
inherit config lib;
pkgs = null;
};
# this is used both as a module and as specialArgs.
# as a module it sets the _module special values, as specialArgs it makes `config`
# unusable. this causes documentation attributes depending on `config` to fail.
config = {
_module.check = false;
_module.args = {};
system.stateVersion = stateVersion;
};
eval = lib.evalModules {
modules = (map (m: "${modulesPath}/${m}") modules) ++ [
config
];
specialArgs = {
inherit config pkgs utils;
};
};
docs = import "${nixosPath}/doc/manual" {
pkgs = pkgs // {
inherit lib;
# duplicate of the declaration in all-packages.nix
buildPackages.nixosOptionsDoc = attrs:
(import "${nixosPath}/lib/make-options-doc")
({ inherit pkgs lib; } // attrs);
};
config = config.config;
options = eval.options;
version = release;
revision = "release-${release}";
prefix = modulesPath;
};
in
docs.optionsNix
51 changes: 37 additions & 14 deletions nixos/lib/make-options-doc/default.nix
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,13 @@
, options
, transformOptions ? lib.id # function for additional tranformations of the options
, revision ? "" # Specify revision for the options
# a set of options the docs we are generating will be merged into, as if by recursiveUpdate.
# used to split the options doc build into a static part (nixos/modules) and a dynamic part
# (non-nixos modules imported via configuration.nix, other module sources).
, baseOptionsJSON ? null
# instead of printing warnings for eg options with missing descriptions (which may be lost
# by nix build unless -L is given), emit errors instead and fail the build
, warningsAreErrors ? true
}:

let
Expand Down Expand Up @@ -51,10 +58,15 @@ let
# ../../../lib/options.nix influences.
#
# Each element of `relatedPackages` can be either
# - a string: that will be interpreted as an attribute name from `pkgs`,
# - a list: that will be interpreted as an attribute path from `pkgs`,
# - an attrset: that can specify `name`, `path`, `package`, `comment`
# - a string: that will be interpreted as an attribute name from `pkgs` and turned into a link
# to search.nixos.org,
# - a list: that will be interpreted as an attribute path from `pkgs` and turned into a link
# to search.nixos.org,
# - an attrset: that can specify `name`, `path`, `comment`
# (either of `name`, `path` is required, the rest are optional).
#
# NOTE: No checks against `pkgs` are made to ensure that the referenced package actually exists.
# Such checks are not compatible with option docs caching.
genRelatedPackages = packages: optName:
let
unpack = p: if lib.isString p then { name = p; }
Expand All @@ -64,16 +76,16 @@ let
let
title = args.title or null;
name = args.name or (lib.concatStringsSep "." args.path);
path = args.path or [ args.name ];
package = args.package or (lib.attrByPath path (throw "Invalid package attribute path `${toString path}' found while evaluating `relatedPackages' of option `${optName}'") pkgs);
in "<listitem>"
+ "<para><literal>${lib.optionalString (title != null) "${title} aka "}pkgs.${name} (${package.meta.name})</literal>"
+ lib.optionalString (!package.meta.available) " <emphasis>[UNAVAILABLE]</emphasis>"
+ ": ${package.meta.description or "???"}.</para>"
+ lib.optionalString (args ? comment) "\n<para>${args.comment}</para>"
# Lots of `longDescription's break DocBook, so we just wrap them into <programlisting>
+ lib.optionalString (package.meta ? longDescription) "\n<programlisting>${package.meta.longDescription}</programlisting>"
+ "</listitem>";
in ''
<listitem>
<para>
<link xlink:href="https://search.nixos.org/packages?show=${name}&amp;sort=relevance&amp;query=${name}">
<literal>${lib.optionalString (title != null) "${title} aka "}pkgs.${name}</literal>
</link>
</para>
${lib.optionalString (args ? comment) "<para>${args.comment}</para>"}
</listitem>
'';
in "<itemizedlist>${lib.concatStringsSep "\n" (map (p: describe (unpack p)) packages)}</itemizedlist>";

# Remove invisible and internal options.
Expand All @@ -99,13 +111,24 @@ in rec {
optionsJSON = pkgs.runCommand "options.json"
{ meta.description = "List of NixOS options in JSON format";
buildInputs = [ pkgs.brotli ];
options = builtins.toFile "options.json"
(builtins.unsafeDiscardStringContext (builtins.toJSON optionsNix));
}
''
# Export list of options in different format.
dst=$out/share/doc/nixos
mkdir -p $dst
cp ${builtins.toFile "options.json" (builtins.unsafeDiscardStringContext (builtins.toJSON optionsNix))} $dst/options.json
${
if baseOptionsJSON == null
then "cp $options $dst/options.json"
else ''
${pkgs.python3Minimal}/bin/python ${./mergeJSON.py} \
${lib.optionalString warningsAreErrors "--warnings-are-errors"} \
${baseOptionsJSON} $options \
> $dst/options.json
''
}
brotli -9 < $dst/options.json > $dst/options.json.br
Expand Down
86 changes: 86 additions & 0 deletions nixos/lib/make-options-doc/mergeJSON.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
import collections
import json
import sys
from typing import Any, Dict, List

JSON = Dict[str, Any]

class Key:
def __init__(self, path: List[str]):
self.path = path
def __hash__(self):
result = 0
for id in self.path:
result ^= hash(id)
return result
def __eq__(self, other):
return type(self) is type(other) and self.path == other.path

Option = collections.namedtuple('Option', ['name', 'value'])

# pivot a dict of options keyed by their display name to a dict keyed by their path
def pivot(options: Dict[str, JSON]) -> Dict[Key, Option]:
result: Dict[Key, Option] = dict()
for (name, opt) in options.items():
result[Key(opt['loc'])] = Option(name, opt)
return result

# pivot back to indexed-by-full-name
# like the docbook build we'll just fail if multiple options with differing locs
# render to the same option name.
def unpivot(options: Dict[Key, Option]) -> Dict[str, JSON]:
result: Dict[str, Dict] = dict()
for (key, opt) in options.items():
if opt.name in result:
raise RuntimeError(
'multiple options with colliding ids found',
opt.name,
result[opt.name]['loc'],
opt.value['loc'],
)
result[opt.name] = opt.value
return result

warningsAreErrors = sys.argv[1] == "--warnings-are-errors"
optOffset = 1 if warningsAreErrors else 0
options = pivot(json.load(open(sys.argv[1 + optOffset], 'r')))
overrides = pivot(json.load(open(sys.argv[2 + optOffset], 'r')))

# fix up declaration paths in lazy options, since we don't eval them from a full nixpkgs dir
for (k, v) in options.items():
v.value['declarations'] = list(map(lambda s: f'nixos/modules/{s}', v.value['declarations']))

# merge both descriptions
for (k, v) in overrides.items():
cur = options.setdefault(k, v).value
for (ok, ov) in v.value.items():
if ok == 'declarations':
decls = cur[ok]
for d in ov:
if d not in decls:
decls += [d]
elif ok == "type":
# ignore types of placeholder options
if ov != "_unspecified" or cur[ok] == "_unspecified":
cur[ok] = ov
elif ov is not None or cur.get(ok, None) is None:
cur[ok] = ov

# check that every option has a description
hasWarnings = False
for (k, v) in options.items():
if v.value.get('description', None) is None:
severity = "error" if warningsAreErrors else "warning"
hasWarnings = True
print(f"\x1b[1;31m{severity}: option {v.name} has no description\x1b[0m", file=sys.stderr)
v.value['description'] = "This option has no description."
if hasWarnings and warningsAreErrors:
print(
"\x1b[1;31m" +
"Treating warnings as errors. Set documentation.nixos.options.warningsAreErrors " +
"to false to ignore these warnings." +
"\x1b[0m",
file=sys.stderr)
sys.exit(1)

json.dump(unpivot(options), fp=sys.stdout)
3 changes: 3 additions & 0 deletions nixos/modules/i18n/input-method/fcitx.nix
Original file line number Diff line number Diff line change
Expand Up @@ -40,4 +40,7 @@ in
};
services.xserver.displayManager.sessionCommands = "${fcitxPackage}/bin/fcitx";
};

# uses attributes of the linked package
meta.buildDocsInSandbox = false;
}
3 changes: 3 additions & 0 deletions nixos/modules/i18n/input-method/ibus.nix
Original file line number Diff line number Diff line change
Expand Up @@ -80,4 +80,7 @@ in
ibusPackage
];
};

# uses attributes of the linked package
meta.buildDocsInSandbox = false;
}
Loading

0 comments on commit 70d2730

Please sign in to comment.