From ab7da439d1c4a0f4d3b281ee2a7a57766d1e8467 Mon Sep 17 00:00:00 2001 From: Fabian Schmitthenner Date: Sun, 21 Feb 2016 04:46:10 +0000 Subject: [PATCH 01/10] Add package parameters to list of packages This uses a heuristic to only include non-package arguments Also, there seems not to be enough laziness to look into all the derivations; so quite some assertions fail Furthermore, this needs a forked nixpkgs --- Makefile | 6 ++-- nixos/packages.tt | 22 +++++++++++- nixpkgs/addAttrs.nix | 71 ++++++++++++++++++++++++++++++++++++++ nixpkgs/get-attributes.nix | 13 +++++++ 4 files changed, 109 insertions(+), 3 deletions(-) create mode 100644 nixpkgs/addAttrs.nix create mode 100644 nixpkgs/get-attributes.nix diff --git a/Makefile b/Makefile index c92f972cbb..54cc178c1c 100644 --- a/Makefile +++ b/Makefile @@ -162,11 +162,13 @@ endif nixpkgs/packages.json.gz: nixpkgs=$$(nix-instantiate --find-file nixpkgs -I nixpkgs=$(NIXPKGS)); \ (echo -n '{ "commit": "' && cat $$nixpkgs/.git-revision && echo -n '","packages":' \ - && nix-env -f '' -I nixpkgs=$(NIXPKGS) -qa --json --arg config '{}' \ + && nix-env -f './nixpkgs/addAttrs.nix' -I nixpkgs=$(NIXPKGS) -qa --json --arg config '{}' \ && echo -n '}') \ | sed "s|$$nixpkgs/||g" | gzip -9 > $@.tmp gunzip < $@.tmp | python -mjson.tool > /dev/null - mv $@.tmp $@ + # I haven't gotten nginx configured to transfer the gzip file + # mv $@.tmp $@ + gunzip < $@.tmp > $@ nixos/options.json.gz: gzip -9 < $$(nix-build --no-out-link '' -I nixpkgs=$(NIXPKGS) -A options)/share/doc/nixos/options.json > $@.tmp diff --git a/nixos/packages.tt b/nixos/packages.tt index 161eb789eb..09c376355b 100644 --- a/nixos/packages.tt +++ b/nixos/packages.tt @@ -60,6 +60,10 @@ Long description: Not specified + + Parameters + Not specified + @@ -205,7 +209,23 @@ function showPackage() { var longDescription = info['meta']['longDescription']; if (typeof longDescription == 'string') $('.long-description', details).empty().addClass("pre").text(longDescription); - + + var parameters = info['meta']['parameters']; + if (typeof parameters == 'object') { + var detailsP = []; + + for (var param in parameters) { + if (parameters.hasOwnProperty(param)) { + detailsP.push($('').append($('').text(param)).append($('').text(parameters[param]))); + } + } + + $('.parameters', details).empty().append($('', {colspan: 2}).addClass('table').append($('').append($('') .addClass('details') .append($('').append($('
').text('Name')).append($('').text('Value'))).append(detailsP)); + + + + } + $(this).after($('
', { colspan: 3 }) diff --git a/nixpkgs/addAttrs.nix b/nixpkgs/addAttrs.nix new file mode 100644 index 0000000000..7a41ab6568 --- /dev/null +++ b/nixpkgs/addAttrs.nix @@ -0,0 +1,71 @@ +#TODO: why do I have to this here? +# (maybe this has to do with more laziness in mkDerivation?) +let config = { + config = { + allowBroken = true; + allowUnfree = true; + }; +}; in +{ nixpkgs ? import ~/nixpkgs config }: + +nixpkgs.overridePackages (self: super: + let get-attribute = package: import ./get-attributes.nix { inherit (nixpkgs) lib; inherit package; }; + inherit (nixpkgs) lib; + blacklisted = x: lib.elem x [ + #TODO: this blacklisting doesn't make sense... + "_.linuxPackages_grsec_stable_server_xen.fusionio-vsl" + "_.linuxPackages_grsec_stable_desktop.fusionio-vsl" + "_.linuxPackages_grsec_stable_server.fusionio-vsl" + #"_.gdb" + "_.gdbGuile" + "_.go_1_4" + "_.gobjectIntrospection" + "_.gnu.libpthreadHeaders" + "_.ocamlPackages.conduit" + "_.ocamlPackages.re2" + "_.ocamlPackages.textutils" + "_.python35Packages.pitz" + "_.python35Packages.pygobject3" + "_.python35Packages.pygtksourceview" + "_.python2" + "_.pythonPackages.python" + "_.python27Packages.python" + "_.python2Full" + "_.python2Packages.python" + "_.pythonFull" + "_.python27Full" + "_.python" + "_.python27" + "_.pythonPackages.wtforms" + "_.python2Packages.wtforms" + "_.python27Packages.wtforms" + "_.python3" + "_.python34" + "_.python35" + # TODO: allow most python packages + "_.python35Packages" + "_.shadow" + "_.gnu.smbfs" + "_.linuxPackages_grsec_stable_server_xen.sysdig" + "_.linuxPackages_grsec_stable_server.sysdig" + "_.linuxPackages_grsec_stable_desktop.sysdig" + "_.linuxPackages_grsec_stable_server_xen.vhba" + "_.linuxPackages_grsec_stable_desktop.vhba" + "_.linuxPackages_grsec_stable_server.vhba" + "_.uclibc" + "_.gnu.unionfs" + "_.gnome3_18.accerciser" + "_.gnome3.accerciser" + ]; + mapRecursively = n: x: + if lib.isDerivation x then + x // { meta = x.meta // { parameters = builtins.trace n (get-attribute x); }; } + else if lib.isAttrs x then + if x ? "recurseForDerivations" then + lib.mapAttrs (name: value: builtins.trace name (if blacklisted "${n}.${name}" then value else mapRecursively "${n}.${name}" value)) x + else + x + else if lib.isList x then + map (mapRecursively "${n}.list") x + else x; in + mapRecursively "_" (nixpkgs // { recurseForDerivations = true; })) \ No newline at end of file diff --git a/nixpkgs/get-attributes.nix b/nixpkgs/get-attributes.nix new file mode 100644 index 0000000000..9d827bde2f --- /dev/null +++ b/nixpkgs/get-attributes.nix @@ -0,0 +1,13 @@ +{ lib +, package }: + +let + arguments = + if !(package ? _function) then { error = "some error happened"; } else + let f = name: isOptional: if builtins.hasAttr name package.origArgs then package.origArgs.${name} else if isOptional then "(optional)" else { type = "derivation"; }; in + lib.mapAttrs f (builtins.functionArgs package._function); + isDerivation = x: + if lib.isList x then lib.any isDerivation x + else lib.isDerivation x || lib.isFunction x; + result = lib.filterAttrs (x: y: !isDerivation y) arguments; +in result \ No newline at end of file From f24b6ec9bfd5617f4b040a5142aecad4f1e75cdd Mon Sep 17 00:00:00 2001 From: Fabian Schmitthenner Date: Sun, 21 Feb 2016 21:39:22 +0000 Subject: [PATCH 02/10] patch nixpkgs --- Makefile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 54cc178c1c..9de649ccc5 100644 --- a/Makefile +++ b/Makefile @@ -161,7 +161,10 @@ endif nixpkgs/packages.json.gz: nixpkgs=$$(nix-instantiate --find-file nixpkgs -I nixpkgs=$(NIXPKGS)); \ - (echo -n '{ "commit": "' && cat $$nixpkgs/.git-revision && echo -n '","packages":' \ + cp -R $$nixpkgs ../home/nixpkgs; \ + patch -d ../home/nixpkgs < nixpkgs/nixpkgs.patch; \ + find ../home/nixpkgs -type f -print0 | xargs -0 perl -pi.back -e 's/assert(.*?);/(if $$1 then builtins.trace "failed assertion" else (x: x))/g'; \ + (echo -n '{ "commit": "' && cat ../home/nixpkgs/.git-revision && echo -n '","packages":' \ && nix-env -f './nixpkgs/addAttrs.nix' -I nixpkgs=$(NIXPKGS) -qa --json --arg config '{}' \ && echo -n '}') \ | sed "s|$$nixpkgs/||g" | gzip -9 > $@.tmp From 0e0cbb3188521a169c75de8917c8b7006c2d492b Mon Sep 17 00:00:00 2001 From: Fabian Schmitthenner Date: Sun, 21 Feb 2016 22:07:04 +0000 Subject: [PATCH 03/10] fix last commit --- Makefile | 5 +++-- nixpkgs/nixpkgs.patch | 19 +++++++++++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) create mode 100644 nixpkgs/nixpkgs.patch diff --git a/Makefile b/Makefile index 9de649ccc5..b0d5aad41e 100644 --- a/Makefile +++ b/Makefile @@ -161,8 +161,9 @@ endif nixpkgs/packages.json.gz: nixpkgs=$$(nix-instantiate --find-file nixpkgs -I nixpkgs=$(NIXPKGS)); \ - cp -R $$nixpkgs ../home/nixpkgs; \ - patch -d ../home/nixpkgs < nixpkgs/nixpkgs.patch; \ + cp -R $$nixpkgs ../home/nixpkgs/*xz; \ + chmod u+w -R ../home/nixpkgs; \ + patch -p1 -d ../home/nixpkgs < nixpkgs/nixpkgs.patch; \ find ../home/nixpkgs -type f -print0 | xargs -0 perl -pi.back -e 's/assert(.*?);/(if $$1 then builtins.trace "failed assertion" else (x: x))/g'; \ (echo -n '{ "commit": "' && cat ../home/nixpkgs/.git-revision && echo -n '","packages":' \ && nix-env -f './nixpkgs/addAttrs.nix' -I nixpkgs=$(NIXPKGS) -qa --json --arg config '{}' \ diff --git a/nixpkgs/nixpkgs.patch b/nixpkgs/nixpkgs.patch new file mode 100644 index 0000000000..08bd6a68da --- /dev/null +++ b/nixpkgs/nixpkgs.patch @@ -0,0 +1,19 @@ +diff --git a/lib/customisation.nix b/lib/customisation.nix +index 5854954..57995a4 100644 +--- a/lib/customisation.nix ++++ b/lib/customisation.nix +@@ -60,9 +60,14 @@ rec { + { override = newArgs: makeOverridable f (overrideWith newArgs); + overrideDerivation = fdrv: + makeOverridable (args: overrideDerivation (f args) fdrv) origArgs; ++ inherit origArgs; ++ _function = if lib.isFunction f then f else f._function; ++ + }) + else if builtins.isFunction ff then + { override = newArgs: makeOverridable f (overrideWith newArgs); ++ inherit origArgs; ++ _function = if lib.isFunction f then f else f._function; + __functor = self: ff; + overrideDerivation = throw "overrideDerivation not yet supported for functors"; + } From b7f5344f0c6176e7f15635def38773a117202ee6 Mon Sep 17 00:00:00 2001 From: Fabian Schmitthenner Date: Mon, 29 Feb 2016 03:08:41 +0000 Subject: [PATCH 04/10] generate package parameter info.This currently uses a hardcoded path to nix (a patched version is needed, see https://github.com/fkz/nix/tree/catch-throw) --- Makefile | 14 +--- nixpkgs/addAttrs.nix | 71 ------------------- nixpkgs/get-attributes.nix | 13 ---- nixpkgs/packages.json-generation/addAttrs.nix | 33 +++++++++ .../generate-packages.json.gzip | 19 +++++ .../get-attributes.nix | 17 +++++ .../nixpkgs.patch | 0 7 files changed, 70 insertions(+), 97 deletions(-) delete mode 100644 nixpkgs/addAttrs.nix delete mode 100644 nixpkgs/get-attributes.nix create mode 100644 nixpkgs/packages.json-generation/addAttrs.nix create mode 100755 nixpkgs/packages.json-generation/generate-packages.json.gzip create mode 100644 nixpkgs/packages.json-generation/get-attributes.nix rename nixpkgs/{ => packages.json-generation}/nixpkgs.patch (100%) diff --git a/Makefile b/Makefile index b0d5aad41e..9b48f22a1b 100644 --- a/Makefile +++ b/Makefile @@ -160,19 +160,7 @@ ifeq ($(UPDATE), 1) endif nixpkgs/packages.json.gz: - nixpkgs=$$(nix-instantiate --find-file nixpkgs -I nixpkgs=$(NIXPKGS)); \ - cp -R $$nixpkgs ../home/nixpkgs/*xz; \ - chmod u+w -R ../home/nixpkgs; \ - patch -p1 -d ../home/nixpkgs < nixpkgs/nixpkgs.patch; \ - find ../home/nixpkgs -type f -print0 | xargs -0 perl -pi.back -e 's/assert(.*?);/(if $$1 then builtins.trace "failed assertion" else (x: x))/g'; \ - (echo -n '{ "commit": "' && cat ../home/nixpkgs/.git-revision && echo -n '","packages":' \ - && nix-env -f './nixpkgs/addAttrs.nix' -I nixpkgs=$(NIXPKGS) -qa --json --arg config '{}' \ - && echo -n '}') \ - | sed "s|$$nixpkgs/||g" | gzip -9 > $@.tmp - gunzip < $@.tmp | python -mjson.tool > /dev/null - # I haven't gotten nginx configured to transfer the gzip file - # mv $@.tmp $@ - gunzip < $@.tmp > $@ + nixpkgs/packages.json-generation/generate-packages.json.gzip $@ $(NIXPKGS) nixos/options.json.gz: gzip -9 < $$(nix-build --no-out-link '' -I nixpkgs=$(NIXPKGS) -A options)/share/doc/nixos/options.json > $@.tmp diff --git a/nixpkgs/addAttrs.nix b/nixpkgs/addAttrs.nix deleted file mode 100644 index 7a41ab6568..0000000000 --- a/nixpkgs/addAttrs.nix +++ /dev/null @@ -1,71 +0,0 @@ -#TODO: why do I have to this here? -# (maybe this has to do with more laziness in mkDerivation?) -let config = { - config = { - allowBroken = true; - allowUnfree = true; - }; -}; in -{ nixpkgs ? import ~/nixpkgs config }: - -nixpkgs.overridePackages (self: super: - let get-attribute = package: import ./get-attributes.nix { inherit (nixpkgs) lib; inherit package; }; - inherit (nixpkgs) lib; - blacklisted = x: lib.elem x [ - #TODO: this blacklisting doesn't make sense... - "_.linuxPackages_grsec_stable_server_xen.fusionio-vsl" - "_.linuxPackages_grsec_stable_desktop.fusionio-vsl" - "_.linuxPackages_grsec_stable_server.fusionio-vsl" - #"_.gdb" - "_.gdbGuile" - "_.go_1_4" - "_.gobjectIntrospection" - "_.gnu.libpthreadHeaders" - "_.ocamlPackages.conduit" - "_.ocamlPackages.re2" - "_.ocamlPackages.textutils" - "_.python35Packages.pitz" - "_.python35Packages.pygobject3" - "_.python35Packages.pygtksourceview" - "_.python2" - "_.pythonPackages.python" - "_.python27Packages.python" - "_.python2Full" - "_.python2Packages.python" - "_.pythonFull" - "_.python27Full" - "_.python" - "_.python27" - "_.pythonPackages.wtforms" - "_.python2Packages.wtforms" - "_.python27Packages.wtforms" - "_.python3" - "_.python34" - "_.python35" - # TODO: allow most python packages - "_.python35Packages" - "_.shadow" - "_.gnu.smbfs" - "_.linuxPackages_grsec_stable_server_xen.sysdig" - "_.linuxPackages_grsec_stable_server.sysdig" - "_.linuxPackages_grsec_stable_desktop.sysdig" - "_.linuxPackages_grsec_stable_server_xen.vhba" - "_.linuxPackages_grsec_stable_desktop.vhba" - "_.linuxPackages_grsec_stable_server.vhba" - "_.uclibc" - "_.gnu.unionfs" - "_.gnome3_18.accerciser" - "_.gnome3.accerciser" - ]; - mapRecursively = n: x: - if lib.isDerivation x then - x // { meta = x.meta // { parameters = builtins.trace n (get-attribute x); }; } - else if lib.isAttrs x then - if x ? "recurseForDerivations" then - lib.mapAttrs (name: value: builtins.trace name (if blacklisted "${n}.${name}" then value else mapRecursively "${n}.${name}" value)) x - else - x - else if lib.isList x then - map (mapRecursively "${n}.list") x - else x; in - mapRecursively "_" (nixpkgs // { recurseForDerivations = true; })) \ No newline at end of file diff --git a/nixpkgs/get-attributes.nix b/nixpkgs/get-attributes.nix deleted file mode 100644 index 9d827bde2f..0000000000 --- a/nixpkgs/get-attributes.nix +++ /dev/null @@ -1,13 +0,0 @@ -{ lib -, package }: - -let - arguments = - if !(package ? _function) then { error = "some error happened"; } else - let f = name: isOptional: if builtins.hasAttr name package.origArgs then package.origArgs.${name} else if isOptional then "(optional)" else { type = "derivation"; }; in - lib.mapAttrs f (builtins.functionArgs package._function); - isDerivation = x: - if lib.isList x then lib.any isDerivation x - else lib.isDerivation x || lib.isFunction x; - result = lib.filterAttrs (x: y: !isDerivation y) arguments; -in result \ No newline at end of file diff --git a/nixpkgs/packages.json-generation/addAttrs.nix b/nixpkgs/packages.json-generation/addAttrs.nix new file mode 100644 index 0000000000..c4f45e0901 --- /dev/null +++ b/nixpkgs/packages.json-generation/addAttrs.nix @@ -0,0 +1,33 @@ + let config = { + config = { + allowBroken = true; + allowUnfree = true; + }; +}; in +{ nixpkgs ? import ./nixpkgs config }: + +let inherit (nixpkgs) lib; + forceCatch = error-handler: x: + let result = builtins.catch (builtins.deepSeq x x); in + if lib.isAttrs result && result ? type && result.type == "error" then error-handler result else result; + get-attribute = package: import ./get-attributes.nix { inherit (nixpkgs) lib; inherit package; }; + info = x: forceCatch (x:x) { + name = x.name; + system = x.system; + meta = x.meta; + parameters = get-attribute x; + }; + mapRecursively = n: x: + forceCatch (error: [ ({ ${n + "(error)"} = { name = n; meta = { longDescription = error.message; }; }; }) ]) + (if lib.isDerivation x then + [(lib.setAttrByPath [n] (info x))] + else if lib.isAttrs x then + if x ? "recurseForDerivations" && x.recurseForDerivations then + lib.concatMap (name: mapRecursively "${n}.${name}" x.${name}) (lib.attrNames x) + else + [] + else if lib.isList x then + [] # map (mapRecursively "${n}.list") x + else []); in +lib.zipAttrsWith (name: values: lib.head values) +(mapRecursively "__elim__" (nixpkgs // { recurseForDerivations = true; })) \ No newline at end of file diff --git a/nixpkgs/packages.json-generation/generate-packages.json.gzip b/nixpkgs/packages.json-generation/generate-packages.json.gzip new file mode 100755 index 0000000000..affd13914e --- /dev/null +++ b/nixpkgs/packages.json-generation/generate-packages.json.gzip @@ -0,0 +1,19 @@ +#!/bin/sh +nixpkgs=$(nix-instantiate --find-file nixpkgs -I nixpkgs=$2); \ +pushd nixpkgs/packages.json-generation; \ +rm -rf nixpkgs +cp -R $nixpkgs nixpkgs +chmod u+w -R nixpkgs +patch -p1 -d nixpkgs < nixpkgs.patch +(echo -n '{ "commit": "' && cat nixpkgs/.git-revision && echo -n '","packages":' \ + && ~/src/working-nix/inst/bin/nix-instantiate --expr --eval 'import ./addAttrs.nix {}' --strict --json --show-trace \ + && echo -n '}') \ +| sed "s|$(pwd)/nixpkgs/||g" | sed "s|__elim__.||g" | gzip -9 > ../../$1.tmp +popd +echo "Written $1.tmp" +# gunzip < $1.tmp | python -mjson.tool + +# I haven't gotten my system configured so it +# does the gzipping right +# mv $1.tmp $1 +gunzip < $1.tmp > $1 diff --git a/nixpkgs/packages.json-generation/get-attributes.nix b/nixpkgs/packages.json-generation/get-attributes.nix new file mode 100644 index 0000000000..e6c03875c9 --- /dev/null +++ b/nixpkgs/packages.json-generation/get-attributes.nix @@ -0,0 +1,17 @@ +{ lib +, package }: + +let + isSimple = x: builtins.isString x || builtins.isBool x || builtins.isInt x || builtins.isNull x; + protect = x: if isDerivation x then { type = "derivation"; } else if isSimple x then x else "(protected)"; + arguments = + if !(package ? _function) then { error = "some error happened"; } else + let f = name: isOptional: if builtins.hasAttr name package.origArgs then protect package.origArgs.${name} else if isOptional then "(optional)" else { type = "derivation"; }; in + lib.mapAttrs f (builtins.functionArgs package._function); + isDerivation = x: + lib.isDerivation x || lib.isFunction x || + (if lib.isList x then lib.any isDerivation x + else if lib.isAttrs x then lib.any isDerivation (lib.attrValues x) + else false); + result = lib.filterAttrs (x: y: !isDerivation y) arguments; +in result \ No newline at end of file diff --git a/nixpkgs/nixpkgs.patch b/nixpkgs/packages.json-generation/nixpkgs.patch similarity index 100% rename from nixpkgs/nixpkgs.patch rename to nixpkgs/packages.json-generation/nixpkgs.patch From d13dd4662dd5f912e792a005e7adce3cf5a16f59 Mon Sep 17 00:00:00 2001 From: Fabian Schmitthenner Date: Mon, 7 Mar 2016 22:30:32 +0000 Subject: [PATCH 05/10] build the patched nix version and a few other additions namely: - also allow override in callPackagesWith (which doesn't use makeOverridable) - support makeOverridable in the functor case - also build 16.03 channel (not displaying yet) - better error message when parameters couldn't be determined --- Makefile | 11 ++++++++++- nixpkgs/packages.json-generation/addAttrs.nix | 4 ++-- .../generate-packages.json.gzip | 17 ++++++++++++++--- .../get-attributes.nix | 4 ++-- nixpkgs/packages.json-generation/nixpkgs.patch | 18 +++++++++++++++--- 5 files changed, 43 insertions(+), 11 deletions(-) diff --git a/Makefile b/Makefile index 9b48f22a1b..a6c67c1364 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,6 @@ NIXOS_VERSION = 16.03 NIXPKGS = https://nixos.org/channels/nixos-$(NIXOS_VERSION)/nixexprs.tar.xz +NIXPKGS_UNSTABLE = https://nixos.org/channels/nixos-16.03-beta/nixexprs.tar.xz rwildcard=$(foreach d,$(wildcard $1*),$(call rwildcard,$d/,$2) $(filter $(subst *,%,$2),$d)) @@ -15,7 +16,11 @@ HTML = index.html news.html \ disnix/index.html disnix/download.html disnix/docs.html \ disnix/extensions.html disnix/examples.html disnix/support.html \ docs/papers.html \ - nixops/index.html + nixops/index.html \ + nixpkgs/packages.json.gz \ + nixpkgs/packages-unstable.json.gz \ + nixos/options.json.gz \ + nix/install ### Prettify the NixOS manual. @@ -162,6 +167,10 @@ endif nixpkgs/packages.json.gz: nixpkgs/packages.json-generation/generate-packages.json.gzip $@ $(NIXPKGS) +nixpkgs/packages-unstable.json.gz: + nixpkgs/packages.json-generation/generate-packages.json.gzip $@ $(NIXPKGS_UNSTABLE) + + nixos/options.json.gz: gzip -9 < $$(nix-build --no-out-link '' -I nixpkgs=$(NIXPKGS) -A options)/share/doc/nixos/options.json > $@.tmp mv $@.tmp $@ diff --git a/nixpkgs/packages.json-generation/addAttrs.nix b/nixpkgs/packages.json-generation/addAttrs.nix index c4f45e0901..0dc8295097 100644 --- a/nixpkgs/packages.json-generation/addAttrs.nix +++ b/nixpkgs/packages.json-generation/addAttrs.nix @@ -8,7 +8,7 @@ let inherit (nixpkgs) lib; forceCatch = error-handler: x: - let result = builtins.catch (builtins.deepSeq x x); in + let result = builtins.catch "EvalError" (builtins.deepSeq x x); in if lib.isAttrs result && result ? type && result.type == "error" then error-handler result else result; get-attribute = package: import ./get-attributes.nix { inherit (nixpkgs) lib; inherit package; }; info = x: forceCatch (x:x) { @@ -18,7 +18,7 @@ let inherit (nixpkgs) lib; parameters = get-attribute x; }; mapRecursively = n: x: - forceCatch (error: [ ({ ${n + "(error)"} = { name = n; meta = { longDescription = error.message; }; }; }) ]) + forceCatch (error: [ ({ ${n + "(error)"} = { name = n; inherit error; meta = { longDescription = error.message; }; }; }) ]) (if lib.isDerivation x then [(lib.setAttrByPath [n] (info x))] else if lib.isAttrs x then diff --git a/nixpkgs/packages.json-generation/generate-packages.json.gzip b/nixpkgs/packages.json-generation/generate-packages.json.gzip index affd13914e..7fd568f1e5 100755 --- a/nixpkgs/packages.json-generation/generate-packages.json.gzip +++ b/nixpkgs/packages.json-generation/generate-packages.json.gzip @@ -1,17 +1,28 @@ #!/bin/sh +set -e +set -o pipefail +# hack to build nix +unset IN_NIX_SHELL +alias curl="curl --insecure" +# download nixpkgs and patch nixpkgs=$(nix-instantiate --find-file nixpkgs -I nixpkgs=$2); \ pushd nixpkgs/packages.json-generation; \ -rm -rf nixpkgs +rm -rf nixpkgs nix cp -R $nixpkgs nixpkgs chmod u+w -R nixpkgs patch -p1 -d nixpkgs < nixpkgs.patch +# download and build patched nix version +rm -rf nix +curl -L https://github.com/fkz/nix/archive/generate-nixos-packages.tar.gz | tar xz +mv nix-generate-nixos-packages nix +nix-build -A nixUnstable ./nixpkgs --arg config 'import ./nixUnstable.nix' (echo -n '{ "commit": "' && cat nixpkgs/.git-revision && echo -n '","packages":' \ - && ~/src/working-nix/inst/bin/nix-instantiate --expr --eval 'import ./addAttrs.nix {}' --strict --json --show-trace \ + && ./result/bin/nix-instantiate --eval --expr 'import ./addAttrs.nix {}' --strict --json --show-trace \ && echo -n '}') \ | sed "s|$(pwd)/nixpkgs/||g" | sed "s|__elim__.||g" | gzip -9 > ../../$1.tmp popd echo "Written $1.tmp" -# gunzip < $1.tmp | python -mjson.tool +# gunzip < $1.stmp | python -mjson.tool # I haven't gotten my system configured so it # does the gzipping right diff --git a/nixpkgs/packages.json-generation/get-attributes.nix b/nixpkgs/packages.json-generation/get-attributes.nix index e6c03875c9..46b917e811 100644 --- a/nixpkgs/packages.json-generation/get-attributes.nix +++ b/nixpkgs/packages.json-generation/get-attributes.nix @@ -5,9 +5,9 @@ let isSimple = x: builtins.isString x || builtins.isBool x || builtins.isInt x || builtins.isNull x; protect = x: if isDerivation x then { type = "derivation"; } else if isSimple x then x else "(protected)"; arguments = - if !(package ? _function) then { error = "some error happened"; } else + if !(package ? _function) then { error = "This derivation seems not to use lib.makeOverridable, so parameters couldn't be determined"; } else let f = name: isOptional: if builtins.hasAttr name package.origArgs then protect package.origArgs.${name} else if isOptional then "(optional)" else { type = "derivation"; }; in - lib.mapAttrs f (builtins.functionArgs package._function); + lib.mapAttrs f package._functionArgs; isDerivation = x: lib.isDerivation x || lib.isFunction x || (if lib.isList x then lib.any isDerivation x diff --git a/nixpkgs/packages.json-generation/nixpkgs.patch b/nixpkgs/packages.json-generation/nixpkgs.patch index 08bd6a68da..17e6908165 100644 --- a/nixpkgs/packages.json-generation/nixpkgs.patch +++ b/nixpkgs/packages.json-generation/nixpkgs.patch @@ -2,18 +2,30 @@ diff --git a/lib/customisation.nix b/lib/customisation.nix index 5854954..57995a4 100644 --- a/lib/customisation.nix +++ b/lib/customisation.nix -@@ -60,9 +60,14 @@ rec { +@@ -60,9 +60,15 @@ rec { { override = newArgs: makeOverridable f (overrideWith newArgs); overrideDerivation = fdrv: makeOverridable (args: overrideDerivation (f args) fdrv) origArgs; + inherit origArgs; + _function = if lib.isFunction f then f else f._function; -+ ++ _functionArgs = builtins.functionArgs f; }) else if builtins.isFunction ff then { override = newArgs: makeOverridable f (overrideWith newArgs); + inherit origArgs; + _function = if lib.isFunction f then f else f._function; - __functor = self: ff; ++ _functionArgs = builtins.functionArgs f; ++ __functor = self: args: let result = makeOverridable ff args; in result // { origArgs = origArgs // result.origArgs; _functionArgs = builtins.functionArgs f // result._functionArgs; }; +- __functor = self: ff; overrideDerivation = throw "overrideDerivation not yet supported for functors"; } +@@ -112,5 +119,9 @@ + pkgs = f finalArgs; + mkAttrOverridable = name: pkg: pkg // { + override = newArgs: mkAttrOverridable name (f (finalArgs // newArgs)).${name}; ++ _function = if lib.isFunction f then f else f._function; ++ _functionArgs = builtins.functionArgs f; ++ _functionArgs = builtins.functionArgs f; ++ origArgs = finalArgs; + }; + in lib.mapAttrs mkAttrOverridable pkgs; From 6fa8a66587bd3ce0f8b98279d0e53c21506543ba Mon Sep 17 00:00:00 2001 From: Fabian Schmitthenner Date: Tue, 8 Mar 2016 02:01:22 +0000 Subject: [PATCH 06/10] search for parameters, better error catching (so an error inside of a parameter only hides it), 15.09 and 16.03 package sets are both displayed --- nixos/packages.tt | 61 ++++++++++++++++--- nixpkgs/packages.json-generation/addAttrs.nix | 6 +- .../get-attributes.nix | 10 ++- .../packages.json-generation/nixpkgs.patch | 8 +-- 4 files changed, 64 insertions(+), 21 deletions(-) diff --git a/nixos/packages.tt b/nixos/packages.tt index 09c376355b..5888c8ab0c 100644 --- a/nixos/packages.tt +++ b/nixos/packages.tt @@ -4,6 +4,17 @@ + +
+ + + + +

@@ -125,6 +136,13 @@ function refilter() { var attrs = Object.keys(packageData); var words = $('#search').val().toLowerCase().split(/ +/).filter(Boolean); + var searchParameters = $('#search-in-parameters').prop('checked'); + var showFailing = $('#show-failing-packages').prop('checked'); + if (!showFailing) { + attrs = attrs.filter(function(attr) { + return !('error' in packageData[attr]); + }); + } if (words.length > 0) { attrs = attrs.filter(function (attr) { @@ -136,7 +154,10 @@ function refilter() { return (attr.toLowerCase().indexOf(word) != -1 || info['name'].toLowerCase().indexOf(word) != -1 || description.toLowerCase().indexOf(word) != -1 - || longDescription.toLowerCase().indexOf(word) != -1); + || longDescription.toLowerCase().indexOf(word) != -1 + || (searchParameters && Object.keys(info['parameters']).some(function(val) { + return val.toLowerCase().indexOf(word) != -1; + }))); }; return words.every(match); @@ -209,8 +230,8 @@ function showPackage() { var longDescription = info['meta']['longDescription']; if (typeof longDescription == 'string') $('.long-description', details).empty().addClass("pre").text(longDescription); - - var parameters = info['meta']['parameters']; + + var parameters = info['parameters']; if (typeof parameters == 'object') { var detailsP = []; @@ -219,13 +240,13 @@ function showPackage() { detailsP.push($('
').text(param)).append($('').text(parameters[param]))); } } - + $('.parameters', details).empty().append($('', {colspan: 2}).addClass('table').append($('').append($('') .addClass('details') .append($('
').text('Name')).append($('').text('Value'))).append(detailsP)); - - + + } - + $(this).after($('
', { colspan: 3 }) @@ -233,8 +254,11 @@ function showPackage() { ).attr('expanded', 1); }; -$.ajax({ - url: '[%root%]nixpkgs/packages.json.gz', +function download(name) { +if (request) request.abort(); +$("#how-many").text("Loading..."); +request = $.ajax({ + url: '[%root%]nixpkgs/' + name + '.json.gz', type: 'GET', dataType: 'json', error: function(a, status, error) { alert('Failed to get package data [' + status + ']: ' + error); }, @@ -250,6 +274,25 @@ $.ajax({ } }); +} + +$().ready(function() { + $("#release-stable").change(function() { + download("packages"); + }); + $("#release-unstable").change(function() { + download("packages-unstable"); + }); + $("#search-in-parameters").change(function() { + refilter(); + }); + $("#show-failing-packages").change(function() { + refilter(); + }); +}); + +request = null; +download("packages"); //]]> diff --git a/nixpkgs/packages.json-generation/addAttrs.nix b/nixpkgs/packages.json-generation/addAttrs.nix index 0dc8295097..06ec54084a 100644 --- a/nixpkgs/packages.json-generation/addAttrs.nix +++ b/nixpkgs/packages.json-generation/addAttrs.nix @@ -10,12 +10,12 @@ let inherit (nixpkgs) lib; forceCatch = error-handler: x: let result = builtins.catch "EvalError" (builtins.deepSeq x x); in if lib.isAttrs result && result ? type && result.type == "error" then error-handler result else result; - get-attribute = package: import ./get-attributes.nix { inherit (nixpkgs) lib; inherit package; }; - info = x: forceCatch (x:x) { + get-attribute = package: import ./get-attributes.nix { inherit package forceCatch lib; }; + info = x: { name = x.name; system = x.system; meta = x.meta; - parameters = get-attribute x; + parameters = forceCatch (x: x) (get-attribute x); }; mapRecursively = n: x: forceCatch (error: [ ({ ${n + "(error)"} = { name = n; inherit error; meta = { longDescription = error.message; }; }; }) ]) diff --git a/nixpkgs/packages.json-generation/get-attributes.nix b/nixpkgs/packages.json-generation/get-attributes.nix index 46b917e811..cfa8533332 100644 --- a/nixpkgs/packages.json-generation/get-attributes.nix +++ b/nixpkgs/packages.json-generation/get-attributes.nix @@ -1,12 +1,16 @@ -{ lib +{ lib, forceCatch , package }: let isSimple = x: builtins.isString x || builtins.isBool x || builtins.isInt x || builtins.isNull x; protect = x: if isDerivation x then { type = "derivation"; } else if isSimple x then x else "(protected)"; arguments = - if !(package ? _function) then { error = "This derivation seems not to use lib.makeOverridable, so parameters couldn't be determined"; } else - let f = name: isOptional: if builtins.hasAttr name package.origArgs then protect package.origArgs.${name} else if isOptional then "(optional)" else { type = "derivation"; }; in + if !(package ? _functionArgs) then { error = "This derivation seems not to use lib.makeOverridable, so parameters couldn't be determined"; } else + let f = name: isOptional: + forceCatch (x: x // { type = "derivation"; }) + (if builtins.hasAttr name package.origArgs + then protect package.origArgs.${name} + else if isOptional then "(optional)" else { type = "derivation"; }); in lib.mapAttrs f package._functionArgs; isDerivation = x: lib.isDerivation x || lib.isFunction x || diff --git a/nixpkgs/packages.json-generation/nixpkgs.patch b/nixpkgs/packages.json-generation/nixpkgs.patch index 17e6908165..d388923f4f 100644 --- a/nixpkgs/packages.json-generation/nixpkgs.patch +++ b/nixpkgs/packages.json-generation/nixpkgs.patch @@ -2,29 +2,25 @@ diff --git a/lib/customisation.nix b/lib/customisation.nix index 5854954..57995a4 100644 --- a/lib/customisation.nix +++ b/lib/customisation.nix -@@ -60,9 +60,15 @@ rec { +@@ -60,9 +60,13 @@ rec { { override = newArgs: makeOverridable f (overrideWith newArgs); overrideDerivation = fdrv: makeOverridable (args: overrideDerivation (f args) fdrv) origArgs; + inherit origArgs; -+ _function = if lib.isFunction f then f else f._function; + _functionArgs = builtins.functionArgs f; }) else if builtins.isFunction ff then { override = newArgs: makeOverridable f (overrideWith newArgs); + inherit origArgs; -+ _function = if lib.isFunction f then f else f._function; + _functionArgs = builtins.functionArgs f; + __functor = self: args: let result = makeOverridable ff args; in result // { origArgs = origArgs // result.origArgs; _functionArgs = builtins.functionArgs f // result._functionArgs; }; - __functor = self: ff; overrideDerivation = throw "overrideDerivation not yet supported for functors"; } -@@ -112,5 +119,9 @@ +@@ -112,5 +119,7 @@ pkgs = f finalArgs; mkAttrOverridable = name: pkg: pkg // { override = newArgs: mkAttrOverridable name (f (finalArgs // newArgs)).${name}; -+ _function = if lib.isFunction f then f else f._function; -+ _functionArgs = builtins.functionArgs f; + _functionArgs = builtins.functionArgs f; + origArgs = finalArgs; }; From 8193c646c3ed4e034142b3c3e594a1c8926876fb Mon Sep 17 00:00:00 2001 From: Fabian Schmitthenner Date: Tue, 8 Mar 2016 02:19:59 +0000 Subject: [PATCH 07/10] do gzipping --- .../packages.json-generation/generate-packages.json.gzip | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/nixpkgs/packages.json-generation/generate-packages.json.gzip b/nixpkgs/packages.json-generation/generate-packages.json.gzip index 7fd568f1e5..5d01a8502b 100755 --- a/nixpkgs/packages.json-generation/generate-packages.json.gzip +++ b/nixpkgs/packages.json-generation/generate-packages.json.gzip @@ -23,8 +23,4 @@ nix-build -A nixUnstable ./nixpkgs --arg config 'import ./nixUnstable.nix' popd echo "Written $1.tmp" # gunzip < $1.stmp | python -mjson.tool - -# I haven't gotten my system configured so it -# does the gzipping right -# mv $1.tmp $1 -gunzip < $1.tmp > $1 +mv $1.tmp $1 From ccc0caf3f3aaaedfa95ff8fedb5f94b3b044ddf3 Mon Sep 17 00:00:00 2001 From: Fabian Schmitthenner Date: Tue, 8 Mar 2016 13:54:55 +0000 Subject: [PATCH 08/10] fix duplicated event handlers --- nixos/packages.tt | 45 ++++++++++++++++++++++++--------------------- 1 file changed, 24 insertions(+), 21 deletions(-) diff --git a/nixos/packages.tt b/nixos/packages.tt index 5888c8ab0c..cf17db4a48 100644 --- a/nixos/packages.tt +++ b/nixos/packages.tt @@ -254,6 +254,25 @@ function showPackage() { ).attr('expanded', 1); }; +function initializeEvents() { + $("#release-stable").change(function() { + download("packages"); + }); + $("#release-unstable").change(function() { + download("packages-unstable"); + }); + + $("#search-in-parameters").change(function() { + refilter(); + }); + $("#show-failing-packages").change(function() { + refilter(); + }); + $('#search').on('input', function() { + refilter(); + }); +}s; + function download(name) { if (request) request.abort(); $("#how-many").text("Loading..."); @@ -263,35 +282,19 @@ request = $.ajax({ dataType: 'json', error: function(a, status, error) { alert('Failed to get package data [' + status + ']: ' + error); }, success: function(data) { + var firstTime = request === null; nixpkgsCommit = data.commit; packageData = data.packages; - refilter(); - - $('#search').on('input', function() { - refilter(); - }); + if (firstTime) { + initializeEvents(); + } } }); - } -$().ready(function() { - $("#release-stable").change(function() { - download("packages"); - }); - $("#release-unstable").change(function() { - download("packages-unstable"); - }); - $("#search-in-parameters").change(function() { - refilter(); - }); - $("#show-failing-packages").change(function() { - refilter(); - }); -}); - request = null; +packageData = null; download("packages"); //]]> From 9080c0d9355c55db5a7df3376ce3086a942c200d Mon Sep 17 00:00:00 2001 From: Fabian Schmitthenner Date: Tue, 8 Mar 2016 22:15:42 +0000 Subject: [PATCH 09/10] fix javascript --- nixos/packages.tt | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/nixos/packages.tt b/nixos/packages.tt index cf17db4a48..8ac5837070 100644 --- a/nixos/packages.tt +++ b/nixos/packages.tt @@ -271,24 +271,28 @@ function initializeEvents() { $('#search').on('input', function() { refilter(); }); -}s; +}; function download(name) { if (request) request.abort(); -$("#how-many").text("Loading..."); +$(function() { + $("#how-many").text("Loading..."); +}); request = $.ajax({ url: '[%root%]nixpkgs/' + name + '.json.gz', type: 'GET', dataType: 'json', error: function(a, status, error) { alert('Failed to get package data [' + status + ']: ' + error); }, success: function(data) { - var firstTime = request === null; + $(function() { + var firstTime = packageData === null; nixpkgsCommit = data.commit; packageData = data.packages; refilter(); if (firstTime) { initializeEvents(); } + }); } }); } From d199294691663cec7725b3e0fe5c54c12bbe59a2 Mon Sep 17 00:00:00 2001 From: Fabian Schmitthenner Date: Tue, 8 Mar 2016 22:16:33 +0000 Subject: [PATCH 10/10] use fetchTarball to build nix --- .../packages.json-generation/generate-packages.json.gzip | 4 ---- nixpkgs/packages.json-generation/nixUnstable.nix | 9 +++++++++ 2 files changed, 9 insertions(+), 4 deletions(-) create mode 100644 nixpkgs/packages.json-generation/nixUnstable.nix diff --git a/nixpkgs/packages.json-generation/generate-packages.json.gzip b/nixpkgs/packages.json-generation/generate-packages.json.gzip index 5d01a8502b..bacdb5c43f 100755 --- a/nixpkgs/packages.json-generation/generate-packages.json.gzip +++ b/nixpkgs/packages.json-generation/generate-packages.json.gzip @@ -3,7 +3,6 @@ set -e set -o pipefail # hack to build nix unset IN_NIX_SHELL -alias curl="curl --insecure" # download nixpkgs and patch nixpkgs=$(nix-instantiate --find-file nixpkgs -I nixpkgs=$2); \ pushd nixpkgs/packages.json-generation; \ @@ -12,9 +11,6 @@ cp -R $nixpkgs nixpkgs chmod u+w -R nixpkgs patch -p1 -d nixpkgs < nixpkgs.patch # download and build patched nix version -rm -rf nix -curl -L https://github.com/fkz/nix/archive/generate-nixos-packages.tar.gz | tar xz -mv nix-generate-nixos-packages nix nix-build -A nixUnstable ./nixpkgs --arg config 'import ./nixUnstable.nix' (echo -n '{ "commit": "' && cat nixpkgs/.git-revision && echo -n '","packages":' \ && ./result/bin/nix-instantiate --eval --expr 'import ./addAttrs.nix {}' --strict --json --show-trace \ diff --git a/nixpkgs/packages.json-generation/nixUnstable.nix b/nixpkgs/packages.json-generation/nixUnstable.nix new file mode 100644 index 0000000000..2e4a43ddc2 --- /dev/null +++ b/nixpkgs/packages.json-generation/nixUnstable.nix @@ -0,0 +1,9 @@ +let path-to-nix = builtins.fetchTarball https://github.com/fkz/nix/archive/generate-nixos-packages.tar.gz; in { + packageOverrides = pkgs: { + nixUnstable = pkgs.lib.overrideDerivation pkgs.nixUnstable (old: rec { + name = "nix-${version}"; + version = builtins.readFile (path-to-nix + "/version"); + src = (import "${path-to-nix}/release.nix" {}).tarball + "/tarballs/${name}pre1234_abcdef.tar.xz"; + }); + }; +}