Merge branch 'master' into ma27-tracking

README.md

@@ -80,7 +80,7 @@ $ nix-build
 You can use the provided shell.nix to get a working development environment:
 ```
 $ nix-shell
-$ ./bootstrap
+$ autoreconfPhase
 $ configurePhase # NOTE: not ./configure
 $ make
 ```

doc/manual/src/configuration.md

@@ -74,6 +74,30 @@ following:
         }
     }
 
+Populating a Cache
+------------------
+
+A common use for Hydra is to pre-build and cache derivations which
+take a long time to build. While it is possible to direcly access the
+Hydra server's store over SSH, a more scalable option is to upload
+built derivations to a remote store like an [S3-compatible object
+store](https://nixos.org/manual/nix/stable/command-ref/new-cli/nix3-help-stores.html#s3-binary-cache-store). Setting
+the `store_uri` parameter will cause Hydra to sign and upload
+derivations as they are built:
+
+```
+store_uri = s3://cache-bucket-name?compression=zstd&parallel-compression=true&write-nar-listing=1&ls-compression=br&log-compression=br&secret-key=/path/to/cache/private/key
+```
+
+This example uses [Zstandard](https://github.com/facebook/zstd)
+compression on derivations to reduce CPU usage on the server, but
+[Brotli](https://brotli.org/) compression for derivation listings and
+build logs because it has better browser support.
+
+See [`nix help
+stores`](https://nixos.org/manual/nix/stable/command-ref/new-cli/nix3-help-stores.html)
+for a description of the store URI format.
+
 Statsd Configuration
 --------------------
 

flake.nix

@@ -2,7 +2,7 @@
   description = "A Nix-based continuous build system";
 
   inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05";
-  inputs.nix.url = "github:NixOS/nix/2.17.0";
+  inputs.nix.url = "github:NixOS/nix/2.19-maintenance";
   inputs.nix.inputs.nixpkgs.follows = "nixpkgs";
 
   outputs = { self, nixpkgs, nix }:
@@ -61,10 +61,11 @@
 
         };
 
-        hydra = with final; let
-          perlDeps = buildEnv {
+        hydra = let
+          inherit (final) lib stdenv;
+          perlDeps = final.buildEnv {
             name = "hydra-perl-deps";
-            paths = with perlPackages; lib.closePropagation
+            paths = with final.perlPackages; lib.closePropagation
               [
                 AuthenSASL
                 CatalystActionREST
@@ -98,8 +99,8 @@
                 FileSlurper
                 FileWhich
                 final.nix.perl-bindings
-                git
                 HTMLTreeBuilderXPath
+                final.git
                 IOCompress
                 IPCRun
                 IPCRun3
@@ -142,15 +143,20 @@
 
           src = self;
 
-          buildInputs =
-            [
+          nativeBuildInputs =
+            with final.buildPackages; [
               makeWrapper
-              autoconf
+              autoreconfHook
               automake
               libtool
-              unzip
               nukeReferences
               pkg-config
+              mdbook
+            ];
+
+          buildInputs =
+            with final; [
+              unzip
               libpqxx
               top-git
               mercurial
@@ -163,7 +169,6 @@
               final.nix
               perlDeps
               perl
-              mdbook
               pixz
               boost
               postgresql_13
@@ -173,7 +178,7 @@
               prometheus-cpp
             ];
 
-          checkInputs = [
+          checkInputs = with final; [
             cacert
             foreman
             glibcLocales
@@ -182,7 +187,7 @@
             python3
           ];
 
-          hydraPath = lib.makeBinPath (
+          hydraPath = with final; lib.makeBinPath (
             [
               subversion
               openssh
@@ -204,7 +209,7 @@
             ] ++ lib.optionals stdenv.isLinux [ rpm dpkg cdrkit ]
           );
 
-          OPENLDAP_ROOT = openldap;
+          OPENLDAP_ROOT = final.openldap;
 
           shellHook = ''
             pushd $(git rev-parse --show-toplevel) >/dev/null
@@ -219,8 +224,6 @@
             popd >/dev/null
           '';
 
-          preConfigure = "autoreconf -vfi";
-
           NIX_LDFLAGS = [ "-lpthread" ];
 
           enableParallelBuilding = true;

hydra-api.yaml

@@ -536,13 +536,13 @@ paths:
               schema:
                 $ref: '#/components/schemas/Error'
 
-  /eval/{build-id}:
+  /eval/{eval-id}:
     get:
-      summary: Retrieves evaluations identified by build id
+      summary: Retrieves evaluations identified by eval id
       parameters:
-        - name: build-id
+        - name: eval-id
           in: path
-          description: build identifier
+          description: eval identifier
           required: true
           schema:
             type: integer
@@ -554,6 +554,24 @@ paths:
               schema:
                 $ref: '#/components/schemas/JobsetEval'
 
+  /eval/{eval-id}/builds:
+    get:
+      summary: Retrieves all builds belonging to an evaluation identified by eval id
+      parameters:
+        - name: eval-id
+          in: path
+          description: eval identifier
+          required: true
+          schema:
+            type: integer
+      responses:
+        '200':
+          description: builds
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/JobsetEvalBuilds'
+
 components:
   schemas:
 
@@ -802,6 +820,13 @@ components:
           additionalProperties:
             $ref: '#/components/schemas/JobsetEvalInput'
 
+    JobsetEvalBuilds:
+      type: array
+      items:
+        type: object
+        additionalProperties:
+          $ref: '#/components/schemas/Build'
+
     JobsetOverview:
       type: array
       items:

src/hydra-eval-jobs/hydra-eval-jobs.cc

@@ -7,6 +7,9 @@
 #include "store-api.hh"
 #include "eval.hh"
 #include "eval-inline.hh"
+#include "eval-settings.hh"
+#include "signals.hh"
+#include "terminal.hh"
 #include "util.hh"
 #include "get-drvs.hh"
 #include "globals.hh"
@@ -53,7 +56,7 @@ using namespace nix;
 static Path gcRootsDir;
 static size_t maxMemorySize;
 
-struct MyArgs : MixEvalArgs, MixCommonArgs
+struct MyArgs : MixEvalArgs, MixCommonArgs, RootArgs
 {
     Path releaseExpr;
     bool flake = false;
@@ -94,7 +97,7 @@ static std::string queryMetaStrings(EvalState & state, DrvInfo & drv, const std:
     rec = [&](Value & v) {
         state.forceValue(v, noPos);
         if (v.type() == nString)
-            res.push_back(v.string.s);
+            res.emplace_back(v.string_view());
         else if (v.isList())
             for (unsigned int n = 0; n < v.listSize(); ++n)
                 rec(*v.listElems()[n]);
@@ -208,20 +211,20 @@ static void worker(
                     for (auto & c : context)
                         std::visit(overloaded {
                             [&](const NixStringContextElem::Built & b) {
-                                job["constituents"].push_back(state.store->printStorePath(b.drvPath));
+                                job["constituents"].push_back(b.drvPath->to_string(*state.store));
                             },
                             [&](const NixStringContextElem::Opaque & o) {
                             },
                             [&](const NixStringContextElem::DrvDeep & d) {
                             },
-                        }, c.raw());
+                        }, c.raw);
 
                     state.forceList(*a->value, a->pos, "while evaluating the `constituents` attribute");
                     for (unsigned int n = 0; n < a->value->listSize(); ++n) {
                         auto v = a->value->listElems()[n];
                         state.forceValue(*v, noPos);
                         if (v->type() == nString)
-                            job["namedConstituents"].push_back(v->str());
+                            job["namedConstituents"].push_back(v->string_view());
                     }
                 }
 
@@ -516,7 +519,7 @@ int main(int argc, char * * argv)
                     auto drvPath2 = store->parseStorePath((std::string) (*job2)["drvPath"]);
                     auto drv2 = store->readDerivation(drvPath2);
                     job["constituents"].push_back(store->printStorePath(drvPath2));
-                    drv.inputDrvs[drvPath2] = {drv2.outputs.begin()->first};
+                    drv.inputDrvs.map[drvPath2].value = {drv2.outputs.begin()->first};
                 }
 
                 if (brokenJobs.empty()) {

src/hydra-evaluator/hydra-evaluator.cc

@@ -2,6 +2,7 @@
 #include "hydra-config.hh"
 #include "pool.hh"
 #include "shared.hh"
+#include "signals.hh"
 
 #include <algorithm>
 #include <thread>