diff --git a/lib/default.nix b/lib/default.nix index d5d47defb8e64..0725a1c9ba3f1 100644 --- a/lib/default.nix +++ b/lib/default.nix @@ -125,7 +125,7 @@ let inherit (self.sources) cleanSourceFilter cleanSource sourceByRegex sourceFilesBySuffices commitIdFromGitRepo cleanSourceWith pathHasContext - canCleanSource pathIsGitRepo; + canCleanSource pathIsGitRepo repoRevToName; inherit (self.modules) evalModules setDefaultModuleLocation unifyModuleSyntax applyModuleArgsIfFunction mergeModules mergeModules' mergeOptionDecls mergeDefinitions diff --git a/lib/sources.nix b/lib/sources.nix index f61ea306aec56..084bdaac49a9f 100644 --- a/lib/sources.nix +++ b/lib/sources.nix @@ -254,6 +254,64 @@ let outPath = builtins.path { inherit filter name; path = origSrc; }; }; + # urlToName : (URL | Path | String) -> String + # + # Transform a URL (or path, or string) into a clean package name. + urlToName = url_: + let + inherit (builtins) stringLength; + url = toString url_; + base = baseNameOf (lib.removeSuffix "/" (lib.last (lib.splitString ":" url))); + # chop away one git or archive-related extension + removeExt = name: let + matchExt = builtins.match "(.*)\\.(git|tar|zip|gz|tgz|bz|tbz|bz2|tbz2|lzma|txz|xz|zstd)" name; + in if matchExt != null then lib.head matchExt else name; + # apply function f to string x while the result shrinks + shrink = f: x: let v = f x; in if stringLength v < stringLength x then shrink f v else x; + in shrink removeExt base; + + # shortRev : (String | Integer) -> String + # + # Given a package revision (like "refs/tags/v12.0"), produce a short revision ("12.0"). + shortRev = rev_: + let + rev = toString rev_; + baseRev = baseNameOf rev; + matchHash = builtins.match "[a-f0-9]+" baseRev; + matchVer = builtins.match "([A-Za-z]+[-_. ]?)*(v)?([0-9.]+.*)" baseRev; + in + if matchHash != null then builtins.substring 0 7 baseRev + else if matchVer != null then lib.last matchVer + else baseRev; + + # repoRevToNameFull : (URL | Path | String) -> (String | Integer | null) -> (String | null) -> String + # + # See `repoRevToName` below. + repoRevToNameFull = repo_: rev_: suffix_: + let + repo = urlToName repo_; + rev = if rev_ != null then "-${shortRev rev_}" else ""; + suffix = if suffix_ != null then "-${suffix_}" else ""; + in "${repo}${rev}${suffix}-source"; + + # repoRevToName : (Bool | String) -> (URL | Path | String) -> (String | Integer | null) -> String -> String + # + # Produce derivation.name attribute for a given repository URL/path/name and (optionally) its revision/version tag. + # + # This is used by fetch(zip|git|FromGitHub|hg|svn|etc) to generate discoverable + # /nix/store paths. + # + # This uses a different implementation depending on the `pretty` argument: + # false -> name everything as "source" + # true -> name everything as "${repo}-${rev}-source" + # "full" -> name everything as "${repo}-${rev}-${fetcher}-source" + repoRevToName = pretty: + # match on `pretty` first to minimize the thunk + if pretty == false then (repo: rev: suffix: "source") + else if pretty == true then (repo: rev: suffix: repoRevToNameFull repo rev null) + else if pretty == "full" then repoRevToNameFull + else throw "invalid value of `pretty'"; + in { pathType = lib.warnIf (lib.isInOldestRelease 2305) @@ -278,6 +336,8 @@ in { pathHasContext canCleanSource + repoRevToName + sourceByRegex sourceFilesBySuffices diff --git a/pkgs/build-support/fetchbitbucket/default.nix b/pkgs/build-support/fetchbitbucket/default.nix index 2f9103f2bb3e0..ceb06d5683c8b 100644 --- a/pkgs/build-support/fetchbitbucket/default.nix +++ b/pkgs/build-support/fetchbitbucket/default.nix @@ -1,7 +1,8 @@ -{ fetchzip, lib }: +{ lib, repoRevToNameMaybe, fetchzip }: lib.makeOverridable ( -{ owner, repo, rev, name ? "source" +{ owner, repo, rev +, name ? repoRevToNameMaybe repo rev "bitbucket" , ... # For hash agility }@args: fetchzip ({ inherit name; diff --git a/pkgs/build-support/fetchgithub/default.nix b/pkgs/build-support/fetchgithub/default.nix index 4ce3c6e84d768..b260f306546c6 100644 --- a/pkgs/build-support/fetchgithub/default.nix +++ b/pkgs/build-support/fetchgithub/default.nix @@ -1,7 +1,8 @@ -{ lib, fetchgit, fetchzip }: +{ lib, repoRevToNameMaybe, fetchgit, fetchzip }: lib.makeOverridable ( -{ owner, repo, rev, name ? "source" +{ owner, repo, rev +, name ? repoRevToNameMaybe repo rev "github" , fetchSubmodules ? false, leaveDotGit ? null , deepClone ? false, private ? false, forceFetchGit ? false , sparseCheckout ? [] diff --git a/pkgs/build-support/fetchgitiles/default.nix b/pkgs/build-support/fetchgitiles/default.nix index be81c6e8a4c21..87c1d2662bc6c 100644 --- a/pkgs/build-support/fetchgitiles/default.nix +++ b/pkgs/build-support/fetchgitiles/default.nix @@ -1,7 +1,10 @@ -{ fetchzip, lib }: +{ fetchzip, repoRevToNameMaybe, lib }: lib.makeOverridable ( -{ url, rev, name ? "source", ... } @ args: +{ url, rev +, name ? repoRevToNameMaybe url rev "gitiles" +, ... +} @ args: fetchzip ({ inherit name; diff --git a/pkgs/build-support/fetchgitlab/default.nix b/pkgs/build-support/fetchgitlab/default.nix index 749883f2365eb..b4daacf705f94 100644 --- a/pkgs/build-support/fetchgitlab/default.nix +++ b/pkgs/build-support/fetchgitlab/default.nix @@ -1,8 +1,9 @@ -{ lib, fetchgit, fetchzip }: +{ lib, repoRevToNameMaybe, fetchgit, fetchzip }: lib.makeOverridable ( # gitlab example -{ owner, repo, rev, protocol ? "https", domain ? "gitlab.com", name ? "source", group ? null +{ owner, repo, rev, protocol ? "https", domain ? "gitlab.com", group ? null +, name ? repoRevToNameMaybe repo rev "gitlab" , fetchSubmodules ? false, leaveDotGit ? false , deepClone ? false, forceFetchGit ? false , sparseCheckout ? [] diff --git a/pkgs/build-support/fetchrepoorcz/default.nix b/pkgs/build-support/fetchrepoorcz/default.nix index 3ac7cace0dcf7..650c930c707ec 100644 --- a/pkgs/build-support/fetchrepoorcz/default.nix +++ b/pkgs/build-support/fetchrepoorcz/default.nix @@ -1,7 +1,8 @@ -{ fetchzip }: +{ lib, repoRevToNameMaybe, fetchzip }: # gitweb example, snapshot support is optional in gitweb -{ repo, rev, name ? "source" +{ repo, rev +, name ? repoRevToNameMaybe repo rev "repoorcz" , ... # For hash agility }@args: fetchzip ({ inherit name; diff --git a/pkgs/build-support/fetchsavannah/default.nix b/pkgs/build-support/fetchsavannah/default.nix index e75e25fc1e70b..3d0a23f87179c 100644 --- a/pkgs/build-support/fetchsavannah/default.nix +++ b/pkgs/build-support/fetchsavannah/default.nix @@ -1,8 +1,9 @@ -{ fetchzip, lib }: +{ lib, repoRevToNameMaybe, fetchzip }: lib.makeOverridable ( # cgit example, snapshot support is optional in cgit -{ repo, rev, name ? "source" +{ repo, rev +, name ? repoRevToNameMaybe repo rev "savannah" , ... # For hash agility }@args: fetchzip ({ inherit name; diff --git a/pkgs/build-support/fetchsourcehut/default.nix b/pkgs/build-support/fetchsourcehut/default.nix index 42d437b3555e6..5df72ac50351f 100644 --- a/pkgs/build-support/fetchsourcehut/default.nix +++ b/pkgs/build-support/fetchsourcehut/default.nix @@ -1,4 +1,4 @@ -{ fetchgit, fetchhg, fetchzip, lib }: +{ lib, repoRevToNameMaybe, fetchgit, fetchhg, fetchzip }: let inherit (lib) @@ -13,7 +13,7 @@ makeOverridable ( , repo, rev , domain ? "sr.ht" , vc ? "git" -, name ? "source" +, name ? repoRevToNameMaybe repo rev "sourcehut" , fetchSubmodules ? false , ... # For hash agility } @ args: diff --git a/pkgs/build-support/fetchsvn/default.nix b/pkgs/build-support/fetchsvn/default.nix index 41752eb55a7a8..b6e61e750a0d3 100644 --- a/pkgs/build-support/fetchsvn/default.nix +++ b/pkgs/build-support/fetchsvn/default.nix @@ -1,42 +1,40 @@ -{ lib, stdenvNoCC, buildPackages +{ lib, config, stdenvNoCC, buildPackages , subversion, glibcLocales, sshSupport ? true, openssh ? null }: -{ url, rev ? "HEAD", sha256 ? "", hash ? "" -, ignoreExternals ? false, ignoreKeywords ? false, name ? null +let + repoToName = url: rev: + let + inherit (lib) removeSuffix splitString reverseList head last elemAt; + base = removeSuffix "/" (last (splitString ":" url)); + path = reverseList (splitString "/" base); + repoName = + # ../repo/trunk -> repo + if head path == "trunk" then elemAt path 1 + # ../repo/branches/branch -> repo-branch + else if elemAt path 1 == "branches" then "${elemAt path 2}-${head path}" + # ../repo/tags/tag -> repo-tag + else if elemAt path 1 == "tags" then "${elemAt path 2}-${head path}" + # ../repo (no trunk) -> repo + else head path; + suffix = lib.optionalString (config.nameSourcesPrettily == "full") "-svn"; + in "${repoName}-r${toString rev}${suffix}-source"; +in + +{ url, rev ? "HEAD" +, name ? repoToName url rev +, sha256 ? "", hash ? "" +, ignoreExternals ? false, ignoreKeywords ? false , preferLocalBuild ? true }: assert sshSupport -> openssh != null; -let - repoName = with lib; - let - fst = head; - snd = l: head (tail l); - trd = l: head (tail (tail l)); - path_ = - (p: if head p == "" then tail p else p) # ~ drop final slash if any - (reverseList (splitString "/" url)); - path = [ (removeSuffix "/" (head path_)) ] ++ (tail path_); - in - # ../repo/trunk -> repo - if fst path == "trunk" then snd path - # ../repo/branches/branch -> repo-branch - else if snd path == "branches" then "${trd path}-${fst path}" - # ../repo/tags/tag -> repo-tag - else if snd path == "tags" then "${trd path}-${fst path}" - # ../repo (no trunk) -> repo - else fst path; - - name_ = if name == null then "${repoName}-r${toString rev}" else name; -in - if hash != "" && sha256 != "" then throw "Only one of sha256 or hash can be set" else stdenvNoCC.mkDerivation { - name = name_; + inherit name; builder = ./builder.sh; nativeBuildInputs = [ subversion glibcLocales ] ++ lib.optional sshSupport openssh; diff --git a/pkgs/build-support/fetchzip/default.nix b/pkgs/build-support/fetchzip/default.nix index dd04ccb6e0931..3cd1abb577ff2 100644 --- a/pkgs/build-support/fetchzip/default.nix +++ b/pkgs/build-support/fetchzip/default.nix @@ -5,11 +5,14 @@ # (e.g. due to minor changes in the compression algorithm, or changes # in timestamps). -{ lib, fetchurl, withUnzip ? true, unzip, glibcLocalesUtf8 }: +{ lib, repoRevToNameMaybe, fetchurl +, withUnzip ? true, unzip +, glibcLocalesUtf8 +}: -{ name ? "source" -, url ? "" +{ url ? "" , urls ? [] +, name ? repoRevToNameMaybe (if url != "" then url else builtins.head urls) null "unpacked" , nativeBuildInputs ? [] , postFetch ? "" , extraPostFetch ? "" @@ -22,7 +25,8 @@ , extension ? null # the rest are given to fetchurl as is -, ... } @ args: +, ... +} @ args: assert (extraPostFetch != "") -> lib.warn "use 'postFetch' instead of 'extraPostFetch' with 'fetchzip' and 'fetchFromGitHub' or 'fetchFromGitLab'." true; diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index c2719dfa4558b..5b87d8b854900 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -1090,6 +1090,9 @@ with pkgs; broadlink-cli = callPackage ../tools/misc/broadlink-cli { }; + # this is used by most `fetch*` functions + repoRevToNameMaybe = lib.repoRevToName config.nameSourcesPrettily; + fetchpatch = callPackage ../build-support/fetchpatch { # 0.3.4 would change hashes: https://github.com/NixOS/nixpkgs/issues/25154 patchutils = buildPackages.patchutils_0_3_3; diff --git a/pkgs/top-level/config.nix b/pkgs/top-level/config.nix index 67a9a60dbaeaa..f239c968686ee 100644 --- a/pkgs/top-level/config.nix +++ b/pkgs/top-level/config.nix @@ -50,6 +50,39 @@ let default = false; }; + nameSourcesPrettily = mkOption { + type = types.uniq (types.either types.bool (types.enum ["full"])); + default = false; + description = lib.mdDoc '' + This controls the default derivation name attribute generated by the + `fetch*` (like `fetchzip`, `fetchFromGitHub`, etc) functions. + + Possible values and the resulting `.name`: + + - `false` -> `"source"` + - `true` -> `"''${repo}-''${rev}-source"` + - `"full"` -> `"''${repo}-''${rev}-''${fetcherName}-source"` + + The default `false` is the best choice for minimal rebuilds, it will + ignore any non-hash changes (like branches being renamed, source URLs + changing, etc). + + Setting this to `true` greatly helps with discoverability of sources + in `/nix/store` at the cost of a single mass-rebuild for all `src` + derivations (but not their referrers, as all `src`s are fixed-output) + and occasional rebuild when a source changes some of its non-hash + attributes. + + Setting this to `"full"` is similar to setting it to `true`, but the + use of `fetcherName` in the derivation name will force a rebuild when + `src` switches between `fetch*` functions, thus forcing `nix` to check + new derivation's `outputHash`, which is useful for debugging. + + Also, `"full"` is useful for easy collection and tracking of + statistics of where the packages you use are hosted. + ''; + }; + doCheckByDefault = mkMassRebuild { feature = "run `checkPhase` by default"; };