darwin: change the SDK pattern, update the SDKs, and update source releases

maintainers/team-list.nix

@@ -194,7 +194,10 @@ with lib.maintainers;
   };
 
   darwin = {
-    members = [ toonn ];
+    members = [
+      reckenrode
+      toonn
+    ];
     githubTeams = [ "darwin-maintainers" ];
     scope = "Maintain Darwin compatibility of packages and Darwin-only packages.";
     shortName = "Darwin";

pkgs/os-specific/darwin/apple-source-releases/default.nix

@@ -228,22 +228,31 @@ in
 self:
 
 let
-  macosPackages_11_0_1 = import ./macos-11.0.1.nix { inherit applePackage'; };
-  developerToolsPackages_11_3_1 = import ./developer-tools-11.3.1.nix { inherit applePackage'; };
+  macosPackages_11_0_1 = import ./macos-11.0.1.nix { inherit applePackage' callPackage; };
+  developerToolsPackages_11_3_1 = import ./developer-tools-11.3.1.nix { inherit applePackage' callPackage; };
 
   applePackage' = namePath: version: sdkName: sha256:
     let
       pname = builtins.head (lib.splitString "/" namePath);
       appleDerivation' = stdenv: appleDerivation'' stdenv pname version sdkName sha256;
       appleDerivation = appleDerivation' stdenv;
-      callPackage = self.newScope { inherit appleDerivation' appleDerivation; python3 = pkgs.buildPackages.python3Minimal; };
+      callPackage = self.newScope {
+        inherit appleDerivation' appleDerivation mkAppleDerivation;
+        python3 = pkgs.buildPackages.python3Minimal;
+      };
     in callPackage (./. + "/${namePath}");
 
   applePackage = namePath: sdkName: sha256: let
     pname = builtins.head (lib.splitString "/" namePath);
     version = versions.${sdkName}.${pname};
   in applePackage' namePath version sdkName sha256;
 
+  callPackage = self.newScope { inherit mkAppleDerivation; };
+
+  mkAppleDerivation = pkgs.callPackage ./mkAppleDerivation.nix {
+    inherit (pkgs.darwin) bootstrapStdenv xcodeProjectCheckHook;
+  };
+
   # Only used for bootstrapping. It’s convenient because it was the last version to come with a real makefile.
   adv_cmds-boot = applePackage "adv_cmds/boot.nix" "osx-10.5.8" "sha256-/OJLNpATyS31W5nWfJgSVO5itp8j55TRwG57/QLT5Fg=" {};
 

pkgs/os-specific/darwin/apple-source-releases/developer-tools-11.3.1.nix

@@ -1,6 +1,6 @@
 # Generated using:  ./generate-sdk-packages.sh developer-tools 11.3.1
 
-{ applePackage' }:
+{ applePackage', callPackage }:
 
 {
 bootstrap_cmds = applePackage' "bootstrap_cmds" "116" "developer-tools-11.3.1" "06nw99ajkd264vdi6n2zv252ppxp3wx3120hqf3jqdh6c1wavy0b" {};

pkgs/os-specific/darwin/apple-source-releases/macos-11.0.1.nix

@@ -1,6 +1,6 @@
 # Generated using:  ./generate-sdk-packages.sh macos 11.0.1
 
-{ applePackage' }:
+{ applePackage', callPackage }:
 
 {
 CommonCrypto = applePackage' "CommonCrypto" "60178.40.2" "macos-11.0.1" "129gsxhhcxqycg0zjrdrz2ay4dv2ih1ckafqh33qrc499z8dam2p" {};

pkgs/os-specific/darwin/apple-source-releases/mkAppleDerivation.nix

@@ -0,0 +1,89 @@
+let
+  versions = builtins.fromJSON (builtins.readFile ./versions.json);
+in
+
+{
+  lib,
+  bootstrapStdenv,
+  fetchFromGitHub,
+  meson,
+  ninja,
+  stdenv,
+  stdenvNoCC,
+  xcodeProjectCheckHook,
+}:
+
+let
+  hasBasenamePrefix = prefix: file: lib.hasPrefix prefix (baseNameOf file);
+in
+lib.makeOverridable (
+  attrs:
+  let
+    attrs' = if lib.isFunction attrs then attrs else _: attrs;
+    attrsFixed = lib.fix attrs';
+    stdenv' =
+      if attrsFixed.noCC or false then
+        stdenvNoCC
+      else if attrsFixed.noBootstrap or false then
+        stdenv
+      else
+        bootstrapStdenv;
+  in
+  stdenv'.mkDerivation (
+    lib.extends (
+      self: super:
+      assert super ? releaseName;
+      let
+        inherit (super) releaseName;
+        info = versions.${releaseName};
+        files = lib.filesystem.listFilesRecursive (lib.path.append ./. releaseName);
+        mesonFiles = lib.filter (hasBasenamePrefix "meson") files;
+      in
+      # You have to have at least `meson.build.in` when using xcodeHash to trigger the Meson
+      # build support in `mkAppleDerivation`.
+      assert super ? xcodeHash -> lib.length mesonFiles > 0;
+      {
+        pname = super.pname or releaseName;
+        inherit (info) version;
+
+        src = super.src or fetchFromGitHub {
+          owner = "apple-oss-distributions";
+          repo = releaseName;
+          rev = info.rev or "${releaseName}-${info.version}";
+          inherit (info) hash;
+        };
+
+        strictDeps = true;
+        __structuredAttrs = true;
+
+        meta = {
+          homepage = "https://opensource.apple.com/releases/";
+          license = lib.licenses.apple-psl20;
+          maintainers = lib.teams.darwin.members;
+          platforms = lib.platforms.darwin;
+        } // super.meta or { };
+      }
+      // lib.optionalAttrs (super ? xcodeHash) {
+        postUnpack =
+          super.postUnpack or ""
+          + lib.concatMapStrings (
+            file:
+            if baseNameOf file == "meson.build.in" then
+              "substitute ${lib.escapeShellArg "${file}"} \"$sourceRoot/meson.build\" --subst-var version\n"
+            else
+              "cp ${lib.escapeShellArg "${file}"} \"$sourceRoot/\"${lib.escapeShellArg (baseNameOf file)}\n"
+          ) mesonFiles;
+
+        xcodeProject = super.xcodeProject or "${releaseName}.xcodeproj";
+
+        nativeBuildInputs = super.nativeBuildInputs or [ ] ++ [
+          meson
+          ninja
+          xcodeProjectCheckHook
+        ];
+
+        mesonBuildType = "release";
+      }
+    ) attrs'
+  )
+)

pkgs/os-specific/darwin/apple-source-releases/versions.json

@@ -0,0 +1,2 @@
+{
+}

pkgs/os-specific/darwin/darwin-min-version-hook/setup-hook.sh

@@ -0,0 +1,26 @@
+local role_post
+getHostRole
+
+# Compare the requested deployment target to the existing one. The deployment target has to be a version number,
+# and this hook tries to do the right thing with deployment targets set outside of it, so it has to parse
+# the version numbers for the comparison manually.
+
+local darwinMinVersion=@deploymentTarget@
+local darwinMinVersionVar=@darwinMinVersionVariable@${role_post}
+
+local currentDeploymentTargetArr
+IFS=. read -a currentDeploymentTargetArr <<< "${!darwinMinVersionVar-0.0.0}"
+
+local darwinMinVersionArr
+IFS=. read -a darwinMinVersionArr <<< "$darwinMinVersion"
+
+local currentDeploymentTarget
+currentDeploymentTarget=$(printf "%02d%02d%02d" "${currentDeploymentTargetArr[0]-0}" "${currentDeploymentTargetArr[1]-0}" "${currentDeploymentTargetArr[2]-0}")
+
+darwinMinVersion=$(printf "%02d%02d%02d" "${darwinMinVersionArr[0]-0}" "${darwinMinVersionArr[1]-0}" "${darwinMinVersionArr[2]-0}")
+
+if [ "$darwinMinVersion" -gt "$currentDeploymentTarget" ]; then
+    export "$darwinMinVersionVar"=@deploymentTarget@
+fi
+
+unset -v role_post currentDeploymentTarget currentDeploymentTargetArr darwinMinVersion darwinMinVersionArr darwinMinVersionVar

pkgs/os-specific/darwin/libSystem/default.nix

@@ -0,0 +1,13 @@
+{ stdenvNoCC }:
+
+# Darwin dynamically determines the `libSystem` to use based on the SDK found at `DEVELOPER_DIR`.
+# By default, this will be `apple-sdk` or one of the versioned variants.
+stdenvNoCC.mkDerivation {
+  pname = "libSystem";
+  version = "B";
+
+  # Silence linker warnings due a missing `lib` (which is added by cc-wrapper).
+  buildCommand = ''
+    mkdir -p "$out/include" "$out/lib"
+  '';
+}

pkgs/os-specific/darwin/xcode-project-check-hook/setup-hook.sh

@@ -0,0 +1,55 @@
+# Verify that the Xcode project has not changed unexpectedly. This is only useful for source releases that are
+# being built with other build systems (e.g., Meson) instead of xcbuild.
+
+verifyXcodeProjectHash() {
+    printHashInstructions() {
+        echo '1. Set xcodeHash to an empty string: `xcodeHash = "";`'
+        echo '2. Build the derivation and wait for it to fail with a hash mismatch'
+        echo '3. Copy the "got: sha256-..." value back into the xcodeHash field'
+        echo '   You should have: xcodeHash = "sha256-XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX=";'
+    }
+
+    if [ -z "${xcodeHash-}" ]; then
+        echo "error: xcodeHash missing"
+        echo
+        echo "To fix the issue:"
+        printHashInstructions
+        exit 1
+    fi
+
+    if [ -z "${xcodeProject-}" ]; then
+        echo "error: xcodeProject missing"
+        echo
+        echo "To fix the issue: Set xcodeProject to the name of the project"
+        exit 1
+    fi
+
+    local xcodeHashArr
+    readarray -t -d - xcodeHashArr < <(printf "$xcodeHash")
+
+    local hashType=${xcodeHashArr[0]}
+    local expectedHash=${xcodeHashArr[1]}
+
+    if [ -z "$hashType" ] || [ -z "$expectedHash" ]; then
+        echo "error: xcodeHash is in invalid format"
+        echo
+        echo "To fix the issue:"
+        printHashInstructions
+        exit 1
+    fi
+
+    local hash
+    hash=$(openssl "$hashType" -binary "$sourceRoot/$xcodeProject/project.pbxproj" | base64)
+
+    if [ "$hash" != "$expectedHash" ]; then
+        echo "error: hash mismatch in $xcodeProject/project.pbxproj"
+        echo "        specified: $xcodeHash"
+        echo "           got:    $hashType-$hash"
+        echo
+        echo 'Upstream Xcode project has changed. Update `meson.build` with any changes, then update `xcodeHash`.'
+        printHashInstructions
+        exit 1
+    fi
+}
+
+postUnpackHooks+=(verifyXcodeProjectHash)

pkgs/top-level/all-packages.nix

@@ -23740,6 +23740,18 @@ with pkgs;
 
   plumed = callPackage ../development/libraries/science/chemistry/plumed { };
 
+  ### DEVELOPMENT / LIBRARIES / DARWIN SDKS
+
+  darwinMinVersionHook =
+    deploymentTarget:
+    makeSetupHook {
+      name = "darwin-deployment-target-hook-${deploymentTarget}";
+      substitutions = {
+        darwinMinVersionVariable = lib.escapeShellArg stdenv.hostPlatform.darwinMinVersionVariable;
+        deploymentTarget = lib.escapeShellArg deploymentTarget;
+      };
+    } ../os-specific/darwin/darwin-min-version-hook/setup-hook.sh;
+
   ### DEVELOPMENT / LIBRARIES / AGDA
 
   agdaPackages = callPackage ./agda-packages.nix {

pkgs/top-level/darwin-packages.nix

@@ -20,6 +20,18 @@ let
   };
 
   aliases = self: super: lib.optionalAttrs config.allowAliases (import ../top-level/darwin-aliases.nix lib self super pkgs);
+
+  mkBootstrapStdenv =
+    stdenv:
+    stdenv.override (old: {
+      extraBuildInputs = map (
+        pkg:
+        if lib.isDerivation pkg && lib.getName pkg == "apple-sdk" then
+          pkg.override { enableBootstrap = true; }
+        else
+          pkg
+      ) (old.extraBuildInputs or [ ]);
+    });
 in
 
 makeScopeWithSplicing' {
@@ -133,6 +145,11 @@ impure-cmds // appleSourcePackages // chooseLibs // {
     bintools = self.binutils-unwrapped;
   };
 
+  # Removes propagated packages from the stdenv, so those packages can be built without depending upon themselves.
+  bootstrapStdenv = mkBootstrapStdenv pkgs.stdenv;
+
+  libSystem = callPackage ../os-specific/darwin/libSystem { };
+
   # TODO(@connorbaker): See https://github.com/NixOS/nixpkgs/issues/229389.
   cf-private = self.apple_sdk.frameworks.CoreFoundation;
 
@@ -225,6 +242,11 @@ impure-cmds // appleSourcePackages // chooseLibs // {
       setupHook = null;
     });
 
+  xcodeProjectCheckHook = pkgs.makeSetupHook {
+    name = "xcode-project-check-hook";
+    propagatedBuildInputs = [ pkgs.pkgsBuildHost.openssl ];
+  } ../os-specific/darwin/xcode-project-check-hook/setup-hook.sh;
+
   # Formerly the CF attribute. Use this is you need the open source release.
   swift-corelibs-foundation = callPackage ../os-specific/darwin/swift-corelibs/corefoundation.nix { };