darwin: change the SDK pattern, update the SDKs, and update source re…

…leases (NixOS#346043)
nxmatic · Nov 25, 2024 · af0f05a · af0f05a
1 parent 8d48200
commit af0f05a
Show file tree

Hide file tree

Showing 28 changed files with 21,064 additions and 1,152 deletions.
diff --git a/pkgs/build-support/bintools-wrapper/add-darwin-ldflags-before.sh b/pkgs/build-support/bintools-wrapper/add-darwin-ldflags-before.sh
@@ -79,3 +79,13 @@ if [ ! "$havePlatformVersionFlag" ]; then
         extraBefore+=(-@darwinPlatform@_version_min "${@darwinMinVersionVariable@_@suffixSalt@:-@darwinMinVersion@}")
     fi
 fi
+
+mangleVarSingle DEVELOPER_DIR ${role_suffixes[@]+"${role_suffixes[@]}"}
+
+# Allow wrapped bintools to do something useful when no `DEVELOPER_DIR` is set, which can happen when
+# the compiler is run outside of a stdenv or intentionally in an environment with no environment variables set.
+DEVELOPER_DIR=${DEVELOPER_DIR_@suffixSalt@:-@fallback_sdk@}
+
+# Darwin looks for frameworks in the SDK located at `DEVELOPER_DIR`.
+extraBefore+=("-F$DEVELOPER_DIR/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk/System/Library/Frameworks")
+extraBefore+=("-L$DEVELOPER_DIR/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk/usr/lib")
diff --git a/pkgs/build-support/bintools-wrapper/default.nix b/pkgs/build-support/bintools-wrapper/default.nix
@@ -32,6 +32,7 @@
 , isCCTools ? bintools.isCCTools or false
 , expand-response-params
 , targetPackages ? {}
+, useMacosReexportHack ? false
 , wrapGas ? false
 
 # Note: the hardening flags are part of the bintools-wrapper, rather than
@@ -136,7 +137,6 @@ let
     else if targetPlatform.isLoongArch64              then "${sharedLibraryLoader}/lib/ld-linux-loongarch*.so.1"
     else if targetPlatform.isDarwin                   then "/usr/lib/dyld"
     else if targetPlatform.isFreeBSD                  then "${sharedLibraryLoader}/libexec/ld-elf.so.1"
-    else if targetPlatform.isOpenBSD                  then "${sharedLibraryLoader}/libexec/ld.so"
     else if hasSuffix "pc-gnu" targetPlatform.config then "ld.so.1"
     else "";
 
@@ -230,9 +230,16 @@ stdenvNoCC.mkDerivation {
         fi
       done
 
+    '' + (if !useMacosReexportHack then ''
       if [ -e ''${ld:-$ldPath/${targetPrefix}ld} ]; then
         wrap ${targetPrefix}ld ${./ld-wrapper.sh} ''${ld:-$ldPath/${targetPrefix}ld}
       fi
+    '' else ''
+      ldInner="${targetPrefix}ld-reexport-delegate"
+      wrap "$ldInner" ${./macos-sierra-reexport-hack.bash} ''${ld:-$ldPath/${targetPrefix}ld}
+      wrap "${targetPrefix}ld" ${./ld-wrapper.sh} "$out/bin/$ldInner"
+      unset ldInner
+    '') + ''
 
       for variant in $ldPath/${targetPrefix}ld.*; do
         basename=$(basename "$variant")
@@ -372,15 +379,24 @@ stdenvNoCC.mkDerivation {
       substituteAll ${./add-flags.sh} $out/nix-support/add-flags.sh
       substituteAll ${./add-hardening.sh} $out/nix-support/add-hardening.sh
       substituteAll ${../wrapper-common/utils.bash} $out/nix-support/utils.bash
-      substituteAll ${../wrapper-common/darwin-sdk-setup.bash} $out/nix-support/darwin-sdk-setup.bash
     ''
 
     ###
     ### Ensure consistent LC_VERSION_MIN_MACOSX
     ###
-    + optionalString targetPlatform.isDarwin ''
-      substituteAll ${./add-darwin-ldflags-before.sh} $out/nix-support/add-local-ldflags-before.sh
-    ''
+    + optionalString targetPlatform.isDarwin (
+      let
+        inherit (targetPlatform)
+          darwinPlatform darwinSdkVersion
+          darwinMinVersion darwinMinVersionVariable;
+      in ''
+        export darwinPlatform=${darwinPlatform}
+        export darwinMinVersion=${darwinMinVersion}
+        export darwinSdkVersion=${darwinSdkVersion}
+        export darwinMinVersionVariable=${darwinMinVersionVariable}
+        substituteAll ${./add-darwin-ldflags-before.sh} $out/nix-support/add-local-ldflags-before.sh
+      ''
+    )
 
     ##
     ## Extra custom steps
@@ -394,17 +410,10 @@ stdenvNoCC.mkDerivation {
     # TODO(@sternenseemann): rename env var via stdenv rebuild
     shell = (getBin runtimeShell + runtimeShell.shellPath or "");
     gnugrep_bin = optionalString (!nativeTools) gnugrep;
-    rm = if nativeTools then "rm" else lib.getExe' coreutils "rm";
-    mktemp = if nativeTools then "mktemp" else lib.getExe' coreutils "mktemp";
     wrapperName = "BINTOOLS_WRAPPER";
     inherit dynamicLinker targetPrefix suffixSalt coreutils_bin;
     inherit bintools_bin libc_bin libc_dev libc_lib;
     default_hardening_flags_str = builtins.toString defaultHardeningFlags;
-  } // lib.mapAttrs (_: lib.optionalString targetPlatform.isDarwin) {
-    # These will become empty strings when not targeting Darwin.
-    inherit (targetPlatform)
-      darwinPlatform darwinSdkVersion
-      darwinMinVersion darwinMinVersionVariable;
   } // lib.optionalAttrs (apple-sdk != null && stdenvNoCC.targetPlatform.isDarwin) {
     # Wrapped compilers should do something useful even when no SDK is provided at `DEVELOPER_DIR`.
     fallback_sdk = apple-sdk.__spliced.buildTarget or apple-sdk;
@@ -417,5 +426,7 @@ stdenvNoCC.mkDerivation {
         attrByPath ["meta" "description"] "System binary utilities" bintools_
         + " (wrapper script)";
       priority = 10;
+  } // optionalAttrs useMacosReexportHack {
+    platforms = platforms.darwin;
   };
 }
diff --git a/pkgs/build-support/cc-wrapper/add-flags.sh b/pkgs/build-support/cc-wrapper/add-flags.sh
@@ -78,12 +78,28 @@ if [ -e @out@/nix-support/cc-cflags-before ]; then
     NIX_CFLAGS_COMPILE_BEFORE_@suffixSalt@="$(< @out@/nix-support/cc-cflags-before) $NIX_CFLAGS_COMPILE_BEFORE_@suffixSalt@"
 fi
 
-# Only add darwin min version flag if a default darwin min version is set,
+# Only add darwin min version flag and set up `DEVELOPER_DIR` if a default darwin min version is set,
 # which is a signal that we're targetting darwin.
 if [ "@darwinMinVersion@" ]; then
     mangleVarSingle @darwinMinVersionVariable@ ${role_suffixes[@]+"${role_suffixes[@]}"}
 
     NIX_CFLAGS_COMPILE_BEFORE_@suffixSalt@="-m@darwinPlatformForCC@-version-min=${@darwinMinVersionVariable@_@suffixSalt@:-@darwinMinVersion@} $NIX_CFLAGS_COMPILE_BEFORE_@suffixSalt@"
+
+    # `DEVELOPER_DIR` is used to dynamically locate libSystem (and the SDK frameworks) based on the SDK at that path.
+    mangleVarSingle DEVELOPER_DIR ${role_suffixes[@]+"${role_suffixes[@]}"}
+
+    # Allow wrapped compilers to do something useful when no `DEVELOPER_DIR` is set, which can happen when
+    # the compiler is run outside of a stdenv or intentionally in an environment with no environment variables set.
+    DEVELOPER_DIR=${DEVELOPER_DIR_@suffixSalt@:-@fallback_sdk@}
+
+    # xcbuild needs `SDKROOT` to be the name of the SDK, which it sets in its own wrapper,
+    # but compilers expect it to point to the absolute path.
+    SDKROOT="$DEVELOPER_DIR/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk"
+
+    # Set up various library paths since compilers may not support (or may have disabled) finding them in the sysroot.
+    NIX_CFLAGS_COMPILE_BEFORE_@suffixSalt@+=" -isysroot $SDKROOT"
+    NIX_CFLAGS_COMPILE_@suffixSalt@+=" -idirafter $SDKROOT/usr/include"
+    NIX_CFLAGS_COMPILE_@suffixSalt@+=" -iframework $SDKROOT/System/Library/Frameworks"
 fi
 
 # That way forked processes will not extend these environment variables again.

diff --git a/pkgs/build-support/cc-wrapper/default.nix b/pkgs/build-support/cc-wrapper/default.nix
@@ -284,6 +284,13 @@ let
     if (targetPlatform.darwinPlatform == "macos" && isGNU) then "macosx"
     else targetPlatform.darwinPlatform
   );
+
+  darwinMinVersion = optionalString targetPlatform.isDarwin (
+    targetPlatform.darwinMinVersion
+  );
+
+  darwinMinVersionVariable = optionalString targetPlatform.isDarwin
+    targetPlatform.darwinMinVersionVariable;
 in
 
 assert includeFortifyHeaders' -> fortify-headers != null;
@@ -700,7 +707,6 @@ stdenvNoCC.mkDerivation {
       substituteAll ${./add-flags.sh} $out/nix-support/add-flags.sh
       substituteAll ${./add-hardening.sh} $out/nix-support/add-hardening.sh
       substituteAll ${../wrapper-common/utils.bash} $out/nix-support/utils.bash
-      substituteAll ${../wrapper-common/darwin-sdk-setup.bash} $out/nix-support/darwin-sdk-setup.bash
     ''
 
     + optionalString cc.langAda or false ''
@@ -733,23 +739,18 @@ stdenvNoCC.mkDerivation {
 
     # for substitution in utils.bash
     # TODO(@sternenseemann): invent something cleaner than passing in "" in case of absence
-    expandResponseParams = lib.optionalString (expand-response-params != "") (lib.getExe expand-response-params);
+    expandResponseParams = "${expand-response-params}/bin/expand-response-params";
     # TODO(@sternenseemann): rename env var via stdenv rebuild
     shell = getBin runtimeShell + runtimeShell.shellPath or "";
     gnugrep_bin = optionalString (!nativeTools) gnugrep;
-    rm = if nativeTools then "rm" else lib.getExe' coreutils "rm";
-    mktemp = if nativeTools then "mktemp" else lib.getExe' coreutils "mktemp";
     # stdenv.cc.cc should not be null and we have nothing better for now.
     # if the native impure bootstrap is gotten rid of this can become `inherit cc;` again.
     cc = optionalString (!nativeTools) cc;
     wrapperName = "CC_WRAPPER";
     inherit suffixSalt coreutils_bin bintools;
     inherit libc_bin libc_dev libc_lib;
-    inherit darwinPlatformForCC;
+    inherit darwinPlatformForCC darwinMinVersion darwinMinVersionVariable;
     default_hardening_flags_str = builtins.toString defaultHardeningFlags;
-  } // lib.mapAttrs (_: lib.optionalString targetPlatform.isDarwin) {
-    # These will become empty strings when not targeting Darwin.
-    inherit (targetPlatform) darwinMinVersion darwinMinVersionVariable;
   } // lib.optionalAttrs (apple-sdk != null && stdenvNoCC.targetPlatform.isDarwin) {
     # Wrapped compilers should do something useful even when no SDK is provided at `DEVELOPER_DIR`.
     fallback_sdk = apple-sdk.__spliced.buildTarget or apple-sdk;

diff --git a/pkgs/by-name/ap/apple-sdk/common/propagate-inputs.nix b/pkgs/by-name/ap/apple-sdk/common/propagate-inputs.nix
@@ -54,8 +54,6 @@ self: super: {
       libiconv
       darwin.libresolv
       darwin.libsbuf
-      # Shipped with the SDK only as a library with no headers
-      (lib.getLib darwin.libutil)
       # Required by some SDK headers
       cupsHeaders
     ]

diff --git a/pkgs/by-name/ap/apple-sdk/metadata/apple-oss-lockfile.json b/pkgs/by-name/ap/apple-sdk/metadata/apple-oss-lockfile.json
@@ -1422,183 +1422,5 @@
       "hash": "sha256-j5Ep1RX5DTJqTGszrF4d/JtzUqZ6nA6XoExqcIQ0RVQ=",
       "version": "10063.101.15"
     }
-  },
-  "15.0": {
-    "CarbonHeaders": {
-      "hash": "sha256-nIPXnLr21yVnpBhx9K5q3l/nPARA6JL/dED08MeyhP8=",
-      "version": "18.1"
-    },
-    "CommonCrypto": {
-      "hash": "sha256-qwQEFoycAw+YLwqpZgJB1Ppg8mrWFnRPDj4I5f2Ggns=",
-      "version": "600032"
-    },
-    "IOAudioFamily": {
-      "hash": "sha256-VSk3jvsITJugtL67Qt0m4qJ879i7Fj6B/NGBFVCwpiU=",
-      "version": "600.2"
-    },
-    "IOBDStorageFamily": {
-      "hash": "sha256-s8hTwX0jq2iPULfBLUwpzqtszWuvJrrLGbmrKa/fY4U=",
-      "version": "24"
-    },
-    "IOCDStorageFamily": {
-      "hash": "sha256-p/2qM5zjXFDRb/DISpEHxQEdvmuLlRGt/Ygc71Yu2rI=",
-      "version": "62"
-    },
-    "IODVDStorageFamily": {
-      "hash": "sha256-1Sa8aZBGNtqJBNHva+YXxET6Wcdm2PgVrTzYT/8qrN4=",
-      "version": "46"
-    },
-    "IOFWDVComponents": {
-      "hash": "sha256-WkfkWnzRupEh20U7vjsTta89clhus6GTkOpXQWXw/bM=",
-      "version": "208"
-    },
-    "IOFireWireAVC": {
-      "hash": "sha256-qR9lSTa7PN5Z9Nis4tfuXlcZGMIU48dete/NPD0UBbE=",
-      "version": "434"
-    },
-    "IOFireWireFamily": {
-      "hash": "sha256-hmErAXjLWIelqJaCrB8J4IiIxyB7S6EHFY+AY9YhmKQ=",
-      "version": "490"
-    },
-    "IOFireWireSBP2": {
-      "hash": "sha256-Xk+PDnUaO9q46nQwHwTKf/QXtGclfs0wTWiUbcV7e4s=",
-      "version": "452"
-    },
-    "IOFireWireSerialBusProtocolTransport": {
-      "hash": "sha256-P7egeaD9SSa+YyrIRzM44gILKbIL7vezXK3M6q3MBOI=",
-      "version": "261"
-    },
-    "IOGraphics": {
-      "hash": "sha256-Ag37fd3tZJLXLVq1yzHOCWGOYYfwwTkC8hnvNaTEaWg=",
-      "version": "598"
-    },
-    "IOHIDFamily": {
-      "hash": "sha256-4hIztdbKpoC0VrRVwZkoCZuByyTGw02lrrcFDBAXyko=",
-      "version": "2102.0.6"
-    },
-    "IOKitUser": {
-      "hash": "sha256-ytMma1ft1fKjCvP0SKdwnVonPEixzthoCR7ML94/pLE=",
-      "version": "100140.0.6"
-    },
-    "IONetworkingFamily": {
-      "hash": "sha256-gZ7Dkk4Iu7AV9K2ioqSeJ1W7bTNxv77bmT18iv3ljLg=",
-      "version": "185"
-    },
-    "IOSerialFamily": {
-      "hash": "sha256-wVS4QTx6MBOS0VrwyCZ3s5Usezwaf8rWzmNnfdDTXTU=",
-      "version": "93"
-    },
-    "IOStorageFamily": {
-      "hash": "sha256-W9H3jzaXLvAb0cziHBpNo5Iom7c7H5bg4MxQIhIsefc=",
-      "version": "317"
-    },
-    "IOUSBFamily": {
-      "hash": "sha256-Z0E3TfKP49toYo1Fo9kElRap8CZ+mVDHy5RIexgJTpA=",
-      "version": "630.4.5"
-    },
-    "Libc": {
-      "hash": "sha256-1t+e8DQUmsrmr1f6QUU4uUm/el3G7EOL+vUO28srPAM=",
-      "version": "1669.0.4"
-    },
-    "Libinfo": {
-      "hash": "sha256-D7JMCakQVCQ9j2zUHQSGB8zZcHD6azwYY3bsJU0JfEE=",
-      "version": "592"
-    },
-    "Libm": {
-      "hash": "sha256-p4BndAag9d0XSMYWQ+c4myGv5qXbKx5E1VghudSbpTk=",
-      "version": "2026"
-    },
-    "Libnotify": {
-      "hash": "sha256-qYox9iQUnU0AGVfPK2p61/2zlNAJmixCE5K7WojMQ3I=",
-      "version": "327.0.5"
-    },
-    "Librpcsvc": {
-      "hash": "sha256-UWYdCQ9QsBqwM01bWr+igINAHSdSluB/FrOclC5AjTI=",
-      "version": "31"
-    },
-    "Libsystem": {
-      "hash": "sha256-nawWJiu2IJ34ek5iOX6CrlqMzev7TuJpUkvDp30ZQ/U=",
-      "version": "1351"
-    },
-    "OpenDirectory": {
-      "hash": "sha256-6fSl8PasCZSBfe0ftaePcBuSEO3syb6kK+mfDI6iR7A=",
-      "version": "146"
-    },
-    "Security": {
-      "hash": "sha256-ptS/IESkJmbcO3H+v6mmN2jvH2mfLXi+fMQfGCLSt7M=",
-      "version": "61439.1.1"
-    },
-    "architecture": {
-      "hash": "sha256-PRNUrhzSOrwmxSPkKmV0LV7yEIik65sdkfKdBqcwFhU=",
-      "version": "282"
-    },
-    "configd": {
-      "hash": "sha256-Wks7d0Kere6YYPJar593ZSC5bbkyKcaTxrHK6Ah6u0c=",
-      "version": "1345"
-    },
-    "copyfile": {
-      "hash": "sha256-lIhl5sr1gewIHkh10hD+H/I7MVPzlHlONGRMOO7OZuA=",
-      "version": "213"
-    },
-    "dtrace": {
-      "hash": "sha256-S0gI+9eTcuZkh0FWmTmZ+AhZ4qoSdnUb5GTp1melk9A=",
-      "version": "409"
-    },
-    "dyld": {
-      "hash": "sha256-q0GN5+4mW8Yxer0XxhlLK7JRM4JDoxeSFDRzbZ738lw=",
-      "version": "1231.3"
-    },
-    "eap8021x": {
-      "hash": "sha256-2FdEb76KBbCAl2iwly4c1Xstar53O8qgGdN/3WXO23U=",
-      "version": "364"
-    },
-    "hfs": {
-      "hash": "sha256-utmIFAW7Gdbbj71oZnHSaTUse9cIN3ZSfXyFTmuxnc4=",
-      "version": "672"
-    },
-    "launchd": {
-      "hash": "sha256-8mW9bnuHmRXCx9py8Wy28C5b2QPICW0rlAps5njYa00=",
-      "version": "842.1.4"
-    },
-    "libclosure": {
-      "hash": "sha256-21OuQearKTN75OgHN+RPLR1VGdf3ZffPYpV51Kj6LYE=",
-      "version": "94"
-    },
-    "libdispatch": {
-      "hash": "sha256-q2oyaEdt8clVLjLwBjAAvLKodpuYThscK3VcQotCmIM=",
-      "version": "1502.0.1"
-    },
-    "libmalloc": {
-      "hash": "sha256-tFaYSvebk4uIIPu/46eMp6QnwiO/SmShjUoFnJjnmsc=",
-      "version": "646.0.13"
-    },
-    "libplatform": {
-      "hash": "sha256-U3TRUGBxuspEPfzdsd+53Kh8E9GmceMhsxxXuQbcdcc=",
-      "version": "340"
-    },
-    "libpthread": {
-      "hash": "sha256-eYHDAt2wNk7hJZJxsC7Y9w4ASKdexidu613kPo7TAKs=",
-      "version": "535"
-    },
-    "mDNSResponder": {
-      "hash": "sha256-w+Pw/VsHl8hkDiS7EEEYKp9P2NVwu8NSVPSn2U15vHM=",
-      "version": "2559.1.1"
-    },
-    "objc4": {
-      "hash": "sha256-Z9UAm/hjjO2K0c7ag/ws4e/Y2nKOWnObPgp4HUZe+W4=",
-      "version": "928.2"
-    },
-    "ppp": {
-      "hash": "sha256-8+QUA79sHf85yvGSPE9qCmGsrZDT3NZnbgZVroJw/Hg=",
-      "version": "1016"
-    },
-    "removefile": {
-      "hash": "sha256-h1jb4DcgDHwi9eiUguc2e5OLP8ZHxCN3B4Myp/DFDBg=",
-      "version": "75"
-    },
-    "xnu": {
-      "hash": "sha256-9cFPrWtTpCb02YrvKX1KWoExoH2VjPdOBU4dscmKL4A=",
-      "version": "11215.1.10"
-    }
   }
 }
diff --git a/pkgs/by-name/ap/apple-sdk/metadata/versions.json b/pkgs/by-name/ap/apple-sdk/metadata/versions.json
@@ -38,10 +38,5 @@
     "url": "https://swcdn.apple.com/content/downloads/14/48/052-59890-A_I0F5YGAY0Y/p9n40hio7892gou31o1v031ng6fnm9sb3c/CLTools_macOSNMOS_SDK.pkg",
     "version": "14.4",
     "hash": "sha256-QozDiwY0Czc0g45vPD7G4v4Ra+3DujCJbSads3fJjjM="
-  },
-  "15": {
-    "url": "https://swcdn.apple.com/content/downloads/33/46/042-32691-A_3MH7S3118O/3dblccqo9ws17dc5lk3hojfbt3s74q0ql6/CLTools_macOSNMOS_SDK.pkg",
-    "version": "15.0",
-    "hash": "sha256-JhaAPyfX46D+9sematdAYAORw40JP3xvleWRz7Hj/1s="
   }
 }
diff --git a/pkgs/by-name/ap/apple-sdk/package.nix b/pkgs/by-name/ap/apple-sdk/package.nix
@@ -48,13 +48,9 @@ let
       (callPackage ./common/propagate-inputs.nix { })
       (callPackage ./common/propagate-xcrun.nix { })
     ]
-    # Older SDKs do not include the libraries re-exported from umbrella frameworks in the umbrellas’ stubs, which causes
-    # link failures for those libraries unless their paths have been rewritten to point to the store.
-    ++ lib.optionals (lib.versionOlder sdkVersion "11.0") [
-      (callPackage ./common/rewrite-sdk-paths.nix { inherit sdkVersion; })
-    ]
-    # This has to happen last.
     ++ [
+      # These have to happen last.
+      (callPackage ./common/rewrite-sdk-paths.nix { inherit sdkVersion; })
       (callPackage ./common/run-build-phase-hooks.nix { })
     ]
   );