Revival of 'Compile core crates before std'

Cargo.toml

@@ -1,6 +1,7 @@
 [workspace]
 members = [
   "src/bootstrap",
+  "src/compiler_builtins_shim",
   "src/rustc",
   "src/libstd",
   "src/libtest",
@@ -59,6 +60,7 @@ cargo = { path = "src/tools/cargo" }
 # that we're shipping as well (to ensure that the rustfmt in RLS and the
 # `rustfmt` executable are the same exact version).
 rustfmt-nightly = { path = "src/tools/rustfmt" }
+hashbrown = { git = "https://github.com/rust-lang/hashbrown" }
 
 # See comments in `src/tools/rustc-workspace-hack/README.md` for what's going on
 # here

src/bootstrap/builder.rs

@@ -781,6 +781,11 @@ impl<'a> Builder<'a> {
         // we copy the libs forward.
         let cmp = self.compiler_for(compiler.stage, compiler.host, target);
 
+        let libcore_stamp = match cmd {
+            "check" | "clippy" | "fix" => check::libcore_stamp(self, cmp, target),
+            _ => compile::libcore_stamp(self, cmp, target),
+        };
+
         let libstd_stamp = match cmd {
             "check" | "clippy" | "fix" => check::libstd_stamp(self, cmp, target),
             _ => compile::libstd_stamp(self, cmp, target),
@@ -816,6 +821,7 @@ impl<'a> Builder<'a> {
                 },
                 Mode::Rustc => {
                     self.clear_if_dirty(&my_out, &self.rustc(compiler));
+                    self.clear_if_dirty(&my_out, &libcore_stamp);
                     self.clear_if_dirty(&my_out, &libstd_stamp);
                     self.clear_if_dirty(&my_out, &libtest_stamp);
                 },

src/bootstrap/check.rs

@@ -1,6 +1,6 @@
 //! Implementation of compiling the compiler and standard library, in "check"-based modes.
 
-use crate::compile::{run_cargo, std_cargo, test_cargo, rustc_cargo, rustc_cargo_env,
+use crate::compile::{run_cargo, core_cargo, std_cargo, test_cargo, rustc_cargo, rustc_cargo_env,
                      add_to_sysroot};
 use crate::builder::{RunConfig, Builder, Kind, ShouldRun, Step};
 use crate::tool::{prepare_tool_cargo, SourceType};
@@ -35,6 +35,8 @@ impl Step for Std {
 
     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
         run.all_krates("std")
+            .path("src/libcore")
+            .path("src/liballoc")
     }
 
     fn make_run(run: RunConfig<'_>) {
@@ -47,19 +49,30 @@ impl Step for Std {
         let target = self.target;
         let compiler = builder.compiler(0, builder.config.build);
 
-        let mut cargo = builder.cargo(compiler, Mode::Std, target, cargo_subcommand(builder.kind));
-        std_cargo(builder, &compiler, target, &mut cargo);
+        let mut core_cargo_invoc = builder.cargo(compiler, Mode::Std, target,
+                                                 cargo_subcommand(builder.kind));
+        core_cargo(builder, &compiler, target, &mut core_cargo_invoc);
+
+        let mut std_cargo_invoc = builder.cargo(compiler, Mode::Std, target,
+                                                cargo_subcommand(builder.kind));
+        std_cargo(builder, &compiler, target, &mut std_cargo_invoc);
 
         let _folder = builder.fold_output(|| format!("stage{}-std", compiler.stage));
+        let libdir = builder.sysroot_libdir(compiler, target);
+        let hostdir = builder.sysroot_libdir(compiler, compiler.host);
         builder.info(&format!("Checking std artifacts ({} -> {})", &compiler.host, target));
         run_cargo(builder,
-                  &mut cargo,
+                  &mut core_cargo_invoc,
+                  args(builder.kind),
+                  &libcore_stamp(builder, compiler, target),
+                  true);
+        add_to_sysroot(&builder, &libdir, &hostdir, &libcore_stamp(builder, compiler, target));
+        run_cargo(builder,
+                  &mut std_cargo_invoc,
                   args(builder.kind),
                   &libstd_stamp(builder, compiler, target),
                   true);
 
-        let libdir = builder.sysroot_libdir(compiler, target);
-        let hostdir = builder.sysroot_libdir(compiler, compiler.host);
         add_to_sysroot(&builder, &libdir, &hostdir, &libstd_stamp(builder, compiler, target));
     }
 }
@@ -254,6 +267,13 @@ impl Step for Rustdoc {
     }
 }
 
+/// Cargo's output path for the core library in a given stage, compiled
+/// by a particular compiler for the specified target.
+pub fn libcore_stamp(builder: &Builder<'_>, compiler: Compiler, target: Interned<String>
+) -> PathBuf {
+    builder.cargo_out(compiler, Mode::Std, target).join(".libcore-check.stamp")
+}
+
 /// Cargo's output path for the standard library in a given stage, compiled
 /// by a particular compiler for the specified target.
 pub fn libstd_stamp(

src/bootstrap/compile.rs

@@ -40,6 +40,8 @@ impl Step for Std {
 
     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
         run.all_krates("std")
+            .path("src/libcore")
+            .path("src/liballoc")
     }
 
     fn make_run(run: RunConfig<'_>) {
@@ -82,6 +84,12 @@ impl Step for Std {
             // still contain the third party objects needed by various targets.
             copy_third_party_objects(builder, &compiler, target);
 
+            builder.ensure(CoreLink {
+                compiler: compiler_to_use,
+                target_compiler: compiler,
+                target,
+            });
+
             builder.ensure(StdLink {
                 compiler: compiler_to_use,
                 target_compiler: compiler,
@@ -92,23 +100,44 @@ impl Step for Std {
 
         copy_third_party_objects(builder, &compiler, target);
 
-        let mut cargo = builder.cargo(compiler, Mode::Std, target, "build");
-        std_cargo(builder, &compiler, target, &mut cargo);
+
+        let out_dir = builder.cargo_out(compiler, Mode::Std, target);
+        builder.clear_if_dirty(&out_dir, &builder.rustc(compiler));
+        let mut core_cargo_invoc = builder.cargo(compiler, Mode::Std, target, "build");
+        core_cargo(builder, &compiler, target, &mut core_cargo_invoc);
+        let std_cargo_invoc = if builder.no_std(target) != Some(true) {
+            let mut std_cargo_invoc = builder.cargo(compiler, Mode::Std, target, "build");
+            std_cargo(builder, &compiler, target, &mut std_cargo_invoc);
+            Some(std_cargo_invoc)
+        } else {
+            None
+        };
 
         let _folder = builder.fold_output(|| format!("stage{}-std", compiler.stage));
         builder.info(&format!("Building stage{} std artifacts ({} -> {})", compiler.stage,
                 &compiler.host, target));
         run_cargo(builder,
-                  &mut cargo,
+                  &mut core_cargo_invoc,
                   vec![],
-                  &libstd_stamp(builder, compiler, target),
+                  &libcore_stamp(builder, compiler, target),
                   false);
-
-        builder.ensure(StdLink {
+        builder.ensure(CoreLink {
             compiler: builder.compiler(compiler.stage, builder.config.build),
             target_compiler: compiler,
             target,
         });
+        if let Some(mut std_cargo_invoc) = std_cargo_invoc {
+            run_cargo(builder,
+                      &mut std_cargo_invoc,
+                      vec![],
+                      &libstd_stamp(builder, compiler, target),
+                      false);
+            builder.ensure(StdLink {
+                compiler: builder.compiler(compiler.stage, builder.config.build),
+                target_compiler: compiler,
+                target,
+            });
+        }
     }
 }
 
@@ -152,15 +181,19 @@ fn copy_third_party_objects(builder: &Builder<'_>, compiler: &Compiler, target:
     }
 }
 
-/// Configure cargo to compile the standard library, adding appropriate env vars
-/// and such.
-pub fn std_cargo(builder: &Builder<'_>,
-                 compiler: &Compiler,
-                 target: Interned<String>,
-                 cargo: &mut Command) {
+fn mac_os_deployment_env_var(cargo: &mut Command) {
     if let Some(target) = env::var_os("MACOSX_STD_DEPLOYMENT_TARGET") {
         cargo.env("MACOSX_DEPLOYMENT_TARGET", target);
     }
+}
+
+/// Configure cargo to compile a few no_std crates like core,
+/// adding appropriate env vars and such.
+pub fn core_cargo(builder: &Builder<'_>,
+                  _compiler: &Compiler,
+                  target: Interned<String>,
+                  cargo: &mut Command) {
+    mac_os_deployment_env_var(cargo);
 
     // Determine if we're going to compile in optimized C intrinsics to
     // the `compiler-builtins` crate. These intrinsics live in LLVM's
@@ -178,27 +211,44 @@ pub fn std_cargo(builder: &Builder<'_>,
     // `compiler-builtins` crate is enabled and it's configured to learn where
     // `compiler-rt` is located.
     let compiler_builtins_root = builder.src.join("src/llvm-project/compiler-rt");
-    let compiler_builtins_c_feature = if compiler_builtins_root.exists() {
+    let mut features = if compiler_builtins_root.exists() {
         cargo.env("RUST_COMPILER_RT_ROOT", &compiler_builtins_root);
-        " compiler-builtins-c".to_string()
+        "compiler-builtins-c".to_string()
     } else {
         String::new()
     };
 
     if builder.no_std(target) == Some(true) {
-        let mut features = "compiler-builtins-mem".to_string();
-        features.push_str(&compiler_builtins_c_feature);
+        features.push_str(" compiler-builtins-mem");
+    }
+
+
+
+    cargo
+        .arg("--manifest-path")
+        .arg(builder.src.join("src/compiler_builtins_shim/Cargo.toml"))
+        .arg("--features")
+        .arg(features);
+}
+
+/// Configure cargo to compile the standard library, adding appropriate env vars
+/// and such.
+pub fn std_cargo(builder: &Builder<'_>,
+                 compiler: &Compiler,
+                 target: Interned<String>,
+                 cargo: &mut Command) {
+    mac_os_deployment_env_var(cargo);
+
+
 
+    if builder.no_std(target) == Some(true) {
         // for no-std targets we only compile a few no_std crates
         cargo
             .args(&["-p", "alloc"])
             .arg("--manifest-path")
-            .arg(builder.src.join("src/liballoc/Cargo.toml"))
-            .arg("--features")
-            .arg("compiler-builtins-mem compiler-builtins-c");
+            .arg(builder.src.join("src/liballoc/Cargo.toml"));
     } else {
-        let mut features = builder.std_features();
-        features.push_str(&compiler_builtins_c_feature);
+        let features = builder.std_features();
 
         if compiler.stage != 0 && builder.config.sanitizers {
             // This variable is used by the sanitizer runtime crates, e.g.
@@ -233,6 +283,44 @@ pub fn std_cargo(builder: &Builder<'_>,
     }
 }
 
+#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
+struct CoreLink {
+    pub compiler: Compiler,
+    pub target_compiler: Compiler,
+    pub target: Interned<String>,
+}
+
+impl Step for CoreLink {
+    type Output = ();
+
+    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+        run.never()
+    }
+
+    /// Link all libcore rlibs/dylibs into the sysroot location.
+    ///
+    /// Links those artifacts generated by `compiler` to a the `stage` compiler's
+    /// sysroot for the specified `host` and `target`.
+    ///
+    /// Note that this assumes that `compiler` has already generated the
+    /// libraries for `target`, and this method will find them in the relevant
+    /// output directory.
+    fn run(self, builder: &Builder<'_>) {
+        let compiler = self.compiler;
+        let target_compiler = self.target_compiler;
+        let target = self.target;
+        builder.info(&format!("Copying stage{} core from stage{} ({} -> {} / {})",
+                target_compiler.stage,
+                compiler.stage,
+                &compiler.host,
+                target_compiler.host,
+                target));
+        let libdir = builder.sysroot_libdir(target_compiler, target);
+        let hostdir = builder.sysroot_libdir(target_compiler, compiler.host);
+        add_to_sysroot(builder, &libdir, &hostdir, &libcore_stamp(builder, compiler, target));
+    }
+}
+
 #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
 struct StdLink {
     pub compiler: Compiler,
@@ -444,9 +532,7 @@ pub fn test_cargo(builder: &Builder<'_>,
                   _compiler: &Compiler,
                   _target: Interned<String>,
                   cargo: &mut Command) {
-    if let Some(target) = env::var_os("MACOSX_STD_DEPLOYMENT_TARGET") {
-        cargo.env("MACOSX_DEPLOYMENT_TARGET", target);
-    }
+    mac_os_deployment_env_var(cargo);
     cargo.arg("--manifest-path")
         .arg(builder.src.join("src/libtest/Cargo.toml"));
 }
@@ -868,6 +954,13 @@ fn copy_lld_to_sysroot(builder: &Builder<'_>,
     builder.copy(&lld_install_root.join("bin").join(&src_exe), &dst.join(&dst_exe));
 }
 
+/// Cargo's output path for libcore in a given stage, compiled
+/// by a particular compiler for the specified target.
+pub fn libcore_stamp(builder: &Builder<'_>, compiler: Compiler, target: Interned<String>
+) -> PathBuf {
+    builder.cargo_out(compiler, Mode::Std, target).join(".libcore.stamp")
+}
+
 /// Cargo's output path for the standard library in a given stage, compiled
 /// by a particular compiler for the specified target.
 pub fn libstd_stamp(

src/bootstrap/doc.rs

@@ -433,7 +433,10 @@ impl Step for Std {
 
     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
         let builder = run.builder;
-        run.all_krates("std").default_condition(builder.config.docs)
+        run.all_krates("std")
+            .path("src/libcore")
+            .path("src/liballoc")
+            .default_condition(builder.config.docs)
     }
 
     fn make_run(run: RunConfig<'_>) {
@@ -476,9 +479,13 @@ impl Step for Std {
         t!(symlink_dir_force(&builder.config, &my_out, &out_dir));
         t!(fs::copy(builder.src.join("src/doc/rust.css"), out.join("rust.css")));
 
-        let run_cargo_rustdoc_for = |package: &str| {
+        let run_cargo_rustdoc_for = |package: &str, std: bool| {
             let mut cargo = builder.cargo(compiler, Mode::Std, target, "rustdoc");
-            compile::std_cargo(builder, &compiler, target, &mut cargo);
+            if std {
+                compile::std_cargo(builder, &compiler, target, &mut cargo);
+            } else {
+                compile::core_cargo(builder, &compiler, target, &mut cargo);
+            }
 
             // Keep a whitelist so we do not build internal stdlib crates, these will be
             // build by the rustc step later if enabled.
@@ -498,9 +505,10 @@ impl Step for Std {
             builder.run(&mut cargo);
             builder.cp_r(&my_out, &out);
         };
-        for krate in &["alloc", "core", "std"] {
-            run_cargo_rustdoc_for(krate);
-        }
+
+        run_cargo_rustdoc_for("alloc", true);
+        run_cargo_rustdoc_for("core", false);
+        run_cargo_rustdoc_for("std", true);
     }
 }
 

src/bootstrap/test.rs

@@ -1671,9 +1671,11 @@ impl Step for Crate {
                 run = run.path(krate.local_path(&builder).to_str().unwrap());
             }
         }
-        run
+        run.path("src/libcore")
+            .path("src/liballoc")
     }
 
+
     fn make_run(run: RunConfig<'_>) {
         let builder = run.builder;
         let compiler = builder.compiler(builder.top_stage, run.host);

src/compiler_builtins_shim/Cargo.toml

@@ -0,0 +1,12 @@
+[package]
+name = "compiler_builtins_shim"
+authors = ["The Rust Project Developers"]
+version = "0.0.0"
+
+
+[dependencies]
+compiler_builtins = { version = "0.1.16", features = ['rustc-dep-of-std']  }
+
+[features]
+compiler-builtins-mem = ['compiler_builtins/mem']
+compiler-builtins-c = ["compiler_builtins/c"]

src/compiler_builtins_shim/src/lib.rs

@@ -0,0 +1,4 @@
+// ignore-tidy
+#![feature(no_core)]
+//#![no_std]
+#![no_core]