Upstream/c index while building and tar indexstore

crosstool/BUILD.tpl

@@ -51,6 +51,7 @@ cc_toolchain_suite(
             ":make_hashed_objlist.py",
             ":wrapped_clang",
             ":wrapped_clang_pp",
+            "@build_bazel_apple_support_index_import//:index-import",
             ":xcrunwrapper.sh",
         ],
     )

crosstool/cc_toolchain_config.bzl

@@ -2154,6 +2154,50 @@ def _impl(ctx):
         ],
     )
 
+    indexstore_files_feature = feature(
+        name = "indexstore_files",
+        flag_sets = [
+            flag_set(
+                actions = [
+                    ACTION_NAMES.c_compile,
+                    ACTION_NAMES.cpp_compile,
+                    ACTION_NAMES.cpp_module_compile,
+                    ACTION_NAMES.objc_compile,
+                    ACTION_NAMES.objcpp_compile,
+                    ACTION_NAMES.cpp_header_parsing,
+                ],
+                flag_groups = [
+                    flag_group(
+                        flags = ["-index-store-path", "%{indexstore_files}", "-index-ignore-system-symbols"],
+                        expand_if_available = "indexstore_files",
+                    ),
+                ],
+            ),
+        ],
+    )
+
+    use_global_indexstore_feature = feature(
+        name = "use_global_indexstore",
+        env_sets = [
+            env_set(
+                actions = [
+                    ACTION_NAMES.c_compile,
+                    ACTION_NAMES.cpp_compile,
+                    ACTION_NAMES.cpp_module_compile,
+                    ACTION_NAMES.objc_compile,
+                    ACTION_NAMES.objcpp_compile,
+                    ACTION_NAMES.cpp_header_parsing,
+                ],
+                env_entries = [
+                    env_entry(
+                        key = "GLOBAL_INDEXSTORE",
+                        value = "true",
+                    ),
+                ],
+            ),
+        ],
+    )
+
     preprocessor_defines_feature = feature(
         name = "preprocessor_defines",
         enabled = True,
@@ -2472,6 +2516,8 @@ def _impl(ctx):
         sysroot_feature,
         dependency_file_feature,
         serialized_diagnostics_file_feature,
+        indexstore_files_feature,
+        use_global_indexstore_feature,
         pic_feature,
         per_object_debug_info_feature,
         preprocessor_defines_feature,

crosstool/osx_cc_configure.bzl

@@ -136,6 +136,7 @@ def configure_osx_toolchain(repository_ctx):
     wrapped_clang_src_path = str(repository_ctx.path(
         Label("@build_bazel_apple_support//crosstool:wrapped_clang.cc"),
     ))
+    index_import = Label("@build_bazel_apple_support_index_import//:index-import")
 
     xcode_toolchains = []
     xcodeloc_err = ""
@@ -162,6 +163,7 @@ def configure_osx_toolchain(repository_ctx):
     )
     repository_ctx.symlink(xcrunwrapper, "xcrunwrapper.sh")
     repository_ctx.symlink(libtool, "libtool")
+    repository_ctx.symlink(index_import, "index-import")
     repository_ctx.symlink(make_hashed_objlist, "make_hashed_objlist.py")
     repository_ctx.symlink(cc_toolchain_config, "cc_toolchain_config.bzl")
     _compile_cc_file(repository_ctx, libtool_check_unique_src_path, "libtool_check_unique")

crosstool/wrapped_clang.cc

@@ -28,6 +28,7 @@
 #include <spawn.h>
 #include <string.h>
 #include <sys/wait.h>
+#include <sys/stat.h>
 #include <unistd.h>
 
 #include <array>
@@ -260,14 +261,16 @@ static std::unique_ptr<TempFile> WriteResponseFile(
 void ProcessArgument(const std::string arg, const std::string developer_dir,
                      const std::string sdk_root, const std::string cwd,
                      bool relative_ast_path, std::string &linked_binary,
-                     std::string &dsym_path, std::string toolchain_path,
-                     std::function<void(const std::string &)> consumer);
+                     std::string &dsym_path, std::string &original_indexstore_path, 
+                     std::string &global_indexstore_path, std::string &output_file_path,
+                     std::string toolchain_path, std::function<void(const std::string &)> consumer);
 
 bool ProcessResponseFile(const std::string arg, const std::string developer_dir,
                          const std::string sdk_root, const std::string cwd,
                          bool relative_ast_path, std::string &linked_binary,
-                         std::string &dsym_path, std::string toolchain_path,
-                         std::function<void(const std::string &)> consumer) {
+                         std::string &dsym_path, std::string &original_indexstore_path, 
+                         std::string &global_indexstore_path, std::string &output_file_path,
+                         std::string toolchain_path, std::function<void(const std::string &)> consumer) {
   auto path = arg.substr(1);
   std::ifstream original_file(path);
   // Ignore non-file args such as '@loader_path/...'
@@ -280,7 +283,8 @@ bool ProcessResponseFile(const std::string arg, const std::string developer_dir,
     // Arguments in response files might be quoted/escaped, so we need to
     // unescape them ourselves.
     ProcessArgument(Unescape(arg_from_file), developer_dir, sdk_root, cwd,
-                    relative_ast_path, linked_binary, dsym_path,
+                    relative_ast_path, linked_binary, dsym_path, 
+                    original_indexstore_path, global_indexstore_path, output_file_path,
                     toolchain_path, consumer);
   }
 
@@ -336,12 +340,14 @@ std::string GetToolchainPath(const std::string &toolchain_id) {
 void ProcessArgument(const std::string arg, const std::string developer_dir,
                      const std::string sdk_root, const std::string cwd,
                      bool relative_ast_path, std::string &linked_binary,
-                     std::string &dsym_path, std::string toolchain_path,
-                     std::function<void(const std::string &)> consumer) {
+                     std::string &dsym_path, std::string &original_indexstore_path, 
+                     std::string &global_indexstore_path, std::string &output_file_path,
+                     std::string toolchain_path, std::function<void(const std::string &)> consumer) {
   auto new_arg = arg;
   if (arg[0] == '@') {
     if (ProcessResponseFile(arg, developer_dir, sdk_root, cwd,
                             relative_ast_path, linked_binary, dsym_path,
+                            original_indexstore_path, global_indexstore_path, output_file_path,
                             toolchain_path, consumer)) {
       return;
     }
@@ -374,11 +380,31 @@ void ProcessArgument(const std::string arg, const std::string developer_dir,
     }
   }
 
+  size_t extension_position = arg.find_last_of(".");
+
+  // global_indexstore_path should be non-empty when global indexstore feature is enabled. 
+  // If enabled, replace --index-store-path flag value with global indexstore path and save the 
+  // original indexstore path for copying back later.
+  if (!global_indexstore_path.empty()) { 
+    if (extension_position != std::string::npos && arg.substr(extension_position) == ".indexstore") {
+      original_indexstore_path = cwd + "/" + arg;
+      new_arg = global_indexstore_path;
+    }
+    if (extension_position != std::string::npos && arg.substr(extension_position) == ".o") {
+      output_file_path = cwd + "/" + arg;
+    }
+  }
+
   consumer(new_arg);
 }
 
 }  // namespace
 
+inline bool path_exists(const std::string& path) {
+  struct stat buffer;   
+  return (stat (path.c_str(), &buffer) == 0); 
+}
+
 int main(int argc, char *argv[]) {
   std::string tool_name;
 
@@ -403,8 +429,23 @@ int main(int argc, char *argv[]) {
   std::string developer_dir = GetMandatoryEnvVar("DEVELOPER_DIR");
   std::string sdk_root = GetMandatoryEnvVar("SDKROOT");
   std::string linked_binary, dsym_path;
+  // variables for indexstore binaries and outputs
+  std::string original_indexstore_path, global_indexstore_path, output_file_path, index_import_path; 
 
   const std::string cwd = GetCurrentDirectory();
+
+  // check if global indexstore feature is used. If true, use global indexstore for all compile actions 
+  // passing --index-store-path flag and copy indexstore data corresponding to the action output file to 
+  // action specific value of --index-store-path flag. This change is modeled after a rules_swift change 
+  // (https://github.com/bazelbuild/rules_swift/pull/567)
+  bool use_global_indexstore = getenv("GLOBAL_INDEXSTORE") != nullptr;
+  global_indexstore_path = "";
+  index_import_path = "";
+  if (use_global_indexstore) {
+    global_indexstore_path = cwd + "/" + "bazel-out" + "/" + "_global_index_store";
+    index_import_path = cwd + "/" + Dirname(argv[0]) + "/" + "index-import";
+  }
+
   std::vector<std::string> invocation_args = {"/usr/bin/xcrun", tool_name};
   std::vector<std::string> processed_args = {};
 
@@ -416,7 +457,8 @@ int main(int argc, char *argv[]) {
     std::string arg(argv[i]);
 
     ProcessArgument(arg, developer_dir, sdk_root, cwd, relative_ast_path,
-                    linked_binary, dsym_path, toolchain_path, consumer);
+                    linked_binary, dsym_path, original_indexstore_path,
+                    global_indexstore_path, output_file_path, toolchain_path, consumer);
   }
 
   // Special mode that only prints the command. Used for testing.
@@ -430,6 +472,7 @@ int main(int argc, char *argv[]) {
   auto response_file = WriteResponseFile(processed_args);
   invocation_args.push_back("@" + response_file->GetPath());
 
+  bool copy_indexstore = use_global_indexstore;
   // Check to see if we should postprocess with dsymutil.
   bool postprocess = false;
   if ((!linked_binary.empty()) || (!dsym_path.empty())) {
@@ -453,11 +496,55 @@ int main(int argc, char *argv[]) {
     return 1;
   }
 
-  if (!postprocess) {
+  if (!postprocess && !copy_indexstore) {
     return 0;
   }
 
-  std::vector<std::string> dsymutil_args = {"/usr/bin/xcrun",
+  if (copy_indexstore) { 
+    auto file_prefix_map = "--file-prefix-map=" + cwd + "=.";
+    std::vector<std::string> index_import_args = {index_import_path, 
+                                            file_prefix_map,
+                                            "-import-output-file",
+                                            output_file_path,
+                                            global_indexstore_path,
+                                            original_indexstore_path};
+    if (!RunSubProcess(index_import_args)) {
+      return 1;
+    }
+
+    std::string original_indexstore_parent_dir = std::filesystem::path(original_indexstore_path).parent_path();
+    std::string original_indexstore_name = std::filesystem::path(original_indexstore_path).filename();
+    std::vector<std::string> zip_indexstore_args = { "/usr/bin/tar",
+    "-C",
+    original_indexstore_parent_dir,
+    "-cf",
+    original_indexstore_path + ".tar", 
+    original_indexstore_name};
+
+    if (!RunSubProcess(zip_indexstore_args)) {
+      return 1;
+    }
+
+    std::vector<std::string> rm_args = { "/bin/rm",
+    "-r",
+    original_indexstore_path};
+
+    if (!RunSubProcess(rm_args)) {
+      return 1;
+    }
+
+    std::vector<std::string> final_args = { "/bin/mv",
+    original_indexstore_path + ".tar", 
+    original_indexstore_path};
+
+    if (!RunSubProcess(final_args)) {
+      return 1;
+    }
+
+  }
+
+  if (postprocess) {
+    std::vector<std::string> dsymutil_args = {"/usr/bin/xcrun",
                                             "dsymutil",
                                             linked_binary,
                                             "-o",

lib/repositories.bzl

@@ -56,4 +56,20 @@ def apple_support_dependencies():
         sha256 = "3a561c99e7bdbe9173aa653fd579fe849f1d8d67395780ab4770b1f381431d51",
     )
 
+    _maybe(
+        http_archive,
+        name = "build_bazel_apple_support_index_import",
+        build_file_content = """
+load("@bazel_skylib//rules:native_binary.bzl", "native_binary")
+package(default_visibility = ["//visibility:public"])
+native_binary(
+    name = "index_import",
+    src = "index-import",
+    out = "index-import",
+)
+""",
+        urls = ["https://github.com/MobileNativeFoundation/index-import/releases/download/5.8.0.1/index-import.tar.gz"],
+        sha256 = "28c1ffa39d99e74ed70623899b207b41f79214c498c603915aef55972a851a15",
+    )
+
     apple_cc_configure()