Merge pull request iovisor#1569 from joelagnel/bcc-cross-compile

BCC cross compilation support

src/cc/export/helpers.h

@@ -572,7 +572,37 @@ struct pt_regs;
 int bpf_usdt_readarg(int argc, struct pt_regs *ctx, void *arg) asm("llvm.bpf.extra");
 int bpf_usdt_readarg_p(int argc, struct pt_regs *ctx, void *buf, u64 len) asm("llvm.bpf.extra");
 
-#ifdef __powerpc__
+/* Scan the ARCH passed in from ARCH env variable (see kbuild_helper.cc) */
+#if defined(__TARGET_ARCH_x86)
+#define bpf_target_x86
+#define bpf_target_defined
+#elif defined(__TARGET_ARCH_s930x)
+#define bpf_target_s930x
+#define bpf_target_defined
+#elif defined(__TARGET_ARCH_arm64)
+#define bpf_target_arm64
+#define bpf_target_defined
+#elif defined(__TARGET_ARCH_powerpc)
+#define bpf_target_powerpc
+#define bpf_target_defined
+#else
+#undef bpf_target_defined
+#endif
+
+/* Fall back to what the compiler says */
+#ifndef bpf_target_defined
+#if defined(__x86_64__)
+#define bpf_target_x86
+#elif defined(__s390x__)
+#define bpf_target_s930x
+#elif defined(__aarch64__)
+#define bpf_target_arm64
+#elif defined(__powerpc__)
+#define bpf_target_powerpc
+#endif
+#endif
+
+#if defined(bpf_target_powerpc)
 #define PT_REGS_PARM1(ctx)	((ctx)->gpr[3])
 #define PT_REGS_PARM2(ctx)	((ctx)->gpr[4])
 #define PT_REGS_PARM3(ctx)	((ctx)->gpr[5])
@@ -582,7 +612,7 @@ int bpf_usdt_readarg_p(int argc, struct pt_regs *ctx, void *buf, u64 len) asm("l
 #define PT_REGS_RC(ctx)		((ctx)->gpr[3])
 #define PT_REGS_IP(ctx)		((ctx)->nip)
 #define PT_REGS_SP(ctx)		((ctx)->gpr[1])
-#elif defined(__s390x__)
+#elif defined(bpf_target_s930x)
 #define PT_REGS_PARM1(x) ((x)->gprs[2])
 #define PT_REGS_PARM2(x) ((x)->gprs[3])
 #define PT_REGS_PARM3(x) ((x)->gprs[4])
@@ -593,7 +623,7 @@ int bpf_usdt_readarg_p(int argc, struct pt_regs *ctx, void *buf, u64 len) asm("l
 #define PT_REGS_RC(x) ((x)->gprs[2])
 #define PT_REGS_SP(x) ((x)->gprs[15])
 #define PT_REGS_IP(x) ((x)->psw.addr)
-#elif defined(__x86_64__)
+#elif defined(bpf_target_x86)
 #define PT_REGS_PARM1(ctx)	((ctx)->di)
 #define PT_REGS_PARM2(ctx)	((ctx)->si)
 #define PT_REGS_PARM3(ctx)	((ctx)->dx)
@@ -604,7 +634,7 @@ int bpf_usdt_readarg_p(int argc, struct pt_regs *ctx, void *buf, u64 len) asm("l
 #define PT_REGS_RC(ctx)		((ctx)->ax)
 #define PT_REGS_IP(ctx)		((ctx)->ip)
 #define PT_REGS_SP(ctx)		((ctx)->sp)
-#elif defined(__aarch64__)
+#elif defined(bpf_target_arm64)
 #define PT_REGS_PARM1(x)	((x)->regs[0])
 #define PT_REGS_PARM2(x)	((x)->regs[1])
 #define PT_REGS_PARM3(x)	((x)->regs[2])

src/cc/frontends/clang/arch_helper.h

@@ -0,0 +1,65 @@
+/*
+ * Copyright (c) 2018 Google, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <string.h>
+#include <stdlib.h>
+
+typedef enum {
+  BCC_ARCH_PPC,
+  BCC_ARCH_PPC_LE,
+  BCC_ARCH_S390X,
+  BCC_ARCH_ARM64,
+  BCC_ARCH_X86
+} bcc_arch_t;
+
+typedef void *(*arch_callback_t)(bcc_arch_t arch);
+
+static void *run_arch_callback(arch_callback_t fn)
+{
+  const char *archenv = getenv("ARCH");
+
+  /* If ARCH is not set, detect from local arch clang is running on */
+  if (!archenv) {
+#if defined(__powerpc64__)
+#if defined(_CALL_ELF) && _CALL_ELF == 2
+    return fn(BCC_ARCH_PPC_LE);
+#else
+    return fn(BCC_ARCH_PPC);
+#endif
+#elif defined(__s390x__)
+    return fn(BCC_ARCH_S390X);
+#elif defined(__aarch64__)
+    return fn(BCC_ARCH_ARM64);
+#else
+    return fn(BCC_ARCH_X86);
+#endif
+  }
+
+  /* Otherwise read it from ARCH */
+  if (!strcmp(archenv, "powerpc")) {
+#if defined(_CALL_ELF) && _CALL_ELF == 2
+    return fn(BCC_ARCH_PPC_LE);
+#else
+    return fn(BCC_ARCH_PPC);
+#endif
+  } else if (!strcmp(archenv, "s390x")) {
+    return fn(BCC_ARCH_S390X);
+  } else if (!strcmp(archenv, "arm64")) {
+    return fn(BCC_ARCH_ARM64);
+  } else {
+    return fn(BCC_ARCH_X86);
+  }
+}

src/cc/frontends/clang/b_frontend_action.cc

@@ -17,6 +17,7 @@
 #include <linux/version.h>
 #include <sys/utsname.h>
 #include <unistd.h>
+#include <stdlib.h>
 
 #include <clang/AST/ASTConsumer.h>
 #include <clang/AST/ASTContext.h>
@@ -30,6 +31,7 @@
 #include "common.h"
 #include "loader.h"
 #include "table_storage.h"
+#include "arch_helper.h"
 
 #include "libbpf.h"
 
@@ -47,16 +49,35 @@ const char *calling_conv_regs_s390x[] = {"gprs[2]", "gprs[3]", "gprs[4]",
 
 const char *calling_conv_regs_arm64[] = {"regs[0]", "regs[1]", "regs[2]",
                                        "regs[3]", "regs[4]", "regs[5]"};
-// todo: support more archs
-#if defined(__powerpc__)
-const char **calling_conv_regs = calling_conv_regs_ppc;
-#elif defined(__s390x__)
-const char **calling_conv_regs = calling_conv_regs_s390x;
-#elif defined(__aarch64__)
-const char **calling_conv_regs = calling_conv_regs_arm64;
-#else
-const char **calling_conv_regs = calling_conv_regs_x86;
-#endif
+
+void *get_call_conv_cb(bcc_arch_t arch)
+{
+  const char **ret;
+
+  switch(arch) {
+    case BCC_ARCH_PPC:
+    case BCC_ARCH_PPC_LE:
+      ret = calling_conv_regs_ppc;
+      break;
+    case BCC_ARCH_S390X:
+      ret = calling_conv_regs_s390x;
+      break;
+    case BCC_ARCH_ARM64:
+      ret = calling_conv_regs_arm64;
+      break;
+    default:
+      ret = calling_conv_regs_x86;
+  }
+
+  return (void *)ret;
+}
+
+const char **get_call_conv(void) {
+  const char **ret;
+
+  ret = (const char **)run_arch_callback(get_call_conv_cb);
+  return ret;
+}
 
 using std::map;
 using std::move;
@@ -256,6 +277,8 @@ BTypeVisitor::BTypeVisitor(ASTContext &C, BFrontendAction &fe)
     : C(C), diag_(C.getDiagnostics()), fe_(fe), rewriter_(fe.rewriter()), out_(llvm::errs()) {}
 
 bool BTypeVisitor::VisitFunctionDecl(FunctionDecl *D) {
+  const char **calling_conv_regs = get_call_conv();
+
   // put each non-static non-inline function decl in its own section, to be
   // extracted by the MemoryManager
   auto real_start_loc = rewriter_.getSourceMgr().getFileLoc(D->getLocStart());

src/cc/frontends/clang/kbuild_helper.cc

@@ -14,7 +14,9 @@
  * limitations under the License.
  */
 #include <fcntl.h>
+#include <stdlib.h>
 #include <ftw.h>
+#include <iostream>
 #include "kbuild_helper.h"
 
 namespace ebpf {
@@ -34,6 +36,8 @@ int KBuildHelper::get_flags(const char *uname_machine, vector<string> *cflags) {
   //               -e s/aarch64.*/arm64/
 
   string arch = uname_machine;
+  const char *archenv;
+
   if (!strncmp(uname_machine, "x86_64", 6)) {
     arch = "x86";
   } else if (uname_machine[0] == 'i' && !strncmp(&uname_machine[2], "86", 2)) {
@@ -56,6 +60,11 @@ int KBuildHelper::get_flags(const char *uname_machine, vector<string> *cflags) {
     arch = "arm64";
   }
 
+  // If ARCH env is defined, use it over uname
+  archenv = getenv("ARCH");
+  if (archenv)
+    arch = string(archenv);
+
   cflags->push_back("-nostdinc");
   cflags->push_back("-isystem");
   cflags->push_back("/virtual/lib/clang/include");
@@ -87,6 +96,11 @@ int KBuildHelper::get_flags(const char *uname_machine, vector<string> *cflags) {
   cflags->push_back("-D__HAVE_BUILTIN_BSWAP16__");
   cflags->push_back("-D__HAVE_BUILTIN_BSWAP32__");
   cflags->push_back("-D__HAVE_BUILTIN_BSWAP64__");
+
+  // If ARCH env variable is set, pass this along.
+  if (archenv)
+	cflags->push_back("-D__TARGET_ARCH_" + arch);
+
   cflags->push_back("-Wno-unused-value");
   cflags->push_back("-Wno-pointer-sign");
   cflags->push_back("-fno-stack-protector");

src/cc/frontends/clang/loader.cc

@@ -29,6 +29,7 @@
 #include <unistd.h>
 #include <utility>
 #include <vector>
+#include <iostream>
 #include <linux/bpf.h>
 
 #include <clang/Basic/FileManager.h>
@@ -56,6 +57,7 @@
 #include "b_frontend_action.h"
 #include "tp_frontend_action.h"
 #include "loader.h"
+#include "arch_helper.h"
 
 using std::map;
 using std::string;
@@ -108,11 +110,24 @@ int ClangLoader::parse(unique_ptr<llvm::Module> *mod, TableStorage &ts,
   unique_ptr<llvm::MemoryBuffer> main_buf;
   struct utsname un;
   uname(&un);
-  string kdir = string(KERNEL_MODULES_DIR) + "/" + un.release;
-  auto kernel_path_info = get_kernel_path_info (kdir);
+  string kdir, kpath;
+  const char *kpath_env = ::getenv("BCC_KERNEL_SOURCE");
+  bool has_kpath_source = false;
+
+  if (kpath_env) {
+    kpath = string(kpath_env);
+  } else {
+    kdir = string(KERNEL_MODULES_DIR) + "/" + un.release;
+    auto kernel_path_info = get_kernel_path_info(kdir);
+    has_kpath_source = kernel_path_info.first;
+    kpath = kdir + "/" + kernel_path_info.second;
+  }
+
+  if (flags_ & DEBUG_PREPROCESSOR)
+    std::cout << "Running from kernel directory at: " << kpath.c_str() << "\n";
 
   // clang needs to run inside the kernel dir
-  DirStack dstack(kdir + "/" + kernel_path_info.second);
+  DirStack dstack(kpath);
   if (!dstack.ok())
     return -1;
 
@@ -143,7 +158,8 @@ int ClangLoader::parse(unique_ptr<llvm::Module> *mod, TableStorage &ts,
                                    "-fno-asynchronous-unwind-tables",
                                    "-x", "c", "-c", abs_file.c_str()});
 
-  KBuildHelper kbuild_helper(kdir, kernel_path_info.first);
+  KBuildHelper kbuild_helper(kpath_env ? kpath : kdir, has_kpath_source);
+
   vector<string> kflags;
   if (kbuild_helper.get_flags(un.machine, &kflags))
     return -1;
@@ -186,6 +202,37 @@ int ClangLoader::parse(unique_ptr<llvm::Module> *mod, TableStorage &ts,
   return 0;
 }
 
+void *get_clang_target_cb(bcc_arch_t arch)
+{
+  const char *ret;
+
+  switch(arch) {
+    case BCC_ARCH_PPC_LE:
+      ret = "powerpc64le-unknown-linux-gnu";
+      break;
+    case BCC_ARCH_PPC:
+      ret = "powerpc64-unknown-linux-gnu";
+      break;
+    case BCC_ARCH_S390X:
+      ret = "s390x-ibm-linux-gnu";
+      break;
+    case BCC_ARCH_ARM64:
+      ret = "aarch64-unknown-linux-gnu";
+      break;
+    default:
+      ret = "x86_64-unknown-linux-gnu";
+  }
+
+  return (void *)ret;
+}
+
+string get_clang_target(void) {
+  const char *ret;
+
+  ret = (const char *)run_arch_callback(get_clang_target_cb);
+  return string(ret);
+}
+
 int ClangLoader::do_compile(unique_ptr<llvm::Module> *mod, TableStorage &ts,
                             bool in_memory,
                             const vector<const char *> &flags_cstr_in,
@@ -212,19 +259,10 @@ int ClangLoader::do_compile(unique_ptr<llvm::Module> *mod, TableStorage &ts,
   DiagnosticsEngine diags(DiagID, &*diag_opts, diag_client);
 
   // set up the command line argument wrapper
-#if defined(__powerpc64__)
-#if defined(_CALL_ELF) && _CALL_ELF == 2
-  driver::Driver drv("", "powerpc64le-unknown-linux-gnu", diags);
-#else
-  driver::Driver drv("", "powerpc64-unknown-linux-gnu", diags);
-#endif
-#elif defined(__s390x__)
-  driver::Driver drv("", "s390x-ibm-linux-gnu", diags);
-#elif defined(__aarch64__)
-  driver::Driver drv("", "aarch64-unknown-linux-gnu", diags);
-#else
-  driver::Driver drv("", "x86_64-unknown-linux-gnu", diags);
-#endif
+
+  string target_triple = get_clang_target();
+  driver::Driver drv("", target_triple, diags);
+
   drv.setTitle("bcc-clang-driver");
   drv.setCheckInputsExist(false);