clang: Add support to build eBPF for user specified ARCH

BCC at the moment builds eBPF only considering the local architecture
instead of the one that the user's target kernel is running on.
For cross-compiler environments, the ARCH environment variable is
used to specify which ARCH to build the kernel for. In this patch we
add support to read ARCH and if that's not set, then fallback to
detecting based on local architecture.

This patch borrows some code from a patch doing a similar thing for
eBPF samples in the kenrel that I submitted recently [1]

[1] https://patchwork.kernel.org/patch/9961801/

Signed-off-by: Joel Fernandes <[email protected]>

src/cc/export/helpers.h

@@ -572,7 +572,37 @@ struct pt_regs;
 int bpf_usdt_readarg(int argc, struct pt_regs *ctx, void *arg) asm("llvm.bpf.extra");
 int bpf_usdt_readarg_p(int argc, struct pt_regs *ctx, void *buf, u64 len) asm("llvm.bpf.extra");
 
-#ifdef __powerpc__
+/* Scan the ARCH passed in from ARCH env variable (see kbuild_helper.cc) */
+#if defined(__TARGET_ARCH_x86)
+#define bpf_target_x86
+#define bpf_target_defined
+#elif defined(__TARGET_ARCH_s930x)
+#define bpf_target_s930x
+#define bpf_target_defined
+#elif defined(__TARGET_ARCH_arm64)
+#define bpf_target_arm64
+#define bpf_target_defined
+#elif defined(__TARGET_ARCH_powerpc)
+#define bpf_target_powerpc
+#define bpf_target_defined
+#else
+#undef bpf_target_defined
+#endif
+
+/* Fall back to what the compiler says */
+#ifndef bpf_target_defined
+#if defined(__x86_64__)
+#define bpf_target_x86
+#elif defined(__s390x__)
+#define bpf_target_s930x
+#elif defined(__aarch64__)
+#define bpf_target_arm64
+#elif defined(__powerpc__)
+#define bpf_target_powerpc
+#endif
+#endif
+
+#if defined(bpf_target_powerpc)
 #define PT_REGS_PARM1(ctx)	((ctx)->gpr[3])
 #define PT_REGS_PARM2(ctx)	((ctx)->gpr[4])
 #define PT_REGS_PARM3(ctx)	((ctx)->gpr[5])
@@ -582,7 +612,7 @@ int bpf_usdt_readarg_p(int argc, struct pt_regs *ctx, void *buf, u64 len) asm("l
 #define PT_REGS_RC(ctx)		((ctx)->gpr[3])
 #define PT_REGS_IP(ctx)		((ctx)->nip)
 #define PT_REGS_SP(ctx)		((ctx)->gpr[1])
-#elif defined(__s390x__)
+#elif defined(bpf_target_s930x)
 #define PT_REGS_PARM1(x) ((x)->gprs[2])
 #define PT_REGS_PARM2(x) ((x)->gprs[3])
 #define PT_REGS_PARM3(x) ((x)->gprs[4])
@@ -593,7 +623,7 @@ int bpf_usdt_readarg_p(int argc, struct pt_regs *ctx, void *buf, u64 len) asm("l
 #define PT_REGS_RC(x) ((x)->gprs[2])
 #define PT_REGS_SP(x) ((x)->gprs[15])
 #define PT_REGS_IP(x) ((x)->psw.addr)
-#elif defined(__x86_64__)
+#elif defined(bpf_target_x86)
 #define PT_REGS_PARM1(ctx)	((ctx)->di)
 #define PT_REGS_PARM2(ctx)	((ctx)->si)
 #define PT_REGS_PARM3(ctx)	((ctx)->dx)
@@ -604,7 +634,7 @@ int bpf_usdt_readarg_p(int argc, struct pt_regs *ctx, void *buf, u64 len) asm("l
 #define PT_REGS_RC(ctx)		((ctx)->ax)
 #define PT_REGS_IP(ctx)		((ctx)->ip)
 #define PT_REGS_SP(ctx)		((ctx)->sp)
-#elif defined(__aarch64__)
+#elif defined(bpf_target_arm64)
 #define PT_REGS_PARM1(x)	((x)->regs[0])
 #define PT_REGS_PARM2(x)	((x)->regs[1])
 #define PT_REGS_PARM3(x)	((x)->regs[2])

src/cc/frontends/clang/arch_helper.h

@@ -0,0 +1,65 @@
+/*
+ * Copyright (c) 2018 Google, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <string.h>
+#include <stdlib.h>
+
+typedef enum {
+  BCC_ARCH_PPC,
+  BCC_ARCH_PPC_LE,
+  BCC_ARCH_S390X,
+  BCC_ARCH_ARM64,
+  BCC_ARCH_X86
+} bcc_arch_t;
+
+typedef void *(*arch_callback_t)(bcc_arch_t arch);
+
+static void *run_arch_callback(arch_callback_t fn)
+{
+  const char *archenv = getenv("ARCH");
+
+  /* If ARCH is not set, detect from local arch clang is running on */
+  if (!archenv) {
+#if defined(__powerpc64__)
+#if defined(_CALL_ELF) && _CALL_ELF == 2
+    return fn(BCC_ARCH_PPC_LE);
+#else
+    return fn(BCC_ARCH_PPC);
+#endif
+#elif defined(__s390x__)
+    return fn(BCC_ARCH_S390X);
+#elif defined(__aarch64__)
+    return fn(BCC_ARCH_ARM64);
+#else
+    return fn(BCC_ARCH_X86);
+#endif
+  }
+
+  /* Otherwise read it from ARCH */
+  if (!strcmp(archenv, "powerpc")) {
+#if defined(_CALL_ELF) && _CALL_ELF == 2
+    return fn(BCC_ARCH_PPC_LE);
+#else
+    return fn(BCC_ARCH_PPC);
+#endif
+  } else if (!strcmp(archenv, "s390x")) {
+    return fn(BCC_ARCH_S390X);
+  } else if (!strcmp(archenv, "arm64")) {
+    return fn(BCC_ARCH_ARM64);
+  } else {
+    return fn(BCC_ARCH_X86);
+  }
+}

src/cc/frontends/clang/b_frontend_action.cc

@@ -17,6 +17,7 @@
 #include <linux/version.h>
 #include <sys/utsname.h>
 #include <unistd.h>
+#include <stdlib.h>
 
 #include <clang/AST/ASTConsumer.h>
 #include <clang/AST/ASTContext.h>
@@ -30,6 +31,7 @@
 #include "common.h"
 #include "loader.h"
 #include "table_storage.h"
+#include "arch_helper.h"
 
 #include "libbpf.h"
 
@@ -47,16 +49,35 @@ const char *calling_conv_regs_s390x[] = {"gprs[2]", "gprs[3]", "gprs[4]",
 
 const char *calling_conv_regs_arm64[] = {"regs[0]", "regs[1]", "regs[2]",
                                        "regs[3]", "regs[4]", "regs[5]"};
-// todo: support more archs
-#if defined(__powerpc__)
-const char **calling_conv_regs = calling_conv_regs_ppc;
-#elif defined(__s390x__)
-const char **calling_conv_regs = calling_conv_regs_s390x;
-#elif defined(__aarch64__)
-const char **calling_conv_regs = calling_conv_regs_arm64;
-#else
-const char **calling_conv_regs = calling_conv_regs_x86;
-#endif
+
+void *get_call_conv_cb(bcc_arch_t arch)
+{
+  const char **ret;
+
+  switch(arch) {
+    case BCC_ARCH_PPC:
+    case BCC_ARCH_PPC_LE:
+      ret = calling_conv_regs_ppc;
+      break;
+    case BCC_ARCH_S390X:
+      ret = calling_conv_regs_s390x;
+      break;
+    case BCC_ARCH_ARM64:
+      ret = calling_conv_regs_arm64;
+      break;
+    default:
+      ret = calling_conv_regs_x86;
+  }
+
+  return (void *)ret;
+}
+
+const char **get_call_conv(void) {
+  const char **ret;
+
+  ret = (const char **)run_arch_callback(get_call_conv_cb);
+  return ret;
+}
 
 using std::map;
 using std::move;
@@ -256,6 +277,8 @@ BTypeVisitor::BTypeVisitor(ASTContext &C, BFrontendAction &fe)
     : C(C), diag_(C.getDiagnostics()), fe_(fe), rewriter_(fe.rewriter()), out_(llvm::errs()) {}
 
 bool BTypeVisitor::VisitFunctionDecl(FunctionDecl *D) {
+  const char **calling_conv_regs = get_call_conv();
+
   // put each non-static non-inline function decl in its own section, to be
   // extracted by the MemoryManager
   auto real_start_loc = rewriter_.getSourceMgr().getFileLoc(D->getLocStart());

src/cc/frontends/clang/kbuild_helper.cc

@@ -14,7 +14,9 @@
  * limitations under the License.
  */
 #include <fcntl.h>
+#include <stdlib.h>
 #include <ftw.h>
+#include <iostream>
 #include "kbuild_helper.h"
 
 namespace ebpf {
@@ -34,6 +36,8 @@ int KBuildHelper::get_flags(const char *uname_machine, vector<string> *cflags) {
   //               -e s/aarch64.*/arm64/
 
   string arch = uname_machine;
+  const char *archenv;
+
   if (!strncmp(uname_machine, "x86_64", 6)) {
     arch = "x86";
   } else if (uname_machine[0] == 'i' && !strncmp(&uname_machine[2], "86", 2)) {
@@ -56,6 +60,11 @@ int KBuildHelper::get_flags(const char *uname_machine, vector<string> *cflags) {
     arch = "arm64";
   }
 
+  // If ARCH env is defined, use it over uname
+  archenv = getenv("ARCH");
+  if (archenv)
+    arch = string(archenv);
+
   cflags->push_back("-nostdinc");
   cflags->push_back("-isystem");
   cflags->push_back("/virtual/lib/clang/include");
@@ -87,6 +96,11 @@ int KBuildHelper::get_flags(const char *uname_machine, vector<string> *cflags) {
   cflags->push_back("-D__HAVE_BUILTIN_BSWAP16__");
   cflags->push_back("-D__HAVE_BUILTIN_BSWAP32__");
   cflags->push_back("-D__HAVE_BUILTIN_BSWAP64__");
+
+  // If ARCH env variable is set, pass this along.
+  if (archenv)
+	cflags->push_back("-D__TARGET_ARCH_" + arch);
+
   cflags->push_back("-Wno-unused-value");
   cflags->push_back("-Wno-pointer-sign");
   cflags->push_back("-fno-stack-protector");

src/cc/frontends/clang/loader.cc

@@ -57,6 +57,7 @@
 #include "b_frontend_action.h"
 #include "tp_frontend_action.h"
 #include "loader.h"
+#include "arch_helper.h"
 
 using std::map;
 using std::string;
@@ -201,6 +202,37 @@ int ClangLoader::parse(unique_ptr<llvm::Module> *mod, TableStorage &ts,
   return 0;
 }
 
+void *get_clang_target_cb(bcc_arch_t arch)
+{
+  const char *ret;
+
+  switch(arch) {
+    case BCC_ARCH_PPC_LE:
+      ret = "powerpc64le-unknown-linux-gnu";
+      break;
+    case BCC_ARCH_PPC:
+      ret = "powerpc64-unknown-linux-gnu";
+      break;
+    case BCC_ARCH_S390X:
+      ret = "s390x-ibm-linux-gnu";
+      break;
+    case BCC_ARCH_ARM64:
+      ret = "aarch64-unknown-linux-gnu";
+      break;
+    default:
+      ret = "x86_64-unknown-linux-gnu";
+  }
+
+  return (void *)ret;
+}
+
+string get_clang_target(void) {
+  const char *ret;
+
+  ret = (const char *)run_arch_callback(get_clang_target_cb);
+  return string(ret);
+}
+
 int ClangLoader::do_compile(unique_ptr<llvm::Module> *mod, TableStorage &ts,
                             bool in_memory,
                             const vector<const char *> &flags_cstr_in,
@@ -227,19 +259,10 @@ int ClangLoader::do_compile(unique_ptr<llvm::Module> *mod, TableStorage &ts,
   DiagnosticsEngine diags(DiagID, &*diag_opts, diag_client);
 
   // set up the command line argument wrapper
-#if defined(__powerpc64__)
-#if defined(_CALL_ELF) && _CALL_ELF == 2
-  driver::Driver drv("", "powerpc64le-unknown-linux-gnu", diags);
-#else
-  driver::Driver drv("", "powerpc64-unknown-linux-gnu", diags);
-#endif
-#elif defined(__s390x__)
-  driver::Driver drv("", "s390x-ibm-linux-gnu", diags);
-#elif defined(__aarch64__)
-  driver::Driver drv("", "aarch64-unknown-linux-gnu", diags);
-#else
-  driver::Driver drv("", "x86_64-unknown-linux-gnu", diags);
-#endif
+
+  string target_triple = get_clang_target();
+  driver::Driver drv("", target_triple, diags);
+
   drv.setTitle("bcc-clang-driver");
   drv.setCheckInputsExist(false);