Merge pull request #107 from Lyt99/feature/btfhack

feat(btfhack): add btfhack & replace initContainer image

Dockerfile

@@ -28,5 +28,7 @@ RUN apk add --no-cache \
 COPY --from=build /go/src/github.com/alibaba/kubeskoop/bin/inspector /bin/inspector
 COPY --from=build /go/src/github.com/alibaba/kubeskoop/bin/pod-collector /bin/pod-collector
 COPY --from=build /go/src/github.com/alibaba/kubeskoop/bin/skoop /bin/skoop
+COPY --from=build /go/src/github.com/alibaba/kubeskoop/bin/btfhack /bin/btfhack
+
 COPY tools/scripts/* /bin/
 COPY deploy/resource/kubeskoop-exporter-dashboard.json /etc/

Makefile

@@ -14,7 +14,7 @@ GIT_COMMIT=${shell git rev-parse HEAD}
 ldflags="-X $(VERSION_PKG).Version=$(TAG) -X $(VERSION_PKG).Commit=${GIT_COMMIT}"
 
 .PHONY: all
-all: build-exporter build-skoop build-collector
+all: build-exporter build-skoop build-collector build-btfhack
 
 .PHONY: fmt
 fmt:
@@ -36,6 +36,10 @@ build-skoop:
 build-collector:
 	CGO_ENABLED=0 go build -o bin/pod-collector -ldflags $(ldflags) ./cmd/collector
 
+.PHONY: build-btfhack
+build-btfhack:
+	CGO_ENABLED=0 go build -o bin/btfhack -ldflags $(ldflags) ./cmd/btfhack
+
 .PHONY: image
 image: ## build kubeskoop image
 	docker build -t $(SKOOP_REPO):$(TAG) .

cmd/btfhack/main.go

@@ -0,0 +1,9 @@
+package main
+
+import (
+	"github.com/alibaba/kubeskoop/pkg/exporter/btfhack"
+)
+
+func main() {
+	btfhack.Execute()
+}

deploy/helm/values.yaml

@@ -11,7 +11,7 @@ image:
 
 initContainer:
   enabled: true
-  repository: registry.cn-hangzhou.aliyuncs.com/acs/btfhack
+  repository: kubeskoop/kubeskoop
   tag: latest
   imagePullPolicy: Always
 

deploy/skoopbundle.yaml

@@ -31,13 +31,13 @@ spec:
       dnsPolicy: ClusterFirstWithHostNet
       initContainers:
         - name: inspector-prepare
-          image: registry.cn-hangzhou.aliyuncs.com/acs/btfhack:latest
+          image: "kubeskoop/kubeskoop:latest"
           volumeMounts:
             - name: btf-rawdata
               mountPath: /etc/net-exporter/btf
             - mountPath: /boot/
               name: boot
-          command: [btfhack, discover,-p ,/etc/net-exporter/btf/]
+          command: [btfhack, discover, -p, /etc/net-exporter/btf/]
       containers:
       - image: "kubeskoop/kubeskoop:latest"
         name: inspector

pkg/exporter/bpfutil/btf.go

@@ -50,6 +50,15 @@ func KernelRelease() (string, error) {
 	return unix.ByteSliceToString(uname.Release[:]), nil
 }
 
+func KernelArch() (string, error) {
+	var uname unix.Utsname
+	if err := unix.Uname(&uname); err != nil {
+		return "", fmt.Errorf("uname failed: %w", err)
+	}
+
+	return unix.ByteSliceToString(uname.Machine[:]), nil
+}
+
 // LoadBTFSpecOrNil once error occurs in load process, return nil and use system raw spec instead
 func LoadBTFSpecOrNil() *btf.Spec {
 	var (
@@ -60,7 +69,7 @@ func LoadBTFSpecOrNil() *btf.Spec {
 		btffile = kernelBTFPath
 	} else if os.IsNotExist(err) {
 		for _, btfPath := range []string{BTFPATH, bpfSharePath, userCustomBtfPath} {
-			btffile, err = findBTFFileWithPath(btfPath)
+			btffile, err = FindBTFFileWithPath(btfPath)
 			if err == nil {
 				break
 			}
@@ -85,7 +94,7 @@ func LoadBTFSpecOrNil() *btf.Spec {
 	return spec
 }
 
-func findBTFFileWithPath(path string) (string, error) {
+func FindBTFFileWithPath(path string) (string, error) {
 	path = filepath.Clean(path)
 
 	v, err := KernelRelease()

pkg/exporter/btfhack/discover.go

@@ -0,0 +1,70 @@
+package btfhack
+
+import (
+	"fmt"
+	"log"
+	"os/exec"
+
+	"github.com/alibaba/kubeskoop/pkg/exporter/bpfutil"
+
+	"github.com/spf13/cobra"
+)
+
+const (
+	defaultBTFDstPath = "/etc/net-exporter/btf"
+)
+
+// cpCmd represents the cp command
+var (
+	cpCmd = &cobra.Command{
+		Use:   "discover",
+		Short: "copy or download appropriate btf file to dst path",
+		Run: func(cmd *cobra.Command, args []string) {
+			if btfSrcPath == "" {
+				btfSrcPath = defaultBTFPath
+			}
+			if btfDstPath == "" {
+				btfDstPath = defaultBTFDstPath
+			}
+
+			btffile, err := bpfutil.FindBTFFileWithPath(btfSrcPath)
+			if err == nil {
+				err := copyBtfFile(btffile, btfDstPath)
+				if err != nil {
+					log.Fatalf("Failed copy btf file: %s\n", err)
+				}
+				log.Printf("Copy btf file %s to %s succeed\n", btffile, btfDstPath)
+				return
+			}
+
+			btffile, err = downloadBTFOnline(btfDstPath)
+			if err != nil {
+				log.Printf("Download btf error: %s\n", err)
+				return
+			}
+			log.Printf("Download btf file %s succeed\n", btffile)
+		},
+	}
+
+	btfDstPath string
+)
+
+func copyBtfFile(path, dstPath string) error {
+	cmdToExecute := exec.Command("cp", path, dstPath)
+	output, err := cmdToExecute.CombinedOutput()
+	if err != nil {
+		return fmt.Errorf("load btf with:%s err:%s", output, err)
+	}
+
+	log.Printf("load btf %s to %s succeed", path, dstPath)
+	return nil
+}
+
+func init() {
+	rootCmd.AddCommand(cpCmd)
+
+	flags := cpCmd.PersistentFlags()
+
+	flags.StringVarP(&btfSrcPath, "src", "s", "", "btf source file")
+	flags.StringVarP(&btfDstPath, "dst", "p", "", "btf destination directory")
+}

pkg/exporter/btfhack/download.go

@@ -0,0 +1,90 @@
+package btfhack
+
+import (
+	"fmt"
+	"io"
+	"log"
+	"net"
+	"net/http"
+	"net/url"
+	"os"
+	"path"
+	"time"
+
+	"github.com/alibaba/kubeskoop/pkg/exporter/bpfutil"
+)
+
+const (
+	EnvBTFDownloadURL = "BTF_DOWNLOAD_URL"
+	OpenBTFURL        = "https://mirrors.openanolis.cn/coolbpf/btf/"
+)
+
+func downloadBTFOnline(btfDstPath string) (string, error) {
+	release, err := bpfutil.KernelRelease()
+	if err != nil {
+		return "", err
+	}
+	arch, err := bpfutil.KernelArch()
+	if err != nil {
+		return "", err
+	}
+
+	filename := fmt.Sprintf("vmlinux-%s", release)
+	dst := path.Join(btfDstPath, filename)
+	urlPath := fmt.Sprintf("%s/%s", arch, filename)
+	if envURL, ok := os.LookupEnv(EnvBTFDownloadURL); ok {
+		downloadURL, err := url.JoinPath(envURL, urlPath)
+		if err != nil {
+			return "", err
+		}
+		err = downloadTo(downloadURL, dst)
+		if err == nil {
+			log.Printf("Downloaded btf file from %s", downloadURL)
+			return dst, nil
+		}
+		log.Printf("Download btf file failed from %s: %s", downloadURL, err)
+	}
+
+	downloadURL, err := url.JoinPath(OpenBTFURL, urlPath)
+	if err != nil {
+		return "", err
+	}
+	err = downloadTo(downloadURL, dst)
+	if err != nil {
+		return "", fmt.Errorf("download btf file failed from %s: %w", downloadURL, err)
+	}
+	return dst, nil
+}
+
+func downloadTo(url, dst string) error {
+	tr := &http.Transport{
+		Dial: (&net.Dialer{
+			Timeout:   1 * time.Second,
+			KeepAlive: 30 * time.Second,
+		}).Dial,
+	}
+
+	client := http.Client{
+		Timeout:   50 * time.Second,
+		Transport: tr,
+	}
+
+	res, err := client.Get(url)
+	if err != nil {
+		return err
+	}
+	defer res.Body.Close()
+	if res.StatusCode != http.StatusOK {
+		return fmt.Errorf("got status code %d", res.StatusCode)
+	}
+
+	f, err := os.Create(dst)
+	if err != nil {
+		return err
+	}
+	_, err = io.Copy(f, res.Body)
+	if err != nil {
+		return err
+	}
+	return nil
+}

pkg/exporter/btfhack/root.go

@@ -0,0 +1,23 @@
+package btfhack
+
+import (
+	"os"
+
+	"github.com/spf13/cobra"
+)
+
+const (
+	defaultBTFPath = "/etc/btf"
+)
+
+var rootCmd = &cobra.Command{
+	Use:   "btfhack",
+	Short: "A tool to automatically discover btf file from local path or online",
+}
+
+func Execute() {
+	err := rootCmd.Execute()
+	if err != nil {
+		os.Exit(1)
+	}
+}

pkg/exporter/btfhack/test.go

@@ -0,0 +1,55 @@
+package btfhack
+
+import (
+	"log"
+
+	"github.com/alibaba/kubeskoop/pkg/exporter/bpfutil"
+	"github.com/alibaba/kubeskoop/pkg/exporter/testbtf"
+
+	"github.com/cilium/ebpf/btf"
+	"github.com/spf13/cobra"
+)
+
+// testCmd represents the test command
+var (
+	testCmd = &cobra.Command{
+		Use:   "test",
+		Short: "test btf support locally",
+		Run: func(cmd *cobra.Command, args []string) {
+			if btfSrcPath == "" {
+				btfSrcPath = defaultBTFPath
+			}
+
+			file, err := bpfutil.FindBTFFileWithPath(btfSrcPath)
+			if err != nil {
+				log.Printf("failed with %s", err)
+				return
+			}
+
+			spec, err := bpfutil.LoadBTFFromFile(file)
+			if err != nil {
+				log.Printf("load btf spec faiild with %s", err)
+				return
+			}
+
+			if err := testBTFAvailable(spec); err != nil {
+				log.Printf("btf test failed: %v", err)
+			} else {
+				log.Printf("btf test ok")
+			}
+		},
+	}
+
+	btfSrcPath string
+)
+
+func init() {
+	rootCmd.AddCommand(testCmd)
+	flags := testCmd.PersistentFlags()
+
+	flags.StringVarP(&btfSrcPath, "src", "s", "", "btf source file")
+}
+
+func testBTFAvailable(spec *btf.Spec) error {
+	return testbtf.RunBTFTest(spec)
+}

pkg/exporter/btfhack/version.go

@@ -0,0 +1,20 @@
+package btfhack
+
+import (
+	"github.com/alibaba/kubeskoop/version"
+	"github.com/spf13/cobra"
+)
+
+var (
+	versionCmd = &cobra.Command{
+		Use:   "version",
+		Short: "show version",
+		Run: func(_ *cobra.Command, args []string) {
+			version.PrintVersion()
+		},
+	}
+)
+
+func init() {
+	rootCmd.AddCommand(versionCmd)
+}