添加日志追踪功能

book/追踪指令集/01多主机日志实时追踪.md

@@ -0,0 +1,66 @@
+## 实时输出多主机日志文件内容
+
+实现跨多主机的类似`tail`功能
+
+### 版本&兼容性
+
+> 版本支持
+
+- [以上]()
+
+> 兼容性
+
+- [x] `CentOS6` 
+- [x] `CentOS7`
+
+### 使用方式
+
+> 生成默认配置文件
+
+```shell
+$ easyctl track tail-log
+I1001 11:13:06.384839  126576 track.go:50] 检测到配置文件参数为空，生成配置文件样例 -> config.yaml
+```
+
+> 修改配置文件
+
+`config.yaml`，修改`server`列表，日志文件`path`
+
+```yaml
+server:
+  - host: 10.10.10.[1:3]
+    username: root
+    password: "123456"
+    port: 22
+excludes:
+  - 192.168.235.132
+tail-log:
+  log-path: "/opt/nginx/logs/access.log" # 日志文件路径
+  whence: 2   # 读取位置： 0为从文件开头读取 1为相对当前位置 2为从文件结尾开始读取
+  offset: 200 # 当whence为1时生效，offset表示从文件第200行读取
+```
+
+建议使用默认值`key`：`whence、offset`
+
+**注意：** 实现原理为通过协程`ssh`至目标机读取文件并输出，本身并不会对日志文件内容进行处理，
+所以读取历史内容是无序的，如有排序需求建议使用`elk`等专业产品。
+
+> 执行
+
+```shell
+$ easyctl track tail-log -c config.yaml
+```
+
+> 查看解析
+
+```shell
+[root@scq-dc01 ~]# cat /etc/hosts
+127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4
+::1         localhost localhost.localdomain localhost6 localhost6.localdomain6
+
+# easyctl hosts BEGIN
+10.10.1.1 scq-dc01
+10.10.1.2 scq-dc02
+10.10.1.3 scq-dc03
+# easyctl hosts END
+```

cmd/exec/asset/1.sh

@@ -1,3 +1,13 @@
 #!/usr/bin/env bash
 
-systemctl restart docker
+#!/bin/sh
+
+orphanedPods=`cat /var/log/messages|grep 'orphaned pod'|awk -F '"' '{print $2}'|uniq`;
+orphanedPodsNum=`echo $orphanedPods|awk -F ' ' '{print NF}'`;
+echo -e "orphanedPods: $orphanedPodsNum \n$orphanedPods";
+
+for i in $orphanedPods
+do
+echo "Deleting Orphaned pod id: $i";
+rm -rf /var/lib/kubelet/pods/$i;
+done

cmd/exec/asset/executor.yaml

@@ -1,7 +1,16 @@
 server:
-  - host: 10.10.10.[1:40]
+  #  - host: 10.79.165.[11:13]
+  #    username: root
+  #    password: neusoft
+  #    port: 22
+  #  - host: 10.79.164.[61:63]
+  #    username: root
+  #    password: neusoft
+  #    port: 22
+  - host: 10.79.166.[1:45]
     username: root
-    password: 123456
+    privateKeyPath: D:\github\easyctl\asset\id_rsa
+    #    password: ""
     port: 22
 excludes:
   - 192.168.235.132

cmd/track/asset/executor.yaml

@@ -0,0 +1,17 @@
+server:
+  #  - host: 10.79.165.[11:13]
+  #    username: root
+  #    password: neusoft
+  #    port: 22
+  #  - host: 10.79.164.[61:63]
+  #    username: root
+  #    password: neusoft
+  #    port: 22
+  - host: 10.79.166.[1:45]
+    username: root
+    privateKeyPath: D:\github\easyctl\asset\id_rsa
+    #    password: ""
+    port: 22
+excludes:
+  - 192.168.235.132
+script: "1.sh"

cmd/track/asset/tail_log.yaml

@@ -0,0 +1,11 @@
+server:
+  - host: 10.10.10.[1:3]
+    username: root
+    password: "123456"
+    port: 22
+excludes:
+  - 192.168.235.132
+tail-log:
+  log-path: "/opt/nginx/logs/access.log" # 日志文件路径
+  whence: 2   # 读取位置： 0为从文件开头读取 1为相对当前位置 2为从文件结尾开始读取
+  offset: 200 # 当whence为1时生效，offset表示从文件第200行读取

cmd/track/tail_log.go

@@ -0,0 +1,21 @@
+package track
+
+import (
+	_ "embed"
+	"github.com/spf13/cobra"
+	"github.com/weiliang-ms/easyctl/pkg/track"
+)
+
+//go:embed asset/tail_log.yaml
+var tailLogConfig []byte
+
+// RootCmd close命令
+var tailLogCmd = &cobra.Command{
+	Use:   "tail-log [flags]",
+	Short: "追踪日志命令",
+	Run: func(cmd *cobra.Command, args []string) {
+		if runErr := Exec(Entity{Cmd: cmd, Fnc: track.TaiLog, DefaultConfig: tailLogConfig}); runErr != nil {
+			panic(runErr)
+		}
+	},
+}

cmd/track/track.go

@@ -0,0 +1,69 @@
+package track
+
+import (
+	_ "embed"
+	"fmt"
+	"github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+	"github.com/weiliang-ms/easyctl/pkg/util"
+	"k8s.io/klog"
+	"os"
+)
+
+var (
+	configFile string
+)
+
+//go:embed asset/executor.yaml
+var config []byte
+
+func init() {
+	RootCmd.PersistentFlags().StringVarP(&configFile, "config", "c", "", "配置文件")
+	RootCmd.AddCommand(tailLogCmd)
+}
+
+// RootCmd track命令
+var RootCmd = &cobra.Command{
+	Use:   "track [flags]",
+	Short: "追踪命令指令集",
+	Args:  cobra.ExactValidArgs(1),
+}
+
+type Entity struct {
+	Cmd           *cobra.Command
+	Fnc           func(b []byte, logger *logrus.Logger) error
+	DefaultConfig []byte
+}
+
+func Exec(entity Entity) error {
+
+	if entity.DefaultConfig == nil {
+		entity.DefaultConfig = config
+	}
+
+	if configFile == "" {
+		klog.Infof("检测到配置文件参数为空，生成配置文件样例 -> %s", util.ConfigFile)
+		_ = os.WriteFile(util.ConfigFile, entity.DefaultConfig, 0666)
+		os.Exit(0)
+	}
+
+	flagset := entity.Cmd.Parent().Parent().PersistentFlags()
+	debug, err := flagset.GetBool("debug")
+	if err != nil {
+		fmt.Println(err)
+	}
+
+	b, readErr := os.ReadFile(configFile)
+	if readErr != nil {
+		klog.Fatalf("读取配置文件失败")
+	}
+
+	logger := logrus.New()
+	if debug {
+		logger.SetLevel(logrus.DebugLevel)
+	} else {
+		logger.SetLevel(logrus.InfoLevel)
+	}
+
+	return entity.Fnc(b, logger)
+}

gotest/runner/run_test.go

@@ -16,7 +16,7 @@ func TestRun(t *testing.T) {
 	}
 
 	executor, err := runner.ParseExecutor(b)
-	executor.Script = "date"
+	executor.Script = "date;ps -ef|grep java"
 	if err != nil {
 		panic(err)
 	}

gotest/track/tail_log_test.go

@@ -0,0 +1,19 @@
+package track
+
+import (
+	"github.com/sirupsen/logrus"
+	"github.com/weiliang-ms/easyctl/pkg/track"
+	"os"
+	"testing"
+)
+
+func TestTailLog(t *testing.T) {
+	b, readErr := os.ReadFile("../../asset/config.yaml")
+	if readErr != nil {
+		panic(readErr)
+	}
+	err := track.TaiLog(b, logrus.New())
+	if err != nil {
+		panic(err)
+	}
+}

main.go

@@ -12,6 +12,7 @@ import (
 	"github.com/weiliang-ms/easyctl/cmd/install"
 	"github.com/weiliang-ms/easyctl/cmd/secure"
 	"github.com/weiliang-ms/easyctl/cmd/set"
+	"github.com/weiliang-ms/easyctl/cmd/track"
 	"github.com/weiliang-ms/easyctl/cmd/upgrade"
 	"github.com/weiliang-ms/easyctl/pkg/logs"
 	"math/rand"
@@ -52,6 +53,7 @@ func init() {
 		//cmd.InitTmplCmd,
 		export.RootCmd,
 		secure.RootCmd,
+		track.RootCmd,
 		versionCmd,
 		completionCmd,
 	}

pkg/runner/parse.go

@@ -18,7 +18,7 @@ func (serverListInternal ServerListInternal) ServerListFilter() []ServerInternal
 	for _, v := range serverListInternal.Servers {
 		re := v.ServerFilter(serverListInternal.Excludes)
 		if len(re) == 1 {
-			servers = append(servers, re[1])
+			servers = append(servers, re[0])
 		} else {
 			for _, s := range re {
 				servers = append(servers, s)
@@ -130,10 +130,10 @@ func (server ServerInternal) ServerFilter(excludes []string) []ServerInternal {
 
 		for i := beginIndex; i <= endIndex; i++ {
 			server := ServerInternal{
-				Host:          fmt.Sprintf("%s%d", baseAddress, i),
-				Port:          server.Port,
-				Username:      server.Username,
-				Password:      server.Password,
+				Host:           fmt.Sprintf("%s%d", baseAddress, i),
+				Port:           server.Port,
+				Username:       server.Username,
+				Password:       server.Password,
 				PrivateKeyPath: server.PrivateKeyPath,
 			}
 

pkg/runner/tail.go

@@ -0,0 +1,100 @@
+package runner
+
+import (
+	"bufio"
+	"fmt"
+	"github.com/pkg/sftp"
+	"io"
+	"log"
+	"os"
+	"strings"
+)
+
+// TailFile all & real time
+func (server ServerInternal) TailFile(path string, offset int64, whence int, stopCh <-chan struct{}) {
+	// init sftp
+	sftp, err := sftpConnect(server.Username, server.Password, server.Host, server.Port)
+	if err != nil {
+		log.Println(err)
+	}
+
+	f, err := sftp.OpenFile(path, os.O_RDONLY)
+	if err != nil {
+		panic(err)
+	}
+
+	f.Seek(offset, whence)
+
+	for {
+		select {
+		case <-stopCh:
+			_ = f.Close()
+			return
+		default:
+		}
+
+		buf := bufio.NewReader(f)
+
+		for {
+			select {
+			case <-stopCh:
+				_ = f.Close()
+				return
+			default:
+			}
+			line, err := buf.ReadString('\n')
+			line = strings.TrimSpace(line)
+			if line != "" {
+				fmt.Printf("[%s] %s\n", server.Host, line)
+			}
+			if err != nil {
+				if err == io.EOF {
+					break
+				} else {
+					fmt.Println("Read file error!", err)
+					return
+				}
+			}
+		}
+	}
+}
+
+func readAtTheBeginning(f *sftp.File, host string, stopCh <-chan struct{}) {
+}
+
+func readAtTheLatest(f *sftp.File, host string, stopCh <-chan struct{}) {
+	for {
+		//f.Seek(0, 2)
+		select {
+		case <-stopCh:
+			_ = f.Close()
+			return
+		default:
+		}
+
+		f.Seek(0, 2)
+		buf := bufio.NewReader(f)
+
+		for {
+			select {
+			case <-stopCh:
+				_ = f.Close()
+				return
+			default:
+			}
+			line, err := buf.ReadString('\n')
+			line = strings.TrimSpace(line)
+			if line != "" {
+				fmt.Printf("[%s] %s\n", host, line)
+			}
+			if err != nil {
+				if err == io.EOF {
+					break
+				} else {
+					fmt.Println("Read file error!", err)
+					return
+				}
+			}
+		}
+	}
+}