diff --git "a/book/\350\277\275\350\270\252\346\214\207\344\273\244\351\233\206/01\345\244\232\344\270\273\346\234\272\346\227\245\345\277\227\345\256\236\346\227\266\350\277\275\350\270\252.md" "b/book/\350\277\275\350\270\252\346\214\207\344\273\244\351\233\206/01\345\244\232\344\270\273\346\234\272\346\227\245\345\277\227\345\256\236\346\227\266\350\277\275\350\270\252.md" new file mode 100644 index 00000000..ea4c6031 --- /dev/null +++ "b/book/\350\277\275\350\270\252\346\214\207\344\273\244\351\233\206/01\345\244\232\344\270\273\346\234\272\346\227\245\345\277\227\345\256\236\346\227\266\350\277\275\350\270\252.md" @@ -0,0 +1,66 @@ +## 实时输出多主机日志文件内容 + +实现跨多主机的类似`tail`功能 + +### 版本&兼容性 + +> 版本支持 + +- [以上]() + +> 兼容性 + +- [x] `CentOS6` +- [x] `CentOS7` + +### 使用方式 + +> 生成默认配置文件 + +```shell +$ easyctl track tail-log +I1001 11:13:06.384839 126576 track.go:50] 检测到配置文件参数为空,生成配置文件样例 -> config.yaml +``` + +> 修改配置文件 + +`config.yaml`,修改`server`列表,日志文件`path` + +```yaml +server: + - host: 10.10.10.[1:3] + username: root + password: "123456" + port: 22 +excludes: + - 192.168.235.132 +tail-log: + log-path: "/opt/nginx/logs/access.log" # 日志文件路径 + whence: 2 # 读取位置: 0为从文件开头读取 1为相对当前位置 2为从文件结尾开始读取 + offset: 200 # 当whence为1时生效,offset表示从文件第200行读取 +``` + +建议使用默认值`key`:`whence、offset` + +**注意:** 实现原理为通过协程`ssh`至目标机读取文件并输出,本身并不会对日志文件内容进行处理, +所以读取历史内容是无序的,如有排序需求建议使用`elk`等专业产品。 + +> 执行 + +```shell +$ easyctl track tail-log -c config.yaml +``` + +> 查看解析 + +```shell +[root@scq-dc01 ~]# cat /etc/hosts +127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4 +::1 localhost localhost.localdomain localhost6 localhost6.localdomain6 + +# easyctl hosts BEGIN +10.10.1.1 scq-dc01 +10.10.1.2 scq-dc02 +10.10.1.3 scq-dc03 +# easyctl hosts END +``` \ No newline at end of file diff --git a/cmd/exec/asset/1.sh b/cmd/exec/asset/1.sh index c4384987..c09a671d 100644 --- a/cmd/exec/asset/1.sh +++ b/cmd/exec/asset/1.sh @@ -1,3 +1,13 @@ #!/usr/bin/env bash -systemctl restart docker \ No newline at end of file +#!/bin/sh + +orphanedPods=`cat /var/log/messages|grep 'orphaned pod'|awk -F '"' '{print $2}'|uniq`; +orphanedPodsNum=`echo $orphanedPods|awk -F ' ' '{print NF}'`; +echo -e "orphanedPods: $orphanedPodsNum \n$orphanedPods"; + +for i in $orphanedPods +do +echo "Deleting Orphaned pod id: $i"; +rm -rf /var/lib/kubelet/pods/$i; +done \ No newline at end of file diff --git a/cmd/exec/asset/executor.yaml b/cmd/exec/asset/executor.yaml index 0b6a4a78..c3686a97 100644 --- a/cmd/exec/asset/executor.yaml +++ b/cmd/exec/asset/executor.yaml @@ -1,7 +1,16 @@ server: - - host: 10.10.10.[1:40] + # - host: 10.79.165.[11:13] + # username: root + # password: neusoft + # port: 22 + # - host: 10.79.164.[61:63] + # username: root + # password: neusoft + # port: 22 + - host: 10.79.166.[1:45] username: root - password: 123456 + privateKeyPath: D:\github\easyctl\asset\id_rsa + # password: "" port: 22 excludes: - 192.168.235.132 diff --git a/cmd/track/asset/executor.yaml b/cmd/track/asset/executor.yaml new file mode 100644 index 00000000..c3686a97 --- /dev/null +++ b/cmd/track/asset/executor.yaml @@ -0,0 +1,17 @@ +server: + # - host: 10.79.165.[11:13] + # username: root + # password: neusoft + # port: 22 + # - host: 10.79.164.[61:63] + # username: root + # password: neusoft + # port: 22 + - host: 10.79.166.[1:45] + username: root + privateKeyPath: D:\github\easyctl\asset\id_rsa + # password: "" + port: 22 +excludes: + - 192.168.235.132 +script: "1.sh" \ No newline at end of file diff --git a/cmd/track/asset/tail_log.yaml b/cmd/track/asset/tail_log.yaml new file mode 100644 index 00000000..e49adc4f --- /dev/null +++ b/cmd/track/asset/tail_log.yaml @@ -0,0 +1,11 @@ +server: + - host: 10.10.10.[1:3] + username: root + password: "123456" + port: 22 +excludes: + - 192.168.235.132 +tail-log: + log-path: "/opt/nginx/logs/access.log" # 日志文件路径 + whence: 2 # 读取位置: 0为从文件开头读取 1为相对当前位置 2为从文件结尾开始读取 + offset: 200 # 当whence为1时生效,offset表示从文件第200行读取 \ No newline at end of file diff --git a/cmd/track/tail_log.go b/cmd/track/tail_log.go new file mode 100644 index 00000000..645c2113 --- /dev/null +++ b/cmd/track/tail_log.go @@ -0,0 +1,21 @@ +package track + +import ( + _ "embed" + "github.com/spf13/cobra" + "github.com/weiliang-ms/easyctl/pkg/track" +) + +//go:embed asset/tail_log.yaml +var tailLogConfig []byte + +// RootCmd close命令 +var tailLogCmd = &cobra.Command{ + Use: "tail-log [flags]", + Short: "追踪日志命令", + Run: func(cmd *cobra.Command, args []string) { + if runErr := Exec(Entity{Cmd: cmd, Fnc: track.TaiLog, DefaultConfig: tailLogConfig}); runErr != nil { + panic(runErr) + } + }, +} diff --git a/cmd/track/track.go b/cmd/track/track.go new file mode 100644 index 00000000..c0ac3b91 --- /dev/null +++ b/cmd/track/track.go @@ -0,0 +1,69 @@ +package track + +import ( + _ "embed" + "fmt" + "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + "github.com/weiliang-ms/easyctl/pkg/util" + "k8s.io/klog" + "os" +) + +var ( + configFile string +) + +//go:embed asset/executor.yaml +var config []byte + +func init() { + RootCmd.PersistentFlags().StringVarP(&configFile, "config", "c", "", "配置文件") + RootCmd.AddCommand(tailLogCmd) +} + +// RootCmd track命令 +var RootCmd = &cobra.Command{ + Use: "track [flags]", + Short: "追踪命令指令集", + Args: cobra.ExactValidArgs(1), +} + +type Entity struct { + Cmd *cobra.Command + Fnc func(b []byte, logger *logrus.Logger) error + DefaultConfig []byte +} + +func Exec(entity Entity) error { + + if entity.DefaultConfig == nil { + entity.DefaultConfig = config + } + + if configFile == "" { + klog.Infof("检测到配置文件参数为空,生成配置文件样例 -> %s", util.ConfigFile) + _ = os.WriteFile(util.ConfigFile, entity.DefaultConfig, 0666) + os.Exit(0) + } + + flagset := entity.Cmd.Parent().Parent().PersistentFlags() + debug, err := flagset.GetBool("debug") + if err != nil { + fmt.Println(err) + } + + b, readErr := os.ReadFile(configFile) + if readErr != nil { + klog.Fatalf("读取配置文件失败") + } + + logger := logrus.New() + if debug { + logger.SetLevel(logrus.DebugLevel) + } else { + logger.SetLevel(logrus.InfoLevel) + } + + return entity.Fnc(b, logger) +} diff --git a/gotest/runner/run_test.go b/gotest/runner/run_test.go index 9c4c6d5a..eec3aa60 100644 --- a/gotest/runner/run_test.go +++ b/gotest/runner/run_test.go @@ -16,7 +16,7 @@ func TestRun(t *testing.T) { } executor, err := runner.ParseExecutor(b) - executor.Script = "date" + executor.Script = "date;ps -ef|grep java" if err != nil { panic(err) } diff --git a/gotest/track/tail_log_test.go b/gotest/track/tail_log_test.go new file mode 100644 index 00000000..68b44af2 --- /dev/null +++ b/gotest/track/tail_log_test.go @@ -0,0 +1,19 @@ +package track + +import ( + "github.com/sirupsen/logrus" + "github.com/weiliang-ms/easyctl/pkg/track" + "os" + "testing" +) + +func TestTailLog(t *testing.T) { + b, readErr := os.ReadFile("../../asset/config.yaml") + if readErr != nil { + panic(readErr) + } + err := track.TaiLog(b, logrus.New()) + if err != nil { + panic(err) + } +} diff --git a/main.go b/main.go index d54d8fc9..3e39d100 100644 --- a/main.go +++ b/main.go @@ -12,6 +12,7 @@ import ( "github.com/weiliang-ms/easyctl/cmd/install" "github.com/weiliang-ms/easyctl/cmd/secure" "github.com/weiliang-ms/easyctl/cmd/set" + "github.com/weiliang-ms/easyctl/cmd/track" "github.com/weiliang-ms/easyctl/cmd/upgrade" "github.com/weiliang-ms/easyctl/pkg/logs" "math/rand" @@ -52,6 +53,7 @@ func init() { //cmd.InitTmplCmd, export.RootCmd, secure.RootCmd, + track.RootCmd, versionCmd, completionCmd, } diff --git a/pkg/runner/parse.go b/pkg/runner/parse.go index 05dcbf56..7c12d0e6 100644 --- a/pkg/runner/parse.go +++ b/pkg/runner/parse.go @@ -18,7 +18,7 @@ func (serverListInternal ServerListInternal) ServerListFilter() []ServerInternal for _, v := range serverListInternal.Servers { re := v.ServerFilter(serverListInternal.Excludes) if len(re) == 1 { - servers = append(servers, re[1]) + servers = append(servers, re[0]) } else { for _, s := range re { servers = append(servers, s) @@ -130,10 +130,10 @@ func (server ServerInternal) ServerFilter(excludes []string) []ServerInternal { for i := beginIndex; i <= endIndex; i++ { server := ServerInternal{ - Host: fmt.Sprintf("%s%d", baseAddress, i), - Port: server.Port, - Username: server.Username, - Password: server.Password, + Host: fmt.Sprintf("%s%d", baseAddress, i), + Port: server.Port, + Username: server.Username, + Password: server.Password, PrivateKeyPath: server.PrivateKeyPath, } diff --git a/pkg/runner/tail.go b/pkg/runner/tail.go new file mode 100644 index 00000000..427afd81 --- /dev/null +++ b/pkg/runner/tail.go @@ -0,0 +1,100 @@ +package runner + +import ( + "bufio" + "fmt" + "github.com/pkg/sftp" + "io" + "log" + "os" + "strings" +) + +// TailFile all & real time +func (server ServerInternal) TailFile(path string, offset int64, whence int, stopCh <-chan struct{}) { + // init sftp + sftp, err := sftpConnect(server.Username, server.Password, server.Host, server.Port) + if err != nil { + log.Println(err) + } + + f, err := sftp.OpenFile(path, os.O_RDONLY) + if err != nil { + panic(err) + } + + f.Seek(offset, whence) + + for { + select { + case <-stopCh: + _ = f.Close() + return + default: + } + + buf := bufio.NewReader(f) + + for { + select { + case <-stopCh: + _ = f.Close() + return + default: + } + line, err := buf.ReadString('\n') + line = strings.TrimSpace(line) + if line != "" { + fmt.Printf("[%s] %s\n", server.Host, line) + } + if err != nil { + if err == io.EOF { + break + } else { + fmt.Println("Read file error!", err) + return + } + } + } + } +} + +func readAtTheBeginning(f *sftp.File, host string, stopCh <-chan struct{}) { +} + +func readAtTheLatest(f *sftp.File, host string, stopCh <-chan struct{}) { + for { + //f.Seek(0, 2) + select { + case <-stopCh: + _ = f.Close() + return + default: + } + + f.Seek(0, 2) + buf := bufio.NewReader(f) + + for { + select { + case <-stopCh: + _ = f.Close() + return + default: + } + line, err := buf.ReadString('\n') + line = strings.TrimSpace(line) + if line != "" { + fmt.Printf("[%s] %s\n", host, line) + } + if err != nil { + if err == io.EOF { + break + } else { + fmt.Println("Read file error!", err) + return + } + } + } + } +} diff --git a/pkg/track/track.go b/pkg/track/track.go new file mode 100644 index 00000000..d7e0e3d4 --- /dev/null +++ b/pkg/track/track.go @@ -0,0 +1,64 @@ +package track + +import ( + "fmt" + "github.com/mitchellh/mapstructure" + "github.com/olekukonko/tablewriter" + "github.com/sirupsen/logrus" + "github.com/weiliang-ms/easyctl/pkg/runner" + "os" + "sort" +) + +type ParseConfig interface { + Parse(b []byte, debug bool) (error, interface{}) +} + +func Config(b []byte, logger *logrus.Logger, cmd string) error { + + results, err := GetResult(b, logger, cmd) + if err != nil { + return err + } + var data [][]string + + for _, v := range results { + data = append(data, []string{v.Host, v.Cmd, fmt.Sprintf("%d", v.Code), v.Status, v.StdOut, v.StdErrMsg}) + } + + table := tablewriter.NewWriter(os.Stdout) + table.SetHeader([]string{"IP ADDRESS", "cmd", "exit code", "result", "output", "exception"}) + table.SetBorders(tablewriter.Border{Left: true, Top: false, Right: true, Bottom: false}) + table.SetCenterSeparator("|") + //table.SetRowLine(true) + table.SetAlignment(tablewriter.ALIGN_CENTER) + table.AppendBulk(data) // Add Bulk Data + table.Render() + + return nil +} + +func GetResult(b []byte, logger *logrus.Logger, cmd string) ([]runner.ShellResult, error) { + + servers, err := runner.ParseServerList(b) + if err != nil { + return []runner.ShellResult{}, err + } + + executor := runner.ExecutorInternal{Servers: servers, Script: cmd} + + ch := executor.ParallelRun(logger) + + results := []runner.ShellResult{} + + for re := range ch { + var result runner.ShellResult + _ = mapstructure.Decode(re, &result) + results = append(results, result) + } + + // todo: ip地址排序 + sort.Sort(runner.ShellResultSlice(results)) + + return results, nil +} diff --git a/pkg/track/track_log.go b/pkg/track/track_log.go new file mode 100644 index 00000000..385cffb6 --- /dev/null +++ b/pkg/track/track_log.go @@ -0,0 +1,51 @@ +package track + +import ( + "github.com/sirupsen/logrus" + "github.com/weiliang-ms/easyctl/pkg/runner" + "gopkg.in/yaml.v2" +) + +type TailLogExecutor struct { + TailLog struct { + LogPath string `yaml:"log-path"` + Whence int `yaml:"whence"` + Offset int64 `yaml:"offset"` + } `yaml:"tail-log"` +} + +func TaiLog(config []byte, logger *logrus.Logger) error { + + servers, err := runner.ParseServerList(config) + stopCh := make(chan struct{}) + if err != nil { + return err + } + + executor, err := parseTailLogExecutor(config) + if err != nil { + return err + } + + executor.Tail(servers, stopCh) + + return nil +} + +func (tail TailLogExecutor) Tail(servers []runner.ServerInternal, stopCh <-chan struct{}) { + for _, v := range servers { + go v.TailFile(tail.TailLog.LogPath, tail.TailLog.Offset, tail.TailLog.Whence, stopCh) + } + + <-stopCh +} + +func parseTailLogExecutor(b []byte) (TailLogExecutor, error) { + executor := TailLogExecutor{} + err := yaml.Unmarshal(b, &executor) + if err != nil { + return TailLogExecutor{}, err + } + + return executor, nil +}