初始化项目

.github/workflows/go.yml

@@ -0,0 +1,32 @@
+# This workflow will build a golang project
+# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-go
+
+name: Go
+
+on:
+  push:
+    branches: [ "main" ]
+  pull_request:
+    branches: [ "main" ]
+
+jobs:
+
+  build:
+    runs-on: ubuntu-latest
+    steps:
+
+    - name: Set up Go1.16
+      uses: actions/setup-go@v4
+      with:
+        go-version: '1.16'
+
+    - uses: actions/checkout@v2
+
+    - name: Get dependencies
+      run: go get -v -t -d ./...
+
+    - name: Build
+      run: go build -v ./...
+
+    - name: Test
+      run: go test -v ./...

.github/workflows/release.yml

@@ -0,0 +1,55 @@
+# 工作流的名称
+name: Release And Build Docker
+
+on:
+  release:
+    # 创建release时触发
+    types: [created]
+
+jobs:
+  releases-matrix:
+    name: Release Go Binary
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        goos: [ linux ]
+        goarch: [ amd64, arm64 ]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: wangyoucao577/[email protected]
+        with:
+          github_token: ${{ secrets.MY_TOKEN }}
+          goos: ${{ matrix.goos }}
+          goarch: ${{ matrix.goarch }}
+          goversion: "https://go.dev/dl/go1.16.6.linux-amd64.tar.gz"
+          project_path: "./src"
+          binary_name: "kafka2es"
+  docker:
+    # 指定的运行器环境
+    runs-on: ubuntu-latest
+
+    steps:
+      # 检出当前代码(触发工作流时的commits)
+      - name: Checkout
+        uses: actions/checkout@v4
+      # 获取docker源信息
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: registry.cn-beijing.aliyuncs.com/whoops/kafka2es
+      # 登录到阿里云容器镜像服务
+      - name: Login to Ali Docker
+        uses: docker/login-action@v3
+        # 配置登录信息，secrets 变量在 github settings -> secrets 中设置
+        with:
+          registry: ${{ secrets.ALI_DOCKER_HUB_REGISTRY }}
+          username: ${{ secrets.ALI_DOCKER_HUB_USN }}
+          password: ${{ secrets.ALI_DOCKER_HUB_PWD }}
+      # 构建镜像并上传到阿里云容器镜像仓库
+      - name: Build and push
+        id: docker_build
+        uses: docker/build-push-action@v5
+        with:
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}

Dockerfile

@@ -0,0 +1,36 @@
+# 第一层是构建一个builder镜像,目的是在其中编译出可执行文件 kafka2es
+FROM golang:alpine AS builder
+
+LABEL stage=gobuilder
+
+# 禁用cgo
+ENV CGO_ENABLED 0
+# 添加代理
+ENV GOPROXY https://goproxy.cn,direct
+RUN apk update --no-cache && apk add --no-cache tzdata
+
+
+WORKDIR /app
+
+ADD go.mod .
+ADD go.sum .
+RUN go mod tidy
+COPY .. .
+COPY ../src/etc/ /app/etc
+RUN go build -ldflags="-s -w" -o /app/kafka2es src/main.go
+
+
+# 从第一个镜像里copy出来可执行文件,并且使用尽可能小的基础镜像 alphine 以保障最终镜像尽可能小
+FROM scratch
+
+COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
+COPY --from=builder /usr/share/zoneinfo/Asia/Shanghai /usr/share/zoneinfo/Asia/Shanghai
+ENV TZ Asia/Shanghai
+
+WORKDIR /app
+COPY --from=builder /app/kafka2es /app/kafka2es
+COPY --from=builder /app/etc /app/etc
+
+EXPOSE "8080"
+
+CMD ["./kafka2es","--config","etc/config.yaml"]

README-CN.md

@@ -0,0 +1,34 @@
+# Kafka2ES
+
+Kafka2ES 是一个从Kafka读取消息，按照配置的格式化规则进行处理，然后写入的ElasticSearch集群的工具。
+![main](img/flow.png)
+### 安装
+
+```shell
+cd src && go build -o kafka2es main.go
+```
+### Quick Start
+
+- 可执行文件方式
+
+```shell
+./kafka2es
+```
+
+- docker 方式，确保配置文件路径正确 [部署在了阿里云容器里]
+
+```shell
+docker run -d -v `pwd`/src/etc:/app/etc registry.cn-beijing.aliyuncs.com/whoops/kafka2es:v1.0
+```
+### 目录介绍
+- src [主要代码逻辑]
+  - api[http路由相关信息]
+  - cmd[程序入口函数]
+  - es_proxy[输出组件es的相关操作]
+  - etc[配置文件]
+  - format[格式化相关操作]
+  - handle[Kafka-handler-ES的逻辑代码]
+  - kafka_proxy[输入组件kafka的相关操作]
+  - logger[日志相关操作]
+  - model[一些常量信息]
+- main.go [函数执行入口]

README.md

@@ -0,0 +1,2 @@
+# kafka2es
+消费Kafka消息格式化处理后写入到ES

go.mod

@@ -0,0 +1,22 @@
+module whoops/kafka2es
+
+go 1.16
+
+require (
+	github.com/Shopify/sarama v1.36.0
+	github.com/antonfisher/nested-logrus-formatter v1.3.1
+	github.com/go-basic/uuid v1.0.0
+	github.com/gorilla/mux v1.8.0
+	github.com/jonboulle/clockwork v0.2.2 // indirect
+	github.com/json-iterator/go v1.1.12
+	github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible
+	github.com/lestrrat-go/strftime v1.0.5 // indirect
+	github.com/olivere/elastic/v7 v7.0.32
+	github.com/panjf2000/ants/v2 v2.9.0
+	github.com/pkg/errors v0.9.1
+	github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5
+	github.com/sirupsen/logrus v1.8.1
+	github.com/spf13/pflag v1.0.5
+	github.com/spf13/viper v1.8.1
+	go.uber.org/automaxprocs v1.5.1
+)