From 89db5aa1451ce62de09e770b4de25d3c9645a511 Mon Sep 17 00:00:00 2001 From: Rory Crispin Date: Fri, 19 Jul 2024 16:04:43 +0100 Subject: [PATCH 1/7] ClickHouse changes patch --- .github/workflows/build-test-publish.yaml | 472 ++++++++++++++++++ Makefile | 4 +- cmd/otelcontribcol/Dockerfile | 5 +- cmd/otelcontribcol/builder-config.yaml | 5 + cmd/otelcontribcol/components.go | 4 + cmd/otelcontribcol/go.mod | 6 + .../example/docker-compose.yml | 80 +-- .../example/otel-collector-config-local.yml | 132 +++++ .../example/otel-collector-config.yml | 161 ++++-- exporter/clickhouseexporter/exporter_logs.go | 6 +- processor/loghouseprocessor/Makefile | 1 + processor/loghouseprocessor/README.md | 3 + processor/loghouseprocessor/config.go | 7 + processor/loghouseprocessor/factory.go | 54 ++ processor/loghouseprocessor/go.mod | 45 ++ processor/loghouseprocessor/go.sum | 132 +++++ processor/loghouseprocessor/processor.go | 275 ++++++++++ processor/loghouseprocessor/processor_test.go | 233 +++++++++ 18 files changed, 1555 insertions(+), 70 deletions(-) create mode 100644 .github/workflows/build-test-publish.yaml create mode 100644 exporter/clickhouseexporter/example/otel-collector-config-local.yml create mode 100644 processor/loghouseprocessor/Makefile create mode 100644 processor/loghouseprocessor/README.md create mode 100644 processor/loghouseprocessor/config.go create mode 100644 processor/loghouseprocessor/factory.go create mode 100644 processor/loghouseprocessor/go.mod create mode 100644 processor/loghouseprocessor/go.sum create mode 100644 processor/loghouseprocessor/processor.go create mode 100644 processor/loghouseprocessor/processor_test.go diff --git a/.github/workflows/build-test-publish.yaml b/.github/workflows/build-test-publish.yaml new file mode 100644 index 000000000000..628525fb08d8 --- /dev/null +++ b/.github/workflows/build-test-publish.yaml @@ -0,0 +1,472 @@ +name: build-test-publish +on: + push: + branches: [main] + tags: + - "v[0-9]+.[0-9]+.[0-9]+*" + pull_request: +env: + TEST_RESULTS: testbed/tests/results/junit/results.xml + # See: https://github.com/actions/cache/issues/810#issuecomment-1222550359 + # Cache downloads for this workflow consistently run in under 1 minute + SEGMENT_DOWNLOAD_TIMEOUT_MINS: 5 + +# Do not cancel this workflow on main. See https://github.com/open-telemetry/opentelemetry-collector-contrib/pull/16616 +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + setup-environment: + timeout-minutes: 30 + runs-on: ubuntu-latest + if: ${{ github.actor != 'dependabot[bot]' }} + steps: + - name: Checkout Repo + uses: actions/checkout@v3 + - name: Setup Go + uses: actions/setup-go@v3 + with: + go-version: "1.20" + - name: Cache Go + id: go-cache + uses: actions/cache@v3 + with: + path: | + ~/go/bin + ~/go/pkg/mod + key: go-cache-${{ runner.os }}-${{ hashFiles('**/go.sum') }} + - name: Install dependencies + if: steps.go-cache.outputs.cache-hit != 'true' + run: make -j2 gomoddownload + - name: Install Tools + if: steps.go-cache.outputs.cache-hit != 'true' + run: make install-tools + check-collector-module-version: + runs-on: ubuntu-latest + needs: [setup-environment] + steps: + - name: Checkout Repo + uses: actions/checkout@v3 + - name: Check Collector Module Version + run: ./.github/workflows/scripts/check-collector-module-version.sh + lint-matrix: + strategy: + matrix: + group: + - receiver-0 + - receiver-1 + - processor + - exporter + - extension + - connector + - internal + - other + runs-on: ubuntu-latest + needs: [setup-environment] + steps: + - name: Checkout Repo + uses: actions/checkout@v3 + - name: Setup Go + uses: actions/setup-go@v3 + with: + go-version: "1.20" + - name: Cache Go + id: go-cache + uses: actions/cache@v3 + with: + path: | + ~/go/bin + ~/go/pkg/mod + key: go-cache-${{ runner.os }}-${{ hashFiles('**/go.sum') }} + - name: Install dependencies + if: steps.go-cache.outputs.cache-hit != 'true' + run: make -j2 gomoddownload + - name: Install Tools + if: steps.go-cache.outputs.cache-hit != 'true' + run: make install-tools + - name: Cache Lint Build + uses: actions/cache@v3 + with: + path: ~/.cache/go-build + key: go-lint-build-${{ matrix.group }}-${{ runner.os }}-${{ hashFiles('**/go.sum') }} + - name: Lint + run: make -j2 golint GROUP=${{ matrix.group }} + lint: + if: ${{ github.actor != 'dependabot[bot]' && always() }} + runs-on: ubuntu-latest + needs: [setup-environment, lint-matrix] + steps: + - name: Print result + run: echo ${{ needs.lint-matrix.result }} + - name: Interpret result + run: | + if [[ success == ${{ needs.lint-matrix.result }} ]] + then + echo "All matrix jobs passed!" + else + echo "One or more matrix jobs failed." + false + fi + checks: + runs-on: ubuntu-latest + needs: [setup-environment] + steps: + - name: Checkout Repo + uses: actions/checkout@v3 + - name: Setup Go + uses: actions/setup-go@v3 + with: + go-version: "1.20" + - name: Cache Go + id: go-cache + uses: actions/cache@v3 + with: + path: | + ~/go/bin + ~/go/pkg/mod + key: go-cache-${{ runner.os }}-${{ hashFiles('**/go.sum') }} + - name: Install dependencies + if: steps.go-cache.outputs.cache-hit != 'true' + run: make -j2 gomoddownload + - name: Install Tools + if: steps.go-cache.outputs.cache-hit != 'true' + run: make install-tools + - name: CheckDoc + run: make checkdoc + - name: Porto + run: | + make -j2 goporto + git diff --exit-code || (echo 'Porto links are out of date, please run "make goporto" and commit the changes in this PR.' && exit 1) + - name: crosslink + run: | + make crosslink + git diff --exit-code || (echo 'Replace statements are out of date, please run "make crosslink" and commit the changes in this PR.' && exit 1) + - name: Check for go mod dependency changes + run: | + make gotidy + git diff --exit-code || (echo 'go.mod/go.sum deps changes detected, please run "make gotidy" and commit the changes in this PR.' && exit 1) + - name: Gen genotelcontribcol + run: | + make genotelcontribcol + git diff -s --exit-code || (echo 'Generated code is out of date, please run "make genotelcontribcol" and commit the changes in this PR.' && exit 1) + - name: Gen genoteltestbedcol + run: | + make genoteltestbedcol + git diff -s --exit-code || (echo 'Generated code is out of date, please run "make genoteltestbedcol" and commit the changes in this PR.' && exit 1) + - name: CodeGen + run: | + make -j2 generate + git diff --exit-code ':!*go.sum' || (echo 'Generated code is out of date, please run "make generate" and commit the changes in this PR.' && exit 1) + - name: Check gendependabot + run: | + make -j2 gendependabot + git diff --exit-code ':!*go.sum' || (echo 'dependabot.yml is out of date, please run "make gendependabot" and commit the changes in this PR.' && exit 1) + - name: MultimodVerify + run: make multimod-verify + - name: Components dropdown in issue templates + run: | + make generate-gh-issue-templates + git diff --exit-code '.github/ISSUE_TEMPLATE' || (echo 'Dropdowns in issue templates are out of date, please run "make generate-gh-issue-templates" and commit the changes in this PR.' && exit 1) + unittest-matrix: + strategy: + matrix: + go-version: ["1.21", "1.20"] # 1.20 is interpreted as 1.2 without quotes + group: + - receiver-0 + - receiver-1 + - processor + - exporter + - extension + - connector + - internal + - other + runs-on: ubuntu-latest + needs: [setup-environment] + steps: + - name: Collect Workflow Telemetry + if: always() + uses: runforesight/foresight-workflow-kit-action@v1 + with: + api_key: ${{ secrets.FORESIGHT_API_KEY }} + - name: Checkout Repo + uses: actions/checkout@v3 + - name: Setup Go + uses: actions/setup-go@v3 + with: + go-version: ${{ matrix.go-version }} + - name: Cache Go + id: go-cache + uses: actions/cache@v3 + with: + path: | + ~/go/bin + ~/go/pkg/mod + key: go-cache-${{ runner.os }}-${{ hashFiles('**/go.sum') }} + - name: Install dependencies + if: steps.go-cache.outputs.cache-hit != 'true' + run: make -j2 gomoddownload + - name: Install Tools + if: steps.go-cache.outputs.cache-hit != 'true' + run: make install-tools + - name: Cache Test Build + uses: actions/cache@v3 + with: + path: ~/.cache/go-build + key: go-test-build-${{ runner.os }}-${{ matrix.go-version }}-${{ hashFiles('**/go.sum') }} + - name: Run Unit Tests + run: make gotest GROUP=${{ matrix.group }} + unittest: + if: ${{ github.actor != 'dependabot[bot]' && always() }} + strategy: + matrix: + go-version: ["1.21", "1.20"] # 1.20 is interpreted as 1.2 without quotes + runs-on: ubuntu-latest + needs: [setup-environment, unittest-matrix] + steps: + - name: Print result + run: echo ${{ needs.unittest-matrix.result }} + - name: Interpret result + run: | + if [[ success == ${{ needs.unittest-matrix.result }} ]] + then + echo "All matrix jobs passed!" + else + echo "One or more matrix jobs failed." + false + fi + + integration-tests: + runs-on: ubuntu-latest + needs: [setup-environment] + steps: + - name: Checkout Repo + uses: actions/checkout@v3 + - name: Setup Go + uses: actions/setup-go@v3 + with: + go-version: "1.20" + - name: Cache Go + id: go-cache + uses: actions/cache@v3 + with: + path: | + ~/go/bin + ~/go/pkg/mod + key: go-cache-${{ runner.os }}-${{ hashFiles('**/go.sum') }} + - name: Run Integration Tests + run: make integration-tests-with-cover + + correctness-traces: + runs-on: ubuntu-latest + needs: [setup-environment] + steps: + - name: Checkout Repo + uses: actions/checkout@v3 + - name: Setup Go + uses: actions/setup-go@v3 + with: + go-version: "1.20" + - name: Cache Go + id: go-cache + uses: actions/cache@v3 + with: + path: | + ~/go/bin + ~/go/pkg/mod + key: go-cache-${{ runner.os }}-${{ hashFiles('**/go.sum') }} + - name: Install dependencies + if: steps.go-cache.outputs.cache-hit != 'true' + run: make -j2 gomoddownload + - name: Install Tools + if: steps.go-cache.outputs.cache-hit != 'true' + run: make install-tools + - name: Correctness + run: make -C testbed run-correctness-traces-tests + + correctness-metrics: + runs-on: ubuntu-latest + needs: [setup-environment] + steps: + - name: Checkout Repo + uses: actions/checkout@v3 + - name: Setup Go + uses: actions/setup-go@v3 + with: + go-version: "1.20" + - name: Cache Go + id: go-cache + uses: actions/cache@v3 + with: + path: | + ~/go/bin + ~/go/pkg/mod + key: go-cache-${{ runner.os }}-${{ hashFiles('**/go.sum') }} + - name: Install dependencies + if: steps.go-cache.outputs.cache-hit != 'true' + run: make -j2 gomoddownload + - name: Install Tools + if: steps.go-cache.outputs.cache-hit != 'true' + run: make install-tools + - name: Correctness + run: make -C testbed run-correctness-metrics-tests + + cross-compile: + runs-on: ubuntu-latest + needs: [unittest, integration-tests, lint] + strategy: + matrix: + os: + #- darwin + - linux + #- windows + arch: + #- 386 + - amd64 + #- arm + - arm64 + #- ppc64le + exclude: + - os: darwin + arch: 386 + - os: darwin + arch: arm + - os: darwin + arch: ppc64le + - os: windows + arch: arm + - os: windows + arch: arm64 + - os: windows + arch: ppc64le + steps: + - name: Checkout Repo + uses: actions/checkout@v3 + - name: Setup Go + uses: actions/setup-go@v3 + with: + go-version: "1.20" + - name: Cache Go + id: go-cache + uses: actions/cache@v3 + with: + path: | + ~/go/bin + ~/go/pkg/mod + key: go-cache-${{ runner.os }}-${{ hashFiles('**/go.sum') }} + - name: Install dependencies + if: steps.go-cache.outputs.cache-hit != 'true' + run: make -j2 gomoddownload + - name: Install Tools + if: steps.go-cache.outputs.cache-hit != 'true' + run: make install-tools + - name: Build Collector ${{ matrix.binary }} + run: make GOOS=${{ matrix.os }} GOARCH=${{ matrix.arch }} otelcontribcol + - name: Upload Collector Binaries + uses: actions/upload-artifact@v3 + with: + name: collector-binaries + path: ./bin/* + + publish-dev: + permissions: + contents: 'read' + id-token: 'write' + runs-on: ubuntu-latest + needs: [lint, unittest, integration-tests, cross-compile] + if: (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v')) + steps: + - name: Checkout Repo + uses: actions/checkout@v3 + - name: Setup Go + uses: actions/setup-go@v3 + with: + go-version: "1.20" + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Mkdir bin and dist + run: | + mkdir bin/ dist/ + - name: Cache Go + id: go-cache + uses: actions/cache@v3 + with: + path: | + ~/go/bin + ~/go/pkg/mod + key: go-cache-${{ runner.os }}-${{ hashFiles('**/go.sum') }} + - name: Install dependencies + if: steps.go-cache.outputs.cache-hit != 'true' + run: make -j2 gomoddownload + - name: Install Tools + if: steps.go-cache.outputs.cache-hit != 'true' + run: make install-tools + - name: Download Binaries + uses: actions/download-artifact@v3 + with: + name: collector-binaries + path: ./bin/ + - run: chmod +x bin/* + - name: Sanitize branch name and create version + id: create-version + env: + BRANCH: ${{github.ref_name}} + RUN_NUMBER: ${{github.run_number}} + BASE_VERSION: "0.0.0" + run: | + # let's simply use the k8s namespace rules (even stricter) and have the same version(-suffix) for everything + # lowercase everything and replace all invalid characters with '-' and trim to 60 characters + SANITIZED_BRANCH=$(echo -n "${BRANCH}" | tr '[:upper:]' '[:lower:]' | tr -C 'a-z0-9' '-') + SANITIZED_BRANCH="${SANITIZED_BRANCH:0:60}" + + BUILD_VERSION="${BASE_VERSION}-${SANITIZED_BRANCH}-${RUN_NUMBER}" + echo "BUILD_VERSION=${BUILD_VERSION}" | tee -a $GITHUB_ENV $GITHUB_OUTPUT + - id: login-gcp + name: Authenticate with Google Cloud + uses: google-github-actions/auth@v1 + with: + token_format: access_token + workload_identity_provider: ${{secrets.GCR_WORKLOAD_IDENTITY_PROVIDER}} + service_account: ${{secrets.GCR_SERVICE_ACCOUNT}} + access_token_lifetime: 1800s + - name: Login to us Artifact Registry + uses: docker/login-action@v2 + with: + registry: us-docker.pkg.dev + username: oauth2accesstoken + password: ${{ steps.login-gcp.outputs.access_token }} + - name: Login to eu Artifact Registry + uses: docker/login-action@v2 + with: + registry: europe-docker.pkg.dev + username: oauth2accesstoken + password: ${{ steps.login-gcp.outputs.access_token }} + - name: Login to asia Artifact Registry + uses: docker/login-action@v2 + with: + registry: asia-docker.pkg.dev + username: oauth2accesstoken + password: ${{ steps.login-gcp.outputs.access_token }} + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1-node16 + with: + aws-access-key-id: ${{secrets.ECR_AWS_ACCESS_KEY_ID}} + aws-secret-access-key: ${{secrets.ECR_AWS_SECRET_ACCESS_KEY}} + aws-region: us-west-2 + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v1 + - name: build and push + uses: docker/build-push-action@v5 + with: + push: true + context: . + file: ./cmd/otelcontribcol/Dockerfile + platforms: linux/amd64,linux/arm64 + tags: | + 609927696493.dkr.ecr.us-west-2.amazonaws.com/opentelemetry-collector-contrib:${{steps.create-version.outputs.BUILD_VERSION}} + ${{secrets.GCR_ASIA_IMAGE}}:${{steps.create-version.outputs.BUILD_VERSION}} + ${{secrets.GCR_EUROPE_IMAGE}}:${{steps.create-version.outputs.BUILD_VERSION}} + ${{secrets.GCR_US_IMAGE}}:${{steps.create-version.outputs.BUILD_VERSION}} diff --git a/Makefile b/Makefile index c033110b6ef9..4123d29409c9 100644 --- a/Makefile +++ b/Makefile @@ -259,8 +259,8 @@ run: .PHONY: docker-component # Not intended to be used directly docker-component: check-component - GOOS=linux GOARCH=amd64 $(MAKE) $(COMPONENT) - cp ./bin/$(COMPONENT)_linux_amd64 ./cmd/$(COMPONENT)/$(COMPONENT) + GOOS=linux GOARCH=$(GOARCH) $(MAKE) $(COMPONENT) + cp ./bin/$(COMPONENT)_linux_$(GOARCH) ./cmd/$(COMPONENT)/$(COMPONENT) docker build -t $(COMPONENT) ./cmd/$(COMPONENT)/ rm ./cmd/$(COMPONENT)/$(COMPONENT) diff --git a/cmd/otelcontribcol/Dockerfile b/cmd/otelcontribcol/Dockerfile index 10b37eaf6e72..d4a9f14cbe1a 100644 --- a/cmd/otelcontribcol/Dockerfile +++ b/cmd/otelcontribcol/Dockerfile @@ -8,8 +8,11 @@ FROM scratch ARG USER_UID=10001 USER ${USER_UID} +ARG TARGETOS +ARG TARGETARCH + COPY --from=prep /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt -COPY otelcontribcol / +COPY ../../bin/otelcontribcol_${TARGETOS}_${TARGETARCH} /otelcontribcol EXPOSE 4317 55680 55679 ENTRYPOINT ["/otelcontribcol"] CMD ["--config", "/etc/otel/config.yaml"] diff --git a/cmd/otelcontribcol/builder-config.yaml b/cmd/otelcontribcol/builder-config.yaml index 294d571463d3..2f19946e6278 100644 --- a/cmd/otelcontribcol/builder-config.yaml +++ b/cmd/otelcontribcol/builder-config.yaml @@ -126,6 +126,9 @@ processors: - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/tailsamplingprocessor v0.105.0 - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/transformprocessor v0.105.0 - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/remotetapprocessor v0.105.0 + - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/logstransformprocessor v0.105.0 + - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/loghouseprocessor v0.105.0 + receivers: - gomod: go.opentelemetry.io/collector/receiver/nopreceiver v0.105.1-0.20240717163034-43ed6184f9fe @@ -328,6 +331,7 @@ replaces: - github.com/open-telemetry/opentelemetry-collector-contrib/receiver/k8sclusterreceiver => ../../receiver/k8sclusterreceiver - github.com/open-telemetry/opentelemetry-collector-contrib/processor/transformprocessor => ../../processor/transformprocessor - github.com/open-telemetry/opentelemetry-collector-contrib/processor/filterprocessor => ../../processor/filterprocessor + - github.com/open-telemetry/opentelemetry-collector-contrib/processor/loghouseprocessor => ../../processor/loghouseprocessor - github.com/open-telemetry/opentelemetry-collector-contrib/extension/observer/hostobserver => ../../extension/observer/hostobserver - github.com/open-telemetry/opentelemetry-collector-contrib/receiver/kafkareceiver => ../../receiver/kafkareceiver - github.com/open-telemetry/opentelemetry-collector-contrib/extension/bearertokenauthextension => ../../extension/bearertokenauthextension @@ -361,6 +365,7 @@ replaces: - github.com/open-telemetry/opentelemetry-collector-contrib/receiver/influxdbreceiver => ../../receiver/influxdbreceiver - github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver => ../../receiver/awscontainerinsightreceiver - github.com/open-telemetry/opentelemetry-collector-contrib/processor/metricstransformprocessor => ../../processor/metricstransformprocessor + - github.com/open-telemetry/opentelemetry-collector-contrib/processor/logstransformprocessor => ../../processor/logstransformprocessor - github.com/open-telemetry/opentelemetry-collector-contrib/extension/sigv4authextension => ../../extension/sigv4authextension - github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/opencensus => ../../pkg/translator/opencensus - github.com/open-telemetry/opentelemetry-collector-contrib/exporter/pulsarexporter => ../../exporter/pulsarexporter diff --git a/cmd/otelcontribcol/components.go b/cmd/otelcontribcol/components.go index fa9632185fff..6cb68e059520 100644 --- a/cmd/otelcontribcol/components.go +++ b/cmd/otelcontribcol/components.go @@ -116,6 +116,8 @@ import ( groupbytraceprocessor "github.com/open-telemetry/opentelemetry-collector-contrib/processor/groupbytraceprocessor" intervalprocessor "github.com/open-telemetry/opentelemetry-collector-contrib/processor/intervalprocessor" k8sattributesprocessor "github.com/open-telemetry/opentelemetry-collector-contrib/processor/k8sattributesprocessor" + loghouseprocessor "github.com/open-telemetry/opentelemetry-collector-contrib/processor/loghouseprocessor" + logstransformprocessor "github.com/open-telemetry/opentelemetry-collector-contrib/processor/logstransformprocessor" metricsgenerationprocessor "github.com/open-telemetry/opentelemetry-collector-contrib/processor/metricsgenerationprocessor" metricstransformprocessor "github.com/open-telemetry/opentelemetry-collector-contrib/processor/metricstransformprocessor" probabilisticsamplerprocessor "github.com/open-telemetry/opentelemetry-collector-contrib/processor/probabilisticsamplerprocessor" @@ -596,12 +598,14 @@ func components() (otelcol.Factories, error) { cumulativetodeltaprocessor.NewFactory(), deltatorateprocessor.NewFactory(), filterprocessor.NewFactory(), + loghouseprocessor.NewFactory(), groupbyattrsprocessor.NewFactory(), groupbytraceprocessor.NewFactory(), intervalprocessor.NewFactory(), k8sattributesprocessor.NewFactory(), metricsgenerationprocessor.NewFactory(), metricstransformprocessor.NewFactory(), + logstransformprocessor.NewFactory(), probabilisticsamplerprocessor.NewFactory(), redactionprocessor.NewFactory(), resourcedetectionprocessor.NewFactory(), diff --git a/cmd/otelcontribcol/go.mod b/cmd/otelcontribcol/go.mod index 851bc13d371b..73ffda7b72e0 100644 --- a/cmd/otelcontribcol/go.mod +++ b/cmd/otelcontribcol/go.mod @@ -106,6 +106,8 @@ require ( github.com/open-telemetry/opentelemetry-collector-contrib/processor/intervalprocessor v0.105.0 github.com/open-telemetry/opentelemetry-collector-contrib/processor/k8sattributesprocessor v0.105.0 github.com/open-telemetry/opentelemetry-collector-contrib/processor/metricsgenerationprocessor v0.105.0 + github.com/open-telemetry/opentelemetry-collector-contrib/processor/loghouseprocessor v0.105.0 + github.com/open-telemetry/opentelemetry-collector-contrib/processor/logstransformprocessor v0.105.0 github.com/open-telemetry/opentelemetry-collector-contrib/processor/metricstransformprocessor v0.105.0 github.com/open-telemetry/opentelemetry-collector-contrib/processor/probabilisticsamplerprocessor v0.105.0 github.com/open-telemetry/opentelemetry-collector-contrib/processor/redactionprocessor v0.105.0 @@ -1014,6 +1016,8 @@ replace github.com/open-telemetry/opentelemetry-collector-contrib/processor/tran replace github.com/open-telemetry/opentelemetry-collector-contrib/processor/filterprocessor => ../../processor/filterprocessor +replace github.com/open-telemetry/opentelemetry-collector-contrib/processor/loghouseprocessor => ../../processor/loghouseprocessor + replace github.com/open-telemetry/opentelemetry-collector-contrib/extension/observer/hostobserver => ../../extension/observer/hostobserver replace github.com/open-telemetry/opentelemetry-collector-contrib/receiver/kafkareceiver => ../../receiver/kafkareceiver @@ -1080,6 +1084,8 @@ replace github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsco replace github.com/open-telemetry/opentelemetry-collector-contrib/processor/metricstransformprocessor => ../../processor/metricstransformprocessor +replace github.com/open-telemetry/opentelemetry-collector-contrib/processor/logstransformprocessor => ../../processor/logstransformprocessor + replace github.com/open-telemetry/opentelemetry-collector-contrib/extension/sigv4authextension => ../../extension/sigv4authextension replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/opencensus => ../../pkg/translator/opencensus diff --git a/exporter/clickhouseexporter/example/docker-compose.yml b/exporter/clickhouseexporter/example/docker-compose.yml index 709d554f8679..d175fc64443c 100644 --- a/exporter/clickhouseexporter/example/docker-compose.yml +++ b/exporter/clickhouseexporter/example/docker-compose.yml @@ -4,18 +4,28 @@ networks: otel-clickhouse: services: +# clickhouse-logger: +# image: alpine:latest +# container_name: alpine +# entrypoint: ["head", "-n", "1000", "/tmp/clickhouse-server.log"] +# volumes: +# - "./clickhouse-server.log:/tmp/clickhouse-server.log" + otelcollector: build: context: ../../../local - dockerfile: ../exporter/clickhouseexporter/example/Dockerfile + dockerfile: Dockerfile # Uncomment the next line to use a preexisting image - # image: otelcontribcol:latest + image: otelcontribcol:latest container_name: otel command: - "--config=/etc/otel-collector-config.yml" - - "--set=service.telemetry.logs.level=INFO" + - "--set=service.telemetry.logs.level=DEBUG" volumes: - ./otel-collector-config.yml:/etc/otel-collector-config.yml + - ./clickhouse-server.log:/tmp/clickhouse-server.log + - ./clickhouse-server.log.short:/tmp/clickhouse-server.log.short + - ./otel-storage:/var/lib/otelcol ports: - "4317:4317" # otlp receiver - "1888:1888" # pprof extension @@ -49,36 +59,36 @@ services: networks: - otel-clickhouse - # Log generator - flog: - image: mingrammer/flog:0.4.3 - # Output fake log in JSON format - command: [ "--format=json", "--loop", "--delay=100ms"] - networks: - - otel-clickhouse - depends_on: - - otelcollector - logging: - driver: fluentd - options: - mode: non-blocking - # Allow time for otel-collector to spin up, then forward fluentd logs to the fluentforwarder receiver. - fluentd-async-connect: "true" - # Use nanosecond precision - fluentd-sub-second-precision: "true" - stop_signal: SIGKILL +# # Log generator +# flog: +# image: mingrammer/flog:0.4.3 +# # Output fake log in JSON format +# command: [ "--format=json", "--loop", "--delay=100ms"] +# networks: +# - otel-clickhouse +# depends_on: +# - otelcollector +# logging: +# driver: fluentd +# options: +# mode: non-blocking +# # Allow time for otel-collector to spin up, then forward fluentd logs to the fluentforwarder receiver. +# fluentd-async-connect: "true" +# # Use nanosecond precision +# fluentd-sub-second-precision: "true" +# stop_signal: SIGKILL - # Traces generator - telemetrygen: - build: - context: ../../../cmd/telemetrygen/ - command: - - traces - - --otlp-endpoint=otelcollector:4317 - - --otlp-insecure - - --rate=100 - - --duration=10000h - networks: - - otel-clickhouse - depends_on: - - otelcollector + +# # Traces generator +# tracegen: +# build: +# context: ../../../cmd/tracegen/ +# command: +# - --otlp-endpoint=otelcollector:4317 +# - --otlp-insecure +# - --rate=100 +# - --duration=10000h +# networks: +# - otel-clickhouse +# depends_on: +# - otelcollector diff --git a/exporter/clickhouseexporter/example/otel-collector-config-local.yml b/exporter/clickhouseexporter/example/otel-collector-config-local.yml new file mode 100644 index 000000000000..820dcaf4e0dc --- /dev/null +++ b/exporter/clickhouseexporter/example/otel-collector-config-local.yml @@ -0,0 +1,132 @@ +receivers: + filelog: + start_at: beginning + include: + - /tmp/clickhouse-server.log.short +# - /Users/rory/var/log/pods/*/*/*.log +processors: + transform: + log_statements: + + - context: log + statements: + # clickhouse server/keeper: + + # extract clickhouse/keeper log level +# - set(severity_text, "TRACE") where IsMatch(body, "[<]Trace[>]") == true +# - set(severity_text, "DEBUG") where IsMatch(body, "[<]Debug[>]") == true +# - set(severity_text, "INFO") where IsMatch(body, "[<]Info(rmation(al)?)?[>]") == true +# - set(severity_xt, "WARN") where IsMatch(body, "[<]Warn(ing)?[>]") == true +# - set(severity_text, "ERROR") where IsMatch(body, "[<]Error[>]") == true +# - set(severity_text, "FATAL") where IsMatch(body, "[<]Fatal[>]") == true + + + # clickpipes/other apps: + + # parse json logs + - merge_maps(attributes, ParseJSON(body), "insert") where IsMatch(body, "^\\s*[{].+?[}]\\s*$") == true + - set(body, attributes["msg"]) where attributes["msg"] != nil + - set(body, attributes["message"]) where attributes["message"] != nil + - delete_key(attributes, "msg") where body == attributes["msg"] + - delete_key(attributes, "message") where body == attributes["message"] + + # parse severity text + - set(severity_text, ConvertCase(attributes["level"], "upper")) where IsMatch(attributes["level"], "(?i)^(trace|debug|info|warn|error|fatal)$") == true + + # Extract error code + - merge_maps(attributes, ExtractPatterns(body, "(?P[A-Z][A-Z0-9]+_[A-Z0-9_]+)"), "insert") + - delete_key(attributes, "level") where attributes["level"] != nil and severity_text == ConvertCase(attributes["level"], "upper") + + # Parse ClickHouse JSON log levels + - set(severity_text, "FATAL") where attributes["level"] == "0" + - set(severity_text, "FATAL") where attributes["level"] == "1" + - set(severity_text, "FATAL") where attributes["level"] == "2" + - set(severity_text, "ERROR") where attributes["level"] == "3" + - set(severity_text, "WARN") where attributes["level"] == "4" + - set(severity_text, "INFO") where attributes["level"] == "5" + - set(severity_text, "INFO") where attributes["level"] == "6" + - set(severity_text, "DEBUG") where attributes["level"] == "7" + - set(severity_text, "TRACE") where attributes["level"] == "8" + logstransform: + operators: + - type: add + field: resource.app + value: "" + if: resource.app == nil + +# - type: add +# field: resource.app +# value: clickhouse-server +# if: resource["k8s.container.name"] matches "^c-.+-server$" +# +# - type: add +# field: resource.app +# value: clickhouse-keeper +# if: resource["k8s.container.name"] matches "^c-.+-keeper$" +# +# # recombine stack trace logs to single log entry +# - type: recombine +# is_first_entry: body matches "^\\s*\\d{4}[.]\\d{2}[.]\\d{2}" +# combine_field: body +# combine_with: "\n" +# source_identifier: attributes["log.file.path"] +# if: resource.app matches "^clickhouse-" +# +# # parse timestamp from log +# - type: regex_parser +# regex: ^\\s*(?P\\d{4}[.]\\d{2}[.]\\d{2}\\s+\\d{2}:\\d{2}:\\d{2}\\.\\d+) +# timestamp: +# parse_from: attributes.ts +# layout_type: strptime +# layout: "%Y.%m.%d %H:%M:%S.%L" +# if: resource.app matches "^clickhouse-" + + # remove tmp fields + - type: remove + field: resource.app + + # Parse timestamp from ClickHouse structured logs + - type: time_parser + parse_from: attributes.date_time + layout_type: epoch + layout: 's.us' + +exporters: + clickhouse: + create_db_and_tables: true + endpoint: http://localhost:8123 + database: otel + logs_table_name: otel_logs_2 + traces_table_name: otel_traces + connection_params: + async_insert: 1 + wait_for_async_insert: 0 + ttl_days: 1 + timeout: 10s + sending_queue: + queue_size: 100 + retry_on_failure: + enabled: true + initial_interval: 5s + max_interval: 30s + max_elapsed_time: 300s +extensions: +# file_storage: +# directory: /var/lib/otelcol + health_check: + pprof: + zpages: + memory_ballast: + size_mib: 1000 + +service: + extensions: + - pprof + - zpages + - health_check +# - file_storage + pipelines: + logs: + receivers: [ filelog ] + processors: [ transform, logstransform ] + exporters: [ clickhouse ] diff --git a/exporter/clickhouseexporter/example/otel-collector-config.yml b/exporter/clickhouseexporter/example/otel-collector-config.yml index e6c13dd5d7b3..4dea1fff4b84 100644 --- a/exporter/clickhouseexporter/example/otel-collector-config.yml +++ b/exporter/clickhouseexporter/example/otel-collector-config.yml @@ -1,34 +1,131 @@ receivers: - fluentforward: - endpoint: 0.0.0.0:24224 - otlp: - protocols: - grpc: - endpoint: 0.0.0.0:4317 + filelog: + start_at: beginning +# storage: file_storage + include: + - /tmp/clickhouse-server.log.short +# operators: +# - id: get-format +# routes: +# - expr: body matches "^\\{" +# output: parser-docker +# - expr: body matches "^[^ Z]+ " +# output: parser-crio +# - expr: body matches "^[^ Z]+Z" +# output: parser-containerd +# type: router +# - id: parser-crio +# output: extract_metadata_from_filepath +# regex: ^(?P