From 7bf24d787f793602fb21fdada7a7ad5c3021eb50 Mon Sep 17 00:00:00 2001 From: Rajat Date: Tue, 12 Dec 2023 10:58:50 -0800 Subject: [PATCH 01/12] fluent-bit addition Signed-off-by: Rajat --- .github/workflows/image_run_pr_v3.yml | 128 ++++++++++++++++ .github/workflows/image_run_v3.yml | 96 ++++++++++++ .github/workflows/monitor.yml | 6 +- .github/workflows/readme_updater.yml | 16 ++ builder.lst | 4 +- community_images/fluent-bit/bitnami/.rfignore | 3 + community_images/fluent-bit/bitnami/README.md | 141 ++++++++++++++++++ .../fluent-bit/bitnami/coverage.sh | 7 + .../fluent-bit/bitnami/dc_coverage.sh | 23 +++ .../fluent-bit/bitnami/docker-compose.yml | 16 ++ community_images/fluent-bit/bitnami/image.yml | 47 ++++++ .../fluent-bit/bitnami/k8s_coverage.sh | 24 +++ .../fluent-bit/bitnami/overrides.yml | 20 +++ .../scripts/fluent-bit_coverage_script.sh | 22 +++ .../fluent-bit/ironbank/.rfignore | 1 + .../fluent-bit/ironbank/README.md | 139 +++++++++++++++++ .../fluent-bit/ironbank/config/fluent-bit.yml | 99 ++++++++++++ .../fluent-bit/ironbank/coverage.sh | 7 + .../fluent-bit/ironbank/dc_coverage.sh | 26 ++++ .../fluent-bit/ironbank/docker-compose.yml | 17 +++ .../fluent-bit/ironbank/image.yml | 46 ++++++ .../fluent-bit/ironbank/k8s_coverage.sh | 27 ++++ .../fluent-bit/ironbank/overrides.yml | 20 +++ .../scripts/fluent-bit_coverage_script.sh | 19 +++ image.lst | 4 +- 25 files changed, 955 insertions(+), 3 deletions(-) create mode 100644 community_images/fluent-bit/bitnami/.rfignore create mode 100644 community_images/fluent-bit/bitnami/README.md create mode 100644 community_images/fluent-bit/bitnami/coverage.sh create mode 100644 community_images/fluent-bit/bitnami/dc_coverage.sh create mode 100644 community_images/fluent-bit/bitnami/docker-compose.yml create mode 100644 community_images/fluent-bit/bitnami/image.yml create mode 100644 community_images/fluent-bit/bitnami/k8s_coverage.sh create mode 100644 community_images/fluent-bit/bitnami/overrides.yml create mode 100644 community_images/fluent-bit/bitnami/scripts/fluent-bit_coverage_script.sh create mode 100644 community_images/fluent-bit/ironbank/.rfignore create mode 100644 community_images/fluent-bit/ironbank/README.md create mode 100644 community_images/fluent-bit/ironbank/config/fluent-bit.yml create mode 100644 community_images/fluent-bit/ironbank/coverage.sh create mode 100644 community_images/fluent-bit/ironbank/dc_coverage.sh create mode 100644 community_images/fluent-bit/ironbank/docker-compose.yml create mode 100644 community_images/fluent-bit/ironbank/image.yml create mode 100644 community_images/fluent-bit/ironbank/k8s_coverage.sh create mode 100644 community_images/fluent-bit/ironbank/overrides.yml create mode 100644 community_images/fluent-bit/ironbank/scripts/fluent-bit_coverage_script.sh diff --git a/.github/workflows/image_run_pr_v3.yml b/.github/workflows/image_run_pr_v3.yml index e056b97270..8ee514b49f 100644 --- a/.github/workflows/image_run_pr_v3.yml +++ b/.github/workflows/image_run_pr_v3.yml @@ -1228,6 +1228,134 @@ jobs: -H "Authorization: Bearer ${PULL_COUNTER_MAGIC_TOKEN}" \ -d '{ "rapidfort/etcd-ib": 1 }' \ https://data-receiver.rapidfort.com/counts/internal_image_pulls + fluent-bit: + runs-on: ubuntu-latest + + environment: actions-cicd-pr + + steps: + - uses: actions/checkout@v3 + + + - name: Check if test required + id: test-required + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + python3 scripts/tests_required.py fluent-bit fluent-bit/bitnami output.txt + cat output.txt >> $GITHUB_OUTPUT + + - name: Start minikube + + if: ${{ github.event_name == 'pull_request' && steps.test-required.outputs.TEST_REQUIRED == 'yes' }} + + with: + memory: 6g + uses: medyagh/setup-minikube@master + - name: Check k8s cluster ! + + if: ${{ github.event_name == 'pull_request' && steps.test-required.outputs.TEST_REQUIRED == 'yes' }} + + run: kubectl get pods -A + - name: Setup ubuntu + env: + RF_ACCESS_ID: ${{ secrets.RF_ACCESS_ID }} + RF_SECRET_ACCESS_KEY: ${{ secrets.RF_SECRET_ACCESS_KEY }} + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }} + RF_PLATFORM_HOST: ${{ secrets.RF_PLATFORM_HOST }} + + if: ${{ github.event_name == 'pull_request' && steps.test-required.outputs.TEST_REQUIRED == 'yes' }} + + run: ./scripts/setup.sh + + - name: Create images for testing PR + env: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }} + RAPIDFORT_ACCOUNT: ${{ secrets.RAPIDFORT_ACCOUNT }} + IB_DOCKER_USERNAME: ${{ secrets.IB_DOCKER_USERNAME }} + IB_DOCKER_PASSWORD: ${{ secrets.IB_DOCKER_PASSWORD }} + if: ${{ steps.test-required.outputs.TEST_REQUIRED == 'yes' }} + run: python3 community_images/common/orchestrator/main.py hourly_run fluent-bit/bitnami + + + - name: Report pulls + env: + PULL_COUNTER_MAGIC_TOKEN: ${{ secrets.PULL_COUNTER_MAGIC_TOKEN }} + + if: ${{ github.event_name == 'pull_request' && steps.test-required.outputs.TEST_REQUIRED == 'yes' }} + + run: | + curl -X POST \ + -H "Accept: application/json" \ + -H "Authorization: Bearer ${PULL_COUNTER_MAGIC_TOKEN}" \ + -d '{ "rapidfort/fluent-bit": 1 }' \ + https://data-receiver.rapidfort.com/counts/internal_image_pulls + fluent-bit-ib: + runs-on: ubuntu-latest + + environment: actions-cicd-pr + + steps: + - uses: actions/checkout@v3 + + + - name: Check if test required + id: test-required + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + python3 scripts/tests_required.py fluent-bit-ib fluent-bit/ironbank output.txt + cat output.txt >> $GITHUB_OUTPUT + + - name: Start minikube + + if: ${{ github.event_name == 'pull_request' && steps.test-required.outputs.TEST_REQUIRED == 'yes' }} + + with: + memory: 6g + uses: medyagh/setup-minikube@master + - name: Check k8s cluster ! + + if: ${{ github.event_name == 'pull_request' && steps.test-required.outputs.TEST_REQUIRED == 'yes' }} + + run: kubectl get pods -A + - name: Setup ubuntu + env: + RF_ACCESS_ID: ${{ secrets.RF_ACCESS_ID }} + RF_SECRET_ACCESS_KEY: ${{ secrets.RF_SECRET_ACCESS_KEY }} + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }} + RF_PLATFORM_HOST: ${{ secrets.RF_PLATFORM_HOST }} + + if: ${{ github.event_name == 'pull_request' && steps.test-required.outputs.TEST_REQUIRED == 'yes' }} + + run: ./scripts/setup.sh + + - name: Create images for testing PR + env: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }} + RAPIDFORT_ACCOUNT: ${{ secrets.RAPIDFORT_ACCOUNT }} + IB_DOCKER_USERNAME: ${{ secrets.IB_DOCKER_USERNAME }} + IB_DOCKER_PASSWORD: ${{ secrets.IB_DOCKER_PASSWORD }} + if: ${{ steps.test-required.outputs.TEST_REQUIRED == 'yes' }} + run: python3 community_images/common/orchestrator/main.py hourly_run fluent-bit/ironbank + + + - name: Report pulls + env: + PULL_COUNTER_MAGIC_TOKEN: ${{ secrets.PULL_COUNTER_MAGIC_TOKEN }} + + if: ${{ github.event_name == 'pull_request' && steps.test-required.outputs.TEST_REQUIRED == 'yes' }} + + run: | + curl -X POST \ + -H "Accept: application/json" \ + -H "Authorization: Bearer ${PULL_COUNTER_MAGIC_TOKEN}" \ + -d '{ "rapidfort/fluent-bit-ib": 1 }' \ + https://data-receiver.rapidfort.com/counts/internal_image_pulls fluentd: runs-on: ubuntu-latest diff --git a/.github/workflows/image_run_v3.yml b/.github/workflows/image_run_v3.yml index 9e34e296a9..ded5bfa03f 100644 --- a/.github/workflows/image_run_v3.yml +++ b/.github/workflows/image_run_v3.yml @@ -923,6 +923,102 @@ jobs: -H "Authorization: Bearer ${PULL_COUNTER_MAGIC_TOKEN}" \ -d '{ "rapidfort/etcd-ib": 1 }' \ https://data-receiver.rapidfort.com/counts/internal_image_pulls + fluent-bit: + runs-on: ubuntu-latest + + environment: actions-cicd + + steps: + - uses: actions/checkout@v3 + + + - name: Start minikube + + with: + memory: 6g + uses: medyagh/setup-minikube@master + - name: Check k8s cluster ! + + run: kubectl get pods -A + - name: Setup ubuntu + env: + RF_ACCESS_ID: ${{ secrets.RF_ACCESS_ID }} + RF_SECRET_ACCESS_KEY: ${{ secrets.RF_SECRET_ACCESS_KEY }} + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }} + RF_PLATFORM_HOST: ${{ secrets.RF_PLATFORM_HOST }} + + run: ./scripts/setup.sh + + - name: Create images and publish to dockerhub + env: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }} + RAPIDFORT_ACCOUNT: ${{ secrets.RAPIDFORT_ACCOUNT }} + IB_DOCKER_USERNAME: ${{ secrets.IB_DOCKER_USERNAME }} + IB_DOCKER_PASSWORD: ${{ secrets.IB_DOCKER_PASSWORD }} + if: github.event_name != 'pull_request' + run: python3 community_images/common/orchestrator/main.py hourly_run fluent-bit/bitnami --publish + + + - name: Report pulls + env: + PULL_COUNTER_MAGIC_TOKEN: ${{ secrets.PULL_COUNTER_MAGIC_TOKEN }} + + run: | + curl -X POST \ + -H "Accept: application/json" \ + -H "Authorization: Bearer ${PULL_COUNTER_MAGIC_TOKEN}" \ + -d '{ "rapidfort/fluent-bit": 1 }' \ + https://data-receiver.rapidfort.com/counts/internal_image_pulls + fluent-bit-ib: + runs-on: ubuntu-latest + + environment: actions-cicd + + steps: + - uses: actions/checkout@v3 + + + - name: Start minikube + + with: + memory: 6g + uses: medyagh/setup-minikube@master + - name: Check k8s cluster ! + + run: kubectl get pods -A + - name: Setup ubuntu + env: + RF_ACCESS_ID: ${{ secrets.RF_ACCESS_ID }} + RF_SECRET_ACCESS_KEY: ${{ secrets.RF_SECRET_ACCESS_KEY }} + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }} + RF_PLATFORM_HOST: ${{ secrets.RF_PLATFORM_HOST }} + + run: ./scripts/setup.sh + + - name: Create images and publish to dockerhub + env: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }} + RAPIDFORT_ACCOUNT: ${{ secrets.RAPIDFORT_ACCOUNT }} + IB_DOCKER_USERNAME: ${{ secrets.IB_DOCKER_USERNAME }} + IB_DOCKER_PASSWORD: ${{ secrets.IB_DOCKER_PASSWORD }} + if: github.event_name != 'pull_request' + run: python3 community_images/common/orchestrator/main.py hourly_run fluent-bit/ironbank --publish + + + - name: Report pulls + env: + PULL_COUNTER_MAGIC_TOKEN: ${{ secrets.PULL_COUNTER_MAGIC_TOKEN }} + + run: | + curl -X POST \ + -H "Accept: application/json" \ + -H "Authorization: Bearer ${PULL_COUNTER_MAGIC_TOKEN}" \ + -d '{ "rapidfort/fluent-bit-ib": 1 }' \ + https://data-receiver.rapidfort.com/counts/internal_image_pulls fluentd: runs-on: ubuntu-latest diff --git a/.github/workflows/monitor.yml b/.github/workflows/monitor.yml index 65bcc86de3..5d19dcccab 100644 --- a/.github/workflows/monitor.yml +++ b/.github/workflows/monitor.yml @@ -54,6 +54,10 @@ jobs: run: docker pull rapidfort/etcd - name: Pull Etcd Ironbank run: docker pull rapidfort/etcd-ib + - name: Pull Fluent-Bit + run: docker pull rapidfort/fluent-bit + - name: Pull Fluent-Bit Ironbank + run: docker pull rapidfort/fluent-bit-ib - name: Pull Fluentd run: docker pull rapidfort/fluentd - name: Pull Fluentd IronBank @@ -164,5 +168,5 @@ jobs: curl -X POST \ -H "Accept: application/json" \ -H "Authorization: Bearer ${PULL_COUNTER_MAGIC_TOKEN}" \ - -d '{ "rapidfort/airflow": 1 , "rapidfort/airflow-ib": 1 , "rapidfort/airflow-scheduler": 1 , "rapidfort/airflow-worker": 1 , "rapidfort/apache": 1 , "rapidfort/apache2-ib": 1 , "rapidfort/apache-official": 1 , "rapidfort/cassandra-official": 1 , "rapidfort/consul": 1 , "rapidfort/consul-ib": 1 , "rapidfort/consul-official": 1 , "rapidfort/couchdb": 1 , "rapidfort/couchdb_3-ib": 1 , "rapidfort/couchdb-official": 1 , "rapidfort/curl": 1 , "rapidfort/elasticsearch": 1 , "rapidfort/elasticsearch-official": 1 , "rapidfort/envoy": 1 , "rapidfort/envoy-official": 1 , "rapidfort/etcd": 1 , "rapidfort/etcd-ib": 1 , "rapidfort/fluentd": 1 , "rapidfort/fluentd-ib": 1 , "rapidfort/fluentd-official": 1 , "rapidfort/ghost": 1 , "rapidfort/grafana-ib": 1 , "rapidfort/haproxy": 1 , "rapidfort/haproxy24-ib": 1 , "rapidfort/haproxy-official": 1 , "rapidfort/influxdb": 1 , "rapidfort/keycloak-official": 1 , "rapidfort/kong": 1 , "rapidfort/mariadb": 1 , "rapidfort/mariadb-ib": 1 , "rapidfort/mariadb-official": 1 , "rapidfort/memcached": 1 , "rapidfort/memcached-ib": 1 , "rapidfort/memcached-official": 1 , "rapidfort/microsoft-sql-server-2019-ib": 1 , "rapidfort/mongodb": 1 , "rapidfort/mongodb-ib": 1 , "rapidfort/mongodb-official": 1 , "rapidfort/mysql": 1 , "rapidfort/mysql8-ib": 1 , "rapidfort/mysql-official": 1 , "rapidfort/nats": 1 , "rapidfort/nats-ib": 1 , "rapidfort/nats-official": 1 , "rapidfort/nginx": 1 , "rapidfort/nginx-ib": 1 , "rapidfort/nginx-official": 1 , "rapidfort/oncall": 1 , "rapidfort/postgresql": 1 , "rapidfort/postgresql12-ib": 1 , "rapidfort/postgresql-official": 1 , "rapidfort/prometheus": 1 , "rapidfort/prometheus-ib": 1 , "rapidfort/rabbitmq": 1 , "rapidfort/redis": 1 , "rapidfort/redis-cluster": 1 , "rapidfort/redis6-ib": 1 , "rapidfort/redis-official": 1 , "rapidfort/telegraf": 1 , "rapidfort/traefik": 1 , "rapidfort/traefik-ib": 1 , "rapidfort/vault": 1 , "rapidfort/wordpress": 1 , "rapidfort/wordpress-ib": 1 , "rapidfort/yourls": 1 , "rapidfort/zookeeper": 1 , "rapidfort/zookeeper-ib": 1 , "rapidfort/zookeeper-official": 1 }' \ + -d '{ "rapidfort/airflow": 1 , "rapidfort/airflow-ib": 1 , "rapidfort/airflow-scheduler": 1 , "rapidfort/airflow-worker": 1 , "rapidfort/apache": 1 , "rapidfort/apache2-ib": 1 , "rapidfort/apache-official": 1 , "rapidfort/cassandra-official": 1 , "rapidfort/consul": 1 , "rapidfort/consul-ib": 1 , "rapidfort/consul-official": 1 , "rapidfort/couchdb": 1 , "rapidfort/couchdb_3-ib": 1 , "rapidfort/couchdb-official": 1 , "rapidfort/curl": 1 , "rapidfort/elasticsearch": 1 , "rapidfort/elasticsearch-official": 1 , "rapidfort/envoy": 1 , "rapidfort/envoy-official": 1 , "rapidfort/etcd": 1 , "rapidfort/etcd-ib": 1 , "rapidfort/fluent-bit": 1 , "rapidfort/fluent-bit-ib": 1 , "rapidfort/fluentd": 1 , "rapidfort/fluentd-ib": 1 , "rapidfort/fluentd-official": 1 , "rapidfort/ghost": 1 , "rapidfort/grafana-ib": 1 , "rapidfort/haproxy": 1 , "rapidfort/haproxy24-ib": 1 , "rapidfort/haproxy-official": 1 , "rapidfort/influxdb": 1 , "rapidfort/keycloak-official": 1 , "rapidfort/kong": 1 , "rapidfort/mariadb": 1 , "rapidfort/mariadb-ib": 1 , "rapidfort/mariadb-official": 1 , "rapidfort/memcached": 1 , "rapidfort/memcached-ib": 1 , "rapidfort/memcached-official": 1 , "rapidfort/microsoft-sql-server-2019-ib": 1 , "rapidfort/mongodb": 1 , "rapidfort/mongodb-ib": 1 , "rapidfort/mongodb-official": 1 , "rapidfort/mysql": 1 , "rapidfort/mysql8-ib": 1 , "rapidfort/mysql-official": 1 , "rapidfort/nats": 1 , "rapidfort/nats-ib": 1 , "rapidfort/nats-official": 1 , "rapidfort/nginx": 1 , "rapidfort/nginx-ib": 1 , "rapidfort/nginx-official": 1 , "rapidfort/oncall": 1 , "rapidfort/postgresql": 1 , "rapidfort/postgresql12-ib": 1 , "rapidfort/postgresql-official": 1 , "rapidfort/prometheus": 1 , "rapidfort/prometheus-ib": 1 , "rapidfort/rabbitmq": 1 , "rapidfort/redis": 1 , "rapidfort/redis-cluster": 1 , "rapidfort/redis6-ib": 1 , "rapidfort/redis-official": 1 , "rapidfort/telegraf": 1 , "rapidfort/traefik": 1 , "rapidfort/traefik-ib": 1 , "rapidfort/vault": 1 , "rapidfort/wordpress": 1 , "rapidfort/wordpress-ib": 1 , "rapidfort/yourls": 1 , "rapidfort/zookeeper": 1 , "rapidfort/zookeeper-ib": 1 , "rapidfort/zookeeper-official": 1 }' \ https://data-receiver.rapidfort.com/counts/internal_image_pulls diff --git a/.github/workflows/readme_updater.yml b/.github/workflows/readme_updater.yml index 1d2ccc6b40..ab727c202b 100644 --- a/.github/workflows/readme_updater.yml +++ b/.github/workflows/readme_updater.yml @@ -210,6 +210,22 @@ jobs: repository: rapidfort/etcd-ib readme-filepath: ./community_images/etcd/ironbank/README.md.dockerhub short-description: RapidFort optimized, hardened image for Etcd Ironbank + - name: Docker Hub Description Fluent-Bit + uses: peter-evans/dockerhub-description@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + repository: rapidfort/fluent-bit + readme-filepath: ./community_images/fluent-bit/bitnami/README.md.dockerhub + short-description: RapidFort optimized, hardened image for Fluent-Bit + - name: Docker Hub Description Fluent-Bit Ironbank + uses: peter-evans/dockerhub-description@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + repository: rapidfort/fluent-bit-ib + readme-filepath: ./community_images/fluent-bit/ironbank/README.md.dockerhub + short-description: RapidFort optimized, hardened image for Fluent-Bit Ironbank - name: Docker Hub Description Fluentd uses: peter-evans/dockerhub-description@v3 with: diff --git a/builder.lst b/builder.lst index bd686dda62..184eafe9e2 100644 --- a/builder.lst +++ b/builder.lst @@ -67,4 +67,6 @@ etcd/ironbank traefik/ironbank nats/ironbank airflow/airflow/ironbank -keycloak/official \ No newline at end of file +keycloak/official +fluent-bit/ironbank +fluent-bit/bitnami \ No newline at end of file diff --git a/community_images/fluent-bit/bitnami/.rfignore b/community_images/fluent-bit/bitnami/.rfignore new file mode 100644 index 0000000000..f6d6006729 --- /dev/null +++ b/community_images/fluent-bit/bitnami/.rfignore @@ -0,0 +1,3 @@ +opt/bitnami/fluent-bit/licenses +opt/bitnami/licenses +opt/bitnami/common/licenses \ No newline at end of file diff --git a/community_images/fluent-bit/bitnami/README.md b/community_images/fluent-bit/bitnami/README.md new file mode 100644 index 0000000000..db819f91a1 --- /dev/null +++ b/community_images/fluent-bit/bitnami/README.md @@ -0,0 +1,141 @@ + +RapidFort + + +
+ +[![rf-h][rf-h-badge]][rf-view-report-button] +[![DH Image][dh-rf-badge]][rf-dh-image-link] +[![Slack][slack-badge]][slack-link] +[![FOSSA Status][fossa-badge]][fossa-link] + +# RapidFort hardened image for Fluent-Bit + +RapidFort’s container optimization process hardened this Fluent-Bit container. This container is free to use and has no license limitations. + +It is the same as the [Bitnami Fluent-Bit][source-image-repo-link] image but more secure. + +Every day, we optimize and harden a variety of Docker Hub’s most famous images. Check out our [entire library](https://hub.docker.com/u/rapidfort) of secured containers. +
+ +[Get the full report here or click on the image below][rf-view-report-link] + +[![Metrics][metrics-link]][rf-image-metrics-link] + +

Vulnerabilities: Original vs. Hardened + +

+ +[![CVE Reduction][cve-reduction-link]][rf-image-cve-reduction-link] + + +View Report + +
+
+ + +## What is Fluent-Bit? + +> Fluent Bit is a Fast and Lightweight Log Processor and Forwarder. It has been made with a strong focus on performance to allow the collection of events from different sources without complexity. + + +[Overview of Fluent-Bit](https://fluentbit.io/) + +Trademarks: This software listing is packaged by RapidFort. The respective trademarks mentioned in the offering are owned by the respective companies, and use of them does not imply any affiliation or endorsement. + + +## How do I use this hardened Fluent-Bit image? + +The runtime instructions for this container are no different from the official release. Follow the instructions in their readme, but use our hardened image. + + +View Detailed Instructions + +
+
+ +```sh +$ helm repo add bitnami https://charts.bitnami.com/bitnami +# install fluent-bit, just replace repository with RapidFort registry +$ helm install my-fluent-bit bitnami/fluent-bit --set image.repository=rapidfort/fluent-bit + +``` + +## What is a hardened image? + +A hardened image is a copy of a container that has been optimized and reduced for significantly improved security. Because every container uses many open-source software components and their dependencies, there’s a lot of extra weight that can be trimmed. + +This image is a hardened version of the official [Bitnami Fluent-Bit][source-image-repo-link] image on Docker Hub. + +RapidFort is an industry-leading container optimization solution that minimizes software attack surfaces by removing unused code. Most containers can be reduced by at least 50%, which reduces the opportunity for malicious attacks and CVE exploits. Learn more at [RapidFort.com][rf-link]. + +Our hardened images are updated daily using the latest vulnerability information available. + + +View on GitHub + +
+
+ +## What’s the difference between the official [Bitnami Fluent-Bit][source-image-repo-link] image and this hardened image? +RapidFort’s hardened [rapidfort/fluent-bit][rf-dh-image-link] image has been optimized by our proprietary scanning and slimming technology. We are big fans of open-source software, containerized infrastructure, and security. + +We are making secure copies of the images we use every day and the most popular ones on Docker Hub. We want to make the world a safer place to operate. + +## Supported tags and respective `Dockerfile` links +* [`2`, `2-debian-11`, `2.2.0`, `2.2.0-debian-11-r` (2/debian-11/Dockerfile)](https://github.com/bitnami/containers/tree/main/bitnami/fluent-bit/2/debian-11/Dockerfile) + +## Need support + +Join our slack community for any questions. + + +RapidFort Community Slack + + +## 🌟 Support this project + +[![](https://user-images.githubusercontent.com/48997634/174794647-0c851917-e5c9-4fb9-bf88-b61d89dc2f4f.gif)](https://github.com/rapidfort/community-images/stargazers) + +### [⏫⭐️ Scroll to the star button](#start-of-content) + +If you believe this project has potential, feel free to **star this repo** just like many [amazing people](https://github.com/rapidfort/community-images/stargazers) +have. + +## Have questions? + +[![RapidFort](https://raw.githubusercontent.com/rapidfort/community-images/main/contrib/github_logo_footer.png)][rf-rapidfort-footer-logo-link] + + +If you'd like to learn more about RapidFort or our container optimization process, visit [RapidFort.com][rf-link]. + +
+
+ + +[dh-rf-badge]: https://img.shields.io/badge/dockerhub-images-important.svg?logo=Docker + +[fossa-badge]: https://app.fossa.com/api/projects/git%2Bgithub.com%2Frapidfort%2Fcommunity-images.svg?type=shield +[fossa-link]: https://app.fossa.com/projects/git%2Bgithub.com%2Frapidfort%2Fcommunity-images?ref=badge_shield + +[rf-link]: https://rapidfort.com?utm_source=github&utm_medium=ci_rf_link&utm_campaign=sep_01_sprint&utm_term=fluent-bit&utm_content=rapidfort_have_questions + +[rf-rapidfort-footer-logo-link]: https://frontrow.rapidfort.com/app/community/imageinfo/docker.io%2Fbitnami%2Ffluent-bit?utm_source=github&utm_medium=ci_view_report&utm_campaign=sep_01_sprint&utm_term=fluent-bit&utm_content=rapidfort_footer_logo +[rf-view-report-button]: https://frontrow.rapidfort.com/app/community/imageinfo/docker.io%2Fbitnami%2Ffluent-bit?utm_source=github&utm_medium=ci_view_report&utm_campaign=sep_01_sprint&utm_term=fluent-bit&utm_content=view_report_button +[rf-view-report-link]: https://frontrow.rapidfort.com/app/community/imageinfo/docker.io%2Fbitnami%2Ffluent-bit?utm_source=github&utm_medium=ci_view_report&utm_campaign=sep_01_sprint&utm_term=fluent-bit&utm_content=view_report_link +[rf-image-metrics-link]: https://frontrow.rapidfort.com/app/community/imageinfo/docker.io%2Fbitnami%2Ffluent-bit?utm_source=github&utm_medium=ci_view_report&utm_campaign=sep_01_sprint&utm_term=fluent-bit&utm_content=image_metrics_link +[rf-image-cve-reduction-link]: https://frontrow.rapidfort.com/app/community/imageinfo/docker.io%2Fbitnami%2Ffluent-bit?utm_source=github&utm_medium=ci_view_report&utm_campaign=sep_01_sprint&utm_term=fluent-bit&utm_content=image_cve_reduction_link + +[dh-img-size-badge]: https://img.shields.io/docker/image-size/rapidfort/fluent-bit?logo=docker&logoColor=white&sort=semver +[dh-img-pulls-badge]: https://img.shields.io/docker/pulls/rapidfort/fluent-bit?logo=docker&logoColor=white + +[slack-badge]: https://img.shields.io/static/v1?label=Join&message=slack&logo=slack&logoColor=E01E5A&color=4A154B +[slack-link]: https://join.slack.com/t/rapidfortcommunity/shared_invite/zt-1g3wy28lv-DaeGexTQ5IjfpbmYW7Rm_Q + +[rf-h-badge]: https://img.shields.io/static/v1?label=RapidFort&labelColor=333F48&message=hardened&color=50B4C4&logo=data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACcAAAAkCAYAAAAKNyObAAAACXBIWXMAACE4AAAhOAFFljFgAAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8YQUAAAHvSURBVHgB7ZjvTcMwEMUvEgNkhNuAjOAR2IAyQbsB2YAyQbsBYoKwQdjA3aAjHA514Xq1Hf9r6QeeFKVJ3tkv+cWOVYCAiKg124b82gZqe0+NNlsHJbLBxthg1o+RASetIEdTJxnBRvtUMCHgM6TIBtMZwY7SiQFfrhUsN+Ao/TJYR3WC5QY88/Nge6oXLBRwO+P/GcnNMZzZteBR0zQfogM0O4Q47Uz9TtSrUIHs71+paugw16Dn+qt5xJ/TD4viEcrE25tepaXPaHxP350GXtD10WwHQWjQxKhl7YUGRg/MuPaY9vxuzPFA+RpEW9rj0yCMbcCsmG9B+Xpk7YRo4RnjQEEttBiBtAefyI23BtoYpBrmRO6ZX0EZWo60c1yfaGBMOKRzdKVocYZO/NpuMss7E9cHitcc0gFS5Qig2LUUtCGkmmJwOsJJvLlokdWtfMFzAvLGctCOooYPtg2USoRQ7HwM2hXzIzuvKQenIxzHm4oWmZ9TKF1AnAR8sI2moB093nKcjoBvtnHFzoXQ8qeMDGcLtUW/i4NYtJ3jJhRcSnRYHMSg1Q5PD5cWHT4/ih0vIpDOf9QrhZtQLsWxlILT8AjXEol/iQRaiVTBX4pO57D6U0WJBFoFtyaLtuqLfwf19G62e7hFWbQKKuoLYovGDo9dW28AAAAASUVORK5CYII= +[metrics-link]: https://github.com/rapidfort/community-images/raw/main/community_images/fluent-bit/bitnami/assets/metrics.webp +[cve-reduction-link]: https://github.com/rapidfort/community-images/raw/main/community_images/fluent-bit/bitnami/assets/cve_reduction.webp + +[source-image-repo-link]: https://hub.docker.com/r/bitnami/fluent-bit +[rf-dh-image-link]: https://hub.docker.com/r/rapidfort/fluent-bit diff --git a/community_images/fluent-bit/bitnami/coverage.sh b/community_images/fluent-bit/bitnami/coverage.sh new file mode 100644 index 0000000000..d3ccfd4b2e --- /dev/null +++ b/community_images/fluent-bit/bitnami/coverage.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +function test_fluent-bit() { + CONTAINER_NAME=$1 + + CMD="docker exec -i ${CONTAINER_NAME} bash -c /tmp/fluent-bit_coverage_script.sh" + $CMD \ No newline at end of file diff --git a/community_images/fluent-bit/bitnami/dc_coverage.sh b/community_images/fluent-bit/bitnami/dc_coverage.sh new file mode 100644 index 0000000000..e1ff10da5f --- /dev/null +++ b/community_images/fluent-bit/bitnami/dc_coverage.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +set -x +set -e + +SCRIPTPATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" + +# shellcheck disable=SC1091 +. "${SCRIPTPATH}"/../../common/scripts/bash_helper.sh + +JSON_PARAMS="$1" + + +PROJECT_NAME=$(jq -r '.project_name' < "$JSON_PARAMS") +CONTAINER_NAME="${PROJECT_NAME}"-fluent-bit-1 + +# Wait +sleep 10 + +# log for debugging +docker inspect "${CONTAINER_NAME}" +docker cp "${SCRIPTPATH}"/scripts/fluent-bit_coverage_script.sh "${CONTAINER_NAME}":/tmp/fluent-bit_coverage_script.sh +docker exec -i ${CONTAINER_NAME} bash -c "bash /tmp/fluent-bit_coverage_script.sh" \ No newline at end of file diff --git a/community_images/fluent-bit/bitnami/docker-compose.yml b/community_images/fluent-bit/bitnami/docker-compose.yml new file mode 100644 index 0000000000..8a86804a84 --- /dev/null +++ b/community_images/fluent-bit/bitnami/docker-compose.yml @@ -0,0 +1,16 @@ +version: '2' + +services: + fluent-bit: + image: ${FLUENT_BIT_IMAGE_REPOSITORY}:${FLUENT_BIT_IMAGE_TAG} + volumes: + - ./scripts/fluent-bit_coverage_script.sh:/tmp/coverage_script.sh + cap_add: + - SYS_PTRACE + ports: + - '2020:2020' + + prometheus: + image: docker.io/rapidfort/prometheus + cap_add: + - SYS_PTRACE \ No newline at end of file diff --git a/community_images/fluent-bit/bitnami/image.yml b/community_images/fluent-bit/bitnami/image.yml new file mode 100644 index 0000000000..f7aeb67219 --- /dev/null +++ b/community_images/fluent-bit/bitnami/image.yml @@ -0,0 +1,47 @@ +name: fluent-bit +official_name: Fluent-Bit +official_website: https://fluentbit.io/ +source_image_provider: Bitnami +source_image_repo: docker.io/bitnami/fluent-bit +source_image_repo_link: https://hub.docker.com/r/bitnami/fluent-bit +source_image_readme: https://github.com/bitnami/containers/blob/main/bitnami/fluent-bit/README.md +rf_docker_link: rapidfort/fluent-bit +image_workflow_name: fluent-bit_bitnami +github_location: fluent-bit/bitnami +report_url: https://frontrow.rapidfort.com/app/community/imageinfo/docker.io%2Fbitnami%2Ffluent-bit +usage_instructions: | + $ helm repo add bitnami https://charts.bitnami.com/bitnami + # install fluent-bit, just replace repository with RapidFort registry + $ helm install my-fluent-bit bitnami/fluent-bit --set image.repository=rapidfort/fluent-bit +what_is_text: | + Fluent Bit is a Fast and Lightweight Log Processor and Forwarder. It has been made with a strong focus on performance to allow the collection of events from different sources without complexity. +disclaimer: | + Trademarks: This software listing is packaged by RapidFort. The respective trademarks mentioned in the offering are owned by the respective companies, and use of them does not imply any affiliation or endorsement. +input_registry: + registry: docker.io + account: bitnami +repo_sets: + - fluent-bit: + input_base_tag: "2.2." + +runtimes: + - type: docker_compose + script: dc_coverage.sh + compose_file: docker-compose.yml + image_keys: + fluent-bit: + repository: "FLUENT_BIT_IMAGE_REPOSITORY" + tag: "FLUENT_BIT_IMAGE_TAG" + + + - type: k8s + script: k8s_coverage.sh + helm: + repo: bitnami + repo_url: https://charts.bitnami.com/bitnami + chart: fluent-bit + image_keys: + fluent-bit: + repository: "image.repository" + tag: "image.tag" + override_file: "overrides.yml" \ No newline at end of file diff --git a/community_images/fluent-bit/bitnami/k8s_coverage.sh b/community_images/fluent-bit/bitnami/k8s_coverage.sh new file mode 100644 index 0000000000..4230d2361c --- /dev/null +++ b/community_images/fluent-bit/bitnami/k8s_coverage.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +set -x +set -e + +# shellcheck disable=SC1091 +SCRIPTPATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" + +# shellcheck disable=SC1091 +. "${SCRIPTPATH}"/../../common/scripts/bash_helper.sh + +JSON_PARAMS="$1" + +JSON=$(cat "$JSON_PARAMS") +NAMESPACE=$(jq -r '.namespace_name' < "$JSON_PARAMS") +RELEASE_NAME=$(jq -r '.release_name' < "$JSON_PARAMS") + +sleep 60 +CONTAINER_NAME="${RELEASE_NAME}-0" + +# copy over the script to the pod +kubectl cp "${SCRIPTPATH}"/scripts/fluent-bit_coverage_script.sh "${CONTAINER_NAME}":/tmp/cvoverage_script.sh -n "${NAMESPACE}" + +test_fluent-bit "${CONTAINER_NAME}" "${NAMESPACE}" "yes" \ No newline at end of file diff --git a/community_images/fluent-bit/bitnami/overrides.yml b/community_images/fluent-bit/bitnami/overrides.yml new file mode 100644 index 0000000000..3dc6dc3b08 --- /dev/null +++ b/community_images/fluent-bit/bitnami/overrides.yml @@ -0,0 +1,20 @@ +image: + pullSecrets: ["rf-regcred"] + pullPolicy: Always +containerSecurityContext: + enabled: true + runAsUser: 1001 + allowPrivilegeEscalation: true + capabilities: + add: ["SYS_PTRACE"] +resourceType: deployment +replicaCount: 1 +extraEnvVars: + - name: "RF_VERBOSE" + value: "0" +livenessProbe: + initialDelaySeconds: 30 + timeoutSeconds: 30 +readinessProbe: + initialDelaySeconds: 30 + timeoutSeconds: 30 \ No newline at end of file diff --git a/community_images/fluent-bit/bitnami/scripts/fluent-bit_coverage_script.sh b/community_images/fluent-bit/bitnami/scripts/fluent-bit_coverage_script.sh new file mode 100644 index 0000000000..f7f73b0f06 --- /dev/null +++ b/community_images/fluent-bit/bitnami/scripts/fluent-bit_coverage_script.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +set -e +set -x + +# Start Fluent Bit with the specified configuration file +/opt/bitnami/fluent-bit/bin/fluent-bit -c /opt/bitnami/fluent-bit/conf/fluent-bit.conf & + +# Add a delay to allow Fluent Bit to run (you can adjust this as needed) +# Capture the PID of the last background process +FLUENT_BIT_PID=$! + +# Add a delay to allow Fluent Bit to run (you can adjust this as needed) +sleep 5 + +# Your additional commands or logic can go here + +# Stop Fluent Bit gracefully by sending a termination signal +kill -TERM $FLUENT_BIT_PID + +# Optionally, wait for Fluent Bit to exit +wait $FLUENT_BIT_PID \ No newline at end of file diff --git a/community_images/fluent-bit/ironbank/.rfignore b/community_images/fluent-bit/ironbank/.rfignore new file mode 100644 index 0000000000..98801c7a94 --- /dev/null +++ b/community_images/fluent-bit/ironbank/.rfignore @@ -0,0 +1 @@ +usr/share/licenses \ No newline at end of file diff --git a/community_images/fluent-bit/ironbank/README.md b/community_images/fluent-bit/ironbank/README.md new file mode 100644 index 0000000000..fd6cdbecd2 --- /dev/null +++ b/community_images/fluent-bit/ironbank/README.md @@ -0,0 +1,139 @@ + +RapidFort + + +
+ +[![rf-h][rf-h-badge]][rf-view-report-button] +[![DH Image][dh-rf-badge]][rf-dh-image-link] +[![Slack][slack-badge]][slack-link] +[![FOSSA Status][fossa-badge]][fossa-link] + +# RapidFort hardened image for Fluent-Bit Ironbank + +RapidFort’s container optimization process hardened this Fluent-Bit Ironbank container. This container is free to use and has no license limitations. + +It is the same as the [Platform One Fluent-Bit Ironbank][source-image-repo-link] image but more secure. + +Every day, we optimize and harden a variety of Docker Hub’s most famous images. Check out our [entire library](https://hub.docker.com/u/rapidfort) of secured containers. +
+ +[Get the full report here or click on the image below][rf-view-report-link] + +[![Metrics][metrics-link]][rf-image-metrics-link] + +

Vulnerabilities: Original vs. Hardened + +

+ +[![CVE Reduction][cve-reduction-link]][rf-image-cve-reduction-link] + + +View Report + +
+
+ + +## What is Fluent-Bit Ironbank? + +> Fluent Bit is a Fast and Lightweight Log Processor and Forwarder. It has been made with a strong focus on performance to allow the collection of events from different sources without complexity. + + +[Overview of Fluent-Bit Ironbank](https://fluentbit.io/) + +Trademarks: This software listing is packaged by RapidFort. The respective trademarks mentioned in the offering are owned by the respective companies, and use of them does not imply any affiliation or endorsement. + + +## How do I use this hardened Fluent-Bit Ironbank image? + +The runtime instructions for this container are no different from the official release. Follow the instructions in their readme, but use our hardened image. + + +View Detailed Instructions + +
+
+ +```sh +# install fluent-bit, using docker run +$ docker run -it --name Fluent-Bit rapidfort/fluent-bit-ib + +``` + +## What is a hardened image? + +A hardened image is a copy of a container that has been optimized and reduced for significantly improved security. Because every container uses many open-source software components and their dependencies, there’s a lot of extra weight that can be trimmed. + +This image is a hardened version of the official [Platform One Fluent-Bit Ironbank][source-image-repo-link] image on Docker Hub. + +RapidFort is an industry-leading container optimization solution that minimizes software attack surfaces by removing unused code. Most containers can be reduced by at least 50%, which reduces the opportunity for malicious attacks and CVE exploits. Learn more at [RapidFort.com][rf-link]. + +Our hardened images are updated daily using the latest vulnerability information available. + + +View on GitHub + +
+
+ +## What’s the difference between the official [Platform One Fluent-Bit Ironbank][source-image-repo-link] image and this hardened image? +RapidFort’s hardened [rapidfort/fluent-bit-ib][rf-dh-image-link] image has been optimized by our proprietary scanning and slimming technology. We are big fans of open-source software, containerized infrastructure, and security. + +We are making secure copies of the images we use every day and the most popular ones on Docker Hub. We want to make the world a safer place to operate. + +## Supported tags and respective `Dockerfile` links + +## Need support + +Join our slack community for any questions. + + +RapidFort Community Slack + + +## 🌟 Support this project + +[![](https://user-images.githubusercontent.com/48997634/174794647-0c851917-e5c9-4fb9-bf88-b61d89dc2f4f.gif)](https://github.com/rapidfort/community-images/stargazers) + +### [⏫⭐️ Scroll to the star button](#start-of-content) + +If you believe this project has potential, feel free to **star this repo** just like many [amazing people](https://github.com/rapidfort/community-images/stargazers) +have. + +## Have questions? + +[![RapidFort](https://raw.githubusercontent.com/rapidfort/community-images/main/contrib/github_logo_footer.png)][rf-rapidfort-footer-logo-link] + + +If you'd like to learn more about RapidFort or our container optimization process, visit [RapidFort.com][rf-link]. + +
+
+ + +[dh-rf-badge]: https://img.shields.io/badge/dockerhub-images-important.svg?logo=Docker + +[fossa-badge]: https://app.fossa.com/api/projects/git%2Bgithub.com%2Frapidfort%2Fcommunity-images.svg?type=shield +[fossa-link]: https://app.fossa.com/projects/git%2Bgithub.com%2Frapidfort%2Fcommunity-images?ref=badge_shield + +[rf-link]: https://rapidfort.com?utm_source=github&utm_medium=ci_rf_link&utm_campaign=sep_01_sprint&utm_term=fluent-bit-ib&utm_content=rapidfort_have_questions + +[rf-rapidfort-footer-logo-link]: https://us01.rapidfort.com/app/community/imageinfo/registry1.dso.mil%2Fironbank%2Fopensource%2Ffluent%2Ffluent-bit?utm_source=github&utm_medium=ci_view_report&utm_campaign=sep_01_sprint&utm_term=fluent-bit-ib&utm_content=rapidfort_footer_logo +[rf-view-report-button]: https://us01.rapidfort.com/app/community/imageinfo/registry1.dso.mil%2Fironbank%2Fopensource%2Ffluent%2Ffluent-bit?utm_source=github&utm_medium=ci_view_report&utm_campaign=sep_01_sprint&utm_term=fluent-bit-ib&utm_content=view_report_button +[rf-view-report-link]: https://us01.rapidfort.com/app/community/imageinfo/registry1.dso.mil%2Fironbank%2Fopensource%2Ffluent%2Ffluent-bit?utm_source=github&utm_medium=ci_view_report&utm_campaign=sep_01_sprint&utm_term=fluent-bit-ib&utm_content=view_report_link +[rf-image-metrics-link]: https://us01.rapidfort.com/app/community/imageinfo/registry1.dso.mil%2Fironbank%2Fopensource%2Ffluent%2Ffluent-bit?utm_source=github&utm_medium=ci_view_report&utm_campaign=sep_01_sprint&utm_term=fluent-bit-ib&utm_content=image_metrics_link +[rf-image-cve-reduction-link]: https://us01.rapidfort.com/app/community/imageinfo/registry1.dso.mil%2Fironbank%2Fopensource%2Ffluent%2Ffluent-bit?utm_source=github&utm_medium=ci_view_report&utm_campaign=sep_01_sprint&utm_term=fluent-bit-ib&utm_content=image_cve_reduction_link + +[dh-img-size-badge]: https://img.shields.io/docker/image-size/rapidfort/fluent-bit-ib?logo=docker&logoColor=white&sort=semver +[dh-img-pulls-badge]: https://img.shields.io/docker/pulls/rapidfort/fluent-bit-ib?logo=docker&logoColor=white + +[slack-badge]: https://img.shields.io/static/v1?label=Join&message=slack&logo=slack&logoColor=E01E5A&color=4A154B +[slack-link]: https://join.slack.com/t/rapidfortcommunity/shared_invite/zt-1g3wy28lv-DaeGexTQ5IjfpbmYW7Rm_Q + +[rf-h-badge]: https://img.shields.io/static/v1?label=RapidFort&labelColor=333F48&message=hardened&color=50B4C4&logo=data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACcAAAAkCAYAAAAKNyObAAAACXBIWXMAACE4AAAhOAFFljFgAAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8YQUAAAHvSURBVHgB7ZjvTcMwEMUvEgNkhNuAjOAR2IAyQbsB2YAyQbsBYoKwQdjA3aAjHA514Xq1Hf9r6QeeFKVJ3tkv+cWOVYCAiKg124b82gZqe0+NNlsHJbLBxthg1o+RASetIEdTJxnBRvtUMCHgM6TIBtMZwY7SiQFfrhUsN+Ao/TJYR3WC5QY88/Nge6oXLBRwO+P/GcnNMZzZteBR0zQfogM0O4Q47Uz9TtSrUIHs71+paugw16Dn+qt5xJ/TD4viEcrE25tepaXPaHxP350GXtD10WwHQWjQxKhl7YUGRg/MuPaY9vxuzPFA+RpEW9rj0yCMbcCsmG9B+Xpk7YRo4RnjQEEttBiBtAefyI23BtoYpBrmRO6ZX0EZWo60c1yfaGBMOKRzdKVocYZO/NpuMss7E9cHitcc0gFS5Qig2LUUtCGkmmJwOsJJvLlokdWtfMFzAvLGctCOooYPtg2USoRQ7HwM2hXzIzuvKQenIxzHm4oWmZ9TKF1AnAR8sI2moB093nKcjoBvtnHFzoXQ8qeMDGcLtUW/i4NYtJ3jJhRcSnRYHMSg1Q5PD5cWHT4/ih0vIpDOf9QrhZtQLsWxlILT8AjXEol/iQRaiVTBX4pO57D6U0WJBFoFtyaLtuqLfwf19G62e7hFWbQKKuoLYovGDo9dW28AAAAASUVORK5CYII= +[metrics-link]: https://github.com/rapidfort/community-images/raw/main/community_images/fluent-bit/ironbank/assets/metrics.webp +[cve-reduction-link]: https://github.com/rapidfort/community-images/raw/main/community_images/fluent-bit/ironbank/assets/cve_reduction.webp + +[source-image-repo-link]: https://registry1.dso.mil/harbor/projects/3/repositories/opensource%2Ffluent%2fluent-bit +[rf-dh-image-link]: https://hub.docker.com/r/rapidfort/fluent-bit-ib diff --git a/community_images/fluent-bit/ironbank/config/fluent-bit.yml b/community_images/fluent-bit/ironbank/config/fluent-bit.yml new file mode 100644 index 0000000000..510d809591 --- /dev/null +++ b/community_images/fluent-bit/ironbank/config/fluent-bit.yml @@ -0,0 +1,99 @@ +service: + # Flush + # ===== + # set an interval of seconds before to flush records to a destination + flush: 1 + + # Daemon + # ====== + # instruct Fluent Bit to run in foreground or background mode. + daemon: Off + + # Log_Level + # ========= + # Set the verbosity level of the service, values can be: + # + # - error + # - warning + # - info + # - debug + # - trace + # + # by default 'info' is set, that means it includes 'error' and 'warning'. + log_level: info + + # Parsers File + # ============ + # specify an optional 'Parsers' configuration file + parsers_file: parsers.conf + + # Plugins File + # ============ + # specify an optional 'Plugins' configuration file to load external plugins. + plugins_file: plugins.conf + + # HTTP Server + # =========== + # Enable/Disable the built-in HTTP Server for metrics + http_server: Off + http_listen: 0.0.0.0 + http_port: 2020 + + # Storage + # ======= + # Fluent Bit can use memory and filesystem buffering based mechanisms + # + # - https://docs.fluentbit.io/manual/administration/buffering-and-storage + # + # storage metrics + # --------------- + # publish storage pipeline metrics in '/api/v1/storage'. The metrics are + # exported only if the 'http_server' option is enabled. + # + storage: + metrics: on + + # storage.path + # ------------ + # absolute file system path to store filesystem data buffers (chunks). + # + # storage.path /tmp/storage + path: /tmp/storage + + # storage.sync + # ------------ + # configure the synchronization mode used to store the data into the + # filesystem. It can take the values normal or full. + # + # storage.sync normal + sync: normal + + # storage.checksum + # ---------------- + # enable the data integrity check when writing and reading data from the + # filesystem. The storage layer uses the CRC32 algorithm. + # + # storage.checksum off + checksum: off + + # storage.backlog.mem_limit + # ------------------------- + # if storage.path is set, Fluent Bit will look for data chunks that were + # not delivered and are still in the storage layer, these are called + # backlog data. This option configure a hint of maximum value of memory + # to use when processing these records. + # + # storage.backlog.mem_limit 5M + backlog: + mem_limit: 5M + +input: + name: cpu + tag: cpu.local + + # Read interval (sec) Default: 1 + interval_sec: 1 + +output: + name: stdout + match: "*" \ No newline at end of file diff --git a/community_images/fluent-bit/ironbank/coverage.sh b/community_images/fluent-bit/ironbank/coverage.sh new file mode 100644 index 0000000000..d3ccfd4b2e --- /dev/null +++ b/community_images/fluent-bit/ironbank/coverage.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +function test_fluent-bit() { + CONTAINER_NAME=$1 + + CMD="docker exec -i ${CONTAINER_NAME} bash -c /tmp/fluent-bit_coverage_script.sh" + $CMD \ No newline at end of file diff --git a/community_images/fluent-bit/ironbank/dc_coverage.sh b/community_images/fluent-bit/ironbank/dc_coverage.sh new file mode 100644 index 0000000000..f925ace156 --- /dev/null +++ b/community_images/fluent-bit/ironbank/dc_coverage.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +set -x +set -e + +SCRIPTPATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" + +# shellcheck disable=SC1091 +. "${SCRIPTPATH}"/../../common/scripts/bash_helper.sh + +JSON_PARAMS="$1" + +JSON=$(cat "$JSON_PARAMS") + +echo "Json params for docker compose coverage = $JSON" + +PROJECT_NAME=$(jq -r '.project_name' < "$JSON_PARAMS") +CONTAINER_NAME="${PROJECT_NAME}"-fluent-bit-1 + +# Wait +sleep 10 + +# log for debugging +docker inspect "${CONTAINER_NAME}" +docker cp "${SCRIPTPATH}"/scripts/fluent-bit_coverage_script.sh "${CONTAINER_NAME}":/tmp/fluent-bit_coverage_script.sh +docker exec -i ${CONTAINER_NAME} bash -c "bash /tmp/fluent-bit_coverage_script.sh" diff --git a/community_images/fluent-bit/ironbank/docker-compose.yml b/community_images/fluent-bit/ironbank/docker-compose.yml new file mode 100644 index 0000000000..ad77a4882b --- /dev/null +++ b/community_images/fluent-bit/ironbank/docker-compose.yml @@ -0,0 +1,17 @@ +version: '2' + +services: + fluent-bit: + # image: 'registry1.dso.mil/ironbank/opensource/fluent/fluent-bit' + image: ${FLUENT_BIT_IMAGE_REPOSITORY}:${FLUENT_BIT_IMAGE_TAG} + volumes: + - ./scripts/fluent-bit_coverage_script.sh:/tmp/coverage_script.sh + cap_add: + - SYS_PTRACE + ports: + - '2020:2020' + + prometheus: + image: docker.io/rapidfort/prometheus + cap_add: + - SYS_PTRACE \ No newline at end of file diff --git a/community_images/fluent-bit/ironbank/image.yml b/community_images/fluent-bit/ironbank/image.yml new file mode 100644 index 0000000000..64cda33c04 --- /dev/null +++ b/community_images/fluent-bit/ironbank/image.yml @@ -0,0 +1,46 @@ +name: fluent-bit-ib +official_name: Fluent-Bit Ironbank +official_website: https://fluentbit.io/ +source_image_provider: Platform One +source_image_repo: registry1.dso.mil/ironbank/opensource/fluent/fluent-bit +source_image_repo_link: https://registry1.dso.mil/harbor/projects/3/repositories/opensource%2Ffluent%2fluent-bit +source_image_readme: https://repo1.dso.mil/dsop/opensource/fluent/fluent-bit/-/blob/development/README.md +rf_docker_link: rapidfort/fluent-bit-ib +image_workflow_name: fluent-bit_ironbank +github_location: fluent-bit/ironbank +report_url: https://us01.rapidfort.com/app/community/imageinfo/registry1.dso.mil%2Fironbank%2Fopensource%2Ffluent%2Ffluent-bit +usage_instructions: | + # install fluent-bit, using docker run + $ docker run -it --name Fluent-Bit rapidfort/fluent-bit-ib +what_is_text: | + Fluent Bit is a Fast and Lightweight Log Processor and Forwarder. It has been made with a strong focus on performance to allow the collection of events from different sources without complexity. +disclaimer: | + Trademarks: This software listing is packaged by RapidFort. The respective trademarks mentioned in the offering are owned by the respective companies, and use of them does not imply any affiliation or endorsement. +input_registry: + registry: registry1.dso.mil + account: ironbank +repo_sets: + - opensource/fluent/fluent-bit: + input_base_tag: "2.2." + output_repo: fluent-bit-ib +runtimes: + - type: docker_compose + script: dc_coverage.sh + compose_file: docker-compose.yml + image_keys: + fluent-bit-ib: + repository: "FLUENT_BIT_IMAGE_REPOSITORY" + tag: "FLUENT_BIT_IMAGE_TAG" + + + - type: k8s + script: k8s_coverage.sh + helm: + repo: fluent + repo_url: https://fluent.github.io/helm-charts + chart: fluent-bit + image_keys: + fluent-bit-ib: + repository: "image.repository" + tag: "image.tag" + override_file: "overrides.yml" \ No newline at end of file diff --git a/community_images/fluent-bit/ironbank/k8s_coverage.sh b/community_images/fluent-bit/ironbank/k8s_coverage.sh new file mode 100644 index 0000000000..33493b146b --- /dev/null +++ b/community_images/fluent-bit/ironbank/k8s_coverage.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +set -x +set -e + +# shellcheck disable=SC1091 +SCRIPTPATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" + +# shellcheck disable=SC1091 +. "${SCRIPTPATH}"/../../common/scripts/bash_helper.sh + + +JSON_PARAMS="$1" + +JSON=$(cat "$JSON_PARAMS") + +NAMESPACE=$(jq -r '.namespace_name' < "$JSON_PARAMS") +RELEASE_NAME=$(jq -r '.release_name' < "$JSON_PARAMS") +echo "NAMESPACE: $NAMESPACE" +echo "RELEASE_NAME: $RELEASE_NAME" + +sleep 60 +CONTAINER_NAME="${RELEASE_NAME}-0" +# copy over the script to the pod +kubectl cp "${SCRIPTPATH}"/scripts/fluent-bit_coverage_script.sh "${CONTAINER_NAME}":/tmp/fluent-bit_cvoverage_script.sh -n "${NAMESPACE}" + +test_fluent-bit "${CONTAINER_NAME}" "${NAMESPACE}" "yes" \ No newline at end of file diff --git a/community_images/fluent-bit/ironbank/overrides.yml b/community_images/fluent-bit/ironbank/overrides.yml new file mode 100644 index 0000000000..3dc6dc3b08 --- /dev/null +++ b/community_images/fluent-bit/ironbank/overrides.yml @@ -0,0 +1,20 @@ +image: + pullSecrets: ["rf-regcred"] + pullPolicy: Always +containerSecurityContext: + enabled: true + runAsUser: 1001 + allowPrivilegeEscalation: true + capabilities: + add: ["SYS_PTRACE"] +resourceType: deployment +replicaCount: 1 +extraEnvVars: + - name: "RF_VERBOSE" + value: "0" +livenessProbe: + initialDelaySeconds: 30 + timeoutSeconds: 30 +readinessProbe: + initialDelaySeconds: 30 + timeoutSeconds: 30 \ No newline at end of file diff --git a/community_images/fluent-bit/ironbank/scripts/fluent-bit_coverage_script.sh b/community_images/fluent-bit/ironbank/scripts/fluent-bit_coverage_script.sh new file mode 100644 index 0000000000..a724f5ecf2 --- /dev/null +++ b/community_images/fluent-bit/ironbank/scripts/fluent-bit_coverage_script.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +set -e +set -x + +# Start Fluent Bit with the specified configuration file +/fluent-bit/bin/fluent-bit -c /fluent-bit/etc/fluent-bit.conf & + + +# Capture the PID of the last background process +FLUENT_BIT_PID=$! + +sleep 5 + +# Stop Fluent Bit gracefully by sending a termination signal +kill -TERM $FLUENT_BIT_PID + +# Optionally, wait for Fluent Bit to exit +wait $FLUENT_BIT_PID \ No newline at end of file diff --git a/image.lst b/image.lst index 396cf0256f..449608234b 100644 --- a/image.lst +++ b/image.lst @@ -69,4 +69,6 @@ etcd/ironbank traefik/ironbank nats/ironbank airflow/airflow/ironbank -keycloak/official \ No newline at end of file +keycloak/official +fluent-bit/ironbank +fluent-bit/bitnami \ No newline at end of file From 322bb394456a517bb6c8fd125e22ea94b07aaf15 Mon Sep 17 00:00:00 2001 From: Rajat Date: Tue, 12 Dec 2023 11:14:13 -0800 Subject: [PATCH 02/12] runner correction Signed-off-by: Rajat --- community_images/fluent-bit/bitnami/dc_coverage.sh | 2 +- community_images/fluent-bit/ironbank/dc_coverage.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/community_images/fluent-bit/bitnami/dc_coverage.sh b/community_images/fluent-bit/bitnami/dc_coverage.sh index e1ff10da5f..cfbbba3a97 100644 --- a/community_images/fluent-bit/bitnami/dc_coverage.sh +++ b/community_images/fluent-bit/bitnami/dc_coverage.sh @@ -20,4 +20,4 @@ sleep 10 # log for debugging docker inspect "${CONTAINER_NAME}" docker cp "${SCRIPTPATH}"/scripts/fluent-bit_coverage_script.sh "${CONTAINER_NAME}":/tmp/fluent-bit_coverage_script.sh -docker exec -i ${CONTAINER_NAME} bash -c "bash /tmp/fluent-bit_coverage_script.sh" \ No newline at end of file +docker exec -i "${CONTAINER_NAME}" bash -c "bash /tmp/fluent-bit_coverage_script.sh" \ No newline at end of file diff --git a/community_images/fluent-bit/ironbank/dc_coverage.sh b/community_images/fluent-bit/ironbank/dc_coverage.sh index f925ace156..b6296d5c31 100644 --- a/community_images/fluent-bit/ironbank/dc_coverage.sh +++ b/community_images/fluent-bit/ironbank/dc_coverage.sh @@ -23,4 +23,4 @@ sleep 10 # log for debugging docker inspect "${CONTAINER_NAME}" docker cp "${SCRIPTPATH}"/scripts/fluent-bit_coverage_script.sh "${CONTAINER_NAME}":/tmp/fluent-bit_coverage_script.sh -docker exec -i ${CONTAINER_NAME} bash -c "bash /tmp/fluent-bit_coverage_script.sh" +docker exec -i "${CONTAINER_NAME}" bash -c "bash /tmp/fluent-bit_coverage_script.sh" From 45ef42831a998fc53499c34f3a287fc9bd4092d0 Mon Sep 17 00:00:00 2001 From: Rajat Date: Wed, 13 Dec 2023 03:20:45 -0800 Subject: [PATCH 03/12] removing error Signed-off-by: Rajat --- community_images/fluent-bit/bitnami/coverage.sh | 3 ++- community_images/fluent-bit/ironbank/coverage.sh | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/community_images/fluent-bit/bitnami/coverage.sh b/community_images/fluent-bit/bitnami/coverage.sh index d3ccfd4b2e..457ed54761 100644 --- a/community_images/fluent-bit/bitnami/coverage.sh +++ b/community_images/fluent-bit/bitnami/coverage.sh @@ -4,4 +4,5 @@ function test_fluent-bit() { CONTAINER_NAME=$1 CMD="docker exec -i ${CONTAINER_NAME} bash -c /tmp/fluent-bit_coverage_script.sh" - $CMD \ No newline at end of file + $CMD +} \ No newline at end of file diff --git a/community_images/fluent-bit/ironbank/coverage.sh b/community_images/fluent-bit/ironbank/coverage.sh index d3ccfd4b2e..457ed54761 100644 --- a/community_images/fluent-bit/ironbank/coverage.sh +++ b/community_images/fluent-bit/ironbank/coverage.sh @@ -4,4 +4,5 @@ function test_fluent-bit() { CONTAINER_NAME=$1 CMD="docker exec -i ${CONTAINER_NAME} bash -c /tmp/fluent-bit_coverage_script.sh" - $CMD \ No newline at end of file + $CMD +} \ No newline at end of file From 1dc40305380e01fcdfee32599d4f6b3089c5e023 Mon Sep 17 00:00:00 2001 From: Rajat Date: Wed, 13 Dec 2023 03:23:06 -0800 Subject: [PATCH 04/12] error fix Signed-off-by: Rajat --- community_images/fluent-bit/bitnami/coverage.sh | 2 +- community_images/fluent-bit/ironbank/coverage.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/community_images/fluent-bit/bitnami/coverage.sh b/community_images/fluent-bit/bitnami/coverage.sh index 457ed54761..79074e5b89 100644 --- a/community_images/fluent-bit/bitnami/coverage.sh +++ b/community_images/fluent-bit/bitnami/coverage.sh @@ -3,6 +3,6 @@ function test_fluent-bit() { CONTAINER_NAME=$1 - CMD="docker exec -i ${CONTAINER_NAME} bash -c /tmp/fluent-bit_coverage_script.sh" + CMD="docker exec -i "${CONTAINER_NAME}" bash -c /tmp/fluent-bit_coverage_script.sh" $CMD } \ No newline at end of file diff --git a/community_images/fluent-bit/ironbank/coverage.sh b/community_images/fluent-bit/ironbank/coverage.sh index 457ed54761..79074e5b89 100644 --- a/community_images/fluent-bit/ironbank/coverage.sh +++ b/community_images/fluent-bit/ironbank/coverage.sh @@ -3,6 +3,6 @@ function test_fluent-bit() { CONTAINER_NAME=$1 - CMD="docker exec -i ${CONTAINER_NAME} bash -c /tmp/fluent-bit_coverage_script.sh" + CMD="docker exec -i "${CONTAINER_NAME}" bash -c /tmp/fluent-bit_coverage_script.sh" $CMD } \ No newline at end of file From 1dc63b0818de432a6bfe32e17507bafc6408770a Mon Sep 17 00:00:00 2001 From: Rajat Date: Wed, 13 Dec 2023 04:55:56 -0800 Subject: [PATCH 05/12] correction in coverage.sh Signed-off-by: Rajat --- community_images/fluent-bit/bitnami/coverage.sh | 2 +- community_images/fluent-bit/bitnami/k8s_coverage.sh | 2 +- community_images/fluent-bit/ironbank/coverage.sh | 2 +- community_images/fluent-bit/ironbank/k8s_coverage.sh | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/community_images/fluent-bit/bitnami/coverage.sh b/community_images/fluent-bit/bitnami/coverage.sh index 79074e5b89..457ed54761 100644 --- a/community_images/fluent-bit/bitnami/coverage.sh +++ b/community_images/fluent-bit/bitnami/coverage.sh @@ -3,6 +3,6 @@ function test_fluent-bit() { CONTAINER_NAME=$1 - CMD="docker exec -i "${CONTAINER_NAME}" bash -c /tmp/fluent-bit_coverage_script.sh" + CMD="docker exec -i ${CONTAINER_NAME} bash -c /tmp/fluent-bit_coverage_script.sh" $CMD } \ No newline at end of file diff --git a/community_images/fluent-bit/bitnami/k8s_coverage.sh b/community_images/fluent-bit/bitnami/k8s_coverage.sh index 4230d2361c..778db20055 100644 --- a/community_images/fluent-bit/bitnami/k8s_coverage.sh +++ b/community_images/fluent-bit/bitnami/k8s_coverage.sh @@ -11,7 +11,7 @@ SCRIPTPATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" JSON_PARAMS="$1" -JSON=$(cat "$JSON_PARAMS") +#JSON=$(cat "$JSON_PARAMS") NAMESPACE=$(jq -r '.namespace_name' < "$JSON_PARAMS") RELEASE_NAME=$(jq -r '.release_name' < "$JSON_PARAMS") diff --git a/community_images/fluent-bit/ironbank/coverage.sh b/community_images/fluent-bit/ironbank/coverage.sh index 79074e5b89..457ed54761 100644 --- a/community_images/fluent-bit/ironbank/coverage.sh +++ b/community_images/fluent-bit/ironbank/coverage.sh @@ -3,6 +3,6 @@ function test_fluent-bit() { CONTAINER_NAME=$1 - CMD="docker exec -i "${CONTAINER_NAME}" bash -c /tmp/fluent-bit_coverage_script.sh" + CMD="docker exec -i ${CONTAINER_NAME} bash -c /tmp/fluent-bit_coverage_script.sh" $CMD } \ No newline at end of file diff --git a/community_images/fluent-bit/ironbank/k8s_coverage.sh b/community_images/fluent-bit/ironbank/k8s_coverage.sh index 33493b146b..ac54de47ad 100644 --- a/community_images/fluent-bit/ironbank/k8s_coverage.sh +++ b/community_images/fluent-bit/ironbank/k8s_coverage.sh @@ -12,7 +12,7 @@ SCRIPTPATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" JSON_PARAMS="$1" -JSON=$(cat "$JSON_PARAMS") +#JSON=$(cat "$JSON_PARAMS") NAMESPACE=$(jq -r '.namespace_name' < "$JSON_PARAMS") RELEASE_NAME=$(jq -r '.release_name' < "$JSON_PARAMS") From 140af36a5f5635b2a540b2baf9d57b73f2deccef Mon Sep 17 00:00:00 2001 From: Rajat Date: Wed, 13 Dec 2023 09:35:34 -0800 Subject: [PATCH 06/12] fixing runtime issue Signed-off-by: Rajat --- .../bitnami/config/fluent-bit.config | 93 +++++++++++++++++ .../fluent-bit/bitnami/dc_coverage.sh | 2 + .../fluent-bit/bitnami/k8s_coverage.sh | 3 +- .../scripts/fluent-bit_coverage_script.sh | 2 +- .../ironbank/config/fluent-bit.config | 93 +++++++++++++++++ .../fluent-bit/ironbank/config/fluent-bit.yml | 99 ------------------- .../fluent-bit/ironbank/dc_coverage.sh | 1 + .../fluent-bit/ironbank/k8s_coverage.sh | 2 + .../scripts/fluent-bit_coverage_script.sh | 2 +- 9 files changed, 195 insertions(+), 102 deletions(-) create mode 100644 community_images/fluent-bit/bitnami/config/fluent-bit.config create mode 100644 community_images/fluent-bit/ironbank/config/fluent-bit.config delete mode 100644 community_images/fluent-bit/ironbank/config/fluent-bit.yml diff --git a/community_images/fluent-bit/bitnami/config/fluent-bit.config b/community_images/fluent-bit/bitnami/config/fluent-bit.config new file mode 100644 index 0000000000..5d3a9deddc --- /dev/null +++ b/community_images/fluent-bit/bitnami/config/fluent-bit.config @@ -0,0 +1,93 @@ +[SERVICE] + # Flush + # ===== + # set an interval of seconds before to flush records to a destination + flush 1 + + # Daemon + # ====== + # instruct Fluent Bit to run in foreground or background mode. + daemon Off + + # Log_Level + # ========= + # Set the verbosity level of the service, values can be: + # + # - error + # - warning + # - info + # - debug + # - trace + # + # by default 'info' is set, that means it includes 'error' and 'warning'. + log_level info + + # Parsers File + # ============ + # specify an optional 'Parsers' configuration file + #parsers_file parsers.conf + + # Plugins File + # ============ + # specify an optional 'Plugins' configuration file to load external plugins. + #plugins_file plugins.conf + + # HTTP Server + # =========== + # Enable/Disable the built-in HTTP Server for metrics + http_server Off + http_listen 0.0.0.0 + http_port 2020 + + # Storage + # ======= + # Fluent Bit can use memory and filesystem buffering based mechanisms + # + # - https://docs.fluentbit.io/manual/administration/buffering-and-storage + # + # storage metrics + # --------------- + # publish storage pipeline metrics in '/api/v1/storage'. The metrics are + # exported only if the 'http_server' option is enabled. + # + storage.metrics on + + # storage.path + # ------------ + # absolute file system path to store filesystem data buffers (chunks). + # + # storage.path /tmp/storage + + # storage.sync + # ------------ + # configure the synchronization mode used to store the data into the + # filesystem. It can take the values normal or full. + # + # storage.sync normal + + # storage.checksum + # ---------------- + # enable the data integrity check when writing and reading data from the + # filesystem. The storage layer uses the CRC32 algorithm. + # + # storage.checksum off + + # storage.backlog.mem_limit + # ------------------------- + # if storage.path is set, Fluent Bit will look for data chunks that were + # not delivered and are still in the storage layer, these are called + # backlog data. This option configure a hint of maximum value of memory + # to use when processing these records. + # + # storage.backlog.mem_limit 5M + +[INPUT] + name cpu + tag cpu.local + + # Read interval (sec) Default: 1 + interval_sec 1 + +[OUTPUT] + name stdout + match * diff --git a/community_images/fluent-bit/bitnami/dc_coverage.sh b/community_images/fluent-bit/bitnami/dc_coverage.sh index cfbbba3a97..6c0bb3f9fb 100644 --- a/community_images/fluent-bit/bitnami/dc_coverage.sh +++ b/community_images/fluent-bit/bitnami/dc_coverage.sh @@ -19,5 +19,7 @@ sleep 10 # log for debugging docker inspect "${CONTAINER_NAME}" +docker cp "${SCRIPTPATH}"/config/fluent-bit.config "${CONTAINER_NAME}":/tmp/fluent-bit.config +sleep 10 docker cp "${SCRIPTPATH}"/scripts/fluent-bit_coverage_script.sh "${CONTAINER_NAME}":/tmp/fluent-bit_coverage_script.sh docker exec -i "${CONTAINER_NAME}" bash -c "bash /tmp/fluent-bit_coverage_script.sh" \ No newline at end of file diff --git a/community_images/fluent-bit/bitnami/k8s_coverage.sh b/community_images/fluent-bit/bitnami/k8s_coverage.sh index 778db20055..694fffe740 100644 --- a/community_images/fluent-bit/bitnami/k8s_coverage.sh +++ b/community_images/fluent-bit/bitnami/k8s_coverage.sh @@ -17,7 +17,8 @@ RELEASE_NAME=$(jq -r '.release_name' < "$JSON_PARAMS") sleep 60 CONTAINER_NAME="${RELEASE_NAME}-0" - +kubectl cp "${SCRIPTPATH}"/config/fluent-bit.config "${CONTAINER_NAME}":/tmp/fluent-bit.config -n "${NAMESPACE}" +sleep 10 # copy over the script to the pod kubectl cp "${SCRIPTPATH}"/scripts/fluent-bit_coverage_script.sh "${CONTAINER_NAME}":/tmp/cvoverage_script.sh -n "${NAMESPACE}" diff --git a/community_images/fluent-bit/bitnami/scripts/fluent-bit_coverage_script.sh b/community_images/fluent-bit/bitnami/scripts/fluent-bit_coverage_script.sh index f7f73b0f06..0861b1ab84 100644 --- a/community_images/fluent-bit/bitnami/scripts/fluent-bit_coverage_script.sh +++ b/community_images/fluent-bit/bitnami/scripts/fluent-bit_coverage_script.sh @@ -4,7 +4,7 @@ set -e set -x # Start Fluent Bit with the specified configuration file -/opt/bitnami/fluent-bit/bin/fluent-bit -c /opt/bitnami/fluent-bit/conf/fluent-bit.conf & +/opt/bitnami/fluent-bit/bin/fluent-bit -c /tmp/fluent-bit.conf & # Add a delay to allow Fluent Bit to run (you can adjust this as needed) # Capture the PID of the last background process diff --git a/community_images/fluent-bit/ironbank/config/fluent-bit.config b/community_images/fluent-bit/ironbank/config/fluent-bit.config new file mode 100644 index 0000000000..5d3a9deddc --- /dev/null +++ b/community_images/fluent-bit/ironbank/config/fluent-bit.config @@ -0,0 +1,93 @@ +[SERVICE] + # Flush + # ===== + # set an interval of seconds before to flush records to a destination + flush 1 + + # Daemon + # ====== + # instruct Fluent Bit to run in foreground or background mode. + daemon Off + + # Log_Level + # ========= + # Set the verbosity level of the service, values can be: + # + # - error + # - warning + # - info + # - debug + # - trace + # + # by default 'info' is set, that means it includes 'error' and 'warning'. + log_level info + + # Parsers File + # ============ + # specify an optional 'Parsers' configuration file + #parsers_file parsers.conf + + # Plugins File + # ============ + # specify an optional 'Plugins' configuration file to load external plugins. + #plugins_file plugins.conf + + # HTTP Server + # =========== + # Enable/Disable the built-in HTTP Server for metrics + http_server Off + http_listen 0.0.0.0 + http_port 2020 + + # Storage + # ======= + # Fluent Bit can use memory and filesystem buffering based mechanisms + # + # - https://docs.fluentbit.io/manual/administration/buffering-and-storage + # + # storage metrics + # --------------- + # publish storage pipeline metrics in '/api/v1/storage'. The metrics are + # exported only if the 'http_server' option is enabled. + # + storage.metrics on + + # storage.path + # ------------ + # absolute file system path to store filesystem data buffers (chunks). + # + # storage.path /tmp/storage + + # storage.sync + # ------------ + # configure the synchronization mode used to store the data into the + # filesystem. It can take the values normal or full. + # + # storage.sync normal + + # storage.checksum + # ---------------- + # enable the data integrity check when writing and reading data from the + # filesystem. The storage layer uses the CRC32 algorithm. + # + # storage.checksum off + + # storage.backlog.mem_limit + # ------------------------- + # if storage.path is set, Fluent Bit will look for data chunks that were + # not delivered and are still in the storage layer, these are called + # backlog data. This option configure a hint of maximum value of memory + # to use when processing these records. + # + # storage.backlog.mem_limit 5M + +[INPUT] + name cpu + tag cpu.local + + # Read interval (sec) Default: 1 + interval_sec 1 + +[OUTPUT] + name stdout + match * diff --git a/community_images/fluent-bit/ironbank/config/fluent-bit.yml b/community_images/fluent-bit/ironbank/config/fluent-bit.yml deleted file mode 100644 index 510d809591..0000000000 --- a/community_images/fluent-bit/ironbank/config/fluent-bit.yml +++ /dev/null @@ -1,99 +0,0 @@ -service: - # Flush - # ===== - # set an interval of seconds before to flush records to a destination - flush: 1 - - # Daemon - # ====== - # instruct Fluent Bit to run in foreground or background mode. - daemon: Off - - # Log_Level - # ========= - # Set the verbosity level of the service, values can be: - # - # - error - # - warning - # - info - # - debug - # - trace - # - # by default 'info' is set, that means it includes 'error' and 'warning'. - log_level: info - - # Parsers File - # ============ - # specify an optional 'Parsers' configuration file - parsers_file: parsers.conf - - # Plugins File - # ============ - # specify an optional 'Plugins' configuration file to load external plugins. - plugins_file: plugins.conf - - # HTTP Server - # =========== - # Enable/Disable the built-in HTTP Server for metrics - http_server: Off - http_listen: 0.0.0.0 - http_port: 2020 - - # Storage - # ======= - # Fluent Bit can use memory and filesystem buffering based mechanisms - # - # - https://docs.fluentbit.io/manual/administration/buffering-and-storage - # - # storage metrics - # --------------- - # publish storage pipeline metrics in '/api/v1/storage'. The metrics are - # exported only if the 'http_server' option is enabled. - # - storage: - metrics: on - - # storage.path - # ------------ - # absolute file system path to store filesystem data buffers (chunks). - # - # storage.path /tmp/storage - path: /tmp/storage - - # storage.sync - # ------------ - # configure the synchronization mode used to store the data into the - # filesystem. It can take the values normal or full. - # - # storage.sync normal - sync: normal - - # storage.checksum - # ---------------- - # enable the data integrity check when writing and reading data from the - # filesystem. The storage layer uses the CRC32 algorithm. - # - # storage.checksum off - checksum: off - - # storage.backlog.mem_limit - # ------------------------- - # if storage.path is set, Fluent Bit will look for data chunks that were - # not delivered and are still in the storage layer, these are called - # backlog data. This option configure a hint of maximum value of memory - # to use when processing these records. - # - # storage.backlog.mem_limit 5M - backlog: - mem_limit: 5M - -input: - name: cpu - tag: cpu.local - - # Read interval (sec) Default: 1 - interval_sec: 1 - -output: - name: stdout - match: "*" \ No newline at end of file diff --git a/community_images/fluent-bit/ironbank/dc_coverage.sh b/community_images/fluent-bit/ironbank/dc_coverage.sh index b6296d5c31..b7255c0b95 100644 --- a/community_images/fluent-bit/ironbank/dc_coverage.sh +++ b/community_images/fluent-bit/ironbank/dc_coverage.sh @@ -23,4 +23,5 @@ sleep 10 # log for debugging docker inspect "${CONTAINER_NAME}" docker cp "${SCRIPTPATH}"/scripts/fluent-bit_coverage_script.sh "${CONTAINER_NAME}":/tmp/fluent-bit_coverage_script.sh +sleep 10 docker exec -i "${CONTAINER_NAME}" bash -c "bash /tmp/fluent-bit_coverage_script.sh" diff --git a/community_images/fluent-bit/ironbank/k8s_coverage.sh b/community_images/fluent-bit/ironbank/k8s_coverage.sh index ac54de47ad..144fffbd8b 100644 --- a/community_images/fluent-bit/ironbank/k8s_coverage.sh +++ b/community_images/fluent-bit/ironbank/k8s_coverage.sh @@ -22,6 +22,8 @@ echo "RELEASE_NAME: $RELEASE_NAME" sleep 60 CONTAINER_NAME="${RELEASE_NAME}-0" # copy over the script to the pod +kubectl cp "${SCRIPTPATH}"/config/fluent-bit.config "${CONTAINER_NAME}":/tmp/fluent-bit.config -n "${NAMESPACE}" +sleep 10 kubectl cp "${SCRIPTPATH}"/scripts/fluent-bit_coverage_script.sh "${CONTAINER_NAME}":/tmp/fluent-bit_cvoverage_script.sh -n "${NAMESPACE}" test_fluent-bit "${CONTAINER_NAME}" "${NAMESPACE}" "yes" \ No newline at end of file diff --git a/community_images/fluent-bit/ironbank/scripts/fluent-bit_coverage_script.sh b/community_images/fluent-bit/ironbank/scripts/fluent-bit_coverage_script.sh index a724f5ecf2..acad4eed2d 100644 --- a/community_images/fluent-bit/ironbank/scripts/fluent-bit_coverage_script.sh +++ b/community_images/fluent-bit/ironbank/scripts/fluent-bit_coverage_script.sh @@ -4,7 +4,7 @@ set -e set -x # Start Fluent Bit with the specified configuration file -/fluent-bit/bin/fluent-bit -c /fluent-bit/etc/fluent-bit.conf & +/fluent-bit/bin/fluent-bit -c /tmp/fluent-bit.conf & # Capture the PID of the last background process From 0316ab13d5883042b5645606cf0e259260a5c0c3 Mon Sep 17 00:00:00 2001 From: Rajat Date: Wed, 13 Dec 2023 10:39:42 -0800 Subject: [PATCH 07/12] fixing bitnami Signed-off-by: Rajat --- .../fluent-bit/bitnami/config/fluent-bit.config | 10 +--------- .../fluent-bit/bitnami/config/plugins.config | 2 ++ community_images/fluent-bit/bitnami/dc_coverage.sh | 2 ++ community_images/fluent-bit/bitnami/docker-compose.yml | 4 +--- community_images/fluent-bit/bitnami/k8s_coverage.sh | 0 5 files changed, 6 insertions(+), 12 deletions(-) create mode 100644 community_images/fluent-bit/bitnami/config/plugins.config mode change 100644 => 100755 community_images/fluent-bit/bitnami/dc_coverage.sh mode change 100644 => 100755 community_images/fluent-bit/bitnami/k8s_coverage.sh diff --git a/community_images/fluent-bit/bitnami/config/fluent-bit.config b/community_images/fluent-bit/bitnami/config/fluent-bit.config index 5d3a9deddc..45c2002402 100644 --- a/community_images/fluent-bit/bitnami/config/fluent-bit.config +++ b/community_images/fluent-bit/bitnami/config/fluent-bit.config @@ -22,15 +22,7 @@ # by default 'info' is set, that means it includes 'error' and 'warning'. log_level info - # Parsers File - # ============ - # specify an optional 'Parsers' configuration file - #parsers_file parsers.conf - - # Plugins File - # ============ - # specify an optional 'Plugins' configuration file to load external plugins. - #plugins_file plugins.conf + # HTTP Server # =========== diff --git a/community_images/fluent-bit/bitnami/config/plugins.config b/community_images/fluent-bit/bitnami/config/plugins.config new file mode 100644 index 0000000000..55c5a8d395 --- /dev/null +++ b/community_images/fluent-bit/bitnami/config/plugins.config @@ -0,0 +1,2 @@ +[PLUGINS] + # Path /path/to/out_gstdout.so diff --git a/community_images/fluent-bit/bitnami/dc_coverage.sh b/community_images/fluent-bit/bitnami/dc_coverage.sh old mode 100644 new mode 100755 index 6c0bb3f9fb..6aaf37e89a --- a/community_images/fluent-bit/bitnami/dc_coverage.sh +++ b/community_images/fluent-bit/bitnami/dc_coverage.sh @@ -19,6 +19,8 @@ sleep 10 # log for debugging docker inspect "${CONTAINER_NAME}" +docker cp "${SCRIPTPATH}"/config/plugins.config "${CONTAINER_NAME}":/opt/bitnami/fluent-bit/conf/plugins.conf +sleep 5 docker cp "${SCRIPTPATH}"/config/fluent-bit.config "${CONTAINER_NAME}":/tmp/fluent-bit.config sleep 10 docker cp "${SCRIPTPATH}"/scripts/fluent-bit_coverage_script.sh "${CONTAINER_NAME}":/tmp/fluent-bit_coverage_script.sh diff --git a/community_images/fluent-bit/bitnami/docker-compose.yml b/community_images/fluent-bit/bitnami/docker-compose.yml index 8a86804a84..35180c4d3a 100644 --- a/community_images/fluent-bit/bitnami/docker-compose.yml +++ b/community_images/fluent-bit/bitnami/docker-compose.yml @@ -3,12 +3,10 @@ version: '2' services: fluent-bit: image: ${FLUENT_BIT_IMAGE_REPOSITORY}:${FLUENT_BIT_IMAGE_TAG} - volumes: - - ./scripts/fluent-bit_coverage_script.sh:/tmp/coverage_script.sh cap_add: - SYS_PTRACE ports: - - '2020:2020' + - '8020:8020' prometheus: image: docker.io/rapidfort/prometheus diff --git a/community_images/fluent-bit/bitnami/k8s_coverage.sh b/community_images/fluent-bit/bitnami/k8s_coverage.sh old mode 100644 new mode 100755 From 406cb5f1321a8bb7782c5253b1157220df1aa984 Mon Sep 17 00:00:00 2001 From: Rajat Date: Wed, 13 Dec 2023 12:55:12 -0800 Subject: [PATCH 08/12] fixing execution Signed-off-by: Rajat --- community_images/fluent-bit/bitnami/dc_coverage.sh | 8 +------- community_images/fluent-bit/bitnami/docker-compose.yml | 2 +- community_images/fluent-bit/bitnami/k8s_coverage.sh | 6 +++--- .../fluent-bit/ironbank/config/plugins.config | 2 ++ community_images/fluent-bit/ironbank/dc_coverage.sh | 4 +--- community_images/fluent-bit/ironbank/docker-compose.yml | 4 +--- community_images/fluent-bit/ironbank/k8s_coverage.sh | 5 ++--- 7 files changed, 11 insertions(+), 20 deletions(-) create mode 100644 community_images/fluent-bit/ironbank/config/plugins.config mode change 100644 => 100755 community_images/fluent-bit/ironbank/dc_coverage.sh mode change 100644 => 100755 community_images/fluent-bit/ironbank/docker-compose.yml mode change 100644 => 100755 community_images/fluent-bit/ironbank/k8s_coverage.sh diff --git a/community_images/fluent-bit/bitnami/dc_coverage.sh b/community_images/fluent-bit/bitnami/dc_coverage.sh index 6aaf37e89a..bd3bab15e4 100755 --- a/community_images/fluent-bit/bitnami/dc_coverage.sh +++ b/community_images/fluent-bit/bitnami/dc_coverage.sh @@ -16,12 +16,6 @@ CONTAINER_NAME="${PROJECT_NAME}"-fluent-bit-1 # Wait sleep 10 - +docker cp "${SCRIPTPATH}"/config/plugins.config "${CONTAINER_NAME}":/opt/bitnami/fluent-bit/conf/plugins.conf # log for debugging docker inspect "${CONTAINER_NAME}" -docker cp "${SCRIPTPATH}"/config/plugins.config "${CONTAINER_NAME}":/opt/bitnami/fluent-bit/conf/plugins.conf -sleep 5 -docker cp "${SCRIPTPATH}"/config/fluent-bit.config "${CONTAINER_NAME}":/tmp/fluent-bit.config -sleep 10 -docker cp "${SCRIPTPATH}"/scripts/fluent-bit_coverage_script.sh "${CONTAINER_NAME}":/tmp/fluent-bit_coverage_script.sh -docker exec -i "${CONTAINER_NAME}" bash -c "bash /tmp/fluent-bit_coverage_script.sh" \ No newline at end of file diff --git a/community_images/fluent-bit/bitnami/docker-compose.yml b/community_images/fluent-bit/bitnami/docker-compose.yml index 35180c4d3a..27e823fa24 100644 --- a/community_images/fluent-bit/bitnami/docker-compose.yml +++ b/community_images/fluent-bit/bitnami/docker-compose.yml @@ -6,7 +6,7 @@ services: cap_add: - SYS_PTRACE ports: - - '8020:8020' + - '2020:2020' prometheus: image: docker.io/rapidfort/prometheus diff --git a/community_images/fluent-bit/bitnami/k8s_coverage.sh b/community_images/fluent-bit/bitnami/k8s_coverage.sh index 694fffe740..64ea45c983 100755 --- a/community_images/fluent-bit/bitnami/k8s_coverage.sh +++ b/community_images/fluent-bit/bitnami/k8s_coverage.sh @@ -15,11 +15,11 @@ JSON_PARAMS="$1" NAMESPACE=$(jq -r '.namespace_name' < "$JSON_PARAMS") RELEASE_NAME=$(jq -r '.release_name' < "$JSON_PARAMS") -sleep 60 -CONTAINER_NAME="${RELEASE_NAME}-0" +sleep 10 +CONTAINER_NAME=$(kubectl get pods -n "$NAMESPACE" -l "app.kubernetes.io/instance=$RELEASE_NAME" -o jsonpath='{.items[0].metadata.name}') + kubectl cp "${SCRIPTPATH}"/config/fluent-bit.config "${CONTAINER_NAME}":/tmp/fluent-bit.config -n "${NAMESPACE}" sleep 10 # copy over the script to the pod kubectl cp "${SCRIPTPATH}"/scripts/fluent-bit_coverage_script.sh "${CONTAINER_NAME}":/tmp/cvoverage_script.sh -n "${NAMESPACE}" -test_fluent-bit "${CONTAINER_NAME}" "${NAMESPACE}" "yes" \ No newline at end of file diff --git a/community_images/fluent-bit/ironbank/config/plugins.config b/community_images/fluent-bit/ironbank/config/plugins.config new file mode 100644 index 0000000000..55c5a8d395 --- /dev/null +++ b/community_images/fluent-bit/ironbank/config/plugins.config @@ -0,0 +1,2 @@ +[PLUGINS] + # Path /path/to/out_gstdout.so diff --git a/community_images/fluent-bit/ironbank/dc_coverage.sh b/community_images/fluent-bit/ironbank/dc_coverage.sh old mode 100644 new mode 100755 index b7255c0b95..61e03564d7 --- a/community_images/fluent-bit/ironbank/dc_coverage.sh +++ b/community_images/fluent-bit/ironbank/dc_coverage.sh @@ -22,6 +22,4 @@ sleep 10 # log for debugging docker inspect "${CONTAINER_NAME}" -docker cp "${SCRIPTPATH}"/scripts/fluent-bit_coverage_script.sh "${CONTAINER_NAME}":/tmp/fluent-bit_coverage_script.sh -sleep 10 -docker exec -i "${CONTAINER_NAME}" bash -c "bash /tmp/fluent-bit_coverage_script.sh" + diff --git a/community_images/fluent-bit/ironbank/docker-compose.yml b/community_images/fluent-bit/ironbank/docker-compose.yml old mode 100644 new mode 100755 index ad77a4882b..edb36d6c07 --- a/community_images/fluent-bit/ironbank/docker-compose.yml +++ b/community_images/fluent-bit/ironbank/docker-compose.yml @@ -2,10 +2,8 @@ version: '2' services: fluent-bit: - # image: 'registry1.dso.mil/ironbank/opensource/fluent/fluent-bit' image: ${FLUENT_BIT_IMAGE_REPOSITORY}:${FLUENT_BIT_IMAGE_TAG} - volumes: - - ./scripts/fluent-bit_coverage_script.sh:/tmp/coverage_script.sh + cap_add: - SYS_PTRACE ports: diff --git a/community_images/fluent-bit/ironbank/k8s_coverage.sh b/community_images/fluent-bit/ironbank/k8s_coverage.sh old mode 100644 new mode 100755 index 144fffbd8b..98a2824903 --- a/community_images/fluent-bit/ironbank/k8s_coverage.sh +++ b/community_images/fluent-bit/ironbank/k8s_coverage.sh @@ -19,11 +19,10 @@ RELEASE_NAME=$(jq -r '.release_name' < "$JSON_PARAMS") echo "NAMESPACE: $NAMESPACE" echo "RELEASE_NAME: $RELEASE_NAME" -sleep 60 -CONTAINER_NAME="${RELEASE_NAME}-0" +sleep 10 +CONTAINER_NAME=$(kubectl get pods -n "$NAMESPACE" -l "app.kubernetes.io/instance=$RELEASE_NAME" -o jsonpath='{.items[0].metadata.name}') # copy over the script to the pod kubectl cp "${SCRIPTPATH}"/config/fluent-bit.config "${CONTAINER_NAME}":/tmp/fluent-bit.config -n "${NAMESPACE}" sleep 10 kubectl cp "${SCRIPTPATH}"/scripts/fluent-bit_coverage_script.sh "${CONTAINER_NAME}":/tmp/fluent-bit_cvoverage_script.sh -n "${NAMESPACE}" -test_fluent-bit "${CONTAINER_NAME}" "${NAMESPACE}" "yes" \ No newline at end of file From 2f2d29673e80c5632d83df9099300af29456f711 Mon Sep 17 00:00:00 2001 From: Rajat Date: Thu, 14 Dec 2023 08:23:21 -0800 Subject: [PATCH 09/12] correction in fluent-bit/bitnami Signed-off-by: Rajat --- .../bitnami/config/fluent-bit.config | 8 +++---- .../fluent-bit/bitnami/coverage.sh | 8 ------- .../fluent-bit/bitnami/dc_coverage.sh | 15 +++++++++---- .../fluent-bit/bitnami/docker-compose.yml | 6 +++-- community_images/fluent-bit/bitnami/image.yml | 3 --- .../fluent-bit/bitnami/k8s_coverage.sh | 15 ++++++------- .../scripts/fluent-bit_coverage_script.sh | 22 ------------------- 7 files changed, 26 insertions(+), 51 deletions(-) delete mode 100644 community_images/fluent-bit/bitnami/coverage.sh delete mode 100644 community_images/fluent-bit/bitnami/scripts/fluent-bit_coverage_script.sh diff --git a/community_images/fluent-bit/bitnami/config/fluent-bit.config b/community_images/fluent-bit/bitnami/config/fluent-bit.config index 45c2002402..a445f89971 100644 --- a/community_images/fluent-bit/bitnami/config/fluent-bit.config +++ b/community_images/fluent-bit/bitnami/config/fluent-bit.config @@ -48,21 +48,21 @@ # ------------ # absolute file system path to store filesystem data buffers (chunks). # - # storage.path /tmp/storage + storage.path /tmp/storage # storage.sync # ------------ # configure the synchronization mode used to store the data into the # filesystem. It can take the values normal or full. # - # storage.sync normal + storage.sync normal # storage.checksum # ---------------- # enable the data integrity check when writing and reading data from the # filesystem. The storage layer uses the CRC32 algorithm. # - # storage.checksum off + storage.checksum off # storage.backlog.mem_limit # ------------------------- @@ -71,7 +71,7 @@ # backlog data. This option configure a hint of maximum value of memory # to use when processing these records. # - # storage.backlog.mem_limit 5M + storage.backlog.mem_limit 5M [INPUT] name cpu diff --git a/community_images/fluent-bit/bitnami/coverage.sh b/community_images/fluent-bit/bitnami/coverage.sh deleted file mode 100644 index 457ed54761..0000000000 --- a/community_images/fluent-bit/bitnami/coverage.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -function test_fluent-bit() { - CONTAINER_NAME=$1 - - CMD="docker exec -i ${CONTAINER_NAME} bash -c /tmp/fluent-bit_coverage_script.sh" - $CMD -} \ No newline at end of file diff --git a/community_images/fluent-bit/bitnami/dc_coverage.sh b/community_images/fluent-bit/bitnami/dc_coverage.sh index bd3bab15e4..6478a8c524 100755 --- a/community_images/fluent-bit/bitnami/dc_coverage.sh +++ b/community_images/fluent-bit/bitnami/dc_coverage.sh @@ -9,13 +9,20 @@ SCRIPTPATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" . "${SCRIPTPATH}"/../../common/scripts/bash_helper.sh JSON_PARAMS="$1" - - PROJECT_NAME=$(jq -r '.project_name' < "$JSON_PARAMS") CONTAINER_NAME="${PROJECT_NAME}"-fluent-bit-1 - # Wait sleep 10 -docker cp "${SCRIPTPATH}"/config/plugins.config "${CONTAINER_NAME}":/opt/bitnami/fluent-bit/conf/plugins.conf # log for debugging docker inspect "${CONTAINER_NAME}" +docker exec -d "${CONTAINER_NAME}" /bin/bash -c "nohup /opt/bitnami/fluent-bit/bin/fluent-bit -c /tmp/fluent-bit.conf > /opt/bitnami/fluent-bit/logs/fluent-bit.log 2>&1 " & + +sleep 10 +# Get the PID of the last background process +DOCKER_PID=$(docker exec "${CONTAINER_NAME}" /bin/bash -c "pgrep -o fluent-bit") +# Check if the process is still running and terminate it if needed +if [ -n "${DOCKER_PID}" ]; then + echo "Fluent Bit process is still running. Terminating..." + docker exec "${CONTAINER_NAME}" kill "${DOCKER_PID}" +fi + diff --git a/community_images/fluent-bit/bitnami/docker-compose.yml b/community_images/fluent-bit/bitnami/docker-compose.yml index 27e823fa24..65988ecd9e 100644 --- a/community_images/fluent-bit/bitnami/docker-compose.yml +++ b/community_images/fluent-bit/bitnami/docker-compose.yml @@ -6,8 +6,10 @@ services: cap_add: - SYS_PTRACE ports: - - '2020:2020' - + - '0.0.0.0::8080' + volumes: + - ./config/fluent-bit.config:/tmp/fluent-bit.config + - ./config/plugin.config:/opt/bitnami/fluent-bit/conf/plugin.config prometheus: image: docker.io/rapidfort/prometheus cap_add: diff --git a/community_images/fluent-bit/bitnami/image.yml b/community_images/fluent-bit/bitnami/image.yml index f7aeb67219..4d7faafae1 100644 --- a/community_images/fluent-bit/bitnami/image.yml +++ b/community_images/fluent-bit/bitnami/image.yml @@ -23,7 +23,6 @@ input_registry: repo_sets: - fluent-bit: input_base_tag: "2.2." - runtimes: - type: docker_compose script: dc_coverage.sh @@ -32,8 +31,6 @@ runtimes: fluent-bit: repository: "FLUENT_BIT_IMAGE_REPOSITORY" tag: "FLUENT_BIT_IMAGE_TAG" - - - type: k8s script: k8s_coverage.sh helm: diff --git a/community_images/fluent-bit/bitnami/k8s_coverage.sh b/community_images/fluent-bit/bitnami/k8s_coverage.sh index 64ea45c983..31e32c2e60 100755 --- a/community_images/fluent-bit/bitnami/k8s_coverage.sh +++ b/community_images/fluent-bit/bitnami/k8s_coverage.sh @@ -10,16 +10,15 @@ SCRIPTPATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" . "${SCRIPTPATH}"/../../common/scripts/bash_helper.sh JSON_PARAMS="$1" - #JSON=$(cat "$JSON_PARAMS") NAMESPACE=$(jq -r '.namespace_name' < "$JSON_PARAMS") RELEASE_NAME=$(jq -r '.release_name' < "$JSON_PARAMS") - -sleep 10 CONTAINER_NAME=$(kubectl get pods -n "$NAMESPACE" -l "app.kubernetes.io/instance=$RELEASE_NAME" -o jsonpath='{.items[0].metadata.name}') - -kubectl cp "${SCRIPTPATH}"/config/fluent-bit.config "${CONTAINER_NAME}":/tmp/fluent-bit.config -n "${NAMESPACE}" -sleep 10 # copy over the script to the pod -kubectl cp "${SCRIPTPATH}"/scripts/fluent-bit_coverage_script.sh "${CONTAINER_NAME}":/tmp/cvoverage_script.sh -n "${NAMESPACE}" - +kubectl exec "${CONTAINER_NAME}" -n "${NAMESPACE}" -- /bin/bash -c "nohup /opt/bitnami/fluent-bit/bin/fluent-bit -c /opt/bitnami/fluent-bit/conf/fluent-bit.conf" & +sleep 10 +# Check if the process is still running and terminate it if needed +if ps -p $! > /dev/null; then + echo "Fluent Bit process is still running. Terminating..." + kill $! +fi \ No newline at end of file diff --git a/community_images/fluent-bit/bitnami/scripts/fluent-bit_coverage_script.sh b/community_images/fluent-bit/bitnami/scripts/fluent-bit_coverage_script.sh deleted file mode 100644 index 0861b1ab84..0000000000 --- a/community_images/fluent-bit/bitnami/scripts/fluent-bit_coverage_script.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash - -set -e -set -x - -# Start Fluent Bit with the specified configuration file -/opt/bitnami/fluent-bit/bin/fluent-bit -c /tmp/fluent-bit.conf & - -# Add a delay to allow Fluent Bit to run (you can adjust this as needed) -# Capture the PID of the last background process -FLUENT_BIT_PID=$! - -# Add a delay to allow Fluent Bit to run (you can adjust this as needed) -sleep 5 - -# Your additional commands or logic can go here - -# Stop Fluent Bit gracefully by sending a termination signal -kill -TERM $FLUENT_BIT_PID - -# Optionally, wait for Fluent Bit to exit -wait $FLUENT_BIT_PID \ No newline at end of file From fae0c409ff22c10b1fb1c8582283b27e912d9817 Mon Sep 17 00:00:00 2001 From: Rajat Date: Thu, 14 Dec 2023 08:53:38 -0800 Subject: [PATCH 10/12] correction for ironbank Signed-off-by: Rajat --- .../ironbank/config/fluent-bit.config | 18 ++++-------------- .../fluent-bit/ironbank/coverage.sh | 8 -------- .../fluent-bit/ironbank/dc_coverage.sh | 11 +++++++++-- .../fluent-bit/ironbank/docker-compose.yml | 7 ++++--- .../fluent-bit/ironbank/image.yml | 2 -- .../fluent-bit/ironbank/k8s_coverage.sh | 10 ++++++---- .../scripts/fluent-bit_coverage_script.sh | 19 ------------------- 7 files changed, 23 insertions(+), 52 deletions(-) delete mode 100644 community_images/fluent-bit/ironbank/coverage.sh mode change 100644 => 100755 community_images/fluent-bit/ironbank/image.yml delete mode 100644 community_images/fluent-bit/ironbank/scripts/fluent-bit_coverage_script.sh diff --git a/community_images/fluent-bit/ironbank/config/fluent-bit.config b/community_images/fluent-bit/ironbank/config/fluent-bit.config index 5d3a9deddc..0a6059cc83 100644 --- a/community_images/fluent-bit/ironbank/config/fluent-bit.config +++ b/community_images/fluent-bit/ironbank/config/fluent-bit.config @@ -22,16 +22,6 @@ # by default 'info' is set, that means it includes 'error' and 'warning'. log_level info - # Parsers File - # ============ - # specify an optional 'Parsers' configuration file - #parsers_file parsers.conf - - # Plugins File - # ============ - # specify an optional 'Plugins' configuration file to load external plugins. - #plugins_file plugins.conf - # HTTP Server # =========== # Enable/Disable the built-in HTTP Server for metrics @@ -56,21 +46,21 @@ # ------------ # absolute file system path to store filesystem data buffers (chunks). # - # storage.path /tmp/storage + storage.path /tmp/storage # storage.sync # ------------ # configure the synchronization mode used to store the data into the # filesystem. It can take the values normal or full. # - # storage.sync normal + storage.sync normal # storage.checksum # ---------------- # enable the data integrity check when writing and reading data from the # filesystem. The storage layer uses the CRC32 algorithm. # - # storage.checksum off + storage.checksum off # storage.backlog.mem_limit # ------------------------- @@ -79,7 +69,7 @@ # backlog data. This option configure a hint of maximum value of memory # to use when processing these records. # - # storage.backlog.mem_limit 5M + storage.backlog.mem_limit 5M [INPUT] name cpu diff --git a/community_images/fluent-bit/ironbank/coverage.sh b/community_images/fluent-bit/ironbank/coverage.sh deleted file mode 100644 index 457ed54761..0000000000 --- a/community_images/fluent-bit/ironbank/coverage.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -function test_fluent-bit() { - CONTAINER_NAME=$1 - - CMD="docker exec -i ${CONTAINER_NAME} bash -c /tmp/fluent-bit_coverage_script.sh" - $CMD -} \ No newline at end of file diff --git a/community_images/fluent-bit/ironbank/dc_coverage.sh b/community_images/fluent-bit/ironbank/dc_coverage.sh index 61e03564d7..d14c3da250 100755 --- a/community_images/fluent-bit/ironbank/dc_coverage.sh +++ b/community_images/fluent-bit/ironbank/dc_coverage.sh @@ -16,10 +16,17 @@ echo "Json params for docker compose coverage = $JSON" PROJECT_NAME=$(jq -r '.project_name' < "$JSON_PARAMS") CONTAINER_NAME="${PROJECT_NAME}"-fluent-bit-1 - # Wait sleep 10 - # log for debugging docker inspect "${CONTAINER_NAME}" +docker exec -d "${CONTAINER_NAME}" /bin/bash -c "nohup /fluent-bit/bin/fluent-bit -c /tmp/fluent-bit.conf" & +sleep 10 +# Get the PID of the last background process +DOCKER_PID=$(docker exec "${CONTAINER_NAME}" ps -e -o pid,cmd | grep '/fluent-bit/bin/fluent-bit' | awk '{print $1}') +# Check if the process is still running and terminate it if needed +if [ -n "${DOCKER_PID}" ]; then + echo "Fluent Bit process is still running. Terminating..." + docker exec "${CONTAINER_NAME}" kill "${DOCKER_PID}" +fi diff --git a/community_images/fluent-bit/ironbank/docker-compose.yml b/community_images/fluent-bit/ironbank/docker-compose.yml index edb36d6c07..8643a078b2 100755 --- a/community_images/fluent-bit/ironbank/docker-compose.yml +++ b/community_images/fluent-bit/ironbank/docker-compose.yml @@ -3,12 +3,13 @@ version: '2' services: fluent-bit: image: ${FLUENT_BIT_IMAGE_REPOSITORY}:${FLUENT_BIT_IMAGE_TAG} - cap_add: - SYS_PTRACE ports: - - '2020:2020' - + - '0.0.0.0::2020' + volumes: + - ./config/fluent-bit.config:/tmp/fluent-bit.config + - ./config/plugin.config:/fluent-bit/etc/plugin.config prometheus: image: docker.io/rapidfort/prometheus cap_add: diff --git a/community_images/fluent-bit/ironbank/image.yml b/community_images/fluent-bit/ironbank/image.yml old mode 100644 new mode 100755 index 64cda33c04..0685822d93 --- a/community_images/fluent-bit/ironbank/image.yml +++ b/community_images/fluent-bit/ironbank/image.yml @@ -31,8 +31,6 @@ runtimes: fluent-bit-ib: repository: "FLUENT_BIT_IMAGE_REPOSITORY" tag: "FLUENT_BIT_IMAGE_TAG" - - - type: k8s script: k8s_coverage.sh helm: diff --git a/community_images/fluent-bit/ironbank/k8s_coverage.sh b/community_images/fluent-bit/ironbank/k8s_coverage.sh index 98a2824903..4d925cb5c9 100755 --- a/community_images/fluent-bit/ironbank/k8s_coverage.sh +++ b/community_images/fluent-bit/ironbank/k8s_coverage.sh @@ -13,7 +13,6 @@ SCRIPTPATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" JSON_PARAMS="$1" #JSON=$(cat "$JSON_PARAMS") - NAMESPACE=$(jq -r '.namespace_name' < "$JSON_PARAMS") RELEASE_NAME=$(jq -r '.release_name' < "$JSON_PARAMS") echo "NAMESPACE: $NAMESPACE" @@ -22,7 +21,10 @@ echo "RELEASE_NAME: $RELEASE_NAME" sleep 10 CONTAINER_NAME=$(kubectl get pods -n "$NAMESPACE" -l "app.kubernetes.io/instance=$RELEASE_NAME" -o jsonpath='{.items[0].metadata.name}') # copy over the script to the pod -kubectl cp "${SCRIPTPATH}"/config/fluent-bit.config "${CONTAINER_NAME}":/tmp/fluent-bit.config -n "${NAMESPACE}" +kubectl exec "${CONTAINER_NAME}" -n "${NAMESPACE}" -- /bin/bash -c "nohup /fluent-bit/bin/fluent-bit -c /fluent-bit/etc/fluent-bit.conf" & sleep 10 -kubectl cp "${SCRIPTPATH}"/scripts/fluent-bit_coverage_script.sh "${CONTAINER_NAME}":/tmp/fluent-bit_cvoverage_script.sh -n "${NAMESPACE}" - +# Check if the process is still running and terminate it if needed +if ps -p $! > /dev/null; then + echo "Fluent Bit process is still running. Terminating..." + kill $! +fi \ No newline at end of file diff --git a/community_images/fluent-bit/ironbank/scripts/fluent-bit_coverage_script.sh b/community_images/fluent-bit/ironbank/scripts/fluent-bit_coverage_script.sh deleted file mode 100644 index acad4eed2d..0000000000 --- a/community_images/fluent-bit/ironbank/scripts/fluent-bit_coverage_script.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash - -set -e -set -x - -# Start Fluent Bit with the specified configuration file -/fluent-bit/bin/fluent-bit -c /tmp/fluent-bit.conf & - - -# Capture the PID of the last background process -FLUENT_BIT_PID=$! - -sleep 5 - -# Stop Fluent Bit gracefully by sending a termination signal -kill -TERM $FLUENT_BIT_PID - -# Optionally, wait for Fluent Bit to exit -wait $FLUENT_BIT_PID \ No newline at end of file From aeb5a20cbf8a1a56bea2a614f1855082e213aed5 Mon Sep 17 00:00:00 2001 From: Rajat Date: Thu, 14 Dec 2023 10:28:13 -0800 Subject: [PATCH 11/12] addition of conf Signed-off-by: Rajat --- .../bitnami/config/fluent-bit.config | 27 ++++---------- .../fluent-bit/bitnami/config/plugins.config | 14 ++++++++ .../fluent-bit/bitnami/docker-compose.yml | 4 +-- .../ironbank/config/fluent-bit.config | 36 ++++++------------- .../fluent-bit/ironbank/config/plugins.config | 14 ++++++++ .../fluent-bit/ironbank/docker-compose.yml | 4 +-- 6 files changed, 47 insertions(+), 52 deletions(-) diff --git a/community_images/fluent-bit/bitnami/config/fluent-bit.config b/community_images/fluent-bit/bitnami/config/fluent-bit.config index a445f89971..b2de868e71 100644 --- a/community_images/fluent-bit/bitnami/config/fluent-bit.config +++ b/community_images/fluent-bit/bitnami/config/fluent-bit.config @@ -1,28 +1,18 @@ [SERVICE] # Flush - # ===== - # set an interval of seconds before to flush records to a destination + flush 1 # Daemon - # ====== - # instruct Fluent Bit to run in foreground or background mode. + daemon Off # Log_Level - # ========= - # Set the verbosity level of the service, values can be: - # - # - error - # - warning - # - info - # - debug - # - trace - # - # by default 'info' is set, that means it includes 'error' and 'warning'. + log_level info - + log_level debug + plugins_file plugins.conf # HTTP Server # =========== @@ -36,12 +26,7 @@ # Fluent Bit can use memory and filesystem buffering based mechanisms # # - https://docs.fluentbit.io/manual/administration/buffering-and-storage - # - # storage metrics - # --------------- - # publish storage pipeline metrics in '/api/v1/storage'. The metrics are - # exported only if the 'http_server' option is enabled. - # + storage.metrics on # storage.path diff --git a/community_images/fluent-bit/bitnami/config/plugins.config b/community_images/fluent-bit/bitnami/config/plugins.config index 55c5a8d395..250e068bba 100644 --- a/community_images/fluent-bit/bitnami/config/plugins.config +++ b/community_images/fluent-bit/bitnami/config/plugins.config @@ -1,2 +1,16 @@ [PLUGINS] # Path /path/to/out_gstdout.so +[SERVICE] + Flush 1 + Log_level info + +[INPUT] + Name node_exporter_metrics + Tag node_metrics + Scrape_interval 2 + +[OUTPUT] + Name prometheus_remote_write + Match node_metrics + Host metric-api.newrelic.com + Port 443 \ No newline at end of file diff --git a/community_images/fluent-bit/bitnami/docker-compose.yml b/community_images/fluent-bit/bitnami/docker-compose.yml index 65988ecd9e..04b59764dc 100644 --- a/community_images/fluent-bit/bitnami/docker-compose.yml +++ b/community_images/fluent-bit/bitnami/docker-compose.yml @@ -11,6 +11,4 @@ services: - ./config/fluent-bit.config:/tmp/fluent-bit.config - ./config/plugin.config:/opt/bitnami/fluent-bit/conf/plugin.config prometheus: - image: docker.io/rapidfort/prometheus - cap_add: - - SYS_PTRACE \ No newline at end of file + image: docker.io/rapidfort/prometheus \ No newline at end of file diff --git a/community_images/fluent-bit/ironbank/config/fluent-bit.config b/community_images/fluent-bit/ironbank/config/fluent-bit.config index 0a6059cc83..69d4d82c70 100644 --- a/community_images/fluent-bit/ironbank/config/fluent-bit.config +++ b/community_images/fluent-bit/ironbank/config/fluent-bit.config @@ -1,12 +1,10 @@ [SERVICE] # Flush - # ===== - # set an interval of seconds before to flush records to a destination + flush 1 # Daemon - # ====== - # instruct Fluent Bit to run in foreground or background mode. + daemon Off # Log_Level @@ -21,7 +19,7 @@ # # by default 'info' is set, that means it includes 'error' and 'warning'. log_level info - + log_level debug # HTTP Server # =========== # Enable/Disable the built-in HTTP Server for metrics @@ -29,17 +27,13 @@ http_listen 0.0.0.0 http_port 2020 + + plugins_file plugins.conf + + parsers_file parsers.conf + # Storage - # ======= - # Fluent Bit can use memory and filesystem buffering based mechanisms - # - # - https://docs.fluentbit.io/manual/administration/buffering-and-storage - # - # storage metrics - # --------------- - # publish storage pipeline metrics in '/api/v1/storage'. The metrics are - # exported only if the 'http_server' option is enabled. - # + storage.metrics on # storage.path @@ -49,10 +43,7 @@ storage.path /tmp/storage # storage.sync - # ------------ - # configure the synchronization mode used to store the data into the - # filesystem. It can take the values normal or full. - # + storage.sync normal # storage.checksum @@ -63,12 +54,7 @@ storage.checksum off # storage.backlog.mem_limit - # ------------------------- - # if storage.path is set, Fluent Bit will look for data chunks that were - # not delivered and are still in the storage layer, these are called - # backlog data. This option configure a hint of maximum value of memory - # to use when processing these records. - # + storage.backlog.mem_limit 5M [INPUT] diff --git a/community_images/fluent-bit/ironbank/config/plugins.config b/community_images/fluent-bit/ironbank/config/plugins.config index 55c5a8d395..250e068bba 100644 --- a/community_images/fluent-bit/ironbank/config/plugins.config +++ b/community_images/fluent-bit/ironbank/config/plugins.config @@ -1,2 +1,16 @@ [PLUGINS] # Path /path/to/out_gstdout.so +[SERVICE] + Flush 1 + Log_level info + +[INPUT] + Name node_exporter_metrics + Tag node_metrics + Scrape_interval 2 + +[OUTPUT] + Name prometheus_remote_write + Match node_metrics + Host metric-api.newrelic.com + Port 443 \ No newline at end of file diff --git a/community_images/fluent-bit/ironbank/docker-compose.yml b/community_images/fluent-bit/ironbank/docker-compose.yml index 8643a078b2..f1fe25577b 100755 --- a/community_images/fluent-bit/ironbank/docker-compose.yml +++ b/community_images/fluent-bit/ironbank/docker-compose.yml @@ -11,6 +11,4 @@ services: - ./config/fluent-bit.config:/tmp/fluent-bit.config - ./config/plugin.config:/fluent-bit/etc/plugin.config prometheus: - image: docker.io/rapidfort/prometheus - cap_add: - - SYS_PTRACE \ No newline at end of file + image: docker.io/rapidfort/prometheus \ No newline at end of file From 777db5af8201e56854c270aa588d2f29e2eec164 Mon Sep 17 00:00:00 2001 From: Rajat Date: Thu, 14 Dec 2023 12:12:27 -0800 Subject: [PATCH 12/12] fix for two config Signed-off-by: Rajat --- .../bitnami/config/fluent-bit2.config | 70 +++++++++++++++++++ .../fluent-bit/bitnami/dc_coverage.sh | 2 +- .../fluent-bit/bitnami/docker-compose.yml | 1 + .../fluent-bit/bitnami/k8s_coverage.sh | 3 +- .../ironbank/config/fluent-bit2.config | 69 ++++++++++++++++++ .../fluent-bit/ironbank/dc_coverage.sh | 2 +- .../fluent-bit/ironbank/docker-compose.yml | 1 + .../fluent-bit/ironbank/k8s_coverage.sh | 5 +- 8 files changed, 148 insertions(+), 5 deletions(-) create mode 100644 community_images/fluent-bit/bitnami/config/fluent-bit2.config create mode 100644 community_images/fluent-bit/ironbank/config/fluent-bit2.config diff --git a/community_images/fluent-bit/bitnami/config/fluent-bit2.config b/community_images/fluent-bit/bitnami/config/fluent-bit2.config new file mode 100644 index 0000000000..4c207dd6d2 --- /dev/null +++ b/community_images/fluent-bit/bitnami/config/fluent-bit2.config @@ -0,0 +1,70 @@ +[SERVICE] + # Flush + + flush 1 + + # Daemon + + daemon on + + # Log_Level + + log_level info + log_level debug + + plugins_file plugins.conf + + # HTTP Server + # =========== + # Enable/Disable the built-in HTTP Server for metrics + http_server Off + http_listen 0.0.0.0 + http_port 2020 + + # Storage + # ======= + # Fluent Bit can use memory and filesystem buffering based mechanisms + # + # - https://docs.fluentbit.io/manual/administration/buffering-and-storage + + storage.metrics on + + # storage.path + # ------------ + # absolute file system path to store filesystem data buffers (chunks). + # + storage.path /tmp/storage + + # storage.sync + # ------------ + # configure the synchronization mode used to store the data into the + # filesystem. It can take the values normal or full. + # + storage.sync normal + + # storage.checksum + # ---------------- + # enable the data integrity check when writing and reading data from the + # filesystem. The storage layer uses the CRC32 algorithm. + # + storage.checksum off + + # storage.backlog.mem_limit + # ------------------------- + # if storage.path is set, Fluent Bit will look for data chunks that were + # not delivered and are still in the storage layer, these are called + # backlog data. This option configure a hint of maximum value of memory + # to use when processing these records. + # + storage.backlog.mem_limit 5M + +[INPUT] + name cpu + tag cpu.local + + # Read interval (sec) Default: 1 + interval_sec 1 + +[OUTPUT] + name stdout + match * diff --git a/community_images/fluent-bit/bitnami/dc_coverage.sh b/community_images/fluent-bit/bitnami/dc_coverage.sh index 6478a8c524..f6e8668743 100755 --- a/community_images/fluent-bit/bitnami/dc_coverage.sh +++ b/community_images/fluent-bit/bitnami/dc_coverage.sh @@ -15,7 +15,7 @@ CONTAINER_NAME="${PROJECT_NAME}"-fluent-bit-1 sleep 10 # log for debugging docker inspect "${CONTAINER_NAME}" -docker exec -d "${CONTAINER_NAME}" /bin/bash -c "nohup /opt/bitnami/fluent-bit/bin/fluent-bit -c /tmp/fluent-bit.conf > /opt/bitnami/fluent-bit/logs/fluent-bit.log 2>&1 " & +docker exec -d "${CONTAINER_NAME}" /bin/bash -c "nohup /opt/bitnami/fluent-bit/bin/fluent-bit -c /tmp/fluent-bit2.config > /opt/bitnami/fluent-bit/logs/fluent-bit.log 2>&1 " & sleep 10 # Get the PID of the last background process diff --git a/community_images/fluent-bit/bitnami/docker-compose.yml b/community_images/fluent-bit/bitnami/docker-compose.yml index 04b59764dc..eb06b3c07e 100644 --- a/community_images/fluent-bit/bitnami/docker-compose.yml +++ b/community_images/fluent-bit/bitnami/docker-compose.yml @@ -10,5 +10,6 @@ services: volumes: - ./config/fluent-bit.config:/tmp/fluent-bit.config - ./config/plugin.config:/opt/bitnami/fluent-bit/conf/plugin.config + - ./config/fluent-bit2.config:/tmp/fluent-bit2.config prometheus: image: docker.io/rapidfort/prometheus \ No newline at end of file diff --git a/community_images/fluent-bit/bitnami/k8s_coverage.sh b/community_images/fluent-bit/bitnami/k8s_coverage.sh index 31e32c2e60..9f4ed54422 100755 --- a/community_images/fluent-bit/bitnami/k8s_coverage.sh +++ b/community_images/fluent-bit/bitnami/k8s_coverage.sh @@ -14,8 +14,9 @@ JSON_PARAMS="$1" NAMESPACE=$(jq -r '.namespace_name' < "$JSON_PARAMS") RELEASE_NAME=$(jq -r '.release_name' < "$JSON_PARAMS") CONTAINER_NAME=$(kubectl get pods -n "$NAMESPACE" -l "app.kubernetes.io/instance=$RELEASE_NAME" -o jsonpath='{.items[0].metadata.name}') +kubectl cp "${SCRIPTPATH}"/config/fluent-bit.config "${CONTAINER_NAME}":/tmp/fluent-bit.config -n "${NAMESPACE}" # copy over the script to the pod -kubectl exec "${CONTAINER_NAME}" -n "${NAMESPACE}" -- /bin/bash -c "nohup /opt/bitnami/fluent-bit/bin/fluent-bit -c /opt/bitnami/fluent-bit/conf/fluent-bit.conf" & +kubectl exec "${CONTAINER_NAME}" -n "${NAMESPACE}" -- /bin/bash -c "nohup /opt/bitnami/fluent-bit/bin/fluent-bit -c /tmp/fluent-bit2.config" & sleep 10 # Check if the process is still running and terminate it if needed if ps -p $! > /dev/null; then diff --git a/community_images/fluent-bit/ironbank/config/fluent-bit2.config b/community_images/fluent-bit/ironbank/config/fluent-bit2.config new file mode 100644 index 0000000000..644cddc6f4 --- /dev/null +++ b/community_images/fluent-bit/ironbank/config/fluent-bit2.config @@ -0,0 +1,69 @@ +[SERVICE] + # Flush + + flush 1 + + # Daemon + + daemon on + + # Log_Level + # ========= + # Set the verbosity level of the service, values can be: + # + # - error + # - warning + # - info + # - debug + # - trace + # + # by default 'info' is set, that means it includes 'error' and 'warning'. + log_level info + log_level debug + # HTTP Server + # =========== + # Enable/Disable the built-in HTTP Server for metrics + http_server Off + http_listen 0.0.0.0 + http_port 2020 + + + plugins_file plugins.conf + + parsers_file parsers.conf + + # Storage + + storage.metrics on + + # storage.path + # ------------ + # absolute file system path to store filesystem data buffers (chunks). + # + storage.path /tmp/storage + + # storage.sync + + storage.sync normal + + # storage.checksum + # ---------------- + # enable the data integrity check when writing and reading data from the + # filesystem. The storage layer uses the CRC32 algorithm. + # + storage.checksum off + + # storage.backlog.mem_limit + + storage.backlog.mem_limit 5M + +[INPUT] + name cpu + tag cpu.local + + # Read interval (sec) Default: 1 + interval_sec 1 + +[OUTPUT] + name stdout + match * diff --git a/community_images/fluent-bit/ironbank/dc_coverage.sh b/community_images/fluent-bit/ironbank/dc_coverage.sh index d14c3da250..015240e27e 100755 --- a/community_images/fluent-bit/ironbank/dc_coverage.sh +++ b/community_images/fluent-bit/ironbank/dc_coverage.sh @@ -20,7 +20,7 @@ CONTAINER_NAME="${PROJECT_NAME}"-fluent-bit-1 sleep 10 # log for debugging docker inspect "${CONTAINER_NAME}" -docker exec -d "${CONTAINER_NAME}" /bin/bash -c "nohup /fluent-bit/bin/fluent-bit -c /tmp/fluent-bit.conf" & +docker exec -d "${CONTAINER_NAME}" /bin/bash -c "nohup /fluent-bit/bin/fluent-bit -c /tmp/fluent-bit2.config" & sleep 10 # Get the PID of the last background process diff --git a/community_images/fluent-bit/ironbank/docker-compose.yml b/community_images/fluent-bit/ironbank/docker-compose.yml index f1fe25577b..3edf64db81 100755 --- a/community_images/fluent-bit/ironbank/docker-compose.yml +++ b/community_images/fluent-bit/ironbank/docker-compose.yml @@ -10,5 +10,6 @@ services: volumes: - ./config/fluent-bit.config:/tmp/fluent-bit.config - ./config/plugin.config:/fluent-bit/etc/plugin.config + - ./config/fluent-bit2.config:/tmp/fluent-bit2.config prometheus: image: docker.io/rapidfort/prometheus \ No newline at end of file diff --git a/community_images/fluent-bit/ironbank/k8s_coverage.sh b/community_images/fluent-bit/ironbank/k8s_coverage.sh index 4d925cb5c9..1db8758c50 100755 --- a/community_images/fluent-bit/ironbank/k8s_coverage.sh +++ b/community_images/fluent-bit/ironbank/k8s_coverage.sh @@ -21,10 +21,11 @@ echo "RELEASE_NAME: $RELEASE_NAME" sleep 10 CONTAINER_NAME=$(kubectl get pods -n "$NAMESPACE" -l "app.kubernetes.io/instance=$RELEASE_NAME" -o jsonpath='{.items[0].metadata.name}') # copy over the script to the pod -kubectl exec "${CONTAINER_NAME}" -n "${NAMESPACE}" -- /bin/bash -c "nohup /fluent-bit/bin/fluent-bit -c /fluent-bit/etc/fluent-bit.conf" & +kubectl cp "${SCRIPTPATH}"/config/fluent-bit.config "${CONTAINER_NAME}":/tmp/fluent-bit.config -n "${NAMESPACE}" +kubectl exec "${CONTAINER_NAME}" -n "${NAMESPACE}" -- /bin/bash -c "nohup /fluent-bit/bin/fluent-bit -c /tmp/fluent-bit.config" & sleep 10 # Check if the process is still running and terminate it if needed if ps -p $! > /dev/null; then echo "Fluent Bit process is still running. Terminating..." kill $! -fi \ No newline at end of file +fi