fix: add docker-compose configuration

fix: update manage_secrets.py and its documentation fix: create docker compose workflow fix: update compose workflow fix: update compose workflow fix: update compose workflow fix: update compose workflow fix: downloading package instruction fix: fix link in 6-env-file-configuration.md
splunk · Jan 3, 2024 · 63fb153 · 63fb153
1 parent 5670d7c
commit 63fb153
Show file tree

Hide file tree

Showing 25 changed files with 987 additions and 1 deletion.
diff --git a/.github/workflows/ci-docker.yaml b/.github/workflows/ci-docker.yaml
@@ -0,0 +1,29 @@
+name: create-compose-files
+on:
+  push:
+    branches:
+      - fix/docker-compose
+
+jobs:
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      packages: write
+    strategy:
+      matrix:
+        python-version:
+          - 3.9
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v4
+      - name: Zip docker_compose directory
+        run: zip -r docker_compose.zip docker_compose
+      - name: Create artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ env.ARTIFACT_NAME }}
+          path: docker_compose.zip
+        env:
+          ARTIFACT_NAME: docker_compose
diff --git a/.gitignore b/.gitignore
@@ -103,6 +103,7 @@ celerybeat.pid
 
 # Environments
 .env
+!docker_compose/.env
 .venv
 env/
 venv/

diff --git a/docker_compose/.env b/docker_compose/.env
@@ -0,0 +1,55 @@
+# Deployment configuration
+SC4SNMP_IMAGE=ghcr.io/splunk/splunk-connect-for-snmp/container
+SC4SNMP_TAG=latest
+SCHEDULER_CONFIG_FILE_ABSOLUTE_PATH=
+TRAPS_CONFIG_FILE_ABSOLUTE_PATH=
+INVENTORY_FILE_ABSOLUTE_PATH=
+COREFILE_ABS_PATH=
+COREDNS_ADDRESS=172.28.0.255
+
+# Splunk instance configuration
+SPLUNK_HEC_HOST=
+SPLUNK_HEC_PROTOCOL=https
+SPLUNK_HEC_PORT=8088
+SPLUNK_HEC_TOKEN=
+SPLUNK_HEC_INSECURESSL=false
+SPLUNK_SOURCETYPE_TRAPS=sc4snmp:traps
+SPLUNK_SOURCETYPE_POLLING_EVENTS=sc4snmp:event
+SPLUNK_SOURCETYPE_POLLING_METRICS=sc4snmp:metric
+SPLUNK_HEC_INDEX_EVENTS=netops
+SPLUNK_HEC_INDEX_METRICS=netmetrics
+SPLUNK_HEC_PATH=/services/collector
+SPLUNK_AGGREGATE_TRAPS_EVENTS=false
+IGNORE_EMPTY_VARBINDS=false
+
+# Workers configration
+WALK_RETRY_MAX_INTERVAL=180
+WALK_MAX_RETRIES=5
+METRICS_INDEXING_ENABLED=false
+POLL_BASE_PROFILES=true
+IGNORE_NOT_INCREASING_OIDS=
+WORKER_LOG_LEVEL=INFO
+UDP_CONNECTION_TIMEOUT=3
+MAX_OID_TO_PROCESS=70
+WORKER_POLLER_CONCURRENCY=4
+WORKER_SENDER_CONCURRENCY=4
+WORKER_TRAP_CONCURRENCY=4
+PREFETCH_POLLER_COUNT=1
+PREFETCH_SENDER_COUNT=30
+PREFETCH_TRAP_COUNT=30
+RESOLVE_TRAP_ADDRESS=false
+MAX_DNS_CACHE_SIZE_TRAPS=500
+TTL_DNS_CACHE_TRAPS=1800
+
+# Inventory configuration
+INVENTORY_LOG_LEVEL=INFO
+CHAIN_OF_TASKS_EXPIRY_TIME=500
+
+# Traps configuration
+SNMP_V3_SECURITY_ENGINE_ID=80003a8c04
+TRAPS_PORT=162
+
+# Scheduler configuration
+SCHEDULER_LOG_LEVEL=INFO
+
+#Secrets
diff --git a/docker_compose/Corefile b/docker_compose/Corefile
@@ -0,0 +1,7 @@
+.:53 {
+    log
+    errors
+    auto
+    reload
+    forward . 8.8.8.8
+}
diff --git a/docker_compose/docker-compose-coredns.yaml b/docker_compose/docker-compose-coredns.yaml
@@ -0,0 +1,15 @@
+version: '3.8'
+services:
+  coredns:
+    image: coredns/coredns:latest
+    command: ["-conf", "/Corefile"]
+    container_name: coredns
+    restart: on-failure
+    expose:
+        - '53'
+        - '53/udp'
+    volumes:
+        - '${COREFILE_ABS_PATH}:/Corefile'
+    networks:
+        my_network:
+            ipv4_address: ${COREDNS_ADDRESS}
diff --git a/docker_compose/docker-compose-dependencies.yaml b/docker_compose/docker-compose-dependencies.yaml
@@ -0,0 +1,38 @@
+version: '3.8'
+services:
+  snmp-mibserver:
+    image: "ghcr.io/pysnmp/mibs/container"
+    container_name: snmp-mibserver
+    environment:
+      - NGINX_ENTRYPOINT_QUIET_LOGS=${NGINX_ENTRYPOINT_QUIET_LOGS:-1}
+    volumes:
+      - snmp-mibserver-tmp:/tmp/
+    depends_on:
+      - coredns
+    networks:
+      my_network:
+    dns:
+      - ${COREDNS_ADDRESS}
+
+  some-redis:
+    image: redis
+    container_name: some-redis
+    restart: always
+    depends_on:
+      - coredns
+    networks:
+      my_network:
+    dns:
+      - ${COREDNS_ADDRESS}
+  example-mongo:
+    image: mongo:4.4.6
+    container_name: example-mongo
+    restart: always
+    depends_on:
+      - coredns
+    networks:
+      my_network:
+    dns:
+      - ${COREDNS_ADDRESS}
+volumes:
+  snmp-mibserver-tmp:
diff --git a/docker_compose/docker-compose-inventory.yaml b/docker_compose/docker-compose-inventory.yaml
@@ -0,0 +1,36 @@
+version: '3.8'
+services:
+  inventory:
+    image: ${SC4SNMP_IMAGE}:${SC4SNMP_TAG:-latest}
+    container_name: sc4snmp-inventory
+    command: ["inventory"]
+    environment:
+      - CONFIG_PATH=/app/config/config.yaml
+      - REDIS_URL=redis://some-redis:6379/1
+      - CELERY_BROKER_URL=redis://some-redis:6379/0
+      - MONGO_URI=mongodb://example-mongo:27017/
+      - MIB_SOURCES=http://snmp-mibserver:8000/asn1/@mib@
+      - MIB_INDEX=http://snmp-mibserver:8000/index.csv
+      - MIB_STANDARD=http://snmp-mibserver:8000/standard.txt
+
+      # Inventory configuration
+      - LOG_LEVEL=${INVENTORY_LOG_LEVEL:-INFO}
+      - CHAIN_OF_TASKS_EXPIRY_TIME=${CHAIN_OF_TASKS_EXPIRY_TIME:-500}
+      - CONFIG_FROM_MONGO=${CONFIG_FROM_MONGO:-false}
+    depends_on:
+      - some-redis
+      - example-mongo
+      - coredns
+    volumes:
+      - ${SCHEDULER_CONFIG_FILE_ABSOLUTE_PATH}:/app/config/config.yaml
+      - ${INVENTORY_FILE_ABSOLUTE_PATH}:/app/inventory/inventory.csv
+      - inventory-pysnmp-cache-volume:/.pysnmp/
+      - inventory-tmp:/tmp/
+    restart: on-failure
+    networks:
+      my_network:
+    dns:
+      - ${COREDNS_ADDRESS}
+volumes:
+  inventory-tmp:
+  inventory-pysnmp-cache-volume:
diff --git a/docker_compose/docker-compose-network.yaml b/docker_compose/docker-compose-network.yaml
@@ -0,0 +1,7 @@
+version: '3.8'
+networks:
+  my_network:
+    ipam:
+      driver: default
+      config:
+        - subnet: 172.28.0.0/16
diff --git a/docker_compose/docker-compose-scheduler.yaml b/docker_compose/docker-compose-scheduler.yaml
@@ -0,0 +1,35 @@
+---
+version: '3.8'
+services:
+  scheduler:
+    image: ${SC4SNMP_IMAGE}:${SC4SNMP_TAG:-latest}
+    container_name: sc4snmp-scheduler
+    command: ["celery", "beat"]
+    environment:
+      - CONFIG_PATH=/app/config/config.yaml
+      - REDIS_URL=redis://some-redis:6379/1
+      - CELERY_BROKER_URL=redis://some-redis:6379/0
+      - MONGO_URI=mongodb://example-mongo:27017/
+      - MIB_SOURCES=http://snmp-mibserver:8000/asn1/@mib@
+      - MIB_INDEX=http://snmp-mibserver:8000/index.csv
+      - MIB_STANDARD=http://snmp-mibserver:8000/standard.txt
+      - INVENTORY_REFRESH_RATE=${INVENTORY_REFRESH_RATE:-600}
+
+      # Scheduler configuration
+      - LOG_LEVEL=${SCHEDULER_LOG_LEVEL:-INFO}
+    depends_on:
+      - some-redis
+      - example-mongo
+      - coredns
+    volumes:
+      - ${SCHEDULER_CONFIG_FILE_ABSOLUTE_PATH}:/app/config/config.yaml
+      - scheduler-pysnmp-cache-volume:/.pysnmp/
+      - scheduler-tmp:/tmp/
+    restart: on-failure
+    networks:
+      my_network:
+    dns:
+      - ${COREDNS_ADDRESS}
+volumes:
+  scheduler-tmp:
+  scheduler-pysnmp-cache-volume:
diff --git a/docker_compose/docker-compose-secrets.yaml b/docker_compose/docker-compose-secrets.yaml
@@ -0,0 +1,2 @@
+secrets: {}
+version: '3.8'
diff --git a/docker_compose/docker-compose-traps.yaml b/docker_compose/docker-compose-traps.yaml
@@ -0,0 +1,46 @@
+services:
+  traps:
+    command:
+    - trap
+    container_name: sc4snmp-traps
+    depends_on:
+    - some-redis
+    - example-mongo
+    - coredns
+    dns:
+    - ${COREDNS_ADDRESS}
+    environment:
+    - CONFIG_PATH=/app/config/config.yaml
+    - REDIS_URL=redis://some-redis:6379/1
+    - CELERY_BROKER_URL=redis://some-redis:6379/0
+    - MONGO_URI=mongodb://example-mongo:27017/
+    - MIB_SOURCES=http://snmp-mibserver:8000/asn1/@mib@
+    - MIB_INDEX=http://snmp-mibserver:8000/index.csv
+    - MIB_STANDARD=http://snmp-mibserver:8000/standard.txt
+    - LOG_LEVEL=${SCHEDULER_LOG_LEVEL:-INFO}
+    - INVENTORY_REFRESH_RATE=${INVENTORY_REFRESH_RATE:-600}
+    - SPLUNK_HEC_HOST=${SPLUNK_HEC_HOST}
+    - SPLUNK_HEC_SCHEME=${SPLUNK_HEC_PROTOCOL:-https}
+    - SPLUNK_HEC_PORT=${SPLUNK_HEC_PORT}
+    - SPLUNK_HEC_TOKEN=${SPLUNK_HEC_TOKEN}
+    - SPLUNK_HEC_INSECURESSL=${SPLUNK_HEC_INSECURESSL:-false}
+    - SPLUNK_HEC_PATH=${SPLUNK_HEC_PATH:-/services/collector}
+    - SNMP_V3_SECURITY_ENGINE_ID=${SNMP_V3_SECURITY_ENGINE_ID:-80003a8c04}
+    image: ${SC4SNMP_IMAGE}:${SC4SNMP_TAG:-latest}
+    networks:
+      my_network: null
+    ports:
+    - mode: host
+      protocol: udp
+      published: ${TRAPS_PORT}
+      target: 2162
+    restart: on-failure
+    secrets: []
+    volumes:
+    - ${TRAPS_CONFIG_FILE_ABSOLUTE_PATH}:/app/config/config.yaml
+    - traps-pysnmp-cache-volume:/.pysnmp/
+    - traps-tmp:/tmp/
+version: '3.8'
+volumes:
+  traps-pysnmp-cache-volume: null
+  traps-tmp: null
diff --git a/docker_compose/docker-compose-worker-poller.yaml b/docker_compose/docker-compose-worker-poller.yaml
@@ -0,0 +1,58 @@
+services:
+  worker-poller:
+    command:
+    - celery
+    - worker-poller
+    container_name: sc4snmp-worker-poller
+    depends_on:
+    - some-redis
+    - example-mongo
+    - coredns
+    dns:
+    - ${COREDNS_ADDRESS}
+    environment:
+    - CONFIG_PATH=/app/config/config.yaml
+    - REDIS_URL=redis://some-redis:6379/1
+    - CELERY_BROKER_URL=redis://some-redis:6379/0
+    - MONGO_URI=mongodb://example-mongo:27017/
+    - SC4SNMP_VERSION=${SC4SNMP_VERSION:-0.0.0}
+    - MIB_SOURCES=http://snmp-mibserver:8000/asn1/@mib@
+    - MIB_INDEX=http://snmp-mibserver:8000/index.csv
+    - MIB_STANDARD=http://snmp-mibserver:8000/standard.txt
+    - SPLUNK_HEC_HOST=${SPLUNK_HEC_HOST}
+    - SPLUNK_HEC_SCHEME=${SPLUNK_HEC_PROTOCOL:-https}
+    - SPLUNK_HEC_PORT=${SPLUNK_HEC_PORT}
+    - SPLUNK_HEC_TOKEN=${SPLUNK_HEC_TOKEN}
+    - SPLUNK_HEC_INSECURESSL=${SPLUNK_HEC_INSECURESSL:-false}
+    - SPLUNK_SOURCETYPE_TRAPS=${SPLUNK_SOURCETYPE_TRAPS:-sc4snmp:traps}
+    - SPLUNK_SOURCETYPE_POLLING_EVENTS=${SPLUNK_SOURCETYPE_POLLING_EVENTS:-sc4snmp:event}
+    - SPLUNK_SOURCETYPE_POLLING_METRICS=${SPLUNK_SOURCETYPE_POLLING_METRICS:-sc4snmp:metric}
+    - SPLUNK_HEC_INDEX_EVENTS=${SPLUNK_HEC_INDEX_EVENTS:-netops}
+    - SPLUNK_HEC_INDEX_METRICS=${SPLUNK_HEC_INDEX_METRICS:-netmetrics}
+    - SPLUNK_HEC_PATH=${SPLUNK_HEC_PATH:-/services/collector}
+    - SPLUNK_AGGREGATE_TRAPS_EVENTS=${SPLUNK_AGGREGATE_TRAPS_EVENTS:-false}
+    - IGNORE_EMPTY_VARBINDS=${IGNORE_EMPTY_VARBINDS:-false}
+    - WALK_RETRY_MAX_INTERVAL=${WALK_RETRY_MAX_INTERVAL:-180}
+    - WALK_MAX_RETRIES=${WALK_MAX_RETRIES:-5}
+    - METRICS_INDEXING_ENABLED=${METRICS_INDEXING_ENABLED:-false}
+    - POLL_BASE_PROFILES=${POLL_BASE_PROFILES:-true}
+    - IGNORE_NOT_INCREASING_OIDS=${IGNORE_NOT_INCREASING_OIDS:-}
+    - LOG_LEVEL=${WORKER_LOG_LEVEL:-INFO}
+    - UDP_CONNECTION_TIMEOUT=${UDP_CONNECTION_TIMEOUT:-3}
+    - MAX_OID_TO_PROCESS=${MAX_OID_TO_PROCESS:-70}
+    - PROFILES_RELOAD_DELAY=${PROFILES_RELOAD_DELAY:-60}
+    - WORKER_CONCURRENCY=${WORKER_POLLER_CONCURRENCY:-2}
+    - PREFETCH_COUNT=${PREFETCH_POLLER_COUNT:-1}
+    image: ${SC4SNMP_IMAGE}:${SC4SNMP_TAG:-latest}
+    networks:
+      my_network: null
+    restart: on-failure
+    secrets: []
+    volumes:
+    - ${SCHEDULER_CONFIG_FILE_ABSOLUTE_PATH}:/app/config/config.yaml
+    - worker-poller-pysnmp-cache-volume:/.pysnmp/
+    - worker-poller-tmp:/tmp/
+version: '3.8'
+volumes:
+  worker-poller-pysnmp-cache-volume: null
+  worker-poller-tmp: null
-Original file line number
+Diff line change
@@ Expand Up / @@ -103,6 +103,7 @@ celerybeat.pid @@
     # Environments
     .env
+    !docker_compose/.env
     .venv
     env/
     venv/
@@ Expand Down @@