diff --git a/.github/workflows/ci-docker.yaml b/.github/workflows/ci-docker.yaml new file mode 100644 index 000000000..24fbb39a0 --- /dev/null +++ b/.github/workflows/ci-docker.yaml @@ -0,0 +1,29 @@ +name: create-compose-files +on: + push: + branches: + - fix/docker-compose + +jobs: + build: + name: Build + runs-on: ubuntu-latest + permissions: + contents: write + packages: write + strategy: + matrix: + python-version: + - 3.9 + steps: + - name: Check out code + uses: actions/checkout@v4 + - name: Zip docker_compose directory + run: zip -r docker_compose.zip docker_compose + - name: Create artifact + uses: actions/upload-artifact@v4 + with: + name: ${{ env.ARTIFACT_NAME }} + path: docker_compose.zip + env: + ARTIFACT_NAME: docker_compose diff --git a/.gitignore b/.gitignore index 7f783c156..43d5f3804 100644 --- a/.gitignore +++ b/.gitignore @@ -103,6 +103,7 @@ celerybeat.pid # Environments .env +!docker_compose/.env .venv env/ venv/ diff --git a/docker_compose/.env b/docker_compose/.env new file mode 100644 index 000000000..82cbd1e75 --- /dev/null +++ b/docker_compose/.env @@ -0,0 +1,66 @@ +# Deployment configuration +SC4SNMP_IMAGE=ghcr.io/splunk/splunk-connect-for-snmp/container +SC4SNMP_TAG=latest +SCHEDULER_CONFIG_FILE_ABSOLUTE_PATH= +TRAPS_CONFIG_FILE_ABSOLUTE_PATH= +INVENTORY_FILE_ABSOLUTE_PATH= +COREFILE_ABS_PATH= +COREDNS_ADDRESS=172.28.0.255 +SC4SNMP_VERSION=1.10.0-beta.4 + +# Dependencies images +COREDNS_IMAGE=coredns/coredns +COREDNS_TAG=1.11.1 +MIBSERVER_IMAGE=ghcr.io/pysnmp/mibs/container +MIBSERVER_TAG=latest +REDIS_IMAGE=docker.io/bitnami/redis +REDIS_TAG=7.2.1-debian-11-r0 +MONGO_IMAGE=docker.io/bitnami/mongodb +MONGO_TAG=6.0.9-debian-11-r5 + +# Splunk instance configuration +SPLUNK_HEC_HOST= +SPLUNK_HEC_PROTOCOL=https +SPLUNK_HEC_PORT=8088 +SPLUNK_HEC_TOKEN= +SPLUNK_HEC_INSECURESSL=false +SPLUNK_SOURCETYPE_TRAPS=sc4snmp:traps +SPLUNK_SOURCETYPE_POLLING_EVENTS=sc4snmp:event +SPLUNK_SOURCETYPE_POLLING_METRICS=sc4snmp:metric +SPLUNK_HEC_INDEX_EVENTS=netops +SPLUNK_HEC_INDEX_METRICS=netmetrics +SPLUNK_HEC_PATH=/services/collector +SPLUNK_AGGREGATE_TRAPS_EVENTS=false +IGNORE_EMPTY_VARBINDS=false + +# Workers configration +WALK_RETRY_MAX_INTERVAL=180 +WALK_MAX_RETRIES=5 +METRICS_INDEXING_ENABLED=false +POLL_BASE_PROFILES=true +IGNORE_NOT_INCREASING_OIDS= +WORKER_LOG_LEVEL=INFO +UDP_CONNECTION_TIMEOUT=3 +MAX_OID_TO_PROCESS=70 +WORKER_POLLER_CONCURRENCY=4 +WORKER_SENDER_CONCURRENCY=4 +WORKER_TRAP_CONCURRENCY=4 +PREFETCH_POLLER_COUNT=1 +PREFETCH_SENDER_COUNT=30 +PREFETCH_TRAP_COUNT=30 +RESOLVE_TRAP_ADDRESS=false +MAX_DNS_CACHE_SIZE_TRAPS=500 +TTL_DNS_CACHE_TRAPS=1800 + +# Inventory configuration +INVENTORY_LOG_LEVEL=INFO +CHAIN_OF_TASKS_EXPIRY_TIME=500 + +# Traps configuration +SNMP_V3_SECURITY_ENGINE_ID=80003a8c04 +TRAPS_PORT=162 + +# Scheduler configuration +SCHEDULER_LOG_LEVEL=INFO + +#Secrets diff --git a/docker_compose/Corefile b/docker_compose/Corefile new file mode 100644 index 000000000..7ea43e1b2 --- /dev/null +++ b/docker_compose/Corefile @@ -0,0 +1,7 @@ +.:53 { + log + errors + auto + reload + forward . 8.8.8.8 +} \ No newline at end of file diff --git a/docker_compose/docker-compose-coredns.yaml b/docker_compose/docker-compose-coredns.yaml new file mode 100644 index 000000000..887991b76 --- /dev/null +++ b/docker_compose/docker-compose-coredns.yaml @@ -0,0 +1,15 @@ +version: '3.8' +services: + coredns: + image: ${COREDNS_IMAGE}:${COREDNS_TAG:-latest} + command: ["-conf", "/Corefile"] + container_name: coredns + restart: on-failure + expose: + - '53' + - '53/udp' + volumes: + - '${COREFILE_ABS_PATH}:/Corefile' + networks: + my_network: + ipv4_address: ${COREDNS_ADDRESS} diff --git a/docker_compose/docker-compose-dependencies.yaml b/docker_compose/docker-compose-dependencies.yaml new file mode 100644 index 000000000..2eebf6d16 --- /dev/null +++ b/docker_compose/docker-compose-dependencies.yaml @@ -0,0 +1,40 @@ +version: '3.8' +services: + snmp-mibserver: + image: ${MIBSERVER_IMAGE}:${MIBSERVER_TAG:-latest} + container_name: snmp-mibserver + environment: + - NGINX_ENTRYPOINT_QUIET_LOGS=${NGINX_ENTRYPOINT_QUIET_LOGS:-1} + volumes: + - snmp-mibserver-tmp:/tmp/ + depends_on: + - coredns + networks: + my_network: + dns: + - ${COREDNS_ADDRESS} + + redis: + image: ${REDIS_IMAGE}:${REDIS_TAG:-latest} + container_name: redis + restart: always + environment: + - ALLOW_EMPTY_PASSWORD=yes + depends_on: + - coredns + networks: + my_network: + dns: + - ${COREDNS_ADDRESS} + mongo: + image: ${MONGO_IMAGE}:${MONGO_TAG:-latest} + container_name: mongo + restart: always + depends_on: + - coredns + networks: + my_network: + dns: + - ${COREDNS_ADDRESS} +volumes: + snmp-mibserver-tmp: diff --git a/docker_compose/docker-compose-inventory.yaml b/docker_compose/docker-compose-inventory.yaml new file mode 100644 index 000000000..0c666f546 --- /dev/null +++ b/docker_compose/docker-compose-inventory.yaml @@ -0,0 +1,36 @@ +version: '3.8' +services: + inventory: + image: ${SC4SNMP_IMAGE}:${SC4SNMP_TAG:-latest} + container_name: sc4snmp-inventory + command: ["inventory"] + environment: + - CONFIG_PATH=/app/config/config.yaml + - REDIS_URL=redis://redis:6379/1 + - CELERY_BROKER_URL=redis://redis:6379/0 + - MONGO_URI=mongodb://mongo:27017/ + - MIB_SOURCES=http://snmp-mibserver:8000/asn1/@mib@ + - MIB_INDEX=http://snmp-mibserver:8000/index.csv + - MIB_STANDARD=http://snmp-mibserver:8000/standard.txt + + # Inventory configuration + - LOG_LEVEL=${INVENTORY_LOG_LEVEL:-INFO} + - CHAIN_OF_TASKS_EXPIRY_TIME=${CHAIN_OF_TASKS_EXPIRY_TIME:-500} + - CONFIG_FROM_MONGO=${CONFIG_FROM_MONGO:-false} + depends_on: + - redis + - mongo + - coredns + volumes: + - ${SCHEDULER_CONFIG_FILE_ABSOLUTE_PATH}:/app/config/config.yaml + - ${INVENTORY_FILE_ABSOLUTE_PATH}:/app/inventory/inventory.csv + - inventory-pysnmp-cache-volume:/.pysnmp/ + - inventory-tmp:/tmp/ + restart: on-failure + networks: + my_network: + dns: + - ${COREDNS_ADDRESS} +volumes: + inventory-tmp: + inventory-pysnmp-cache-volume: diff --git a/docker_compose/docker-compose-network.yaml b/docker_compose/docker-compose-network.yaml new file mode 100644 index 000000000..ce09f5a6a --- /dev/null +++ b/docker_compose/docker-compose-network.yaml @@ -0,0 +1,7 @@ +version: '3.8' +networks: + my_network: + ipam: + driver: default + config: + - subnet: 172.28.0.0/16 \ No newline at end of file diff --git a/docker_compose/docker-compose-scheduler.yaml b/docker_compose/docker-compose-scheduler.yaml new file mode 100644 index 000000000..c2a951815 --- /dev/null +++ b/docker_compose/docker-compose-scheduler.yaml @@ -0,0 +1,34 @@ +version: '3.8' +services: + scheduler: + image: ${SC4SNMP_IMAGE}:${SC4SNMP_TAG:-latest} + container_name: sc4snmp-scheduler + command: ["celery", "beat"] + environment: + - CONFIG_PATH=/app/config/config.yaml + - REDIS_URL=redis://redis:6379/1 + - CELERY_BROKER_URL=redis://redis:6379/0 + - MONGO_URI=mongodb://mongo:27017/ + - MIB_SOURCES=http://snmp-mibserver:8000/asn1/@mib@ + - MIB_INDEX=http://snmp-mibserver:8000/index.csv + - MIB_STANDARD=http://snmp-mibserver:8000/standard.txt + - INVENTORY_REFRESH_RATE=${INVENTORY_REFRESH_RATE:-600} + + # Scheduler configuration + - LOG_LEVEL=${SCHEDULER_LOG_LEVEL:-INFO} + depends_on: + - redis + - mongo + - coredns + volumes: + - ${SCHEDULER_CONFIG_FILE_ABSOLUTE_PATH}:/app/config/config.yaml + - scheduler-pysnmp-cache-volume:/.pysnmp/ + - scheduler-tmp:/tmp/ + restart: on-failure + networks: + my_network: + dns: + - ${COREDNS_ADDRESS} +volumes: + scheduler-tmp: + scheduler-pysnmp-cache-volume: \ No newline at end of file diff --git a/docker_compose/docker-compose-secrets.yaml b/docker_compose/docker-compose-secrets.yaml new file mode 100644 index 000000000..c1dae5281 --- /dev/null +++ b/docker_compose/docker-compose-secrets.yaml @@ -0,0 +1,2 @@ +secrets: {} +version: '3.8' diff --git a/docker_compose/docker-compose-traps.yaml b/docker_compose/docker-compose-traps.yaml new file mode 100644 index 000000000..4249f73aa --- /dev/null +++ b/docker_compose/docker-compose-traps.yaml @@ -0,0 +1,46 @@ +services: + traps: + command: + - trap + container_name: sc4snmp-traps + depends_on: + - redis + - mongo + - coredns + dns: + - ${COREDNS_ADDRESS} + environment: + - CONFIG_PATH=/app/config/config.yaml + - REDIS_URL=redis://redis:6379/1 + - CELERY_BROKER_URL=redis://redis:6379/0 + - MONGO_URI=mongodb://mongo:27017/ + - MIB_SOURCES=http://snmp-mibserver:8000/asn1/@mib@ + - MIB_INDEX=http://snmp-mibserver:8000/index.csv + - MIB_STANDARD=http://snmp-mibserver:8000/standard.txt + - LOG_LEVEL=${SCHEDULER_LOG_LEVEL:-INFO} + - INVENTORY_REFRESH_RATE=${INVENTORY_REFRESH_RATE:-600} + - SPLUNK_HEC_HOST=${SPLUNK_HEC_HOST} + - SPLUNK_HEC_SCHEME=${SPLUNK_HEC_PROTOCOL:-https} + - SPLUNK_HEC_PORT=${SPLUNK_HEC_PORT} + - SPLUNK_HEC_TOKEN=${SPLUNK_HEC_TOKEN} + - SPLUNK_HEC_INSECURESSL=${SPLUNK_HEC_INSECURESSL:-false} + - SPLUNK_HEC_PATH=${SPLUNK_HEC_PATH:-/services/collector} + - SNMP_V3_SECURITY_ENGINE_ID=${SNMP_V3_SECURITY_ENGINE_ID:-80003a8c04} + image: ${SC4SNMP_IMAGE}:${SC4SNMP_TAG:-latest} + networks: + my_network: null + ports: + - mode: host + protocol: udp + published: ${TRAPS_PORT} + target: 2162 + restart: on-failure + secrets: [] + volumes: + - ${TRAPS_CONFIG_FILE_ABSOLUTE_PATH}:/app/config/config.yaml + - traps-pysnmp-cache-volume:/.pysnmp/ + - traps-tmp:/tmp/ +version: '3.8' +volumes: + traps-pysnmp-cache-volume: null + traps-tmp: null diff --git a/docker_compose/docker-compose-worker-poller.yaml b/docker_compose/docker-compose-worker-poller.yaml new file mode 100644 index 000000000..3bf4197da --- /dev/null +++ b/docker_compose/docker-compose-worker-poller.yaml @@ -0,0 +1,58 @@ +services: + worker-poller: + command: + - celery + - worker-poller + container_name: sc4snmp-worker-poller + depends_on: + - redis + - mongo + - coredns + dns: + - ${COREDNS_ADDRESS} + environment: + - CONFIG_PATH=/app/config/config.yaml + - REDIS_URL=redis://redis:6379/1 + - CELERY_BROKER_URL=redis://redis:6379/0 + - MONGO_URI=mongodb://mongo:27017/ + - SC4SNMP_VERSION=${SC4SNMP_VERSION:-0.0.0} + - MIB_SOURCES=http://snmp-mibserver:8000/asn1/@mib@ + - MIB_INDEX=http://snmp-mibserver:8000/index.csv + - MIB_STANDARD=http://snmp-mibserver:8000/standard.txt + - SPLUNK_HEC_HOST=${SPLUNK_HEC_HOST} + - SPLUNK_HEC_SCHEME=${SPLUNK_HEC_PROTOCOL:-https} + - SPLUNK_HEC_PORT=${SPLUNK_HEC_PORT} + - SPLUNK_HEC_TOKEN=${SPLUNK_HEC_TOKEN} + - SPLUNK_HEC_INSECURESSL=${SPLUNK_HEC_INSECURESSL:-false} + - SPLUNK_SOURCETYPE_TRAPS=${SPLUNK_SOURCETYPE_TRAPS:-sc4snmp:traps} + - SPLUNK_SOURCETYPE_POLLING_EVENTS=${SPLUNK_SOURCETYPE_POLLING_EVENTS:-sc4snmp:event} + - SPLUNK_SOURCETYPE_POLLING_METRICS=${SPLUNK_SOURCETYPE_POLLING_METRICS:-sc4snmp:metric} + - SPLUNK_HEC_INDEX_EVENTS=${SPLUNK_HEC_INDEX_EVENTS:-netops} + - SPLUNK_HEC_INDEX_METRICS=${SPLUNK_HEC_INDEX_METRICS:-netmetrics} + - SPLUNK_HEC_PATH=${SPLUNK_HEC_PATH:-/services/collector} + - SPLUNK_AGGREGATE_TRAPS_EVENTS=${SPLUNK_AGGREGATE_TRAPS_EVENTS:-false} + - IGNORE_EMPTY_VARBINDS=${IGNORE_EMPTY_VARBINDS:-false} + - WALK_RETRY_MAX_INTERVAL=${WALK_RETRY_MAX_INTERVAL:-180} + - WALK_MAX_RETRIES=${WALK_MAX_RETRIES:-5} + - METRICS_INDEXING_ENABLED=${METRICS_INDEXING_ENABLED:-false} + - POLL_BASE_PROFILES=${POLL_BASE_PROFILES:-true} + - IGNORE_NOT_INCREASING_OIDS=${IGNORE_NOT_INCREASING_OIDS:-} + - LOG_LEVEL=${WORKER_LOG_LEVEL:-INFO} + - UDP_CONNECTION_TIMEOUT=${UDP_CONNECTION_TIMEOUT:-3} + - MAX_OID_TO_PROCESS=${MAX_OID_TO_PROCESS:-70} + - PROFILES_RELOAD_DELAY=${PROFILES_RELOAD_DELAY:-60} + - WORKER_CONCURRENCY=${WORKER_POLLER_CONCURRENCY:-2} + - PREFETCH_COUNT=${PREFETCH_POLLER_COUNT:-1} + image: ${SC4SNMP_IMAGE}:${SC4SNMP_TAG:-latest} + networks: + my_network: null + restart: on-failure + secrets: [] + volumes: + - ${SCHEDULER_CONFIG_FILE_ABSOLUTE_PATH}:/app/config/config.yaml + - worker-poller-pysnmp-cache-volume:/.pysnmp/ + - worker-poller-tmp:/tmp/ +version: '3.8' +volumes: + worker-poller-pysnmp-cache-volume: null + worker-poller-tmp: null diff --git a/docker_compose/docker-compose-worker-sender.yaml b/docker_compose/docker-compose-worker-sender.yaml new file mode 100644 index 000000000..dbb5f68a9 --- /dev/null +++ b/docker_compose/docker-compose-worker-sender.yaml @@ -0,0 +1,60 @@ +version: '3.8' +services: + worker-sender: + image: ${SC4SNMP_IMAGE}:${SC4SNMP_TAG:-latest} + container_name: sc4snmp-worker-sender + command: ["celery", "worker-sender"] + environment: + - CONFIG_PATH=/app/config/config.yaml + - REDIS_URL=redis://redis:6379/1 + - CELERY_BROKER_URL=redis://redis:6379/0 + - MONGO_URI=mongodb://mongo:27017/ + - SC4SNMP_VERSION=${SC4SNMP_VERSION:-0.0.0} + - MIB_SOURCES=http://snmp-mibserver:8000/asn1/@mib@ + - MIB_INDEX=http://snmp-mibserver:8000/index.csv + - MIB_STANDARD=http://snmp-mibserver:8000/standard.txt + #- OTEL_METRICS_URL= #If sim enabled + + # Splunk instance configuration + - SPLUNK_HEC_HOST=${SPLUNK_HEC_HOST} + - SPLUNK_HEC_SCHEME=${SPLUNK_HEC_PROTOCOL:-https} + - SPLUNK_HEC_PORT=${SPLUNK_HEC_PORT} + - SPLUNK_HEC_TOKEN=${SPLUNK_HEC_TOKEN} + - SPLUNK_HEC_INSECURESSL=${SPLUNK_HEC_INSECURESSL:-false} + - SPLUNK_SOURCETYPE_TRAPS=${SPLUNK_SOURCETYPE_TRAPS:-sc4snmp:traps} + - SPLUNK_SOURCETYPE_POLLING_EVENTS=${SPLUNK_SOURCETYPE_POLLING_EVENTS:-sc4snmp:event} + - SPLUNK_SOURCETYPE_POLLING_METRICS=${SPLUNK_SOURCETYPE_POLLING_METRICS:-sc4snmp:metric} + - SPLUNK_HEC_INDEX_EVENTS=${SPLUNK_HEC_INDEX_EVENTS:-netops} + - SPLUNK_HEC_INDEX_METRICS=${SPLUNK_HEC_INDEX_METRICS:-netmetrics} + - SPLUNK_HEC_PATH=${SPLUNK_HEC_PATH:-/services/collector} + - SPLUNK_AGGREGATE_TRAPS_EVENTS=${SPLUNK_AGGREGATE_TRAPS_EVENTS:-false} + - IGNORE_EMPTY_VARBINDS=${IGNORE_EMPTY_VARBINDS:-false} + + # Workers configuration + - WALK_RETRY_MAX_INTERVAL=${WALK_RETRY_MAX_INTERVAL:-180} + - WALK_MAX_RETRIES=${WALK_MAX_RETRIES:-5} + - METRICS_INDEXING_ENABLED=${METRICS_INDEXING_ENABLED:-false} + - POLL_BASE_PROFILES=${POLL_BASE_PROFILES:-true} + - IGNORE_NOT_INCREASING_OIDS=${IGNORE_NOT_INCREASING_OIDS:-} + - LOG_LEVEL=${WORKER_LOG_LEVEL:-INFO} + - UDP_CONNECTION_TIMEOUT=${UDP_CONNECTION_TIMEOUT:-3} + - MAX_OID_TO_PROCESS=${MAX_OID_TO_PROCESS:-70} + - PROFILES_RELOAD_DELAY=${PROFILES_RELOAD_DELAY:-60} + - WORKER_CONCURRENCY=${WORKER_SENDER_CONCURRENCY:-2} + - PREFETCH_COUNT=${PREFETCH_SENDER_COUNT:-1} + depends_on: + - redis + - mongo + - coredns + volumes: + - ${SCHEDULER_CONFIG_FILE_ABSOLUTE_PATH}:/app/config/config.yaml + - worker-sender-pysnmp-cache-volume:/.pysnmp/ + - worker-sender-tmp:/tmp/ + restart: on-failure + networks: + my_network: + dns: + - ${COREDNS_ADDRESS} +volumes: + worker-sender-tmp: + worker-sender-pysnmp-cache-volume: \ No newline at end of file diff --git a/docker_compose/docker-compose-worker-trap.yaml b/docker_compose/docker-compose-worker-trap.yaml new file mode 100644 index 000000000..4d1089e57 --- /dev/null +++ b/docker_compose/docker-compose-worker-trap.yaml @@ -0,0 +1,63 @@ +version: '3.8' +services: + worker-trap: + image: ${SC4SNMP_IMAGE}:${SC4SNMP_TAG:-latest} + container_name: sc4snmp-worker-trap + command: ["celery", "worker-trap"] + environment: + - CONFIG_PATH=/app/config/config.yaml + - REDIS_URL=redis://redis:6379/1 + - CELERY_BROKER_URL=redis://redis:6379/0 + - MONGO_URI=mongodb://mongo:27017/ + - SC4SNMP_VERSION=${SC4SNMP_VERSION:-0.0.0} + - MIB_SOURCES=http://snmp-mibserver:8000/asn1/@mib@ + - MIB_INDEX=http://snmp-mibserver:8000/index.csv + - MIB_STANDARD=http://snmp-mibserver:8000/standard.txt + #- OTEL_METRICS_URL= #If sim enabled + + # Splunk instance configuration + - SPLUNK_HEC_HOST=${SPLUNK_HEC_HOST} + - SPLUNK_HEC_SCHEME=${SPLUNK_HEC_PROTOCOL:-https} + - SPLUNK_HEC_PORT=${SPLUNK_HEC_PORT} + - SPLUNK_HEC_TOKEN=${SPLUNK_HEC_TOKEN} + - SPLUNK_HEC_INSECURESSL=${SPLUNK_HEC_INSECURESSL:-false} + - SPLUNK_SOURCETYPE_TRAPS=${SPLUNK_SOURCETYPE_TRAPS:-sc4snmp:traps} + - SPLUNK_SOURCETYPE_POLLING_EVENTS=${SPLUNK_SOURCETYPE_POLLING_EVENTS:-sc4snmp:event} + - SPLUNK_SOURCETYPE_POLLING_METRICS=${SPLUNK_SOURCETYPE_POLLING_METRICS:-sc4snmp:metric} + - SPLUNK_HEC_INDEX_EVENTS=${SPLUNK_HEC_INDEX_EVENTS:-netops} + - SPLUNK_HEC_INDEX_METRICS=${SPLUNK_HEC_INDEX_METRICS:-netmetrics} + - SPLUNK_HEC_PATH=${SPLUNK_HEC_PATH:-/services/collector} + - SPLUNK_AGGREGATE_TRAPS_EVENTS=${SPLUNK_AGGREGATE_TRAPS_EVENTS:-false} + - IGNORE_EMPTY_VARBINDS=${IGNORE_EMPTY_VARBINDS:-false} + + # Workers configuration + - WALK_RETRY_MAX_INTERVAL=${WALK_RETRY_MAX_INTERVAL:-180} + - WALK_MAX_RETRIES=${WALK_MAX_RETRIES:-5} + - METRICS_INDEXING_ENABLED=${METRICS_INDEXING_ENABLED:-false} + - POLL_BASE_PROFILES=${POLL_BASE_PROFILES:-true} + - IGNORE_NOT_INCREASING_OIDS=${IGNORE_NOT_INCREASING_OIDS:-} + - LOG_LEVEL=${WORKER_LOG_LEVEL:-INFO} + - UDP_CONNECTION_TIMEOUT=${UDP_CONNECTION_TIMEOUT:-3} + - MAX_OID_TO_PROCESS=${MAX_OID_TO_PROCESS:-70} + - PROFILES_RELOAD_DELAY=${PROFILES_RELOAD_DELAY:-60} + - WORKER_CONCURRENCY=${WORKER_TRAP_CONCURRENCY:-2} + - PREFETCH_COUNT=${PREFETCH_TRAP_COUNT:-1} + - RESOLVE_TRAP_ADDRESS=${RESOLVE_TRAP_ADDRESS:-false} + - MAX_DNS_CACHE_SIZE_TRAPS=${MAX_DNS_CACHE_SIZE_TRAPS:-500} + - TTL_DNS_CACHE_TRAPS=${TTL_DNS_CACHE_TRAPS:-1800} + depends_on: + - redis + - mongo + - coredns + volumes: + - ${SCHEDULER_CONFIG_FILE_ABSOLUTE_PATH}:/app/config/config.yaml + - worker-trap-pysnmp-cache-volume:/.pysnmp/ + - worker-trap-tmp:/tmp/ + restart: on-failure + networks: + my_network: + dns: + - ${COREDNS_ADDRESS} +volumes: + worker-trap-tmp: + worker-trap-pysnmp-cache-volume: \ No newline at end of file diff --git a/docker_compose/manage_secrets.py b/docker_compose/manage_secrets.py new file mode 100644 index 000000000..9ce469b13 --- /dev/null +++ b/docker_compose/manage_secrets.py @@ -0,0 +1,223 @@ +import argparse +import os +from typing import Union +import yaml + + +def human_bool(flag: Union[str, bool], default: bool = False) -> bool: + if flag is None: + return False + if isinstance(flag, bool): + return flag + if flag.lower() in [ + "true", + "1", + "t", + "y", + "yes", + ]: + return True + elif flag.lower() in [ + "false", + "0", + "f", + "n", + "no", + ]: + return False + else: + return default + + +def remove_variables_from_env(file_path, variables_to_remove): + try: + with open(file_path, 'r') as env_file: + lines = env_file.readlines() + + with open(file_path, 'w') as env_file: + for line in lines: + key = line.split('=')[0].strip() + if key not in variables_to_remove: + env_file.write(line) + + print("Variables removed successfully from .env file.") + except Exception as e: + print(f"Error: {e}") + + +def create_secrets(variables, path_to_compose_files, secret_name, worker_poller, traps): + for k, v in variables.items(): + if k != "contextEngineId" and not v: + raise ValueError(f"Value {k} is not set") + + new_secrets = [] + new_secrets_in_workers = [] + + for k, v in variables.items(): + if v: + new_secrets.append( + {"secret_name": f"{secret_name}_{k}", "secret_config": {"environment": f"{secret_name}_{k}"}}) + new_secrets_in_workers.append( + {"source": f"{secret_name}_{k}", "target": f"/app/secrets/snmpv3/{secret_name}/{k}"}) + + # Open secrets file + try: + with open(os.path.join(path_to_compose_files, "docker-compose-secrets.yaml"), 'r') as file: + secrets_file = yaml.load(file, Loader=yaml.FullLoader) + if secrets_file["secrets"] is None: + secrets_file["secrets"] = {} + for new_secret in new_secrets: + if new_secret["secret_name"] in secrets_file["secrets"]: + print(f"Secret {secret_name} already configured. New secret not added.") + return + secrets_file["secrets"][new_secret["secret_name"]] = new_secret["secret_config"] + secrets_file_ready = True + except: + print("Problem with editing docker-compose-secrets.yaml. Secret not added.") + secrets_file_ready = False + + if worker_poller: + try: + # Open poller configuration + with open(os.path.join(path_to_compose_files, "docker-compose-worker-poller.yaml"), 'r') as file: + worker_poller_file = yaml.load(file, Loader=yaml.FullLoader) + if 'secrets' not in worker_poller_file['services']['worker-poller']: + worker_poller_file['services']['worker-poller']["secrets"] = [] + worker_poller_file['services']['worker-poller']["secrets"].extend(new_secrets_in_workers) + worker_poller_file_ready = True + except: + print("Problem with editing docker-compose-worker-poller.yaml. Secret not added.") + worker_poller_file_ready = False + else: + worker_poller_file_ready = True + + if traps: + try: + # Open traps configuration + with open(os.path.join(path_to_compose_files, "docker-compose-traps.yaml"), 'r') as file: + traps_file = yaml.load(file, Loader=yaml.FullLoader) + if 'secrets' not in traps_file['services']['traps']: + traps_file['services']['traps']["secrets"] = [] + traps_file['services']['traps']["secrets"].extend(new_secrets_in_workers) + traps_file_ready = True + except: + print("Problem with editing docker-compose-traps.yaml. Secret not added.") + traps_file_ready = False + else: + traps_file_ready = True + + if secrets_file_ready and worker_poller_file_ready and traps_file_ready: + with open(os.path.join(path_to_compose_files, "docker-compose-secrets.yaml"), 'w') as file: + yaml.dump(secrets_file, file, default_flow_style=False) + with open(os.path.join(path_to_compose_files, ".env"), 'a') as file: + for k, v in variables.items(): + if v: + file.write(f'\n{secret_name}_{k}={v}') + if worker_poller: + with open(os.path.join(path_to_compose_files, "docker-compose-worker-poller.yaml"), 'w') as file: + yaml.dump(worker_poller_file, file, default_flow_style=False) + if traps: + with open(os.path.join(path_to_compose_files, "docker-compose-traps.yaml"), 'w') as file: + yaml.dump(traps_file, file, default_flow_style=False) + + +def delete_secrets(variables, path_to_compose_files, secret_name, worker_poller, traps): + secrets = [] + for key in variables.keys(): + secrets.append(f"{secret_name}_{key}") + + # Open secrets file + with open(os.path.join(path_to_compose_files, "docker-compose-secrets.yaml"), 'r') as file: + secrets_file = yaml.load(file, Loader=yaml.FullLoader) + for secret in secrets: + if secret in secrets_file["secrets"]: + del secrets_file["secrets"][secret] + with open(os.path.join(path_to_compose_files, "docker-compose-secrets.yaml"), 'w') as file: + yaml.dump(secrets_file, file, default_flow_style=False) + + # Delete secrets from .env + try: + with open(os.path.join(path_to_compose_files, ".env"), 'r') as env_file: + lines = env_file.readlines() + with open(os.path.join(path_to_compose_files, ".env"), 'w') as env_file: + lines_to_write = [] + for line in lines: + key = line.split('=')[0].strip() + if key not in secrets: + lines_to_write.append(line.strip()) + for i, line in enumerate(lines_to_write): + if i < len(lines_to_write) - 1: + env_file.write(f"{line}\n") + else: + env_file.write(line) + except Exception as e: + print(f"Error: {e}") + + if worker_poller: + # Open poller configuration + with open(os.path.join(path_to_compose_files, "docker-compose-worker-poller.yaml"), 'r') as file: + worker_poller_file = yaml.load(file, Loader=yaml.FullLoader) + worker_poller_file['services']['worker-poller']["secrets"] = list( + filter(lambda el: el["source"] not in secrets, + worker_poller_file['services']['worker-poller']["secrets"])) + with open(os.path.join(path_to_compose_files, "docker-compose-worker-poller.yaml"), 'w') as file: + yaml.dump(worker_poller_file, file, default_flow_style=False) + if traps: + # Open traps configuration + with open(os.path.join(path_to_compose_files, "docker-compose-traps.yaml"), 'r') as file: + traps_file = yaml.load(file, Loader=yaml.FullLoader) + traps_file['services']['traps']["secrets"] = list( + filter(lambda el: el["source"] not in secrets, + traps_file['services']['traps']["secrets"])) + with open(os.path.join(path_to_compose_files, "docker-compose-traps.yaml"), 'w') as file: + yaml.dump(traps_file, file, default_flow_style=False) + + +def main(): + parser = argparse.ArgumentParser(description='Manage secrets in docker compose') + parser.add_argument('--delete', default='false', help='If true, delete the secret') + parser.add_argument('--secret_name', help='Secret name') + parser.add_argument('--path_to_compose', help='Path to dockerfiles') + parser.add_argument('--worker_poller', default='true', help='Add secret to worker poller') + parser.add_argument('--traps', default='true', help='Add secret to traps') + parser.add_argument('--userName', default='', help='SNMPV3 username') + parser.add_argument('--privProtocol', default='', help='SNMPV3 privProtocol') + parser.add_argument('--privKey', default='', help='SNMPV3 privKey') + parser.add_argument('--authProtocol', default='', help='SNMPV3 authProtocol') + parser.add_argument('--authKey', default='', help='SNMPV3 authKey') + parser.add_argument('--contextEngineId', default='', help='SNMPV3 contextEngineId') + + args = parser.parse_args() + + delete_secret = human_bool(args.delete) + secret_name = args.secret_name + path_to_compose_files = args.path_to_compose + worker_poller = human_bool(args.worker_poller) + traps = human_bool(args.traps) + variables = { + "userName": args.userName, + "privProtocol": args.privProtocol, + "privKey": args.privKey, + "authProtocol": args.authProtocol, + "authKey": args.authKey, + "contextEngineId": args.contextEngineId + } + + if not os.path.exists(path_to_compose_files): + print("Path to compose files doesn't exist") + return + if not secret_name: + print("Secret name not specified") + return + + if not delete_secret: + try: + create_secrets(variables, path_to_compose_files, secret_name, worker_poller, traps) + except ValueError as e: + print(e) + else: + delete_secrets(variables, path_to_compose_files, secret_name, worker_poller, traps) + + +if __name__ == "__main__": + main() diff --git a/docs/dockercompose/1-install-docker.md b/docs/dockercompose/1-install-docker.md new file mode 100644 index 000000000..7951d433e --- /dev/null +++ b/docs/dockercompose/1-install-docker.md @@ -0,0 +1,5 @@ +# Install Docker + +To install `Docker` in your environment follow steps from the `Install using the apt repository` section from +the Docker [documentation](https://docs.docker.com/engine/install/ubuntu/#install-using-the-repository). Install the +latest version. \ No newline at end of file diff --git a/docs/dockercompose/2-download-package.md b/docs/dockercompose/2-download-package.md new file mode 100644 index 000000000..3b8dcb706 --- /dev/null +++ b/docs/dockercompose/2-download-package.md @@ -0,0 +1,35 @@ +# Download package with docker compose files + +## Downloading a package +Currently, docker compose installation is not released. Package with the necessary files can be downloaded form +the GitHub branch [fix/docker-compose](https://github.com/splunk/splunk-connect-for-snmp/pull/937/checks). After entering +this link, click the `create-compose-files` workflow which can be seen on the left side: + +![Workflows](../images/dockercompose/workflows.png){style="border:2px solid; width:500px; height:auto" } + +