From d708df9046771dd4a6e413c47fd5bda7421a02af Mon Sep 17 00:00:00 2001 From: ajasnosz <139114006+ajasnosz@users.noreply.github.com> Date: Mon, 22 Jul 2024 08:55:18 +0200 Subject: [PATCH] fix: add docker logs (#1038) --- CHANGELOG.md | 1 + docker_compose/.env | 1 + docker_compose/manage_logs.py | 200 ++++++++++++++++++ .../dockercompose/6-env-file-configuration.md | 31 +-- docs/dockercompose/9-splunk-logging.md | 66 ++++++ mkdocs.yml | 1 + 6 files changed, 285 insertions(+), 15 deletions(-) create mode 100644 docker_compose/manage_logs.py create mode 100644 docs/dockercompose/9-splunk-logging.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 47fd7140c..26af0c1e7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ - released beta version of improved polling performance - added `yamllint` validation for the `values.yaml` formatting - added "in code" validation of groups and profiles +- added logs configuration to docker compose deployment ### Fixed - fixed a bug with configuration from values.yaml not being transferred to the UI while migrating to SC4SNMP-UI diff --git a/docker_compose/.env b/docker_compose/.env index 0180f675f..917c02645 100644 --- a/docker_compose/.env +++ b/docker_compose/.env @@ -33,6 +33,7 @@ SPLUNK_HEC_INDEX_METRICS=netmetrics SPLUNK_HEC_PATH=/services/collector SPLUNK_AGGREGATE_TRAPS_EVENTS=false IGNORE_EMPTY_VARBINDS=false +SPLUNK_LOG_INDEX= # Workers configration WALK_RETRY_MAX_INTERVAL=180 diff --git a/docker_compose/manage_logs.py b/docker_compose/manage_logs.py new file mode 100644 index 000000000..c5d4ca8be --- /dev/null +++ b/docker_compose/manage_logs.py @@ -0,0 +1,200 @@ +import argparse +import os +import re +from typing import Union + +import ruamel.yaml + +DEPENDENCIES = ["snmp-mibserver", "redis", "mongo"] + + +def human_bool(flag: Union[str, bool], default: bool = False) -> bool: + if flag is None: + return False + if isinstance(flag, bool): + return flag + if flag.lower() in [ + "true", + "1", + "t", + "y", + "yes", + ]: + return True + elif flag.lower() in [ + "false", + "0", + "f", + "n", + "no", + ]: + return False + else: + return default + + +def read_var_from_env(path_to_compose_files: str) -> dict: + logging_keys = [ + "SPLUNK_HEC_TOKEN", + "SPLUNK_HEC_PROTOCOL", + "SPLUNK_HEC_HOST", + "SPLUNK_HEC_PORT", + "SPLUNK_LOG_INDEX", + "SPLUNK_HEC_INSECURESSL", + ] + environment = dict() + try: + with open(path_to_compose_files + "/.env") as env_file: + for line in env_file.readlines(): + if any(k in line for k in logging_keys): + line = line.removesuffix("\n") + split_line = line.split("=", 1) + environment[split_line[0]] = split_line[1] + return environment + except Exception as e: + print(f"Error occurred: {e}") + raise Exception(f"Error occurred: {e}") + + +def load_template(environment: dict, service_name: str) -> dict: + yaml = ruamel.yaml.YAML() + template = f""" + logging: + driver: "splunk" + options: + splunk-token: "{environment['SPLUNK_HEC_TOKEN']}" + splunk-url: "{environment['SPLUNK_HEC_PROTOCOL']}://{environment['SPLUNK_HEC_HOST']}:{environment['SPLUNK_HEC_PORT']}" + splunk-index: "{environment['SPLUNK_LOG_INDEX']}" + splunk-insecureskipverify: "{environment['SPLUNK_HEC_INSECURESSL']}" + splunk-sourcetype: "docker:container:splunk-connect-for-snmp-{service_name}" + """ + template_yaml = yaml.load(template) + return template_yaml + + +def create_logs(environment, path_to_compose_files): + files_list = os.listdir(path_to_compose_files) + compose_files = [ + f + for f in files_list + if re.match(r"docker-compose-(?!dependencies|network|secrets).*.yaml", f) + ] + + for filename in compose_files: + service_name = filename.removeprefix("docker-compose-").removesuffix(".yaml") + template_yaml = load_template(environment, service_name) + try: + yaml = ruamel.yaml.YAML() + with open(os.path.join(path_to_compose_files, filename)) as file: + yaml_file = yaml.load(file) + yaml_file["services"][service_name].update(template_yaml) + + with open(os.path.join(path_to_compose_files, filename), "w") as file: + yaml.dump(yaml_file, file) + except Exception as e: + print( + f"Problem with editing docker-compose-{service_name}.yaml. Error: {e}" + ) + + try: + yaml2 = ruamel.yaml.YAML() + with open( + os.path.join(path_to_compose_files, "docker-compose-dependencies.yaml") + ) as file: + yaml_file = yaml2.load(file) + + for service_name in DEPENDENCIES: + template_yaml = load_template(environment, service_name) + yaml_file["services"][service_name].update(template_yaml) + + with open( + os.path.join(path_to_compose_files, "docker-compose-dependencies.yaml"), "w" + ) as file: + yaml2.dump(yaml_file, file) + except Exception as e: + print(f"Problem with editing docker-compose-dependencies.yaml. Error: {e}") + + +def delete_logs(path_to_compose_files): + files_list = os.listdir(path_to_compose_files) + compose_files = [ + f + for f in files_list + if re.match(r"docker-compose-(?!dependencies|network|secrets).*.yaml", f) + ] + + for filename in compose_files: + service_name = filename.removeprefix("docker-compose-").removesuffix(".yaml") + try: + with open(os.path.join(path_to_compose_files, filename)) as file: + yaml = ruamel.yaml.YAML() + yaml_file = yaml.load(file) + + yaml_file["services"][service_name]["logging"]["driver"] = "json-file" + yaml_file["services"][service_name]["logging"].pop("options") + + with open(os.path.join(path_to_compose_files, filename), "w") as file: + yaml.dump(yaml_file, file) + except Exception as e: + print( + f"Problem with editing docker-compose-{service_name}.yaml. Error: {e}" + ) + + try: + with open( + os.path.join(path_to_compose_files, "docker-compose-dependencies.yaml") + ) as file: + yaml2 = ruamel.yaml.YAML() + yaml_file = yaml2.load(file) + + for service_name in DEPENDENCIES: + yaml_file["services"][service_name]["logging"]["driver"] = "json-file" + yaml_file["services"][service_name]["logging"].pop("options") + + with open( + os.path.join(path_to_compose_files, "docker-compose-dependencies.yaml"), "w" + ) as file: + yaml2.dump(yaml_file, file) + except Exception as e: + print(f"Problem with editing docker-compose-dependencies.yaml. Error: {e}") + + +def main(): + parser = argparse.ArgumentParser(description="Manage logs in docker compose") + parser.add_argument( + "-e", "--enable_logs", action="store_true", help="Enables the logs" + ) + parser.add_argument( + "-p", "--path_to_compose", required=True, help="Path to dockerfiles" + ) + parser.add_argument( + "-d", "--disable_logs", action="store_true", help="Disables the logs" + ) + + args = parser.parse_args() + + # Assign inputs from command line to variables + enable_logs = human_bool(args.enable_logs) + path_to_compose_files = args.path_to_compose + disable_logs = human_bool(args.disable_logs) + + if not os.path.exists(path_to_compose_files): + print("Path to compose files doesn't exist") + return + + env = read_var_from_env(path_to_compose_files) + + if enable_logs: + try: + create_logs(env, path_to_compose_files) + except ValueError as e: + print(e) + if disable_logs: + try: + delete_logs(path_to_compose_files) + except ValueError as e: + print(e) + + +if __name__ == "__main__": + main() diff --git a/docs/dockercompose/6-env-file-configuration.md b/docs/dockercompose/6-env-file-configuration.md index 8a78e133d..283b88af7 100644 --- a/docs/dockercompose/6-env-file-configuration.md +++ b/docs/dockercompose/6-env-file-configuration.md @@ -30,21 +30,22 @@ Inside the directory with the docker compose files, there is a `.env`. Variables ## Splunk instance -| Variable | Description | -|-------------------------------------|----------------------------------------------------------------------------------------------------------------------| -| `SPLUNK_HEC_HOST` | IP address or a domain name of a Splunk instance to send data to | -| `SPLUNK_HEC_PROTOCOL` | The protocol of the HEC endpoint: `https` or `http` | -| `SPLUNK_HEC_PORT` | The port of the HEC endpoint | -| `SPLUNK_HEC_TOKEN` | Splunk HTTP Event Collector token | -| `SPLUNK_HEC_INSECURESSL` | Whether to skip checking the certificate of the HEC endpoint when sending data over HTTPS | -| `SPLUNK_SOURCETYPE_TRAPS` | Splunk sourcetype for trap events | -| `SPLUNK_SOURCETYPE_POLLING_EVENTS` | Splunk sourcetype for non-metric polling events | -| `SPLUNK_SOURCETYPE_POLLING_METRICS` | Splunk sourcetype for metric polling events | -| `SPLUNK_HEC_INDEX_EVENTS` | Name of the Splunk event index | -| `SPLUNK_HEC_INDEX_METRICS` | Name of the Splunk metrics index | -| `SPLUNK_HEC_PATH` | Path for the HEC endpoint | -| `SPLUNK_AGGREGATE_TRAPS_EVENTS` | When set to true makes traps events collected as one event inside splunk | -| `IGNORE_EMPTY_VARBINDS` | Details can be found in [empty snmp response message issue](../bestpractices.md#empty-snmp-response-message-problem) | +| Variable | Description | +|-------------------------------------|-----------------------------------------------------------------------------------------------------------------------| +| `SPLUNK_HEC_HOST` | IP address or a domain name of a Splunk instance to send data to | +| `SPLUNK_HEC_PROTOCOL` | The protocol of the HEC endpoint: `https` or `http` | +| `SPLUNK_HEC_PORT` | The port of the HEC endpoint | +| `SPLUNK_HEC_TOKEN` | Splunk HTTP Event Collector token | +| `SPLUNK_HEC_INSECURESSL` | Whether to skip checking the certificate of the HEC endpoint when sending data over HTTPS | +| `SPLUNK_SOURCETYPE_TRAPS` | Splunk sourcetype for trap events | +| `SPLUNK_SOURCETYPE_POLLING_EVENTS` | Splunk sourcetype for non-metric polling events | +| `SPLUNK_SOURCETYPE_POLLING_METRICS` | Splunk sourcetype for metric polling events | +| `SPLUNK_HEC_INDEX_EVENTS` | Name of the Splunk event index | +| `SPLUNK_HEC_INDEX_METRICS` | Name of the Splunk metrics index | +| `SPLUNK_HEC_PATH` | Path for the HEC endpoint | +| `SPLUNK_AGGREGATE_TRAPS_EVENTS` | When set to true makes traps events collected as one event inside splunk | +| `IGNORE_EMPTY_VARBINDS` | Details can be found in [empty snmp response message issue](../bestpractices.md#empty-snmp-response-message-problem) | +| `SPLUNK_LOG_INDEX` | Event index in Splunk where logs from docker containers would be sent | ## Workers diff --git a/docs/dockercompose/9-splunk-logging.md b/docs/dockercompose/9-splunk-logging.md new file mode 100644 index 000000000..229f8862e --- /dev/null +++ b/docs/dockercompose/9-splunk-logging.md @@ -0,0 +1,66 @@ +# Logging + +The default configuration of docker compose is not sending the logs to Splunk. Container logs can be accessed with command: +``` +docker logs +``` + +Creating logs requires updating configuration of several docker compose files. To simplify this process, inside the +`docker_compose` package there is a `manage_logs.py` file which will automatically manage logs. + +## Prerequisites + +Running script requires installation of `ruamel.yaml` package for python. It can be done with command: +``` +pip3 install ruamel.yaml +``` + +The following parameters have to be configured in `.env` file: +`SPLUNK_HEC_TOKEN`, `SPLUNK_HEC_PROTOCOL`, `SPLUNK_HEC_HOST`, `SPLUNK_HEC_PORT`, `SPLUNK_LOG_INDEX`, `SPLUNK_HEC_INSECURESSL`. + +More about `.env` configuration can be found in [.env file configuration](./6-env-file-configuration.md). + +## Enabling logging + +To enable a logging `manage_logs.py` must be run with the following flags: + +| Flag | Description | +|---------------------------|------------------------------------------------------| +| `-e`, `--enable_logs` | Flag enabling the logs | +| `-p`, `--path_to_compose` | Absolute path to directory with docker compose files | + +Example of enabling logs: +``` +python3 manage_logs.py --path_to_compose /home/ubuntu/docker_compose --enable_logs +``` + +The script will add required configuration for logging under services in docker compose files. +To apply the changes run the +``` +sudo docker compose $(find docker* | sed -e 's/^/-f /') up -d +``` +command inside the `docker_compose` directory. + +## Disabling the logs + +To disable logs `manage_logs.py` must be run with the following flags: + +| Flag | Description | +|---------------------------|------------------------------------------------------| +| `-d`, `--disable_logs` | Flag disabling the logs | +| `-p`, `--path_to_compose` | Absolute path to directory with docker compose files | + +Running the disable command will replace the `logging.driver` section with default docker driver `json-file`. + +Example of disabling logs: +``` +python3 manage_logs.py --path_to_compose /home/ubuntu/docker_compose --disable_logs +``` + +To apply the changes run the +``` +sudo docker compose $(find docker* | sed -e 's/^/-f /') up -d +``` +command inside the `docker_compose` directory. + +After that the logs can be reached with `docker logs` command. diff --git a/mkdocs.yml b/mkdocs.yml index d6a78f71d..7af1c79c3 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -77,6 +77,7 @@ nav: - .env file configuration: "dockercompose/6-env-file-configuration.md" - SNMPv3 secrets configuration: "dockercompose/7-snmpv3-secrets.md" - Offline installation: "dockercompose/8-offline-installation.md" + - Sending logs to Splunk: "dockercompose/9-splunk-logging.md" - Lightweight installation: "small-environment.md" - Planning: "planning.md" - Security: "security.md"