diff --git a/appliance/README.md b/appliance/README.md index 14166fd..7680384 100644 --- a/appliance/README.md +++ b/appliance/README.md @@ -12,6 +12,43 @@ more info: - https://github.com/rsyslog/rsyslog/issues/2368 - https://github.com/rsyslog/rsyslog/projects/5 +## Deployment example for Kubernetes: +To run your rsyslog-container in production under Kubernetes, you can start with this template +- ../rsyslog-deployment.yaml + +### Kubernetes infos for high load traffic +If you have much traffic from the same source-ip, you have to tune your Loadbalancer-settings. +I case of a ipvs-implementation in Kubernetes: +``` +ipvsadm -Ln --timeout +### e.g.: Timeout (tcp tcpfin udp): 900 120 300 +### => if you would like Load-Distribution for the same Source-IP => set UDP-Timeout to 1: +ipvsadm --set 900 120 1 +``` + +To make this settings permanent, you can do it e.g. in this way: +``` +cat < /etc/systemd/system/ipvs-config.service +[Unit] +Description=Configure IPVS +After=network.target + +[Service] +Type=oneshot +ExecStart=/sbin/ipvsadm --set 900 120 1 +RemainAfterExit=false +StandardOutput=journal + +[Install] +WantedBy=multi-user.target +EOF + + +systemctl daemon-reload +systemctl enable ipvs-config +systemctl start ipvs-config +``` + ## projects that provide docker containers: - https://github.com/deoren/rsyslog-docker (based on @halfer provided files) diff --git a/appliance/alpine/Dockerfile b/appliance/alpine/Dockerfile index d02c10e..9afbf1f 100644 --- a/appliance/alpine/Dockerfile +++ b/appliance/alpine/Dockerfile @@ -1,29 +1,69 @@ -FROM alpine:3.7 +FROM alpine:3.15 LABEL maintainer="rgerhards@adiscon.com" -COPY rsyslog@lists.adiscon.com-5a55e598.rsa.pub /etc/apk/keys/rsyslog@lists.adiscon.com-5a55e598.rsa.pub -RUN echo "http://alpine.rsyslog.com/3.7/stable" >> /etc/apk/repositories \ - && apk --no-cache update \ - && apk add --no-cache \ - rsyslog \ - rsyslog-elasticsearch \ - rsyslog-imptcp \ - rsyslog-imrelp \ - rsyslog-mmjsonparse \ - rsyslog-mmutf8fix \ - rsyslog-omrelp \ - rsyslog-omstdout -RUN adduser -s /bin/ash -D rsyslog rsyslog \ - && echo "rsyslog ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers -VOLUME /config /work /logs -CMD ["rsyslog"] -ENTRYPOINT ["/home/appliance/starter.sh"] -COPY rsyslog.conf /etc/rsyslog.conf -COPY rsyslog.conf.d/*.conf /etc/rsyslog.conf.d/ +COPY rsyslog@lists.adiscon.com-5a55e598.rsa.pub /etc/apk/keys/rsyslog@lists.adiscon.com-5a55e598.rsa.pub + +USER root +RUN sed -i 's/https/http/' /etc/apk/repositories + +#RUN echo "http://alpine.rsyslog.com/3.15/stable" >> /etc/apk/repositories \ +RUN apk --no-cache update \ + && apk add --no-cache \ + rsyslog \ + rsyslog-mmjsonparse \ + rsyslog-pgsql \ + rsyslog-hiredis \ + rsyslog-snmp \ + rsyslog-mmnormalize \ + rsyslog-dbg \ + rsyslog-clickhouse \ + rsyslog-mmsequence \ + rsyslog-gssapi \ + rsyslog-pmlastmsg \ + rsyslog-mmfields \ + rsyslog-tls \ + rsyslog-mmpstrucdata \ + rsyslog-elasticsearch \ + rsyslog-mmsnmptrapd \ + rsyslog-udpspoof \ + rsyslog-uxsock \ + rsyslog-http \ + rsyslog-imdocker \ + rsyslog-relp \ + rsyslog-mmaudit \ + rsyslog-mysql \ + rsyslog-mmrm1stspace \ + rsyslog-mmutf8fix \ + rsyslog-crypto \ + rsyslog-libdbi \ + rsyslog-pmsnare \ + rsyslog-openrc \ + rsyslog-mmcount \ + rsyslog-zmq \ + rsyslog-mmanon \ + rsyslog-rabbitmq \ + rsyslog-mmdblookup \ + rsyslog-pmaixforwardedfrom \ + libc-utils +RUN adduser -s /bin/ash -D rsyslog rsyslog \ + && echo "rsyslog ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers +VOLUME /config /work /logs +COPY rsyslog.conf /etc/rsyslog.conf +COPY rsyslog.conf.d/*.conf /etc/rsyslog.conf.d/ # done base system setup WORKDIR /home/appliance -COPY starter.sh CONTAINER.* ./ -COPY internal/* ./internal/ -COPY tools/* ./tools/ -RUN echo "`date +%F` (`date +%s`)" > CONTAINER.release \ - && chown -R rsyslog:rsyslog * +COPY starter.sh CONTAINER.* ./ +COPY internal/* ./internal/ +COPY internal/container_config /config/ +COPY internal/droprules.conf /config/ +COPY tools/* ./tools/ +COPY custom-builds/* /usr/lib/rsyslog/ +COPY custom-builds-liblog/* /usr/lib/ +RUN echo "`date +%F` (`date +%s`)" > CONTAINER.release \ + && chown -R rsyslog:rsyslog * && \ + chmod +x starter.sh + +USER rsyslog +CMD ["rsyslog"] +ENTRYPOINT ["/home/appliance/starter.sh"] + diff --git a/appliance/alpine/custom-builds-liblog/liblognorm.a b/appliance/alpine/custom-builds-liblog/liblognorm.a new file mode 100644 index 0000000..fe3b2be Binary files /dev/null and b/appliance/alpine/custom-builds-liblog/liblognorm.a differ diff --git a/appliance/alpine/custom-builds-liblog/liblognorm.so.5.1.0 b/appliance/alpine/custom-builds-liblog/liblognorm.so.5.1.0 new file mode 100644 index 0000000..772389e Binary files /dev/null and b/appliance/alpine/custom-builds-liblog/liblognorm.so.5.1.0 differ diff --git a/appliance/alpine/custom-builds-liblog/lognormalizer b/appliance/alpine/custom-builds-liblog/lognormalizer new file mode 100644 index 0000000..c9cd871 Binary files /dev/null and b/appliance/alpine/custom-builds-liblog/lognormalizer differ diff --git a/appliance/alpine/custom-builds/omelasticsearch.a b/appliance/alpine/custom-builds/omelasticsearch.a new file mode 100644 index 0000000..33f2481 Binary files /dev/null and b/appliance/alpine/custom-builds/omelasticsearch.a differ diff --git a/appliance/alpine/custom-builds/omelasticsearch.la b/appliance/alpine/custom-builds/omelasticsearch.la new file mode 100644 index 0000000..427c60b --- /dev/null +++ b/appliance/alpine/custom-builds/omelasticsearch.la @@ -0,0 +1,41 @@ +# omelasticsearch.la - a libtool library file +# Generated by libtool (GNU libtool) 2.4.6 +# +# Please DO NOT delete this file! +# It is necessary for linking the library. + +# The name that we can dlopen(3). +dlname='' + +# Names of this library. +library_names='' + +# The name of the static archive. +old_library='omelasticsearch.a' + +# Linker flags that cannot go in dependency_libs. +inherited_linker_flags='' + +# Libraries that this one depends upon. +dependency_libs=' -lcurl -lm' + +# Names of additional weak libraries provided by this library +weak_library_names='' + +# Version information for omelasticsearch. +current=0 +age=0 +revision=0 + +# Is this an already installed library? +installed=yes + +# Should we warn about portability when linking against -modules? +shouldnotlink=yes + +# Files to dlopen/dlpreopen +dlopen='' +dlpreopen='' + +# Directory that this library needs to be installed in: +libdir='/usr/local/lib/rsyslog' diff --git a/appliance/alpine/custom-builds/omelasticsearch.so b/appliance/alpine/custom-builds/omelasticsearch.so new file mode 100644 index 0000000..0dc1c6f Binary files /dev/null and b/appliance/alpine/custom-builds/omelasticsearch.so differ diff --git a/appliance/alpine/internal/container_config b/appliance/alpine/internal/container_config index d336b62..6d12e5a 100644 --- a/appliance/alpine/internal/container_config +++ b/appliance/alpine/internal/container_config @@ -9,7 +9,7 @@ export TZ=UTC export ENABLE_STATISTICS=on # Do we write log files? -export ENABLE_LOGFILES=on # yes, we do (comment out to disable) +export ENABLE_LOGFILES=off # (comment out to disable) # Where do we write to? # path for host-specific files is: /logs/hosts/HOSTNAME export LOGFILES_STORE="/logs/hosts/%hostname:::secpath-replace%/messages.log" diff --git a/appliance/alpine/starter.sh b/appliance/alpine/starter.sh index 3731db4..4195a23 100755 --- a/appliance/alpine/starter.sh +++ b/appliance/alpine/starter.sh @@ -6,7 +6,8 @@ if [ "$CONTAINER_SILENT" != "on" ]; then echo `cat CONTAINER.name` version `cat CONTAINER.release` - `cat CONTAINER.homepage` echo `cat CONTAINER.copyright` echo - echo "WARNING: this is an experimental container - do not use in production" + echo "This container is build for production." + echo "See online: https://github.com/EHerzog76/rsyslog-docker" echo fi @@ -40,6 +41,7 @@ echo "Using rsyslog configuration file: $RSYSLOG_CONF" if [ -f tools/$1 ]; then + #rsyslogd -D -N 1 -f /etc/rsyslog.conf source tools/$1 else echo "ERROR: command not known: $*" diff --git a/appliance/alpine/tools/CONTAINER.copyright b/appliance/alpine/tools/CONTAINER.copyright new file mode 100644 index 0000000..975666e --- /dev/null +++ b/appliance/alpine/tools/CONTAINER.copyright @@ -0,0 +1 @@ +Copyright (C) 2018 by Rainer Gerhards and Adiscon GmbH - released under ASL 2.0 diff --git a/appliance/alpine/tools/CONTAINER.homepage b/appliance/alpine/tools/CONTAINER.homepage new file mode 100644 index 0000000..d895ff5 --- /dev/null +++ b/appliance/alpine/tools/CONTAINER.homepage @@ -0,0 +1 @@ +http://www.syslogappliance.de diff --git a/appliance/alpine/tools/CONTAINER.name b/appliance/alpine/tools/CONTAINER.name new file mode 100644 index 0000000..83f5c37 --- /dev/null +++ b/appliance/alpine/tools/CONTAINER.name @@ -0,0 +1 @@ +rsyslog appliance diff --git a/appliance/alpine/tools/Dockerfile b/appliance/alpine/tools/Dockerfile new file mode 100644 index 0000000..d02c10e --- /dev/null +++ b/appliance/alpine/tools/Dockerfile @@ -0,0 +1,29 @@ +FROM alpine:3.7 +LABEL maintainer="rgerhards@adiscon.com" +COPY rsyslog@lists.adiscon.com-5a55e598.rsa.pub /etc/apk/keys/rsyslog@lists.adiscon.com-5a55e598.rsa.pub +RUN echo "http://alpine.rsyslog.com/3.7/stable" >> /etc/apk/repositories \ + && apk --no-cache update \ + && apk add --no-cache \ + rsyslog \ + rsyslog-elasticsearch \ + rsyslog-imptcp \ + rsyslog-imrelp \ + rsyslog-mmjsonparse \ + rsyslog-mmutf8fix \ + rsyslog-omrelp \ + rsyslog-omstdout +RUN adduser -s /bin/ash -D rsyslog rsyslog \ + && echo "rsyslog ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers +VOLUME /config /work /logs +CMD ["rsyslog"] +ENTRYPOINT ["/home/appliance/starter.sh"] +COPY rsyslog.conf /etc/rsyslog.conf +COPY rsyslog.conf.d/*.conf /etc/rsyslog.conf.d/ +# done base system setup + +WORKDIR /home/appliance +COPY starter.sh CONTAINER.* ./ +COPY internal/* ./internal/ +COPY tools/* ./tools/ +RUN echo "`date +%F` (`date +%s`)" > CONTAINER.release \ + && chown -R rsyslog:rsyslog * diff --git a/appliance/alpine/tools/README.md b/appliance/alpine/tools/README.md new file mode 100644 index 0000000..803722d --- /dev/null +++ b/appliance/alpine/tools/README.md @@ -0,0 +1,89 @@ +## Configuring the syslog appliance + +### Files + +Container and rsyslog configuration are read from the /config directory. You should +mount it to a volume: + + $ docker run -v myconfig:/config ... + +Upon initial creation of the volume, it is populated with a default file that +you can modify. Note that this happens only for **volume** mounts. + +To show the current container config, run + + $ docker run ... tools/show-config + +You can also use this to get a template for your own container config if you use +bind mounts instead of volumes. In this case, make sure that you have **not** +mounted /config - the container will then use it's own default file. + +Note: volumes are automatically populated with the default file upon creation. + +### Environment Variables + +- TZ + + Default: /etc/localtime + + Change it to set a specific timezone, e.g. TZ=UTC + +- RSYSLOG_CONF + + Default: /etc/rsyslog.conf + + If you want to totally replace the default rsyslog configuration with + your custom config, + + 1. create a config file in /config volume or bind mount, e.g. myrsyslog.conf + 2. set RSYSLOG_CONF=/config/myrsyslog.conf + + Keep in mind that the myconfig: volume is accessible via /config inside the + container. + +- LOGSENE_TOKEN + + Default: disabled + + If you are using Sematext Logsene, set this to your Logsene token. + +- LOGSENE_URL + + Default: disabled + + If you are using Sematext Logsene, set this to the Logsene URL. Ex: logsene-receiver.sematext.com or logsene-receiver.eu.sematext.com + +- RSYSLOG_CONFIG_BASE64 + + Default: disabled + + If you would like to overwite the `/etc/rsyslog.conf` file, _without_ mounting a configuration file into the container, you can use this variable. The contents are the base64 encoded `rsyslog.conf` file contents, without newlines. This can be generated with the following command: `cat rsyslog.conf | base64 | tr -d '\n'`. On startup, the contents of the environment variable will be decoded and overwrite the `/etc/rsyslog.conf` file. + +# Runtime Environment + +## Volumes + +### /config + +Holds the container configuration, also the recommended place for overwriting +the rsyslog configuration. + +This volume can be mounted read-only after initial population with sample files. + +### /work + +The rsyslog work directory. This is used for spool files and other files that +rsyslog needs to be persisted over runs. + +This volume needs to be mounted writable and **must** be persisted between +container invocations. + +**Warning: this volume is specific to one rsyslog instance.** It **must not** +be shared between multiple container instances, else strange problems may +occur. + +### /logs + +This holds log files if the container is configured to write them. + +Needs to be mounted writable. diff --git a/appliance/alpine/tools/build.sh b/appliance/alpine/tools/build.sh new file mode 100644 index 0000000..18150e8 --- /dev/null +++ b/appliance/alpine/tools/build.sh @@ -0,0 +1 @@ +docker build $* -t rsyslog/syslog_appliance_alpine:latest . diff --git a/appliance/alpine/tools/custom-builds/omelasticsearch.a b/appliance/alpine/tools/custom-builds/omelasticsearch.a new file mode 100644 index 0000000..33f2481 Binary files /dev/null and b/appliance/alpine/tools/custom-builds/omelasticsearch.a differ diff --git a/appliance/alpine/tools/custom-builds/omelasticsearch.la b/appliance/alpine/tools/custom-builds/omelasticsearch.la new file mode 100644 index 0000000..427c60b --- /dev/null +++ b/appliance/alpine/tools/custom-builds/omelasticsearch.la @@ -0,0 +1,41 @@ +# omelasticsearch.la - a libtool library file +# Generated by libtool (GNU libtool) 2.4.6 +# +# Please DO NOT delete this file! +# It is necessary for linking the library. + +# The name that we can dlopen(3). +dlname='' + +# Names of this library. +library_names='' + +# The name of the static archive. +old_library='omelasticsearch.a' + +# Linker flags that cannot go in dependency_libs. +inherited_linker_flags='' + +# Libraries that this one depends upon. +dependency_libs=' -lcurl -lm' + +# Names of additional weak libraries provided by this library +weak_library_names='' + +# Version information for omelasticsearch. +current=0 +age=0 +revision=0 + +# Is this an already installed library? +installed=yes + +# Should we warn about portability when linking against -modules? +shouldnotlink=yes + +# Files to dlopen/dlpreopen +dlopen='' +dlpreopen='' + +# Directory that this library needs to be installed in: +libdir='/usr/local/lib/rsyslog' diff --git a/appliance/alpine/tools/custom-builds/omelasticsearch.so b/appliance/alpine/tools/custom-builds/omelasticsearch.so new file mode 100644 index 0000000..0dc1c6f Binary files /dev/null and b/appliance/alpine/tools/custom-builds/omelasticsearch.so differ diff --git a/appliance/alpine/tools/internal/build-config b/appliance/alpine/tools/internal/build-config new file mode 100644 index 0000000..8bff1a0 --- /dev/null +++ b/appliance/alpine/tools/internal/build-config @@ -0,0 +1,25 @@ +#!/bin/ash +# This reads the meta config (environment variables) and prepares environment +# variables used by rsyslog.conf so that we can finally startup rsyslog. +# Errors are detected and will lead to abort of the startup. + +export CNF_CALL_LOG_TO_LOGFILES= +if [ "$ENABLE_LOGFILES" == "on" ]; then + if [ -z $LOGFILES_STORE ]; then + echo "Error: ENABLE_LOGFILES set, but LOGFILES_STORE not set - please correct" + exit 1 + fi + export CNF_CALL_LOG_TO_LOGFILES="call log_to_files" +fi # end LOGFILES + +export CNF_CALL_LOG_TO_LOGSENE= +export ENABLE_LOGSENE=off +if [ "$LOGSENE_TOKEN" != "" ]; then + if [ -z $LOGSENE_URL ]; then + echo "Error: LOGSENE_TOKEN set, but LOGSENE_URL not set - please correct" + exit 1 + fi + export ENABLE_LOGSENE=on + export CNF_CALL_LOG_TO_LOGSENE="call log_to_logsene" +fi # end LOGSENE +env diff --git a/appliance/alpine/tools/internal/container_config b/appliance/alpine/tools/internal/container_config new file mode 100644 index 0000000..d336b62 --- /dev/null +++ b/appliance/alpine/tools/internal/container_config @@ -0,0 +1,29 @@ +# rsyslog syslog appliance container configuration. +# This file will be sourced upon container startup. +# Uncomment those settings that you need and set them to your +# desired values. + +# general container app settings: +export TZ=UTC +#export CONTAINER_SILENT=on # do not emit startup message +export ENABLE_STATISTICS=on + +# Do we write log files? +export ENABLE_LOGFILES=on # yes, we do (comment out to disable) +# Where do we write to? +# path for host-specific files is: /logs/hosts/HOSTNAME +export LOGFILES_STORE="/logs/hosts/%hostname:::secpath-replace%/messages.log" +# you can of course overwrite this. For example, the below definition +# uses the program name instead of a fixed name "messages.log". That means +# for each host, a separate file for each program will be generated. +#export LOGFILES_STORE="/logs/hosts/%hostname:::secpath-replace%/%programname:::secpath-replace%.log" + +# If you have an account with Logsene, enter your access +# information below: +#export LOGSENE_TOKEN= +#export LOGSENE_URL=logsene-receiver.eu.sematext.com + +# Settings for debugging the container +#export USE_VALGRIND=on +#export RSYSLOG_DEBUG="debug nostdout" +#export RSYSLOG_DEBUGLOG="/logs/rsyslog-internal-debug.log" diff --git a/appliance/alpine/tools/internal/droprules.conf b/appliance/alpine/tools/internal/droprules.conf new file mode 100644 index 0000000..6c6da9c --- /dev/null +++ b/appliance/alpine/tools/internal/droprules.conf @@ -0,0 +1,5 @@ +# This file permits to do drop unneded messages +# Uncomment the samples below to activate them. You can +# add as many drop rules as you like. + +#if $rawmsg contains "somejunk" then stop diff --git a/appliance/alpine/tools/internal/install-valgrind b/appliance/alpine/tools/internal/install-valgrind new file mode 100644 index 0000000..6993571 --- /dev/null +++ b/appliance/alpine/tools/internal/install-valgrind @@ -0,0 +1,11 @@ +#!/bin/ash +touch /Are-we-Writable +if [ ! -f /Are-we-Writable ]; then + echo "ERROR: we need to install valgrind, so the container must" + echo " NOT be read-only for this debug session." + exit 1 +fi +rm /Are-we-Writable +echo installing valgrind for debugging -- need to access Alpine repositories +apk update +apk add --no-cache rsyslog-dbg musl-dbg valgrind diff --git a/appliance/alpine/tools/internal/set-defaults b/appliance/alpine/tools/internal/set-defaults new file mode 100644 index 0000000..d838a44 --- /dev/null +++ b/appliance/alpine/tools/internal/set-defaults @@ -0,0 +1,8 @@ +# set defaults if not present. This enables us to pre-populate bind +# mounts, which makes sense in our case. +if [ ! -f /config/container_config ]; then + cp internal/container_config /config +fi +if [ ! -f /config/droprules.conf ]; then + cp internal/droprules.conf /config +fi diff --git a/appliance/alpine/tools/rsyslog b/appliance/alpine/tools/rsyslog index 20d166c..51a97ae 100644 --- a/appliance/alpine/tools/rsyslog +++ b/appliance/alpine/tools/rsyslog @@ -1,3 +1,7 @@ # This NEEDS to be sourced (else we don't get PID 1!) source internal/build-config -exec $VALGRIND /usr/sbin/rsyslogd -n $RSYSLOG_DEBUG_FLAG -f$RSYSLOG_CONF +if [ "$USE_VALGRIND" = "on" ]; then + exec $VALGRIND /usr/sbin/rsyslogd -n $RSYSLOG_DEBUG_FLAG -f$RSYSLOG_CONF +else + exec /usr/sbin/rsyslogd -n $RSYSLOG_DEBUG_FLAG -f$RSYSLOG_CONF +fi diff --git a/appliance/alpine/tools/rsyslog.conf b/appliance/alpine/tools/rsyslog.conf new file mode 100644 index 0000000..dac55a6 --- /dev/null +++ b/appliance/alpine/tools/rsyslog.conf @@ -0,0 +1,34 @@ +global(processInternalMessages="on") + +#module(load="imtcp" StreamDriver.AuthMode="anon" StreamDriver.Mode="1") +module(load="impstats") # config.enabled=`echo $ENABLE_STATISTICS`) +module(load="imrelp") +module(load="imptcp") +module(load="imudp" TimeRequery="500") + +module(load="omstdout") +module(load="omelasticsearch") + +module(load="mmjsonparse") +module(load="mmutf8fix") + + +input(type="imptcp" port="514") +input(type="imudp" port="514") +input(type="imrelp" port="1601") + +# includes done explicitely +include(file="/etc/rsyslog.conf.d/log_to_logsene.conf" config.enabled=`echo $ENABLE_LOGSENE`) +include(file="/etc/rsyslog.conf.d/log_to_files.conf" config.enabled=`echo $ENABLE_LOGFILES`) + +#################### default ruleset begins #################### + +# we emit our own messages to docker console: +syslog.* :omstdout: + +include(file="/config/droprules.conf" mode="optional") # this permits the user to easily drop unwanted messages + +action(name="main_utf8fix" type="mmutf8fix" replacementChar="?") + +include(text=`echo $CNF_CALL_LOG_TO_LOGFILES`) +include(text=`echo $CNF_CALL_LOG_TO_LOGSENE`) diff --git a/appliance/alpine/tools/rsyslog.conf.d/log_to_files.conf b/appliance/alpine/tools/rsyslog.conf.d/log_to_files.conf new file mode 100644 index 0000000..bce4628 --- /dev/null +++ b/appliance/alpine/tools/rsyslog.conf.d/log_to_files.conf @@ -0,0 +1,7 @@ +# logging to files in /logs + +template(name="log_to_files_dynafile" type="string" string=`echo $LOGFILES_STORE`) +ruleset(name="log_to_files") { + /logs/debug;RSYSLOG_DebugFormat + action(type="omfile" dynafile="log_to_files_dynafile" name="log_to_logfiles") +} diff --git a/appliance/alpine/tools/rsyslog.conf.d/log_to_logsene.conf b/appliance/alpine/tools/rsyslog.conf.d/log_to_logsene.conf new file mode 100644 index 0000000..5ff0240 --- /dev/null +++ b/appliance/alpine/tools/rsyslog.conf.d/log_to_logsene.conf @@ -0,0 +1,63 @@ +# config snippet for the Logsene service +# https://sematext.com/logsene/ + +template(name="logsene_json" type="list") { + constant(value="{") + constant(value="\"@timestamp\":\"") property(name="timereported" dateFormat="rfc3339") + constant(value="\",\"host\":\"") property(name="hostname") + constant(value="\",\"severity\":\"") property(name="syslogseverity-text" caseConversion="upper") + constant(value="\",\"facility\":\"") property(name="syslogfacility-text") + constant(value="\",\"syslog-tag\":\"") property(name="syslogtag" format="json") + constant(value="\",\"source\":\"") property(name="app-name" format="json") + constant(value="\",") property(name="$!all-json" position.from="2") +} + +template(name="logsene_plain" type="list") { + constant(value="{") + constant(value="\"@timestamp\":\"") property(name="timereported" dateFormat="rfc3339") + constant(value="\",\"host\":\"") property(name="hostname") + constant(value="\",\"severity\":\"") property(name="syslogseverity-text" caseConversion="upper") + constant(value="\",\"facility\":\"") property(name="syslogfacility-text") + constant(value="\",\"syslog-tag\":\"") property(name="syslogtag" format="json") + constant(value="\",\"source\":\"") property(name="app-name" format="json") + constant(value="\",\"message\":\"") property(name="msg" format="json") + constant(value="\"}") +} + + +ruleset(name="log_to_logsene" + queue.type="LinkedList" queue.dequeueBatchSize="2000" + queue.size="100000" # do not use a too-high values, else we get trouble with 10s shutdown period! + queue.spoolDirectory="/work" + queue.filename="q_log_to_logsene" queue.maxfilesize="100m" queue.maxdiskspace="2g" + queue.saveonshutdown="on" + ) { + action(name="main_cee_parser" type="mmjsonparse") + if $parsesuccess == "OK" then { + action( name="logsene_json" + type="omelasticsearch" + server=`echo $LOGSENE_URL` + serverport="443" + usehttps="on" + template="logsene_json" + searchIndex=`echo $LOGSENE_TOKEN` + searchType="syslog-cee" + bulkmode="on" + action.resumeRetryCount="5" + action.resumeInterval="60" + ) + } else { + action( name="logsene_plain" + type="omelasticsearch" + server=`echo $LOGSENE_URL` + serverport="443" + usehttps="on" + template="logsene_plain" + searchIndex=`echo $LOGSENE_TOKEN` + searchType="syslog" + bulkmode="on" + action.resumeRetryCount="5" + action.resumeInterval="60" + ) + } +} diff --git a/appliance/alpine/tools/rsyslog@lists.adiscon.com-5a55e598.rsa.pub b/appliance/alpine/tools/rsyslog@lists.adiscon.com-5a55e598.rsa.pub new file mode 100644 index 0000000..86516c8 --- /dev/null +++ b/appliance/alpine/tools/rsyslog@lists.adiscon.com-5a55e598.rsa.pub @@ -0,0 +1,9 @@ +-----BEGIN PUBLIC KEY----- +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAuQTzWGgcuPw17HLwTe52 +nlUL+PgTRs8D6lKa47QxzoOjSf6XCAXDSKfhlTRoe456A1L0Lu0lIZ6RjIJx6qgP ++hdV2uYAMa1Z5Y2OdBXERHt36ZmHctACXQbFOYA66jcyiSMbDrpggnNQ5NOTfNpR +GE38P+BJ2nQ+rikFQMEQebqAMr2zvaU1YwU/HqhVpJsepy3iBmeUmWlmqiCaXHtC +yOZqqOA8jMwNnqNX4NF5HunhTzOIF9UzdgX0MGhxtUaUupxV4h3fNOtXcCgGx+EM +EsT3S2XDul5W9PKR1U5tmg7iIqPpYKQYT6GAdGDKfZ/RyWrFqz8xXnEDVVPNdl/l +PwIDAQAB +-----END PUBLIC KEY----- diff --git a/appliance/alpine/tools/starter.sh b/appliance/alpine/tools/starter.sh new file mode 100644 index 0000000..3731db4 --- /dev/null +++ b/appliance/alpine/tools/starter.sh @@ -0,0 +1,49 @@ +#!/bin/ash +source internal/set-defaults +source /config/container_config + +if [ "$CONTAINER_SILENT" != "on" ]; then + echo `cat CONTAINER.name` version `cat CONTAINER.release` - `cat CONTAINER.homepage` + echo `cat CONTAINER.copyright` + echo + echo "WARNING: this is an experimental container - do not use in production" + echo +fi + +if [ "$USE_VALGRIND" = "on" ]; then + source internal/install-valgrind + export VALGRIND=valgrind +fi + +if [ "$SYSLOG_APPLIANCE_DEBUG" = "on" ]; then + echo APPLIANCE IS IN DEBUG MODE + env + ls -l /config + #RSYSLOG_DEBUG_FLAG=-d +fi + +# Set $RSYSLOG_CONFIG_BASE64 to overwrite /etc/rsyslog.conf with base64 encoded config +# Useful to inject rsyslog.conf file without mounting it into the container +if [ ! -z "$RSYSLOG_CONFIG_BASE64" ]; then + echo "Writing RSYSLOG_CONFIG_BASE64" + echo "$RSYSLOG_CONFIG_BASE64" | base64 -d > /config/rsyslog.conf + export RSYSLOG_CONF="/config/rsyslog.conf" +fi + +# If RSYSLOG_CONF is empty, we want to default to /etc/rsyslog.conf +# This allows the user to overwrite what configuration file is being used +if [ -z "$RSYSLOG_CONF" ]; then + export RSYSLOG_CONF="/etc/rsyslog.conf" +fi + +echo "Using rsyslog configuration file: $RSYSLOG_CONF" + + +if [ -f tools/$1 ]; then + source tools/$1 +else + echo "ERROR: command not known: $*" + echo + source tools/help + exit 1 +fi diff --git a/appliance/alpine/tools/tools/bash b/appliance/alpine/tools/tools/bash new file mode 100644 index 0000000..b84dd7e --- /dev/null +++ b/appliance/alpine/tools/tools/bash @@ -0,0 +1,2 @@ +echo "You are now in Alpine's ash" +/bin/ash diff --git a/appliance/alpine/tools/tools/edit-config b/appliance/alpine/tools/tools/edit-config new file mode 100644 index 0000000..f8d9be1 --- /dev/null +++ b/appliance/alpine/tools/tools/edit-config @@ -0,0 +1 @@ +vi /config/container_config diff --git a/appliance/alpine/tools/tools/help b/appliance/alpine/tools/tools/help new file mode 100644 index 0000000..c8acb6e --- /dev/null +++ b/appliance/alpine/tools/tools/help @@ -0,0 +1,19 @@ +echo 'docker run --read-only command +where command is one of: +rsyslog - run rsyslog (default) +help - display this help message +show-work - list the /work area +show-config - display the current container config +edit-config - edit container config (with vi editor, press ZZ to quit) +bash - start an interactive shell (-d terminates) + +volumes: +/logs - log file store (if used) +/work - rsyslog work files; do not share with multiple instances +/config - configuration files, can be mounted read-only once finalized + +listening ports: +514/udp - syslog (udp) +514/tcp - syslog (plain tcp) +1601/tcp - RELP +' diff --git a/appliance/alpine/tools/tools/rsyslog b/appliance/alpine/tools/tools/rsyslog new file mode 100644 index 0000000..51a97ae --- /dev/null +++ b/appliance/alpine/tools/tools/rsyslog @@ -0,0 +1,7 @@ +# This NEEDS to be sourced (else we don't get PID 1!) +source internal/build-config +if [ "$USE_VALGRIND" = "on" ]; then + exec $VALGRIND /usr/sbin/rsyslogd -n $RSYSLOG_DEBUG_FLAG -f$RSYSLOG_CONF +else + exec /usr/sbin/rsyslogd -n $RSYSLOG_DEBUG_FLAG -f$RSYSLOG_CONF +fi diff --git a/appliance/alpine/tools/tools/show-config b/appliance/alpine/tools/tools/show-config new file mode 100644 index 0000000..b9a4034 --- /dev/null +++ b/appliance/alpine/tools/tools/show-config @@ -0,0 +1 @@ +cat /config/container_config diff --git a/appliance/rsyslog-ubuntu/.dockignore b/appliance/rsyslog-ubuntu/.dockignore new file mode 100644 index 0000000..6d1a6a4 --- /dev/null +++ b/appliance/rsyslog-ubuntu/.dockignore @@ -0,0 +1,4 @@ +tmp* +.tmp +.sw? +README.* diff --git a/appliance/rsyslog-ubuntu/CONTAINER.copyright b/appliance/rsyslog-ubuntu/CONTAINER.copyright new file mode 100644 index 0000000..975666e --- /dev/null +++ b/appliance/rsyslog-ubuntu/CONTAINER.copyright @@ -0,0 +1 @@ +Copyright (C) 2018 by Rainer Gerhards and Adiscon GmbH - released under ASL 2.0 diff --git a/appliance/rsyslog-ubuntu/CONTAINER.homepage b/appliance/rsyslog-ubuntu/CONTAINER.homepage new file mode 100644 index 0000000..d895ff5 --- /dev/null +++ b/appliance/rsyslog-ubuntu/CONTAINER.homepage @@ -0,0 +1 @@ +http://www.syslogappliance.de diff --git a/appliance/rsyslog-ubuntu/CONTAINER.name b/appliance/rsyslog-ubuntu/CONTAINER.name new file mode 100644 index 0000000..83f5c37 --- /dev/null +++ b/appliance/rsyslog-ubuntu/CONTAINER.name @@ -0,0 +1 @@ +rsyslog appliance diff --git a/appliance/rsyslog-ubuntu/Dockerfile b/appliance/rsyslog-ubuntu/Dockerfile new file mode 100644 index 0000000..91cc9ba --- /dev/null +++ b/appliance/rsyslog-ubuntu/Dockerfile @@ -0,0 +1,50 @@ +FROM ubuntu:20.04 +LABEL maintainer="rgerhards@adiscon.com" + + +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get -y update \ + && apt-get -y install --no-install-recommends --yes python3 apt-utils software-properties-common curl \ + && apt-get -y upgrade \ + && add-apt-repository -y ppa:adiscon/v8-stable \ + && apt-get -y update \ + && apt-get -y install libfastjson4 \ + libmaxminddb0 \ + rsyslog \ + rsyslog-mmjsonparse \ + rsyslog-pgsql \ + rsyslog-mmnormalize \ + rsyslog-mmfields \ + rsyslog-elasticsearch \ + rsyslog-imdocker \ + rsyslog-relp \ + rsyslog-mysql \ + rsyslog-mmrm1stspace \ + rsyslog-mmutf8fix \ + rsyslog-omstdout \ + rsyslog-mmanon +RUN groupadd rsyslog \ + && useradd -g rsyslog -s /bin/bash rsyslog \ + && echo "rsyslog ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers +VOLUME /config /work /logs +COPY rsyslog.conf /etc/rsyslog.conf +COPY rsyslog.conf.d/*.conf /etc/rsyslog.conf.d/ +# done base system setup + +WORKDIR /home/appliance +COPY starter.sh CONTAINER.* ./ +COPY internal/* ./internal/ +COPY internal/container_config /config/ +COPY internal/droprules.conf /config/ +COPY tools/* ./tools/ +COPY custom-builds/* /usr/lib/rsyslog/ +COPY custom-builds-liblog/* /usr/lib/ +RUN echo "`date +%F` (`date +%s`)" > CONTAINER.release \ + && chown -R rsyslog:rsyslog * && \ + chmod +x starter.sh && \ + chmod +x /usr/sbin/rsyslogd + +USER rsyslog +CMD ["rsyslog"] +ENTRYPOINT ["/home/appliance/starter.sh"] + diff --git a/appliance/rsyslog-ubuntu/README.md b/appliance/rsyslog-ubuntu/README.md new file mode 100644 index 0000000..6424ac9 --- /dev/null +++ b/appliance/rsyslog-ubuntu/README.md @@ -0,0 +1,13 @@ +Note: we currently focus on the Alpine container, because + +- Alpine is frequently used in docker due to the small footprint +- We have a firm grip on Alpine packaging and can quickly build + ourselfs all components that we need (quick turn-around cycle) +- the appliance container is not yet mature, so duplicating work + sounds counter-productive + +Once the Alpine container is really production grade and if there is +demand for others, we plan to create appliances with other base OS. +Ubuntu and CentOS would be choices. + +see also: https://github.com/rsyslog/rsyslog-docker/issues/3 diff --git a/appliance/rsyslog-ubuntu/build.sh b/appliance/rsyslog-ubuntu/build.sh new file mode 100644 index 0000000..940150b --- /dev/null +++ b/appliance/rsyslog-ubuntu/build.sh @@ -0,0 +1,2 @@ +printf "use ./build.sh --no-cache to disable cache\n" +docker build $1 -t rsyslog-container:ubuntu-20.04 . diff --git a/appliance/rsyslog-ubuntu/custom-builds-liblog/liblognorm.a b/appliance/rsyslog-ubuntu/custom-builds-liblog/liblognorm.a new file mode 100644 index 0000000..fe3b2be Binary files /dev/null and b/appliance/rsyslog-ubuntu/custom-builds-liblog/liblognorm.a differ diff --git a/appliance/rsyslog-ubuntu/custom-builds-liblog/liblognorm.so.5.1.0 b/appliance/rsyslog-ubuntu/custom-builds-liblog/liblognorm.so.5.1.0 new file mode 100644 index 0000000..772389e Binary files /dev/null and b/appliance/rsyslog-ubuntu/custom-builds-liblog/liblognorm.so.5.1.0 differ diff --git a/appliance/rsyslog-ubuntu/custom-builds-liblog/lognormalizer b/appliance/rsyslog-ubuntu/custom-builds-liblog/lognormalizer new file mode 100644 index 0000000..c9cd871 Binary files /dev/null and b/appliance/rsyslog-ubuntu/custom-builds-liblog/lognormalizer differ diff --git a/appliance/rsyslog-ubuntu/custom-builds/mmdblookup.la b/appliance/rsyslog-ubuntu/custom-builds/mmdblookup.la new file mode 100644 index 0000000..8ab2b38 --- /dev/null +++ b/appliance/rsyslog-ubuntu/custom-builds/mmdblookup.la @@ -0,0 +1,41 @@ +# mmdblookup.la - a libtool library file +# Generated by libtool (GNU libtool) 2.4.6 Debian-2.4.6-14 +# +# Please DO NOT delete this file! +# It is necessary for linking the library. + +# The name that we can dlopen(3). +dlname='mmdblookup.so' + +# Names of this library. +library_names='mmdblookup.so mmdblookup.so mmdblookup.so' + +# The name of the static archive. +old_library='' + +# Linker flags that cannot go in dependency_libs. +inherited_linker_flags='' + +# Libraries that this one depends upon. +dependency_libs=' -lmaxminddb' + +# Names of additional weak libraries provided by this library +weak_library_names='' + +# Version information for mmdblookup. +current=0 +age=0 +revision=0 + +# Is this an already installed library? +installed=no + +# Should we warn about portability when linking against -modules? +shouldnotlink=yes + +# Files to dlopen/dlpreopen +dlopen='' +dlpreopen='' + +# Directory that this library needs to be installed in: +libdir='/usr/local/lib/rsyslog' diff --git a/appliance/rsyslog-ubuntu/custom-builds/mmdblookup.so b/appliance/rsyslog-ubuntu/custom-builds/mmdblookup.so new file mode 100644 index 0000000..fe6e5c9 Binary files /dev/null and b/appliance/rsyslog-ubuntu/custom-builds/mmdblookup.so differ diff --git a/appliance/rsyslog-ubuntu/custom-builds/omelasticsearch.la b/appliance/rsyslog-ubuntu/custom-builds/omelasticsearch.la new file mode 100644 index 0000000..ffb5c0b --- /dev/null +++ b/appliance/rsyslog-ubuntu/custom-builds/omelasticsearch.la @@ -0,0 +1,41 @@ +# omelasticsearch.la - a libtool library file +# Generated by libtool (GNU libtool) 2.4.6 Debian-2.4.6-14 +# +# Please DO NOT delete this file! +# It is necessary for linking the library. + +# The name that we can dlopen(3). +dlname='omelasticsearch.so' + +# Names of this library. +library_names='omelasticsearch.so omelasticsearch.so omelasticsearch.so' + +# The name of the static archive. +old_library='' + +# Linker flags that cannot go in dependency_libs. +inherited_linker_flags=' -pthread' + +# Libraries that this one depends upon. +dependency_libs=' /usr/lib/x86_64-linux-gnu/libcurl.la -lm' + +# Names of additional weak libraries provided by this library +weak_library_names='' + +# Version information for omelasticsearch. +current=0 +age=0 +revision=0 + +# Is this an already installed library? +installed=yes + +# Should we warn about portability when linking against -modules? +shouldnotlink=yes + +# Files to dlopen/dlpreopen +dlopen='' +dlpreopen='' + +# Directory that this library needs to be installed in: +libdir='/usr/local/lib/rsyslog' diff --git a/appliance/rsyslog-ubuntu/custom-builds/omelasticsearch.so b/appliance/rsyslog-ubuntu/custom-builds/omelasticsearch.so new file mode 100644 index 0000000..204744a Binary files /dev/null and b/appliance/rsyslog-ubuntu/custom-builds/omelasticsearch.so differ diff --git a/appliance/rsyslog-ubuntu/custom-builds/omhttp.la b/appliance/rsyslog-ubuntu/custom-builds/omhttp.la new file mode 100644 index 0000000..f5ba66c --- /dev/null +++ b/appliance/rsyslog-ubuntu/custom-builds/omhttp.la @@ -0,0 +1,41 @@ +# omhttp.la - a libtool library file +# Generated by libtool (GNU libtool) 2.4.6 Debian-2.4.6-14 +# +# Please DO NOT delete this file! +# It is necessary for linking the library. + +# The name that we can dlopen(3). +dlname='omhttp.so' + +# Names of this library. +library_names='omhttp.so omhttp.so omhttp.so' + +# The name of the static archive. +old_library='' + +# Linker flags that cannot go in dependency_libs. +inherited_linker_flags=' -pthread' + +# Libraries that this one depends upon. +dependency_libs=' /usr/lib/x86_64-linux-gnu/libcurl.la -lm' + +# Names of additional weak libraries provided by this library +weak_library_names='' + +# Version information for omhttp. +current=0 +age=0 +revision=0 + +# Is this an already installed library? +installed=no + +# Should we warn about portability when linking against -modules? +shouldnotlink=yes + +# Files to dlopen/dlpreopen +dlopen='' +dlpreopen='' + +# Directory that this library needs to be installed in: +libdir='/usr/local/lib/rsyslog' diff --git a/appliance/rsyslog-ubuntu/custom-builds/omhttp.so b/appliance/rsyslog-ubuntu/custom-builds/omhttp.so new file mode 100644 index 0000000..828ec83 Binary files /dev/null and b/appliance/rsyslog-ubuntu/custom-builds/omhttp.so differ diff --git a/appliance/rsyslog-ubuntu/internal/build-config b/appliance/rsyslog-ubuntu/internal/build-config new file mode 100644 index 0000000..8bff1a0 --- /dev/null +++ b/appliance/rsyslog-ubuntu/internal/build-config @@ -0,0 +1,25 @@ +#!/bin/ash +# This reads the meta config (environment variables) and prepares environment +# variables used by rsyslog.conf so that we can finally startup rsyslog. +# Errors are detected and will lead to abort of the startup. + +export CNF_CALL_LOG_TO_LOGFILES= +if [ "$ENABLE_LOGFILES" == "on" ]; then + if [ -z $LOGFILES_STORE ]; then + echo "Error: ENABLE_LOGFILES set, but LOGFILES_STORE not set - please correct" + exit 1 + fi + export CNF_CALL_LOG_TO_LOGFILES="call log_to_files" +fi # end LOGFILES + +export CNF_CALL_LOG_TO_LOGSENE= +export ENABLE_LOGSENE=off +if [ "$LOGSENE_TOKEN" != "" ]; then + if [ -z $LOGSENE_URL ]; then + echo "Error: LOGSENE_TOKEN set, but LOGSENE_URL not set - please correct" + exit 1 + fi + export ENABLE_LOGSENE=on + export CNF_CALL_LOG_TO_LOGSENE="call log_to_logsene" +fi # end LOGSENE +env diff --git a/appliance/rsyslog-ubuntu/internal/container_config b/appliance/rsyslog-ubuntu/internal/container_config new file mode 100644 index 0000000..6d12e5a --- /dev/null +++ b/appliance/rsyslog-ubuntu/internal/container_config @@ -0,0 +1,29 @@ +# rsyslog syslog appliance container configuration. +# This file will be sourced upon container startup. +# Uncomment those settings that you need and set them to your +# desired values. + +# general container app settings: +export TZ=UTC +#export CONTAINER_SILENT=on # do not emit startup message +export ENABLE_STATISTICS=on + +# Do we write log files? +export ENABLE_LOGFILES=off # (comment out to disable) +# Where do we write to? +# path for host-specific files is: /logs/hosts/HOSTNAME +export LOGFILES_STORE="/logs/hosts/%hostname:::secpath-replace%/messages.log" +# you can of course overwrite this. For example, the below definition +# uses the program name instead of a fixed name "messages.log". That means +# for each host, a separate file for each program will be generated. +#export LOGFILES_STORE="/logs/hosts/%hostname:::secpath-replace%/%programname:::secpath-replace%.log" + +# If you have an account with Logsene, enter your access +# information below: +#export LOGSENE_TOKEN= +#export LOGSENE_URL=logsene-receiver.eu.sematext.com + +# Settings for debugging the container +#export USE_VALGRIND=on +#export RSYSLOG_DEBUG="debug nostdout" +#export RSYSLOG_DEBUGLOG="/logs/rsyslog-internal-debug.log" diff --git a/appliance/rsyslog-ubuntu/internal/droprules.conf b/appliance/rsyslog-ubuntu/internal/droprules.conf new file mode 100644 index 0000000..6c6da9c --- /dev/null +++ b/appliance/rsyslog-ubuntu/internal/droprules.conf @@ -0,0 +1,5 @@ +# This file permits to do drop unneded messages +# Uncomment the samples below to activate them. You can +# add as many drop rules as you like. + +#if $rawmsg contains "somejunk" then stop diff --git a/appliance/rsyslog-ubuntu/internal/install-valgrind b/appliance/rsyslog-ubuntu/internal/install-valgrind new file mode 100644 index 0000000..6993571 --- /dev/null +++ b/appliance/rsyslog-ubuntu/internal/install-valgrind @@ -0,0 +1,11 @@ +#!/bin/ash +touch /Are-we-Writable +if [ ! -f /Are-we-Writable ]; then + echo "ERROR: we need to install valgrind, so the container must" + echo " NOT be read-only for this debug session." + exit 1 +fi +rm /Are-we-Writable +echo installing valgrind for debugging -- need to access Alpine repositories +apk update +apk add --no-cache rsyslog-dbg musl-dbg valgrind diff --git a/appliance/rsyslog-ubuntu/internal/set-defaults b/appliance/rsyslog-ubuntu/internal/set-defaults new file mode 100644 index 0000000..d838a44 --- /dev/null +++ b/appliance/rsyslog-ubuntu/internal/set-defaults @@ -0,0 +1,8 @@ +# set defaults if not present. This enables us to pre-populate bind +# mounts, which makes sense in our case. +if [ ! -f /config/container_config ]; then + cp internal/container_config /config +fi +if [ ! -f /config/droprules.conf ]; then + cp internal/droprules.conf /config +fi diff --git a/appliance/rsyslog-ubuntu/rsyslog.conf b/appliance/rsyslog-ubuntu/rsyslog.conf new file mode 100644 index 0000000..d24be0c --- /dev/null +++ b/appliance/rsyslog-ubuntu/rsyslog.conf @@ -0,0 +1,34 @@ +global(processInternalMessages="on") + +#module(load="imtcp" StreamDriver.AuthMode="anon" StreamDriver.Mode="1") +module(load="impstats") # config.enabled=`echo $ENABLE_STATISTICS`) +module(load="imrelp") +module(load="imtcp") +module(load="imudp" TimeRequery="500") + +module(load="omstdout") +module(load="omelasticsearch") + +module(load="mmjsonparse") +module(load="mmutf8fix") + + +input(type="imtcp" port="514") +input(type="imudp" port="514") +input(type="imrelp" port="1601") + +# includes done explicitely +include(file="/etc/rsyslog.conf.d/log_to_logsene.conf" config.enabled=`echo $ENABLE_LOGSENE`) +include(file="/etc/rsyslog.conf.d/log_to_files.conf" config.enabled=`echo $ENABLE_LOGFILES`) + +#################### default ruleset begins #################### + +# we emit our own messages to docker console: +syslog.* :omstdout: + +include(file="/config/droprules.conf" mode="optional") # this permits the user to easily drop unwanted messages + +action(name="main_utf8fix" type="mmutf8fix" replacementChar="?") + +include(text=`echo $CNF_CALL_LOG_TO_LOGFILES`) +include(text=`echo $CNF_CALL_LOG_TO_LOGSENE`) diff --git a/appliance/rsyslog-ubuntu/rsyslog.conf.d/log_to_files.conf b/appliance/rsyslog-ubuntu/rsyslog.conf.d/log_to_files.conf new file mode 100644 index 0000000..bce4628 --- /dev/null +++ b/appliance/rsyslog-ubuntu/rsyslog.conf.d/log_to_files.conf @@ -0,0 +1,7 @@ +# logging to files in /logs + +template(name="log_to_files_dynafile" type="string" string=`echo $LOGFILES_STORE`) +ruleset(name="log_to_files") { + /logs/debug;RSYSLOG_DebugFormat + action(type="omfile" dynafile="log_to_files_dynafile" name="log_to_logfiles") +} diff --git a/appliance/rsyslog-ubuntu/rsyslog.conf.d/log_to_logsene.conf b/appliance/rsyslog-ubuntu/rsyslog.conf.d/log_to_logsene.conf new file mode 100644 index 0000000..5ff0240 --- /dev/null +++ b/appliance/rsyslog-ubuntu/rsyslog.conf.d/log_to_logsene.conf @@ -0,0 +1,63 @@ +# config snippet for the Logsene service +# https://sematext.com/logsene/ + +template(name="logsene_json" type="list") { + constant(value="{") + constant(value="\"@timestamp\":\"") property(name="timereported" dateFormat="rfc3339") + constant(value="\",\"host\":\"") property(name="hostname") + constant(value="\",\"severity\":\"") property(name="syslogseverity-text" caseConversion="upper") + constant(value="\",\"facility\":\"") property(name="syslogfacility-text") + constant(value="\",\"syslog-tag\":\"") property(name="syslogtag" format="json") + constant(value="\",\"source\":\"") property(name="app-name" format="json") + constant(value="\",") property(name="$!all-json" position.from="2") +} + +template(name="logsene_plain" type="list") { + constant(value="{") + constant(value="\"@timestamp\":\"") property(name="timereported" dateFormat="rfc3339") + constant(value="\",\"host\":\"") property(name="hostname") + constant(value="\",\"severity\":\"") property(name="syslogseverity-text" caseConversion="upper") + constant(value="\",\"facility\":\"") property(name="syslogfacility-text") + constant(value="\",\"syslog-tag\":\"") property(name="syslogtag" format="json") + constant(value="\",\"source\":\"") property(name="app-name" format="json") + constant(value="\",\"message\":\"") property(name="msg" format="json") + constant(value="\"}") +} + + +ruleset(name="log_to_logsene" + queue.type="LinkedList" queue.dequeueBatchSize="2000" + queue.size="100000" # do not use a too-high values, else we get trouble with 10s shutdown period! + queue.spoolDirectory="/work" + queue.filename="q_log_to_logsene" queue.maxfilesize="100m" queue.maxdiskspace="2g" + queue.saveonshutdown="on" + ) { + action(name="main_cee_parser" type="mmjsonparse") + if $parsesuccess == "OK" then { + action( name="logsene_json" + type="omelasticsearch" + server=`echo $LOGSENE_URL` + serverport="443" + usehttps="on" + template="logsene_json" + searchIndex=`echo $LOGSENE_TOKEN` + searchType="syslog-cee" + bulkmode="on" + action.resumeRetryCount="5" + action.resumeInterval="60" + ) + } else { + action( name="logsene_plain" + type="omelasticsearch" + server=`echo $LOGSENE_URL` + serverport="443" + usehttps="on" + template="logsene_plain" + searchIndex=`echo $LOGSENE_TOKEN` + searchType="syslog" + bulkmode="on" + action.resumeRetryCount="5" + action.resumeInterval="60" + ) + } +} diff --git a/appliance/rsyslog-ubuntu/starter.sh b/appliance/rsyslog-ubuntu/starter.sh new file mode 100644 index 0000000..d8ea6b7 --- /dev/null +++ b/appliance/rsyslog-ubuntu/starter.sh @@ -0,0 +1,51 @@ +#!/bin/bash +source internal/set-defaults +source /config/container_config + +if [ "$CONTAINER_SILENT" != "on" ]; then + echo `cat CONTAINER.name` version `cat CONTAINER.release` - `cat CONTAINER.homepage` + echo `cat CONTAINER.copyright` + echo + echo "This container is build for production." + echo "See online: https://github.com/EHerzog76/rsyslog-docker" + echo +fi + +if [ "$USE_VALGRIND" = "on" ]; then + source internal/install-valgrind + export VALGRIND=valgrind +fi + +if [ "$SYSLOG_APPLIANCE_DEBUG" = "on" ]; then + echo APPLIANCE IS IN DEBUG MODE + env + ls -l /config + #RSYSLOG_DEBUG_FLAG=-d +fi + +# Set $RSYSLOG_CONFIG_BASE64 to overwrite /etc/rsyslog.conf with base64 encoded config +# Useful to inject rsyslog.conf file without mounting it into the container +if [ ! -z "$RSYSLOG_CONFIG_BASE64" ]; then + echo "Writing RSYSLOG_CONFIG_BASE64" + echo "$RSYSLOG_CONFIG_BASE64" | base64 -d > /config/rsyslog.conf + export RSYSLOG_CONF="/config/rsyslog.conf" +fi + +# If RSYSLOG_CONF is empty, we want to default to /etc/rsyslog.conf +# This allows the user to overwrite what configuration file is being used +if [ -z "$RSYSLOG_CONF" ]; then + export RSYSLOG_CONF="/etc/rsyslog.conf" +fi + +echo "Using rsyslog configuration file: $RSYSLOG_CONF" + + +if [ -f tools/$1 ]; then + #rsyslogd -D -N 1 -f /etc/rsyslog.conf + source tools/$1 +else + echo "ERROR: command not known: $*" + echo + source tools/help + exit 1 +fi diff --git a/appliance/rsyslog-ubuntu/tools/bash b/appliance/rsyslog-ubuntu/tools/bash new file mode 100644 index 0000000..01dca2d --- /dev/null +++ b/appliance/rsyslog-ubuntu/tools/bash @@ -0,0 +1 @@ +/bin/bash diff --git a/appliance/rsyslog-ubuntu/tools/edit-config b/appliance/rsyslog-ubuntu/tools/edit-config new file mode 100644 index 0000000..f8d9be1 --- /dev/null +++ b/appliance/rsyslog-ubuntu/tools/edit-config @@ -0,0 +1 @@ +vi /config/container_config diff --git a/appliance/rsyslog-ubuntu/tools/help b/appliance/rsyslog-ubuntu/tools/help new file mode 100644 index 0000000..c8acb6e --- /dev/null +++ b/appliance/rsyslog-ubuntu/tools/help @@ -0,0 +1,19 @@ +echo 'docker run --read-only command +where command is one of: +rsyslog - run rsyslog (default) +help - display this help message +show-work - list the /work area +show-config - display the current container config +edit-config - edit container config (with vi editor, press ZZ to quit) +bash - start an interactive shell (-d terminates) + +volumes: +/logs - log file store (if used) +/work - rsyslog work files; do not share with multiple instances +/config - configuration files, can be mounted read-only once finalized + +listening ports: +514/udp - syslog (udp) +514/tcp - syslog (plain tcp) +1601/tcp - RELP +' diff --git a/appliance/rsyslog-ubuntu/tools/rsyslog b/appliance/rsyslog-ubuntu/tools/rsyslog new file mode 100644 index 0000000..51a97ae --- /dev/null +++ b/appliance/rsyslog-ubuntu/tools/rsyslog @@ -0,0 +1,7 @@ +# This NEEDS to be sourced (else we don't get PID 1!) +source internal/build-config +if [ "$USE_VALGRIND" = "on" ]; then + exec $VALGRIND /usr/sbin/rsyslogd -n $RSYSLOG_DEBUG_FLAG -f$RSYSLOG_CONF +else + exec /usr/sbin/rsyslogd -n $RSYSLOG_DEBUG_FLAG -f$RSYSLOG_CONF +fi diff --git a/appliance/rsyslog-ubuntu/tools/show-config b/appliance/rsyslog-ubuntu/tools/show-config new file mode 100644 index 0000000..b9a4034 --- /dev/null +++ b/appliance/rsyslog-ubuntu/tools/show-config @@ -0,0 +1 @@ +cat /config/container_config diff --git a/rsyslog-deployment.yaml b/rsyslog-deployment.yaml new file mode 100644 index 0000000..02c8265 --- /dev/null +++ b/rsyslog-deployment.yaml @@ -0,0 +1,847 @@ +--- +apiVersion: v1 +kind: Service +metadata: + namespace: "logging" + name: "service-rsyslog" +spec: + internalTrafficPolicy: Cluster + ports: + - name: syslog514tcp + port: 514 + targetPort: 514 + protocol: TCP + - name: syslog514udp + port: 514 + targetPort: 514 + protocol: UDP + - name: ciscoftd + port: 9003 + targetPort: 9003 + protocol: UDP + - name: fortinetfw + port: 9004 + targetPort: 9004 + protocol: UDP + - name: checkpoint + port: 9501 + targetPort: 9501 + protocol: UDP + - name: f5ltm + port: 9544 + targetPort: 9544 + protocol: UDP + sessionAffinity: None + type: ClusterIP + selector: + app: "rsyslog4elk" +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: rsyslog-configmap + labels: + app: rsyslog4elk +data: + rsyslog.conf: | + # For more information see /usr/share/doc/rsyslog-*/rsyslog_conf.html + #### MODULES #### + # omfwd is a Bultin-Modul, so no Modul-Load is needed. + #module(load="omfwd") # syslog Forwarding Output Module for TCP and UDP + # Provides UDP syslog reception + # for parameters see http://www.rsyslog.com/doc/imudp.html + module(load="imudp" threads="4" timeRequery="2" batchSize="128") # needs to be done just once + # Provides TCP syslog reception + # for parameters see http://www.rsyslog.com/doc/imtcp.html + module(load="imtcp") # needs to be done just once + # Provides Message-Formating and Elasticsearch-Forwarding + #module(load="mmpstrucdata") # RFC5424 Parsingmodul + module(load="mmnormalize") + #module(load="mmexternal") + module(load="omelasticsearch") + #### GLOBAL DIRECTIVES #### + global(net.enableDNS="off") + main_queue( + # buffer up to 1M messages in memory + queue.size="1000000" + # these threads process messages and send them to Elasticsearch + queue.workerThreads="5" + queue.type="LinkedList" + # FixedArray/LinkedList/Direct/Disk + # rsyslog processes messages in batches to avoid queue contention + queue.dequeueBatchSize="1000" + queue.highWatermark="610000" # max no. of events to hold in memory + queue.lowWatermark="300000" # use memory queue again, when it's back to this level + queue.spoolDirectory="/rsyslog-queues" # where to write on disk + queue.fileName="main-queue" + queue.maxDiskSpace="10g" # it will stop at this much disk space + queue.saveOnShutdown="off" # save memory queue contents to disk when rsyslog is exiting. User persistent Disk. + ) + # Use default timestamp format + $ActionFileDefaultTemplate RSYSLOG_TraditionalFileFormat + # Include all config files in /etc/rsyslog.d/ + $IncludeConfig /etc/rsyslog.d/*.conf +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: rsyslog-d + labels: + app: rsyslog4elk +data: + global.conf: | + #module(load="mmdblookup" container="!geoip") + #### Lookup Tables #### + lookup_table(name="ftd-iptable" file="/etc/rsyslog.d/lookup_ftd.tbl") + #### TEMPLATES #### + template(name="tplRawMsg" type="string" string="%timegenerated% %fromhost% %rawmsg:::drop-last-lf%\n") + template(name="tplPlainMsg" type="string" string="%rawmsg:::drop-last-lf%\n") + template(name="syslogmsg" type="list") { + constant(value="syslogmsg") + } + template(name="plain-syslog" type="list" option.json="on") { + constant(value="{") + constant(value="\"@timestamp\":\"") property(name="timereported" dateFormat="rfc3339") + constant(value="\",\"host\":\"") property(name="hostname") + constant(value="\",\"severity\":\"") property(name="syslogseverity-text") + constant(value="\",\"facility\":\"") property(name="syslogfacility-text") + constant(value="\",\"tag\":\"") property(name="syslogtag") + constant(value="\",\"message\":\"") property(name="msg") + constant(value="\"}") + } + # Checkpoint-Syslog to Elastic ConnectionEvents + template(name="CPSyslog-IPConEvents" type="list" option.json="on") { + constant(value="{") + constant(value="\"@timestamp\":\"") property(name="timegenerated" dateFormat="rfc3339") + # constant(value="\"@timestamp\":\"") property(name="$!eventdate" dateFormat="rfc3339") + # constant(value="\",\"eventsource\":\"") property(name="$!loghost") + constant(value="\",\"eventsource\":\"") property(name="fromhost") + constant(value="\",\"eventorigin\":\"") property(name="$!logsource") + constant(value="\",\"action\":\"") property(name="$.action") + constant(value="\",\"uuid\":\"") property(name="$.uuid") + constant(value="\",\"serviceid\":\"") property(name="$.service_id") + constant(value="\",\"service\":\"") property(name="$.service") + constant(value="\",\"srcport\":\"") property(name="$.s_port") + constant(value="\",\"src\":\"") property(name="$.src") + constant(value="\",\"dst\":\"") property(name="$.dst") + constant(value="\",\"proto\":\"") property(name="$.proto") + constant(value="\",\"natsrc\":\"") property(name="$.xlatesrc") + constant(value="\",\"natdst\":\"") property(name="$.xlatedst") + constant(value="\",\"natsrcport\":\"") property(name="$.xlatesport") + constant(value="\",\"natdstport\":\"") property(name="$.xlatedport") + constant(value="\",\"natrulenr\":\"") property(name="$.nat_rulenum") + constant(value="\",\"natrulenr2\":\"") property(name="$.nat_addtnl_rulenum") + constant(value="\",\"user\":\"") property(name="$.user") + constant(value="\",\"user_dst\":\"") property(name="$.user_dst") + constant(value="\",\"rulenr\":\"") property(name="$.rule") + constant(value="\",\"msginfo\":\"") property(name="$.message_info") + constant(value="\",\"productfam\":\"") property(name="$.product_family") + constant(value="\",\"product\":\"") property(name="$.product") + constant(value="\",\"url\":\"") property(name="$.resource") + constant(value="\"}") + } + # Cisco Firepower Syslog to Elastic ConnectionEvents + template(name="FPSyslog-IPConEvents" type="list" option.json="on") { + constant(value="{") + # constant(value="\"@timestamp\":\"") property(name="timereported" dateFormat="year") + # constant(value="-") property(name="timereported" dateFormat="month") + # constant(value="-") property(name="timereported" dateFormat="day") + # constant(value="T") property(name="timereported" dateFormat="hour") + # constant(value=":") property(name="timereported" dateFormat="minute") + # constant(value=":") property(name="timereported" dateFormat="second") + # constant(value="+00:00") + # constant(value="\"@timestamp\":\"") property(name="timereported" dateFormat="rfc3339") + constant(value="\"@timestamp\":\"") property(name="$!eventdate" dateFormat="rfc3339") + constant(value="\",\"data_stream\": {\"namespace\": \"default\",\"type\": \"logs\",\"dataset\": \"cisco_ftd63.log\"}") + constant(value=",\"observer.name\":\"") property(name="$!syslog5424_host") + constant(value="\",\"observer.vendor\":\"Cisco") + constant(value="\",\"observer.product\":\"Firepower") + constant(value="\",\"observer.type\":\"firewall") + constant(value="\",\"rule.ruleset\":\"") property(name="$.rule_ruleset") + constant(value="\",\"rule.name\":\"") property(name="$.rule_name") + constant(value="\",\"event.outcome\":\"") property(name="$.event_outcome") + constant(value="\",\"event.action\":\"") property(name="$.event_action") + constant(value="\",\"access_control_rule_reason\":\"") property(name="$.event_outcome") + constant(value="\",\"network.protocol\":\"") property(name="$.network_protocol") + constant(value="\",\"archive_depth\":\"") property(name="$.archive_depth") + constant(value="\",\"archive_file_name\":\"") property(name="$.archive_file_name") + constant(value="\",\"archive_file_status\":\"") property(name="$.archive_file_status") + constant(value="\",\"classification\":\"") property(name="$.classification") + constant(value="\",\"network.application\":\"") property(name="$.network_application") + constant(value="\",\"client_version\":\"") property(name="$.client_version") + constant(value="\",\"event.duration\":\"") property(name="$.event_duration") + constant(value="\",\"dns_sinkhole\":\"") property(name="$.dns_sinkhole") + constant(value="\",\"dns_ttl\":\"") property(name="$.dns_ttl") + constant(value="\",\"dns.question.name\":\"") property(name="$.dns_question_name") + constant(value="\",\"dns.question.type\":\"") property(name="$.dns_question_type") + constant(value="\",\"dns.response_code\":\"") property(name="$.dns_response_code") + constant(value="\",\"dnssi_category\":\"") property(name="$.dnssi_category") + constant(value="\",\"destination.ip\":\"") property(name="$.destination_ip") #destination.address + constant(value="\",\"destination.port\":\"") property(name="$.destination_port") + constant(value="\",\"observer.egress.interface.name\":\"") property(name="$.observer_egress_interface_name") + constant(value="\",\"observer.egress.zone\":\"") property(name="$.observer.egress.zone") + constant(value="\",\"endpoint_profile\":\"") property(name="$.endpoint_profile") + constant(value="\",\"file_action\":\"") property(name="$.file_action") + constant(value="\",\"file_count\":\"") property(name="$.file_count") + constant(value="\",\"file_direction\":\"") property(name="$.file_direction") + constant(value="\",\"file.name\":\"") property(name="$.file_name") + constant(value="\",\"file_policy\":\"") property(name="$.file_policy") + constant(value="\",\"file.hash.sha256\":\"") property(name="$.file_hash_sha256") + constant(value="\",\"file_sandbox_status\":\"") property(name="$.file_sandbox_status") + constant(value="\",\"file.size\":\"") property(name="$.file_size") + constant(value="\",\"file_storage_status\":\"") property(name="$.file_storage_status") + constant(value="\",\"event.start\":\"") property(name="$.event_start") + constant(value="\",\"service.id\":\"") property(name="$.service_id") + constant(value="\",\"http.request.referrer\":\"") property(name="$.http_request_referrer") + constant(value="\",\"http.response.status_code\":\"") property(name="$.http_response_status_code") + constant(value="\",\"icmp_code\":\"") property(name="$.icmp_code") + constant(value="\",\"icmp_type\":\"") property(name="$.icmp_type") + constant(value="\",\"ip_reputation_si_category\":\"") property(name="$.ip_reputation_si_category") + constant(value="\",\"ips_count\":\"") property(name="$.ips_count") + constant(value="\",\"observer.ingress.interface.name\":\"") property(name="$.observer_ingress_interface_name") + constant(value="\",\"observer.ingress.zone\":\"") property(name="$.observer.ingress.zone") + constant(value="\",\"source.bytes\":\"") property(name="$.source_bytes") + constant(value="\",\"source.packets\":\"") property(name="$.source_packets") + constant(value="\",\"intrusion_policy\":\"") property(name="$.intrusion_policy") + constant(value="\",\"mpls_label\":\"") property(name="$.mpls_label") + constant(value="\",\"message\":\"") property(name="$.message") + constant(value="\",\"nap_policy\":\"") property(name="$.nap_policy") + constant(value="\",\"net_bios_domain\":\"") property(name="$.net_bios_domain") + constant(value="\",\"num_ioc\":\"") property(name="$.num_ioc") + constant(value="\",\"prefilter_policy\":\"") property(name="$.prefilter_policy") + constant(value="\",\"priority\":\"") property(name="$.priority") + constant(value="\",\"network.transport\":\"") property(name="$.network_transport") + constant(value="\",\"url.domain\":\"") property(name="$.url_domain") + constant(value="\",\"destination.bytes\":\"") property(name="$.destination_bytes") + constant(value="\",\"destination.packets\":\"") property(name="$.destination_packets") + constant(value="\",\"revision\":\"") property(name="$.revision") + constant(value="\",\"sha_disposition\":\"") property(name="$.sha_disposition") + constant(value="\",\"sid\":\"") property(name="$.sid") + constant(value="\",\"ssl_certificate\":\"") property(name="$.ssl_certificate") + constant(value="\",\"ssl_expected_action\":\"") property(name="$.ssl_expected_action") + constant(value="\",\"ssl_flow_status\":\"") property(name="$.ssl_flow_status") + constant(value="\",\"ssl_policy\":\"") property(name="$.ssl_policy") + constant(value="\",\"ssl_rule_name\":\"") property(name="$.ssl_rule_name") + constant(value="\",\"ssl_server_cert_status\":\"") property(name="$.ssl_server_cert_status") + constant(value="\",\"server.domain\":\"") property(name="$.server.domain") + constant(value="\",\"ssl_session_id\":\"") property(name="$.ssl_session_id") + constant(value="\",\"ssl_ticket_id\":\"") property(name="$.ssl_ticket_id") + constant(value="\",\"sslurl_category\":\"") property(name="$.sslurl_category") + constant(value="\",\"ssl_version\":\"") property(name="$.ssl_version") + constant(value="\",\"ssl_cipher_suite\":\"") property(name="$.ssl_cipher_suite") + constant(value="\",\"sec_int_matching_ip\":\"") property(name="$.sec_int_matching_ip") + constant(value="\",\"security_group\":\"") property(name="$.security_group") + constant(value="\",\"spero_disposition\":\"") property(name="$.spero_disposition") + constant(value="\",\"source.ip\":\"") property(name="$.source_ip") #source.address + constant(value="\",\"source.port\":\"") property(name="$.source_port") + constant(value="\",\"tcp_flags\":\"") property(name="$.tcp_flags") + constant(value="\",\"threat_name\":\"") property(name="$.threat_name") + constant(value="\",\"threat_score\":\"") property(name="$.threat_score") + constant(value="\",\"tunnel_or_prefilter_rule\":\"") property(name="$.tunnel_or_prefilter_rule") + constant(value="\",\"url.original\":\"") property(name="$.url_original") + constant(value="\",\"url_category\":\"") property(name="$.url_category") + constant(value="\",\"url_reputation\":\"") property(name="$.url_reputation") + constant(value="\",\"urlsi_category\":\"") property(name="$.urlsi_category") + constant(value="\",\"user.name\":\"") property(name="$.user_name") #user.id + constant(value="\",\"user_agent.original\":\"") property(name="$.user_agent.original") + constant(value="\",\"vlan_id\":\"") property(name="$.vlan_id") + constant(value="\",\"client.ip\":\"") property(name="$.client_ip") #client.address + constant(value="\",\"source.nat.ip\":\"") property(name="$.source_nat_ip") + constant(value="\",\"source.nat.port\":\"") property(name="$.source_nat_port") + constant(value="\",\"destination.nat.ip\":\"") property(name="$.destination_nat_ip") + constant(value="\",\"destination.nat.port\":\"") property(name="$.destination_nat_port") + constant(value="\"}") + } + # f5-LTM-Syslog to Elastic ConnectionEvents + template(name="f5-ltmEvents" type="list" option.json="on") { + constant(value="{") + constant(value="\"@timestamp\":\"") property(name="timegenerated" dateFormat="rfc3339") + # constant(value="\"@timestamp\":\"") property(name="$!eventdate" dateFormat="rfc3339") + # constant(value="\",\"eventsource\":\"") property(name="$!loghost") + constant(value="\",\"eventsource\":\"") property(name="fromhost") + constant(value="\",\"eventorigin\":\"") property(name="$!logsource") + constant(value="\",\"observer.vendor\":\"f5") + constant(value="\",\"observer.product\":\"BigIP") + constant(value="\",\"observer.type\":\"loadbalancer") + constant(value="\",\"event.kind\":\"event") + constant(value="\",\"event.category\":\"web") + constant(value="\",\"event.outcome\":\"") property(name="$.event_outcome") + constant(value="\",\"http.request.method\":\"") property(name="$.method") + constant(value="\",\"destination.domain\":\"") property(name="$.host") + constant(value="\",\"url\":\"") property(name="$.uri") + constant(value="\",\"source.ip\":\"") property(name="$.srcip") + constant(value="\",\"user_agent.original\":\"") property(name="$.uagent") + constant(value="\",\"destination.ip\":\"") property(name="$.vsip") + constant(value="\",\"related.ip\":[") property(name="$.related_ip") + constant(value="],\"vs\":\"") property(name="$.vs") + constant(value="\",\"statusmsg\":\"") property(name="$.statusmsg") + constant(value="\",\"realserver\":\"") property(name="$.realserver") + constant(value="\",\"pool\":\"") property(name="$.pool") + constant(value="\",\"unknown\":\"") property(name="$.unknown") + constant(value="\",\"sessiontime\":") property(name="$.sessiontime") + constant(value=",\"event.duration\":") property(name="$.rtt") + constant(value=",\"reqts\":\"") property(name="$.reqtime") # in UNIX_MS + constant(value=",\"responsetime\":") property(name="$.responsetime") + constant(value=",\"realport\":") property(name="$.realport") + constant(value=",\"http.response.status_code\":") property(name="$.status") + constant(value=",\"http.response.bytes\":") property(name="$.bytes") + constant(value="}") + #geoip: source.ip => source.geo, source.as.asn => source.as.number, source.as.organization_name => source.as.organization.name + #related.hosts += destination.domain + } + # RFC5424 to Elastic + template(name="rfc5424-syslog" type="list" option.json="on") { + constant(value="{") + constant(value="\"@timestamp\":\"") property(name="timereported" dateFormat="rfc3339") + constant(value="\",\"host\":\"") property(name="hostname") + constant(value="\",\"severity\":\"") property(name="syslogseverity-text") + constant(value="\",\"facility\":\"") property(name="syslogfacility-text") + constant(value="\",\"tag\":\"") property(name="syslogtag") + constant(value="\",\"message\":\"") property(name="msg") + constant(value="\"}") + } + imtcp.conf: | + #module(load="imtcp") + input(type="imtcp" port="514") + imudp.conf: | + #module(load="imudp") + #### RULES #### + ruleset(name="rulesetUDPCheckPoint") { + action(type="mmnormalize" rulebase="/etc/rsyslog.d/cpfw.rb" userawmsg="on") + #ToDo: ... + + ### Forwarding to remote-syslog-server via UDP + #action(type="omfwd" Target="172.1.2.3" Port="514" Protocol="udp" Template="tplPlainMsg" + # queue.type="LinkedList" + # queue.dequeuebatchsize="1000" # bulk size + # queue.size="10000" + # queue.workerthreads="1" + #) + } + ####################################### + # + ruleset(name="rulesetUDPf5LTM") { + action(type="mmnormalize" rulebase="/etc/rsyslog.d/f5.rb" userawmsg="on") + if ($parsesuccess == "OK") then { + set $.related_ip = ""; + foreach ($.f in $!kvfields) do { + if ($.f!k == "srcip") then { + set $.srcip = $.f!v; + if ($.related_ip == "") then { + set $.related_ip = '"' & $.srcip & '"'; + } else { + set $.related_ip = $.related_ip & ',"' & $.srcip & '"'; + } + } else if ($.f!k == 'method') then { + set $.method = $.f!v; + } else if ($.f!k == 'host') then { + set $.host = $.f!v; + } else if ($.f!k == 'uri') then { + set $.uri = $.f!v; + } else if ($.f!k == 'uagent') then { + set $.uagent = $.f!v; + } else if ($.f!k == 'vsip') then { + set $.vsip = $.f!v; + if ($.related_ip == "") then { + set $.related_ip = '"' & $.vsip & '"'; + } else { + set $.related_ip = $.related_ip & ',"' & $.vsip & '"'; + } + } else if ($.f!k == 'vsname') then { + set $.vsname = $.f!v; + } else if ($.f!k == 'vs') then { + set $.vs = $.f!v; + } else if ($.f!k == 'status') then { + set $.status = $.f!v; + if (cnum($.status) < 400) then { + set $.event_outcome = "success"; + } else { + set $.event_outcome = "failure"; + } + } else if ($.f!k == 'responsetime') then { + set $.responsetime = $.f!v; + } else if ($.f!k == 'bytes') then { + set $.bytes = $.f!v; + } else if ($.f!k == 'realserver') then { + set $.realserver = $.f!v; + if ($.related_ip == "") then { + set $.related_ip = '"' & $.realserver & '"'; + } else { + set $.related_ip = $.related_ip & ',"' & $.realserver & '"'; + } + } else if ($.f!k == 'realport') then { + set $.realport = $.f!v; + } else if ($.f!k == 'pool') then { + set $.pool = $.f!v; + } else if ($.f!k == 'rtt') then { + set $.rtt = cnum($.f!v)*1000000; + } else if ($.f!k == 'sessiontime') then { + set $.sessiontime = $.f!v; + } else if ($.f!k == 'reqtime') then { + set $.reqtime = $.f!v; + } else { + set $.unknown = $.f!k; + } + } + #action(type="mmdblookup" mmdbfile="/etc/rsyslog.d/GeoLite2-City.mmdb" fields=[":continent:!continent!code", "!location", ":cityname:!city!names!en"] key="$.srcip") + #pipelineName="geoip-info" + action(type="omelasticsearch" server="elk-es-http.elk.svc.cluster.local:9200/logs-f5ltm.log" template="f5-ltmEvents" + usehttps="on" skipverifyhost="on" allowunsignedcerts="on" uid="rsyslog" pwd="******" + writeoperation="create" bulkmode="on" action.resumeretrycount="-1" + queue.type="LinkedList" + queue.dequeuebatchsize="100" # ES bulk size + queue.size="100000" + queue.workerthreads="3" + ) + #unset $!kvfields + } #else { + # action(type="omfile" file="/var/log/debug-f5-rawgmsg.log" template="tplPlainMs") + #} + } + ####################################### + # + ruleset(name="rulesetUDPCiscoFTD") { + set $.tmpmsg = $rawmsg & ", "; + action(type="mmnormalize" rulebase="/etc/rsyslog.d/ciscoFP.rb" userawmsg="off" variable="$.tmpmsg") + if strlen($!unparsed-data) == 0 then { + foreach ($.f in $!kvfields) do { + if ($.f!k == 'ACPolicy') then { + set $.rule.ruleset = $.f!v; + } else if ($.f!k == 'AccessControlRuleName') then { + set $.rule_name = $.f!v; + } else if ($.f!k == 'list_id') then { + set $.rule_name2 = $.f!v; + } else if ($.f!k == 'AccessControlRuleAction') then { + set $.event_outcome = $.f!v; + } else if ($.f!k == 'AccessControlRuleReason') then { + set $.access_control_rule_reason = $.f!v; + } else if ($.f!k == 'ApplicationProtocol') then { + set $.network_protocol = $.f!v; + } else if ($.f!k == 'ArchiveDepth') then { + set $.archive_depth = $.f!v; + } else if ($.f!k == 'ArchiveFileName') then { + set $.archive_file_name = $.f!v; + } else if ($.f!k == 'ArchiveFileStatus') then { + set $.archive_file_status = $.f!v; + } else if ($.f!k == 'ArchiveSHA256') then { + set $.file_hash_sha256 = $.f!v; + } else if ($.f!k == 'Classification') then { + set $.classification = $.f!v; + } else if ($.f!k == 'Client') then { + set $.network_application = $.f!v; + } else if ($.f!k == 'ClientVersion') then { + set $.client_version = $.f!v; + } else if ($.f!k == 'ConnectionDuration') then { + set $.event_duration = $.f!v; + } else if ($.f!k == 'DNS_Sinkhole') then { + set $.dns_sinkhole = $.f!v; + } else if ($.f!k == 'DNS_TTL') then { + set $.dns_ttl = $.f!v; + } else if ($.f!k == 'DNSQuery') then { + set $.dns_question_name = $.f!v; + } else if ($.f!k == 'DNSRecordType') then { + set $.dns_question_type = $.f!v; + } else if ($.f!k == 'DNSResponseType') then { + set $.dns_response_code = $.f!v; + } else if ($.f!k == 'DNSResponseType') then { + set $.dns_response_code = $.f!v; + } else if ($.f!k == 'DNSSICategory') then { + set $.dnssi_category = $.f!v; + } else if ($.f!k == 'DstIP') then { + #destination.address=DstIP + set $.destination_ip = $.f!v; + } else if ($.f!k == 'DstPort') then { + set $.destination_port = $.f!v; + } else if ($.f!k == 'destination_interface') then { + set $.observer_egress_interface_name = $.f!v; + } else if ($.f!k == 'EgressInterface') then { + set $.observer_egress_interface_name = $.f!v; + } else if ($.f!k == 'EgressZone') then { + set $.observer_egress_zone = $.f!v; + } else if ($.f!k == 'Endpoint Profile') then { + set $.endpoint_profile = $.f!v; + } else if ($.f!k == 'FileAction') then { + set $.file_action = $.f!v; + } else if ($.f!k == 'FileCount') then { + set $.file_count = $.f!v; + } else if ($.f!k == 'FileDirection') then { + set $.file_direction = $.f!v; + } else if ($.f!k == 'FileName') then { + set $.file_name = $.f!v; + } else if ($.f!k == 'FilePolicy') then { + set $.file_policy = $.f!v; + } else if ($.f!k == 'FileSHA256') then { + set $.file_hash_sha256 = $.f!v; + } else if ($.f!k == 'FileSandboxStatus') then { + set $.file_sandbox_status = $.f!v; + } else if ($.f!k == 'FileSize') then { + set $.file_size = $.f!v; + } else if ($.f!k == 'FileStorageStatus') then { + set $.file_storage_status = $.f!v; + } else if ($.f!k == 'FirstPacketSecond') then { + set $.event_start = $.f!v; + } else if ($.f!k == 'GID') then { + set $.service_id = $.f!v; + } else if ($.f!k == 'HTTPReferer') then { + set $.http_request_referrer = $.f!v; + } else if ($.f!k == 'HTTPResponse') then { + set $.http_response_status_code = $.f!v; + } else if ($.f!k == 'ICMPCode') then { + set $.icmp_code = $.f!v; + } else if ($.f!k == 'ICMPType') then { + set $.icmp_type = $.f!v; + } else if ($.f!k == 'IPReputationSICategory') then { + set $.ip_reputation_si_category = $.f!v; + } else if ($.f!k == 'IPSCount') then { + set $.ips_count = $.f!v; + } else if ($.f!k == 'source_interface') then { + set $.observer_ingress_interface_name = $.f!v; + } else if ($.f!k == 'IngressInterface') then { + set $.observer_ingress_interface_name = $.f!v; + } else if ($.f!k == 'IngressZone') then { + set $.observer_ingress_zone = $.f!v; + } else if ($.f!k == 'InitiatorBytes') then { + set $.source_bytes = $.f!v; + } else if ($.f!k == 'InitiatorPackets') then { + set $.source_packets = $.f!v; + } else if ($.f!k == 'InlineResult') then { + set $.event_outcome = $.f!v; + } else if ($.f!k == 'IntrusionPolicy') then { + set $.intrusion_policy = $.f!v; + } else if ($.f!k == 'MPLS_Label') then { + set $.mpls_label = $.f!v; + } else if ($.f!k == 'Message') then { + set $.message = $.f!v; + } else if ($.f!k == 'NAPPolicy') then { + set $.nap_policy = $.f!v; + } else if ($.f!k == 'NetBIOSDomain') then { + #target_field: host.hostname + set $.net_bios_domain = $.f!v; + } else if ($.f!k == 'NumIOC') then { + set $.num_ioc = $.f!v; + } else if ($.f!k == 'Prefilter Policy') then { + set $.prefilter_policy = $.f!v; + } else if ($.f!k == 'Priority') then { + set $.priority = $.f!v; + } else if ($.f!k == 'Protocol') then { + set $.network_transport = $.f!v; + } else if ($.f!k == 'ReferencedHost') then { + set $.url_domain = $.f!v; + } else if ($.f!k == 'ResponderBytes') then { + set $.destination_bytes = $.f!v; + } else if ($.f!k == 'ResponderPackets') then { + set $.destination_packets = $.f!v; + } else if ($.f!k == 'Revision') then { + set $.revision = $.f!v; + } else if ($.f!k == 'SHA_Disposition') then { + set $.sha_disposition = $.f!v; + } else if ($.f!k == 'SID') then { + set $.sid = $.f!v; + } else if ($.f!k == 'SSLActualAction') then { + set $.event_outcome = $.f!v; + } else if ($.f!k == 'SSLCertificate') then { + set $.ssl_certificate = $.f!v; + } else if ($.f!k == 'SSLExpectedAction') then { + set $.ssl_expected_action = $.f!v; + } else if ($.f!k == 'SSLFlowStatus') then { + set $.ssl_flow_status = $.f!v; + } else if ($.f!k == 'SSLPolicy') then { + set $.ssl_policy = $.f!v; + } else if ($.f!k == 'SSLRuleName') then { + set $.ssl_rule_name = $.f!v; + } else if ($.f!k == 'SSLServerCertStatus') then { + set $.ssl_server_cert_status = $.f!v; + } else if ($.f!k == 'SSLServerName') then { + set $.server_domain = $.f!v; + } else if ($.f!k == 'SSLSessionID') then { + set $.ssl_session_id = $.f!v; + } else if ($.f!k == 'SSLTicketID') then { + set $.ssl_ticket_id = $.f!v; + } else if ($.f!k == 'SSLURLCategory') then { + set $.sslurl_category = $.f!v; + } else if ($.f!k == 'SSLVersion') then { + set $.ssl_version = $.f!v; + } else if ($.f!k == 'SSSLCipherSuite') then { + set $.ssl_cipher_suite = $.f!v; + } else if ($.f!k == 'SecIntMatchingIP') then { + set $.sec_int_matching_ip = $.f!v; + } else if ($.f!k == 'Security Group') then { + set $.security_group = $.f!v; + } else if ($.f!k == 'SperoDisposition') then { + set $.spero_disposition = $.f!v; + } else if ($.f!k == 'SrcIP') then { + #source.address=SrcIP + set $.source_ip = $.f!v; + } else if ($.f!k == 'SrcPort') then { + set $.source_port = $.f!v; + } else if ($.f!k == 'TCPFlags') then { + set $.tcp_flags = $.f!v; + } else if ($.f!k == 'ThreatName') then { + set $.threat_name = $.f!v; + } else if ($.f!k == 'ThreatScore') then { + set $.threat_score = $.f!v; + } else if ($.f!k == 'Tunnel or Prefilter Rule') then { + set $.tunnel_or_prefilter_rule = $.f!v; + } else if ($.f!k == 'URI') then { + set $.url_original = $.f!v; + } else if ($.f!k == 'URL') then { + set $.url_original = $.f!v; + } else if ($.f!k == 'URLCategory') then { + set $.url_category = $.f!v; + } else if ($.f!k == 'URLReputation') then { + set $.url_reputation = $.f!v; + } else if ($.f!k == 'URLSICategory') then { + set $.urlsi_category = $.f!v; + } else if ($.f!k == 'User') then { + #target_field: user.id + set $.user_name = $.f!v; + } else if ($.f!k == 'UserAgent') then { + set $.user_agent_original = $.f!v; + } else if ($.f!k == 'VLAN_ID') then { + set $.vlan_id = $.f!v; + } else if ($.f!k == 'WebApplication') then { + set $.network_application = $.f!v; + } else if ($.f!k == 'originalClientSrcIP') then { + #client.address=originalClientSrcIP + set $.client_ip = $.f!v; + } else if ($.f!k == 'mapped_source_ip') then { + set $.source_nat_ip = $.f!v; + } else if ($.f!k == 'mapped_source_port') then { + set $.source_nat_port = $.f!v; + } else if ($.f!k == 'mapped_destination_ip') then { + set $.destination_nat_ip = $.f!v; + } else if ($.f!k == 'mapped_destination_port') then { + set $.destination_nat_port = $.f!v; + } else { + set $.unknown = $.f!k; + } + } + if ($.event_code == '430001') then { + set $.event_action = "intrusion-detected"; + } else if ($.event_code == '430002') then { + set $.event_action = "connection-started"; + } else if ($.event_code == '430003') then { + set $.event_action = "connection-finished"; + } else if ($.event_code == '430004') then { + set $.event_action = "file-detected"; + } else if ($.event_code == '430005') then { + set $.event_action = "malware-detected"; + } else { + set $.event_action = "unknown"; + } + #https://github.com/rsyslog/rsyslog/issues/3938 + #action(type="mmdblookup" mmdbfile="/etc/rsyslog.d/GeoLite2-City.mmdb" fields=[":continent:!continent!code", "!location", ":cityname:!city!names!en"] key="$.srcip") + #action(type="mmdblookup" mmdbfile="/etc/rsyslog.d/GeoLite2-City.mmdb" fields=[":continent:!continent!code", "!location", ":cityname:!city!names!en"] key="$.dstip") + # pipelineName="geoip-info" + action(type="omelasticsearch" server="elk-es-http.elk.svc.cluster.local:9200/logs-cisco_ftd63.log" template="FPSyslog-IPConEvents" + usehttps="on" skipverifyhost="on" allowunsignedcerts="on" uid="rsyslog" pwd="******" + writeoperation="create" bulkmode="on" action.resumeretrycount="-1" + queue.type="LinkedList" + queue.dequeuebatchsize="100" # ES bulk size + queue.size="100000" + queue.workerthreads="3" + ) + } + ### Forwarding to remote-syslog-server via TCP + #action(type="omfwd" Target="10.1.2.3" Port="514" Protocol="tcp" Template="tplPlainMsg" + # queue.type="LinkedList" + # queue.dequeuebatchsize="1000" # bulk size + # queue.size="10000" + # queue.workerthreads="1" + #) + } + ####################################### + # + ruleset(name="rulesetUDPGeneric") { + action(type="omelasticsearch" server="elk-es-http.elk.svc.cluster.local:9200/logs-network.log" template="plain-syslog" + usehttps="on" skipverifyhost="on" allowunsignedcerts="on" uid="rsyslog" pwd="******" + writeoperation="create" bulkmode="on" action.resumeretrycount="-1" + queue.type="LinkedList" + queue.dequeuebatchsize="100" # ES bulk size + queue.size="100000" + queue.workerthreads="3" + ) + } + #### RULESET-Bindings #### + input(type="imudp" port="9501" ruleset="rulesetUDPCheckPoint") + input(type="imudp" port="9544" ruleset="rulesetUDPf5LTM") + input(type="imudp" port="9003" ruleset="rulesetUDPCiscoFTD") + #input(type="imudp" port="9004" ruleset="rulesetUDPFortinetFW") + input(type="imudp" port="514" ruleset="rulesetUDPGeneric") + f5.rb: | + version=2 + #http://www.liblognorm.com/files/manual/configuration.html + # rule=:%date:date-rfc3164% %host:word% %tag:char-to:\x3a%: no longer listening on %ip:ipv4%#%port:number%' + # rule=[[,...]]: + # %:[{}]% + #{"type":"string-to", "name":"-", "extradata":":<"}, + rule=:%[ {"type":"literal", "name":"-", "text":"<"}, + {"type":"number", "name":"field1"}, + {"type":"literal", "name":"-", "text":">"}, + {"type":"number", "name":"PRI"}, + {"type":"whitespace", "name":"-"}, + {"type":"date-rfc5424", "name":"eventdate"}, + {"type":"whitespace", "name":"-"}, + {"type":"word", "name":"syslog5424_host"}, + {"type":"whitespace", "name":"-"}, + {"type":"word", "name":"logsource"}, + {"type":"whitespace", "name":"-"}, + {"type":"number", "name":"procid"}, + {"type":"literal", "name":"-", "text":" ["}, + {"name":"kvfields", "type":"repeat", "option.permitMismatchInParser":"true", "parser": [ + {"type":"char-to", "name":"k", "extradata":"="}, + {"type":"literal", "name":"-", "text":"="}, + {"type":"op-quoted-string", "name":"v"} + ], + "while": { + "type":"alternative", "parser": [ + {"type":"literal", "text":" "}, + {"type":"literal", "text":" "} + ] + } + }, + {"type":"rest", "name":"msg"} + ]% + ciscoFP.rb: | + version=2 + #http://www.liblognorm.com/files/manual/configuration.html + rule=:%[ + {"type":"literal", "name":"-", "text":"<"}, + {"type":"number", "name":"field1"}, + {"type":"literal", "name":"-", "text":">"}, + {"type":"date-rfc5424", "name":"eventdate"}, + {"type":"whitespace", "name":"-"}, + {"type":"char-to", "name":"syslog5424_host", "extradata":" "}, + {"type":"whitespace", "name":"-"}, + {"type":"char-to", "name":"ciscoftd", "extradata":"-"}, + {"type":"literal", "name":"-", "text":"-"}, + {"type":"number", "name":"event.severity"}, + {"type":"literal", "name":"-", "text":"-"}, + {"type":"number", "name":"event_code"}, + {"type":"literal", "name":"-", "text":":"}, + {"type":"whitespace", "name":"-"}, + {"name":"kvfields", "type":"repeat", "option.permitMismatchInParser":"false", "parser": [ + {"type":"char-to", "name":"k", "extradata":":"}, + {"type":"literal", "name":"-", "text":":"}, + {"type":"string", "name":"v", "quoting.char.begin":" ", "quoting.char.end":","} + ], + "while": { + "type":"alternative", "parser": [ + {"type":"literal", "text":" "} + ] + } + }, + {"type":"rest", "name":"msg"} + ]% + cpfw.rb: | + version=2 + #http://www.liblognorm.com/files/manual/configuration.html + rule=:%[ {"type":"literal", "name":"-", "text":"<"}, + {"type":"number", "name":"field1"}, + {"type":"literal", "name":"-", "text":">"}, + {"type":"number", "name":"PRI"}, + {"type":"whitespace", "name":"-"}, + {"type":"date-rfc5424", "name":"eventdate"}, + {"type":"whitespace", "name":"-"}, + {"type":"word", "name":"syslog5424_host"}, + {"type":"whitespace", "name":"-"}, + {"type":"word", "name":"logsource"}, + {"type":"whitespace", "name":"-"}, + {"type":"number", "name":"procid"}, + {"type":"literal", "name":"-", "text":" - ["}, + {"name":"kvfields", "type":"repeat", "option.permitMismatchInParser":"false", "parser": [ + {"type":"char-to", "name":"k", "extradata":":"}, + {"type":"literal", "name":"-", "text":":"}, + {"type":"op-quoted-string", "name":"v"}, + {"type":"literal", "name":"-", "text":";"} + ], + "while": { + "type":"alternative", "parser": [ + {"type":"literal", "text":" "}, + {"type":"literal", "text":" "} + ] + } + }, + {"type":"rest", "name":"-"} + ]% + lookup_ftd.tbl: | + { + "version" : 1, + "nomatch" : "UnknownFW", + "type" : "string", + "table" : [ + {"index" : "172.19.1.2", "value" : "Selfdefined-FirewallName" } + ] + } + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: rsyslog-deployment + namespace: logging + labels: + app: rsyslog4elk +spec: + replicas: 3 + selector: + matchLabels: + app: rsyslog4elk + template: + metadata: + labels: + app: rsyslog4elk + spec: + containers: + - name: rsyslog + #image: rsyslog/syslog_appliance_alpine + image: private-repository.default.svc.cluster.local:5000/rsyslog-container:alpine-3.15 + ports: + - containerPort: 514 + env: + - name: ENABLE_LOGFILES + value: "off" + - name: ENABLE_LOGSENE + value: "off" + - name: ENABLE_STATISTICS + value: "on" + #- name: RSYSLOG_DEBUG_FLAG + # value: "-d" + livenessProbe: + #exec: + # command: + # - /bin/sh + # - -c + # - ps aux | grep rsyslog[d] + tcpSocket: + port: 514 + initialDelaySeconds: 30 + periodSeconds: 5 + timeoutSeconds: 2 #Default 1 + resources: + requests: + cpu: 500m + memory: 524Mi + volumeMounts: + - name: rsyslogconfig-volume + mountPath: /etc/rsyslog.conf + subPath: rsyslog.conf + - name: config-volume + mountPath: /etc/rsyslog.d + - name: spool-volume + mountPath: /rsyslog-queues + restartPolicy: Always + terminationGracePeriodSeconds: 60 + volumes: + - name: rsyslogconfig-volume + configMap: + name: rsyslog-configmap + items: + - key: rsyslog.conf + path: rsyslog.conf + # Attention: configMap to File-Path will only be updated when a POD is created !!! + - name: config-volume + configMap: + name: rsyslog-d + - emptyDir: {} + name: spool-volume