Full rsyslog Alpine-Appliance

appliance/README.md

@@ -12,6 +12,43 @@ more info:
 - https://github.com/rsyslog/rsyslog/issues/2368
 - https://github.com/rsyslog/rsyslog/projects/5
 
+## Deployment example for Kubernetes:
+To run your rsyslog-container in production under Kubernetes, you can start with this template
+- ../rsyslog-deployment.yaml
+
+### Kubernetes infos for high load traffic
+If you have much traffic from the same source-ip, you have to tune your Loadbalancer-settings.
+I case of a ipvs-implementation in Kubernetes:
+```
+ipvsadm -Ln --timeout
+### e.g.: Timeout (tcp tcpfin udp): 900 120 300
+###        => if you would like Load-Distribution for the same Source-IP => set UDP-Timeout to 1:
+ipvsadm --set 900 120 1
+```
+
+To make this settings permanent, you can do it e.g. in this way:
+```
+cat <<EOF > /etc/systemd/system/ipvs-config.service
+[Unit]
+Description=Configure IPVS
+After=network.target
+
+[Service]
+Type=oneshot
+ExecStart=/sbin/ipvsadm --set 900 120 1
+RemainAfterExit=false
+StandardOutput=journal
+
+[Install]
+WantedBy=multi-user.target
+EOF
+
+
+systemctl daemon-reload
+systemctl enable ipvs-config
+systemctl start ipvs-config
+```
+
 ## projects that provide docker containers:
 
 - https://github.com/deoren/rsyslog-docker (based on @halfer provided files)

appliance/alpine/Dockerfile

@@ -1,29 +1,69 @@
-FROM	alpine:3.7
+FROM    alpine:3.15
 LABEL   maintainer="[email protected]"
-COPY	[email protected] /etc/apk/keys/[email protected]
-RUN	echo "http://alpine.rsyslog.com/3.7/stable" >> /etc/apk/repositories \
-	&& apk --no-cache update  \
-	&& apk add --no-cache \
-	   rsyslog \
-	   rsyslog-elasticsearch \
-	   rsyslog-imptcp \
-	   rsyslog-imrelp \
-	   rsyslog-mmjsonparse \
-	   rsyslog-mmutf8fix \
-	   rsyslog-omrelp \
-	   rsyslog-omstdout
-RUN	adduser -s /bin/ash -D rsyslog rsyslog \
-	&& echo "rsyslog ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers
-VOLUME	/config /work /logs
-CMD	["rsyslog"]
-ENTRYPOINT ["/home/appliance/starter.sh"]
-COPY	rsyslog.conf /etc/rsyslog.conf
-COPY	rsyslog.conf.d/*.conf /etc/rsyslog.conf.d/
+COPY    [email protected] /etc/apk/keys/[email protected]
+
+USER root
+RUN sed -i 's/https/http/' /etc/apk/repositories
+
+#RUN     echo "http://alpine.rsyslog.com/3.15/stable" >> /etc/apk/repositories \
+RUN     apk --no-cache update  \
+        && apk add --no-cache \
+           rsyslog \
+           rsyslog-mmjsonparse \
+           rsyslog-pgsql \
+           rsyslog-hiredis \
+           rsyslog-snmp \
+           rsyslog-mmnormalize \
+           rsyslog-dbg \
+           rsyslog-clickhouse \
+           rsyslog-mmsequence \
+           rsyslog-gssapi \
+           rsyslog-pmlastmsg \
+           rsyslog-mmfields \
+           rsyslog-tls \
+           rsyslog-mmpstrucdata \
+           rsyslog-elasticsearch \
+           rsyslog-mmsnmptrapd \
+           rsyslog-udpspoof \
+           rsyslog-uxsock \
+           rsyslog-http \
+           rsyslog-imdocker \
+           rsyslog-relp \
+           rsyslog-mmaudit \
+           rsyslog-mysql \
+           rsyslog-mmrm1stspace \
+           rsyslog-mmutf8fix \
+           rsyslog-crypto \
+           rsyslog-libdbi \
+           rsyslog-pmsnare \
+           rsyslog-openrc \
+           rsyslog-mmcount \
+           rsyslog-zmq \
+           rsyslog-mmanon \
+           rsyslog-rabbitmq \
+           rsyslog-mmdblookup \
+           rsyslog-pmaixforwardedfrom \
+           libc-utils
+RUN     adduser -s /bin/ash -D rsyslog rsyslog \
+        && echo "rsyslog ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers
+VOLUME  /config /work /logs
+COPY    rsyslog.conf /etc/rsyslog.conf
+COPY    rsyslog.conf.d/*.conf /etc/rsyslog.conf.d/
 # done base system setup
 
 WORKDIR /home/appliance
-COPY	starter.sh CONTAINER.* ./
-COPY	internal/* ./internal/
-COPY	tools/* ./tools/
-RUN	echo "`date +%F` (`date +%s`)" > CONTAINER.release \
-	&& chown -R rsyslog:rsyslog *
+COPY    starter.sh CONTAINER.* ./
+COPY    internal/* ./internal/
+COPY    internal/container_config /config/
+COPY    internal/droprules.conf /config/
+COPY    tools/* ./tools/
+COPY    custom-builds/* /usr/lib/rsyslog/
+COPY    custom-builds-liblog/* /usr/lib/
+RUN     echo "`date +%F` (`date +%s`)" > CONTAINER.release \
+        && chown -R rsyslog:rsyslog * && \
+        chmod +x starter.sh
+
+USER rsyslog
+CMD     ["rsyslog"]
+ENTRYPOINT ["/home/appliance/starter.sh"]
+

appliance/alpine/custom-builds/omelasticsearch.la

@@ -0,0 +1,41 @@
+# omelasticsearch.la - a libtool library file
+# Generated by libtool (GNU libtool) 2.4.6
+#
+# Please DO NOT delete this file!
+# It is necessary for linking the library.
+
+# The name that we can dlopen(3).
+dlname=''
+
+# Names of this library.
+library_names=''
+
+# The name of the static archive.
+old_library='omelasticsearch.a'
+
+# Linker flags that cannot go in dependency_libs.
+inherited_linker_flags=''
+
+# Libraries that this one depends upon.
+dependency_libs=' -lcurl -lm'
+
+# Names of additional weak libraries provided by this library
+weak_library_names=''
+
+# Version information for omelasticsearch.
+current=0
+age=0
+revision=0
+
+# Is this an already installed library?
+installed=yes
+
+# Should we warn about portability when linking against -modules?
+shouldnotlink=yes
+
+# Files to dlopen/dlpreopen
+dlopen=''
+dlpreopen=''
+
+# Directory that this library needs to be installed in:
+libdir='/usr/local/lib/rsyslog'

appliance/alpine/internal/container_config

@@ -9,7 +9,7 @@ export TZ=UTC
 export ENABLE_STATISTICS=on
 
 # Do we write log files?
-export ENABLE_LOGFILES=on # yes, we do (comment out to disable)
+export ENABLE_LOGFILES=off # (comment out to disable)
 # Where do we write to?
 # path for host-specific files is: /logs/hosts/HOSTNAME
 export LOGFILES_STORE="/logs/hosts/%hostname:::secpath-replace%/messages.log"

appliance/alpine/starter.sh

@@ -6,7 +6,8 @@ if [ "$CONTAINER_SILENT" != "on" ]; then
 	echo `cat CONTAINER.name` version `cat CONTAINER.release` - `cat CONTAINER.homepage`
 	echo `cat CONTAINER.copyright`
 	echo
-	echo "WARNING: this is an experimental container - do not use in production"
+	echo "This container is build for production."
+	echo "See online: https://github.com/EHerzog76/rsyslog-docker"
 	echo
 fi
 
@@ -40,6 +41,7 @@ echo "Using rsyslog configuration file: $RSYSLOG_CONF"
 
 
 if [ -f tools/$1 ]; then
+        #rsyslogd -D -N 1 -f /etc/rsyslog.conf
 	source tools/$1
 else
 	echo "ERROR: command not known: $*"

appliance/alpine/tools/CONTAINER.copyright

@@ -0,0 +1 @@
+Copyright (C) 2018 by Rainer Gerhards and Adiscon GmbH - released under ASL 2.0

appliance/alpine/tools/CONTAINER.homepage

@@ -0,0 +1 @@
+http://www.syslogappliance.de

appliance/alpine/tools/CONTAINER.name

@@ -0,0 +1 @@
+rsyslog appliance

appliance/alpine/tools/Dockerfile

@@ -0,0 +1,29 @@
+FROM	alpine:3.7
+LABEL   maintainer="[email protected]"
+COPY	[email protected] /etc/apk/keys/[email protected]
+RUN	echo "http://alpine.rsyslog.com/3.7/stable" >> /etc/apk/repositories \
+	&& apk --no-cache update  \
+	&& apk add --no-cache \
+	   rsyslog \
+	   rsyslog-elasticsearch \
+	   rsyslog-imptcp \
+	   rsyslog-imrelp \
+	   rsyslog-mmjsonparse \
+	   rsyslog-mmutf8fix \
+	   rsyslog-omrelp \
+	   rsyslog-omstdout
+RUN	adduser -s /bin/ash -D rsyslog rsyslog \
+	&& echo "rsyslog ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers
+VOLUME	/config /work /logs
+CMD	["rsyslog"]
+ENTRYPOINT ["/home/appliance/starter.sh"]
+COPY	rsyslog.conf /etc/rsyslog.conf
+COPY	rsyslog.conf.d/*.conf /etc/rsyslog.conf.d/
+# done base system setup
+
+WORKDIR /home/appliance
+COPY	starter.sh CONTAINER.* ./
+COPY	internal/* ./internal/
+COPY	tools/* ./tools/
+RUN	echo "`date +%F` (`date +%s`)" > CONTAINER.release \
+	&& chown -R rsyslog:rsyslog *

appliance/alpine/tools/README.md

@@ -0,0 +1,89 @@
+## Configuring the syslog appliance
+
+### Files
+
+Container and rsyslog configuration are read from the /config directory. You should
+mount it to a volume:
+
+  $ docker run -v myconfig:/config ...
+
+Upon initial creation of the volume, it is populated with a default file that
+you can modify. Note that this happens only for **volume** mounts.
+
+To show the current container config, run
+
+  $ docker run ... tools/show-config
+
+You can also use this to get a template for your own container config if you use
+bind mounts instead of volumes. In this case, make sure that you have **not**
+mounted /config - the container will then use it's own default file.
+
+Note: volumes are automatically populated with the default file upon creation.
+
+### Environment Variables
+
+- TZ
+
+  Default: /etc/localtime
+
+  Change it to set a specific timezone, e.g. TZ=UTC
+
+- RSYSLOG_CONF
+
+  Default: /etc/rsyslog.conf
+
+  If you want to totally replace the default rsyslog configuration with
+  your custom config,
+
+  1. create a config file in /config volume or bind mount, e.g. myrsyslog.conf
+  2. set RSYSLOG_CONF=/config/myrsyslog.conf
+
+  Keep in mind that the myconfig: volume is accessible via /config inside the
+  container.
+
+- LOGSENE_TOKEN
+
+  Default: disabled
+
+  If you are using Sematext Logsene, set this to your Logsene token.
+
+- LOGSENE_URL
+
+  Default: disabled
+
+  If you are using Sematext Logsene, set this to the Logsene URL. Ex: logsene-receiver.sematext.com or logsene-receiver.eu.sematext.com
+
+- RSYSLOG_CONFIG_BASE64
+
+  Default: disabled
+
+  If you would like to overwite the `/etc/rsyslog.conf` file, _without_ mounting a configuration file into the container, you can use this variable. The contents are the base64 encoded `rsyslog.conf` file contents, without newlines. This can be generated with the following command: `cat rsyslog.conf | base64 | tr -d '\n'`. On startup, the contents of the environment variable will be decoded and overwrite the `/etc/rsyslog.conf` file.
+
+# Runtime Environment
+
+## Volumes
+
+### /config
+
+Holds the container configuration, also the recommended place for overwriting
+the rsyslog configuration.
+
+This volume can be mounted read-only after initial population with sample files.
+
+### /work
+
+The rsyslog work directory. This is used for spool files and other files that
+rsyslog needs to be persisted over runs.
+
+This volume needs to be mounted writable and **must** be persisted between
+container invocations.
+
+**Warning: this volume is specific to one rsyslog instance.** It **must not**
+be shared between multiple container instances, else strange problems may
+occur.
+
+### /logs
+
+This holds log files if the container is configured to write them.
+
+Needs to be mounted writable.

appliance/alpine/tools/build.sh

@@ -0,0 +1 @@
+docker build $* -t rsyslog/syslog_appliance_alpine:latest .

appliance/alpine/tools/custom-builds/omelasticsearch.la

@@ -0,0 +1,41 @@
+# omelasticsearch.la - a libtool library file
+# Generated by libtool (GNU libtool) 2.4.6
+#
+# Please DO NOT delete this file!
+# It is necessary for linking the library.
+
+# The name that we can dlopen(3).
+dlname=''
+
+# Names of this library.
+library_names=''
+
+# The name of the static archive.
+old_library='omelasticsearch.a'
+
+# Linker flags that cannot go in dependency_libs.
+inherited_linker_flags=''
+
+# Libraries that this one depends upon.
+dependency_libs=' -lcurl -lm'
+
+# Names of additional weak libraries provided by this library
+weak_library_names=''
+
+# Version information for omelasticsearch.
+current=0
+age=0
+revision=0
+
+# Is this an already installed library?
+installed=yes
+
+# Should we warn about portability when linking against -modules?
+shouldnotlink=yes
+
+# Files to dlopen/dlpreopen
+dlopen=''
+dlpreopen=''
+
+# Directory that this library needs to be installed in:
+libdir='/usr/local/lib/rsyslog'