Align to newest Snap, Event Collector Changes (AO-19422) (#33)

solarwinds · Mar 29, 2021 · 077ea8d · 077ea8d
1 parent 1adf21a
commit 077ea8d
Show file tree

Hide file tree

Showing 19 changed files with 453 additions and 364 deletions.
diff --git a/CODEOWNERS b/CODEOWNERS
@@ -1,2 +1,2 @@
-* @melord @d-maslyk @tomekkolo @swi-snap-agent
+* @swisnap-krk 
 
diff --git a/Dockerfile b/Dockerfile
@@ -1,6 +1,6 @@
 FROM ubuntu:bionic
 
-LABEL authors='SolarWinds AppOptics team <support@appoptics.com>'
+LABEL authors='SolarWinds AppOptics team <technicalsupport@solarwinds.com>'
 
 USER root
 ARG DEBIAN_FRONTEND=noninteractive
@@ -37,6 +37,5 @@ COPY ./conf/swisnap-init.sh /opt/SolarWinds/Snap/etc/init.sh
 WORKDIR /opt/SolarWinds/Snap
 
 EXPOSE 21413
-
 # Run SolarWinds Snap Agent
 CMD ["/opt/SolarWinds/Snap/etc/init.sh"]
diff --git a/Makefile b/Makefile
@@ -1,5 +1,5 @@
-DOCKERFILE_VERSION=4.2.0
-SWISNAP_VERSION=4.0.0.863
+DOCKERFILE_VERSION=4.3.0
+SWISNAP_VERSION=4.1.0.1024
 TAG=$(DOCKERFILE_VERSION)-$(SWISNAP_VERSION)
 USER=solarwinds
 REPOSITORY=solarwinds-snap-agent-docker

diff --git a/README.md b/README.md
diff --git a/conf/swisnap-init.sh b/conf/swisnap-init.sh
@@ -8,6 +8,7 @@ TASK_AUTOLOAD_DIR="${SWISNAP_HOME}/etc/tasks-autoload.d"
 CONFIG_FILE="${SWISNAP_HOME}/etc/config.yaml"
 PUBLISHER_PROCESSES_CONFIG="${PLUGINS_DIR}/publisher-processes.yaml"
 PUBLISHER_APPOPTICS_CONFIG="${PLUGINS_DIR}/publisher-appoptics.yaml"
+PUBLISHER_LOGS_CONFIG="${PLUGINS_DIR}/publisher-logs.yaml"
 
 swisnap_config_setup() {
     # SOLARWINDS_TOKEN is required. Please note, that APPOPTICS_TOKEN is left for preserving backward compatibility
@@ -24,8 +25,12 @@ swisnap_config_setup() {
     yq w -i "${PUBLISHER_APPOPTICS_CONFIG}" v2.publisher.publisher-appoptics.all.endpoint.token -- "${SWI_TOKEN}"
     yq w -i "${PUBLISHER_PROCESSES_CONFIG}" v2.publisher.publisher-processes.all.endpoint.token -- "${SWI_TOKEN}"
 
-    yq w -i ${CONFIG_FILE} log_path "${LOG_PATH:-/proc/self/fd/1}"
-    yq w -i ${CONFIG_FILE} restapi.addr "tcp://0.0.0.0:21413"
+    # Use APPOPTICS_HOSTNAME as hostname_alias
+    if [ -n "${APPOPTICS_HOSTNAME}" ]; then
+        yq w -i "${PUBLISHER_APPOPTICS_CONFIG}" v1.publisher.publisher-appoptics.all.hostname_alias "${APPOPTICS_HOSTNAME}"
+        yq w -i "${PUBLISHER_APPOPTICS_CONFIG}" v2.publisher.publisher-appoptics.all.endpoint.hostname_alias "${APPOPTICS_HOSTNAME}"
+        yq w -i "${PUBLISHER_PROCESSES_CONFIG}" v2.publisher.publisher-processes.all.endpoint.hostname_alias "${APPOPTICS_HOSTNAME}"
+    fi
 
     if [ -n "${LOG_LEVEL}" ]; then
         yq w -i $CONFIG_FILE log_level "${LOG_LEVEL}"
@@ -37,10 +42,35 @@ swisnap_config_setup() {
         yq w -i ${CONFIG_FILE} control.plugin_trust_level 0
     fi
 
-    # Use APPOPTICS_HOSTNAME as hostname_alias
-    if [ -n "${APPOPTICS_HOSTNAME}" ]; then
-        yq w -i "${CONFIG_FILE}" control.plugins.publisher.publisher-appoptics.all.hostname_alias "${APPOPTICS_HOSTNAME}"
+    yq w -i ${CONFIG_FILE} log_path "${LOG_PATH:-/proc/self/fd/1}"
+    yq w -i ${CONFIG_FILE} restapi.addr "tcp://0.0.0.0:21413"
+
+    # Logs Publishers releated configs
+    if [ -n "${LOGGLY_TOKEN}" ] && [ "${LOGGLY_TOKEN}" != 'LOGGLY_TOKEN' ]; then
+        LOGGLY_PUBL_TOKEN="${LOGGLY_TOKEN}"
+    else
+        LOGGLY_PUBL_TOKEN="${SWI_TOKEN}"
     fi
+
+    yq w -i "${PUBLISHER_LOGS_CONFIG}" v2.publisher.loggly-http.all.token -- "${LOGGLY_PUBL_TOKEN}"
+    yq w -i "${PUBLISHER_LOGS_CONFIG}" v2.publisher.loggly-http-bulk.all.token -- "${LOGGLY_PUBL_TOKEN}"
+    yq w -i "${PUBLISHER_LOGS_CONFIG}" v2.publisher.loggly-syslog.all.token -- "${LOGGLY_PUBL_TOKEN}"
+
+    if [ -n "${PAPERTRAIL_TOKEN}" ] && [ "${PAPERTRAIL_TOKEN}" != 'PAPERTRAIL_TOKEN' ]; then
+        PAPERTRAIL_PUBL_TOKEN="${PAPERTRAIL_TOKEN}"
+    else
+        PAPERTRAIL_PUBL_TOKEN="${SWI_TOKEN}"
+    fi
+
+    yq w -i "${PUBLISHER_LOGS_CONFIG}" v2.publisher.swi-logs-http-bulk.all.token -- "${PAPERTRAIL_PUBL_TOKEN}"
+    yq w -i "${PUBLISHER_LOGS_CONFIG}" v2.publisher.swi-logs-http.all.token -- "${PAPERTRAIL_PUBL_TOKEN}"
+
+    if [ -n "${PAPERTRAIL_HOST}" ] && [ -n "${PAPERTRAIL_PORT}" ]; then
+       yq w -i "${PUBLISHER_LOGS_CONFIG}" v2.publisher.papertrail-syslog.all.host "${PAPERTRAIL_HOST}"
+       yq w -i "${PUBLISHER_LOGS_CONFIG}" v2.publisher.papertrail-syslog.all.port "${PAPERTRAIL_PORT}"
+    fi
+
+
 }
 
 run_plugins_with_default_configs() {
@@ -49,13 +79,29 @@ run_plugins_with_default_configs() {
         mv "${PLUGINS_DIR}/apache.yaml.example" "${PLUGINS_DIR}/apache.yaml"
     fi
 
-    if [ "$SWISNAP_ENABLE_DOCKER" = "true" ]; then
+    if [ "${SWISNAP_ENABLE_DOCKER}" = "true" ]; then
         mv "${PLUGINS_DIR}/docker.yaml.example" "${PLUGINS_DIR}/docker.yaml"
         if [[ -n "${HOST_PROC}" ]]; then
             sed -i 's,procfs: "/proc",procfs: "'"${HOST_PROC}"'",g' "${PLUGINS_DIR}/docker.yaml"
         fi
     fi
 
+    if [ "${SWISNAP_ENABLE_DOCKER_LOGS}" = "true" ] && [ -n "${SWISNAP_DOCKER_LOGS_CONTAINER_NAMES}" ]; then
+        DOCKER_LOGS_CONFIG="${TASK_AUTOLOAD_DIR}/task-logs-docker.yaml"
+        mv "${DOCKER_LOGS_CONFIG}.example" "${DOCKER_LOGS_CONFIG}"
+        yq d -i "${DOCKER_LOGS_CONFIG}" 'plugins.(plugin_name==docker-logs).config.logs'
+        for cont_name in ${SWISNAP_DOCKER_LOGS_CONTAINER_NAMES}; do
+            yq w -i "${DOCKER_LOGS_CONFIG}" "plugins.(plugin_name==docker-logs).config.logs[+].filters.name.${cont_name}" true
+        done
+
+        yq w -i "${DOCKER_LOGS_CONFIG}" 'plugins.(plugin_name==docker-logs).config.logs[*].options.showstdout' true
+        yq w -i "${DOCKER_LOGS_CONFIG}" 'plugins.(plugin_name==docker-logs).config.logs[*].options.showstderr' true
+        yq w -i "${DOCKER_LOGS_CONFIG}" 'plugins.(plugin_name==docker-logs).config.logs[*].options.follow' true
+        yq w -i "${DOCKER_LOGS_CONFIG}" 'plugins.(plugin_name==docker-logs).config.logs[*].options.tail' all
+        yq w -i "${DOCKER_LOGS_CONFIG}" 'plugins.(plugin_name==docker-logs).config.logs[*].options.since' --tag '!!str' ""
+    fi
+
+
     if [ "${SWISNAP_ENABLE_ELASTICSEARCH}" = "true" ]; then
         mv "${PLUGINS_DIR}/elasticsearch.yaml.example" "${PLUGINS_DIR}/elasticsearch.yaml"
     fi
@@ -64,6 +110,15 @@ run_plugins_with_default_configs() {
         mv "${PLUGINS_DIR}/kubernetes.yaml.example" "${PLUGINS_DIR}/kubernetes.yaml"
     fi
 
+    if [ "${SWISNAP_ENABLE_KUBERNETES_LOGS}" = "true" ]; then
+        KUBERNETES_LOGS_CONFIG="${TASK_AUTOLOAD_DIR}/task-logs-k8s-events.yaml"
+        mv "${KUBERNETES_LOGS_CONFIG}.example" "${KUBERNETES_LOGS_CONFIG}"
+        yq w -i "${KUBERNETES_LOGS_CONFIG}" 'plugins.(plugin_name==k8s-events).config.incluster' 'true'
+        yq w -i "${KUBERNETES_LOGS_CONFIG}" 'plugins.(plugin_name==k8s-events).config.filters[+].namespace' 'default'
+        yq w -i "${KUBERNETES_LOGS_CONFIG}" 'plugins.(plugin_name==k8s-events).config.filters[*].watch_only' 'true'
+        yq w -i "${KUBERNETES_LOGS_CONFIG}" 'plugins.(plugin_name==k8s-events).config.filters[*].options.fieldSelector' 'type==Normal'
+    fi
+
     if [ "${SWISNAP_ENABLE_NGINX}" = "true" ]; then
         NGINX_CONFIG="${TASK_AUTOLOAD_DIR}/task-bridge-nginx.yaml"
         mv "${NGINX_CONFIG}.example" "${NGINX_CONFIG}"
@@ -174,7 +229,6 @@ run_plugins_with_default_configs() {
     fi
 
     if [ "${SWISNAP_DISABLE_HOSTAGENT}" = "true" ]; then
-        rm "${TASK_AUTOLOAD_DIR}/task-aosystem-warmup.yaml"
         rm "${TASK_AUTOLOAD_DIR}/task-aosystem.yaml"
     fi
 

diff --git a/deploy/base/daemonset/kustomization.yaml b/deploy/base/daemonset/kustomization.yaml
@@ -24,7 +24,7 @@ configMapGenerator:
       - SWISNAP_ENABLE_ZOOKEEPER=false
       - SWISNAP_DISABLE_HOSTAGENT=false
       - SWISNAP_DISABLE_PROCESSES=false
-      - SWISNAP_SECURE=false
+      - SWISNAP_SECURE=true
       - HOST_PROC=/host/proc
       - LOG_LEVEL=WARN
 

diff --git a/deploy/base/daemonset/swisnap-agent-daemonset.yaml b/deploy/base/daemonset/swisnap-agent-daemonset.yaml
@@ -44,6 +44,30 @@ spec:
                   name: solarwinds-token
                   key: SOLARWINDS_TOKEN
                   optional: true
+            - name: LOGGLY_TOKEN
+              valueFrom:
+                secretKeyRef:
+                  name: loggly-token
+                  key: LOGGLY_TOKEN
+                  optional: true
+            - name: PAPERTRAIL_TOKEN
+              valueFrom:
+                secretKeyRef:
+                  name: papertrail-token
+                  key: PAPERTRAIL_TOKEN
+                  optional: true
+            - name: PAPERTRAIL_HOST
+              valueFrom:
+                secretKeyRef:
+                  name: papertrail-publisher-settings
+                  key: PAPERTRAIL_HOST
+                  optional: true
+            - name: PAPERTRAIL_PORT
+              valueFrom:
+                secretKeyRef:
+                  name: papertrail-publisher-settings
+                  key: PAPERTRAIL_PORT
+                  optional: true
           envFrom:
             - configMapRef:
                 name: swisnap-host-configmap

diff --git a/deploy/base/deployment/kustomization.yaml b/deploy/base/deployment/kustomization.yaml
@@ -25,7 +25,7 @@ configMapGenerator:
       - SWISNAP_ENABLE_ZOOKEEPER=false
       - SWISNAP_DISABLE_HOSTAGENT=true
       - SWISNAP_DISABLE_PROCESSES=true
-      - SWISNAP_SECURE=false
+      - SWISNAP_SECURE=true
       - HOST_PROC=/host/proc
       - LOG_LEVEL=WARN
 

diff --git a/deploy/base/deployment/swisnap-agent-deployment.yaml b/deploy/base/deployment/swisnap-agent-deployment.yaml
@@ -48,6 +48,30 @@ spec:
                   name: solarwinds-token
                   key: SOLARWINDS_TOKEN
                   optional: true
+            - name: LOGGLY_TOKEN
+              valueFrom:
+                secretKeyRef:
+                  name: loggly-token 
+                  key: LOGGLY_TOKEN
+                  optional: true
+            - name: PAPERTRAIL_TOKEN
+              valueFrom:
+                secretKeyRef:
+                  name: papertrail-token
+                  key: PAPERTRAIL_TOKEN
+                  optional: true
+            - name: PAPERTRAIL_HOST
+              valueFrom:
+                secretKeyRef:
+                  name: papertrail-publisher-settings
+                  key: PAPERTRAIL_HOST
+                  optional: true
+            - name: PAPERTRAIL_PORT
+              valueFrom:
+                secretKeyRef:
+                  name: papertrail-publisher-settings
+                  key: PAPERTRAIL_PORT
+                  optional: true
           envFrom:
             - configMapRef:
                 name: swisnap-k8s-configmap

diff --git a/deploy/base/events-collector/kustomization.yaml b/deploy/base/events-collector/kustomization.yaml
@@ -28,5 +28,5 @@ configMapGenerator:
       - SWISNAP_ENABLE_ZOOKEEPER='false'
       - SWISNAP_DISABLE_HOSTAGENT='true'
       - SWISNAP_DISABLE_PROCESSES='true'
-      - SWISNAP_SECURE='false'
+      - SWISNAP_SECURE='true'
       - HOST_PROC='/host/proc'
diff --git a/deploy/base/events-collector/swisnap-agent-deployment-event-collector.yaml b/deploy/base/events-collector/swisnap-agent-deployment-event-collector.yaml
@@ -22,18 +22,9 @@ spec:
           image: 'solarwinds/solarwinds-snap-agent-docker:latest'
           imagePullPolicy: IfNotPresent
           volumeMounts:
-            - name: plugins-logs
-              mountPath: /opt/SolarWinds/Snap/etc/plugins.d/logs-v2.yaml
-              subPath: logs-v2.yaml
-            - name: plugins-kubernetes
-              mountPath: /opt/SolarWinds/Snap/etc/plugins.d/kubernetes.yaml
-              subPath: kubernetes.yaml
-            - name: tasks-aokubernetes
-              mountPath: /opt/SolarWinds/Snap/etc/tasks.d/task-aokubernetes.yaml
-              subPath: task-aokubernetes.yaml
-            - name: tasks-autoload-logs
-              mountPath: /opt/SolarWinds/Snap/etc/tasks-autoload.d/task-logs-k8s.yaml
-              subPath: task-logs-k8s.yaml
+            - name: task-logs-k8s-events
+              mountPath: /opt/SolarWinds/Snap/etc/tasks-autoload.d/task-logs-k8s-events.yaml
+              subPath: task-logs-k8s-events.yaml
           env:
             - name: APPOPTICS_TOKEN
               valueFrom:
@@ -47,6 +38,30 @@ spec:
                   name: solarwinds-token
                   key: SOLARWINDS_TOKEN
                   optional: true
+            - name: LOGGLY_TOKEN
+              valueFrom:
+                secretKeyRef:
+                  name: loggly-token
+                  key: LOGGLY_TOKEN
+                  optional: true
+            - name: PAPERTRAIL_TOKEN
+              valueFrom:
+                secretKeyRef:
+                  name: papertrail-token
+                  key: PAPERTRAIL_TOKEN
+                  optional: true
+            - name: PAPERTRAIL_HOST
+              valueFrom:
+                secretKeyRef:
+                  name: papertrail-publisher-settings
+                  key: PAPERTRAIL_HOST
+                  optional: true
+            - name: PAPERTRAIL_PORT
+              valueFrom:
+                secretKeyRef:
+                  name: papertrail-publisher-settings
+                  key: PAPERTRAIL_PORT
+                  optional: true
           envFrom:
             - configMapRef:
                 name: swisnap-agent-k8s-events-configmap
@@ -70,30 +85,12 @@ spec:
             initialDelaySeconds: 5
             timeoutSeconds: 5
       volumes:
-        - name: plugins-logs
-          configMap:
-            name: plugin-configs
-            items:
-              - key: logs-v2.yaml
-                path: logs-v2.yaml
-        - name: plugins-kubernetes
-          configMap:
-            name: plugin-configs
-            items:
-              - key: kubernetes.yaml
-                path: kubernetes.yaml
-        - name: tasks-autoload-logs
+        - name: task-logs-k8s-events
           configMap:
             name: task-autoload
             items:
-              - key: task-logs-k8s.yaml
-                path: task-logs-k8s.yaml
-        - name: tasks-aokubernetes
-          configMap:
-            name: task-manifests
-            items:
-              - key: task-aokubernetes.yaml
-                path: task-aokubernetes.yaml
+              - key: task-logs-k8s-events.yaml 
+                path: task-logs-k8s-events.yaml
   strategy:
     type: RollingUpdate
     rollingUpdate:

diff --git a/deploy/overlays/stable/daemonset/kustomization.yaml b/deploy/overlays/stable/daemonset/kustomization.yaml
@@ -5,7 +5,15 @@ kind: Kustomization
 bases:
   - ../../../base/daemonset
 
+namespace: kube-system
+
+configMapGenerator:
+  - name: swisnap-host-configmap
+    behavior: merge
+    literals:
+      - SWISNAP_ENABLE_DOCKER_LOGS=false
+      - SWISNAP_DOCKER_LOGS_CONTAINER_NAMES=""
 
 images:
   - name: solarwinds/solarwinds-snap-agent-docker
-    newTag: 4.2.0-4.0.0.863
+    newTag: 4.3.0-4.1.0.1024
diff --git a/deploy/overlays/stable/deployment/kustomization.yaml b/deploy/overlays/stable/deployment/kustomization.yaml
@@ -5,6 +5,14 @@ kind: Kustomization
 bases:
   - ../../../base/deployment
 
+namespace: kube-system
+
+configMapGenerator:
+  - name: swisnap-k8s-configmap
+    behavior: merge
+    literals:
+      - SWISNAP_ENABLE_KUBERNETES_LOGS=false
+
 images:
   - name: solarwinds/solarwinds-snap-agent-docker
-    newTag: 4.2.0-4.0.0.863
+    newTag: 4.3.0-4.1.0.1024
diff --git a/deploy/overlays/stable/events-collector/kustomization.yaml b/deploy/overlays/stable/events-collector/kustomization.yaml
@@ -7,4 +7,4 @@ bases:
 
 images:
   - name: solarwinds/solarwinds-snap-agent-docker
-    newTag: 4.2.0-4.0.0.863
+    newTag: 4.3.0-4.1.0.1024
diff --git a/examples/event-collector-configs/kubernetes.yaml b/examples/event-collector-configs/kubernetes.yaml
Original file line number	Diff line number	Diff line change
		@@ -1,2 +1,2 @@
		* @melord @d-maslyk @tomekkolo @swi-snap-agent
		* @swisnap-krk