From 31856472076cffeda8c3276ba8d73348e77076f0 Mon Sep 17 00:00:00 2001 From: Ben Harosh Date: Thu, 24 Oct 2024 16:06:28 -0700 Subject: [PATCH] PTRENG-6489 - FluentD sidecar version bumped to 4.9 Add support for a configurable http request timeout, with request_timeout FluentD param Add support for metrics and logs outbound json payload compression, with gzip_compression FluentD param Add support for a configurable verify_ssl FluentD param as part of fluent-plugin-jfrog-metrics --- CHANGELOG.md | 7 +++++++ README.md | 20 ++++++++++++++++++-- app/jfrog-logs/README.md | 8 ++++---- docker-build/docker.env | 7 ++----- fluent.conf.rt | 9 ++++++--- fluent.conf.xray | 6 +++++- helm/artifactory-ha-values.yaml | 8 +++++++- helm/artifactory-values.yaml | 8 +++++++- helm/jfrog_helm.env | 1 + helm/xray-values.yaml | 8 +++++++- jfrog.env | 2 ++ 11 files changed, 66 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f389bac..c43dffb 100755 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All changes to the log analytics integration will be documented in this file. +## [1.0.7] - October 25, 2024 + +* Add support for metrics and logs outbound payload compression, with `gzip_compression` FluentD param as part of `fluent-plugin-jfrog-sent-metrics` and `fluent-plugin-splunk-hec` plugins +* Add support for a configurable http request timeout, with `request_timeout` FluentD param as part of `fluent-plugin-jfrog-metrics` and `fluent-plugin-jfrog-sent-metrics` plugins +* Add support for a configurable `verify_ssl` FluentD param as part of `fluent-plugin-jfrog-metrics` +* FluentD sidecar version bumped to 4.9, to incorporate the above changes + ## [1.0.6] - August 8, 2024 * Fix metrics configuration due to deprication of `artifactory.openMetrics` as part of Artifactory 7.87.x charts and renaming it to `artifactory.metrics` diff --git a/README.md b/README.md index e7a260a..98514f8 100644 --- a/README.md +++ b/README.md @@ -176,7 +176,9 @@ We rely heavily on environment variables so that the correct log files are strea * **SPLUNK_HEC_PORT**: Splunk HEC configured port * **SPLUNK_HEC_TOKEN**: Splunk HEC Token for sending logs to Splunk * **SPLUNK_METRICS_HEC_TOKEN**: Splunk HEC Token for sending metrics to Splunk -* **SPLUNK_INSECURE_SSL**: false for test environments only or if http scheme. +* **SPLUNK_INSECURE_SSL**: false for test environments only or if http scheme +* **SPLUNK_VERIFY_SSL**: false for disabling ssl validation (useful for proxy forwarding or bypassing ssl certificate validation) +* **SPLUNK_COMPRESS_DATA**: true for compressing logs and metrics json payloads on outbound to Splunk * **JPD_URL**: Artifactory JPD URL of the format `http://` * **JPD_ADMIN_USERNAME**: Artifactory username for authentication * **JFROG_ADMIN_TOKEN**: Artifactory [Access Token](https://jfrog.com/help/r/how-to-generate-an-access-token-video/artifactory-creating-access-tokens-in-artifactory) for authentication @@ -226,7 +228,9 @@ For Splunk as the observability platform, execute these commands to setup the do **SPLUNK_HEC_TOKEN**: Splunk HEC Token for sending logs to Splunk **SPLUNK_METRICS_HEC_TOKEN**: Splunk HEC Token for sending metrics to Splunk **SPLUNK_INSECURE_SSL**: false for test environments only or if http scheme - **JPD_URL: Artifactory JPD URL of the format `http://` + **SPLUNK_VERIFY_SSL**: false for disabling ssl validation (useful for proxy forwarding or bypassing ssl certificate validation) + **SPLUNK_COMPRESS_DATA**: true for compressing logs and metrics payloads that are sent to Splunk + **JPD_URL**: Artifactory JPD URL of the format `http://` **JPD_ADMIN_USERNAME**: Artifactory username for authentication **JFROG_ADMIN_TOKEN**: Artifactory [Access Token](https://jfrog.com/help/r/how-to-generate-an-access-token-video/artifactory-creating-access-tokens-in-artifactory) for authentication **COMMON_JPD**: This flag should be set as true only for non-kubernetes installations or installations where JPD base URL is same to access both Artifactory and Xray (ex: https://sample_base_url/artifactory or https://sample_base_url/xray) @@ -316,6 +320,8 @@ export MASTER_KEY=$(openssl rand -hex 32) * **SPLUNK_HEC_TOKEN**: Splunk HEC Token for sending logs to Splunk * **SPLUNK_METRICS_HEC_TOKEN**: Splunk HEC Token for sending metrics to Splunk * **SPLUNK_INSECURE_SSL**: false for test environments only or if http scheme + * **SPLUNK_VERIFY_SSL**: false for disabling ssl validation (useful for proxy forwarding or bypassing ssl certificate validation) + * **SPLUNK_COMPRESS_DATA**: true for compressing logs and metrics json payloads on outbound to Splunk * **JPD_URL**: Artifactory JPD URL of the format `http://` * **JPD_ADMIN_USERNAME**: Artifactory username for authentication * **COMMON_JPD**: This flag should be set as true only for non-kubernetes installations or installations where JPD base URL is same to access both Artifactory and Xray (ex: https://sample_base_url/artifactory or https://sample_base_url/xray) @@ -340,8 +346,10 @@ export MASTER_KEY=$(openssl rand -hex 32) --set splunk.port=$SPLUNK_HEC_PORT \ --set splunk.logs_token=$SPLUNK_HEC_TOKEN \ --set splunk.metrics_token=$SPLUNK_METRICS_HEC_TOKEN \ + --set splunk.compress_data=$SPLUNK_COMPRESS_DATA \ --set splunk.com_protocol=$SPLUNK_COM_PROTOCOL \ --set splunk.insecure_ssl=$SPLUNK_INSECURE_SSL \ + --set splunk.verify_ssl=$SPLUNK_VERIFY_SSL \ --set jfrog.observability.jpd_url=$JPD_URL \ --set jfrog.observability.username=$JPD_ADMIN_USERNAME \ --set jfrog.observability.common_jpd=$COMMON_JPD \ @@ -386,6 +394,8 @@ export MASTER_KEY=$(openssl rand -hex 32) * **SPLUNK_HEC_TOKEN**: Splunk HEC Token for sending logs to Splunk * **SPLUNK_METRICS_HEC_TOKEN**: Splunk HEC Token for sending metrics to Splunk * **SPLUNK_INSECURE_SSL**: false for test environments only or if http scheme + * **SPLUNK_VERIFY_SSL**: false for disabling ssl validation (useful for proxy forwarding or bypassing ssl certificate validation) + * **SPLUNK_COMPRESS_DATA**: true for compressing logs and metrics json payloads on outbound to Splunk * **JPD_URL**: Artifactory JPD URL of the format `http://` * **JPD_ADMIN_USERNAME**: Artifactory username for authentication * **COMMON_JPD**: This flag should be set as true only for non-kubernetes installations or installations where JPD base URL is same to access both Artifactory and Xray (ex: https://sample_base_url/artifactory or https://sample_base_url/xray) @@ -412,6 +422,8 @@ export MASTER_KEY=$(openssl rand -hex 32) --set splunk.metrics_token=$SPLUNK_METRICS_HEC_TOKEN \ --set splunk.com_protocol=$SPLUNK_COM_PROTOCOL \ --set splunk.insecure_ssl=$SPLUNK_INSECURE_SSL \ + --set splunk.verify_ssl=$SPLUNK_VERIFY_SSL \ + --set splunk.compress_data=$SPLUNK_COMPRESS_DATA \ --set jfrog.observability.jpd_url=$JPD_URL \ --set jfrog.observability.username=$JPD_ADMIN_USERNAME \ --set jfrog.observability.common_jpd=$COMMON_JPD \ @@ -439,6 +451,8 @@ For Xray installation, download the .env file from [here](https://raw.githubuser * **SPLUNK_HEC_TOKEN**: Splunk HEC Token for sending logs to Splunk * **SPLUNK_METRICS_HEC_TOKEN**: Splunk HEC Token for sending metrics to Splunk * **SPLUNK_INSECURE_SSL**: false for test environments only or if http scheme +* **SPLUNK_VERIFY_SSL**: false for disabling ssl validation (useful for proxy forwarding or bypassing ssl certificate validation) +* **SPLUNK_COMPRESS_DATA**: true for compressing logs and metrics json payloads on outbound to Splunk * **JPD_URL**: Artifactory JPD URL of the format `http://` * **JPD_ADMIN_USERNAME**: Artifactory username for authentication * **JFROG_ADMIN_TOKEN**: For security reasons, this value will be pulled from the secret jfrog-admin-token created in the step above @@ -468,6 +482,8 @@ helm upgrade --install xray jfrog/xray --set xray.jfrogUrl=$JPD_URL \ --set splunk.metrics_token=$SPLUNK_METRICS_HEC_TOKEN \ --set splunk.com_protocol=$SPLUNK_COM_PROTOCOL \ --set splunk.insecure_ssl=$SPLUNK_INSECURE_SSL \ + --set splunk.verify_ssl=$SPLUNK_VERIFY_SSL \ + --set splunk.compress_data=$SPLUNK_COMPRESS_DATA \ --set jfrog.observability.jpd_url=$JPD_URL \ --set jfrog.observability.username=$JPD_ADMIN_USERNAME \ --set jfrog.observability.common_jpd=$COMMON_JPD \ diff --git a/app/jfrog-logs/README.md b/app/jfrog-logs/README.md index 324bb18..0f24858 100644 --- a/app/jfrog-logs/README.md +++ b/app/jfrog-logs/README.md @@ -4,7 +4,6 @@ Install the app in your Splunk instance. Then restart your Splunk instance by going to _Server Controls > Restart_. ## Splunk Setup - 1. Create new Events index `jfrog_splunk` at _Settings > Indexes > New Index > Save_ 2. Create new Metrics index `jfrog_splunk_metrics` at _Settings > Indexes > New Index > Metrics > Save_ 3. Create a new HTTP Event Collector data input for logs at _Settings > Data Inputs > HTTP Event Collector > New Token > jfrog_splunk index > Save_ @@ -13,7 +12,7 @@ Install the app in your Splunk instance. Then restart your Splunk instance by go ## Setup Fluentd FluentD is used to send log events to Splunk. This [repo](https://github.com/jfrog/log-analytics-splunk) contains instructions on various installations options for Fluentd as a logging agent. -Download the .env file from [here](https://raw.githubusercontent.com/jfrog/log-analytics-splunk/master/.env_jfrog) and fill in the .env_jfrog file with Splunk and JPD information +Download the .env file from [here](https://raw.githubusercontent.com/jfrog/log-analytics-splunk/master/jfrog.env) and fill in the jfrog.env file with Splunk and JPD information ``` export JF_PRODUCT_DATA_INTERNAL=JF_PRODUCT_DATA_INTERNAL @@ -23,19 +22,21 @@ export SPLUNK_HEC_PORT=8088 export SPLUNK_HEC_TOKEN=SPLUNK_HEC_TOKEN export SPLUNK_METRICS_HEC_TOKEN=SPLUNK_METRICS_HEC_TOKEN export SPLUNK_INSECURE_SSL=false +export SPLUNK_COMPRESS_DATA=true export JPD_URL=http://abc.jfrog.io export JPD_ADMIN_USERNAME=admin export JFROG_ADMIN_TOKEN=JFROG_ADMIN_TOKEN export COMMON_JPD=false ``` -* **JF_PRODUCT_DATA_INTERNAL**: This environment variable must be defined to the correct location. For each JFrog service you will find its active log files in the `$JFROG_HOME//var/log` directory. Helm based installs will already have this defined based upon the underlying docker images. Not a required field for k8s installation +* **JF_PRODUCT_DATA_INTERNAL**: The environment variable JF_PRODUCT_DATA_INTERNAL must be defined to the correct location. For each JFrog service you will find its active log files in the `$JFROG_HOME//var/log` directory * **SPLUNK_COM_PROTOCOL**: HTTP Scheme, http or https * **SPLUNK_HEC_HOST**: Splunk Instance URL * **SPLUNK_HEC_PORT**: Splunk HEC configured port * **SPLUNK_HEC_TOKEN**: Splunk HEC Token for sending logs to Splunk * **SPLUNK_METRICS_HEC_TOKEN**: Splunk HEC Token for sending metrics to Splunk * **SPLUNK_INSECURE_SSL**: false for test environments only or if http scheme +* **SPLUNK_COMPRESS_DATA**: true for compressing logs and metrics json payloads on outbound to Splunk * **JPD_URL**: Artifactory JPD URL of the format `http://` * **JPD_ADMIN_USERNAME**: Artifactory username for authentication * **JFROG_ADMIN_TOKEN**: Artifactory [Access Token](https://jfrog.com/help/r/how-to-generate-an-access-token-video/artifactory-creating-access-tokens-in-artifactory) for authentication @@ -71,5 +72,4 @@ Log data from JFrog platform logs is translated to pre-defined Common Informatio ``` ## Additional Setup - For complete instructions on setup of the integration between JFrog Artifactory & Xray to Splunk visit our Github [repo](https://github.com/jfrog/log-analytics-splunk) diff --git a/docker-build/docker.env b/docker-build/docker.env index 76a1f66..39cf600 100644 --- a/docker-build/docker.env +++ b/docker-build/docker.env @@ -5,12 +5,9 @@ SPLUNK_HEC_PORT=8088 SPLUNK_HEC_TOKEN=change_me SPLUNK_METRICS_HEC_TOKEN=change_me SPLUNK_INSECURE_SSL=change_me +SPLUNK_VERIFY_SSL=change_me +SPLUNK_COMPRESS_DATA=change_me JPD_URL=https://change_me.jfrog.io JPD_ADMIN_USERNAME=admin JFROG_ADMIN_TOKEN=change_me COMMON_JPD=false - - - - - diff --git a/fluent.conf.rt b/fluent.conf.rt index 2e10119..0733c18 100644 --- a/fluent.conf.rt +++ b/fluent.conf.rt @@ -3,12 +3,14 @@ @type jfrog_metrics @id metrics_http_jfrt tag jfrog.metrics.artifactory - interval 5s + execution_interval 60s metric_prefix 'jfrog.artifactory' jpd_url "#{ENV['JPD_URL']}" username "#{ENV['JPD_ADMIN_USERNAME']}" token "#{ENV['JFROG_ADMIN_TOKEN']}" common_jpd "#{ENV['COMMON_JPD']}" + # request_timeout 30s + # verify_ssl "#{ENV['SPLUNK_VERIFY_SSL']}" # SPLUNK ARTIFACTORY METRICS OUTPUT @@ -24,9 +26,10 @@ metric_name_key metric_name metric_value_key value insecure_ssl "#{ENV['SPLUNK_INSECURE_SSL']}" + gzip_compression "#{ENV['SPLUNK_COMPRESS_DATA']}" -# ALL CALLHOME +# CALLHOME @type exec tag callhome @@ -256,7 +259,6 @@ - # FILTER DIRECTIVE ## ALL LOGS @@ -397,6 +399,7 @@ sourcetype_key log_source use_fluentd_time false insecure_ssl "#{ENV['SPLUNK_INSECURE_SSL']}" + gzip_compression "#{ENV['SPLUNK_COMPRESS_DATA']}" # ssl parameter # use_ssl true # ca_file /path/to/ca.pem diff --git a/fluent.conf.xray b/fluent.conf.xray index f043266..87bcdb6 100644 --- a/fluent.conf.xray +++ b/fluent.conf.xray @@ -3,12 +3,14 @@ @type jfrog_metrics @id metrics_http_jfrt tag jfrog.metrics.xray - interval 5s + execution_interval 5s metric_prefix 'jfrog.xray' jpd_url "#{ENV['JPD_URL']}" username "#{ENV['JPD_ADMIN_USERNAME']}" token "#{ENV['JFROG_ADMIN_TOKEN']}" common_jpd "#{ENV['COMMON_JPD']}" + # request_timeout 30s + # verify_ssl "#{ENV['SPLUNK_VERIFY_SSL']}" # SPLUNK XRAY METRICS OUTPUT @@ -24,6 +26,7 @@ metric_name_key metric_name metric_value_key value insecure_ssl "#{ENV['SPLUNK_INSECURE_SSL']}" + gzip_compression "#{ENV['SPLUNK_COMPRESS_DATA']}" # ALL CALLHOME @@ -358,6 +361,7 @@ sourcetype_key log_source use_fluentd_time false insecure_ssl "#{ENV['SPLUNK_INSECURE_SSL']}" + gzip_compression "#{ENV['SPLUNK_COMPRESS_DATA']}" # ssl parameter # use_ssl true # ca_file /path/to/ca.pem diff --git a/helm/artifactory-ha-values.yaml b/helm/artifactory-ha-values.yaml index 9b11ec5..c743642 100644 --- a/helm/artifactory-ha-values.yaml +++ b/helm/artifactory-ha-values.yaml @@ -18,7 +18,7 @@ artifactory: name: volume customSidecarContainers: | - name: "artifactory-fluentd-sidecar" - image: "releases-pts-observability-fluentd.jfrog.io/fluentd:4.5" + image: "releases-pts-observability-fluentd.jfrog.io/fluentd:4.9" imagePullPolicy: "IfNotPresent" volumeMounts: - mountPath: "{{ .Values.artifactory.persistence.mountPath }}" @@ -49,6 +49,10 @@ artifactory: value: {{ .Values.splunk.logs_token }} - name: SPLUNK_INSECURE_SSL value: {{ .Values.splunk.insecure_ssl | quote}} + - name: SPLUNK_VERIFY_SSL + value: {{ .Values.splunk.verify_ssl | quote}} + - name: SPLUNK_COMPRESS_DATA + value: {{ .Values.splunk.compress_data | quote}} - name: FLUENTD_CONF value: ../../../..{{ .Values.artifactory.persistence.mountPath }}/etc/fluentd/fluentd.conf splunk: @@ -58,6 +62,8 @@ splunk: metrics_token: SPLUNK_METRICS_HEC_TOKEN com_protocol: SPLUNK_COM_PROTOCOL insecure_ssl: SPLUNK_INSECURE_SSL + verify_ssl: SPLUNK_VERIFY_SSL + compress_data: SPLUNK_COMPRESS_DATA jfrog: observability: jpd_url: JPD_URL diff --git a/helm/artifactory-values.yaml b/helm/artifactory-values.yaml index 3151d0a..3be98a0 100644 --- a/helm/artifactory-values.yaml +++ b/helm/artifactory-values.yaml @@ -18,7 +18,7 @@ artifactory: name: artifactory-volume customSidecarContainers: | - name: "artifactory-fluentd-sidecar" - image: "releases-pts-observability-fluentd.jfrog.io/fluentd:4.5" + image: "releases-pts-observability-fluentd.jfrog.io/fluentd:4.9" imagePullPolicy: "IfNotPresent" volumeMounts: - mountPath: "{{ .Values.artifactory.persistence.mountPath }}" @@ -49,6 +49,10 @@ artifactory: value: {{ .Values.splunk.logs_token }} - name: SPLUNK_INSECURE_SSL value: {{ .Values.splunk.insecure_ssl | quote}} + - name: SPLUNK_VERIFY_SSL + value: {{ .Values.splunk.verify_ssl | quote}} + - name: SPLUNK_COMPRESS_DATA + value: {{ .Values.splunk.compress_data | quote }} - name: FLUENTD_CONF value: ../../../..{{ .Values.artifactory.persistence.mountPath }}/etc/fluentd/fluentd.conf splunk: @@ -58,6 +62,8 @@ splunk: metrics_token: SPLUNK_METRICS_HEC_TOKEN com_protocol: SPLUNK_COM_PROTOCOL insecure_ssl: SPLUNK_INSECURE_SSL + verify_ssl: SPLUNK_VERIFY_SSL + compress_data: SPLUNK_COMPRESS_DATA jfrog: observability: jpd_url: JPD_URL diff --git a/helm/jfrog_helm.env b/helm/jfrog_helm.env index 45ffd97..a788f91 100644 --- a/helm/jfrog_helm.env +++ b/helm/jfrog_helm.env @@ -4,6 +4,7 @@ export SPLUNK_HEC_PORT=8088 export SPLUNK_HEC_TOKEN=SPLUNK_HEC_TOKEN export SPLUNK_METRICS_HEC_TOKEN=SPLUNK_METRICS_HEC_TOKEN export SPLUNK_INSECURE_SSL=false +export SPLUNK_COMPRESS_DATA=true export JPD_URL=http://abc.jfrog.io export JPD_ADMIN_USERNAME=admin export COMMON_JPD=false \ No newline at end of file diff --git a/helm/xray-values.yaml b/helm/xray-values.yaml index b1bfa58..7c8598d 100644 --- a/helm/xray-values.yaml +++ b/helm/xray-values.yaml @@ -19,7 +19,7 @@ common: name: data-volume customSidecarContainers: | - name: "xray-platform-fluentd-sidecar" - image: "releases-pts-observability-fluentd.jfrog.io/fluentd:4.5" + image: "releases-pts-observability-fluentd.jfrog.io/fluentd:4.9" imagePullPolicy: "IfNotPresent" volumeMounts: - mountPath: "{{ .Values.xray.persistence.mountPath }}" @@ -52,6 +52,10 @@ common: value: {{ .Values.splunk.logs_token }} - name: SPLUNK_INSECURE_SSL value: {{ .Values.splunk.insecure_ssl | quote}} + - name: SPLUNK_VERIFY_SSL + value: {{ .Values.splunk.verify_ssl | quote}} + - name: SPLUNK_COMPRESS_DATA + value: {{ .Values.splunk.compress_data | quote }} splunk: host: SPLUNK_HEC_HOST port: SPLUNK_HEC_PORT @@ -59,6 +63,8 @@ splunk: metrics_token: SPLUNK_METRICS_HEC_TOKEN com_protocol: SPLUNK_COM_PROTOCOL insecure_ssl: SPLUNK_INSECURE_SSL + verify_ssl: SPLUNK_VERIFY_SSL + compress_data: SPLUNK_COMPRESS_DATA jfrog: observability: jpd_url: JPD_URL diff --git a/jfrog.env b/jfrog.env index 1f9d444..73ecb65 100644 --- a/jfrog.env +++ b/jfrog.env @@ -5,6 +5,8 @@ export SPLUNK_HEC_PORT=8088 export SPLUNK_HEC_TOKEN=SPLUNK_HEC_TOKEN export SPLUNK_METRICS_HEC_TOKEN=SPLUNK_METRICS_HEC_TOKEN export SPLUNK_INSECURE_SSL=false +export SPLUNK_VERIFY_SSL=true +export SPLUNK_COMPRESS_DATA=true export JPD_URL=http://abc.jfrog.io export JPD_ADMIN_USERNAME=admin export JFROG_ADMIN_TOKEN=JFROG_ADMIN_TOKEN