From 81042c54e104b6de1262a840e3bf02b91272078e Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 21 Oct 2024 16:28:36 +0200 Subject: [PATCH 1/6] chore(deps): update dependency mkdocs-material to v9.5.42 (#2624) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- poetry.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index b057aa3a3c..def4dd7cec 100644 --- a/poetry.lock +++ b/poetry.lock @@ -680,13 +680,13 @@ mkdocs = ">=1.0.4" [[package]] name = "mkdocs-material" -version = "9.5.41" +version = "9.5.42" description = "Documentation that simply works" optional = false python-versions = ">=3.8" files = [ - {file = "mkdocs_material-9.5.41-py3-none-any.whl", hash = "sha256:990bc138c33342b5b73e7545915ebc0136e501bfbd8e365735144f5120891d83"}, - {file = "mkdocs_material-9.5.41.tar.gz", hash = "sha256:30fa5d459b4b8130848ecd8e1c908878345d9d8268f7ddbc31eebe88d462d97b"}, + {file = "mkdocs_material-9.5.42-py3-none-any.whl", hash = "sha256:452a7c5d21284b373f36b981a2cbebfff59263feebeede1bc28652e9c5bbe316"}, + {file = "mkdocs_material-9.5.42.tar.gz", hash = "sha256:92779b5e9b5934540c574c11647131d217dc540dce72b05feeda088c8eb1b8f2"}, ] [package.dependencies] From e32e5195332ebbf079fa28a4054100c810153468 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 21 Oct 2024 16:29:23 +0200 Subject: [PATCH 2/6] chore(deps): update splunk/addonfactory-test-matrix-action action to v2.1.9 (#2620) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- .github/workflows/ci-lite.yaml | 2 +- .github/workflows/ci-main.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-lite.yaml b/.github/workflows/ci-lite.yaml index 96cc4e7b29..5ebcf2532f 100644 --- a/.github/workflows/ci-lite.yaml +++ b/.github/workflows/ci-lite.yaml @@ -86,7 +86,7 @@ jobs: type=ref,event=tag - name: matrix id: matrix - uses: splunk/addonfactory-test-matrix-action@v2.1.8 + uses: splunk/addonfactory-test-matrix-action@v2.1.9 security-fossa-scan: continue-on-error: true diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index 9d02ba3fc7..0a44116cd7 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -86,7 +86,7 @@ jobs: type=ref,event=tag - name: matrix id: matrix - uses: splunk/addonfactory-test-matrix-action@v2.1.8 + uses: splunk/addonfactory-test-matrix-action@v2.1.9 security-fossa-scan: continue-on-error: true From 3ee8fa06853816a225eee2181ae60bedcab3e24f Mon Sep 17 00:00:00 2001 From: wojtekzyla <108660584+wojtekzyla@users.noreply.github.com> Date: Mon, 21 Oct 2024 16:31:52 +0200 Subject: [PATCH 3/6] fix: fix CISE_Alarm messages parsing (#2609) --- .../post-filter/app-postfilter-cisco_ise.conf | 3 +- .../conflib/syslog/app-syslog-cisco_ise.conf | 29 ++++- .../cisco/app-postfilter-cisco_ise.conf | 3 +- .../addons/cisco/app-syslog-cisco_ise.conf | 29 ++++- tests/test_cisco_ise.py | 101 +++++++++++++++++- 5 files changed, 154 insertions(+), 11 deletions(-) diff --git a/package/etc/conf.d/conflib/post-filter/app-postfilter-cisco_ise.conf b/package/etc/conf.d/conflib/post-filter/app-postfilter-cisco_ise.conf index d47f3ce445..640705d906 100644 --- a/package/etc/conf.d/conflib/post-filter/app-postfilter-cisco_ise.conf +++ b/package/etc/conf.d/conflib/post-filter/app-postfilter-cisco_ise.conf @@ -57,7 +57,8 @@ block parser app-postfilter-cisco_ise() { application app-postfilter-cisco_ise[sc4s-finalfilter] { filter { program('CISE_' type(string) flags(prefix)) - and "${.values.num}" != 1; + and "${.values.num}" != 1 + and not program('CISE_Alarm'); }; parser { app-postfilter-cisco_ise(); }; }; diff --git a/package/etc/conf.d/conflib/syslog/app-syslog-cisco_ise.conf b/package/etc/conf.d/conflib/syslog/app-syslog-cisco_ise.conf index d4f2ff2079..ebdb9136bd 100644 --- a/package/etc/conf.d/conflib/syslog/app-syslog-cisco_ise.conf +++ b/package/etc/conf.d/conflib/syslog/app-syslog-cisco_ise.conf @@ -18,6 +18,27 @@ parser ise_event_time { block parser app-syslog-cisco_ise() { channel { + + if { + parser { + regexp-parser( + template("${MESSAGE}") + patterns("^(?\\d{2}) (?\\d{2}:\\d{2}:\\d{2}) (?[^ ]+) (?[^ ]+) (?.*)") + prefix(".parsed.") + ); + + date-parser-nofilter( + format('%b %d %H:%M:%S') + template("${PROGRAM} ${.parsed.real_day} ${.parsed.real_hour}") + ); + }; + rewrite { + set("${.parsed.real_host}" value("HOST")); + set("${.parsed.real_program}" value("PROGRAM")); + set("${.parsed.rest_of_message}" value("MESSAGE")); + }; + }; + parser { csv-parser( columns(serial, num, seq, message) @@ -44,13 +65,13 @@ block parser app-syslog-cisco_ise() { product('ise') ); }; - - - }; + }; }; + application app-syslog-cisco_ise[sc4s-syslog-pgm] { filter { - program('CISE_' type(string) flags(prefix)); + program('CISE_' type(string) flags(prefix)) + or message('CISE_' type(string) flags(substring)); }; parser { app-syslog-cisco_ise(); }; }; diff --git a/package/lite/etc/addons/cisco/app-postfilter-cisco_ise.conf b/package/lite/etc/addons/cisco/app-postfilter-cisco_ise.conf index d47f3ce445..640705d906 100644 --- a/package/lite/etc/addons/cisco/app-postfilter-cisco_ise.conf +++ b/package/lite/etc/addons/cisco/app-postfilter-cisco_ise.conf @@ -57,7 +57,8 @@ block parser app-postfilter-cisco_ise() { application app-postfilter-cisco_ise[sc4s-finalfilter] { filter { program('CISE_' type(string) flags(prefix)) - and "${.values.num}" != 1; + and "${.values.num}" != 1 + and not program('CISE_Alarm'); }; parser { app-postfilter-cisco_ise(); }; }; diff --git a/package/lite/etc/addons/cisco/app-syslog-cisco_ise.conf b/package/lite/etc/addons/cisco/app-syslog-cisco_ise.conf index d4f2ff2079..ebdb9136bd 100644 --- a/package/lite/etc/addons/cisco/app-syslog-cisco_ise.conf +++ b/package/lite/etc/addons/cisco/app-syslog-cisco_ise.conf @@ -18,6 +18,27 @@ parser ise_event_time { block parser app-syslog-cisco_ise() { channel { + + if { + parser { + regexp-parser( + template("${MESSAGE}") + patterns("^(?\\d{2}) (?\\d{2}:\\d{2}:\\d{2}) (?[^ ]+) (?[^ ]+) (?.*)") + prefix(".parsed.") + ); + + date-parser-nofilter( + format('%b %d %H:%M:%S') + template("${PROGRAM} ${.parsed.real_day} ${.parsed.real_hour}") + ); + }; + rewrite { + set("${.parsed.real_host}" value("HOST")); + set("${.parsed.real_program}" value("PROGRAM")); + set("${.parsed.rest_of_message}" value("MESSAGE")); + }; + }; + parser { csv-parser( columns(serial, num, seq, message) @@ -44,13 +65,13 @@ block parser app-syslog-cisco_ise() { product('ise') ); }; - - - }; + }; }; + application app-syslog-cisco_ise[sc4s-syslog-pgm] { filter { - program('CISE_' type(string) flags(prefix)); + program('CISE_' type(string) flags(prefix)) + or message('CISE_' type(string) flags(substring)); }; parser { app-syslog-cisco_ise(); }; }; diff --git a/tests/test_cisco_ise.py b/tests/test_cisco_ise.py index 8958efb963..2b4ee45819 100644 --- a/tests/test_cisco_ise.py +++ b/tests/test_cisco_ise.py @@ -207,7 +207,7 @@ def test_cisco_ise_cise_alarm_single( sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string( - 'search index=netauth host="{{ host }}" sourcetype="cisco:ise:syslog" "Server=10.0.0.5"' + 'search index=netauth host="{{ host }}" sourcetype="cisco:ise:syslog" "CISE_Alarm WARN: RADIUS Authentication Request dropped : Server=10.0.0.5;"' ) search = st.render(epoch=epoch, host=host) @@ -218,3 +218,102 @@ def test_cisco_ise_cise_alarm_single( record_property("message", message) assert result_count == 1 + +@pytest.mark.addons("cisco") +def test_cisco_ise_double_timestamp_and_hostname( + record_property, setup_splunk, setup_sc4s +): + host = f"{shortuuid.ShortUUID().random(length=5).lower()}-{shortuuid.ShortUUID().random(length=5).lower()}" + + dt = datetime.datetime.now() + _, bsd, time, date, tzoffset, _, epoch = time_operations(dt) + + # Tune time functions for Cisco ISE + time = time[:-3] + tzoffset = tzoffset[0:3] + ":" + tzoffset[3:] + epoch = epoch[:-3] + + mt = env.from_string( + "{{ mark }}{{ bsd }} wrong_host {{ bsd }} {{ host }} CISE_System_Statistics 0000001313 1 4 2020-01-01 10:00:00.000000 +00:00 0000015291 70501 NOTICE System-Stats: ISE Counters, ConfigVersionId=1, OperationCounters=Counter=1_LocalEndPointReads:1]\n" + ) + message = mt.render( + mark="<165>", bsd=bsd, host=host, date=date, time=time, tzoffset=tzoffset + ) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + st = env.from_string( + 'search index=netauth host="{{ host }}" sourcetype="cisco:ise:syslog" "CISE_System_Statistics: 0000001313 1 4 2020-01-01 10:00:00.000000"' + ) + search = st.render(epoch=epoch, host=host) + + result_count, _ = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", result_count) + record_property("message", message) + + assert result_count == 1 + +@pytest.mark.addons("cisco") +def test_cisco_ise_double_timestamp_and_hostname_sequence_eq_0( + record_property, setup_splunk, setup_sc4s +): + host = f"{shortuuid.ShortUUID().random(length=5).lower()}-{shortuuid.ShortUUID().random(length=5).lower()}" + + dt = datetime.datetime.now() + _, bsd, time, date, tzoffset, _, epoch = time_operations(dt) + + # Tune time functions for Cisco ISE + time = time[:-3] + tzoffset = tzoffset[0:3] + ":" + tzoffset[3:] + epoch = epoch[:-3] + + mt = env.from_string( + "{{ mark }}{{ bsd }} wrong_host {{ bsd }} {{ host }} CISE_System_Statistics 0000001313 4 0 {{ date }} {{ time }} {{ tzoffset }} 0000015291 70501 NOTICE System-Stats: part one,\n" + ) + + message = mt.render( + mark="<165>", bsd=bsd, host=host, date=date, time=time, tzoffset=tzoffset + ) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + # Generate new datetime for subsequent messages; not used in log path parser so actually could be anything + dt = datetime.datetime.now() + datetime.timedelta(seconds=1) + bsd = dt.strftime("%b %d %H:%M:%S") + + mt = env.from_string( + "{{ mark }}{{ bsd }} wrong_host {{ bsd }} {{ host }} CISE_System_Statistics 0000001313 4 1 part two,\n" + ) + message = mt.render( + mark="<111>", bsd=bsd, host=host, date=date, time=time, tzoffset=tzoffset + ) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + mt = env.from_string( + "{{ mark }}{{ bsd }} wrong_host {{ bsd }} {{ host }} CISE_System_Statistics 0000001313 4 2 part three,\n" + ) + message = mt.render( + mark="<111>", bsd=bsd, host=host, date=date, time=time, tzoffset=tzoffset + ) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + mt = env.from_string( + "{{ mark }}{{ bsd }} wrong_host {{ bsd }} {{ host }} CISE_System_Statistics 0000001313 4 3 part four,\n" + ) + message = mt.render( + mark="<111>", bsd=bsd, host=host, date=date, time=time, tzoffset=tzoffset + ) + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + st = env.from_string( + 'search _time={{ epoch }} index=netauth host="{{ host }}" sourcetype="cisco:ise:syslog" one two three four' + ) + search = st.render(epoch=epoch, host=host) + + result_count, _ = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", result_count) + record_property("message", message) + + assert result_count == 1 \ No newline at end of file From e43315d1f75fb911fde5f447d7e038d2560d5820 Mon Sep 17 00:00:00 2001 From: mstopa-splunk <139441697+mstopa-splunk@users.noreply.github.com> Date: Mon, 21 Oct 2024 17:10:12 +0200 Subject: [PATCH 4/6] fix: improve SC4S Dashboard performance (#2592) --- dashboard/dashboard.xml | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/dashboard/dashboard.xml b/dashboard/dashboard.xml index 88382ff68f..d17d725e04 100644 --- a/dashboard/dashboard.xml +++ b/dashboard/dashboard.xml @@ -18,7 +18,7 @@ - index=* sc4s_container=$sc4s_instance$ + | tstats count where index=* sc4s_container=$sc4s_instance$ by index _time $time_range.earliest$ $time_range.latest$ @@ -43,8 +43,8 @@ - rt-15m - rt + -15m + now @@ -220,6 +220,7 @@ + @@ -310,7 +311,7 @@ Total volume of actual syslog traffic delivered by this SC4S instance to Splunk - | stats count + | stats sum(count) @@ -318,6 +319,7 @@ + @@ -336,7 +338,7 @@ Distributions of events by index - | stats count by index + | stats sum(count) as count by index @@ -366,6 +368,7 @@ + @@ -375,7 +378,7 @@ Trends of events by index - | chart sparkline(count) AS "Indexes Trend" count AS Total BY index + | stats sparkline(sum(count)) as "Indexes Trend" sum(count) as Total by index @@ -393,7 +396,7 @@ - index=* sc4s_container=$sc4s_instance$ | eval tags=split(sc4s_tags,"|") | mvexpand tags | search tags=".app.*" | timechart count by tags + | tstats count where index=* sc4s_container=$sc4s_instance$ by sc4s_tags _time | eval tags=split(sc4s_tags,"|") | mvexpand tags | search tags=".app.*" | timechart sum(count) by tags $time_range.earliest$ $time_range.latest$ @@ -439,7 +442,7 @@
- index=* sc4s_container=$sc4s_instance$ | eval tags=split(sc4s_tags,"|") | mvexpand tags | chart count by tags + | tstats count where index=* sc4s_container=$sc4s_instance$ by sc4s_tags _time | eval tags=split(sc4s_tags,"|") | mvexpand tags | stats sum(count) as eventCount by tags | sort - eventCount $time_range.earliest$ $time_range.latest$ @@ -449,4 +452,4 @@
- \ No newline at end of file + From ed8999580875a7e70f8da0e47153fd77206df52e Mon Sep 17 00:00:00 2001 From: cwadhwani-splunk Date: Tue, 29 Oct 2024 16:33:26 +0530 Subject: [PATCH 5/6] docs: Removed reference of Cisco eStreamer for Splunk app from ASA/FTD doc (#2629) * docs: Removed reference of Cisco eStreamer for Splunk app --- docs/sources/vendor/Cisco/cisco_asa.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/sources/vendor/Cisco/cisco_asa.md b/docs/sources/vendor/Cisco/cisco_asa.md index adf03eab8c..8cafb37184 100644 --- a/docs/sources/vendor/Cisco/cisco_asa.md +++ b/docs/sources/vendor/Cisco/cisco_asa.md @@ -11,7 +11,6 @@ | Ref | Link | |----------------|---------------------------------------------------------------------------------------------------------| | Splunk Add-on for ASA (No long supports FWSM and PIX) | | -| Cisco eStreamer for Splunk | | | Product Manual | | ## Sourcetypes From 0ee9941295415d1a08a9fe8093428aa5e0a5487d Mon Sep 17 00:00:00 2001 From: Wojciech Zyla Date: Thu, 7 Nov 2024 11:51:02 +0100 Subject: [PATCH 6/6] fix: add nodeAffinity and externalTrafficPolicy configuration to values.yaml and change the default value of externalTrafficPolicy to Cluster in order to enable loadbalancing between nodes while using metallb --- charts/splunk-connect-for-syslog/templates/service.yaml | 4 ++-- charts/splunk-connect-for-syslog/templates/statefulset.yaml | 4 ++++ charts/splunk-connect-for-syslog/values.yaml | 4 ++++ 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/charts/splunk-connect-for-syslog/templates/service.yaml b/charts/splunk-connect-for-syslog/templates/service.yaml index cfe7b56483..633d1775ee 100644 --- a/charts/splunk-connect-for-syslog/templates/service.yaml +++ b/charts/splunk-connect-for-syslog/templates/service.yaml @@ -14,7 +14,7 @@ metadata: spec: type: {{ .Values.service.type }} - externalTrafficPolicy: Local + externalTrafficPolicy: {{ .Values.externalTrafficPolicy }} {{- if .Values.service.loadBalancerIP }} loadBalancerIP: {{ .Values.service.loadBalancerIP }} {{- end }} @@ -92,7 +92,7 @@ metadata: {{- include "splunk-connect-for-syslog.labels" . | nindent 4 }} spec: type: {{ .Values.service.type }} - externalTrafficPolicy: Local + externalTrafficPolicy: {{ .Values.externalTrafficPolicy }} {{- if .Values.service.loadBalancerIP }} loadBalancerIP: {{ .Values.service.loadBalancerIP }} {{- end }} diff --git a/charts/splunk-connect-for-syslog/templates/statefulset.yaml b/charts/splunk-connect-for-syslog/templates/statefulset.yaml index 02b2433da8..7e8020f1a4 100644 --- a/charts/splunk-connect-for-syslog/templates/statefulset.yaml +++ b/charts/splunk-connect-for-syslog/templates/statefulset.yaml @@ -267,6 +267,10 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} affinity: + {{- with .Values.nodeAffinity }} + nodeAffinity: + {{- toYaml . | nindent 10 }} + {{- end }} podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution: - weight: 100 diff --git a/charts/splunk-connect-for-syslog/values.yaml b/charts/splunk-connect-for-syslog/values.yaml index 33aa7b3f85..814b1e55e8 100644 --- a/charts/splunk-connect-for-syslog/values.yaml +++ b/charts/splunk-connect-for-syslog/values.yaml @@ -206,4 +206,8 @@ autoscaling: nodeSelector: {} +nodeAffinity: {} + tolerations: [] + +externalTrafficPolicy: Cluster