diff --git a/.github/workflows/cd-ghcr.io.yaml b/.github/workflows/cd-ghcr.io.yaml index b2e0ae97e9..3930fff758 100644 --- a/.github/workflows/cd-ghcr.io.yaml +++ b/.github/workflows/cd-ghcr.io.yaml @@ -34,7 +34,7 @@ jobs: strategy: max-parallel: 1 matrix: - container: [container3, container3lite] + container: [container3, container3lite , enterprise] steps: - uses: actions/checkout@v4 with: diff --git a/.github/workflows/ci-enterprise.yaml b/.github/workflows/ci-enterprise.yaml new file mode 100644 index 0000000000..9c7adfa0c3 --- /dev/null +++ b/.github/workflows/ci-enterprise.yaml @@ -0,0 +1,382 @@ +# ######################################################################## +# Copyright 2021 Splunk Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ######################################################################## + +name: ci-enterprise +on: + push: + branches: + - "main" + - "enterprise/*" + pull_request: + branches: + - "main" + - "enterprise/*" + + +permissions: + actions: read + contents: write + deployments: write + packages: write + +jobs: + meta: + runs-on: ubuntu-latest + outputs: + sc4s: ghcr.io/${{ github.repository }}/enterprise:${{ fromJSON(steps.docker_action_meta.outputs.json).labels['org.opencontainers.image.version'] }} + container_tags: ${{ steps.docker_action_meta.outputs.tags }} + container_labels: ${{ steps.docker_action_meta.outputs.labels }} + container_buildtime: ${{ fromJSON(steps.docker_action_meta.outputs.json).labels['org.opencontainers.image.created'] }} + container_version: ${{ fromJSON(steps.docker_action_meta.outputs.json).labels['org.opencontainers.image.version'] }} + container_revision: ${{ fromJSON(steps.docker_action_meta.outputs.json).labels['org.opencontainers.image.revision'] }} + container_base: ${{ fromJSON(steps.docker_action_meta.outputs.json).tags[0] }} + matrix_supportedSplunk: ${{ steps.matrix.outputs.supportedSplunk }} + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + submodules: false + persist-credentials: false + - uses: actions/setup-node@v4 + with: + node-version: "16" + - name: Semantic Release + id: version + uses: cycjimmy/semantic-release-action@v3 + with: + semantic_version: 18 + extra_plugins: | + @semantic-release/exec + @semantic-release/git + semantic-release-helm + @google/semantic-release-replace-plugin@1.2.0 + conventional-changelog-conventionalcommits@6.1.0 + dry_run: true + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Docker meta + id: docker_action_meta + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository }}/enterprise + tags: | + type=sha,format=long + type=sha + type=semver,pattern={{version}},value=${{ steps.version.outputs.new_release_version }} + type=semver,pattern={{major}},value=${{ steps.version.outputs.new_release_version }} + type=semver,pattern={{major}}.{{minor}},value=${{ steps.version.outputs.new_release_version }} + type=ref,event=branch + type=ref,event=pr + type=ref,event=tag + - name: matrix + id: matrix + uses: splunk/addonfactory-test-matrix-action@v2.1.9 + + security-fossa-scan: + continue-on-error: true + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: run fossa anlyze and create report + run: | + curl -H 'Cache-Control: no-cache' https://raw.githubusercontent.com/fossas/fossa-cli/master/install-latest.sh | bash + fossa analyze --debug + fossa report attribution --format text > /tmp/THIRDPARTY + env: + FOSSA_API_KEY: ${{ secrets.FOSSA_API_KEY }} + - name: upload THIRDPARTY file + uses: actions/upload-artifact@v4 + with: + name: THIRDPARTY + path: /tmp/THIRDPARTY + - name: run fossa test + run: | + fossa test --debug + env: + FOSSA_API_KEY: ${{ secrets.FOSSA_API_KEY }} + + build_action: + runs-on: ubuntu-latest + name: Build Action + needs: + - meta + steps: + # To use this repository's private action, + # you must check out the repository + - name: Checkout + uses: actions/checkout@v4 + with: + submodules: false + persist-credentials: false + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Login to GitHub Packages Docker Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Build and push action + id: docker_action_build + uses: docker/build-push-action@v6 + with: + context: . + provenance: false + file: package/Dockerfile.enterprise + #platforms: linux/amd64,linux/arm64 + platforms: linux/amd64,linux/arm64 + push: true + #tags: ${{ needs.meta.outputs.container_tags }} + tags: ${{ needs.meta.outputs.container_base }} + labels: ${{ needs.meta.outputs.container_labels }} + build-args: | + BUILDTIME=${{ needs.meta.outputs.container_buildtime }} + VERSION=${{ needs.meta.outputs.container_version }} + REVISION=${{ needs.meta.outputs.container_revision }} + cache-from: type=registry,ref=${{ needs.meta.outputs.container_base }} + cache-to: type=inline + + scan-docker-image-cves: + runs-on: ubuntu-latest + name: Scan docker image on CVEs + needs: + - meta + - build_action + steps: + # To use .trivyignore file, you must check out the repository + - name: Checkout + uses: actions/checkout@v4 + with: + submodules: false + persist-credentials: false + - name: Run docker vulnerability scanner + uses: aquasecurity/trivy-action@master + with: + image-ref: ${{ needs.meta.outputs.container_base }} + format: 'table' + exit-code: '1' + severity: 'CRITICAL,HIGH,MEDIUM,LOW' + trivyignores: '.trivyignore' + scanners: "vuln" + + test-container: + runs-on: ubuntu-latest + needs: + - meta + - build_action + # runs all of the steps inside the specified container rather than on the VM host. + # Because of this the network configuration changes from host based network to a container network. + container: + image: python:3.9-buster + + services: + splunk: + image: splunk/splunk:${{ fromJson(needs.meta.outputs.matrix_supportedSplunk)[0].version }} + ports: + - 8000:8000 + - 8088:8088 + - 8089:8089 + env: + SPLUNK_HEC_TOKEN: 70b6ae71-76b3-4c38-9597-0c5b37ad9630 + SPLUNK_PASSWORD: Changed@11 + SPLUNK_START_ARGS: --accept-license + SPLUNK_APPS_URL: https://github.com/splunk/splunk-configurations-base-indexes/releases/download/v1.0.0/splunk_configurations_base_indexes-1.0.0.tar.gz + + sc4s: + image: ${{ needs.meta.outputs.container_base }} + ports: + - 514:514 + - 601:601 + - 5614:5514 + - 5601:5601 + - 6000:6000 + - 6002:6002 + - 9000:9000 + env: + SC4S_DEST_SPLUNK_HEC_DEFAULT_URL: https://splunk:8088 + SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN: 70b6ae71-76b3-4c38-9597-0c5b37ad9630 + SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_VERIFY: "no" + SC4S_DEST_SPLUNK_HEC_DEFAULT_HTTP_COMPRESSION: "yes" + SC4S_LISTEN_PFSENSE_FIREWALL_TCP_PORT: 6000 + SC4S_LISTEN_SIMPLE_TEST_ONE_TCP_PORT: 5514 + SC4S_LISTEN_SIMPLE_TEST_ONE_UDP_PORT: 5514 + SC4S_LISTEN_SIMPLE_TEST_TWO_TCP_PORT: 5601 + SC4S_LISTEN_SPECTRACOM_NTP_TCP_PORT: 6002 + SC4S_LISTEN_CISCO_ESA_TCP_PORT: 9000 + SC4S_LISTEN_RARITAN_DSX_TCP_PORT: 9001 + SC4S_LISTEN_CHECKPOINT_SPLUNK_NOISE_CONTROL: "yes" + SC4S_SOURCE_RICOH_SYSLOG_FIXHOST: "yes" + TEST_SC4S_ACTIVATE_EXAMPLES: "yes" + SC4S_DEBUG_CONTAINER: "yes" + SC4S_SOURCE_VMWARE_VSPHERE_GROUPMSG: "yes" + SC4S_NETAPP_ONTAP_NEW_FORMAT: "yes" + SC4S_USE_VPS_CACHE: "yes" + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + submodules: false + persist-credentials: false + - name: Run tests + run: | + pip3 install poetry + poetry install + mkdir -p test-results || true + poetry run pytest -v --tb=long \ + --splunk_type=external \ + --splunk_hec_token=70b6ae71-76b3-4c38-9597-0c5b37ad9630 \ + --splunk_host=splunk \ + --sc4s_host=sc4s \ + --junitxml=test-results/test.xml \ + -n 14 \ + -k 'not lite and not name_cache' + + test-ipv4-name-cache: + runs-on: ubuntu-latest + needs: + - meta + - build_action + # runs all of the steps inside the specified container rather than on the VM host. + # Because of this the network configuration changes from host based network to a container network. + container: + image: python:3.9-buster + services: + splunk: + image: splunk/splunk:${{ fromJson(needs.meta.outputs.matrix_supportedSplunk)[0].version }} + ports: + - 8088:8088 + - 8089:8089 + env: + SPLUNK_HEC_TOKEN: 70b6ae71-76b3-4c38-9597-0c5b37ad9630 + SPLUNK_PASSWORD: Changed@11 + SPLUNK_START_ARGS: --accept-license + SPLUNK_APPS_URL: https://github.com/splunk/splunk-configurations-base-indexes/releases/download/v1.0.0/splunk_configurations_base_indexes-1.0.0.tar.gz + + sc4s: + image: ${{ needs.meta.outputs.container_base }} + ports: + - 514:514 + env: + SC4S_DEST_SPLUNK_HEC_DEFAULT_URL: https://splunk:8088 + SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN: 70b6ae71-76b3-4c38-9597-0c5b37ad9630 + SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_VERIFY: "no" + SC4S_USE_NAME_CACHE: "yes" + SC4S_CLEAR_NAME_CACHE: "yes" + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + submodules: false + persist-credentials: false + - name: Run tests + run: | + pip3 install poetry + poetry install + mkdir -p test-results || true + poetry run pytest -v --tb=long \ + --splunk_type=external \ + --splunk_hec_token=70b6ae71-76b3-4c38-9597-0c5b37ad9630 \ + --splunk_host=splunk \ + --sc4s_host=sc4s \ + --junitxml=test-results/test.xml \ + -n 1 \ + -m 'name_cache' + test-ipv6-name-cache: + runs-on: ubuntu-latest + needs: + - meta + - build_action + # runs all of the steps inside the specified container rather than on the VM host. + # Because of this the network configuration changes from host based network to a container network. + container: + image: python:3.9-buster + services: + splunk: + image: splunk/splunk:${{ fromJson(needs.meta.outputs.matrix_supportedSplunk)[0].version }} + ports: + - 8088:8088 + - 8089:8089 + env: + SPLUNK_HEC_TOKEN: 70b6ae71-76b3-4c38-9597-0c5b37ad9630 + SPLUNK_PASSWORD: Changed@11 + SPLUNK_START_ARGS: --accept-license + SPLUNK_APPS_URL: https://github.com/splunk/splunk-configurations-base-indexes/releases/download/v1.0.0/splunk_configurations_base_indexes-1.0.0.tar.gz + + sc4s: + image: ${{ needs.meta.outputs.container_base }} + ports: + - 514:514 + env: + SC4S_DEST_SPLUNK_HEC_DEFAULT_URL: https://splunk:8088 + SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN: 70b6ae71-76b3-4c38-9597-0c5b37ad9630 + SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_VERIFY: "no" + SC4S_USE_NAME_CACHE: "yes" + SC4S_CLEAR_NAME_CACHE: "yes" + SC4S_IPV6_ENABLE: "yes" + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + submodules: false + persist-credentials: false + - name: Run tests + run: | + pip3 install poetry + poetry install + mkdir -p test-results || true + poetry run pytest -v --tb=long \ + --splunk_type=external \ + --splunk_hec_token=70b6ae71-76b3-4c38-9597-0c5b37ad9630 \ + --splunk_host=splunk \ + --sc4s_host=sc4s \ + --junitxml=test-results/test.xml \ + -n 1 \ + -m 'name_cache' + + release: + name: Release + runs-on: ubuntu-latest + needs: + - meta + - build_action + - test-container + - test-ipv4-name-cache + steps: + - uses: actions/checkout@v4 + with: + submodules: false + persist-credentials: false + - uses: actions/setup-node@v4 + with: + node-version: "16" + - name: Semantic Release + id: version + uses: cycjimmy/semantic-release-action@v3 + with: + semantic_version: 18 + extra_plugins: | + @semantic-release/exec + @semantic-release/git + semantic-release-helm + @google/semantic-release-replace-plugin@1.2.0 + conventional-changelog-conventionalcommits@6.1.0 + env: + GITHUB_TOKEN: ${{ secrets.GH_TOKEN_ADMIN }} diff --git a/docs/enterprise.md b/docs/enterprise.md new file mode 100644 index 0000000000..2da07714ec --- /dev/null +++ b/docs/enterprise.md @@ -0,0 +1,26 @@ +# SC4S Enterprise + +## About SC4S Enterprise +Introducing SC4S Enterprise, a robust and reliable solution crafted for organizations that prioritize stability over frequent updates. This release represents a shift towards predictable, stable, and streamlined software development, focusing on delivering a high-quality experience with fewer, carefully curated updates,this is initial version of the product, next version will be more matured with feature, patches and removal of experimental feature + +## Release Cycle +1. Patch Release: Quarterly +2. Version Update: Annualy + + + +## Security considerations +SC4S Enterprise is built on an Alpine lightweight container which has very little vulnerability. SC4S Enterprise supports secure syslog data transmission protocols such as RELP and TLS over TCP to protect your data in transit. + + +## Implement SC4S Enterprise +To implementat of SC4S Enterprise: + +1. Set up the SC4S Enterprise environment. +2. Install SC4S Enterprise following the [instructions for your chosen environment](../gettingstarted/) except microk8s and Kubernetes with the following changes: + +* In the service file for Podman or Docker replace references of standard container image (`container2` or `container3`) with `enterprise`. + + +3. Configure source systems to send syslog data to SC4S Enterprise. +4. Test the setup to ensure that your syslog data is correctly received, processed, and forwarded to Splunk. diff --git a/package/Dockerfile.enterprise b/package/Dockerfile.enterprise new file mode 100644 index 0000000000..38f5b30871 --- /dev/null +++ b/package/Dockerfile.enterprise @@ -0,0 +1,85 @@ +#Splunk Connect for Syslog (SC4S) by Splunk, Inc. +# +#To the extent possible under law, the person who associated CC0 with +#Splunk Connect for Syslog (SC4S) has waived all copyright and related or neighboring rights +#to Splunk Connect for Syslog (SC4S). +# +#You should have received a copy of the CC0 legalcode along with this +#work. If not, see . +#Splunk Syslog-NG Container Image +# +#To the extent possible under law, the person who associated CC0 with +#Splunk Connect for Syslog (SC4S) has waived all copyright and related or neighboring rights +#to Splunk Syslog-NG Container image. +# +#You should have received a copy of the CC0 legalcode along with this +#work. If not, see . + + +ARG SYSLOGNG_VERSION=4.9.0 +FROM ghcr.io/axoflow/axosyslog:${SYSLOGNG_VERSION} + + +RUN apk add -U --upgrade --no-cache \ + bash \ + build-base \ + curl \ + grep \ + less \ + net-tools \ + netcat-openbsd \ + "openssl>=3.3.2-r1" \ + procps \ + py3-pip \ + python3 \ + python3-dev \ + libffi-dev \ + shadow \ + socat \ + tzdata \ + wget \ + cargo \ + postgresql-libs \ + ca-certificates \ + poetry \ + gdb \ + && groupadd --gid 1024 syslog \ + && useradd -M -g 1024 -u 1024 syslog \ + && usermod -L syslog \ + && touch /var/log/syslog-ng.out /var/log/syslog-ng.err \ + && chmod 755 /var/log/syslog-ng.* + +EXPOSE 514 +EXPOSE 601/tcp +EXPOSE 6514/tcp + +#Note this is commented out because the default syslog-ng config will try to read +#/dev/log a low priv user cannot read this and the container will fail in SC4S +#and other uses the low user may be selected + +HEALTHCHECK --interval=2m --timeout=5s --start-period=30s CMD /usr/sbin/syslog-ng-ctl healthcheck --timeout 5 + +COPY pyproject.toml / +COPY poetry.lock / + +RUN python3 -m venv /var/lib/python-venv \ + && poetry export --format requirements.txt --without-hashes | /var/lib/python-venv/bin/pip3 --no-cache-dir install -r /dev/stdin \ + && /var/lib/python-venv/bin/pip3 install --no-cache-dir --upgrade tornado==6.4.2 \ + && apk del build-base python3-dev libffi-dev + +COPY package/enterprise/etc/syslog-ng.conf /etc/syslog-ng/syslog-ng.conf +COPY package/enterprise/etc/conf.d /etc/syslog-ng/conf.d +COPY package/enterprise/etc/pylib /etc/syslog-ng/pylib +COPY package/enterprise/etc/context_templates /etc/syslog-ng/context_templates +COPY package/etc/test_parsers /etc/syslog-ng/test_parsers +COPY package/enterprise/etc/local_config /etc/syslog-ng/local_config +COPY package/enterprise/etc/local_config /etc/syslog-ng/local_config +COPY package/sbin/entrypoint.sh / +COPY package/sbin/source_ports_validator.py / + +ENV SC4S_CONTAINER_OPTS=--no-caps +ARG VERSION=unknown +RUN echo $VERSION>/etc/syslog-ng/VERSION + +ENTRYPOINT ["/entrypoint.sh"] + diff --git a/package/enterprise/etc/conf.d/conflib/_common/p_compliance_meta.conf b/package/enterprise/etc/conf.d/conflib/_common/p_compliance_meta.conf new file mode 100644 index 0000000000..66de51b873 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/_common/p_compliance_meta.conf @@ -0,0 +1,8 @@ +parser compliance_meta_by_source { + add-contextual-data( + selector(filters("`syslog-ng-sysconfdir`/conf.d/local/context/compliance_meta_by_source.conf")), + database("`syslog-ng-sysconfdir`/conf.d/local/context/compliance_meta_by_source.csv") + ignore-case(yes) + ); +}; + diff --git a/package/enterprise/etc/conf.d/conflib/_common/p_date-parser_nofilter.conf b/package/enterprise/etc/conf.d/conflib/_common/p_date-parser_nofilter.conf new file mode 100644 index 0000000000..81c9848c4b --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/_common/p_date-parser_nofilter.conf @@ -0,0 +1,52 @@ +block parser date-parser-nofilter( + format() + template() + ) { + channel { + if { + parser { + date-parser(format(`format`) template("`template`") + ); + }; + rewrite { + #We originally used this feaure as part of the date-parser above + #However when date-parser is called twice due to a non standard header + #The second call does not correctly guess the tz IF its part of date-parser + #But does work as a re-write. We think this is a syslog-ng bug + guess-time-zone(); + }; + + } else { + rewrite { set("dtparse: Expected: `format`; Actual: `template`" value("fields.sc4s_error")); }; + }; + }; + }; + +block parser date-parser-alts-nofilter( + fmt_one() + fmt_two() + template()) { + channel { + if { + parser { + date-parser(format(`fmt_one`) template("`template`") + flags(guess-timezone)); + }; + filter { + '$(round $(- "$S_UNIXTIME" "$R_UNIXTIME"))' < 43200 + and '$(round $(- "$S_UNIXTIME" "$R_UNIXTIME"))' > -43200 + }; + } elif { + parser { + date-parser(format(`fmt_two`) template("`template`") + flags(guess-timezone)); + }; + filter { + '$(round $(- "$S_UNIXTIME" "$R_UNIXTIME"))' < 43200 + and '$(round $(- "$S_UNIXTIME" "$R_UNIXTIME"))' > -43200 + }; + } else { + rewrite { set("dtparse: Expected: `fmt_one` OR `fmt_two`; Actual: `template`" value("fields.sc4s_error")); }; + }; + }; +}; diff --git a/package/enterprise/etc/conf.d/conflib/_common/syslog_format.conf b/package/enterprise/etc/conf.d/conflib/_common/syslog_format.conf new file mode 100644 index 0000000000..09e972d12d --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/_common/syslog_format.conf @@ -0,0 +1,44 @@ +filter f_is_nevermatch{ + tags("nevermatch"); +}; + +rewrite set_rfc{ + set-tag("wireformat:rfc"); +}; +filter f_is_rfc{ + tags("wireformat:rfc"); +}; + +rewrite set_rfc3164{ + set-tag("wireformat:rfc3164"); +}; +filter f_is_rfc3164{ + tags("wireformat:rfc3164"); +}; + +rewrite set_rfc5424{ + set-tag("wireformat:rfc5424"); +}; +filter f_is_rfc5424{ + tags("wireformat:rfc5424"); +}; + +rewrite set_raw{ + set-tag("wireformat:raw"); +}; +filter f_is_raw{ + tags("wireformat:raw"); +}; + +filter f_host_is_nil_or_ip{ + ( + host('^-') or + host('^((((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9]))|((([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9]))))$') + ) + and not '$HOST' eq "127.0.0.1" + +}; + +filter f_host_is_ip{ + host('^((((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9]))|((([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9]))))$') +}; diff --git a/package/enterprise/etc/conf.d/conflib/_common/t_splunk_hec_event_legacy.conf b/package/enterprise/etc/conf.d/conflib/_common/t_splunk_hec_event_legacy.conf new file mode 100644 index 0000000000..fd4728bbc6 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/_common/t_splunk_hec_event_legacy.conf @@ -0,0 +1,12 @@ +template t_splunk_hec_event_legacy { + template('$(format-json + time=$S_UNIXTIME + host=${HOST} + source=${.splunk.source} + sourcetype=${.splunk.sourcetype} + index=${.splunk.index} + event="$MSG" + fields.* + ) + '); +}; \ No newline at end of file diff --git a/package/enterprise/etc/conf.d/conflib/_common/t_splunk_hec_metric_event.conf b/package/enterprise/etc/conf.d/conflib/_common/t_splunk_hec_metric_event.conf new file mode 100644 index 0000000000..22c6c99403 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/_common/t_splunk_hec_metric_event.conf @@ -0,0 +1,56 @@ +python { +import syslogng +import json + +def splunk_hec_metric_event(log_message): + logger = syslogng.Logger() + logger.debug(log_message) + + m = {} + m['time']=log_message.get_as_str('S_UNIXTIME', '', repr='internal') + m['host']=log_message.get_as_str('HOST', '').lower() + m['source']=log_message.get_as_str('.splunk.source', '') + m['sourcetype']=log_message.get_as_str('.splunk.sourcetype', '') + m['index']=log_message.get_as_str('.splunk.index', '') + m['fields']={} + for field in log_message.keys(): + dfield = field.decode("utf-8") + if dfield.startswith('fields.'): + sfield=dfield[7:] + m['fields'][sfield]=log_message.get_as_str(field, '', repr='internal') + rawmetrics = log_message.get_as_str('MESSAGE', '').rstrip(", ").split(", ") + + metrics = {} + metriclist = [] + event = "" + for rm in rawmetrics: + # print(rm) + metric, locator = rm.split("=", 1) + locator, value = locator.strip("'").split("=") + t, instance = locator.split("(", 1) + instance = ( + instance.rstrip(")") + .replace("(", "_") + .replace("/services/collector/event", "") + .replace(".", "_") + .replace(",", ".") + .replace(":", "_") + .replace("//", "_") + .replace("/", "_") + .replace("-", "_") + .replace("#", "") + ) + + result = f"{t}.{instance}.{metric.replace('.', '_')}={value}" + if metric != "not_matched": + event = f"{event}spl.sc4syslog.{t}.{instance}.{metric.replace('.', '_')}={value}" + "\n" + m['event'] = event + + message = json.dumps(m) + logger.debug(message) + return message +}; + +template t_splunk_hec_metric_event { + template('$(python splunk_hec_metric_event)'); +}; diff --git a/package/enterprise/etc/conf.d/conflib/_common/t_splunk_hec_metric_multi.conf b/package/enterprise/etc/conf.d/conflib/_common/t_splunk_hec_metric_multi.conf new file mode 100644 index 0000000000..36a21f0f19 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/_common/t_splunk_hec_metric_multi.conf @@ -0,0 +1,68 @@ +python { +import syslogng +import json +from io import StringIO + +def splunk_hec_metric_multi(log_message): + logger = syslogng.Logger() + logger.debug(log_message) + output = StringIO() + + m = {} + m['time']=log_message.get_as_str('S_UNIXTIME', '', repr='internal') + m['host']=log_message.get_as_str('HOST', '').lower() + m['source']=log_message.get_as_str('.splunk.source', '') + m['sourcetype']=log_message.get_as_str('.splunk.sourcetype', '') + m['index']=log_message.get_as_str('.splunk.index', '') + m['fields']={} + for field in log_message.keys(): + dfield = field.decode("utf-8") + if dfield.startswith('fields.'): + sfield=dfield[7:] + m['fields'][sfield]=log_message.get_as_str(field, '', repr='internal') + rawmetrics = log_message.get_as_str('MESSAGE', '').rstrip(", ").split(", ") + + metricset = {} + metrics = {} + metriclist = [] + for rm in rawmetrics: + # print(rm) + metric, locator = rm.split("=", 1) + locator, value = locator.strip("'").split("=") + t, instance = locator.split("(", 1) + instance = ( + instance.rstrip(")") + .replace("(", "_") + .replace("/services/collector/event", "") + .replace(".", "_") + .replace(",", ".") + .replace(":", "_") + .replace("//", "_") + .replace("/", "_") + .replace("-", "_") + .replace("#", "") + ) + sk = f"{t}.{instance}" + if sk not in metricset: + metricset[sk]=m + + if metric != "not_matched": + try : + float(value) + metricset[sk]['fields'][f"metric_name:spl.sc4syslog.{t}.{instance}.{metric.replace('.', '_')}"] = value + except : + pass + feed=False + for k,v in metricset.items(): + if feed: + output.write("\n") + feed=True + output.write(json.dumps(v)) + message = output.getvalue() + logger.debug(message) + return message +}; + +template t_splunk_hec_metric_multi { + template('$(python splunk_hec_metric_multi)'); +}; diff --git a/package/enterprise/etc/conf.d/conflib/_common/t_splunk_hec_metric_multi_v2.conf b/package/enterprise/etc/conf.d/conflib/_common/t_splunk_hec_metric_multi_v2.conf new file mode 100644 index 0000000000..92168497bf --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/_common/t_splunk_hec_metric_multi_v2.conf @@ -0,0 +1,129 @@ +python { +import syslogng +import json +from io import StringIO +import re +from urllib.parse import urlparse + +def splunk_hec_metric_multi_v2(log_message): + logger = syslogng.Logger() + logger.debug(log_message) + output = StringIO() + + rawmetrics = log_message.get_as_str('MESSAGE', '').rstrip(", ").split(", ") + + metricset = {} + + for rm in rawmetrics: + # print(rm) + metric, locator = rm.split("=", 1) + locator, value = locator.strip("'").split("=") + t, instance = locator.split("(", 1) + instance = instance.rstrip(")").replace("(", "_") + + dims={} + + if "." in t: + tparts=t.split('.') + t=tparts[0] + dims['module'] = tparts[1] + else: + dims['module'] = t + + iparts=[] + + if t in ["center", "global"]: + mn=f"{t}.{instance}" + else: + if "," in instance: + iparts = instance.split(',') + if '#' in iparts[0]: + dims['name'] = iparts[0].split('#')[0] + dims['workernum'] = iparts[0].split('#')[1] + else: + dims['name'] = iparts[0] + dims['workernum'] = "agg" + else: + dims['name'] = instance + mn=f"{t}" + + if t in ["center", "global"]: + mk=f"{t}" + elif t=="dst": + if len(iparts)==2 and iparts[1].startswith('/'): + dims['path']=iparts[1] + elif len(iparts)==3: + dims['proto']=iparts[1] + dims['uri']=iparts[2] + if dims['module'] in ['syslog', 'network']: + dest = iparts[2].split(':') + dims['dest']=dest[0] + dims['dest_port']=dest[1] + elif dims['module'] in ['http']: + parsed=urlparse(iparts[2]) + dims['dest']=parsed.hostname + dims['dest_port']=parsed.port + dims['scheme']=parsed.scheme + else: + dims['name']=instance + mk = f"{t}.{instance}" + elif t=="src": + if dims['module'] == "internal": + if '#' in instance: + dims['name'] = instance.split('#')[0] + dims['workernum'] = instance.split('#')[1] + elif len(iparts)==2 and dims['module']=="file" and iparts[1].startswith('/'): + dims['path']=iparts[1] + elif len(iparts)==2 and dims['module'] in ["syslog", "network"]: + if iparts[1].startswith('afsocket_sd'): + #afsocket_sd.DEFAULT_6587_601_1 + #afsocket_sd.PFSENSE_tcp_6000_1 + socketparts = iparts[1].split('_') + dims['proto']=socketparts[-3] + dims['port']=socketparts[-2] + else: + dims['dest']=iparts[1] + elif len(iparts)==3: + dims['proto']=iparts[1] + if dims['module'] in ['syslog', 'network']: + dims['src']=iparts[2] + else: + dims['name']=instance + mk = f"{t}.{instance}" + else: + mk = f"{t}.{instance}" + + if mk not in metricset: + metricset[mk]={} + metricset[mk]['time']=log_message.get_as_str('S_UNIXTIME', '', repr='internal') + metricset[mk]['host']=log_message.get_as_str('HOST', '').lower() + metricset[mk]['source']=log_message.get_as_str('.splunk.source', '') + metricset[mk]['sourcetype']=log_message.get_as_str('.splunk.sourcetype', '') + metricset[mk]['index']=log_message.get_as_str('.splunk.index', '') + metricset[mk]['fields']={} + + for field in log_message.keys(): + dfield = field.decode("utf-8") + if dfield.startswith('fields.'): + sfield=dfield[7:] + metricset[mk]['fields'][sfield]=log_message.get_as_str(field, '', repr='internal') + + for k,v in dims.items(): + metricset[mk]['fields'][k]=v + + + metricset[mk]['fields'][f"metric_name:spl.sc4syslog.{mn}.{metric.replace('.', '_')}"] = value + feed=False + for k,v in metricset.items(): + if feed: + output.write("\n") + feed=True + output.write(json.dumps(v)) + message = output.getvalue() + logger.debug(message) + return message +}; + +template t_splunk_hec_metric_multi_v2 { + template('$(python splunk_hec_metric_multi_v2)'); +}; diff --git a/package/enterprise/etc/conf.d/conflib/_common/t_splunk_hec_metric_single.conf b/package/enterprise/etc/conf.d/conflib/_common/t_splunk_hec_metric_single.conf new file mode 100644 index 0000000000..ad5522fd74 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/_common/t_splunk_hec_metric_single.conf @@ -0,0 +1,65 @@ + +python { +import syslogng +import json +from io import StringIO + +def splunk_hec_metric_single(log_message): + logger = syslogng.Logger() + logger.debug(log_message) + output = StringIO() + + mb = {} + mb['time']=log_message.get_as_str('S_UNIXTIME', '', repr='internal') + mb['host']=log_message.get_as_str('HOST', '').lower() + mb['source']=log_message.get_as_str('.splunk.source', '') + mb['sourcetype']=log_message.get_as_str('.splunk.sourcetype', '') + mb['index']=log_message.get_as_str('.splunk.index', '') + mb['fields']={} + for field in log_message.keys(): + dfield = field.decode("utf-8") + if dfield.startswith('fields.'): + sfield=dfield[7:] + mb['fields'][sfield]=log_message.get_as_str(field, '', repr='internal') + + + + rawmetrics = log_message.get_as_str('MESSAGE', '').rstrip(", ").split(", ") + feed = False + for rm in rawmetrics: + + metric, locator = rm.split("=", 1) + if metric != "not_matched": + m = mb.copy() + + locator, value = locator.strip("'").split("=") + t, instance = locator.split("(", 1) + instance = ( + instance.rstrip(")") + .replace("(", "_") + .replace("/services/collector/event", "") + .replace(".", "_") + .replace(",", ".") + .replace(":", "_") + .replace("//", "_") + .replace("/", "_") + .replace("-", "_") + .replace("#", "") + ) + + result = f"{t}.{instance}.{metric.replace('.', '_')}={value}" + m['fields']["metric_name"]=f"spl.sc4syslog.{t}.{instance}.{metric.replace('.', '_')}" + m['fields']["_value"] = value + if feed: + output.write("\n") + feed=True + output.write(json.dumps(m)) + + message = output.getvalue() + logger.debug(message) + return message +}; + +template t_splunk_hec_metric_single { + template('$(python splunk_hec_metric_single)'); +}; diff --git a/package/enterprise/etc/conf.d/conflib/_common/t_templates.conf b/package/enterprise/etc/conf.d/conflib/_common/t_templates.conf new file mode 100644 index 0000000000..0dc09795b3 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/_common/t_templates.conf @@ -0,0 +1,176 @@ +# =============================================================================================== +# Templates for message formatting. Overall JSON formatting for HEC/Kafka is handled in the +# destination blocks. +# =============================================================================================== + +# =============================================================================================== +# Standard format; works for nearly all messages +# =============================================================================================== + +template t_standard { + template("${MONTH_ABBREV} ${DAY} ${HOUR}:${MIN}:${SEC} ${HOST} ${MSGHDR}${MESSAGE}"); +}; + +# =============================================================================================== +# Message Only; useful for bluecoat proxy, palo alto, and others +# =============================================================================================== + +template t_msg_only { + template("${MSGONLY}"); + }; + +# =============================================================================================== +# Message Only trim leading or trailing white space +# =============================================================================================== + +template t_msg_trim { + template("$(strip $MSGONLY )"); + }; +# =============================================================================================== +# Everything; useful for bluecoat proxy and others +# =============================================================================================== + +template t_everything { + template("${ISODATE} ${HOST} ${LEGACY_MSGHDR}${MESSAGE}"); + }; + +# =============================================================================================== +# Message Header with Message; for Palo Alto +# =============================================================================================== + +template t_hdr_msg { + template("${MSGHDR}${MESSAGE}"); + }; + +template t_legacy_hdr_msg { + template("${LEGACY_MSGHDR}${MESSAGE}"); + }; + +template t_program_msg { + template("${PROGRAM}[${PID}]: ${MESSAGE}"); + }; + +template t_program_nopid_msg { + template("${PROGRAM}: ${MESSAGE}"); + }; + +# =============================================================================================== +# Message Header, Structured Data (from RFC5424 parse) and Message; for Juniper +# =============================================================================================== + +template t_hdr_sdata_msg { + template("${MSGHDR}${MSGID} ${SDATA} ${MESSAGE}"); + }; +#Send RFC5424 message +template t_5424_hdr_sdata_msg { + template('${ISODATE} $(if ("${HOST}" ne "") "${HOST}" "-") ${PROGRAM:--} ${PID:--} ${MSGID:--} ${SDATA:--}${MESSAGE}'); + }; +#drop sdata send message +template t_5424_hdr_msg { + template('${ISODATE} $(if ("${HOST}" ne "") "${HOST}" "-") ${PROGRAM:--} ${PID:--} ${MSGID:--} ${MESSAGE}'); + }; +#Drop message send sdata +template t_5424_hdr_sdata { + template('${ISODATE} $(if ("${HOST}" ne "") "${HOST}" "-") ${PROGRAM:--} ${PID:--} ${MSGID:--} ${SDATA}'); + }; + +#Send - for host and date to conserve bytes useful for destinations such as splunk where the fields are repetitive +template t_5424_hdr_sdata_compact { + template('- - ${PROGRAM:--} ${PID:--} ${MSGID:--} ${SDATA:--}${MESSAGE}'); + }; + +# =============================================================================================== +# JSON_3164; for JSON pretty-printing (for debugging) +# =============================================================================================== + +template t_JSON_3164 { + template('$(format-json --scope rfc3164 + --pair PRI="<$PRI>" + --key LEGACY_MSGHDR + --key RAWMSG + --exclude FACILITY + --exclude PRIORITY + )'); + }; + +# =============================================================================================== +# JSON_5424; for JSON pretty-printing (standard output for RFC5424 messages in production) +# Fields passed as indexed fields or as Splunk metadata are excluded +# =============================================================================================== + +template t_JSON_5424 { + template('$(format-json --scope rfc5424 + --key RAWMSG + --exclude DATE + --exclude FACILITY + --exclude PRIORITY + --exclude HOST + )'); + }; + +# =============================================================================================== +# JSON_5424_ALL; for JSON pretty-printing (for printing all fields in RFC5424 message and ISO date) +# Useful for log path development +# =============================================================================================== + +template t_JSON_5424_ALL { + template('$(format-json --scope rfc5424 + --exclude DATE + --key ISODATE + --key RAWMSG + )'); + }; + +# =============================================================================================== +# JSON_5424_SDATA; for JSON pretty-printing (for RFC5424 messages with duplicate data in MESSAGE) +# =============================================================================================== + +template t_JSON_5424_SDATA { + template('$(format-json --scope rfc5424 + --exclude DATE + --exclude HOST + --exclude FACILITY + --exclude PRIORITY + --exclude MESSAGE + --exclude RAWMSG + )'); + }; + +template t_splunk_hec { + template('$(format-json + time=$(if ("${.netsource.sc4s_use_recv_time}" eq "yes") ${R_UNIXTIME} ${S_UNIXTIME}) + host=$(lowercase ${HOST}) + source=${.splunk.source:-SC4S} + sourcetype=${.splunk.sourcetype:-sc4s:fallback} + index=${.splunk.index:-main} + event="$(template ${.splunk.sc4s_template:-t_hdr_msg})" + fields.sc4s_tags="$(implode "|" $(explode "," "$TAGS"))" + fields.* + )'); +}; + + + +template t_fallback_kv { + template("$(if (\"${RAWMSG}\" ne \"\") \"RAWMSG=${RAWMSG}\n\" \"\")$(if (\"${PRI}\" ne \"\") \"PRI=${PRI}\n\" \"\")$(if (\"${LEGACY_MSGHDR}\" ne \"\") \"LEGACY_MSGHDR=${LEGACY_MSGHDR}\n\" \"\")$(if (\"${PROGRAM}\" ne \"\") \"PROGRAM=${PROGRAM}\n\" \"\")$(if (\"${PID}\" ne \"\") \"PID=${PID}\n\" \"\")$(if (\"${MESSAGE}\" ne \"\") \"MESSAGE=${MESSAGE}\n\" \"\")"); +}; + + +template-function "format-ewmm" "<$PRI>1 $ISODATE $LOGHOST @syslog-ng - - ${SDATA:--} $(format-json --leave-initial-dot --scope all-nv-pairs --exclude 0* --exclude 1* --exclude 2* --exclude 3* --exclude 4* --exclude 5* --exclude 6* --exclude 7* --exclude 8* --exclude 9* --exclude SOURCE --exclude .SDATA.* ._TAGS=${TAGS})\n"; + + +template t_json_values { + template('$(format-json --key .metadata.* --key .SDATA.* --key .values.* --shift-levels 2)'); +}; + +template t_kv_values { + template('$(format-welf --key .metadata.* --key .SDATA.* --key .values.* --shift-levels 2)'); +}; + +template t_json_values_msg { + template('$(format-json --key .metadata.* --key .SDATA.* --key .values.* --shift-levels 2 message=$MSG)'); +}; + +template t_kv_values_msg { + template('$(format-welf --key .metadata.* --key .SDATA.* --key .values.* --shift-levels 2 message=$MSG)'); +}; \ No newline at end of file diff --git a/package/enterprise/etc/conf.d/conflib/_splunk/fix_dns.conf b/package/enterprise/etc/conf.d/conflib/_splunk/fix_dns.conf new file mode 100644 index 0000000000..2cff5875ca --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/_splunk/fix_dns.conf @@ -0,0 +1,30 @@ +parser p_fix_hostname_resolver { + python( + class("parser_fix_dns.FixHostnameResolver") + ); +}; + +parser p_fix_fqdn_resolver { + python( + class("parser_fix_dns.FixFQDNResolver") + ); +}; + +parser p_host_cache { + python( + class("parser_source_cache.psc_parse") + ); +}; + +parser p_vpst_cache { + python( + class("parser_vps_cache.vpsc_parse") + ); +}; + +parser p_add_context_host { + add-contextual-data( + selector("${SOURCEIP}"), + database("conf.d/local/context/host.csv"), + ); +}; diff --git a/package/enterprise/etc/conf.d/conflib/_splunk/netsourcefields.conf b/package/enterprise/etc/conf.d/conflib/_splunk/netsourcefields.conf new file mode 100644 index 0000000000..041b8fa89f --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/_splunk/netsourcefields.conf @@ -0,0 +1,15 @@ +block parser p_set_netsource_fields( + vendor() + product() +) { + channel { + rewrite { + set("`vendor`", value(".netsource.sc4s_vendor") condition('`vendor`' ne "")); + set("`product`", value(".netsource.sc4s_product") condition('`product`' ne "")); + set("`vendor`_`product`", value(".netsource.sc4s_vendor_product")); + set-tag("vps"); + set-tag("ns_vendor:`vendor`"); + set-tag("ns_product:`product`"); + }; + }; +}; diff --git a/package/enterprise/etc/conf.d/conflib/_splunk/splunkfields.conf b/package/enterprise/etc/conf.d/conflib/_splunk/splunkfields.conf new file mode 100644 index 0000000000..cd100f9c41 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/_splunk/splunkfields.conf @@ -0,0 +1,135 @@ +#Used to set indexed fields we will always use to global defaults +rewrite r_set_splunk_default { + channel { + rewrite + { + set("sc4s", value(".splunk.source") condition(not match('^.' value('.splunk.source')))); + set("sc4s:unknown", value(".splunk.sourcetype") condition(not match('^.' value('.splunk.sourcetype')))); + set("main", value(".splunk.index") condition(not match('^.' value('.splunk.index')))); + + set("`SC4S_CONTAINER_HOST`", value("fields.sc4s_loghost") condition('`SC4S_CONTAINER_HOST`' ne "")); + set($DESTPORT, value("fields.sc4s_destport") condition(match('destport' template('`SC4S_DEST_SPLUNK_INDEXED_FIELDS`') type(string) flags(substring)) )); + set($LOGHOST, value("fields.sc4s_container") condition(match('container' template('`SC4S_DEST_SPLUNK_INDEXED_FIELDS`') type(string) flags(substring)) )); + set($SOURCEIP, value("fields.sc4s_fromhostip") condition(match('fromhostip' template('`SC4S_DEST_SPLUNK_INDEXED_FIELDS`') type(string) flags(substring)) )); + set($R_UNIXTIME, value("fields.sc4s_recv_time") condition(match('r_unixtime' template('`SC4S_DEST_SPLUNK_INDEXED_FIELDS`') type(string) flags(substring)) )); + }; + + if ("$PROTO" == 6) { + rewrite { set("TCP", value("fields.sc4s_proto") condition(match('proto' template('`SC4S_DEST_SPLUNK_INDEXED_FIELDS`') type(string) flags(substring)) )); }; + } elif ("$PROTO" == 17) { + rewrite { set("UDP", value("fields.sc4s_proto") condition(match('proto' template('`SC4S_DEST_SPLUNK_INDEXED_FIELDS`') type(string) flags(substring)) )); }; + } else { + rewrite { set($PROTO, value("fields.sc4s_proto") condition(match('proto' template('`SC4S_DEST_SPLUNK_INDEXED_FIELDS`') type(string) flags(substring)) )); }; + }; + }; +}; +#used by each log-path to set source and sourcetype which may be +#overridden by user defined values +block rewrite r_set_splunk_dest_default( + #While the following is not used it remains to prevent breaking changes in content + index("main") + source("${.splunk.source}") + sourcetype("${.splunk.sourcetype}") + template('$(if (tags("wireformat:rfc5424")) "t_5424_hdr_sdata_compact" "t_hdr_msg")') + tags("default") + vendor("${fields.sc4s_vendor}") + product("${fields.sc4s_product}") + class("${fields.sc4s_class}") + ) { + set("`index`", value(".splunk.index")); + set("`source`", value(".splunk.source")); + set("`sourcetype`", value(".splunk.sourcetype")); + set("`template`", value(".splunk.sc4s_template")); + set("`vendor`", value("fields.sc4s_vendor") condition('`vendor`' ne "")); + set("`product`", value("fields.sc4s_product") condition('`product`' ne "")); + set("`class`", value("fields.sc4s_class") condition('`class`' ne "")); +}; + + +block rewrite r_set_splunk_dest_update( + #While the following is not used it remains to prevent breaking changes in content + index("${.splunk.index}") + source("${.splunk.source}") + sourcetype("${.splunk.sourcetype}") + template("${.splunk.sc4s_template}") + template_hec("${.splunk.sc4s_hec_template}") + vendor("${fields.sc4s_vendor}") + product("${fields.sc4s_product}") + class("${fields.sc4s_class}") + tag("") + condition("1" eq "1") + ) { + channel { + if (`condition`){ + rewrite { + set("`index`", value(".splunk.index")); + set("`source`", value(".splunk.source")); + set("`sourcetype`", value(".splunk.sourcetype")); + set("`template`", value(".splunk.sc4s_template")); + set("`template_hec`", value(".splunk.sc4s_hec_template")); + set("`vendor`", value("fields.sc4s_vendor") condition('`vendor`' ne "")); + set("`product`", value("fields.sc4s_product") condition('`product`' ne "")); + set("`class`", value("fields.sc4s_class") condition('`class`' ne "")); + }; + }; + if { + filter { + "`vendor`" eq "null" + }; + rewrite(r_set_dest_splunk_null_queue); + }; + flags(final); + }; +}; + +block rewrite r_set_splunk_dest_update_v2( + #While the following is not used it remains to prevent breaking changes in content + index("${.splunk.index}") + source("${.splunk.source}") + sourcetype("${.splunk.sourcetype}") + template("${.splunk.sc4s_template}") + template_hec("${.splunk.sc4s_hec_template}") + class("${fields.sc4s_class}") + tags("") + condition("1" eq "1") + ) { + channel { + if (`condition`){ + rewrite { + set("`index`", value(".splunk.index")); + set("`source`", value(".splunk.source")); + set("`sourcetype`", value(".splunk.sourcetype")); + set("`template`", value(".splunk.sc4s_template")); + set("`template_hec`", value(".splunk.sc4s_hec_template")); + set("`class`", value("fields.sc4s_class") condition('`class`' ne "")); + }; + }; + flags(final); + }; +}; + +rewrite r_set_dest_splunk_null_queue{ + set-tag("dest_splunk:null_queue"); +}; +# +filter f_is_dest_splunk_null_queue{ + tags("dest_splunk:null_queue"); +}; +# + +rewrite r_set_source_identified{ + set-tag("source_identified"); +}; +# +filter f_is_source_identified{ + tags("source_identified"); +}; + +filter f_is_agg{ + tags("agg"); +}; + +filter f_is_proxy_ip{ + "$HOST" eq "$SOURCEIP" + and "$PROXIED_SRCIP" ne "" +}; \ No newline at end of file diff --git a/package/enterprise/etc/conf.d/conflib/almost-syslog/app-almost-syslog-airwatch.conf b/package/enterprise/etc/conf.d/conflib/almost-syslog/app-almost-syslog-airwatch.conf new file mode 100644 index 0000000000..dc6da6b22e --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/almost-syslog/app-almost-syslog-airwatch.conf @@ -0,0 +1,47 @@ +rewrite set_rfc3164_airwatch_raw{ + set-tag("wireformat:rfc3164_airwatch_raw"); +}; +filter f_is_rfc3164_airwatch_netscreen_raw{ + tags("wireformat:rfc3164_airwatch_raw"); +}; + + +block parser app-almost-syslog-airwatch_raw() { + channel { + parser { + regexp-parser( + template("${MESSAGE}") + prefix(".tmp.") + patterns( + '(?\<\d+\>) (?:(?\w+ \d{2} \d\d:\d\d:\d\d))\s*(?[^ ]+)\s*(?AirWatch+)\s*(?.*)' + ) + ); + date-parser( + format('%B %d %H:%M:%S') + template("${.tmp.timestamp}") + ); + syslog-parser( + flags(assume-utf8) + template("${$.tmp.pri} $S_ISODATE ${.tmp.host} ${.tmp.program} ${.tmp.message}") + ); + + }; + rewrite { + r_set_splunk_dest_default( + index('epintel') + sourcetype('vmware:airwatch') + vendor("vmware") + product("airwatch") + template('t_msg_only') + ); + }; + + rewrite(set_rfc); + rewrite(set_rfc3164); + rewrite(set_rfc3164_airwatch_raw); + rewrite(r_set_source_identified); + }; +}; +application app-almost-syslog-airwatch_raw[sc4s-almost-syslog] { + parser { app-almost-syslog-airwatch_raw(); }; +}; \ No newline at end of file diff --git a/package/enterprise/etc/conf.d/conflib/almost-syslog/app-almost-syslog-barracuda_waf.conf b/package/enterprise/etc/conf.d/conflib/almost-syslog/app-almost-syslog-barracuda_waf.conf new file mode 100644 index 0000000000..b08d6a6d9f --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/almost-syslog/app-almost-syslog-barracuda_waf.conf @@ -0,0 +1,22 @@ +block parser app-almost-syslog-barracuda_waf() { + channel { + parser { + regexp-parser( + prefix(".tmp.") + patterns('^(?\<\d+\>)(?: *)(?\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3} (\+|-)\d{4})\s{1,2}(?[^ ]+) (?(SYS|WF|TR|AUDIT|NF)) (?.*)') + ); + date-parser( + format('%Y-%m-%d %H:%M:%S.%f %z') + template("${.tmp.timestamp}") + ); + syslog-parser( + flags(assume-utf8, guess-timezone) + template("${.tmp.pri} $S_ISODATE ${.tmp.host} ${.tmp.log_type} ${.tmp.message}") + ); + }; + }; +}; + +application app-almost-syslog-barracuda_waf[sc4s-almost-syslog] { + parser { app-almost-syslog-barracuda_waf(); }; +}; diff --git a/package/enterprise/etc/conf.d/conflib/almost-syslog/app-almost-syslog-cisco_ms.conf b/package/enterprise/etc/conf.d/conflib/almost-syslog/app-almost-syslog-cisco_ms.conf new file mode 100644 index 0000000000..cac0db3e71 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/almost-syslog/app-almost-syslog-cisco_ms.conf @@ -0,0 +1,28 @@ +rewrite set_rfc3164_cisco_ms{ + set-tag("wireformat:rfc3164_cisco_ms"); +}; +filter f_is_rfc3164_cisco_ms{ + tags("wireformat:rfc3164_cisco_ms"); +}; + +block parser app-almost-syslog-cisco_ms() { + channel { + parser { + regexp-parser( + prefix(".tmp.") + patterns('^(?\<\d+\>) ?(?[A-Z][a-z]{2} *\d{1,2} \d\d:\d\d:\d\d) (?[a-z0-9]+\.[a-z]+) (?[^ ]+) (?.*)') + ); + syslog-parser( + flags(assume-utf8, guess-timezone) + template("${.tmp.pri} ${.tmp.timestamp} ${.tmp.host} ${.tmp.message}") + ); + }; + rewrite(set_rfc); + rewrite(set_rfc3164); + rewrite(set_rfc3164_cisco_ms); + + }; +}; +application app-almost-syslog-cisco_ms[sc4s-almost-syslog] { + parser { app-almost-syslog-cisco_ms(); }; +}; diff --git a/package/enterprise/etc/conf.d/conflib/almost-syslog/app-almost-syslog-cisco_syslog.conf b/package/enterprise/etc/conf.d/conflib/almost-syslog/app-almost-syslog-cisco_syslog.conf new file mode 100644 index 0000000000..72da93bb0d --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/almost-syslog/app-almost-syslog-cisco_syslog.conf @@ -0,0 +1,173 @@ +rewrite set_rfc3164_cisco_syslog{ + set-tag("wireformat:rfc3164_cisco_syslog"); +}; +filter f_is_rfc3164_cisco_syslog{ + tags("wireformat:rfc3164_cisco_syslog"); +}; + +block parser app-almost-syslog-cisco_syslog() { + channel { + if ("`SC4S_DISABLE_DROP_INVALID_CISCO`" eq "yes") { + parser { + syslog-parser( + flags(assume-utf8, no-header) + ); + # split msg and header right before the '%', Cisco messages may + # have a variable number of ': ' terminated values + csv-parser(delimiters(chars('') strings(': %')) + columns('header', 'message') + prefix('.tmp.') + flags(greedy)); + regexp-parser( + prefix(".values.") + patterns('^(%(?.+)-(?[0-7])-(?[^\: ]+))(? ?[: ]) *(?.*)') + template("%${.tmp.message}") + ); + + }; + } elif { + parser { + syslog-parser( + flags(assume-utf8, no-header) + ); + # split msg and header right before the '%', Cisco messages may + # have a variable number of ': ' terminated values + csv-parser(delimiters(chars('') strings(': %')) + columns('header', 'message') + prefix('.tmp.') + flags(greedy, drop-invalid)); + regexp-parser( + prefix(".values.") + patterns('^(%(?.+)-(?[0-7])-(?[^\: ]+))(? ?[: ]) *(?.*)') + template("%${.tmp.message}") + ); + + }; + } else {}; + + filter { + #Arista EOS uses a valid semi program syntax this should avoid catching Arista which is otherwise similar to cisco logs + ( + ( + "${.values.identifier}" eq "ACE" + or "${.values.identifier}" eq "ASA" + or "${.values.identifier}" eq "FWSM" + or "${.values.identifier}" eq "PIX" + + or match(': ?|\*', value('.tmp.header')) + or match('[A-Z]{3,4}:?$', value('.tmp.header')) + or match('\d+:?$', value('.tmp.header')) + ) + and not match('[a-z]\S+$', value('.tmp.header')) + and not match(' \w+\[\d+\]$', value('.tmp.header')); + ) + or ( + match('LICENSE', value('.values.identifier') type(string) flags(prefix)) + or match('MGBL', value('.values.identifier') type(string) flags(prefix)) + or match('SECURITY', value('.values.identifier') type(string) flags(prefix)) + ) + }; + + if { + parser { + regexp-parser( + prefix(".tmp.") + patterns('(?:[ ]|^(?\<\d+\>)|^)(?(?[\*\.])?(?:(?\d+-\d+-\d+T\d+:\d+:\d+(?:\.\d+)?(?:Z|[\+-] *\d+:\d+)|[A-Z][a-z]{2} [ 0123]\d(?: \d{4})? \d\d:\d\d:\d\d(?: [AP]M)?(?:[^ :]+)?)(?: (?[A-Za-z]{1,4}T))?)') + template('${.tmp.header}') + ); + }; + if { + filter { + "${.tmp.timestatus}" eq "." + or "${.tmp.timestatus}" eq "*" + }; + rewrite { + set("0" value("fields.timesync")); + }; + } else { + rewrite { + set("1" value("fields.timesync")); + }; + }; + parser { date-parser-nofilter(format( + '%b %d %H:%M:%S.%f', + '%b %d %H:%M:%S', + '%b %d %I:%M:%S %p.%f', + '%b %d %I:%M:%S %p', + '%b %d %Y %I:%M:%S %p.%f' + '%b %d %Y %H:%M:%S.%f', + '%b %d %Y %H:%M:%S', + '%Y-%m-%dT%T%z', + ) + template("${.tmp.timestamp}")); + }; + } elif { + #This is "uptime" if we match this isn't a time stamp + parser { + regexp-parser( + prefix(".values.") + patterns('(?<=: )(?\d\d:\d\d:\d\d|\d+ \d)') + template('${.tmp.header}') + ); + }; + } else {}; + + if ("`SC4S_ENABLE_CISCO_IOS_RAW_MSG`" eq "yes") { + rewrite { + r_set_splunk_dest_default( + index('netops') + sourcetype('cisco:ios') + vendor('cisco') + product('ios') + ); + }; + } elif { + rewrite { + r_set_splunk_dest_default( + index('netops') + sourcetype('cisco:ios') + vendor('cisco') + product('ios') + ); + set('%${.tmp.message}' value("MESSAGE")); + }; + } else {}; + + rewrite(set_rfc); + rewrite(set_rfc3164); + rewrite(set_rfc3164_cisco_syslog); + rewrite(r_set_source_identified); + + if { + parser { app-parser(topic(cisco_syslog)); }; + }; + + }; +}; +application app-almost-syslog-cisco_syslog[sc4s-almost-syslog] { + filter { + message(': %' type(string) flags(substring)) and not message('junos' type(string) flags(substring)) + ; + }; + + parser { app-almost-syslog-cisco_syslog(); }; + + +}; + diff --git a/package/enterprise/etc/conf.d/conflib/almost-syslog/app-almost-syslog-citrix_netscaler.conf b/package/enterprise/etc/conf.d/conflib/almost-syslog/app-almost-syslog-citrix_netscaler.conf new file mode 100644 index 0000000000..0408a0ff77 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/almost-syslog/app-almost-syslog-citrix_netscaler.conf @@ -0,0 +1,64 @@ +rewrite set_rfc3164_citrix_netscaler{ + set-tag("wireformat:rfc3164_citrix_netscaler"); +}; +filter f_is_rfc3164_citrix_netscaler{ + tags("wireformat:rfc3164_citrix_netscaler"); +}; + +block parser app-almost-syslog-citrix_netscaler() { + channel { + parser { + regexp-parser( + prefix(".tmp.") + patterns('^(?\<\d+>) (?(?\d\d)\/\d\d\/\d\d\d\d:\d\d:\d\d:\d\d ?(?\w+))? (?[^ ]+) (?[A-Z\-0-9]+ : .*)') + ); + }; + parser { + syslog-parser( + flags(assume-utf8, no-header) + ); + }; + + if { + filter { "${.tmp.tspart1}" eq "$R_DAY"}; + parser { + date-parser-nofilter( + format('%d/%m/%Y:%H:%M:%S %z','%d/%m/%Y:%H:%M:%S') + template("${.tmp.timestamp}") + ); + }; + } else { + parser { + date-parser-nofilter( + format('%m/%d/%Y:%H:%M:%S %z','%m/%d/%Y:%H:%M:%S') + template("${.tmp.timestamp}") + ); + }; + + }; + + rewrite { + r_set_splunk_dest_default( + index('netfw') + sourcetype('citrix:netscaler:syslog') + vendor('citrix') + product('netscaler') + ); + }; + rewrite { + r_set_splunk_dest_update_v2( + sourcetype('citrix:netscaler:appfw') condition(message(':(\s+\S+)?\s+APPFW(\s+\S+){3}\s+:')) + ); + }; + rewrite { + set("${.tmp.host}", value("HOST")); + }; + rewrite(set_rfc); + rewrite(set_rfc3164); + rewrite(set_rfc3164_citrix_netscaler); + rewrite(r_set_source_identified); + }; +}; +application app-almost-syslog-citrix_netscaler[sc4s-almost-syslog] { + parser { app-almost-syslog-citrix_netscaler(); }; +}; \ No newline at end of file diff --git a/package/enterprise/etc/conf.d/conflib/almost-syslog/app-almost-syslog-juniper_netscreen_raw.conf b/package/enterprise/etc/conf.d/conflib/almost-syslog/app-almost-syslog-juniper_netscreen_raw.conf new file mode 100644 index 0000000000..4f0527fbea --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/almost-syslog/app-almost-syslog-juniper_netscreen_raw.conf @@ -0,0 +1,46 @@ +rewrite set_rfc3164_juniper_netscreen_raw{ + set-tag("wireformat:rfc3164_juniper_netscreen_raw"); +}; +filter f_is_rfc3164_juniper_netscreen_raw{ + tags("wireformat:rfc3164_juniper_netscreen_raw"); +}; + + +block parser app-almost-syslog-juniper_netscreen_raw() { + channel { + parser { + regexp-parser( + template("${MESSAGE}") + prefix(".tmp.") + patterns( + '(?\<\d+\>) ?(?:(?\w\w\w \d\d \d\d:\d\d:\d\d) )?(?[^: ]+)(?:: | (?[^:]+): )(?NetScreen.*)' + ) + ); + syslog-parser( + flags(assume-utf8, guess-timezone) + template('${.tmp.pri} ${.tmp.timestamp} ${.tmp.host} ${.tmp.message}') + ); + + }; + + rewrite { + r_set_splunk_dest_default( + index('netfw') + sourcetype('netscreen:firewall') + vendor("juniper") + product("netscreen") + class("raw") + ); + set("${.tmp.host}", value("HOST")); + set("${.tmp.model}", value(".metadata.juniper.model") condition("${.tmp.model}" ne "")); + }; + + rewrite(set_rfc); + rewrite(set_rfc3164); + rewrite(set_rfc3164_juniper_netscreen_raw); + rewrite(r_set_source_identified); + }; +}; +application app-almost-syslog-juniper_netscreen_raw[sc4s-almost-syslog] { + parser { app-almost-syslog-juniper_netscreen_raw(); }; +}; \ No newline at end of file diff --git a/package/enterprise/etc/conf.d/conflib/almost-syslog/app-almost-syslog-vmware_vsphere_bad5424.conf b/package/enterprise/etc/conf.d/conflib/almost-syslog/app-almost-syslog-vmware_vsphere_bad5424.conf new file mode 100644 index 0000000000..bed2b6e955 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/almost-syslog/app-almost-syslog-vmware_vsphere_bad5424.conf @@ -0,0 +1,89 @@ +rewrite set_rfc3164_vmware_vsphere_bad5424{ + set-tag("wireformat:rfc3164_vmware_vsphere_bad5424"); +}; +filter f_is_rfc3164_vmware_vsphere_bad5424{ + tags("wireformat:rfc3164_vmware_vsphere_bad5424"); +}; + +block parser app-almost-syslog-vmware_vsphere_bad5424() { + channel { + parser { + regexp-parser( + prefix(".tmp.") + patterns('^(?\<\d+\>)(?\d\d\d\d-\d\d-\d\dT\d\d[^ ]+) (?[^ ]+) [\w]+: \w+ (?[^\[]+)\[(?\d+)\] (?\[[a-zA-Z]+@6876[^\]]+\]) *(?.*)') + ); + }; + if ("`SC4S_DISABLE_DROP_INVALID_VMWARE_VSPHERE`" eq "yes") { + if { + parser { + syslog-parser( + drop-invalid(no) + flags(assume-utf8, syslog-protocol) + template("${.tmp.pri}1 ${.tmp.timestamp} ${.tmp.host} ${.tmp.program} ${.tmp.pid} - ${.tmp.sdata} ${.tmp.message}") + ); + }; + filter { + not program('syslog-ng' type(string)) + }; + + } else { + parser { + syslog-parser( + drop-invalid(no) + flags(assume-utf8, syslog-protocol) + template("${.tmp.pri}1 ${.tmp.timestamp} ${.tmp.host} ${.tmp.program} ${.tmp.pid} - - ${.tmp.message}") + ); + regexp-parser( + prefix(".tmp.") + patterns('\[(?[^@]+)@(?[^ ]+) (?.*)\]$') + template("${.tmp.sdata}") + ); + kv-parser( + prefix(".SDATA.originator@6876.") + pair-separator(" ") + template("${.tmp.kv}") + ); + }; + }; + } elif { + if { + parser { + syslog-parser( + drop-invalid(yes) + flags(assume-utf8, syslog-protocol) + template("${.tmp.pri}1 ${.tmp.timestamp} ${.tmp.host} ${.tmp.program} ${.tmp.pid} - ${.tmp.sdata} ${.tmp.message}") + ); + }; + filter { + not program('syslog-ng' type(string)) + }; + + } else { + parser { + syslog-parser( + drop-invalid(yes) + flags(assume-utf8, syslog-protocol) + template("${.tmp.pri}1 ${.tmp.timestamp} ${.tmp.host} ${.tmp.program} ${.tmp.pid} - - ${.tmp.message}") + ); + regexp-parser( + prefix(".tmp.") + patterns('\[(?[^@]+)@(?[^ ]+) (?.*)\]$') + template("${.tmp.sdata}") + ); + kv-parser( + prefix(".SDATA.originator@6876.") + pair-separator(" ") + template("${.tmp.kv}") + ); + }; + }; + } else {}; + + rewrite(set_rfc); + rewrite(set_rfc5424); + rewrite(set_rfc3164_vmware_vsphere_bad5424); + }; +}; +application app-almost-syslog-vmware_vsphere_bad5424[sc4s-almost-syslog] { + parser { app-almost-syslog-vmware_vsphere_bad5424(); }; +}; \ No newline at end of file diff --git a/package/enterprise/etc/conf.d/conflib/almost-syslog/app-almost-syslog-vmware_vsphere_dataslope.conf b/package/enterprise/etc/conf.d/conflib/almost-syslog/app-almost-syslog-vmware_vsphere_dataslope.conf new file mode 100644 index 0000000000..273163a32b --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/almost-syslog/app-almost-syslog-vmware_vsphere_dataslope.conf @@ -0,0 +1,29 @@ +rewrite set_rfc3164_vmware_vsphere_dataslope{ + set-tag("wireformat:rfc3164_vmware_vsphere_dataslope"); +}; +filter f_is_rfc3164_vmware_vsphere_dataslope{ + tags("wireformat:rfc3164_vmware_vsphere_dataslope"); +}; + +block parser app-almost-syslog-vmware_vsphere_dataslope() { + channel { + parser { + regexp-parser( + prefix(".tmp.") + patterns('^(?\<\d+\>)(?datastoreSlopeUS\[\d+\]: \d+,) *(?[^ ]+) (?[^:]+:) (?a.*)') + ); + syslog-parser( + flags(assume-utf8) + template("${.tmp.pri} $R_DATE ${.tmp.host} ${.tmp.program} ${.tmp.prefix}d${.tmp.message}") + ); + }; + rewrite(set_rfc); + rewrite(set_rfc3164); + rewrite(set_rfc3164_vmware_vsphere_dataslope); + + + }; +}; +application app-almost-syslog-vmware_vsphere_dataslope[sc4s-almost-syslog] { + parser { app-almost-syslog-vmware_vsphere_dataslope(); }; +}; \ No newline at end of file diff --git a/package/enterprise/etc/conf.d/conflib/almost-syslog/app-almost-syslog-vmware_vsphere_guiddate.conf b/package/enterprise/etc/conf.d/conflib/almost-syslog/app-almost-syslog-vmware_vsphere_guiddate.conf new file mode 100644 index 0000000000..c119bdf1fc --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/almost-syslog/app-almost-syslog-vmware_vsphere_guiddate.conf @@ -0,0 +1,28 @@ +rewrite set_rfc3164_vmware_vsphere_guiddate{ + set-tag("wireformat:rfc3164_vmware_vsphere_guiddate"); +}; +filter f_is_rfc3164_vmware_vsphere_guiddate{ + tags("wireformat:rfc3164_vmware_vsphere_guiddate"); +}; + +block parser app-almost-syslog-vmware_vsphere_guiddate() { + channel { + parser { + regexp-parser( + prefix(".tmp.") + patterns('^(?\<\d+\>) ?[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{3}Z (?[^ ]+) (?.*)') + ); + syslog-parser( + flags(assume-utf8) + template("${.tmp.pri} $R_DATE ${.tmp.host} ${.tmp.message}") + ); + }; + rewrite(set_rfc); + rewrite(set_rfc3164); + rewrite(set_rfc3164_vmware_vsphere_guiddate); + + }; +}; +application app-almost-syslog-ciscvmware_vsphere_guiddate[sc4s-almost-syslog] { + parser { app-almost-syslog-vmware_vsphere_guiddate(); }; +}; \ No newline at end of file diff --git a/package/enterprise/etc/conf.d/conflib/almost-syslog/app-almost-syslog-zscaler-nss-alerts.conf b/package/enterprise/etc/conf.d/conflib/almost-syslog/app-almost-syslog-zscaler-nss-alerts.conf new file mode 100644 index 0000000000..6dc52bb558 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/almost-syslog/app-almost-syslog-zscaler-nss-alerts.conf @@ -0,0 +1,33 @@ +rewrite set_rfc3164_zscaler-nss-alerts{ + set-tag("wireformat:rfc3164_zscaler-nss-alerts"); +}; +filter f_is_rfc3164_zscaler-nss-alerts{ + tags("wireformat:rfc3164_zscaler-nss-alerts"); +}; + +block parser app-almost-syslog-zscaler-nss-alerts() { + channel { + # + parser { + regexp-parser( + template("${MESSAGE}") + prefix(".tmp.") + patterns('^(?\<\d+>) ?(?\w\w\w [ \d]?\d \d\d:\d\d:\d\d) \[(?[^\]]+)] (?ZscalerNSS: .*)') + ); + syslog-parser( + flags(assume-utf8, guess-timezone) + template("${.tmp.pri} ${.tmp.timestamp} ${.tmp.host} ${.tmp.message}") + ); + + }; + rewrite(set_rfc); + rewrite(set_rfc3164); + rewrite(set_rfc3164_zscaler-nss-alerts); + }; +}; +application app-almost-syslog-zscaler-nss-alerts[sc4s-almost-syslog] { + parser { app-almost-syslog-zscaler-nss-alerts(); }; +}; + + + diff --git a/package/enterprise/etc/conf.d/conflib/almost-syslog/app-almost-syslogy-wlevelword.conf b/package/enterprise/etc/conf.d/conflib/almost-syslog/app-almost-syslogy-wlevelword.conf new file mode 100644 index 0000000000..7ea0ace9fc --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/almost-syslog/app-almost-syslogy-wlevelword.conf @@ -0,0 +1,31 @@ +rewrite set_rfc3164_wlevel{ + set-tag("wireformat:rfc3164_wlevel"); +}; +filter f_is_rfc3164_wlevel{ + tags("wireformat:rfc3164_wlevel"); +}; + +block parser app-almost-syslogz-wlevelword() { + channel { + parser { + regexp-parser( + prefix(".tmp.") + patterns('^(?\<\d+>) ?(?\w\w\w +\d{1,2} \d\d:\d\d:\d\d) (?[^ ]+) (?alert|crit|debug|emerg|error|err|info|notice|warning) (?.*)') + ); + syslog-parser( + flags(assume-utf8, expect-hostname) + template("${.tmp.pri} ${.tmp.timestamp} ${.tmp.host} ${.tmp.message}") + ); + }; + rewrite { + guess-time-zone(); + set("${.tmp.level}", value(".metadata.header.log_level")); + }; + rewrite(set_rfc); + rewrite(set_rfc3164); + rewrite(set_rfc3164_wlevel); + }; +}; +application app-almost-syslogz-wlevelword[sc4s-almost-syslog] { + parser { app-almost-syslogz-wlevelword(); }; +}; diff --git a/package/enterprise/etc/conf.d/conflib/almost-syslog/app-almost-syslogz-bsd-epoch.conf b/package/enterprise/etc/conf.d/conflib/almost-syslog/app-almost-syslogz-bsd-epoch.conf new file mode 100644 index 0000000000..4a015f3f09 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/almost-syslog/app-almost-syslogz-bsd-epoch.conf @@ -0,0 +1,32 @@ +rewrite set_rfc3164_epoch{ + set-tag("wireformat:rfc3164_epoch"); +}; +filter f_is_rfc3164_epoch{ + tags("wireformat:rfc3164_epoch"); +}; + +block parser app-almost-syslogz-bsd-epoch() { + channel { + parser { + regexp-parser( + prefix(".tmp.") + patterns('^(?\<\d+\>)(?:1(?= ))? ?(?\d{10,13}(?:\.\d+)?) (?.*)') + ); + date-parser( + format('%s.%f', '%s') + template("${.tmp.timestamp}") + ); + syslog-parser( + flags(assume-utf8, guess-timezone) + template("${.tmp.pri} $S_ISODATE ${.tmp.message}") + ); + }; + rewrite(set_rfc); + rewrite(set_rfc3164); + rewrite(set_rfc3164_epoch); + + }; +}; +application app-almost-syslogz-bsd-epoch[sc4s-almost-syslog] { + parser { app-almost-syslogz-bsd-epoch(); }; +}; diff --git a/package/enterprise/etc/conf.d/conflib/almost-syslog/app-almost-syslogz-bsd-isotime.conf b/package/enterprise/etc/conf.d/conflib/almost-syslog/app-almost-syslogz-bsd-isotime.conf new file mode 100644 index 0000000000..a6c5c627ed --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/almost-syslog/app-almost-syslogz-bsd-isotime.conf @@ -0,0 +1,33 @@ +rewrite set_rfc3164_bsddate_isotime{ + set-tag("wireformat:rfc3164_bsddate_isotime"); +}; +filter f_is_rfc3164_bsddate_isotime{ + tags("wireformat:rfc3164_bsddate_isotime"); +}; + + +block parser app-almost-syslogz-bsd-isotime() { + channel { + parser { + regexp-parser( + prefix(".tmp.") + patterns('^(?\<\d+\>) ?(?[A-Z][a-z]{2} ?\d{1,2}(?: \d{4})? \d\d:\d\d:\d\d\.\d+(?:Z|(?:-|\+)\d{4})) (?[^ ]+) (?.*)') + ); + date-parser( + format('%b %d %Y %H:%M:%S.%f%z') + template("${.tmp.timestamp}") + ); + syslog-parser( + flags(assume-utf8) + template("${$.tmp.pri} $S_ISODATE ${.tmp.host} ${.tmp.message}") + ); + }; + rewrite(set_rfc); + rewrite(set_rfc3164); + rewrite(set_rfc3164_bsddate_isotime); + + }; +}; +application app-almost-syslogz-bsd-isotime[sc4s-almost-syslog] { + parser { app-almost-syslogz-bsd-isotime(); }; +}; diff --git a/package/enterprise/etc/conf.d/conflib/almost-syslog/app-almost-syslogz-bsd-onedigitday.conf b/package/enterprise/etc/conf.d/conflib/almost-syslog/app-almost-syslogz-bsd-onedigitday.conf new file mode 100644 index 0000000000..e95ce4acbe --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/almost-syslog/app-almost-syslogz-bsd-onedigitday.conf @@ -0,0 +1,27 @@ +rewrite set_rfc3164_bsd-onedigitday{ + set-tag("wireformat:rfc3164_bsd-onedigitday"); +}; +filter f_is_rfc3164_bsd-onedigitday{ + tags("wireformat:rfc3164_bsd-onedigitday"); +}; + +block parser app-almost-syslogz-bsd-onedigitday() { + channel { + parser { + regexp-parser( + prefix(".tmp.") + patterns('^(?\<\d+\>) ?(?[A-Z][a-z]{2}) (?\d \d\d:\d\d:\d\d) (?.*)') + ); + syslog-parser( + flags(assume-utf8, guess-timezone) + template("${.tmp.pri} ${.tmp.tsp1} ${.tmp.tsp2} ${.tmp.message}") + ); + }; + rewrite(set_rfc); + rewrite(set_rfc3164); + rewrite(set_rfc3164_bsd-onedigitday); + }; +}; +application app-almost-syslogz-bsd-onedigitday[sc4s-almost-syslog] { + parser { app-almost-syslogz-bsd-onedigitday(); }; +}; diff --git a/package/enterprise/etc/conf.d/conflib/almost-syslog/app-almost-syslogz-bsd-wrongver.conf b/package/enterprise/etc/conf.d/conflib/almost-syslog/app-almost-syslogz-bsd-wrongver.conf new file mode 100644 index 0000000000..8b5a78de51 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/almost-syslog/app-almost-syslogz-bsd-wrongver.conf @@ -0,0 +1,36 @@ +rewrite set_rfc3164_wrongver{ + set-tag("wireformat:rfc3164_wrongver"); +}; +filter f_is_rfc3164_wrongver{ + tags("wireformat:rfc3164_wrongver"); +}; + +block parser app-almost-syslogz-bsd-wrongver() { + channel { + parser { + regexp-parser( + prefix(".tmp.") + patterns('^(?\<\d+\>)1 *(?
.*)') + ); + }; + rewrite{ + subst('^(\<\d+\>)1(.*)', '$0' , value("MESSAGE")); + }; + parser { + syslog-parser( + flags(assume-utf8,guess-timezone) + template('${.tmp.pri}${.tmp.header}') + ); + }; + + rewrite(set_rfc); + rewrite(set_rfc3164); + rewrite(set_rfc3164_wrongver); + }; +}; +application app-almost-syslogz-bsd-wrongver[sc4s-almost-syslog] { + filter { + message('^\<\d+\>1 ') + }; + parser { app-almost-syslogz-bsd-wrongver(); }; +}; diff --git a/package/enterprise/etc/conf.d/conflib/almost-syslog/app-almost-syslogz-hip.conf b/package/enterprise/etc/conf.d/conflib/almost-syslog/app-almost-syslogz-hip.conf new file mode 100644 index 0000000000..e06222f944 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/almost-syslog/app-almost-syslogz-hip.conf @@ -0,0 +1,27 @@ +rewrite set_rfc3164_host_ip{ + set-tag("wireformat:rfc3164_host_ip"); +}; +filter f_is_rfc3164_host_ip{ + tags("wireformat:rfc3164_host_ip"); +}; + +block parser app-almost-syslogz-hip() { + channel { + parser { + regexp-parser( + prefix(".tmp.") + patterns('^(?<\d+>) ?(?\w\w\w \d\d \d\d:\d\d:\d\d) (?[a-z0-9\.-]+) \d+\.\d+\.\d+\.\d +(?.*)') + ); + syslog-parser( + flags(assume-utf8, guess-timezone) + template("${.tmp.pri} ${.tmp.timestamp} ${.tmp.host} ${.tmp.message}") + ); + }; + rewrite(set_rfc); + rewrite(set_rfc3164); + rewrite(set_rfc3164_host_ip); + }; +}; +application app-almost-syslogz-hip[sc4s-almost-syslog] { + parser { app-almost-syslogz-hip(); }; +}; diff --git a/package/enterprise/etc/conf.d/conflib/almost-syslog/app-almost-syslogz-hostnodate.conf b/package/enterprise/etc/conf.d/conflib/almost-syslog/app-almost-syslogz-hostnodate.conf new file mode 100644 index 0000000000..cbcb91a9e4 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/almost-syslog/app-almost-syslogz-hostnodate.conf @@ -0,0 +1,30 @@ +rewrite set_rfc3164_nodate{ + set-tag("wireformat:rfc3164_nodate"); +}; +filter f_is_rfc3164_nodate{ + tags("wireformat:rfc3164_nodate"); +}; + +block parser app-almost-syslogz-hostnodate() { + channel { + filter { + not message('^\<\d+\> ?[A-Z][a-z]{2} '); + }; + parser { + regexp-parser( + prefix(".tmp.") + patterns('^(?\<\d+\>) ?(?[a-zA-Z][a-zA-Z0-9\.-]+) (?[^ ]+ .*)') + ); + syslog-parser( + flags(assume-utf8) + template("${.tmp.pri} $R_DATE ${.tmp.host} ${.tmp.message}") + ); + }; + rewrite(set_rfc); + rewrite(set_rfc3164); + rewrite(set_rfc3164_nodate); + }; +}; +application app-almost-syslogz-hostnodate[sc4s-almost-syslog] { + parser { app-almost-syslogz-hostnodate(); }; +}; diff --git a/package/enterprise/etc/conf.d/conflib/almost-syslog/app-almost-syslogz-isodate.conf b/package/enterprise/etc/conf.d/conflib/almost-syslog/app-almost-syslogz-isodate.conf new file mode 100644 index 0000000000..833c12ea1b --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/almost-syslog/app-almost-syslogz-isodate.conf @@ -0,0 +1,31 @@ +rewrite set_rfc3164_isodate{ + set-tag("wireformat:rfc3164_isodate"); +}; +filter f_is_rfc3164_isodate{ + tags("wireformat:rfc3164_isodate"); +}; + +block parser app-almost-syslogz-isodate() { + channel { + parser { + regexp-parser( + prefix(".tmp.") + patterns('^(?\<\d+\>)1?(?: *)(?

\d{4}-\d{2}-\d{2}(?: |T)?\d{2}:\d{2}:\d{2}),(?

\d+.*)') + ); + }; + parser { + syslog-parser( + flags(assume-utf8, guess-timezone) + template("${.tmp.pri}${.tmp.h1}.${.tmp.h2}") + ); + }; + + + rewrite(set_rfc); + rewrite(set_rfc3164); + rewrite(set_rfc3164_isodate); + }; +}; +application app-almost-syslogz-isodate[sc4s-almost-syslog] { + parser { app-almost-syslogz-isodate(); }; +}; diff --git a/package/enterprise/etc/conf.d/conflib/almost-syslog/app-almost-syslogz-json.conf b/package/enterprise/etc/conf.d/conflib/almost-syslog/app-almost-syslogz-json.conf new file mode 100644 index 0000000000..da3b100f3d --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/almost-syslog/app-almost-syslogz-json.conf @@ -0,0 +1,25 @@ +rewrite set_rfc3164_json{ + set-tag("wireformat:rfc3164_json"); +}; +filter f_is_rfc3164_json{ + tags("wireformat:rfc3164_json"); +}; + +block parser app-almost-syslogz-json() { + channel { + parser { + syslog-parser( + flags(assume-utf8, no-header) + ); + }; + rewrite(set_rfc); + rewrite(set_rfc3164); + rewrite(set_rfc3164_json); + }; +}; +application app-almost-syslogz-json[sc4s-almost-syslog] { + filter { + message('^(?:\<\d+>)? ?{.*}$'); + }; + parser { app-almost-syslogz-json(); }; +}; diff --git a/package/enterprise/etc/conf.d/conflib/app-fix-invalid-program/app-fix-invalid-program-dashdash.conf b/package/enterprise/etc/conf.d/conflib/app-fix-invalid-program/app-fix-invalid-program-dashdash.conf new file mode 100644 index 0000000000..7d0f5e83e3 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/app-fix-invalid-program/app-fix-invalid-program-dashdash.conf @@ -0,0 +1,6 @@ + +application app-fix-invalid-program-dashdash[fix-invalid-program] { + filter { + program('--' type(string) flags(prefix)) + }; +}; diff --git a/package/enterprise/etc/conf.d/conflib/app-fix-invalid-program/app-fix-invalid-program-epoch.conf b/package/enterprise/etc/conf.d/conflib/app-fix-invalid-program/app-fix-invalid-program-epoch.conf new file mode 100644 index 0000000000..54e7bc2607 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/app-fix-invalid-program/app-fix-invalid-program-epoch.conf @@ -0,0 +1,6 @@ + +application app-fix-invalid-program-epoch[fix-invalid-program] { + filter { + program('^\d{10}(?:\.\d+)?') + }; +}; diff --git a/package/enterprise/etc/conf.d/conflib/app-fix-invalid-program/app-fix-invalid-program-ipv4.conf b/package/enterprise/etc/conf.d/conflib/app-fix-invalid-program/app-fix-invalid-program-ipv4.conf new file mode 100644 index 0000000000..b1f8fdfdb5 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/app-fix-invalid-program/app-fix-invalid-program-ipv4.conf @@ -0,0 +1,7 @@ + + +application app-fix-invalid-program-ipv4[fix-invalid-program] { + filter { + program('(\b25[0-5]|\b2[0-4][0-9]|\b[01]?[0-9][0-9]?)(\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)){3}') + }; +}; diff --git a/package/enterprise/etc/conf.d/conflib/app-fix-invalid-program/app-fix-invalid-program-ne.conf b/package/enterprise/etc/conf.d/conflib/app-fix-invalid-program/app-fix-invalid-program-ne.conf new file mode 100644 index 0000000000..4b79f26b4c --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/app-fix-invalid-program/app-fix-invalid-program-ne.conf @@ -0,0 +1,7 @@ + + +application app-fix-invalid-program-ne[fix-invalid-program] { + filter { + "${MSGHDR}" ne "${LEGACY_MSGHDR}" + }; +}; diff --git a/package/enterprise/etc/conf.d/conflib/app-fix-invalid-program/app-fix-invalid-program-z_bsdconvention.conf b/package/enterprise/etc/conf.d/conflib/app-fix-invalid-program/app-fix-invalid-program-z_bsdconvention.conf new file mode 100644 index 0000000000..0b3ac762a4 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/app-fix-invalid-program/app-fix-invalid-program-z_bsdconvention.conf @@ -0,0 +1,5 @@ +application app-fix-invalid-program-z_bsdconvention[fix-invalid-program] { + filter { + not program('^[a-zA-Z0-9-_\/\(\)\.]+$') + }; +}; diff --git a/package/enterprise/etc/conf.d/conflib/app-hec-debug/app-hec-debug.conf b/package/enterprise/etc/conf.d/conflib/app-hec-debug/app-hec-debug.conf new file mode 100644 index 0000000000..a74d50ad82 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/app-hec-debug/app-hec-debug.conf @@ -0,0 +1,6 @@ +#This "app" parser will always match unless disabled by SC4S_ARCHIVE_GLOBAL=no +application app-hec-debug[sc4s-hec-debug] { + filter { + '$(if ("`SC4S_DEBUG_HEC`" ne "") "`SC4S_DEBUG_HEC`" "no")' eq "yes" + }; +}; \ No newline at end of file diff --git a/package/enterprise/etc/conf.d/conflib/app-lp-archive/app-lp-global_archive.conf b/package/enterprise/etc/conf.d/conflib/app-lp-archive/app-lp-global_archive.conf new file mode 100644 index 0000000000..511647d1aa --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/app-lp-archive/app-lp-global_archive.conf @@ -0,0 +1,6 @@ +#This "app" parser will always match unless disabled by SC4S_ARCHIVE_GLOBAL=no +application app-lp-global_archive[sc4s-lp-archive] { + filter { + '$(if ("`SC4S_ARCHIVE_GLOBAL`" ne "") "`SC4S_ARCHIVE_GLOBAL`" "no")' eq "yes" + }; +}; \ No newline at end of file diff --git a/package/enterprise/etc/conf.d/conflib/app-lp_dest_archive/plugin.conf b/package/enterprise/etc/conf.d/conflib/app-lp_dest_archive/plugin.conf new file mode 100644 index 0000000000..ee21c4753c --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/app-lp_dest_archive/plugin.conf @@ -0,0 +1,4 @@ + +@module confgen context(root) name(app-lp_dest_archive) exec("`SC4S_ETC`/conf.d/conflib/app-lp_dest_archive/plugin.py") + +app-lp_dest_archive() \ No newline at end of file diff --git a/package/enterprise/etc/conf.d/conflib/app-lp_dest_archive/plugin.jinja b/package/enterprise/etc/conf.d/conflib/app-lp_dest_archive/plugin.jinja new file mode 100644 index 0000000000..017551fbc2 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/app-lp_dest_archive/plugin.jinja @@ -0,0 +1,6 @@ + +application sc4s-lp-dest-archive-{{ source }}[sc4s-lp-archive] { + filter { + '{{ source }}' eq "${fields.sc4s_vendor}_${fields.sc4s_product}" + }; +}; diff --git a/package/enterprise/etc/conf.d/conflib/app-lp_dest_archive/plugin.py b/package/enterprise/etc/conf.d/conflib/app-lp_dest_archive/plugin.py new file mode 100755 index 0000000000..8e39a48c44 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/app-lp_dest_archive/plugin.py @@ -0,0 +1,25 @@ +#! /usr/bin/env python3 +import os +import jinja2 +import re + +plugin_path = os.path.dirname(os.path.abspath(__file__)) + +templateLoader = jinja2.FileSystemLoader(searchpath=plugin_path) +templateEnv = jinja2.Environment( + loader=templateLoader, + autoescape=jinja2.select_autoescape(default_for_string=False), +) +tm = templateEnv.get_template("plugin.jinja") + + +regexfa = r"SC4S_DEST_(.*)(?Viptela)-(?.*)-(?.*)$') + template('${.values.identifier}') + ); + }; + rewrite{ + unset(value('PROGRAM')); + unset(value('PID')); + set('${.tmp.host}' value('HOST') condition(host('^\d+\.\d+\.\d+\.\d+'))); + set('${.tmp.id1}-${.tmp.id2}' value('.values.identifier')); + subst('[A-Z]+-','', value('.values.mnemonic')); + set('%${.values.identifier}-${.values.severity}-${.values.mnemonic}${.values.seperator} ${.values.message}' value('MESSAGE')); + }; + }; + + }; +}; +application app-cisco-cisco_viptela[cisco_syslog] { + filter { + message('%Viptela' type(string) flags(prefix)); + }; + parser { app-cisco-cisco_viptela(); }; +}; diff --git a/package/enterprise/etc/conf.d/conflib/cisco-syslog/app-cisco-cisco_xr.conf b/package/enterprise/etc/conf.d/conflib/cisco-syslog/app-cisco-cisco_xr.conf new file mode 100644 index 0000000000..995ad4cc2f --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/cisco-syslog/app-cisco-cisco_xr.conf @@ -0,0 +1,21 @@ +block parser app-cisco-cisco_xr() { + channel { + rewrite { + r_set_splunk_dest_default( + index("netops") + sourcetype('cisco:xr') + vendor('cisco') + product('xr') + ); + }; + }; +}; + +application app-cisco-cisco_xr[cisco_syslog] { + filter { + message('%LICENSE-' type(string) flags(prefix)) + or message('%MGBL-' type(string) flags(prefix)) + or message('%SECURITY-' type(string) flags(prefix)); + }; + parser { app-cisco-cisco_xr(); }; +}; diff --git a/package/enterprise/etc/conf.d/conflib/fallback/app-fallback-json.conf b/package/enterprise/etc/conf.d/conflib/fallback/app-fallback-json.conf new file mode 100644 index 0000000000..109af134dc --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/fallback/app-fallback-json.conf @@ -0,0 +1,32 @@ +block parser app-fallback-json() { + channel { + rewrite { + r_set_splunk_dest_default( + sourcetype('json') + vendor("json") + product('generic') + ); + set("t_msg_trim", value(".splunk.sc4s_template")); + + }; + parser { + json-parser( + prefix('.values.') + ); + }; + + if { + parser { app-parser(topic(json)); }; + }; + rewrite { + groupunset(values('.values.*')); + }; + }; +}; +application app-fallback-json[sc4s-syslog] { + filter { + "${PROGRAM}" eq "" + and message('{' type(string) flags(prefix)); + }; + parser { app-fallback-json(); }; +}; diff --git a/package/enterprise/etc/conf.d/conflib/fallback/app-fallback-netsource.conf b/package/enterprise/etc/conf.d/conflib/fallback/app-fallback-netsource.conf new file mode 100644 index 0000000000..83d43286b2 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/fallback/app-fallback-netsource.conf @@ -0,0 +1,37 @@ +block parser app-fallback-netsource() { + channel { + rewrite { + r_set_splunk_dest_default( + index("osnix") + sourcetype('nix:syslog') + ); + }; + rewrite { + set("${PROGRAM}", value(".PROGRAM")); + subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM")); + r_set_splunk_dest_default( + index("osnix") + sourcetype('nix:syslog') + source('program:${.PROGRAM}') + vendor('${.netsource.sc4s_vendor}') + product('${.netsource.sc4s_product}') + class('nix_syslog') + ); + }; + }; +}; +application app-fallback-netsource[fallback] { + filter { + ( + filter(f_is_rfc5424); + or ( + filter(f_is_rfc3164) + and "${PROGRAM}" ne "" + and "${MESSAGE}" ne "" + ) + ) + and "${.netsource.sc4s_vendor}" ne "" + and "${.netsource.sc4s_product}" ne "" + }; + parser { app-fallback-netsource(); }; +}; diff --git a/package/enterprise/etc/conf.d/conflib/fallback/app-fallback-nix_syslog.conf b/package/enterprise/etc/conf.d/conflib/fallback/app-fallback-nix_syslog.conf new file mode 100644 index 0000000000..570012d746 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/fallback/app-fallback-nix_syslog.conf @@ -0,0 +1,32 @@ +block parser app-fallback-nix_syslog() { + channel { + rewrite { + r_set_splunk_dest_default( + index("osnix") + sourcetype('nix:syslog') + vendor('nix') + product('syslog') + ); + }; + rewrite { + set("${PROGRAM}", value(".PROGRAM")); + subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM")); + r_set_splunk_dest_update_v2( + source('program:${.PROGRAM}') + ); + }; + }; +}; +application app-fallback-nix_syslog[fallback] { + filter { + ( + filter(f_is_rfc5424); + or ( + filter(f_is_rfc3164) + and "${PROGRAM}" ne "" + and "${MESSAGE}" ne "" + ) + ) + }; + parser { app-fallback-nix_syslog(); }; +}; diff --git a/package/enterprise/etc/conf.d/conflib/fallback/app-fallbackz-lastchance.conf b/package/enterprise/etc/conf.d/conflib/fallback/app-fallbackz-lastchance.conf new file mode 100644 index 0000000000..ea1b19219f --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/fallback/app-fallbackz-lastchance.conf @@ -0,0 +1,19 @@ +block parser app-fallbackz-lastchance() { + channel { + rewrite { + r_set_splunk_dest_default( + sourcetype('sc4s:fallback') + vendor('splunk') + product('sc4s') + class("fallback") + template('t_fallback_kv') + ); + }; + + #rewrite { set("$(template t_fallback_kv)" value("MSG")); }; + + }; +}; +application app-fallbackz-lastchance[fallback] { + parser { app-fallbackz-lastchance(); }; +}; diff --git a/package/enterprise/etc/conf.d/conflib/json/app-json-novell_netiq.conf b/package/enterprise/etc/conf.d/conflib/json/app-json-novell_netiq.conf new file mode 100644 index 0000000000..2ac923a70e --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/json/app-json-novell_netiq.conf @@ -0,0 +1,28 @@ +block parser app-json-novell_netiq() { + channel { + + rewrite { + r_set_splunk_dest_default( + vendor('novell') + product('netiq') + index('netauth'), + source('novell:netiq:${.values.component}'), + sourcetype('novell:netiq') + ); + }; + + parser { + date-parser-nofilter(format( + '%a, %d %b %Y %H:%M:%S %z', + ) + template("${.values.timeStamp}") + ); + }; + }; +}; +application app-json-novell_netiq[json] { + filter{ + "${.values.appName}" eq "Novell Access Manager"; + }; + parser { app-json-novell_netiq(); }; +}; diff --git a/package/enterprise/etc/conf.d/conflib/json/app-json-zscaler_lss.conf b/package/enterprise/etc/conf.d/conflib/json/app-json-zscaler_lss.conf new file mode 100644 index 0000000000..880ca8e15e --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/json/app-json-zscaler_lss.conf @@ -0,0 +1,118 @@ +block parser app-json-zscaler_lss() { + channel { + if { + filter { + match('.' value('.values.ClientZEN')) + and match('.' value('.values.AppGroup')) + and match('.' value('.values.Application')) + }; + rewrite { + r_set_splunk_dest_default( + index("netproxy") + sourcetype('zscalerlss-zpa-app') + vendor('zscaler') + product('lss') + ); + }; + } elif { + filter { + match('.' value('.values.Exporter')) + and match('.' value('.values.Customer')) + and match('.' value('.values.ConnectionID')) + }; + rewrite { + r_set_splunk_dest_default( + index("netproxy") + sourcetype('zscalerlss-zpa-bba') + vendor('zscaler') + product('lss') + ); + }; + } elif { + filter { + match('.' value('.values.Connector')) + and( + ( + match('.' value('.values.Customer')) and + match('.' value('.values.ConnectorGroup')) + ) + or + ( + match('.' value('.values.ProcessMemoryUtilization')) and + match('.' value('.values.SystemMemoryUtilization')) + ) + ) + }; + rewrite { + r_set_splunk_dest_default( + index("netproxy") + sourcetype('zscalerlss-zpa-connector') + vendor('zscaler') + product('lss') + ); + }; + } elif { + filter { + match('.*' value('.values.SAMLAttributes')) + and match('.' value('.values.Customer')) + }; + rewrite { + r_set_splunk_dest_default( + index("netproxy") + sourcetype('zscalerlss-zpa-auth') + vendor('zscaler') + product('lss') + ); + }; + }; + parser { + #.jsonLog.Timestamp Mar 04 20:37:53 2020 + date-parser( + format('%a %b %d %H:%M:%S %Y', + '%a %b %d %k:%M:%S %Y') + template("${.values.LogTimestamp}") + flags(guess-timezone) + ); + }; + }; +}; + +block parser app-json-zscaler_lss_audit() { + channel { + if { + filter { + match('.' value('.values.ClientAuditUpdate')) + }; + rewrite { + r_set_splunk_dest_default( + index("netproxy") + sourcetype('zscalerlss-zpa-audit') + vendor('zscaler') + product('lss') + ); + }; + }; + parser { + date-parser( + format('%Y-%m-%dT%H:%M:%S.%fZ', + '%a %b %d %k:%M:%S %Y') + template("${.values.CreationTime}") + flags(guess-timezone) + ); + }; + }; +}; + +block parser app-json-zscaler() { + channel { + if { + parser { app-json-zscaler_lss(); }; + } else { + parser { app-json-zscaler_lss_audit(); }; + }; + }; +}; + +application app-json-zscaler_lss[json] { + parser { app-json-zscaler(); }; +}; \ No newline at end of file diff --git a/package/enterprise/etc/conf.d/conflib/leef/app-leef-kaspersky_es.conf b/package/enterprise/etc/conf.d/conflib/leef/app-leef-kaspersky_es.conf new file mode 100644 index 0000000000..5d406f2c49 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/leef/app-leef-kaspersky_es.conf @@ -0,0 +1,54 @@ +block parser app-leef-kaspersky_es() { + channel { + + rewrite { + r_set_splunk_dest_default( + index('epav'), + sourcetype('kaspersky:leef') + ); + }; + rewrite { + r_set_splunk_dest_update_v2( + sourcetype('kaspersky:klaud') + condition(match('^KLAUD_' value('.metadata.leef.EventID'))) + ); + }; + rewrite { + r_set_splunk_dest_update_v2( + sourcetype('kaspersky:klsrv') + condition(match('^KLSRV_' value('.metadata.leef.EventID'))) + ); + }; + rewrite { + r_set_splunk_dest_update_v2( + sourcetype('kaspersky:gnrl') + condition(match('^GNRL_' value('.metadata.leef.EventID'))) + ); + }; + rewrite { + r_set_splunk_dest_update_v2( + sourcetype('kaspersky:klnag') + condition(match('^KLNAG_' value('.metadata.leef.EventID'))) + ); + }; + rewrite { + r_set_splunk_dest_update_v2( + sourcetype('kaspersky:klprci') + condition(match('^KLPRCI_' value('.metadata.leef.EventID'))) + ); + }; + rewrite { + r_set_splunk_dest_update_v2( + sourcetype('kaspersky:klbl') + condition(match('^KLBL_' value('.metadata.leef.EventID'))) + ); + }; + }; +}; +application app-leef-kaspersky_es[leef] { + filter{ + match("KasperskyLab" value(".metadata.leef.vendor")) + and match("SecurityCenter" value(".metadata.leef.product")); + }; + parser { app-leef-kaspersky_es(); }; +}; diff --git a/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-aruba_clearpass.conf b/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-aruba_clearpass.conf new file mode 100644 index 0000000000..451a0c7fdf --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-aruba_clearpass.conf @@ -0,0 +1,107 @@ +block parser app-netsource-aruba_clearpass() { + channel { + rewrite { + r_set_splunk_dest_default( + index('netops') + sourcetype('aruba:clearpass') + vendor('aruba') + product('clearpass') + template('t_hdr_msg') + ); + }; + + rewrite { + r_set_splunk_dest_update_v2( + class("endpoint-profile") + condition(program("CPPM_Endpoint_Profile", type(string) flags(ignore-case, prefix))) + ); + }; + + rewrite { + r_set_splunk_dest_update_v2( + class("alert") + condition(program("CPPM_Alert", type(string) flags(ignore-case, prefix))) + ); + }; + + rewrite { + r_set_splunk_dest_update_v2( + class("endpoint-audit-record") + condition(program("CPPM_Audit_Record", type(string) flags(ignore-case, prefix))) + ); + }; + + rewrite { + r_set_splunk_dest_update_v2( + class("policy-server-session") + condition(program("CPPM_Policy_Server_Session", type(string) flags(ignore-case, prefix))) + ); + }; + + rewrite { + r_set_splunk_dest_update_v2( + class("post-auth-monit-config") + condition(program("CPPM_Post_Auth_Monit_Config", type(string) flags(ignore-case, prefix))) + ); + }; + + rewrite { + r_set_splunk_dest_update_v2( + class("radius-session") + condition(program("CPPM_RADIUS_Session", type(string) flags(ignore-case, prefix))) + ); + }; + + rewrite { + r_set_splunk_dest_update_v2( + class("snmp-session-log") + condition(program("CPPM_SNMP_Session_Log", type(string) flags(ignore-case, prefix))) + ); + }; + + rewrite { + r_set_splunk_dest_update_v2( + class("system-event") + condition(program("CPPM_System_Event", type(string) flags(ignore-case, prefix))) + ); + }; + + rewrite { + r_set_splunk_dest_update_v2( + class("tacacs-accounting-detail") + condition(program("CPPM_TACACS_Accounting_Detail", type(string) flags(ignore-case, prefix))) + ); + }; + + rewrite { + r_set_splunk_dest_update_v2( + class("tacacs-accounting-record") + condition(program("CPPM_TACACS_Accouting_Record", type(string) flags(ignore-case, prefix))) + ); + }; + }; +}; + + +application app-netsource-aruba_clearpass[sc4s-network-source] { + filter { + not filter(f_is_source_identified) + and ( + program('CPPM_' type(string) flags(prefix)) + or ( + ( + match("aruba", value('.netsource.sc4s_vendor'), type(string)) + and match("clearpass", value('.netsource.sc4s_product'), type(string)) + ) + and (tags("ns_vendor:aruba") and tags("ns_product:clearpass")) + or tags(".source.s_ARUBA_CLEARPASS") + or "${.netsource.sc4s_vendor_product}" eq "aruba_clearpass" + ) + + ) + ; + }; + parser { app-netsource-aruba_clearpass(); }; +}; + + diff --git a/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-aruba_silverpeak.conf b/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-aruba_silverpeak.conf new file mode 100644 index 0000000000..1a54d11b77 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-aruba_silverpeak.conf @@ -0,0 +1,32 @@ +block parser app-netsource-aruba_silverpeak() { + channel { + rewrite { + r_set_splunk_dest_default( + index('netops') + sourcetype('aruba:silverpeak') + vendor('aruba') + product('silverpeak') + ); + }; + }; +}; + + +application app-netsource-aruba_silverpeak[sc4s-network-source] { + filter { + not filter(f_is_source_identified) + and ( + ( + match("aruba", value('.netsource.sc4s_vendor'), type(string)) + and match("silverpeak", value('.netsource.sc4s_product'), type(string)) + ) + and (tags("ns_vendor:aruba") and tags("ns_product:silverpeak")) + or tags(".source.s_ARUBA_SILVERPEAK") + or "${.netsource.sc4s_vendor_product}" eq "aruba_silverpeak" + ) + ; + }; + parser { app-netsource-aruba_silverpeak(); }; +}; + + diff --git a/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-barracuda_syslog.conf b/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-barracuda_syslog.conf new file mode 100644 index 0000000000..b168ba6e3c --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-barracuda_syslog.conf @@ -0,0 +1,69 @@ +block parser app-netsource-barracuda_syslog() { + channel { + rewrite { + r_set_splunk_dest_default( + index("netops") + sourcetype('barracuda:syslog') + vendor("barracuda") + product("syslog") + template("t_msg_only") + ); + }; + parser { + kv-parser( + prefix(".values.") + pair-separator(" ") + ); + }; + # Web Firewall Logs (WF) web_firewall_logs + # Access Logs (TR) access_logs + # Audit Logs (AUDIT) audit_logs + # Network Firewall Logs (NF) network_firewall_logs + if { + filter { + "${.values.logType}" ne ""; + }; + rewrite { + r_set_splunk_dest_update_v2( + sourcetype('barracuda:$(lowercase ${.values.logType})') + class('$(lowercase ${.values.logType})') + ); + }; + rewrite { + r_set_splunk_dest_update_v2( + index('netfw') + condition(match('nf' value('.values.logType') type(string) ,flags(ignore-case) )) + ); + }; + rewrite { + r_set_splunk_dest_update_v2( + index('netwaf') + condition(match('wf' value('.values.logType') type(string) ,flags(ignore-case) )) + ); + }; + rewrite { + r_set_splunk_dest_update_v2( + index('netwaf') + condition(match('tr' value('.values.logType') type(string) ,flags(ignore-case) )) + ); + }; + }; + }; +}; +application app-netsource-barracuda_syslog[sc4s-network-source] { + filter { + not filter(f_is_source_identified) + and ( + ( + match("barracuda", value('.netsource.sc4s_vendor'), type(string)) + and match("syslog", value('.netsource.sc4s_product'), type(string)) + ) + or (tags("ns_vendor:barracuda") and tags("ns_product:syslog")) + or tags(".source.s_BARRACUDA_SYSLOG") + ) + ; + }; + parser { app-netsource-barracuda_syslog(); }; +}; + + diff --git a/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-brocade_syslog.conf b/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-brocade_syslog.conf new file mode 100644 index 0000000000..270a7ecdaf --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-brocade_syslog.conf @@ -0,0 +1,35 @@ +block parser app-netsource-brocade_syslog() { + channel { + rewrite { + set("${PROGRAM}", value(".PROGRAM")); + subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM")); + r_set_splunk_dest_default( + index('netops') + source('brocade:${.PROGRAM}') + sourcetype('brocade:syslog') + vendor("brocade") + product("syslog") + template('t_hdr_msg') + ); + }; + + + }; +}; +application app-netsource-brocade_syslog[sc4s-network-source] { + filter { + not filter(f_is_source_identified) + and ( + ( + match("brocade", value('.netsource.sc4s_vendor'), type(string)) + and match("syslog", value('.netsource.sc4s_product'), type(string)) + ) + or (tags("ns_vendor:brocade") and tags("ns_product:syslog")) + or tags(".source.s_BROCADE") + or "${.netsource.sc4s_vendor_product}" eq "brocade_syslog" + ) + + + }; + parser { app-netsource-brocade_syslog(); }; +}; diff --git a/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-buffalo_terastation.conf b/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-buffalo_terastation.conf new file mode 100644 index 0000000000..d4a9bfb09b --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-buffalo_terastation.conf @@ -0,0 +1,32 @@ +block parser app-netsource-buffalo_terastation() { + channel { + rewrite { + r_set_splunk_dest_default( + index('infraops') + sourcetype('buffalo:terastation:$(lowercase $PROGRAM)') + vendor('buffalo') + product('terastation') + class("$PROGRAM") + template('t_hdr_msg') + ); + }; + }; +}; +application app-netsource-buffalo_terastation[sc4s-network-source] { + filter { + not filter(f_is_source_identified) + and ( + ( + match("buffalo", value('.netsource.sc4s_vendor'), type(string)) + and match("terastation", value('.netsource.sc4s_product'), type(string)) + ) + or (tags("ns_vendor:buffalo") and tags("ns_product:terastation")) + or tags(".source.s_BUFFALO_TERASTATION") + or "${.netsource.sc4s_vendor_product}" eq "buffalo_terastation" + ) + ; + }; + parser { app-netsource-buffalo_terastation(); }; +}; + + diff --git a/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-checkpoint_fw.conf b/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-checkpoint_fw.conf new file mode 100644 index 0000000000..64a80eabd1 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-checkpoint_fw.conf @@ -0,0 +1,34 @@ +block parser app-netsource-checkpoint_fw() { + channel { + rewrite { + r_set_splunk_dest_default( + index('netops') + sourcetype('cp_log:fw:syslog') + vendor('checkpoint') + product('fw') + template('t_hdr_msg') + ); + }; + + + + }; +}; +application app-netsource-checkpoint_fw[sc4s-network-source] { + filter { + not filter(f_is_source_identified) + and ( + ( + match("checkpoint", value('.netsource.sc4s_vendor'), type(string)) + and match("fw", value('.netsource.sc4s_product'), type(string)) + ) + or (tags("ns_vendor:checkpoint") and tags("ns_product:fw")) + or tags(".source.s_CHECKPOINT_FW") + or "${.netsource.sc4s_vendor_product}" eq "checkpoint_fw" + ) + ; + }; + parser { app-netsource-checkpoint_fw(); }; +}; + + diff --git a/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-cisco_esa.conf b/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-cisco_esa.conf new file mode 100644 index 0000000000..5af68b6ed5 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-cisco_esa.conf @@ -0,0 +1,249 @@ +filter f_cisco_esa_http { + program('gui_logs' type(string) flags(prefix)) + or program('sc4s_gui_logs' type(string) flags(prefix)) + or match('https?\s*service\s*', value("MESSAGE")) + or match('[sS]ession\s*\w+\s*from\s*', value("MESSAGE")) + or match('\s*PERIODIC\s*REPORTS\s*:\s*', value("MESSAGE")) + or match('\s*req\s*:.+\s*user\s*:\s*.+\s*id\s*:\s*', value("MESSAGE")) + or match('testmaillog\s*:\s*[iI]nfo\s*:', value("MESSAGE")) + or match('[sS]ystem\s*is\s*coming\s*up', value("MESSAGE")) +}; + +filter f_cisco_esa_textmail { + program('mail_logs' type(string) flags(prefix)) + or program('sc4s_mail_logs' type(string) flags(prefix)) + or match('mail_logs:', value("MESSAGE")) + or match('MID\s+\d+', value("MESSAGE")) + or match('ICID\s+\d+', value("MESSAGE")) + or match('DCID\s+\d+', value("MESSAGE")) + or match('RID\s+\[?(\d+(?:\s*,\s*\d+)*)', value("MESSAGE")) + or match('(?:SplunkMailSyslog|MAIL_SecurityAudit|CES_VPN_Mail_SecurityAudit)\s*:', value("MESSAGE")) +}; + +filter f_cisco_esa_amp { + program('amp' type(string) flags(prefix)) + or program('sc4s_amp' type(string) flags(prefix)) + or match('SHA256:', value("MESSAGE")) + or match('File\s*not\s*uploaded\s*for\s*analysis', value("MESSAGE")) + or match('Response\s*received\s*for\s*file\s*reputation\s*query\s*from\s*Cloud', value("MESSAGE")) + or match('File\s*reputation\s*query\s*initiating', value("MESSAGE")) +}; + +filter f_cisco_esa_authentication { + program('sc4s_audit_log' type(string) flags(prefix)) + or match('[Ii]nfo\s*:\s*Begin\s*Logfile', value("MESSAGE")) + or match('authenticated\s*successfully.', value("MESSAGE")) + or match('successfully\s*logged\s*on\s*from', value("MESSAGE")) + or match('An\s*authentication\s*attempt', value("MESSAGE")) + or match('failed\s*authentication.', value("MESSAGE")) + or match('Time\s*offset\s*from\s*UTC:', value("MESSAGE")) + or match('[vV]ersion:\s+.*\s+SN:\s+.*', value("MESSAGE")) + or match('\s*login|logout:\s*[^\s]*\s*[Uu]ser:[\w\-]+\s*session:[^\s]+', value("MESSAGE")) + or match('\slogged\s*out\s*', value("MESSAGE")) + or match('MAR_SecurityAudit', value("MESSAGE")) +}; + +filter f_cisco_esa_antispam { + program('antispam' type(string) flags(prefix)) + or program('sc4s_antispam' type(string) flags(prefix)) +}; + +filter f_cisco_esa_euq_logs { + program('euq_logs' type(string) flags(prefix)) + or program('sc4s_euq_logs' type(string) flags(prefix)) +}; + +filter f_cisco_esa_service_logs { + program('service_logs' type(string) flags(prefix)) + or program('sc4s_service_logs' type(string) flags(prefix)) +}; + +filter f_cisco_esa_reportd_logs { + program('reportd_logs' type(string) flags(prefix)) + or program('sc4s_reportd_logs' type(string) flags(prefix)) +}; + +filter f_cisco_esa_sntpd_logs { + program('sntpd_logs' type(string) flags(prefix)) + or program('sc4s_sntpd_logs' type(string) flags(prefix)) +}; + +filter f_cisco_esa_smartlicense { + program('smartlicense' type(string) flags(prefix)) + or program('sc4s_smartlicense' type(string) flags(prefix)) +}; + +filter f_cisco_esa_updater_logs { + program('updater_logs' type(string) flags(prefix)) + or program('sc4s_updater_logs' type(string) flags(prefix)) +}; + +filter f_cisco_esa_content_scanner { + program('content_scanner' type(string) flags(prefix)) + or program('sc4s_content_scanner' type(string) flags(prefix)) +}; + +filter f_cisco_esa_error_logs { + program('error_logs' type(string) flags(prefix)) + or program('sc4s_error_logs' type(string) flags(prefix)) +}; + +filter f_cisco_esa_system_logs { + program('system_logs' type(string) flags(prefix)) + or program('sc4s_system_logs' type(string) flags(prefix)) +}; + + +block parser app-netsource-cisco_esa() { + channel { + rewrite { + set("${PROGRAM}", value(".PROGRAM")); + subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM")); + r_set_splunk_dest_default( + index('email') + source('program:${.PROGRAM}') + sourcetype('cisco:esa') + vendor('cisco') + product('esa') + template('t_msg_only') + ); + }; + + if { + filter(f_cisco_esa_amp); + rewrite { + r_set_splunk_dest_update_v2( + source('esa:amp') + sourcetype('cisco:esa:amp') + ); + }; + } elif { + filter(f_cisco_esa_http); + rewrite { + r_set_splunk_dest_update_v2( + source('esa:http') + sourcetype('cisco:esa:http') + ); + }; + } elif { + filter(f_cisco_esa_euq_logs); + rewrite { + r_set_splunk_dest_update_v2( + source('esa:euq_logs') + sourcetype('cisco:esa:system_logs') + ); + }; + } elif { + filter(f_cisco_esa_service_logs); + rewrite { + r_set_splunk_dest_update_v2( + source('esa:service_logs') + sourcetype('cisco:esa:system_logs') + ); + }; + } elif { + filter(f_cisco_esa_reportd_logs); + rewrite { + r_set_splunk_dest_update_v2( + source('esa:reportd_logs') + sourcetype('cisco:esa:system_logs') + ); + }; + } elif { + filter(f_cisco_esa_sntpd_logs); + rewrite { + r_set_splunk_dest_update_v2( + source('esa:sntpd_logs') + sourcetype('cisco:esa:system_logs') + ); + }; + } elif { + filter(f_cisco_esa_smartlicense); + rewrite { + r_set_splunk_dest_update_v2( + source('esa:smartlicense') + sourcetype('cisco:esa:system_logs') + ); + }; + } elif { + filter(f_cisco_esa_updater_logs); + rewrite { + r_set_splunk_dest_update_v2( + source('esa:updater_logs') + sourcetype('cisco:esa:error_logs') + ); + }; + } elif { + filter(f_cisco_esa_textmail); + rewrite { + r_set_splunk_dest_update_v2( + source('esa:textmail') + sourcetype('cisco:esa:textmail') + ); + }; + } elif { + filter(f_cisco_esa_antispam); + rewrite { + r_set_splunk_dest_update_v2( + source('esa:antispam') + sourcetype('cisco:esa:antispam') + ); + }; + } elif { + filter(f_cisco_esa_content_scanner); + rewrite { + r_set_splunk_dest_update_v2( + source('esa:content_scanner') + sourcetype('cisco:esa:content_scanner') + ); + }; + } elif { + filter(f_cisco_esa_error_logs); + rewrite { + r_set_splunk_dest_update_v2( + source('esa:error_logs') + sourcetype('cisco:esa:error_logs') + ); + }; + } elif { + filter(f_cisco_esa_system_logs); + rewrite { + r_set_splunk_dest_update_v2( + source('esa:system_logs') + sourcetype('cisco:esa:system_logs') + ); + }; + } elif { + filter(f_cisco_esa_authentication); + rewrite { + r_set_splunk_dest_update_v2( + source('esa:authentication') + sourcetype('cisco:esa:authentication') + ); + }; + } else { + + rewrite { + r_set_splunk_dest_update_v2( + template("t_hdr_msg") + ); + }; + }; + + }; +}; +application app-netsource-cisco_esa[sc4s-network-source] { + filter { + not filter(f_is_source_identified) + and ( + ( + match("cisco", value('.netsource.sc4s_vendor'), type(string)) + and match("esa", value('.netsource.sc4s_product'), type(string)) + ) + or ( tags("ns_vendor:cisco") and tags("ns_product:esa")) + or tags(".source.s_CISCO_ESA") + or "${.netsource.sc4s_vendor_product}" eq "cisco_esa" + ) + }; + parser { app-netsource-cisco_esa(); }; +}; diff --git a/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-cisco_meraki.conf b/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-cisco_meraki.conf new file mode 100644 index 0000000000..6d54b000d5 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-cisco_meraki.conf @@ -0,0 +1,88 @@ +block filter f_cisco_meraki(vendor() product()) { + ( + match(`vendor`, value('.netsource.sc4s_vendor'), type(string)) + and match(`product`, value('.netsource.sc4s_product'), type(string)) + ) + or (tags('ns_vendor:`vendor`') and tags('ns_product:`product`')) + or tags('`vendor`_`product`') + or "${.netsource.sc4s_vendor_product}" eq '`vendor`_`product`' +}; + +block parser app-netsource-cisco_meraki() { + channel { + if { + filter { + f_cisco_meraki( + vendor('cisco') + product('meraki') + ); + }; + rewrite { + r_set_splunk_dest_default( + index('netfw') + sourcetype('meraki') + vendor('cisco') + product('meraki') + ); + }; + } elif { + filter { + f_cisco_meraki( + vendor('meraki') + product('securityappliances') + ); + }; + rewrite { + r_set_splunk_dest_default( + index('netfw') + sourcetype('meraki:securityappliances') + vendor('meraki') + product('securityappliances') + ); + }; + } elif { + filter { + f_cisco_meraki( + vendor('meraki') + product('accesspoints') + ); + }; + rewrite { + r_set_splunk_dest_default( + index('netfw') + sourcetype('meraki:accesspoints') + vendor('meraki') + product('accesspoints') + ); + }; + } elif { + filter { + f_cisco_meraki( + vendor('meraki') + product('switches') + ); + }; + rewrite { + r_set_splunk_dest_default( + index('netfw') + sourcetype('meraki:switches') + vendor('meraki') + product('switches') + ); + }; + }; + }; +}; + +application app-netsource-cisco_meraki[sc4s-network-source] { + filter { + not filter(f_is_source_identified) + and ( + f_cisco_meraki(vendor('cisco') product('meraki')) + or f_cisco_meraki(vendor('meraki') product('securityappliances')) + or f_cisco_meraki(vendor('meraki') product('accesspoints')) + or f_cisco_meraki(vendor('meraki') product('switches')) + ) + }; + parser { app-netsource-cisco_meraki(); }; +}; \ No newline at end of file diff --git a/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-cisco_mm.conf b/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-cisco_mm.conf new file mode 100644 index 0000000000..05efc43b84 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-cisco_mm.conf @@ -0,0 +1,35 @@ +block parser app-netsource-cisco_mm() { + channel { + rewrite { + r_set_splunk_dest_default( + index('netops') + sourcetype('cisco:mm') + sourcetype('cisco:mm:audit') + vendor('cisco') + product('mm') + class('audit') + template('t_hdr_msg') + ); + }; + }; +}; +application app-netsource-cisco_mm[sc4s-network-source] { + filter { + not filter(f_is_source_identified) + and not program('cmm-' type(string) flags(prefix)) + and + ( + ( + match("cisco", value('.netsource.sc4s_vendor'), type(string)) + and match("mm", value('.netsource.sc4s_product'), type(string)) + ) + or ( tags("ns_vendor:cisco") and tags("ns_product:mm")) + or tags(".source.s_CISCO_MM") + or "${.netsource.sc4s_vendor_product}" eq "cisco_mm" + ) + ; + }; + parser { app-netsource-cisco_mm(); }; +}; + + diff --git a/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-cisco_ms.conf b/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-cisco_ms.conf new file mode 100644 index 0000000000..7830ee0eb0 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-cisco_ms.conf @@ -0,0 +1,32 @@ +block parser app-netsource-cisco_ms() { + channel { + rewrite { + r_set_splunk_dest_default( + index('netops') + source('program:${PROGRAM}') + sourcetype('cisco:ms') + vendor('cisco') + product('ms') + template('t_hdr_msg') + ); + }; + }; +}; +application app-netsource-cisco_ms[sc4s-network-source] { + filter { + not filter(f_is_source_identified) + and ( + ( + match("cisco", value('.netsource.sc4s_vendor'), type(string)) + and match("ms", value('.netsource.sc4s_product'), type(string)) + ) + or (tags("ns_vendor:cisco") and tags("ns_product:ms")) + or tags(".source.s_CISCO_ms") + or "${.netsource.sc4s_vendor_product}" eq "cisco_ms" + ) + ; + }; + parser { app-netsource-cisco_ms(); }; +}; + + diff --git a/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-cisco_wsa.conf b/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-cisco_wsa.conf new file mode 100644 index 0000000000..b9e974da23 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-cisco_wsa.conf @@ -0,0 +1,61 @@ + + +block parser app-netsource-cisco_wsa() { + channel { + rewrite { + r_set_splunk_dest_default( + index('netproxy') + source("cisco_wsa") + sourcetype('cisco:wsa') + vendor('cisco') + product('wsa') + ); + }; + if{ + filter{ + message('(?:^| )(Info|Warning|Alert|Debug):.*') + }; + rewrite { + r_set_splunk_dest_update_v2( + sourcetype('cisco:wsa:l4tm') + ); + }; + rewrite { + set("$PROGRAM $MSG" value("MSG")); + unset(value("PROGRAM")); + }; + } elif { + parser { + regexp-parser( + prefix(".tmp.") + patterns('^(?:(?\d{10})(?:\.(?\d{1,9}))?)') + ); + date-parser-nofilter(format('%s.%f') + template("${.tmp.epoch}.${.tmp.frac}") + ); + }; + rewrite { + r_set_splunk_dest_update_v2( + sourcetype('cisco:wsa:squid') + ); + }; + } else { + }; + }; +}; +application app-netsource-cisco_wsa[sc4s-network-source] { + filter { + not filter(f_is_source_identified) + and ( + ( + match("cisco", value('.netsource.sc4s_vendor'), type(string)) + and match("wsa", value('.netsource.sc4s_product'), type(string)) + ) + or (tags("ns_vendor:cisco") and tags("ns_product:wsa")) + or tags(".source.s_CISCO_WSA") + or "${.netsource.sc4s_vendor_product}" eq "cisco_wsa" + ) + + }; + parser { app-netsource-cisco_wsa(); }; +}; diff --git a/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-cisco_wsa_11_7.conf b/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-cisco_wsa_11_7.conf new file mode 100644 index 0000000000..ab2fe47e68 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-cisco_wsa_11_7.conf @@ -0,0 +1,61 @@ + + +block parser app-netsource-cisco_wsa_11_7() { + channel { + rewrite { + r_set_splunk_dest_default( + index('netproxy') + source("cisco_wsa_11.7") + sourcetype('cisco:wsa:squid:new') + vendor('cisco') + product('wsa') + ); + }; + if{ + filter{ + message('(?:^| )(Info|Warning|Alert|Debug):.*') + }; + rewrite { + r_set_splunk_dest_update_v2( + sourcetype('cisco:wsa:l4tm') + ); + }; + rewrite { + set("$PROGRAM $MSG" value("MSG")); + unset(value("PROGRAM")); + }; + } elif { + parser { + regexp-parser( + prefix(".tmp.") + patterns('^(?:(?\d{10})(?:\.(?\d{1,9}))?)') + ); + date-parser-nofilter(format('%s.%f') + template("${.tmp.epoch}.${.tmp.frac}") + ); + }; + rewrite { + r_set_splunk_dest_update_v2( + sourcetype('cisco:wsa:squid:new') + ); + }; + } else { + }; + }; +}; +application app-netsource-cisco_wsa_11_7[sc4s-network-source] { + filter { + not filter(f_is_source_identified) + and ( + ( + match("cisco", value('.netsource.sc4s_vendor'), type(string)) + and match("wsa11-7", value('.netsource.sc4s_product'), type(string)) + ) + or (tags("ns_vendor:cisco") and tags("ns_product:wsa11-7")) + or tags(".source.s_CISCO_WSA") + or "${.netsource.sc4s_vendor_product}" eq "cisco_wsa11-7" + ); + }; + parser { app-netsource-cisco_wsa_11_7(); }; +}; + diff --git a/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-cisco_wsa_splunk.conf b/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-cisco_wsa_splunk.conf new file mode 100644 index 0000000000..00fc5bfb20 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-cisco_wsa_splunk.conf @@ -0,0 +1,62 @@ + + +block parser app-netsource-cisco_wsa_splunk() { + channel { + rewrite { + r_set_splunk_dest_default( + index('netproxy') + source("cisco_wsa_splunk") + sourcetype('cisco:wsa') + vendor('cisco') + product('wsa') + ); + }; + if{ + filter{ + message('(?:^| )(Info|Warning|Alert|Debug):.*') + }; + rewrite { + r_set_splunk_dest_update_v2( + sourcetype('cisco:wsa:l4tm') + ); + }; + rewrite { + set("$PROGRAM $MSG" value("MSG")); + unset(value("PROGRAM")); + }; + } elif { + parser { + regexp-parser( + prefix(".tmp.") + patterns('^(?:(?\d{10})(?:\.(?\d{1,9}))?)') + ); + date-parser-nofilter(format('%s.%f') + template("${.tmp.epoch}.${.tmp.frac}") + ); + }; + + rewrite { + r_set_splunk_dest_update_v2( + sourcetype('cisco:wsa:w3c:recommended') + ); + }; + } else { + }; + }; +}; +application app-netsource-cisco_wsa_splunk[sc4s-network-source] { + filter { + not filter(f_is_source_identified) + and ( + ( + match("cisco", value('.netsource.sc4s_vendor'), type(string)) + and match("wsa_recommended", value('.netsource.sc4s_product'), type(string)) + ) + or (tags("ns_vendor:cisco") and tags("ns_product:wsa_recommended")) + or tags(".source.s_CISCO_WSA") + or "${.netsource.sc4s_vendor_product}" eq "cisco_wsa_recommended" + ) + }; + parser { app-netsource-cisco_wsa_splunk(); }; +}; + diff --git a/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-clearswift.conf b/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-clearswift.conf new file mode 100644 index 0000000000..035d61be81 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-clearswift.conf @@ -0,0 +1,29 @@ +block parser app-netsource-clearswift() { + channel { + rewrite { + r_set_splunk_dest_default( + index('email') + sourcetype('clearswift:$(lowercase $PROGRAM)') + vendor('clearswift') + class("$PROGRAM") + template("t_msg_only") + ); + }; + }; +}; +application app-netsource-app-netsource-clearswift[sc4s-network-source] { + filter { + not filter(f_is_source_identified) + and ( + ( + match("clearswift", value('.netsource.sc4s_vendor'), type(string)) + and match("clearswift", value('.netsource.sc4s_product'), type(string)) + ) + or (tags("ns_vendor:clearswift") and tags("ns_product:clearswift")) + or tags(".source.s_CLEARSWIFT_CLEARSWIFT") + or "${.netsource.sc4s_vendor_product}" eq "clearswift" + ) + ; + }; + parser { app-netsource-clearswift(); }; +}; diff --git a/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-dell_poweredge_cmc.conf b/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-dell_poweredge_cmc.conf new file mode 100644 index 0000000000..4a82c252e7 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-dell_poweredge_cmc.conf @@ -0,0 +1,32 @@ +block parser app-netsource-dell_poweredge_cmc() { + channel { + + rewrite { + r_set_splunk_dest_default( + index('infraops') + sourcetype('dell:poweredge:cmc:syslog') + vendor('dell') + product('poweredge') + class('cmc') + + ); + }; + }; +}; +application app-netsource-dell_poweredge_cmc[sc4s-network-source] { + filter { + not filter(f_is_source_identified) + and ( + ( + match("dell", value('.netsource.sc4s_vendor'), type(string)) + and match("poweredge_cmc", value('.netsource.sc4s_product'), type(string)) + ) + or tags("ns_vendor:dell") and tags("ns_product:poweredge_cmc") + or tags(".source.s_DELL_POWEREDGE_CMC") + or "${.netsource.sc4s_vendor_product}" eq "dell_poweredge_cmc" + ) + }; + + parser { app-netsource-dell_poweredge_cmc(); }; +}; + diff --git a/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-dell_rsa_secureid.conf b/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-dell_rsa_secureid.conf new file mode 100644 index 0000000000..e46cf06d76 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-dell_rsa_secureid.conf @@ -0,0 +1,98 @@ +block parser app-netsource-dell_rsa_secureid() { + channel { + rewrite { + r_set_splunk_dest_default( + index('netauth') + sourcetype('rsa:securid:trace') + vendor('dell-rsa') + product('secureid') + + ); + }; + + if { + filter{ + message('audit\.|system\.'); + }; + parser { + #basic parsing + + #we need to actual even time from the field GeneratedTime. Use csv-parser to extract it. + csv-parser( + columns("time","ms","host","type") + prefix(".tmp.") + delimiters(',') + ); + #2012/04/10 04:39:55 + #parse the date + date-parser-nofilter(format( + '%Y-%m-%d %H:%M:%S,%f') + template("${LEGACY_MSGHDR} ${.tmp.time},${.tmp.ms}") + ); + }; + rewrite { + r_set_splunk_dest_update_v2( + sourcetype('rsa:securid:syslog') + ); + }; + rewrite { + r_set_splunk_dest_update_v2( + sourcetype('rsa:securid:admin:syslog') + condition(match('audit\.admin' value('.tmp.type'))) + ); + }; + rewrite { + r_set_splunk_dest_update_v2( + sourcetype('rsa:securid:system:syslog') + condition(match('\s*system\.+\w+' value('.tmp.type'))) + ); + }; + rewrite { + r_set_splunk_dest_update_v2( + sourcetype('rsa:securid:runtime:syslog') + condition(match('\s*audit\.runtime\.(com\.rsa|sa\.ims)' value('.tmp.type'))) + ); + }; + } elif { + filter{ + program('...*') + and not program('at' type(string)) + and not program('Caused' type(string)) + }; + rewrite { + subst("^[^\t]+\t", "", value("MESSAGE"), flags("global")); + set("${PROGRAM}", value(".PROGRAM")); + subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM")); + r_set_splunk_dest_update_v2( + sourcetype('nix:syslog') + ); + }; + } else { + rewrite { + r_set_splunk_dest_update_v2( + source('program:${.PROGRAM}') + sourcetype('rsa:securid:trace') + class('trace') + ); + }; + }; + }; +}; +application app-netsource-dell_rsa_secureid[sc4s-network-source] { + filter { + not filter(f_is_source_identified) + and ( + ( + match("dell", value('.netsource.sc4s_vendor'), type(string)) + and match("rsa_secureid", value('.netsource.sc4s_product'), type(string)) + ) + or ( tags("ns_vendor:dell") and tags("ns_product:rsa_secureid")) + or tags(".source.s_DELL_RSA_SECUREID") + or "${.netsource.sc4s_vendor_product}" eq "dell_rsa_secureid" + ) + + + + }; + parser { app-netsource-dell_rsa_secureid(); }; +}; diff --git a/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-dell_sonic.conf b/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-dell_sonic.conf new file mode 100644 index 0000000000..227be6afed --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-dell_sonic.conf @@ -0,0 +1,20 @@ +block parser app-netsource-dell_sonic() { + channel { + rewrite { + r_set_splunk_dest_default( + index('netops') + sourcetype('dell:sonic') + vendor('dell') + product('sonic') + ); + }; + }; +}; +application app-netsource-dell_sonic[sc4s-network-source] { + filter { + ("${.netsource.sc4s_vendor_product}" eq "dell_sonic" + or "${SOURCE}" eq "s_DELL_SONIC") + and "${fields.sc4s_vendor_product}" eq "" + }; + parser { app-netsource-dell_sonic(); }; +}; \ No newline at end of file diff --git a/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-dell_switch_n.conf b/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-dell_switch_n.conf new file mode 100644 index 0000000000..9cf3932efe --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-dell_switch_n.conf @@ -0,0 +1,21 @@ +block parser app-netsource-dell_switch_n() { + channel { + rewrite { + r_set_splunk_dest_default( + index('netops') + sourcetype('dell:emc:powerswitch:n') + vendor('dellemc') + product('powerswitch') + class('n') + ); + }; + }; +}; +application app-netsource-dell_switch_n[sc4s-network-source] { + filter { + ("${.netsource.sc4s_vendor_product}" eq "dellemc_powerswitch_n" + or "${SOURCE}" eq "s_DELLEMC_POWERSWITCH_N") + and "${fields.sc4s_vendor_product}" eq "" + }; + parser { app-netsource-dell_switch_n(); }; +}; \ No newline at end of file diff --git a/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-f5_bigip.conf b/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-f5_bigip.conf new file mode 100644 index 0000000000..09ac6f9d0f --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-f5_bigip.conf @@ -0,0 +1,30 @@ +block parser app-netsource-f5_bigip() { + channel { + rewrite { + r_set_splunk_dest_default( + index("netops") + source("program:${PROGRAM}") + sourcetype('f5:bigip:syslog') + vendor("f5") + product("bigip") + ); + }; + + }; +}; +application app-netsource-f5_bigip[sc4s-network-source] { + filter { + not filter(f_is_source_identified) + and ( + ( + match("f5", value('.netsource.sc4s_vendor'), type(string)) + and match("bigip", value('.netsource.sc4s_product'), type(string)) + ) + or ( tags("ns_vendor:f5") and tags("ns_product:bigip")) + or tags(".source.s_F5_BIGIP") + or "${.netsource.sc4s_vendor_product}" eq "f5_bigip" + ) + ; + }; + parser { app-netsource-f5_bigip(); }; +}; \ No newline at end of file diff --git a/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-ibm_datapower.conf b/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-ibm_datapower.conf new file mode 100644 index 0000000000..98e684e6e6 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-ibm_datapower.conf @@ -0,0 +1,45 @@ +block parser app-netsource-ibm_datapower() { + channel { + rewrite { + r_set_splunk_dest_default( + index('infraops') + sourcetype('ibm:datapower:syslog') + vendor('ibm') + product('datapower') + template('t_msg_only') + ); + }; + # + if { + parser { + regexp-parser( + prefix(".tmp.") + patterns('\[[^\]]*\]\[(?[^\]]*)\]\[[^\]]*\]\s') + ); + }; + + rewrite{ + r_set_splunk_dest_update_v2( + sourcetype('ibm:datapower:${.tmp.category}') + class("${.tmp.category}") + ); + }; + }; + }; +}; + +application app-netsource-ibm_datapower[sc4s-network-source] { + filter { + not filter(f_is_source_identified) + and ( + ( + match("ibm", value('.netsource.sc4s_vendor'), type(string)) + and match("datapower", value('.netsource.sc4s_product'), type(string)) + ) + or (tags("ns_vendor:ibm") and tags("ns_product:datapower")) + or tags(".source.s_IBM_DATAPOWER") + or "${.netsource.sc4s_vendor_product}" eq "ibm_datapower" + ) + }; + parser { app-netsource-ibm_datapower(); }; +}; diff --git a/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-infoblox.conf b/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-infoblox.conf new file mode 100644 index 0000000000..b8547bfe1d --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-infoblox.conf @@ -0,0 +1,85 @@ +block parser app-netsource-infoblox() { + channel { + rewrite { + set("${PROGRAM}", value(".PROGRAM")); + subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM")); + r_set_splunk_dest_default( + index('netops') + source('program:${.PROGRAM}') + sourcetype('infoblox') + vendor("infoblox") + product("nios") + template('t_hdr_msg') + ); + }; + + if { + filter{program('named' type(string) flags(prefix))}; + rewrite { + r_set_splunk_dest_update_v2( + index('netdns') + class('dns') + source('program:${PROGRAM}') + sourcetype('infoblox:dns') + ); + }; + } elif { + filter{program('dhcpd' type(string) flags(prefix))}; + rewrite { + r_set_splunk_dest_update_v2( + index('netipam') + class('dhcp') + source('program:${PROGRAM}') + sourcetype('infoblox:dhcp') + ); + }; + } elif { + filter{program('threat-protect-log' type(string) flags(prefix))}; + rewrite { + r_set_splunk_dest_update_v2( + index('netids') + class('threatprotect') + source('program:${PROGRAM}') + sourcetype('infoblox:threatprotect') + ); + }; + } elif { + filter{message('^\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2}:\d{2}(\.\d+)Z\s+\[[^\]]+\]:')}; + rewrite { + r_set_splunk_dest_update_v2( + class('audit') + source('program:${PROGRAM}') + sourcetype('infoblox:audit') + ); + }; + } else { + rewrite { + r_set_splunk_dest_update_v2( + class('fallback') + source('program:${PROGRAM}') + sourcetype('infoblox:port') + ); + }; + rewrite { + subst("^[^\t]+\t", "", value("MESSAGE"), flags("global")); + }; + }; + + + }; +}; +application app-netsource-infoblox[sc4s-network-source] { + filter { + ( + ( + match("infoblox", value('.netsource.sc4s_vendor'), type(string)) + and match("nios", value('.netsource.sc4s_product'), type(string)) + ) + or ( tags("ns_vendor:infoblox") and tags("ns_product:nios")) + or tags(".source.s_INFOBLOX") + or "${.netsource.sc4s_vendor_product}" eq "infoblox_nios" + ) + and not message('CEF:0', type('string') flags(prefix)) + }; + parser { app-netsource-infoblox(); }; +}; diff --git a/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-mikrotik_routeros.conf b/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-mikrotik_routeros.conf new file mode 100644 index 0000000000..cca8e71010 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-mikrotik_routeros.conf @@ -0,0 +1,41 @@ +block parser app-netsource-mikrotik_routeros() { + channel { + rewrite { + r_set_splunk_dest_default( + index('netops') + source('program:${.PROGRAM}') + sourcetype('routeros') + vendor("mikrotik") + product("routeros") + template('t_hdr_msg') + ); + }; + + if { + filter{program('forward' type(string) flags(prefix))}; + rewrite { + r_set_splunk_dest_update_v2( + index('netfw') + class('fw') + ); + }; + }; + + + }; +}; +application app-netsource-mikrotik_routeros[sc4s-network-source] { + filter { + not filter(f_is_source_identified) + and ( + ( + match("mikrotik", value('.netsource.sc4s_vendor'), type(string)) + and match("routeros", value('.netsource.sc4s_product'), type(string)) + ) + or (tags("ns_vendor:mikrotik") and tags("ns_product:routeros")) + or tags(".source.s_MIKROTIK_ROUTEROS") + or "${.netsource.sc4s_vendor_product}" eq "mikrotik_routeros" + ) + }; + parser { app-netsource-mikrotik_routeros(); }; +}; diff --git a/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-netapp_ontap.conf b/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-netapp_ontap.conf new file mode 100644 index 0000000000..08cd806164 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-netapp_ontap.conf @@ -0,0 +1,55 @@ +block parser app-netsource-netapp_ontap() { + channel { + rewrite { + r_set_splunk_dest_default( + index("infraops") + vendor("netapp") + product("ontap") + ); + }; + + if { + parser { + regexp-parser( + prefix(".tmp.") + patterns('^[A-Za-z0-9\-\_\.]+: [0-9a-f]+\.[0-9a-f]+ [0-9a-f]+ [A-Z][a-z][a-z] (?[A-Z][a-z][a-z] \d\d \d\d\d\d \d\d:\d\d:\d\d [+-]?\d{1,2}:\d\d)') + ); + date-parser-nofilter( + format( + '%b %d %Y %H:%M:%S %z', + ) + template("${.tmp.timestamp}") + ); + }; + + rewrite { + set('$PROGRAM: $MESSAGE', value(MESSAGE)); + set('$PROGRAM', value(HOST)); + unset(value(PROGRAM)); + }; + + rewrite { + r_set_splunk_dest_update_v2( + sourcetype('netapp:ontap:audit') + class('audit') + ); + }; + } else { + rewrite { + r_set_splunk_dest_update_v2( + sourcetype('netapp:ontap:ems') + class('ems') + ); + }; + }; + }; +}; + +application app-netsource-netapp_ontap[sc4s-network-source] { + filter { + match("netapp", value('.netsource.sc4s_vendor'), type(string)) + and match("ontap", value('.netsource.sc4s_product'), type(string)) + and "`SC4S_NETAPP_ONTAP_NEW_FORMAT`" eq "yes" + }; + parser { app-netsource-netapp_ontap(); }; +}; diff --git a/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-nix_syslog.conf b/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-nix_syslog.conf new file mode 100644 index 0000000000..7666e09344 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-nix_syslog.conf @@ -0,0 +1,32 @@ +block parser app-netsource-nix_syslog() { + channel { + rewrite { + set("${PROGRAM}", value(".PROGRAM")); + subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM")); + r_set_splunk_dest_default( + index('osnix') + source('program:${.PROGRAM}') + sourcetype('nix:syslog') + vendor('nix') + product('syslog') + ); + + }; + + }; +}; +application app-netsource-nix_syslog[sc4s-network-source] { + filter { + not filter(f_is_source_identified) + and ( + ( + match("nix", value('.netsource.sc4s_vendor'), type(string)) + and match("syslog", value('.netsource.sc4s_product'), type(string)) + ) + or (tags("ns_vendor:nix") and tags("ns_product:syslog")) + or tags(".source.s_NIX_SYSLOG") + or "${.netsource.sc4s_vendor_product}" eq "nix_syslog" + ) + }; + parser { app-netsource-nix_syslog(); }; +}; diff --git a/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-pfsense.conf b/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-pfsense.conf new file mode 100644 index 0000000000..0ec0e349f7 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-pfsense.conf @@ -0,0 +1,42 @@ + +block parser app-netsource-pfsense() { + channel { + rewrite { + set("${PROGRAM}", value(".PROGRAM")); + subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM")); + r_set_splunk_dest_default( + index('netops') + source('program:${.PROGRAM}') + sourcetype('pfsense:${PROGRAM}') + vendor("pfsense") + product("firewall") + template('t_hdr_msg') + ); + + r_set_splunk_dest_update_v2( + index('netfw') + sourcetype('pfsense:filterlog') + class('filterlog') + condition( program('filterlog' type(string) flags(prefix)) ) + ); + + }; + + + }; +}; +application app-netsource-pfsense[sc4s-network-source] { + filter { + not filter(f_is_source_identified) + and ( + ( + match("pfsense", value('.netsource.sc4s_vendor'), type(string)) + and match("firewall", value('.netsource.sc4s_product'), type(string)) + ) + or (tags("ns_vendor:pfsense") and tags("ns_product:firewall")) + or tags(".source.s_PFSENSE") + or "${.netsource.sc4s_vendor_product}" eq "pfsense_firewall" + ) + }; + parser { app-netsource-pfsense(); }; +}; diff --git a/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-proofpoint_pps.conf b/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-proofpoint_pps.conf new file mode 100644 index 0000000000..368ba59ed7 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-proofpoint_pps.conf @@ -0,0 +1,56 @@ + +filter f_proofpoint_pps_filter { + match('^(background|cvt|filter|pps)_instance\d+$' value("PROGRAM") type("pcre")) + or match('^\/opt\/proofpoint\/pps-\d\.\d\.\d\.\d+\/\S' value("PROGRAM") type("pcre")) + or match('^queued-(alert|default|digest|reinject|released)$' value("PROGRAM") type("pcre")) + ; +}; + +block parser app-netsource-proofpoint_pps() { + channel { + + rewrite { + set("${PROGRAM}", value(".PROGRAM")); + subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM")); + r_set_splunk_dest_default( + index('email') + sourcetype('pps_mail_log') + vendor("proofpoint") + product("pps") + template('t_hdr_msg') + ); + }; + + rewrite { + r_set_splunk_dest_update_v2( + class('sendmail') + ); + }; + rewrite { + r_set_splunk_dest_update_v2( + class('filter') + sourcetype('pps_filter_log') + condition(filter(f_proofpoint_pps_filter)) + ); + }; + }; +}; +application app-netsource-proofpoint_pps[sc4s-network-source] { + filter { + ( + ( + match("proofpoint", value('.netsource.sc4s_vendor'), type(string)) + and match("pps", value('.netsource.sc4s_product'), type(string)) + ) + or ( tags("ns_vendor:proofpoint") and tags("ns_product:pps")) + or tags(".source.s_PROOFPOINT_PPS") + or "${.netsource.sc4s_vendor_product}" eq "proofpoint_pps" + ) + or ( + match('^(background|cvt|filter|pps)_instance\d+$' value("PROGRAM") type("pcre")) or + match('^\/opt\/proofpoint\/pps-\d\.\d\.\d\.\d+\/\S' value("PROGRAM") type("pcre")) or + match('^queued-(alert|default|digest|reinject|released)$' value("PROGRAM") type("pcre")) + ) + }; + parser { app-netsource-proofpoint_pps(); }; +}; diff --git a/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-raritan_dsx.conf b/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-raritan_dsx.conf new file mode 100644 index 0000000000..114c032436 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-raritan_dsx.conf @@ -0,0 +1,28 @@ +block parser app-netsource-raritan_dsx() { + channel { + rewrite { + r_set_splunk_dest_default( + index('infraops') + sourcetype('raritan:dsx') + vendor('raritan') + product('dsx') + template('t_hdr_msg') + ); + }; + }; +}; +application app-netsource-raritan_dsx[sc4s-network-source] { + filter { + not filter(f_is_source_identified) + and ( + ( + match("raritan", value('.netsource.sc4s_vendor'), type(string)) + and match("dsx", value('.netsource.sc4s_product'), type(string)) + ) + or (tags("ns_vendor:raritan") and tags("ns_product:dsx")) + or tags(".source.s_RARITAN_DSX") + or "${.netsource.sc4s_vendor_product}" eq "raritan_dsx" + ) + }; + parser { app-netsource-raritan_dsx(); }; +}; diff --git a/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-riverbed_syslog.conf b/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-riverbed_syslog.conf new file mode 100644 index 0000000000..d1bd12823a --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-riverbed_syslog.conf @@ -0,0 +1,62 @@ +block parser app-netsource-riverbed_syslog() { + channel { + rewrite { + r_set_splunk_dest_default( + index('netops') + sourcetype('riverbed:syslog') + vendor('riverbed') + product('syslog') + ); + }; + + rewrite { + r_set_splunk_dest_update_v2( + sourcetype('riverbed:steelhead') + class("steelhead") + condition( + program('alarmd', type(string)) + or program('cifs', type(string)) + or program('cmcfc', type(string)) + or program('rgp', type(string)) + or program('rgpd', type(string)) + or program('cli', type(string)) + or program('mgmtd', type(string)) + or program('http', type(string)) + or program('hald', type(string)) + or program('notes', type(string)) + or program('mapi', type(string)) + or program('nfs', type(string)) + or program('pm', type(string)) + or program('sched', type(string)) + or program('virtwrapperd', type(string)) + or program('vspd', type(string)) + or program('statsd', type(string)) + or program('wdt', type(string)) + or program('webasd', type(string)) + or program('domain', type(string) flags(prefix)) + or program('sport', type(string)) + ) + ); + }; + + + }; +}; +application app-netsource-riverbed_syslog[sc4s-network-source] { + filter { + not filter(f_is_source_identified) + and ( + ( + match("riverbed", value('.netsource.sc4s_vendor'), type(string)) + and match("syslog", value('.netsource.sc4s_product'), type(string)) + ) + and (tags("ns_vendor:syslog") and tags("ns_product:syslog")) + or tags(".source.s_RIVERBED_SYSLOG") + or "${.netsource.sc4s_vendor_product}" eq "riverbed_syslog" + ) + ; + }; + parser { app-netsource-riverbed_syslog(); }; +}; + + diff --git a/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-schneider_apc.conf b/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-schneider_apc.conf new file mode 100644 index 0000000000..e883fd3d93 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-schneider_apc.conf @@ -0,0 +1,30 @@ +block parser app-netsource-schneider_apc() { + channel { + rewrite { + r_set_splunk_dest_default( + index('main') + sourcetype('apc:syslog') + vendor('schneider') + product('apc') + template('t_hdr_msg') + ); + }; + + + }; +}; +application app-netsource-schneider_apc[sc4s-network-source] { + filter { + not filter(f_is_source_identified) + and ( + ( + match("schneider", value('.netsource.sc4s_vendor'), type(string)) + and match("apc", value('.netsource.sc4s_product'), type(string)) + ) + or (tags("ns_vendor:schneider") and tags("ns_product:apc")) + or tags(".source.s_SCHNEIDER_APC") + or "${.netsource.sc4s_vendor_product}" eq "schneider_apc" + ) + }; + parser { app-netsource-schneider_apc(); }; +}; diff --git a/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-simple_source.conf b/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-simple_source.conf new file mode 100644 index 0000000000..5ac832f7f3 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-simple_source.conf @@ -0,0 +1,32 @@ +block parser app-netsource-simple_source() { + channel { + rewrite { + set("${PROGRAM}", value(".PROGRAM")); + subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM")); + set('$SOURCE' value('.simple_dest_key')); + subst('^s_SIMPLE_', "" , value(".simple_dest_key")); + }; + parser { + regexp-parser( + prefix(".tmp.") + patterns('^(?[^_]+)_(?.+)') + template('${.simple_dest_key}') + ); + }; + rewrite { + r_set_splunk_dest_default( + source('program:${.PROGRAM}') + sourcetype('sc4s:simple:$(lowercase ${.simple_dest_key})') + vendor("$(lowercase ${.tmp.vendor})") + product("$(lowercase ${.tmp.product})") + ); + }; + }; +}; +application app-netsource-simple_source[sc4s-network-source] { + filter { + not filter(f_is_source_identified) + and match('^s_SIMPLE_', value('SOURCE')) + }; + parser { app-netsource-simple_source(); }; +}; \ No newline at end of file diff --git a/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-sophos_webappliance.conf b/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-sophos_webappliance.conf new file mode 100644 index 0000000000..ae22601df5 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-sophos_webappliance.conf @@ -0,0 +1,40 @@ +block parser app-netsource-sophos_webappliance() { + channel { + rewrite { + r_set_splunk_dest_default( + index("netproxy") + sourcetype('sophos:webappliance') + vendor("sophos") + product("webappliance") + ); + }; + rewrite{ + subst(' [^=]+=(?:"-"|-)', '' flags(global)); + }; + parser { + kv-parser( + prefix(".values.") + pair-separator(" ") + ); + }; + parser { + date-parser-nofilter(format("%s") template("${.values.t}")); + }; + }; +}; +application app-netsource-sophos_webappliance[sc4s-network-source] { + filter { + not filter(f_is_source_identified) + and ( + ( + match("sophos", value('.netsource.sc4s_vendor'), type(string)) + and match("webappliance", value('.netsource.sc4s_product'), type(string)) + ) + or (tags("ns_vendor:sophos") and tags("ns_product:webappliance")) + or tags(".source.s_SCHNEIDER_APC") + or "${.netsource.sc4s_vendor_product}" eq "sophos_webappliance" + ) + ; + }; + parser { app-netsource-sophos_webappliance(); }; +}; \ No newline at end of file diff --git a/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-spectracom-ntp.conf b/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-spectracom-ntp.conf new file mode 100644 index 0000000000..a14919efe5 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-spectracom-ntp.conf @@ -0,0 +1,33 @@ +block parser app-netsource-spectracom-ntp() { + channel { + rewrite{ + set('$PROGRAM' value('HOST')); + unset(value('PROGRAM')); + }; + rewrite { + r_set_splunk_dest_default( + index("netops") + sourcetype('spectracom:ntp') + vendor("spectracom") + product("ntp") + ); + }; + }; +}; +application app-netsource-spectracom-ntp[sc4s-network-source] { + filter { + not filter(f_is_source_identified) + and ( + ( + match("spectracom", value('.netsource.sc4s_vendor'), type(string)) + and match("ntp", value('.netsource.sc4s_product'), type(string)) + ) + or (tags("ns_vendor:spectracom") and tags("ns_product:ntp")) + or tags(".source.s_SPECTRACOM_NTP") + or "${.netsource.sc4s_vendor_product}" eq "spectracom_ntp" + ) + and message('^\[[^\]]+\] ') + ; + }; + parser { app-netsource-spectracom-ntp(); }; +}; \ No newline at end of file diff --git a/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-symantec_dlp.conf b/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-symantec_dlp.conf new file mode 100644 index 0000000000..24a8554040 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-symantec_dlp.conf @@ -0,0 +1,40 @@ +block parser app-netsource-symantec_dlp() { + channel { + rewrite { + r_set_splunk_dest_default( + index('netdlp') + sourcetype('symantec:dlp:system') + vendor("symantec") + product('dlp') + template('t_msg_only') + ); + }; + if { + filter { + message(' Incident_id=', type(string) flags(substring, ignore-case)) + }; + rewrite { + r_set_splunk_dest_update_v2( + sourcetype('symantec:dlp:syslog') + class('alert') + ); + }; + }; + + }; +}; +application app-netsource-symantec_dlp[sc4s-network-source] { + filter { + not filter(f_is_source_identified) + and ( + ( + match("symantec", value('.netsource.sc4s_vendor'), type(string)) + and match("dlp", value('.netsource.sc4s_product'), type(string)) + ) + or (tags("ns_vendor:symantec") and tags("ns_product:dlp")) + or tags(".source.s_SYMANTEC_DLP") + or "${.netsource.sc4s_vendor_product}" eq "symantec_dlp" + ) + }; + parser { app-netsource-symantec_dlp(); }; +}; diff --git a/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-ubiquiti_unifi.conf b/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-ubiquiti_unifi.conf new file mode 100644 index 0000000000..839a2b1a72 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-ubiquiti_unifi.conf @@ -0,0 +1,58 @@ +block parser app-netsource-ubiquiti_unifi() { + channel { + rewrite { + r_set_splunk_dest_default( + index('netfw') + sourcetype('ubnt:fw') + vendor("ubiquiti") + product("unifi") + ); + set("${LEGACY_MSGHDR}${MSG}" value("MSG")); + unset(value("PROGRAM")); + }; + + if (match("[^)]\s\S+\skernel:\s[^ll\sheader][^\[\d+.\d+\]]\S+\s\w+:" value("MSG"))) { + rewrite { + r_set_splunk_dest_update_v2( + class('threat') + index('netids') + sourcetype('ubnt:threat') + ); + }; + } elif (match("\S+\slinkcheck:" value("MSG"))) { + rewrite { + r_set_splunk_dest_update_v2( + class('link') + index('netops') + sourcetype('ubnt:link') + ); + }; + } elif (match("\d+:\d+:\d+\s\S+\ssudo:" value("MSG"))) { + rewrite { + r_set_splunk_dest_update_v2( + class('sudo') + index('netops') + sourcetype('ubnt:sudo') + ); + }; + } else { + }; + + + }; +}; +application app-netsource-ubiquiti_unifi[sc4s-network-source] { + filter { + not filter(f_is_source_identified) + and ( + ( + match("ubiquiti", value('.netsource.sc4s_vendor'), type(string)) + and match("unifi", value('.netsource.sc4s_product'), type(string)) + ) + or (tags("ns_vendor:ubiquiti") and tags("ns_product:unifi")) + or tags(".source.s_UBIQUITI_UNIFI") + or "${.netsource.sc4s_vendor_product}" eq "ubiquiti_unifi" + ) + }; + parser { app-netsource-ubiquiti_unifi(); }; +}; diff --git a/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-vmware_esx_invalidmultiline.conf b/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-vmware_esx_invalidmultiline.conf new file mode 100644 index 0000000000..352afb64fb --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-vmware_esx_invalidmultiline.conf @@ -0,0 +1,41 @@ +block parser app-netsource-vmware_esx_invalidmultiline() { + channel { + rewrite { + r_set_splunk_dest_default( + index('infraops') + source('vmware:esxlog:$(lowercase ${PROGRAM})') + sourcetype('vmware:esxlog:$(lowercase ${PROGRAM})') + vendor("vmware") + product("vsphere") + class("esx") + ); + set("t_msg_trim", value(".splunk.sc4s_template")); + }; + }; +}; +application app-netsource-vmware_esx_invalidmultiline[sc4s-network-source] { + filter { + not filter(f_is_source_identified) + and ( + ( + match("vmware", value('.netsource.sc4s_vendor'), type(string)) + and match("vsphere", value('.netsource.sc4s_product'), type(string)) + ) + or (tags("ns_vendor:vmware") + and ( + tags("ns_product:esx") or tags("ns_product:vcenter") + ) + ) + or tags(".source.s_VMWARE_VCENTER") + or tags(".source.s_VMWARE_VSPHERE") + or tags(".source.s_VMWARE_ESX") + or "${.netsource.sc4s_vendor_product}" eq "vmware_vsphere" + or "${.netsource.sc4s_vendor_product}" eq "vmware_vcenter" + or "${.netsource.sc4s_vendor_product}" eq "vmware_esx" + ) + and ("$MESSAGE" eq "" or message('^-->')) + ; + + }; + parser { app-netsource-vmware_esx_invalidmultiline(); }; +}; \ No newline at end of file diff --git a/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-vmware_esx_json.conf b/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-vmware_esx_json.conf new file mode 100644 index 0000000000..cf9f09d622 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-vmware_esx_json.conf @@ -0,0 +1,43 @@ +block parser app-netsource-vmware_esx_json() { + channel { + rewrite { + r_set_splunk_dest_default( + index('infraops') + sourcetype('vmware:vcenter:json') + vendor("vmware") + product('vsphere') + class('esx-json') + ); + set("t_msg_trim", value(".splunk.sc4s_template")); + }; + parser { + json-parser( + prefix('.values.') + ); + }; + }; +}; +application app-netsource-vmware_esx_json[sc4s-network-source] { + filter { + not filter(f_is_source_identified) + and ( + ( + match("vmware", value('.netsource.sc4s_vendor'), type(string)) + and match("vsphere", value('.netsource.sc4s_product'), type(string)) + ) + or (tags("ns_vendor:vmware") + and ( + tags("ns_product:esx") or tags("ns_product:vsphere") + ) + ) + or tags(".source.s_VMWARE_VCENTER") + or tags(".source.s_VMWARE_VSPHERE") + or tags(".source.s_VMWARE_ESX") + or "${.netsource.sc4s_vendor_product}" eq "vmware_vsphere" + or "${.netsource.sc4s_vendor_product}" eq "vmware_vcenter" + or "${.netsource.sc4s_vendor_product}" eq "vmware_esx" + ) + and message('^{') + }; + parser { app-netsource-vmware_esx_json(); }; +}; diff --git a/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-vmware_vsphere.conf b/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-vmware_vsphere.conf new file mode 100644 index 0000000000..e19d7ec3ae --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/netsource/app-netsource-vmware_vsphere.conf @@ -0,0 +1,88 @@ +filter syslog-netsource-vmware_vsphere-vcenter-pgm{ + #Used by TA + program("trust", type(string) flags(ignore-case,prefix)) + or program("analytics", type(string) flags(ignore-case,prefix)) + or program("ui-", type(string) flags(ignore-case,prefix)) + or program("vum", type(string) flags(ignore-case,prefix)) + or program("sso", type(string) flags(ignore-case,prefix)) + or program("sca", type(string) flags(ignore-case,prefix)) + or program("content-library", type(string) flags(ignore-case)) + or program("lookup", type(string) flags(ignore-case,prefix)) + or program("envoy", type(string) flags(ignore-case,prefix)) + or program("vapi", type(string) flags(ignore-case,prefix)) + or program("observ", type(string) flags(ignore-case,prefix)) + or program("sps", type(string) flags(ignore-case,prefix)) + or program("firewall-pktlog", type(string) flags(ignore-case,prefix)) + + # Requested by community + or program("applmgmt", type(string) flags(ignore-case,prefix)) + or program("certificatemanagement-svcs", type(string) flags(ignore-case)) + or program("cis-license", type(string) flags(ignore-case)) + or program("dnsmasq", type(string) flags(ignore-case)) + or program("eam-", type(string) flags(ignore-case,prefix)) + or program("gclog", type(string) flags(ignore-case)) + or program("perfcharts-localhost_access", type(string) flags(ignore-case)) + or program("postgres-archiver", type(string) flags(ignore-case)) + or program("procstate", type(string) flags(ignore-case)) + or program("rsyslogd", type(string) flags(ignore-case)) + or program("updatemgr", type(string) flags(ignore-case)) + or program("vmafdd", type(string) flags(ignore-case)) + or program("vmcad", type(string) flags(ignore-case)) + or program("vmdird", type(string) flags(ignore-case)) + or program("vmon", type(string) flags(ignore-case)) + or program("vpxd-", type(string) flags(ignore-case,prefix)) + or program("vsan-health-main", type(string) flags(ignore-case)) + or program("vsm-main", type(string) flags(ignore-case)) + or program("vstats", type(string) flags(ignore-case)) + or program("wcpsvc", type(string) flags(ignore-case)) +}; + +block parser app-netsource-vmware_vsphere() { + channel { + rewrite { + r_set_splunk_dest_default( + index('infraops') + sourcetype('vmware:esxlog:$(lowercase ${PROGRAM})') + vendor("vmware") + product('vsphere') + class('esx') + ); + }; + + rewrite { + r_set_splunk_dest_update( + source('vmware:vclog:$(lowercase ${PROGRAM})') + sourcetype('vmware:vclog:$(lowercase ${PROGRAM})') + class("vc") + condition(filter(syslog-netsource-vmware_vsphere-vcenter-pgm)) + ); + }; + + }; +}; +application app-netsource-vmware_vsphere[sc4s-network-source] { + filter { + not filter(f_is_source_identified) + and ( + ( + match("vmware", value('.netsource.sc4s_vendor'), type(string)) + and match("vsphere", value('.netsource.sc4s_product'), type(string)) + ) + or (tags("ns_vendor:vmware") + and ( + tags("ns_product:esx") or tags("ns_product:vsphere") + ) + ) + or tags(".source.s_VMWARE_VCENTER") + or tags(".source.s_VMWARE_VSPHERE") + or tags(".source.s_VMWARE_ESX") + or "${.netsource.sc4s_vendor_product}" eq "vmware_vsphere" + or "${.netsource.sc4s_vendor_product}" eq "vmware_vcenter" + or "${.netsource.sc4s_vendor_product}" eq "vmware_esx" + ) + and ( + filter(syslog-netsource-vmware_vsphere-vcenter-pgm) + ) + }; + parser { app-netsource-vmware_vsphere(); }; +}; diff --git a/package/enterprise/etc/conf.d/conflib/post-filter/app-postfilter-beyondtrust_sra.conf b/package/enterprise/etc/conf.d/conflib/post-filter/app-postfilter-beyondtrust_sra.conf new file mode 100644 index 0000000000..546e0c3986 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/post-filter/app-postfilter-beyondtrust_sra.conf @@ -0,0 +1,43 @@ +template t_beyondtrust_sra { + template("${.values.siteid}:${.values.seq}:${values.num}:${.gb.message}"); +}; + +block parser app-postfilter-beyondtrust_sra() { + channel { + + parser{ + grouping-by( + scope(program) + key("${.values.siteid}") + trigger("$(context-length)" >= "${.metadata.sc4s.countOfParts}") + sort-key("${.values.seq}") + aggregate( + tags("agg") + value(".gb.message" "$(implode ';' $(list-slice 0:-1 $(context-values ${.message})))") + value(".values.siteid", "00") + value(".metadata.sc4s.countOfParts", "${.values.siteid}@1") + value("PROGRAM" "${PROGRAM}@1") + value(".splunk.sc4s_template", "t_beyondtrust_sra") + value(".metadata.sc4s.countOfParts", "$(context-length)") + tags("isStructured") + inherit-mode(context) + ) + timeout(10) + ); + }; + + if { + filter{ not filter(f_is_agg) }; + rewrite(r_set_dest_splunk_null_queue); + }; + + + }; +}; +application app-postfilter-beyondtrust_sra[sc4s-postfilter] { + filter { + program('BG' type(string)) + and "${.metadata.sc4s.countOfParts}" > 1; + }; + parser { app-postfilter-beyondtrust_sra(); }; +}; diff --git a/package/enterprise/etc/conf.d/conflib/post-filter/app-postfilter-checkpoint_splunk_01-partials.conf b/package/enterprise/etc/conf.d/conflib/post-filter/app-postfilter-checkpoint_splunk_01-partials.conf new file mode 100644 index 0000000000..d90184e22b --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/post-filter/app-postfilter-checkpoint_splunk_01-partials.conf @@ -0,0 +1,19 @@ +block parser app-postfilter-checkpoint_splunk_01-partials() { + channel { + rewrite(r_set_dest_splunk_null_queue); + }; +}; + +application app-postfilter-checkpoint_splunk_01-partials[sc4s-postfilter] { + filter { + match('checkpoint', value('fields.sc4s_vendor') type(string)) + and match('splunk', value('fields.sc4s_product') type(string)) + and "`SC4S_LISTEN_CHECKPOINT_SPLUNK_NOISE_INCOMPLETE_EVENTS`" eq "yes" + and "${.values.loguid}" ne "" + and "${.values.bytes}" eq "" + and ( "${.values.product}" eq "Application Control" or "${.values.product}" eq "Firewall" or "${.values.product}" eq "URL Filtering") + and ( "${.values.rule_action}" eq "Accept" or "${.values.rule_action}" eq "Inline") + ; + }; + parser { app-postfilter-checkpoint_splunk_01-partials(); }; +}; \ No newline at end of file diff --git a/package/enterprise/etc/conf.d/conflib/post-filter/app-postfilter-checkpoint_splunk_02-group.conf b/package/enterprise/etc/conf.d/conflib/post-filter/app-postfilter-checkpoint_splunk_02-group.conf new file mode 100644 index 0000000000..946d508578 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/post-filter/app-postfilter-checkpoint_splunk_02-group.conf @@ -0,0 +1,32 @@ +block parser app-postfilter-checkpoint_splunk_02-group() { + channel { + parser { + grouping-by( + key("${.values.loguid}") + #This looks silly but we have no way of knowing if an event is complete so + #We must make an impossible condition and rely on time out + trigger(1 == 2) + aggregate( + tags("agg") + inherit-mode(last-message) + ) + timeout(`SC4S_LISTEN_CHECKPOINT_SPLUNK_NOISE_CONTROL_SECONDS`) + ); + }; + if { + filter{ not filter(f_is_agg) }; + rewrite(r_set_dest_splunk_null_queue); + }; + + }; +}; + +application app-postfilter-checkpoint_splunk_02-group[sc4s-postfilter] { + filter { + match('checkpoint', value('fields.sc4s_vendor') type(string)) + and match('splunk', value('fields.sc4s_product') type(string)) + and "`SC4S_LISTEN_CHECKPOINT_SPLUNK_NOISE_CONTROL`" eq "yes" + and "${.values.loguid}" ne ""; + }; + parser { app-postfilter-checkpoint_splunk_02-group(); }; +}; diff --git a/package/enterprise/etc/conf.d/conflib/post-filter/app-postfilter-checkpoint_syslog_01-partials.conf b/package/enterprise/etc/conf.d/conflib/post-filter/app-postfilter-checkpoint_syslog_01-partials.conf new file mode 100644 index 0000000000..3b7ac11c4b --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/post-filter/app-postfilter-checkpoint_syslog_01-partials.conf @@ -0,0 +1,19 @@ +block parser app-postfilter-checkpoint_syslog_01-partials() { + channel { + rewrite(r_set_dest_splunk_null_queue); + + }; +}; + +application app-postfilter-checkpoint_syslog_01-partials[sc4s-postfilter] { + filter { + program('CheckPoint' type(string)) + and "`SC4S_LISTEN_CHECKPOINT_SPLUNK_NOISE_INCOMPLETE_EVENTS`" eq "yes" + and ("${.SDATA.sc4s@2620.loguid}" ne "" + and "${.SDATA.sc4s@2620.bytes}" eq "" + and ( "${.SDATA.sc4s@2620.product}" eq "Application Control" or "${.SDATA.sc4s@2620.product}" eq "Firewall" or "${.SDATA.sc4s@2620.product}" eq "URL Filtering") + and ( "${.SDATA.sc4s@2620.rule_action}" eq "Accept" or "${.SDATA.sc4s@2620.rule_action}" eq "Inline") + ); + }; + parser { app-postfilter-checkpoint_syslog_01-partials(); }; +}; \ No newline at end of file diff --git a/package/enterprise/etc/conf.d/conflib/post-filter/app-postfilter-checkpoint_syslog_02-group.conf b/package/enterprise/etc/conf.d/conflib/post-filter/app-postfilter-checkpoint_syslog_02-group.conf new file mode 100644 index 0000000000..77b11304df --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/post-filter/app-postfilter-checkpoint_syslog_02-group.conf @@ -0,0 +1,32 @@ +block parser app-postfilter-checkpoint_syslog_02-group() { + channel { + + parser { + grouping-by( + key("${.SDATA.sc4s@2620.loguid}") + #This looks silly but we have no way of knowing if an event is complete so + #We must make an impossible condition and rely on time out + trigger(1 == 2) + aggregate( + tags("agg") + inherit-mode(last-message) + ) + timeout(`SC4S_LISTEN_CHECKPOINT_SPLUNK_NOISE_CONTROL_SECONDS`) + ); + }; + + if { + filter{ not filter(f_is_agg) }; + rewrite(r_set_dest_splunk_null_queue); + }; + + }; +}; +application app-postfilter-checkpoint_syslog_02-group[sc4s-postfilter] { + filter { + program('CheckPoint' type(string)) + and "`SC4S_LISTEN_CHECKPOINT_SPLUNK_NOISE_CONTROL`" eq "yes" + and "${.SDATA.sc4s@2620.loguid}" ne ""; + }; + parser { app-postfilter-checkpoint_syslog_02-group(); }; +}; diff --git a/package/enterprise/etc/conf.d/conflib/post-filter/app-postfilter-cisco_acs.conf b/package/enterprise/etc/conf.d/conflib/post-filter/app-postfilter-cisco_acs.conf new file mode 100644 index 0000000000..d3897c0f83 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/post-filter/app-postfilter-cisco_acs.conf @@ -0,0 +1,64 @@ + +parser p_acs_event_time_multi { + csv-parser( + columns(DATE, TIME, TZ) + prefix(".tmp.") + delimiters(chars(" ")) + template('${.tmp.date_seg}') + ); + + date-parser-nofilter( + #YYYY- MM-DD hh:mm:ss:xxx +/-zh:zm + format('%Y-%m-%d %H:%M:%S.%f %z') + template("${.tmp.DATE} ${.tmp.TIME} ${.tmp.TZ}") + ); +}; + +template t_acs_message { + template("${PROGRAM} ${.values.serial} 1 0 ${.gb.message}"); +}; + +block parser app-postfilter-cisco_acs() { + channel { + + parser{ + grouping-by( + scope(program) + key("${.values.serial}") + trigger("$(context-length)" >= "${.values.num}") + sort-key("${.values.seq}") + aggregate( + tags("agg") + #value(".gb.message" "$(context-lookup ("1" eq "1" ) ${.values.message})") + value(".gb.message" "$(implode '' $(list-slice 0:-1 $(context-values ${.values.message})))") + value("PROGRAM" "${PROGRAM}@1") + value(".values.serial" "${.values.serial}@1") + value(".tmp.date_seg" "$(list-head $(context-values ${.values.message}))") + value(".splunk.sc4s_template", "t_acs_message") + value(".metadata.sc4s.countOfParts", "$(context-length)") + inherit-mode(context) + ) + timeout(10) + ); + }; + + if { + parser(p_acs_event_time_multi); + }; + + if { + filter{ not filter(f_is_agg) }; + rewrite(r_set_dest_splunk_null_queue); + }; + + + + }; +}; +application app-postfilter-cisco_acs[sc4s-postfilter] { + filter { + program('CSCOacs' type(string) flags(prefix)) + and "${.values.num}" > 1; + }; + parser { app-postfilter-cisco_acs(); }; +}; diff --git a/package/enterprise/etc/conf.d/conflib/post-filter/app-postfilter-cisco_ise.conf b/package/enterprise/etc/conf.d/conflib/post-filter/app-postfilter-cisco_ise.conf new file mode 100644 index 0000000000..640705d906 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/post-filter/app-postfilter-cisco_ise.conf @@ -0,0 +1,64 @@ + +parser p_ise_event_time_multi { + csv-parser( + columns(DATE, TIME, TZ) + prefix(".tmp.") + delimiters(chars(" ")) + template('${.tmp.date_seg}') + ); + + date-parser-nofilter( + #YYYY- MM-DD hh:mm:ss:xxx +/-zh:zm + format('%Y-%m-%d %H:%M:%S.%f %z') + template("${.tmp.DATE} ${.tmp.TIME} ${.tmp.TZ}") + ); +}; + +template t_ise_message { + template("${PROGRAM} ${.values.serial} 1 0 ${.gb.message}"); +}; + +block parser app-postfilter-cisco_ise() { + channel { + + parser{ + grouping-by( + scope(host) + key("${.values.serial}") + trigger("$(context-length)" >= "${.values.num}") + sort-key("${.values.seq}") + aggregate( + tags("agg") + value(".gb.message" "$(implode '' $(list-slice 0:-1 $(context-values ${.values.message})))") + value("PROGRAM" "${PROGRAM}@1") + value(".values.serial" "${.values.serial}@1") + value(".tmp.date_seg" "$(list-head $(context-values ${.values.message}))") + value(".splunk.sc4s_template", "t_ise_message") + value(".metadata.sc4s.countOfParts", "$(context-length)") + inherit-mode(context) + ) + timeout(10) + ); + }; + + if { + parser(p_ise_event_time_multi); + }; + + if { + filter{ not filter(f_is_agg) }; + rewrite(r_set_dest_splunk_null_queue); + }; + + + + }; +}; +application app-postfilter-cisco_ise[sc4s-finalfilter] { + filter { + program('CISE_' type(string) flags(prefix)) + and "${.values.num}" != 1 + and not program('CISE_Alarm'); + }; + parser { app-postfilter-cisco_ise(); }; +}; diff --git a/package/enterprise/etc/conf.d/conflib/post-filter/app-postfilter-dell_rsa_secureid.conf b/package/enterprise/etc/conf.d/conflib/post-filter/app-postfilter-dell_rsa_secureid.conf new file mode 100644 index 0000000000..ffafd6726d --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/post-filter/app-postfilter-dell_rsa_secureid.conf @@ -0,0 +1,34 @@ + +block parser app-postfilter-dell_rsa_secureid() { + channel { + + parser{ + grouping-by( + scope(host) + key('x') + timeout(1) + aggregate( + tags("agg") + value("MESSAGE" "$(implode '\n' $(context-values ${LEGACY_MSGHDR}${MESSAGE}))") + ) + ); + + }; + + if { + filter{ not filter(f_is_agg) }; + rewrite(r_set_dest_splunk_null_queue); + }; + + + }; +}; +application app-postfilter-dell_rsa_secureid[sc4s-postfilter] { + filter { + match('dell-rsa', value('fields.sc4s_vendor') type(string)) + and match('secureid', value('fields.sc4s_product') type(string)) + and match('trace', value('fields.sc4s_class') type(string)) + }; + parser { app-postfilter-dell_rsa_secureid(); }; +}; + diff --git a/package/enterprise/etc/conf.d/conflib/post-filter/app-postfilter-mark.conf b/package/enterprise/etc/conf.d/conflib/post-filter/app-postfilter-mark.conf new file mode 100644 index 0000000000..5ec849cbf6 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/post-filter/app-postfilter-mark.conf @@ -0,0 +1,26 @@ + +block parser app-syslog-mark() { + channel { + rewrite { + r_set_splunk_dest_default( + index("main") + sourcetype('sc4s:remote_mark') + vendor('splunk') + product('sc4s') + class("events") + ); + }; + if { + filter{ "`SC4S_SOURCE_MARK_MESSAGE_NULLQUEUE`" ne "no" }; + rewrite(r_set_dest_splunk_null_queue); + }; + }; +}; +application app-syslog-mark[sc4s-postfilter] { + filter { + message('-- MARK --$') + ; + }; + parser { app-syslog-mark(); }; +}; + diff --git a/package/enterprise/etc/conf.d/conflib/post-filter/app-postfilter-symantec_brightmail.conf b/package/enterprise/etc/conf.d/conflib/post-filter/app-postfilter-symantec_brightmail.conf new file mode 100644 index 0000000000..70a8c2ae35 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/post-filter/app-postfilter-symantec_brightmail.conf @@ -0,0 +1,53 @@ +template t_symantec_brightmail { + template("${.metadata.seq}|${.metadata.id}|${.gb.message}"); +}; + + + +block parser app-postfilter-symantec_brightmail() { + channel { + + parser{ + grouping-by( + scope(program) + key("${.tmp.id}") + aggregate( + tags("agg") + value(".metadata.seq" "${.metadata.seq}@1") + value(".metadata.id" "${.metadata.id}@1") + value(".gb.message" "$(implode ';' $(list-slice 0:-1 $(context-values ${.message})))") + value("PROGRAM" "${PROGRAM}@1") + value(".splunk.sc4s_template", "t_symantec_brightmail") + value(".splunk.sourcetype", "symantec:smg:mail") + value(".metadata.sc4s.countOfParts", "$(context-length)") + tags("isStructured") + inherit-mode(context) + ) + timeout(2) + ); + }; + if { + filter{ not filter(f_is_agg) }; + rewrite(r_set_dest_splunk_null_queue); + }; + rewrite { + r_set_splunk_dest_update_v2( + class('merged') + condition(not filter(f_is_agg)) + ); + }; + + + + }; +}; +application app-postfilter-symantec_brightmail[sc4s-postfilter] { + filter { + program('bmserver' type(string) flags(prefix)) and not message('[Brightmail]' type(string) flags(substring)) + and "`SC4S_SOURCE_FF_SYMANTEC_BRIGHTMAIL_GROUPMSG`" ne "no" + + }; + + parser { app-postfilter-symantec_brightmail(); }; +}; + diff --git a/package/enterprise/etc/conf.d/conflib/post-filter/app-postfilter-vmware_vsphere_cmmdsTimeMachineDump.conf b/package/enterprise/etc/conf.d/conflib/post-filter/app-postfilter-vmware_vsphere_cmmdsTimeMachineDump.conf new file mode 100644 index 0000000000..ea365aeb05 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/post-filter/app-postfilter-vmware_vsphere_cmmdsTimeMachineDump.conf @@ -0,0 +1,92 @@ +# template t_vmware_vsphere_cmmdsTimeMachineDump { +# template("${.metadata.seq}|${.metadata.id}|${.gb.message}"); +# }; + + + +block parser app-postfilter-vmware_vsphere_cmmdsTimeMachineDump() { + channel { + + rewrite { + subst('\.$', '', value('MESSAGE')); + }; + + parser{ + grouping-by( + scope(host) + key('$SOURCEIP') + trigger(message('\\q$')) + aggregate( + tags("agg") + value("MESSAGE" "$(implode '' $(list-slice 0:-1 $(context-values ${MESSAGE})))") + value("PROGRAM" "${PROGRAM}@1") + value("HOST" "${HOST}@1") + value(".splunk.sc4s_template", "${.splunk.sc4s_template}@1") + value(".splunk.index", "${.splunk.index}@1") + value(".splunk.source", "${.splunk.source}@1") + value(".splunk.sourcetype", "${.splunk.sourcetype}@1") + value("fields.sc4s_vendor", "${fields.sc4s_vendor}@1") + value("fields.sc4s_product", "${fields.sc4s_product}@1") + value("fields.sc4s_class", "${fields.sc4s_class}@1") + value(".metadata.sc4s.countOfParts", "$(context-length)") + tags("isStructured") + inherit-mode(context) + ) + timeout(5) + ); + }; + + if { + filter { + filter(f_is_agg) + }; + + rewrite { + subst('\q', '', type(string) flags(substring) value('MESSAGE')); + }; + if { + parser { + csv-parser( + columns("timestamp","guid1","value1","value2","guid2","value3","message") + prefix(".metadata.") + delimiters(',') + flags(greedy) + #template('${MESSAGE}') + ); + json-parser( + prefix('.values.') + template('${.metadata.message}') + ); + }; + rewrite { + unset(value(".metadata.message")); + r_set_splunk_dest_update_v2( + template('t_json_values') + ); + }; + } else { + rewrite { + set( + '${.gb.message}', + value('MESSAGE') + ); + unset(value(".gb.message")); + }; + }; + + } else { + rewrite(r_set_dest_splunk_null_queue); + }; + + + }; +}; +application app-postfilter-vmware_vsphere_cmmdsTimeMachineDump[sc4s-postfilter] { + filter { + program('cmmdsTimeMachineDump' type(string)) + and "`SC4S_SOURCE_VMWARE_VSPHERE_GROUPMSG`" eq "yes" + + }; + + parser { app-postfilter-vmware_vsphere_cmmdsTimeMachineDump(); }; +}; diff --git a/package/enterprise/etc/conf.d/conflib/post-filter/app-postfilter-vmware_vsphere_invalidmultiline.conf b/package/enterprise/etc/conf.d/conflib/post-filter/app-postfilter-vmware_vsphere_invalidmultiline.conf new file mode 100644 index 0000000000..0cbe8bb1ef --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/post-filter/app-postfilter-vmware_vsphere_invalidmultiline.conf @@ -0,0 +1,63 @@ +# template t_vmware_vsphere_cmmdsTimeMachineDump { +# template("${.metadata.seq}|${.metadata.id}|${.gb.message}"); +# }; + + + +block parser app-postfilter-vmware_vsphere_invalidmultiline() { + channel { + + parser{ + grouping-by( + scope(program) + key('$SOURCEIP') + aggregate( + tags("agg") + value(".gb.message" "$(implode '\n' $(list-slice 0:-1 $(context-values ${MESSAGE})))") + value("PROGRAM" "${PROGRAM}@1") + value("HOST" "${HOST}@1") + value(".splunk.sc4s_template", "${.splunk.sc4s_template}@1") + value(".splunk.index", "${.splunk.index}@1") + value(".splunk.source", "${.splunk.source}@1") + value(".splunk.sourcetype", "${.splunk.sourcetype}@1") + value("fields.sc4s_vendor", "${fields.sc4s_vendor}@1") + value("fields.sc4s_product", "${fields.sc4s_product}@1") + value("fields.sc4s_class", "${fields.sc4s_class}@1") + value(".metadata.sc4s.countOfParts", "$(context-length)") + tags("isStructured") + inherit-mode(context) + ) + timeout(2) + ); + }; + + if { + filter { + filter(f_is_agg) + }; + } else { + rewrite(r_set_dest_splunk_null_queue); + }; + + + }; +}; +application app-postfilter-vmware_vsphere_invalidmultiline[sc4s-postfilter] { + filter { + match('vmware', value('fields.sc4s_vendor') type(string)) + and ( + match('vsphere', value('fields.sc4s_product') type(string)) + or match('vcenter', value('fields.sc4s_product') type(string) ) + or match('esx', value('fields.sc4s_product') type(string)) + ) + and "`SC4S_SOURCE_VMWARE_VSPHERE_GROUPMSG`" eq "yes" + and ( + message('^--> ') + or message('[{:}] *$') + ) + + + }; + + parser { app-postfilter-vmware_vsphere_invalidmultiline(); }; +}; diff --git a/package/enterprise/etc/conf.d/conflib/post-filter/app-postfilter-vmware_vsphere_localcli.conf b/package/enterprise/etc/conf.d/conflib/post-filter/app-postfilter-vmware_vsphere_localcli.conf new file mode 100644 index 0000000000..9412962a05 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/post-filter/app-postfilter-vmware_vsphere_localcli.conf @@ -0,0 +1,60 @@ +# template t_vmware_vsphere_cmmdsTimeMachineDump { +# template("${.metadata.seq}|${.metadata.id}|${.gb.message}"); +# }; + + + +block parser app-postfilter-vmware_vsphere_localcli() { + channel { + + parser{ + grouping-by( + scope(process) + key('$SOURCEIP') + aggregate( + tags("agg") + value(".gb.message" "$(implode '\n' $(list-slice 0:-1 $(context-values ${MESSAGE})))") + value("PROGRAM" "${PROGRAM}@1") + value("HOST" "${HOST}@1") + value(".splunk.sc4s_template", "${.splunk.sc4s_template}@1") + value(".splunk.index", "${.splunk.index}@1") + value(".splunk.source", "${.splunk.source}@1") + value(".splunk.sourcetype", "${.splunk.sourcetype}@1") + value("fields.sc4s_vendor", "${fields.sc4s_vendor}@1") + value("fields.sc4s_product", "${fields.sc4s_product}@1") + value("fields.sc4s_class", "${fields.sc4s_class}@1") + value(".metadata.sc4s.countOfParts", "$(context-length)") + tags("isStructured") + inherit-mode(context) + ) + timeout(2) + ); + }; + if { + filter { + filter(f_is_agg) + }; + } else { + rewrite(r_set_dest_splunk_null_queue); + }; + + + }; +}; +application app-postfilter-vmware_vsphere_localcli[sc4s-postfilter] { + filter { + match('vmware', value('fields.sc4s_vendor') type(string)) + and ( + match('vcenter', value('fields.sc4s_product') type(string) ) + or match('vsphere', value('fields.sc4s_product') type(string)) + or match('esx', value('fields.sc4s_product') type(string)) + ) + and "`SC4S_SOURCE_VMWARE_VSPHERE_GROUPMSG`" eq "yes" + and program('localcli' type(string)) + ; + + + }; + + parser { app-postfilter-vmware_vsphere_localcli(); }; +}; \ No newline at end of file diff --git a/package/enterprise/etc/conf.d/conflib/post-filter/app-postfilter-vmware_vsphere_sdrsInjector.conf b/package/enterprise/etc/conf.d/conflib/post-filter/app-postfilter-vmware_vsphere_sdrsInjector.conf new file mode 100644 index 0000000000..16b8dd598a --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/post-filter/app-postfilter-vmware_vsphere_sdrsInjector.conf @@ -0,0 +1,55 @@ +# template t_vmware_vsphere_cmmdsTimeMachineDump { +# template("${.metadata.seq}|${.metadata.id}|${.gb.message}"); +# }; + + + +block parser app-postfilter-vmware_vsphere_sdrsInjector() { + channel { + + parser{ + grouping-by( + scope(program) + key('$FULLHOST') + aggregate( + tags("agg") + value("MESSAGE" "$(implode '\n' $(list-slice 0:-1 $(context-values ${MESSAGE})))") + value("PROGRAM" "${PROGRAM}@1") + value("HOST" "${HOST}@1") + value(".splunk.sc4s_template", "${.splunk.sc4s_template}@1") + value(".splunk.index", "${.splunk.index}@1") + value(".splunk.source", "${.splunk.source}@1") + value(".splunk.sourcetype", "${.splunk.sourcetype}@1") + value("fields.sc4s_vendor", "${fields.sc4s_vendor}@1") + value("fields.sc4s_product", "${fields.sc4s_product}@1") + value("fields.sc4s_class", "${fields.sc4s_class}@1") + value(".metadata.sc4s.countOfParts", "$(context-length)") + tags("isStructured") + inherit-mode(context) + ) + timeout(2) + ); + }; + + if { + filter { + filter(f_is_agg) + }; + } else { + rewrite(r_set_dest_splunk_null_queue); + }; + + + }; +}; +application app-postfilter-vmware_vsphere_sdrsInjector[sc4s-finalfilter] { + filter { + ( + program('sdrsInjector' type(string)) + ) + and "`SC4S_SOURCE_VMWARE_VSPHERE_GROUPMSG`" eq "yes" + + }; + + parser { app-postfilter-vmware_vsphere_sdrsInjector(); }; +}; diff --git a/package/enterprise/etc/conf.d/conflib/post-filter/app-postfilter-vmware_vsphere_storageRM.conf b/package/enterprise/etc/conf.d/conflib/post-filter/app-postfilter-vmware_vsphere_storageRM.conf new file mode 100644 index 0000000000..9c6dcb46b6 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/post-filter/app-postfilter-vmware_vsphere_storageRM.conf @@ -0,0 +1,55 @@ +# template t_vmware_vsphere_cmmdsTimeMachineDump { +# template("${.metadata.seq}|${.metadata.id}|${.gb.message}"); +# }; + + + +block parser app-postfilter-vmware_vsphere_storageRM() { + channel { + + parser{ + grouping-by( + scope(program) + key('$FULLHOST_FROM') + trigger(message('datastoreSlopeUS\[5\]')) + aggregate( + tags("agg") + value("MESSAGE" "$(implode '\n' $(list-slice 0:-1 $(context-values ${MESSAGE})))") + value("PROGRAM" "${PROGRAM}@1") + value("HOST" "${HOST}@1") + value(".splunk.sc4s_template", "${.splunk.sc4s_template}@1") + value(".splunk.index", "${.splunk.index}@1") + value(".splunk.source", "${.splunk.source}@1") + value(".splunk.sourcetype", "${.splunk.sourcetype}@1") + value("fields.sc4s_vendor", "${fields.sc4s_vendor}@1") + value("fields.sc4s_product", "${fields.sc4s_product}@1") + value("fields.sc4s_class", "${fields.sc4s_class}@1") + value(".metadata.sc4s.countOfParts", "$(context-length)") + tags("isStructured") + inherit-mode(context) + ) + timeout(2) + ); + }; + + if { + filter { + filter(f_is_agg) + }; + } else { + rewrite(r_set_dest_splunk_null_queue); + }; + + }; +}; +application app-postfilter-vmware_vsphere_storageRM[sc4s-postfilter] { + filter { + ( + program('storageRM' type(string)) + ) + and "`SC4S_SOURCE_VMWARE_VSPHERE_GROUPMSG`" eq "yes" + + }; + + parser { app-postfilter-vmware_vsphere_storageRM(); }; +}; diff --git a/package/enterprise/etc/conf.d/conflib/raw/app-raw-bsd_nopri.conf b/package/enterprise/etc/conf.d/conflib/raw/app-raw-bsd_nopri.conf new file mode 100644 index 0000000000..7f66031700 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/raw/app-raw-bsd_nopri.conf @@ -0,0 +1,59 @@ +rewrite set_raw_nopri_bsd{ + set-tag("wireformat:raw_nopri_bsd"); +}; +filter f_is_raw_nopri_bsd{ + tags("wireformat:raw_nopri_bsd"); +}; + +block parser app-raw-bsd_nopri() { + channel { + if ("`SC4S_DISABLE_DROP_INVALID_RAW_BSD`" eq "yes") { + parser { + syslog-parser( + drop-invalid(no) + flags(assume-utf8, guess-timezone)); + }; + } elif { + parser { + syslog-parser( + drop-invalid(yes) + flags(assume-utf8, guess-timezone)); + }; + } else {}; + + if { + filter{ + not host('^[a-zA-Z0-9_\-\.]+$') + }; + rewrite { + set("${HOST} $(template t_hdr_msg)" value("MSG")); + unset(value("LEGACY_MSGHDR")); + unset(value("PID")); + unset(value("PROGRAM")); + set('$SOURCEIP' value('HOST')); + }; + }; + if { + # If program is probably not valid cleanup MESSAGE so log paths don't have too + # This isn't great for performance but is reliable good reason to use 5424 + parser(app-plugin-syslog-fix-program); + rewrite { + set("$(template t_hdr_msg)" value("MSG")); + unset(value("LEGACY_MSGHDR")); + unset(value("PID")); + unset(value("PROGRAM")); + }; + }; + #Note this should be RFC this tag is being set to allow routing to rfc destinations + rewrite(set_rfc); + rewrite(set_rfc3164); + rewrite(set_raw); + rewrite(set_raw_nopri_bsd); + }; +}; +application app-raw-bsd_nopri[sc4s-raw-syslog] { + filter { + message('^\w{3} ([0 ][1-9]|[12]\d|3[01]) (0\d|1\d|2[0-3]):([0-5]\d):([0-5]\d) '); + }; + parser { app-raw-bsd_nopri(); }; +}; diff --git a/package/enterprise/etc/conf.d/conflib/raw/app-raw-checkpoint_splunk.conf b/package/enterprise/etc/conf.d/conflib/raw/app-raw-checkpoint_splunk.conf new file mode 100644 index 0000000000..67bd1ae2d7 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/raw/app-raw-checkpoint_splunk.conf @@ -0,0 +1,342 @@ +rewrite set_raw_raw_cp_splunk{ + set-tag("wireformat:raw_cp_splunk"); +}; +filter f_is_raw_raw_cp_splunk{ + tags("wireformat:raw_raw_cp_splunk"); +}; + +block parser app-raw-checkpoint_splunk() { + channel { + + rewrite{ + r_set_splunk_dest_default( + index('netfw') + sourcetype('cp_log') + vendor("checkpoint") + product("splunk") + ); + }; + parser { + kv-parser(prefix(".values.") pair-separator("|") template(t_hdr_msg)); + + }; + if { + + parser { + date-parser-nofilter(format("%s") template("${.values.time}")); + }; + + rewrite { + set("${.values.hostname}", value("HOST")); + set("${.values.hostname}", value("fields.cp_lm")); + }; + + if ( + "`SC4S_LISTEN_CHECKPOINT_SPLUNK_OLD_HOST_RULES`" eq "yes") + { + if { + #Get the FW host from the originsicname + parser { + regexp-parser( + patterns('^[Cc][Nn]\\?=(?[^,]+)') + template("${.values.originsicname}") + ); + }; + } elif { + #Get the FW host from the origin_sic_name + parser { + regexp-parser( + patterns('^[Cc][Nn]\\?=(?[^,]+)') + template("${.values.origin_sic_name}") + ); + }; + } elif { + # If hostname field is not empty, host should be evaluated from there + parser { + regexp-parser( + patterns('(?\S+)') + template("${.values.hostname}") + ); + }; + }; + } else { + if { + # If hostname field is not empty, host should be evaluated from there + parser { + regexp-parser( + patterns('(?\S+)') + template("${.values.hostname}") + ); + }; + } elif { + #Get the FW host from the originsicname + parser { + regexp-parser( + patterns('^[Cc][Nn]\\?=(?[^,]+)') + template("${.values.originsicname}") + ); + }; + } elif { + #Get the FW host from the origin_sic_name + parser { + regexp-parser( + patterns('^[Cc][Nn]\\?=(?[^,]+)') + template("${.values.origin_sic_name}") + ); + }; + }; + }; + # If this device is a v host we need to get the firewall name + if { + parser { + regexp-parser( + patterns('-v_(?.*)') + template("$HOST") + ); + }; + }; + + if { + filter { + "${.values.product}" eq "Syslog" + }; + parser { + syslog-parser( + flags(assume-utf8, no-hostname) + template("${.values.default_device_message}") + ); + }; + parser { + date-parser-nofilter(format("%s") template("${.values.time}")); + }; + rewrite { + + r_set_splunk_dest_update_v2( + index('netops') + source('program:${PROGRAM}') + sourcetype('nix:syslog') + template('t_hdr_msg') + ); + }; + } elif { + filter { + match('Firewall' value('.values.product')) + or match('Application\h+Control' value('.values.product')) + or match('RAD' value('.values.product')) + or match('HTTPS\h+Inspection' value('.values.product')) + or match('Compliance\h+Blade' value('.values.product')) + or match('^Compliance' value('.values.product')) + or match('VPN-1\h+&\h+Fire[wW]all-1' value('.values.product')) + or match('Network\h+Security' value('.values.product')) + or match('FG\h+VPN-1\h+&\h+FireWall-1' value('.values.product')) + or match('QoS' value('.values.product')) + and not match('VPN' value('.values.fw_subproduct')) + }; + rewrite { + r_set_splunk_dest_update_v2( + class('firewall') + index('netfw') + source('checkpoint:firewall') + ); + }; + } elif { + filter { + match('U[rR][lL]\h+Filtering' value('.values.product')) + }; + rewrite { + r_set_splunk_dest_update_v2( + class('web') + index('netproxy') + source('checkpoint:web') + ); + }; + } elif { + filter { + match('Scheduled\h+system\h+update' value('.values.product')) + or match('WEB_API' value('.values.product')) + or match('SmartDefense' value('.values.product')) + or match('Smart\h+Defense' value('.values.product')) + or match('W[eE][bB]-UI' value('.values.product')) + or match('SmartDashboard' value('.values.product')) + or match('System\h+Monitor' value('.values.product')) + or match('Log\h+Update' value('.values.product')) + or match('license-mgmt' value('.values.product')) + or match('smart_event' value('.values.product')) + or match('SmartConsole' value('.values.product')) + or match('SmartEvent\h+Client' value('.values.product')) + or match('SmartUpdate' value('.values.product')) + or match('SmartView' value('.values.product')) + or match('Security\h+Gateway\/Management' value('.values.product')) + or match('Smart\h+Defense' value('.values.product')) + or match('WEB_API_INTERNAL' value('.values.product')) + or match('Eventia\h+Analyzer\h+Client' value('.values.product')) + or match('SmartProvisioning\h+Connector' value('.values.product')) + or match('SmartLSM\h+Endpoint\h+Security\h+Console' value('.values.product')) + or match('SmartLSM' value('.values.product')) + or match('ROBO\h+GUI' value('.values.product')) + or match('Management\h+Blade' value('.values.product')) + or match('Connectra' value('.values.product')) + or match('Check\h+Point\h+Security\h+Management\h+Server' value('.values.product')) + or match('CLI' value('.values.product')) + or match('Check\h+Point\h+GO\h+Password\h+Reset' value('.values.product')) + or match('Database\h+Tool' value('.values.product')) + or match('cpmidu_update_tool' value('.values.product')) + or match('query-database' value('.values.product')) + }; + rewrite { + r_set_splunk_dest_update_v2( + class('audit') + index('netops') + source('checkpoint:audit') + ); + }; + } elif { + filter { + match('Endpoint\h+Management' value('.values.product')) + or match('Core' value('.values.product')) + or match('Endpoint\h+Compliance' value('.values.product')) + or match('MEPP' value('.values.product')) + or match('Media\h+Encryption\h+&\h+Port\h+Protection' value('.values.product')) + or match('Endpoint\h+Security\h+Console' value('.values.product')) + or match('endpoint' value('.values.product')) + }; + rewrite { + r_set_splunk_dest_update_v2( + class('endpoint') + index('netops') + source('checkpoint:endpoint') + ); + }; + } elif { + filter { + match('^VPN$' value('.values.product')) + or match('^Mobile$' value('.values.product')) + or match('Mobile\h+App' value('.values.product')) + or match('^VPN' value('.values.fw_subproduct')) + or match('^VPN-1' value('.values.fw_subproduct')) + or match('Identity\h+Awareness' value('.values.product')) + }; + rewrite { + r_set_splunk_dest_update_v2( + class('sessions') + index('netops') + source('checkpoint:sessions') + ); + }; + } elif { + filter { + match('IOS\h+Profile' value('.values.product')) + or match('iOS\h+Profiles' value('.values.product')) + or match('Device' value('.values.product')) + or match('WIFI\h+Network' value('.values.product')) + or match('Mobile\h+Access' value('.values.product')) + }; + rewrite { + r_set_splunk_dest_update_v2( + class('network') + index('netops') + source('checkpoint:network') + ); + }; + } elif { + filter { + match('Threat\h+Emulation' value('.values.product')) + or match('Anti-Virus' value('.values.product')) + or match('New\h+Anti\h+Virus' value('.values.product')) + or match('Anti-Bot' value('.values.product')) + or match('Threat\h+Extraction' value('.values.product')) + or match('Anti-Ransomware' value('.values.product')) + or match('Anti-Exploit' value('.values.product')) + or match('Forensics' value('.values.product')) + or match('OS\h+Exploit' value('.values.product')) + or match('OS\h+Exploits' value('.values.product')) + or (match('Application' value('.values.product')) and not match('Application Control' value('.values.product'))) + or match('Text\h+Message' value('.values.product')) + or match('Network\h+Access' value('.values.product')) + or match('Zero\h+Phishing' value('.values.product')) + or match('Anti-Malware' value('.values.product')) + or match('Anti\h+Malware' value('.values.product')) + or match('Anti\h+Malware\h+New\h+Anti\h+Virus' value('.values.product')) + or match('New\h+Anti\h+Virus\h+Anti\h+Malware' value('.values.product')) + }; + rewrite { + r_set_splunk_dest_update_v2( + class('ids_malware') + index('netids') + source('checkpoint:ids_malware') + ); + }; + } elif { + filter { + match('IPS' value('.values.product')) + or match('W[iI][fF][iI]' value('.values.product')) + or match('Cellular' value('.values.product')) + }; + rewrite { + r_set_splunk_dest_update_v2( + class('ids') + index('netids') + source('checkpoint:ids') + ); + }; + } elif { + filter { + match('MTA' value('.values.product')) + or match('Anti-Spam' value('.values.product')) + or match('Anti\h+Spam' value('.values.product')) + or match('Anti-Phishing' value('.values.product')) + or match('Anti\h+Phishing' value('.values.product')) + or match('Anti-Spam\h+and\h+Email\h+Security' value('.values.product')) + or match('Anti\h+Spam\h+and\h+Email\h+Security' value('.values.product')) + }; + rewrite { + r_set_splunk_dest_update_v2( + class('email') + index('email') + source('checkpoint:email') + ); + }; + } elif { + filter { + match('DLP' value('.values.product')) + }; + rewrite { + r_set_splunk_dest_update_v2( + class('dlp') + index('netfw') + source('checkpoint:firewall') + ); + }; + } elif { + filter { + match('Syslog' value('.values.product')) + }; + rewrite { + r_set_splunk_dest_update_v2( + class('os') + index('netops') + ); + }; + } else { + rewrite { + r_set_splunk_dest_update_v2( + index('netops') + source('checkpoint:cp_default') + ); + }; + }; + }; + rewrite(set_raw); + rewrite(set_raw_raw_cp_splunk); + rewrite(r_set_source_identified); + }; + +}; +application app-raw-checkpoint_splunk[sc4s-raw-syslog] { + filter { + message('^time=\d{10}\|hostname='); + }; + + parser { app-raw-checkpoint_splunk(); }; +}; + diff --git a/package/enterprise/etc/conf.d/conflib/raw/app-raw-cisco_ftd_nopri-bsd.conf b/package/enterprise/etc/conf.d/conflib/raw/app-raw-cisco_ftd_nopri-bsd.conf new file mode 100644 index 0000000000..e808c4cdd2 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/raw/app-raw-cisco_ftd_nopri-bsd.conf @@ -0,0 +1,43 @@ +rewrite set_raw_cisco_ftd_nopri_bsddate{ + set-tag("wireformat:raw_cisco_ftd_nopri_bsddate"); +}; +filter f_is_raw_cisco_ftd_nopri_bsddate{ + tags("wireformat:raw_cisco_ftd_nopri_bsddate"); +}; + +block parser app-raw-cisco_ftd_nopri_bsddate-bsd() { + channel { + parser { + regexp-parser( + prefix(".tmp.") + patterns('(?[A-Z][a-z]{2} *\d{1,2} \d{4} \d\d:\d\d:\d\d) (?[^ ]+)?(?: : | *)(?%FTD-\d-\d+:.*)') + ); + }; + rewrite { + set("${.tmp.message}", value("MESSAGE")); + set("${.tmp.host}", value("HOST")); + }; + parser { + date-parser-nofilter(format( + '%b %d %H:%M:%S.%f', + '%b %d %H:%M:%S', + '%b %d %I:%M:%S %p.%f', + '%b %d %I:%M:%S %p', + '%b %d %Y %I:%M:%S %p.%f' + '%b %d %Y %H:%M:%S.%f', + '%b %d %Y %H:%M:%S', + ) + template("${.tmp.timeStamp}") + ); + }; + + #Note this should be RFC this tag is being set to allow routing to rfc destinations + rewrite(set_rfc); + rewrite(set_rfc3164); + rewrite(set_raw); + rewrite(set_raw_cisco_ftd_nopri_bsddate); + }; +}; +application app-raw-cisco_ftd_nopri_bsddate-bsd[sc4s-raw-syslog] { + parser { app-raw-cisco_ftd_nopri_bsddate-bsd(); }; +}; diff --git a/package/enterprise/etc/conf.d/conflib/raw/app-raw-cisco_ftd_nopri-iso.conf b/package/enterprise/etc/conf.d/conflib/raw/app-raw-cisco_ftd_nopri-iso.conf new file mode 100644 index 0000000000..e11c3805d3 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/raw/app-raw-cisco_ftd_nopri-iso.conf @@ -0,0 +1,43 @@ +rewrite set_raw_cisco_ftd_nopri_isodate{ + set-tag("wireformat:raw_cisco_ftd_nopri_isodate"); +}; +filter f_is_raw_cisco_ftd_nopri_isodate{ + tags("wireformat:raw_cisco_ftd_nopri_isodate"); +}; + +block parser app-raw-cisco_ftd_nopri_isodate-iso() { + channel { + parser { + regexp-parser( + prefix(".tmp.") + patterns('(?\d{4}-\d\d-\d\d.\d\d:\d\d:\d\d[^ ]*) (?[^ ]+)(?: : | *)(?%FTD-\d-\d+:.*)') + ); + }; + rewrite { + set("${.tmp.message}", value("MESSAGE")); + set("${.tmp.host}", value("HOST")); + }; + parser { + date-parser( + format( + '%Y-%m-%dT%T.%f%z', + '%Y-%m-%dT%T%z', + '%Y-%m-%d %T.%f%z', + '%Y-%m-%d %T%z', + '%Y-%m-%d%T.%f%z', + '%Y-%m-%d%T%z', + ) + template("${.tmp.timestamp}") + ); + }; + + #Note this should be RFC this tag is being set to allow routing to rfc destinations + rewrite(set_rfc); + rewrite(set_rfc3164); + rewrite(set_raw); + rewrite(set_raw_cisco_ftd_nopri_isodate); + }; +}; +application app-raw-cisco_ftd_nopri_isodate-iso[sc4s-raw-syslog] { + parser { app-raw-cisco_ftd_nopri_isodate-iso(); }; +}; diff --git a/package/enterprise/etc/conf.d/conflib/raw/app-raw-isodate.conf b/package/enterprise/etc/conf.d/conflib/raw/app-raw-isodate.conf new file mode 100644 index 0000000000..94c8208033 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/raw/app-raw-isodate.conf @@ -0,0 +1,57 @@ +rewrite set_raw_raw_isodate{ + set-tag("wireformat:raw_raw_isodate"); +}; +filter f_is_raw_raw_isodate{ + tags("wireformat:raw_raw_isodate"); +}; + +block parser app-raw-isodate() { + channel { + parser { + regexp-parser( + prefix(".tmp.") + patterns('^(?\d{4}-\d{2}-\d{2}(?: |T)?\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:[+-]\d{2}:?\d{2}|Z)?)\s?(?.*)') + ); + }; + rewrite { + set("${.tmp.message}", value("MESSAGE")); + }; + if { + parser { + date-parser( + format( + '%Y-%m-%dT%T.%f%z', + '%Y-%m-%dT%T%z', + '%Y-%m-%d %T.%f%z', + '%Y-%m-%d %T%z', + '%Y-%m-%d%T.%f%z', + '%Y-%m-%d%T%z', + ) + template("${.tmp.timestamp}") + ); + }; + } else { + parser { + date-parser( + format( + '%Y-%m-%dT%T.%f', + '%Y-%m-%dT%T', + '%Y-%m-%d %T.%f', + '%Y-%m-%d %T', + '%Y-%m-%d%T.%f', + '%Y-%m-%d%T', + ) + flags(guess-timezone) + template("${.tmp.timestamp}") + ); + }; + }; + + + rewrite(set_raw); + rewrite(set_raw_raw_isodate); + }; +}; +application app-raw-isodate[sc4s-raw-syslog] { + parser { app-raw-isodate(); }; +}; diff --git a/package/enterprise/etc/conf.d/conflib/raw/app-raw-xml.conf b/package/enterprise/etc/conf.d/conflib/raw/app-raw-xml.conf new file mode 100644 index 0000000000..9bb195cca6 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/raw/app-raw-xml.conf @@ -0,0 +1,48 @@ +rewrite set_raw_xml{ + set-tag("wireformat:raw_xml"); +}; +filter f_is_raw_xml{ + tags("wireformat:raw_xml"); +}; +block parser app-raw-xml() { + channel { + if ("`SC4S_DISABLE_DROP_INVALID_XML`" eq "yes") { + parser { + regexp-parser( + prefix(".tmp.") + patterns('^ *(?:<\d+>)?(?.*)') + ); + xml( + prefix('.values.') + template('${.tmp.message}') + drop-invalid(no) + ); + }; + } elif { + parser { + regexp-parser( + prefix(".tmp.") + patterns('^ *(?:<\d+>)?(?.*)') + ); + xml( + prefix('.values.') + template('${.tmp.message}') + drop-invalid(yes) + ); + }; + } else {}; + + rewrite { + set("${.tmp.message}" value("MESSAGE")); + }; + rewrite(set_raw); + rewrite(set_raw_xml); + }; +}; +application app-raw-xml[sc4s-raw-syslog] { + filter { + message('^ *<(?![0-9]+\>)'); + }; + + parser { app-raw-xml(); }; +}; diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-a10.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-a10.conf new file mode 100644 index 0000000000..1bd1784386 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-a10.conf @@ -0,0 +1,32 @@ +block parser app-syslog-a10() { + channel { + rewrite { + r_set_splunk_dest_default( + index('netops') + source('a10networks:vThunder') + sourcetype('a10networks:vThunder:syslog') + vendor('a10networks') + product('vThunder') + ); + }; + }; +}; + +application app-syslog-a10[sc4s-syslog-pgm] { + filter { + program('a10logd' type(string) flags(prefix)) or + program('a10timer' type(string) flags(prefix)) or + program('a10stat' type(string) flags(prefix)) or + program('a10switch' type(string) flags(prefix)) or + program('a10hm' type(string) flags(prefix)) or + program('a10rt' type(string) flags(prefix)) or + program('a10rip' type(string) flags(prefix)) or + program('a10ospf' type(string) flags(prefix)) or + program('a10snmpd' type(string) flags(prefix)) or + program('a10wa' type(string) flags(prefix)) or + program('a10gmpd' type(string) flags(prefix)) or + program('a10snpm_trapd' type(string) flags(prefix)) or + program('a10lb' type(string) flags(prefix)) + }; + parser { app-syslog-a10(); }; +}; diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-airwatch.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-airwatch.conf new file mode 100644 index 0000000000..72c94afeee --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-airwatch.conf @@ -0,0 +1,26 @@ +block parser app-syslog-vmware_airwatch() { + channel { + rewrite { + r_set_splunk_dest_default( + index('epintel') + sourcetype('vmware:airwatch') + vendor("vmware") + product("airwatch") + template('t_msg_only') + ); + }; + rewrite { + set("${PROGRAM}", value(".PROGRAM")); + subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM")); + r_set_splunk_dest_update_v2( + source('program:${.PROGRAM}') + ); + }; + }; +}; +application app-syslog-vmware_airwatch[sc4s-syslog-pgm] { + filter { + program('Airwatch' type(string) flags(ignore-case)); + }; + parser { app-syslog-vmware_airwatch(); }; +}; diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-alcatel_switch.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-alcatel_switch.conf new file mode 100644 index 0000000000..4d29c2d3e8 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-alcatel_switch.conf @@ -0,0 +1,22 @@ +block parser app-syslog-alcatel_switch() { + channel { + rewrite { + r_set_splunk_dest_default( + index('netops') + sourcetype('alcatel:switch') + vendor("alcatel") + product("switch") + template('t_hdr_msg') + ); + }; + + + }; +}; +application app-syslog-alcatel_switch[sc4s-syslog-pgm] { + filter { + program('swlogd' type(string) flags(prefix)); + }; + parser { app-syslog-alcatel_switch(); }; +}; + diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-alsid_syslog.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-alsid_syslog.conf new file mode 100644 index 0000000000..6a1091344c --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-alsid_syslog.conf @@ -0,0 +1,21 @@ +block parser app-syslog-alsid_syslog() { + channel { + rewrite { + r_set_splunk_dest_default( + index('oswinsec') + sourcetype('alsid:syslog') + vendor("alsid") + product("syslog") + template('t_standard') + ); + }; + + }; +}; +application app-syslog-alsid_syslog[sc4s-syslog-pgm] { + filter { + "${PROGRAM}" eq "AlsidForAD"; + }; + parser { app-syslog-alsid_syslog(); }; +}; + diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-arista_eos.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-arista_eos.conf new file mode 100644 index 0000000000..c6ce3752fb --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-arista_eos.conf @@ -0,0 +1,35 @@ +block parser app-syslog-arista_eos() { + channel { + # + parser { + regexp-parser( + prefix(".tmp.") + patterns('^(?: ?(?\d+): )?(?%[A-Z]+-\d+-[^: ]+: .*)') + flags(ignore-case) + ); + }; + rewrite { + r_set_splunk_dest_default( + index('netops') + sourcetype('arista:eos') + source('arista:eos:$(lowercase ${PROGRAM})') + vendor("arista") + product("eos") + class('$(lowercase ${.tmp.program})') + ); + }; + rewrite { + set('${.tmp.seq}' value('fields.seq') condition("${.tmp.seq}" ne "")); + set('${.tmp.message}' value('MESSAGE') condition("${.tmp.seq}" ne "")); + }; + }; +}; + +application app-syslog-arista_eos[sc4s-syslog] { + filter { + program('^[A-Z]\S+$') + and message('%' type(string) flags(prefix)); + }; + + parser { app-syslog-arista_eos(); }; +}; diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-aruba_ap.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-aruba_ap.conf new file mode 100644 index 0000000000..ac9d68039e --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-aruba_ap.conf @@ -0,0 +1,39 @@ +block parser app-syslog-aruba_ap() { + channel { + rewrite { + r_set_splunk_dest_default( + index('netops') + source('aruba:${PROGRAM}') + sourcetype('aruba:syslog') + vendor("aruba") + product("ap") + ); + }; + + + }; +}; +application app-syslog-aruba_ap[sc4s-syslog-pgm] { + filter { + ( + ( + program('authmgr' type(string) flags(prefix)) + or program('stm' type(string) flags(prefix)) + or program('wms' type(string) flags(prefix)) + or program('dot1x-proc' type(string) flags(prefix)) + or program('nanny' type(string) flags(prefix)) + ) + and message('^\<\d+\> \<\d+\> \<[A-Z]+\>') + ) + or ( + ( + program('sapd' type(string) flags(prefix)) + ) + and message('^\<\d+\> \<[A-Z]+\>') + ) + ; + }; + + parser { app-syslog-aruba_ap(); }; +}; + diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-avaya_sipmgr.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-avaya_sipmgr.conf new file mode 100644 index 0000000000..51d670b201 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-avaya_sipmgr.conf @@ -0,0 +1,24 @@ +block parser app-syslog-avaya_sipmgr() { + channel { + rewrite { + r_set_splunk_dest_default( + index('main') + source('avaya:${PROGRAM}') + sourcetype('avaya:sipmgr') + vendor("avaya") + product("sipmgr") + ); + }; + + + }; +}; +application app-syslog-avaya_sipmgr[sc4s-syslog-pgm] { + filter { + program('AasSipMgr' type(string) flags(prefix)) + ; + }; + + parser { app-syslog-avaya_sipmgr(); }; +}; + diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-avi_controller_legacy.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-avi_controller_legacy.conf new file mode 100644 index 0000000000..5631d4db20 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-avi_controller_legacy.conf @@ -0,0 +1,33 @@ +block parser app-syslog-avi_controller_legacy() { + channel { + + rewrite { + r_set_splunk_dest_default( + index('netops') + sourcetype('avi:events') + vendor("avi") + product("vantage") + ); + }; + + parser { + regexp-parser( + prefix(".tmp.") + patterns('^\[(?[^ ]+ [^,]+,[^:]+)') + ); + date-parser-nofilter( + format('%Y-%m-%d %H:%M:%S,%f') + template("${.tmp.timestamp}") + ); + }; + }; +}; +application app-syslog-avi_controller_legacy[sc4s-syslog] { + filter { + message('^\[') + and message(": Avi-Controller: " type(string) flags(substring)); + }; + + parser { app-syslog-avi_controller_legacy(); }; +}; + diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-avi_controller_rfc5424.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-avi_controller_rfc5424.conf new file mode 100644 index 0000000000..7397369241 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-avi_controller_rfc5424.conf @@ -0,0 +1,29 @@ +block parser app-syslog-avi_controller_rfc5424() { + channel { + parser { + syslog-parser( + + flags(assume-utf8, syslog-protocol) + template("<${PRI}>1 $S_ISODATE $HOST $PROGRAM $MESSAGE") + ); + }; + + rewrite { + r_set_splunk_dest_default( + index('netops') + sourcetype('avi:events') + vendor("avi") + product("vantage") + ); + }; + }; +}; +application app-syslog-avi_controller_rfc5424[sc4s-syslog-pgm] { + filter { + program('Avi-Controller' type(string) flags(prefix)) + and message('-' type(string) flags(prefix)); + }; + + parser { app-syslog-avi_controller_rfc5424(); }; +}; + diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-avi_vantage.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-avi_vantage.conf new file mode 100644 index 0000000000..951087608f --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-avi_vantage.conf @@ -0,0 +1,23 @@ +block parser app-syslog-avi_vantage() { + channel { + rewrite { + r_set_splunk_dest_default( + index('netops') + sourcetype('avi:logs') + vendor("avi") + product("vantage") + class('logs') + template('t_hdr_msg') + ); + }; + + + }; +}; +application app-syslog-avi_vantage[sc4s-syslog-pgm] { + filter { + program('aer01-' type(string) flags(prefix)); + }; + parser { app-syslog-avi_vantage(); }; +}; + diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-aviatrix.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-aviatrix.conf new file mode 100644 index 0000000000..4847ee72c5 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-aviatrix.conf @@ -0,0 +1,43 @@ +block parser app-syslog-aviatrix() { + channel { + # /usr/local/bin/avx-gw-state-sync -> avx-gw-state-sync + rewrite { + set("${PROGRAM}", value(".PROGRAM")); + subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM")); + }; + + rewrite { + set("cloudx-cli", value(".PROGRAM") condition("${.PROGRAM}" eq "cloudx_cli")); + set("perfmon", value(".PROGRAM") condition("${.PROGRAM}" eq "perfmon.py")); + }; + + rewrite { + r_set_splunk_dest_default( + index('netops') + sourcetype('aviatrix:${.PROGRAM}') + vendor("aviatrix") + product("${.PROGRAM}") + template('t_msg_only') + ); + }; + }; +}; + +application app-syslog-aviatrix[sc4s-syslog-pgm] { + filter { + program('cloudxd' type(string) flags(ignore-case)) + or program('avx-gw-state-sync' type(string) flags(substring, ignore-case)) + or program('cloudx_cli' type(string) flags(ignore-case)) + or program('perfmon.py' type(string) flags(ignore-case)) + or program('avx-nfq' type(string) flags(ignore-case)) + + or ( + program('kernel' type(string) flags(ignore-case)) + and ( + message('AvxRl' type(string) flags(substring)) + or message('AviatrixUser' type(string) flags(substring)) + ) + ) + }; + parser { app-syslog-aviatrix(); }; +}; diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-barracuda_waf.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-barracuda_waf.conf new file mode 100644 index 0000000000..b85bfdcb8c --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-barracuda_waf.conf @@ -0,0 +1,56 @@ +block parser app-syslog-barracuda_waf() { + channel { + rewrite { + r_set_splunk_dest_default( + index("netwaf") + vendor("barracuda") + product("waf") + sourcetype('barracuda:syslog') + ) + }; + + if (program('SYS')) { + rewrite { + r_set_splunk_dest_update_v2( + sourcetype('barracuda:system') + ); + }; + } elif (program('WF')) { + rewrite { + r_set_splunk_dest_update_v2( + sourcetype('barracuda:waf') + ); + }; + } elif (program('TR')) { + rewrite { + r_set_splunk_dest_update_v2( + sourcetype('barracuda:web') + ); + }; + } elif (program('AUDIT')) { + rewrite { + r_set_splunk_dest_update_v2( + sourcetype('barracuda:audit') + ); + }; + } elif (program('NF')) { + rewrite { + r_set_splunk_dest_update_v2( + sourcetype('barracuda:firewall') + ); + }; + }; + }; +}; + + +application app-syslog-barracuda_waf[sc4s-syslog-pgm] { + filter { + program('SYS' type(string)) + or program('WF' type(string)) + or program('TR' type(string)) + or program('AUDIT' type(string)) + or program('NF' type(string)) + }; + parser { app-syslog-barracuda_waf(); }; +}; diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-beyondtrust_sra.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-beyondtrust_sra.conf new file mode 100644 index 0000000000..ed8717d0a6 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-beyondtrust_sra.conf @@ -0,0 +1,97 @@ +block parser app-syslog-beyondtrust_sra() { + channel { + rewrite { + r_set_splunk_dest_default( + index('infraops') + sourcetype('beyondtrust:sra') + vendor("beyondtrust") + product("sra") + template('t_msg_only') + ); + set-tag("isStructured"); + }; + parser { + regexp-parser( + prefix(".tmp.") + patterns('^(?\d+):(?\d+):(?\d+):(?.*)') + ); + kv-parser( + prefix(".values.") + pair-separator(";") + template('${.tmp.message}') + ); + }; + rewrite { + set('${.tmp.siteid}' value('.values.siteid')); + set('${.tmp.seq}' value('.values.seq')); + set(int("${.tmp.num}") value('.metadata.sc4s.countOfParts')); + set('${.tmp.message}' value('.message')); + }; + }; +}; +application app-syslog-beyondtrust_sra[sc4s-syslog-pgm] { + filter { + "${PROGRAM}" eq "BG" + }; + parser { app-syslog-beyondtrust_sra(); }; +}; + + + +#Destination type formatters +block parser app-syslog-beyondtrust_sra-splunk_hec() { + channel { + rewrite{ + set("t_json_values", value(".splunk.sc4s_template")); + }; + }; +}; +application app-syslog-beyondtrust_sra-splunk_hec[sc4s-lp-dest-format-hec_fmt] { + filter { + match('beyondtrust', value('fields.sc4s_vendor') type(string)) + and match('sra', value('fields.sc4s_product') type(string)) + and ( + "`SC4S_DEST_BEYONDTRUST_SRA_SPLUNK_HEC_FMT`" eq "" + or "`SC4S_DEST_BEYONDTRUST_SRA_SPLUNK_HEC_FMT`" eq "JSON" + ) + }; + parser { app-syslog-beyondtrust_sra-splunk_hec(); }; +}; + +block parser app-syslog-beyondtrust_sra-syslog() { + channel { + parser { + map-value-pairs( + key('.values.*' rekey(shift-levels(2) add-prefix(".SDATA.sc4sfields@27389."))) + ); + map-value-pairs( + key('fields.*' rekey(shift-levels(2) add-prefix(".SDATA.sc4smeta@27389."))) + ); + }; + rewrite{ + unset(value('MESSAGE')); + }; + }; +}; +application app-syslog-beyondtrust_sra-syslog[sc4s-lp-dest-format-syslog] { + filter { + match('beyondtrust', value('fields.sc4s_vendor') type(string)) + and match('sra', value('fields.sc4s_product') type(string)) + and ( + "`SC4S_DEST_BEYONDTRUST_SRA_SYSLOG_FMT`" eq "" + or "`SC4S_DEST_BEYONDTRUST_SRA_SYSLOG_FMT`" eq "SDATA" + ) + }; + parser { app-syslog-beyondtrust_sra-syslog(); }; +}; + +block parser app-syslog-beyondtrust_sra-bsd() { + channel { + rewrite{ + set('$(template ${.splunk.sc4s_template:-t_hdr_msg})' value('MESSAGE')); + }; + }; +}; +application app-syslog-beyondtrust_sra-bsd[sc4s-lp-dest-format-bsd] { + parser { app-syslog-beyondtrust_sra-bsd(); }; +}; diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-broadcom_sslva.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-broadcom_sslva.conf new file mode 100644 index 0000000000..d35b406a5f --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-broadcom_sslva.conf @@ -0,0 +1,25 @@ +block parser app-syslog-broadcom_sslva() { + channel { + rewrite { + r_set_splunk_dest_default( + index('netproxy') + sourcetype('broadcom:sslva') + vendor("broadcom") + product("sslva") + template('t_hdr_msg') + ); + }; + + + }; +}; +application app-syslog-broadcom_sslva[sc4s-syslog-pgm] { + filter { + program('ssldata' type(string) flags(prefix)) + or program('sslpilot' type(string) flags(prefix)) + ; + }; + parser { app-syslog-broadcom_sslva(); }; +}; + + diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-cef.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-cef.conf new file mode 100644 index 0000000000..8e53230d92 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-cef.conf @@ -0,0 +1,155 @@ +parser p_cef_kv { + python( + class("parser_cef.cef_kv") + ); +}; + + +parser p_cef_ts_rt { + date-parser-nofilter(format( + '%s.%f', + '%s', + '%b %d %T', + '%b %d %T UTC', + '%b %d %T %Z', + '%b %d %Y %T', + '%b %d %Y %T UTC', + '%b %d %Y %T %Z', + ) + template("${.values.rt}") + ); +}; +parser p_cef_ts_end { + date-parser-nofilter(format( + '%s.%f', + '%s', + '%b %d %T', + '%b %d %T UTC', + '%b %d %T %Z', + '%b %d %Y %T', + '%b %d %Y %T UTC', + '%b %d %Y %T %Z', + ) + template("${.values.end}") + ); +}; + +block parser app-syslog-cef() { + channel { + rewrite { + r_set_splunk_dest_default( + index('main') + sourcetype('cef') + vendor("cef") + product('generic') + ); + + }; + if (message('CEF:0', type('string') flags(prefix))) { + rewrite{ + unset(value('PROGRAM')); + }; + }; + if (program('CEF:0', type('string') flags(prefix))) { + rewrite{ + set('${.values.message}' value('MESSAGE')); + unset(value('PROGRAM')); + }; + }; + if { + parser { + regexp-parser( + prefix(".tmp.") + patterns('^(?(?!-)[a-zA-Z-\/0-9\.]+).*(?CEF:0.*)') + ); + }; + rewrite { + set('${.tmp.message}' value('MESSAGE')); + set('${.tmp.preamble}' value('HOST')); + }; + }; + + + if ("`SC4S_DISABLE_DROP_INVALID_CEF`" eq "yes") { + parser { + csv-parser( + delimiters(chars('') strings('|')) + columns('version', 'device_vendor', 'device_product', 'device_version', 'device_event_class', 'name', 'severity', 'ext') + prefix('.metadata.cef.') + flags(greedy)); + }; + } elif { + parser { + csv-parser( + delimiters(chars('') strings('|')) + columns('version', 'device_vendor', 'device_product', 'device_version', 'device_event_class', 'name', 'severity', 'ext') + prefix('.metadata.cef.') + flags(greedy, drop-invalid)); + }; + } else {}; + parser(p_cef_kv); + rewrite { + unset(value('.metadata.cef.ext')); + set-tag("isStructured"); + r_set_splunk_dest_default( + vendor('${.metadata.cef.device_vendor}') + product('${.metadata.cef.device_product}') + class('${.metadata.cef.device_event_class}') + template('t_hdr_msg') + ); + }; + + # We already have the syslog msg time stamp however that may not be the best one + # If we have an rt or end field that is best we use the If trick here so if this parser fails + # We don't get sent to fallback. + if { + # 12 digit epoch timestamps are non-standard; when used they often indicate the fields are misused + # Non-standard strptime formats also choke the syslog-ng date parser, which outputs wildy random timestamps + # Simply filter and ignore + filter{ + match('^\d{12}', value('.values.start')) or match('^\d{12}', value('.values.end')) or match('^\d{12}', value('.values.rt')); + }; + } elif { + filter{ + match('^.', value('.values.rt')) + }; + parser (p_cef_ts_rt); + } elif { + filter{ + match('^.', value('.values.end')) + }; + parser (p_cef_ts_end); + } else { }; + + #Do nothing this is allows for both rt and end to be missing and still pass with the message ts + rewrite { + set( + "${.values.dvchost}", + value("HOST") + condition("${.values.dvchost}" ne "") + ); + }; + # CEF TAs use the source as their bounds in props.conf + # CEF source entries in splunk_metadata.csv should reflect the proper source as well as sourcetype + + if { + parser { app-parser(topic(cef)); }; + }; + + + }; +}; + +application app-syslog-cef-pgm[sc4s-syslog-pgm] { + filter { + program('CEF' type(string) flags(prefix)); + }; + parser { app-syslog-cef(); }; +}; + +application app-syslog-cef[sc4s-syslog] { + filter { + message('CEF:0' type(string) flags(substring)); + }; + parser { app-syslog-cef(); }; +}; \ No newline at end of file diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-checkpoint_syslog.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-checkpoint_syslog.conf new file mode 100644 index 0000000000..8f6408f40d --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-checkpoint_syslog.conf @@ -0,0 +1,325 @@ +block parser app-syslog-checkpoint_syslog() { + channel { + + parser { + date-parser-nofilter(format("%s") template("${.SDATA.sc4s@2620.time}")); + }; + + rewrite { + set("${.SDATA.sc4s@2620.hostname}", value("HOST") condition("${.SDATA.sc4s@2620.hostname}" ne "")); + set("${.SDATA.sc4s@2620.hostname}", value("fields.cp_lm") condition("${.SDATA.sc4s@2620.hostname}" ne "")); + + r_set_splunk_dest_default( + index('netfw') + sourcetype('cp_log:syslog') + vendor("checkpoint") + product("syslog") + template('t_5424_hdr_sdata_compact') + ); + }; + + + if ( + "`SC4S_LISTEN_CHECKPOINT_SPLUNK_OLD_HOST_RULES`" eq "yes") + { + if { + parser { + regexp-parser( + patterns('^[Cc][Nn]\\?=(?[^,]+)') + template('${.SDATA.sc4s@2620.originsicname}') + ); + }; + } elif { + #Get the FW host from the origin_sic_name + parser { + regexp-parser( + patterns('^[Cc][Nn]\\?=(?[^,]+)') + template('${.SDATA.sc4s@2620.origin_sic_name}') + ); + }; + } elif { + # If hostname field is not empty, host should be evaluated from there + parser { + regexp-parser( + patterns('(?\S+)') + template('${.SDATA.sc4s@2620.hostname}') + ); + }; + }; + } else { + if { + # If hostname field is not empty, host should be evaluated from there + parser { + regexp-parser( + patterns('(?\S+)') + template('${.SDATA.sc4s@2620.hostname}') + ); + }; + } elif { + #Get the FW host from the originsicname + parser { + regexp-parser( + patterns('^[Cc][Nn]\\?=(?[^,]+)') + template('${.SDATA.sc4s@2620.originsicname}') + ); + }; + } elif { + #Get the FW host from the origin_sic_name + parser { + regexp-parser( + patterns('^[Cc][Nn]\\?=(?[^,]+)') + template('${.SDATA.sc4s@2620.origin_sic_name}') + ); + }; + }; + }; + # If this device is a v host we need to get the firewall name + if { + parser { + regexp-parser( + patterns('-v_(?.*)') + template("$HOST") + ); + }; + }; + + if { + filter { + "${.SDATA.sc4s@2620.product}" eq "Syslog" + }; + parser { + syslog-parser( + flags(assume-utf8, no-hostname) + template("${.SDATA.sc4s@2620.default_device_message}") + ); + }; + parser { + date-parser-nofilter(format("%s") template("${.SDATA.sc4s@2620.time}")); + }; + rewrite { + r_set_splunk_dest_update_v2( + class('nix_${PROGRAM}') + index('netops') + source('program:${PROGRAM}') + sourcetype('nix:syslog') + ); + + }; + } elif { + filter { + match('Firewall' value('.SDATA.sc4s@2620.product')) + or match('Application\h+Control' value('.SDATA.sc4s@2620.product')) + or match('RAD' value('.SDATA.sc4s@2620.product')) + or match('HTTPS\h+Inspection' value('.SDATA.sc4s@2620.product')) + or match('Compliance\h+Blade' value('.SDATA.sc4s@2620.product')) + or match('^Compliance' value('.SDATA.sc4s@2620.product')) + or match('VPN-1\h+&\h+Fire[wW]all-1' value('.SDATA.sc4s@2620.product')) + or match('Network\h+Security' value('.SDATA.sc4s@2620.product')) + or match('FG\h+VPN-1\h+&\h+FireWall-1' value('.SDATA.sc4s@2620.product')) + or match('QoS' value('.SDATA.sc4s@2620.product')) + and not match('VPN' value('.SDATA.sc4s@2620.fw_subproduct')) + }; + rewrite { + r_set_splunk_dest_update_v2( + class('firewall') + index('netfw') + source('checkpoint:firewall') + ); + }; + } elif { + filter { + match('U[rR][lL]\h+Filtering' value('.SDATA.sc4s@2620.product')) + }; + rewrite { + r_set_splunk_dest_update_v2( + class('web') + index('netproxy') + source('checkpoint:web') + ); + }; + } elif { + filter { + match('Scheduled\h+system\h+update' value('.SDATA.sc4s@2620.product')) + or match('WEB_API' value('.SDATA.sc4s@2620.product')) + or match('SmartDefense' value('.SDATA.sc4s@2620.product')) + or match('Smart\h+Defense' value('.SDATA.sc4s@2620.product')) + or match('W[eE][bB]-UI' value('.SDATA.sc4s@2620.product')) + or match('SmartDashboard' value('.SDATA.sc4s@2620.product')) + or match('System\h+Monitor' value('.SDATA.sc4s@2620.product')) + or match('Log\h+Update' value('.SDATA.sc4s@2620.product')) + or match('license-mgmt' value('.SDATA.sc4s@2620.product')) + or match('smart_event' value('.SDATA.sc4s@2620.product')) + or match('SmartConsole' value('.SDATA.sc4s@2620.product')) + or match('SmartEvent\h+Client' value('.SDATA.sc4s@2620.product')) + or match('SmartUpdate' value('.SDATA.sc4s@2620.product')) + or match('SmartView' value('.SDATA.sc4s@2620.product')) + or match('Security\h+Gateway\/Management' value('.SDATA.sc4s@2620.product')) + or match('Smart\h+Defense' value('.SDATA.sc4s@2620.product')) + or match('WEB_API_INTERNAL' value('.SDATA.sc4s@2620.product')) + or match('Eventia\h+Analyzer\h+Client' value('.SDATA.sc4s@2620.product')) + or match('SmartProvisioning\h+Connector' value('.SDATA.sc4s@2620.product')) + or match('SmartLSM\h+Endpoint\h+Security\h+Console' value('.SDATA.sc4s@2620.product')) + or match('SmartLSM' value('.SDATA.sc4s@2620.product')) + or match('ROBO\h+GUI' value('.SDATA.sc4s@2620.product')) + or match('Management\h+Blade' value('.SDATA.sc4s@2620.product')) + or match('Connectra' value('.SDATA.sc4s@2620.product')) + or match('Check\h+Point\h+Security\h+Management\h+Server' value('.SDATA.sc4s@2620.product')) + or match('CLI' value('.SDATA.sc4s@2620.product')) + or match('Check\h+Point\h+GO\h+Password\h+Reset' value('.SDATA.sc4s@2620.product')) + or match('Database\h+Tool' value('.SDATA.sc4s@2620.product')) + or match('cpmidu_update_tool' value('.SDATA.sc4s@2620.product')) + or match('query-database' value('.SDATA.sc4s@2620.product')) + }; + rewrite { + r_set_splunk_dest_update_v2( + class('audit') + index('netops') + source('checkpoint:audit') + ); + }; + } elif { + filter { + match('Endpoint\h+Management' value('.SDATA.sc4s@2620.product')) + or match('Core' value('.SDATA.sc4s@2620.product')) + or match('Endpoint\h+Compliance' value('.SDATA.sc4s@2620.product')) + or match('MEPP' value('.SDATA.sc4s@2620.product')) + or match('Media\h+Encryption\h+&\h+Port\h+Protection' value('.SDATA.sc4s@2620.product')) + or match('Endpoint\h+Security\h+Console' value('.SDATA.sc4s@2620.product')) + or match('endpoint' value('.SDATA.sc4s@2620.product')) + }; + rewrite { + r_set_splunk_dest_update_v2( + class('endpoint') + index('netops') + source('checkpoint:endpoint') + ); + }; + } elif { + filter { + match('^VPN$' value('.SDATA.sc4s@2620.product')) + or match('^Mobile$' value('.SDATA.sc4s@2620.product')) + or match('Mobile\h+App' value('.SDATA.sc4s@2620.product')) + or match('VPN' value('.SDATA.sc4s@2620.fw_subproduct')) + or match('VPN-1' value('.SDATA.sc4s@2620.fw_subproduct')) + or match('Identity\h+Awareness' value('.SDATA.sc4s@2620.product')) + }; + rewrite { + r_set_splunk_dest_update_v2( + class('sessions') + index('netops') + source('checkpoint:sessions') + ); + }; + } elif { + filter { + match('IOS\h+Profile' value('.SDATA.sc4s@2620.product')) + or match('iOS\h+Profiles' value('.SDATA.sc4s@2620.product')) + or match('Device' value('.SDATA.sc4s@2620.product')) + or match('WIFI\h+Network' value('.SDATA.sc4s@2620.product')) + or match('Mobile\h+Access' value('.SDATA.sc4s@2620.product')) + }; + rewrite { + r_set_splunk_dest_update_v2( + class('network') + index('netops') + source('checkpoint:network') + ); + }; + } elif { + filter { + match('Threat\h+Emulation' value('.SDATA.sc4s@2620.product')) + or match('Anti-Virus' value('.SDATA.sc4s@2620.product')) + or match('New\h+Anti\h+Virus' value('.SDATA.sc4s@2620.product')) + or match('Anti-Bot' value('.SDATA.sc4s@2620.product')) + or match('Threat\h+Extraction' value('.SDATA.sc4s@2620.product')) + or match('Anti-Ransomware' value('.SDATA.sc4s@2620.product')) + or match('Anti-Exploit' value('.SDATA.sc4s@2620.product')) + or match('Forensics' value('.SDATA.sc4s@2620.product')) + or match('OS\h+Exploit' value('.SDATA.sc4s@2620.product')) + or match('OS\h+Exploits' value('.SDATA.sc4s@2620.product')) + or (match('Application' value('.SDATA.sc4s@2620.product')) and not match('Application Control' value('.SDATA.sc4s@2620.product'))) + or match('Text\h+Message' value('.SDATA.sc4s@2620.product')) + or match('Network\h+Access' value('.SDATA.sc4s@2620.product')) + or match('Zero\h+Phishing' value('.SDATA.sc4s@2620.product')) + or match('Anti-Malware' value('.SDATA.sc4s@2620.product')) + or match('Anti\h+Malware' value('.SDATA.sc4s@2620.product')) + or match('Anti\h+Malware\h+New\h+Anti\h+Virus' value('.SDATA.sc4s@2620.product')) + or match('New\h+Anti\h+Virus\h+Anti\h+Malware' value('.SDATA.sc4s@2620.product')) + }; + rewrite { + r_set_splunk_dest_update_v2( + class('ids_malware') + index('netids') + source('checkpoint:ids_malware') + ); + }; + } elif { + filter { + match('IPS' value('.SDATA.sc4s@2620.product')) + or match('W[iI][fF][iI]' value('.SDATA.sc4s@2620.product')) + or match('Cellular' value('.SDATA.sc4s@2620.product')) + }; + rewrite { + r_set_splunk_dest_update_v2( + class('ids') + index('netids') + source('checkpoint:ids') + ); + }; + } elif { + filter { + match('MTA' value('.SDATA.sc4s@2620.product')) + or match('Anti-Spam' value('.SDATA.sc4s@2620.product')) + or match('Anti\h+Spam' value('.SDATA.sc4s@2620.product')) + or match('Anti-Phishing' value('.SDATA.sc4s@2620.product')) + or match('Anti\h+Phishing' value('.SDATA.sc4s@2620.product')) + or match('Anti-Spam\h+and\h+Email\h+Security' value('.SDATA.sc4s@2620.product')) + or match('Anti\h+Spam\h+and\h+Email\h+Security' value('.SDATA.sc4s@2620.product')) + }; + rewrite { + r_set_splunk_dest_update_v2( + class('email') + index('email') + source('checkpoint:email') + ); + }; + } elif { + filter { + match('DLP' value('.SDATA.sc4s@2620.product')) + }; + rewrite { + r_set_splunk_dest_update_v2( + class('dlp') + index('netfw') + source('checkpoint:firewall') + ); + }; + } elif { + filter { + match('Syslog' value('.SDATA.sc4s@2620.product')) + }; + rewrite { + r_set_splunk_dest_update_v2( + class('os') + index('netops') + ); + }; + } else { + rewrite { + r_set_splunk_dest_update_v2( + index('netops') + source('checkpoint:cp_default') + ); + }; + }; + }; + +}; +application app-syslog-checkpoint_syslog[sc4s-syslog-pgm] { + filter { + program('CheckPoint' type(string) flags(prefix)); + }; + + parser { app-syslog-checkpoint_syslog(); }; +}; + diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-cisco_acs.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-cisco_acs.conf new file mode 100644 index 0000000000..19e840e444 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-cisco_acs.conf @@ -0,0 +1,56 @@ +#This parser adds messages from ACS to a context without sending them on to destination + +parser p_acs_event_time { + csv-parser( + columns(DATE, TIME, TZ) + prefix(".tmp.") + delimiters(chars(" ")) + template('${.values.message}') + ); + + date-parser-nofilter( + #YYYY- MM-DD hh:mm:ss:xxx +/-zh:zm + format('%Y-%m-%d %H:%M:%S.%f %z') + template("${.tmp.DATE} ${.tmp.TIME} ${.tmp.TZ}") + ); +}; + +block parser app-syslog-cisco_acs() { + channel { + + parser { + csv-parser( + columns(serial, num, seq, message) + prefix(".values.") + delimiters(chars(" ")) + flags(greedy) + ); + }; + + rewrite { + set(int("${.values.num:-0}") value(".values.num")); + }; + + if { + filter {"${.values.seq}" eq "0"}; + parser(p_acs_event_time); + }; + + rewrite { + r_set_splunk_dest_default( + index('netauth') + sourcetype('cisco:acs') + vendor("cisco") + product('acs') + ); + }; + + + }; +}; +application app-syslog-cisco_acs[sc4s-syslog-pgm] { + filter { + program('CSCOacs' type(string) flags(prefix)); + }; + parser { app-syslog-cisco_acs(); }; +}; diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-cisco_dna_syslog.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-cisco_dna_syslog.conf new file mode 100644 index 0000000000..226bc6c151 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-cisco_dna_syslog.conf @@ -0,0 +1,24 @@ +block parser app-syslog-cisco_dna_syslog() { + channel { + rewrite { + r_set_splunk_dest_default( + index("netops") + sourcetype('cisco:dna') + vendor('cisco') + product('dna') + template("t_msg_only") + ); + }; + rewrite { + set("$SOURCEIP", value("HOST") condition("`SC4S_SOURCE_CISCO_DNA_FIXHOST`" eq "yes")); + }; + + }; +}; +application app-syslog-cisco_dna_syslog[sc4s-syslog-pgm] { + filter { + program('DNAC' type(string) ,flags(ignore-case)) + ; + }; + parser { app-syslog-cisco_dna_syslog(); }; +}; diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-cisco_firepower.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-cisco_firepower.conf new file mode 100644 index 0000000000..9d776a7032 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-cisco_firepower.conf @@ -0,0 +1,22 @@ +block parser app-syslog-cisco_firepower() { + channel { + rewrite { + r_set_splunk_dest_default( + index('netids') + sourcetype('cisco:firepower:syslog') + vendor("cisco") + product('firepower') + template('t_hdr_msg') + ); + }; + + + }; +}; +application app-syslog-cisco_firepower[sc4s-syslog-pgm] { + filter { + program('SFIMS' type(string) flags(prefix)); + }; + parser { app-syslog-cisco_firepower(); }; +}; + diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-cisco_ise.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-cisco_ise.conf new file mode 100644 index 0000000000..ebdb9136bd --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-cisco_ise.conf @@ -0,0 +1,77 @@ +#This parser adds messages from ACS to a context without sending them on to destination + +parser ise_event_time { + csv-parser( + columns(DATE, TIME, TZ) + prefix(".tmp.") + delimiters(chars(" ")) + template('${.values.message}') + ); + + date-parser-nofilter( + #YYYY- MM-DD hh:mm:ss:xxx +/-zh:zm + format('%Y-%m-%d %H:%M:%S.%f %z') + template("${.tmp.DATE} ${.tmp.TIME} ${.tmp.TZ}") + ); +}; + +block parser app-syslog-cisco_ise() { + + channel { + + if { + parser { + regexp-parser( + template("${MESSAGE}") + patterns("^(?\\d{2}) (?\\d{2}:\\d{2}:\\d{2}) (?[^ ]+) (?[^ ]+) (?.*)") + prefix(".parsed.") + ); + + date-parser-nofilter( + format('%b %d %H:%M:%S') + template("${PROGRAM} ${.parsed.real_day} ${.parsed.real_hour}") + ); + }; + rewrite { + set("${.parsed.real_host}" value("HOST")); + set("${.parsed.real_program}" value("PROGRAM")); + set("${.parsed.rest_of_message}" value("MESSAGE")); + }; + }; + + parser { + csv-parser( + columns(serial, num, seq, message) + prefix(".values.") + delimiters(chars(" ")) + flags(greedy) + ); + }; + + rewrite { + set(int("${.values.num:-0}") value(".values.num")); + }; + + if { + filter {"${.values.seq}" eq "0"}; + parser(ise_event_time); + }; + + rewrite { + r_set_splunk_dest_default( + index('netauth') + sourcetype('cisco:ise:syslog') + vendor("cisco") + product('ise') + ); + }; + }; +}; + +application app-syslog-cisco_ise[sc4s-syslog-pgm] { + filter { + program('CISE_' type(string) flags(prefix)) + or message('CISE_' type(string) flags(substring)); + }; + parser { app-syslog-cisco_ise(); }; +}; diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-cisco_mm.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-cisco_mm.conf new file mode 100644 index 0000000000..52a0a591fa --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-cisco_mm.conf @@ -0,0 +1,36 @@ +block parser app-syslog-cisco_mm() { + channel { + rewrite { + r_set_splunk_dest_default( + index('netops') + sourcetype('cisco:mm') + sourcetype('cisco:mm:system') + vendor("cisco") + product("mm") + class('system') + template('t_msg_only') + ); + }; + if { + parser { + regexp-parser( + prefix(".tmp.") + patterns('^cmm-(?.*)') + template('$PROGRAM') + ); + }; + rewrite{ + r_set_splunk_dest_update_v2( + sourcetype('cisco:mm:system:${.tmp.source}') + ); + }; + }; + }; +}; +application app-syslog-cisco_mm[sc4s-syslog-pgm] { + filter { + program('cmm-' type(string) flags(prefix)) + }; + parser { app-syslog-cisco_mm(); }; +}; + diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-cisco_syslog.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-cisco_syslog.conf new file mode 100644 index 0000000000..76512862ee --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-cisco_syslog.conf @@ -0,0 +1,35 @@ +block parser app-syslog-cisco_syslog() { + channel { + rewrite { + subst('^: ', "" , value("MESSAGE")); + }; + parser { + regexp-parser( + prefix(".values.") + patterns('^(%(?.+)-(?[0-7])-(?[^\: ]+))(? ?[: ]) *(?.*)') + ); + }; + + rewrite { + r_set_splunk_dest_default( + index('netops') + sourcetype('cisco:ios') + vendor("cisco") + product('ios') + ); + }; + + if { + parser { app-parser(topic(cisco_syslog)); }; + }; + + }; +}; +application app-syslog-cisco_syslog[sc4s-syslog] { + filter { + not program('[a-z][^\[\] ]+$') + and message('(?:: )?%'); + }; + parser { app-syslog-cisco_syslog(); }; +}; + diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-cisco_tvcs.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-cisco_tvcs.conf new file mode 100644 index 0000000000..b5a0ec25b6 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-cisco_tvcs.conf @@ -0,0 +1,20 @@ +block parser app-syslog-cisco_tvcs() { + channel { + rewrite { + r_set_splunk_dest_default( + sourcetype('cisco:tvcs') + vendor("cisco") + product('tvcs') + ); + }; + + + }; +}; +application app-syslog-cisco_tvcs[sc4s-syslog-pgm] { + filter { + program('tvcs' type(string) flags(prefix)); + }; + parser { app-syslog-cisco_tvcs(); }; +}; + diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-cisco_ucs_hx.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-cisco_ucs_hx.conf new file mode 100644 index 0000000000..cdc0d0a390 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-cisco_ucs_hx.conf @@ -0,0 +1,24 @@ +block parser app-syslog-cisco_ucs_hx() { + channel { + rewrite { + r_set_splunk_dest_default( + index("infraops") + source("program:${PROGRAM}") + sourcetype('cisco:ucs:hx') + vendor("cisco") + product('ucs') + class("hx") + ); + }; + + + }; +}; +application app-syslog-cisco_ucs_hx[sc4s-syslog-pgm] { + filter { + program('hx-' type(string) flags(prefix)); + }; + parser { app-syslog-cisco_ucs_hx(); }; +}; + + diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-citrix-netscaler_aaa.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-citrix-netscaler_aaa.conf new file mode 100644 index 0000000000..a7ec7bd680 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-citrix-netscaler_aaa.conf @@ -0,0 +1,25 @@ +block parser app-syslog-citrix-netscaler_aaa() { + channel { + rewrite { + set("$(template t_hdr_msg)" value("MSG")); + unset(value("PROGRAM")); + unset(value("PID")); + r_set_splunk_dest_default( + index('netfw') + sourcetype('citrix:netscaler:syslog') + vendor("citrix") + product("netscaler") + template("t_hdr_msg") + ); + }; + + + }; +}; +application app-syslog-citrix-netscaler_aaa[sc4s-syslog] { + filter { + message('^\[\d+\]: [^:]+: ') + and "$PROGRAM" eq ""; + }; + parser { app-syslog-citrix-netscaler_aaa(); }; +}; diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-citrix_netscaler_svm.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-citrix_netscaler_svm.conf new file mode 100644 index 0000000000..e70a1ebfc1 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-citrix_netscaler_svm.conf @@ -0,0 +1,50 @@ +block parser app-syslog-citrix_netscaler_svm() { + channel { + + parser { + regexp-parser( + template("${MESSAGE}") + prefix(".tmp.") + patterns('(?[^\s]*)\s+(?(?\d\d)[^\s]*(?:\s[^\s]+)?)(?:\s+\S+){2}\s+:') + ); + }; + if { + filter { "${.tmp.tspart1}" eq "$S_DAY"}; + parser { + date-parser-nofilter( + format('%d/%m/%Y:%H:%M:%S %z','%d/%m/%Y:%H:%M:%S') + template("${.tmp.timestamp}") + ); + }; + } else { + parser { + date-parser-nofilter( + format('%m/%d/%Y:%H:%M:%S %z','%m/%d/%Y:%H:%M:%S') + template("${.tmp.timestamp}") + ); + }; + + }; + + rewrite { + set('${.tmp.host}' value('HOST')); + r_set_splunk_dest_default( + index('netfw') + sourcetype('citrix:netscaler:syslog') + vendor("citrix") + product("netscaler") + template("t_hdr_msg") + ); + r_set_splunk_dest_update_v2( + sourcetype('citrix:netscaler:appfw') condition(message(':(\s+\S+)?\s+APPFW(\s+\S+){3}\s+:')) + ); + }; + }; +}; +application app-syslog-citrix_netscaler_svm[sc4s-syslog-pgm] { + filter { + program('svm_' type(string) flags(prefix)) + }; + parser { app-syslog-citrix_netscaler_svm(); }; +}; + diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-cohesity_alerts.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-cohesity_alerts.conf new file mode 100644 index 0000000000..af4680465f --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-cohesity_alerts.conf @@ -0,0 +1,26 @@ +block parser app-syslog-cohesity_alerts() { + channel { + parser { + json-parser( + prefix('.values.') + ); + }; + rewrite { + r_set_splunk_dest_default( + index('infraops') + sourcetype('cohesity:alerts') + vendor("cohesity") + product("alerts") + template('t_msg_only') + ); + }; + }; +}; + +application app-syslog-cohesity_alerts[sc4s-syslog-pgm] { + filter { + program("cohesity_alerts" type(string) flags(prefix)); + }; + parser { app-syslog-cohesity_alerts(); }; +}; + diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-cohesity_api_audit.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-cohesity_api_audit.conf new file mode 100644 index 0000000000..705ca09b04 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-cohesity_api_audit.conf @@ -0,0 +1,27 @@ +block parser app-syslog-cohesity_api_audit() { + channel { + parser { + json-parser( + prefix('.values.') + ); + }; + rewrite { + r_set_splunk_dest_default( + index('infraops') + sourcetype('cohesity:api:audit') + vendor("cohesity") + product("api") + class('audit') + template('t_msg_only') + ); + }; + + }; +}; +application app-syslog-cohesity_api_audit[sc4s-syslog-pgm] { + filter { + program("api_audit" type(string) flags(prefix)); + }; + parser { app-syslog-cohesity_api_audit(); }; +}; + diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-cohesity_cluster_audit.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-cohesity_cluster_audit.conf new file mode 100644 index 0000000000..26fe060ab5 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-cohesity_cluster_audit.conf @@ -0,0 +1,37 @@ +block parser app-syslog-cohesity_cluster_audit() { + channel { + parser { + json-parser( + prefix('.values.') + ); + date-parser-nofilter( + format( + '%Y-%m-%dT%T.%f%z', + '%Y-%m-%dT%T%z', + ) + template("${.values.Timestamp}") + ); + }; + filter { + "${.values.ClusterInfo}" ne ""; + }; + rewrite { + r_set_splunk_dest_default( + index('infraops') + sourcetype('cohesity:cluster:audit') + vendor("cohesity") + product("cluster") + class('audit') + template('t_msg_only') + ); + }; + + }; +}; +application app-syslog-cohesity_cluster_audit[sc4s-syslog-pgm] { + filter { + "${PROGRAM}" eq "cluster_audit"; + }; + parser { app-syslog-cohesity_cluster_audit(); }; +}; + diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-cohesity_cluster_dataprotection.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-cohesity_cluster_dataprotection.conf new file mode 100644 index 0000000000..c5f48a7e17 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-cohesity_cluster_dataprotection.conf @@ -0,0 +1,37 @@ +block parser app-syslog-cohesity_cluster_dataprotection() { + channel { + parser { + json-parser( + prefix('.values.') + ); + date-parser-nofilter( + format( + '%Y-%m-%dT%T.%f%z', + '%Y-%m-%dT%T%z', + ) + template("${.values.Timestamp}") + ); + }; + filter { + "${.values.EventType}" ne ""; + }; + rewrite { + r_set_splunk_dest_default( + index('infraops') + sourcetype('cohesity:cluster:dataprotection') + vendor("cohesity") + product("cluster") + class('dataprotection') + template('t_msg_only') + ); + }; + + }; +}; +application app-syslog-cohesity_cluster_dataprotection[sc4s-syslog-pgm] { + filter { + "${PROGRAM}" eq "dataprotection_events"; + }; + parser { app-syslog-cohesity_cluster_dataprotection(); }; +}; + diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-cylance_protect.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-cylance_protect.conf new file mode 100644 index 0000000000..4d041b606d --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-cylance_protect.conf @@ -0,0 +1,120 @@ +block parser app-syslog-cylance_protect() { + channel { + + + rewrite { + r_set_splunk_dest_default( + index('epintel') + sourcetype('syslog_protect') + vendor("cylance") + product("protect") + ); + }; + if { + filter{ + message('Event\sType:\s+AuditLog'); + }; + rewrite { + r_set_splunk_dest_update_v2( + class('auditlog') + index('epintel') + sourcetype('syslog_audit_log') + ); + }; + } elif { + filter{ + message('Event\sType:\s+ThreatClassification'); + }; + rewrite { + r_set_splunk_dest_update_v2( + class('threatclassification') + index('epintel') + sourcetype('syslog_threat_classification') + ); + }; + } elif { + filter{ + message('Event\sType:\s+ExploitAttempt'); + }; + rewrite { + r_set_splunk_dest_update_v2( + class('exploitattempt') + index('epintel') + sourcetype('syslog_exploit') + ); + }; + } elif { + filter{ + message('Event\sType:\s+AppControl'); + }; + rewrite { + r_set_splunk_dest_update_v2( + class('appcontrol') + index('epintel') + sourcetype('syslog_app_control') + ); + }; + } elif { + filter{ + message('Event\sType:\s+Threat,'); + }; + rewrite { + r_set_splunk_dest_update_v2( + class('threat') + index('epintel') + sourcetype('syslog_threat') + ); + }; + } elif { + filter{ + message('Event\sType:\s+Device'); + }; + rewrite { + r_set_splunk_dest_update_v2( + class('device') + index('epintel') + sourcetype('syslog_device') + ); + }; + } elif { + filter{ + message('Event\sType:\s+DeviceControl'); + }; + rewrite { + r_set_splunk_dest_update_v2( + class('devicecontrol') + index('epintel') + sourcetype('syslog_device_control') + ); + }; + } elif { + filter{ + message('Event\sType:\s+ScriptControl'); + }; + rewrite { + r_set_splunk_dest_update_v2( + class('scriptcontrol') + index('epintel') + sourcetype('syslog_script_control') + ); + }; + } elif { + filter{ + message('Event\sType:\s+Optics'); + }; + rewrite { + r_set_splunk_dest_update_v2( + class('optics') + index('epintel') + sourcetype('syslog_optics') + ); + }; + }; + }; +}; +application app-syslog-cylance_protect[sc4s-syslog-pgm] { + filter { + program('CylancePROTECT' type(string)) + }; + parser { app-syslog-cylance_protect(); }; +}; diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-darktrace_syslog.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-darktrace_syslog.conf new file mode 100644 index 0000000000..ee41eef2a8 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-darktrace_syslog.conf @@ -0,0 +1,37 @@ +block parser app-syslog-darktrace_syslog() { + channel { + rewrite { + r_set_splunk_dest_default( + index("netids") + sourcetype('darktrace') + vendor('darktrace') + product('syslog') + template('t_msg_only') + ); + }; + if { + filter { + program('darktrace_audit' type(string)) + }; + rewrite { + r_set_splunk_dest_update_v2( + class('audit') + sourcetype('darktrace:audit') + ); + }; + }; + if { + parser { + json-parser( + prefix('.values.') + ); + }; + }; + }; +}; +application app-syslog-darktrace_syslog[sc4s-syslog-pgm] { + filter { + program('darktrace' type(string) flags(prefix)) + }; + parser { app-syslog-darktrace_syslog(); }; +}; diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-dell_avamar.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-dell_avamar.conf new file mode 100644 index 0000000000..0e987d22d8 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-dell_avamar.conf @@ -0,0 +1,36 @@ +block parser app-syslog-dell_avamar() { + channel { + parser { + regexp-parser( + template("${MESSAGE}") + prefix(".tmp.") + patterns('<(?.*)') + ); + kv-parser( + prefix(".values.") + pair-separator("<") + value-separator(">") + template("${.tmp.message}") + ); + }; + rewrite { + r_set_splunk_dest_default( + index('netops') + sourcetype('dell:avamar:msc') + vendor('dell') + product('avamar') + class('msc') + template("t_json_values") + ); + }; + }; +}; + +application app-syslog-dell_avamar[sc4s-syslog] { + filter { + message('' type(string) flags(substring)) and + message('' type(string) flags(substring)) and + message('' type(string) flags(substring)); + }; + parser { app-syslog-dell_avamar(); }; +}; diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-dell_poweredge_idrac.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-dell_poweredge_idrac.conf new file mode 100644 index 0000000000..9733c627e9 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-dell_poweredge_idrac.conf @@ -0,0 +1,44 @@ +block parser app-syslog-dell_poweredge_idrac() { + channel { + + parser { + kv-parser( + value-separator(":") + prefix(".values.") + template("${PROGRAM}: ${MESSAGE}") + ); + + }; + + filter { + "${.values.Severity}" ne "" + and "${.values.Category}" ne "" + and "${.values.MessageID}" ne "" + and "${.values.Message}" ne "" + and match('[A-Z]{1,3}\d{1,4}' , value(".values.MessageID")) + }; + + rewrite { + r_set_splunk_dest_default( + index('infraops') + sourcetype('dell:poweredge:idrac:syslog') + vendor("dell") + product("poweredge") + class('idrac') + ); + }; + rewrite{ + set("${PROGRAM}: $MSG" value("MSG")); + unset(value('PROGRAM')); + }; + + + }; +}; +application app-syslog-dell_poweredge_idrac[sc4s-syslog] { + filter { + program('Severity' type(string) flags(prefix)); + }; + parser { app-syslog-dell_poweredge_idrac(); }; +}; + diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-dell_sonicwall-firewall.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-dell_sonicwall-firewall.conf new file mode 100644 index 0000000000..46db8976e9 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-dell_sonicwall-firewall.conf @@ -0,0 +1,100 @@ + +block parser app-syslog-dell_sonicwall-firewall() { + channel { + rewrite { + r_set_splunk_dest_default( + index("netfw") + sourcetype("dell:sonicwall") + vendor("dell") + product('sonicwall-firewall') + ); + }; + + if { + parser { + kv-parser( + prefix(".values.") + pair-separator(" ") + ); + regexp-parser( + prefix(".tmp.") + patterns('^(?[^ ]+ [^ ]+)') + template('${.values.time}') + ); + date-parser-nofilter( + format("%Y-%m-%d %H:%M:%S") + template("${.tmp.timestamp}") + ); + }; + }; + rewrite { + set('${.values.fw}' value('HOST') condition("${.values.fw}" ne "") ); + }; + }; +}; +application app-syslog-dell_sonicwall-firewall[sc4s-syslog] { + filter { + "$PROGRAM" eq "" + and message('^id=\S+\ssn\=\S+\stime='); + }; + + parser { app-syslog-dell_sonicwall-firewall(); }; +}; + +#Destination type formatters +block parser app-syslog-dell_sonicwall-firewall-splunk_hec() { + channel { + rewrite{ + set("t_json_values", value(".splunk.sc4s_template")); + }; + }; +}; +application app-syslog-dell_sonicwall-firewall-splunk_hec[sc4s-lp-dest-format-hec_fmt] { + filter { + match('dell', value('fields.sc4s_vendor') type(string)) + and match('sonicwall-firewall', value('fields.sc4s_product') type(string)) + and ( + "`SC4S_DEST_DELL_SONICWALL-FIREWALL_SPLUNK_HEC_FMT`" eq "" + or "`SC4S_DEST_DELL_SONICWALL-FIREWALL_SPLUNK_HEC_FMT`" eq "JSON" + ) + }; + parser { app-syslog-dell_sonicwall-firewall-splunk_hec(); }; +}; + +block parser app-syslog-dell_sonicwall-firewall-syslog() { + channel { + parser { + map-value-pairs( + key('.values.*' rekey(shift-levels(2) add-prefix(".SDATA.sc4sfields@27389."))) + ); + map-value-pairs( + key('fields.*' rekey(shift-levels(2) add-prefix(".SDATA.sc4smeta@27389."))) + ); + }; + rewrite{ + unset(value('MESSAGE')); + }; + }; +}; +application app-syslog-dell_sonicwall-firewall-syslog[sc4s-lp-dest-format-syslog] { + filter { + match('dell', value('fields.sc4s_vendor') type(string)) + and match('sonicwall-firewall', value('fields.sc4s_product') type(string)) + and ( + "`SC4S_DEST_DELL_SONICWALL-FIREWALL_SYSLOG_FMT`" eq "" + or "`SC4S_DEST_DELL_SONICWALL-FIREWALL_SYSLOG_FMT`" eq "SDATA" + ) + }; + parser { app-syslog-dell_sonicwall-firewall-syslog(); }; +}; + +block parser app-syslog-dell_sonicwall-firewall-bsd() { + channel { + rewrite{ + set('$(template ${.splunk.sc4s_template:-t_hdr_msg})' value('MESSAGE')); + }; + }; +}; +application app-syslog-dell_sonicwall-firewall-bsd[sc4s-lp-dest-format-bsd] { + parser { app-syslog-dell_sonicwall-firewall-bsd(); }; +}; diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-epic_ehr.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-epic_ehr.conf new file mode 100644 index 0000000000..e8d69fd0fc --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-epic_ehr.conf @@ -0,0 +1,28 @@ +block parser app-syslog-epic_ehr() { + channel { + rewrite { + r_set_splunk_dest_default( + index("main") + source("epic-ehr") + sourcetype('epic:epic-ehr:syslog') + vendor("epic") + product("epic-ehr") + template('t_json_values') + ); + }; + parser { + xml( + prefix('.values.') + drop-invalid(no) + ); + }; + }; +}; + + +application app-syslog-epic_ehr[sc4s-syslog-pgm] { + filter { + program('Epic' type(string) flags(prefix)); + }; + parser { app-syslog-epic_ehr(); }; +}; diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-epp.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-epp.conf new file mode 100644 index 0000000000..d13b82587f --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-epp.conf @@ -0,0 +1,23 @@ +block parser app-syslog-netwrix_epp() { + channel { + rewrite { + r_set_splunk_dest_default( + index('netops') + source('netwrix:epp') + sourcetype('netwrix:epp') + vendor("netwrix") + product("epp") + ); + }; + + + }; +}; + +application app-syslog-netwrix_epp[sc4s-syslog-pgm] { + filter { + program('EPP-' type(string) flags(prefix)) + }; + + parser { app-syslog-netwrix_epp(); }; +}; \ No newline at end of file diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-example.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-example.conf new file mode 100644 index 0000000000..2d3a4fe22c --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-example.conf @@ -0,0 +1,20 @@ +block parser app-syslog-example() { + channel { + rewrite { + r_set_splunk_dest_default( + index('main') + sourcetype('sc4s:local_example') + vendor('local') + product('example') + ); + }; + }; +}; +application app-syslog-example[sc4s-syslog] { + filter { + program('sc4splugin' type(string)) + ; + }; + parser { app-syslog-example(); }; +}; + diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-f5_bigip.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-f5_bigip.conf new file mode 100644 index 0000000000..4f8ed26166 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-f5_bigip.conf @@ -0,0 +1,96 @@ +block parser app-syslog-f5_bigip() { + channel { + rewrite { + r_set_splunk_dest_default( + index("netops") + source("program:${PROGRAM}") + sourcetype('f5:bigip:syslog') + vendor('f5') + product('bigip') + ); + }; + if { + parser { + regexp-parser( + prefix(".tmp.") + template("$HOST") + patterns('^(?.*)\/(?.*)') + ); + }; + rewrite { + set('${.tmp.slot}' value('fields.slot')); + set('${.tmp.host}' value('HOST')); + }; + }; + + if { + filter{ + program('tmm' type(string) flags(prefix)) + }; + rewrite { + r_set_splunk_dest_update_v2( + sourcetype('f5:bigip:ltm:ssl:error') + condition(message('[0-9A-Fa-f]{4}\d+:\d{1,2}:\sConnection\serror:\s\w*:?:.*\(\d+\)')) + ); + }; + rewrite{ + r_set_splunk_dest_update_v2( + sourcetype('f5:bigip:ltm:tcl:error') + condition(message('[0-9A-Fa-f]{4}\d+:\d{1,2}:\sTCL\serror:\s.+\/(?:\s[^\s]+.*ERR_NOT_SUPPORTED)?')) + ); + }; + rewrite{ + r_set_splunk_dest_update_v2( + sourcetype('f5:bigip:apm:syslog') + condition(program('tmm' type(string) flags(prefix)) and message('^(?:\d{4})[0-9A-Fa-f]{4}:\d+:\s+(?:[^:\s]{1,100}:[^:\s]{1,100}:\s*)?[0-9a-f]{8}:')) + ); + }; + rewrite{ + r_set_splunk_dest_update_v2( + sourcetype('f5:bigip:ltm:traffic') + condition(message('[0-9A-Fa-f]{4}\d+:\d{1,2}:\sLimiting.+from\s\d+\sto\s\d+\spackets\/sec')) + ); + }; + rewrite{ + r_set_splunk_dest_update_v2( + sourcetype('f5:bigip:ltm:log:error') + condition(message('[0-9A-Fa-f]{4}\d+:\d{1,2}:\s\w+\s-\sInvalid action:\w+\s.*\((?:[0-9]{1,3}\.){3}[0-9]{1,3}:\d{1,5}\s->\s(?:[0-9]{1,3}\.){3}[0-9]{1,3}:\d{1,5}\)\sclientside\s\((?:[0-9]{1,3}\.){3}[0-9]{1,3}:\d{1,5}\s->\s(?:[0-9]{1,3}\.){3}[0-9]{1,3}:\d{1,5}\)')) + ); + }; + } elif { + filter{ + not (program('iControlPortal.cgi' type(string) flags(prefix)) + or program('tmsh' type(string) flags(prefix)) + or program('mcpd' type(string) flags(prefix)) + or program('mprov' type(string) flags(prefix)) + or program('apmd' type(string) flags(prefix)) + or program('sshd\(pam_audit\)') + or message('pam_radius_auth|authenticated|pam_bigip_authz|mod_auth_pam') + ); + }; + rewrite { + r_set_splunk_dest_update_v2( + class('nix_syslog') + sourcetype('nix:syslog') + ); + }; + } else {}; + + + }; +}; +application app-syslog-f5_bigip[sc4s-syslog-pgm] { + filter { + ( + program('iControlPortal.cgi' type(string) flags(prefix)) + or program('tmsh' type(string) flags(prefix)) + or program('mcpd' type(string) flags(prefix)) + or program('mprov' type(string) flags(prefix)) + or program('apmd' type(string) flags(prefix)) + or program('tmm' type(string) flags(prefix)) + or (program('F5' type(string) flags(prefix)) and not match('access_json' value('MSGID'))) + ) + ; + }; + parser { app-syslog-f5_bigip(); }; +}; diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-f5_bigip_irule_asm.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-f5_bigip_irule_asm.conf new file mode 100644 index 0000000000..6e79f98ed7 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-f5_bigip_irule_asm.conf @@ -0,0 +1,22 @@ +block parser app-syslog-f5_bigip_irule_asm() { + channel { + rewrite { + r_set_splunk_dest_default( + index("netwaf") + sourcetype('f5:bigip:asm:syslog') + vendor('f5') + product('bigip') + class("asm") + ); + }; + }; +}; +application app-syslog-f5_bigip_irule_asm[sc4s-syslog-pgm] { + filter { + program('ASM' type(string) flags(prefix)) + or message('^f5_asm=Splunk-F5-ASM') + ; + }; + parser { app-syslog-f5_bigip_irule_asm(); }; +}; + diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-f5_bigip_irule_json.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-f5_bigip_irule_json.conf new file mode 100644 index 0000000000..95ecf5d80e --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-f5_bigip_irule_json.conf @@ -0,0 +1,22 @@ +block parser app-syslog-f5_bigip_irule_json() { + channel { + rewrite { + r_set_splunk_dest_default( + index("netops") + sourcetype('f5:bigip:ltm:access_json') + vendor('f5') + product('bigip') + class("access_json") + ); + }; + + }; +}; +application app-syslog-f5_bigip_irule_json[sc4s-syslog-pgm] { + filter { + program('F5' type(string) flags(prefix)) and match('access_json' value('MSGID')) + ; + }; + parser { app-syslog-f5_bigip_irule_json(); }; +}; + diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-f5_bigip_irule_splunk.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-f5_bigip_irule_splunk.conf new file mode 100644 index 0000000000..4be8ad90d2 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-f5_bigip_irule_splunk.conf @@ -0,0 +1,47 @@ +block parser app-syslog-f5_bigip_irule_splunk() { + channel { + parser { + regexp-parser( + prefix(".tmp.") + patterns('^(?[^,]+),f5_irule=(?Splunk-[^,]+)') + ); + }; + rewrite { + r_set_splunk_dest_default( + index('netops') + source('f5:hsl') + sourcetype('f5:bigip:irule') + vendor('f5') + product('bigip') + class("irule") + ); + set("${.tmp.host}", value("HOST")); + set("${.tmp.category}", value(".f5.irule")); + }; + rewrite { + r_set_splunk_dest_update_v2( + sourcetype('f5:bigip:gtm:dns:request:irule') condition("${.f5.irule}" eq "Splunk-iRule-DNS_REQUEST") + ); + }; + rewrite { + r_set_splunk_dest_update_v2( + sourcetype('f5:bigip:gtm:dns:response:irule') condition("${.f5.irule}" eq "Splunk-iRule-DNS_RESPONSE") + ); + }; + rewrite { + r_set_splunk_dest_update_v2( + sourcetype('f5:bigip:ltm:http:irule') condition("${.f5.irule}" eq "Splunk-iRule-HTTP") + ); + }; + rewrite { + r_set_splunk_dest_update_v2( + sourcetype('f5:bigip:ltm:failed:irule') condition("${.f5.irule}" eq "Splunk-iRule-LB_FAILED") + ); + }; + }; +}; +application app-syslog-f5_bigip_irule_splunk[sc4s-syslog] { + parser { app-syslog-f5_bigip_irule_splunk(); }; +}; + + diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-f5_bigip_structured.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-f5_bigip_structured.conf new file mode 100644 index 0000000000..f7a6c563f1 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-f5_bigip_structured.conf @@ -0,0 +1,42 @@ +block parser app-syslog-f5_bigip_structured() { + channel { + rewrite { + r_set_splunk_dest_default( + index("netops") + sourcetype('f5:bigip:syslog') + vendor('f5') + product('bigip') + ); + + }; + + if { + filter{ + not (program('iControlPortal.cgi' type(string) flags(prefix)) + or program('tmsh' type(string) flags(prefix)) + or program('mcpd' type(string) flags(prefix)) + or program('mprov' type(string) flags(prefix)) + or program('apmd' type(string) flags(prefix)) + or program('tmm' type(string) flags(prefix)) + or program('sshd\(pam_audit\)') + or message("pam_radius_auth|authenticated|pam_bigip_authz|mod_auth_pam") + ); + }; + rewrite { + r_set_splunk_dest_update_v2( + class('nix_syslog') + sourcetype('nix:syslog') + ); + }; + }; + + + }; +}; +application app-syslog-f5_bigip_structured[sc4s-syslog-sdata] { + filter { + match('^\[F5@12276' value("SDATA")) + ; + }; + parser { app-syslog-f5_bigip_structured(); }; +}; diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-fireeye-json.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-fireeye-json.conf new file mode 100644 index 0000000000..75c1b07efd --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-fireeye-json.conf @@ -0,0 +1,41 @@ +block parser app-syslog-fireeye-json() { + channel { + parser { + json-parser( + prefix('.values.') + ); + }; + rewrite { + r_set_splunk_dest_default( + index('fireeye') + sourcetype('fe_json') + vendor("fireeye") + product('${.values.product}') + template('t_msg_only') + + ); + set("hx_json", value(".splunk.sourcetype") condition( "${.values.product}" eq "HX" )); + set("${.values.alert.host.hostname}", value("HOST") condition( "${.values.alert.host.hostname}" ne "")); + }; + if { + filter { "${.values.alert.event_values.processEvent/timestamp}" ne "" }; + parser { + date-parser-nofilter(format('%Y-%m-%dT%H:%M:%S.%f%z') + template("${.values.alert.event_values.processEvent/timestamp}")); + }; + } elif { + filter { "${.values.alert.event_at}" ne "" }; + parser { + date-parser-nofilter(format('%Y-%m-%dT%H:%M:%S.%f%z') + template("${.values.alert.event_at}")); + }; + } else {}; + + }; +}; +application app-syslog-fireeye-json[sc4s-syslog-pgm] { + filter { + program('^fe[a-z]+-\d+\.[a-z]+') + }; + parser { app-syslog-fireeye-json(); }; +}; diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-forcepoint_webprotect.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-forcepoint_webprotect.conf new file mode 100644 index 0000000000..cf1054bc2c --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-forcepoint_webprotect.conf @@ -0,0 +1,57 @@ +block parser app-syslog-forcepoint_webprotect() { + channel { + parser { + kv-parser(prefix(".values.") pair-separator(" ") ); + + }; + + if { + filter { + "${.values.product}" eq "Security" + }; + + rewrite { + r_set_splunk_dest_default( + sourcetype('websense:cg:kv') + template('t_msg_only') + index("netproxy") + product("webprotect") + vendor("forcepoint") + ); + }; + } elif { + filter { + "${.values.product}" eq "Email Security" + }; + + rewrite { + r_set_splunk_dest_default( + sourcetype('forcepoint:email:kv') + template('t_msg_only') + index('email') + product('email') + vendor("forcepoint") + ); + }; + } else { + rewrite { + r_set_splunk_dest_default( + index("netproxy") + sourcetype('websense:cg:kv') + vendor("forcepoint") + product("${.values.product}") + ); + subst(" [^ =]+\=\-", "", value("MESSAGE"), flags("global")); + }; + + }; + }; +}; +application app-syslog-forcepoint_webprotect[sc4s-syslog] { + filter { + message('vendor=Websense' type(string) flags(prefix,ignore-case)); + or message('vendor=Forcepoint' type(string) flags(prefix,ignore-case)); + }; + parser { app-syslog-forcepoint_webprotect(); }; +}; + diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-fortigate_fortios.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-fortigate_fortios.conf new file mode 100644 index 0000000000..4c162c0c24 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-fortigate_fortios.conf @@ -0,0 +1,108 @@ +block parser app-syslog-fortigate_fortios() { + channel { + + parser { + kv-parser(prefix(".values.") template("$(template t_hdr_msg)")); + }; + filter { + "${.values.eventtime}" ne "" + or ( + "${.values.date}" ne "" + and "${.values.time}" ne "" + ) + }; + parser { + regexp-parser( + patterns('^F(?[A-Z]|\dK)(?.+)') + prefix(".tmp.") + template('${.values.devid}') + ); + }; + + rewrite { + r_set_splunk_dest_default( + index('netops') + sourcetype('`SC4S_OPTION_FORTINET_SOURCETYPE_PREFIX`_log') + vendor("fortinet") + product("fortios") + class("log") + template('t_msg_only') + ); + set("${.values.devname}", value("HOST")); + set("$(template t_hdr_msg)" value("MESSAGE")); + unset(value("PROGRAM")); + }; + + if { + filter { + "${.values.eventtime}" ne "" + }; + parser { + regexp-parser( + prefix(".tmp.") + template("${.values.eventtime}") + patterns('^(?\d{10})\.?(?\d{3,6})') + ); + }; + parser { + date-parser-nofilter( + format( + '%s.%f', + ) + template("${.tmp.timestamp}.${.tmp.fraq:-000}") + ); + }; + } else { + parser { + date-parser-nofilter( + format( + '%Y-%m-%d:%H:%M:%S%z', + '%Y-%m-%d:%H:%M:%S' + ) + template("${.values.date}:${.values.time}${.values.tz}") + ); + }; + }; + if { + filter{ + "${.values.type}" eq "event" + }; + rewrite { + r_set_splunk_dest_update_v2( + class('${.values.type}') + index('netops') + sourcetype('`SC4S_OPTION_FORTINET_SOURCETYPE_PREFIX`_${.values.type}') + ); + }; + } elif { + filter { + match("traffic|utm|anomaly" value(".values.type")); + }; + rewrite { + r_set_splunk_dest_update_v2( + class('${.values.type}') + index('netfw') + sourcetype('`SC4S_OPTION_FORTINET_SOURCETYPE_PREFIX`_${.values.type}') + ); + }; + } else { + rewrite { + r_set_splunk_dest_update_v2( + class('${.values.type}') + sourcetype('`SC4S_OPTION_FORTINET_SOURCETYPE_PREFIX`_${.values.type}') + ); + }; + }; + }; +}; + +application fortinet_fortios[sc4s-syslog] { + filter { + ( + message("eventtype=") + or message("time=") + ) + and message("devid=") + }; + parser { app-syslog-fortigate_fortios(); }; +}; diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-fortigate_fortiweb.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-fortigate_fortiweb.conf new file mode 100644 index 0000000000..af83f21cc8 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-fortigate_fortiweb.conf @@ -0,0 +1,147 @@ +block parser app-syslog-fortigate_fortiweb() { + channel { + + parser { + kv-parser(prefix(".values.") template("$(template t_hdr_msg)")); + }; + filter { + ( + "${.values.eventtime}" ne "" + or ("${.values.date}" ne "" and "${.values.time}" ne "") + ) + and "${.values.device_id}" ne "" + and ( + "${.values.type}" ne "traffic" + or "${.values.type}" ne "attack" + or "${.values.type}" ne "event" + ) + }; + parser { + regexp-parser( + patterns('^F(?[A-Z]|\dK)(?.+)') + prefix(".tmp.") + template('${.values.device_id}') + ); + }; + + rewrite { + r_set_splunk_dest_default( + index('netops') + sourcetype('fwb_log') + vendor("fortinet") + product("fortiweb") + template('t_msg_only') + ); + + set("${.values.devname}", value("HOST")); + subst('.*([\+-]\d+:\d+).*', $1, value(".values.timezone")); + subst('([\+-])(\d)(?=:)(:\d+)', "${1}0${2}${3}", value(".values.timezone")); + + }; + if { + filter { + "${.values.eventtime}" ne "" + }; + parser { + regexp-parser( + prefix(".tmp.") + template("${.values.eventtime}") + patterns('^(?\d{10})\.?(?\d{3,6})') + ); + }; + parser { + date-parser-nofilter( + format( + '%s.%f', + ) + template("${.tmp.timestamp}.${.tmp.fraq:-000}") + ); + }; + } else { + parser { + date-parser-nofilter( + format( + '%Y-%m-%dT%H:%M:%S%z', + '%Y-%m-%dT%H:%M:%S.%f%z', + '%Y-%m-%dT%H:%M:%S', + '%Y-%m-%dT%H:%M:%S.%f' + ) + template("${.values.date}T${.values.time}${.values.tz}") + ); + }; + }; + if { + filter { + "${.tmp.dev_type}" eq "E" + }; + rewrite{ + r_set_splunk_dest_default( + index('email') + sourcetype('fml:log') + vendor("fortinet") + product("fortimail") + class('log') + template('t_msg_only') + ); + }; + rewrite { + r_set_splunk_dest_update_v2( + class('${.values.type}') + index('email') + sourcetype('fml:${.values.type}') + condition( + match('^.' value('.values.type')) + ) + ); + }; + } else { + if (match("traffic" value(".values.type"))) { + rewrite { + r_set_splunk_dest_update_v2( + class('traffic') + index('netfw') + sourcetype('fwb_traffic') + ); + }; + } elif (match("attack" value(".values.type"))) { + rewrite { + r_set_splunk_dest_update_v2( + class('attack') + index('netids') + sourcetype('fwb_attack') + ); + }; + } elif (match("event" value(".values.type"))) { + rewrite { + r_set_splunk_dest_update_v2( + class('event') + index('netops') + sourcetype('fwb_event') + ); + }; + } else { + rewrite { + r_set_splunk_dest_update_v2( + class('log') + index('netops') + sourcetype('fwb_log') + ); + }; + }; + }; + + }; +}; +application app-syslog-fortigate_fortiweb[sc4s-syslog] { + filter { + ( + message('eventtime=') + or ( + message('date=') + and message('time=') + ) + ) + and message('device_id=') + }; + parser { app-syslog-fortigate_fortiweb(); }; +}; diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-github_ent.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-github_ent.conf new file mode 100644 index 0000000000..ed7fc2798c --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-github_ent.conf @@ -0,0 +1,22 @@ +block parser app-syslog-github_ent() { + channel { + rewrite { + r_set_splunk_dest_default( + index("gitops") + source("github:enterprise:audit") + sourcetype('github:enterprise:audit') + vendor("github") + product("ent") + class('audit') + template("t_msg_only") + ); + }; + }; +}; + +application app-syslog-github_ent[sc4s-syslog-pgm] { + filter { + program('github_audit' type(string) flags(prefix)); + }; + parser { app-syslog-github_ent(); }; +}; \ No newline at end of file diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-haproxy.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-haproxy.conf new file mode 100644 index 0000000000..c0b54d7d80 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-haproxy.conf @@ -0,0 +1,31 @@ + +block parser app-syslog-haproxy() { + channel { + rewrite { + r_set_splunk_dest_default( + index("netlb") + source("program:haproxy") + sourcetype('haproxy:tcp') + vendor("haproxy") + product("syslog") + class('idrac') + template('t_hdr_msg') + ); + }; + + rewrite { + r_set_splunk_dest_update_v2( + sourcetype('haproxy:splunk:http') + condition(message('client_ip=' type('string') flags(prefix))) + ); + }; + + }; +}; +application app-syslog-haproxy[sc4s-syslog-pgm] { + filter { + program('haproxy' type(string) flags(prefix)); + }; + parser { app-syslog-haproxy(); }; + +}; \ No newline at end of file diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-hpe_ilo.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-hpe_ilo.conf new file mode 100644 index 0000000000..095030060f --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-hpe_ilo.conf @@ -0,0 +1,21 @@ +block parser app-syslog-hpe_ilo() { + channel { + rewrite { + r_set_splunk_dest_default( + index("infraops") + sourcetype('hpe:ilo') + vendor("hpe") + product("ilo") + template('t_5424_hdr_sdata_compact') + ); + }; + + }; +}; +application app-syslog-hpe_ilo[sc4s-syslog-pgm] { + filter { + program('(#ILO|[iI][lL][oO])\d{1}+') + }; + parser { app-syslog-hpe_ilo(); }; + +}; diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-hpe_jetdirect.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-hpe_jetdirect.conf new file mode 100644 index 0000000000..4c95123144 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-hpe_jetdirect.conf @@ -0,0 +1,71 @@ +block parser app-syslog-hpe_jetdirect() { + channel { + rewrite { + set("$(template t_hdr_msg)" value("MSG")); + unset(value("PROGRAM")); + }; + + if ("`SC4S_DISABLE_DROP_INVALID_HPE`" eq "yes") { + parser { + csv-parser(delimiters(chars('') strings(';')) + columns('message', 'values') + prefix('.tmp.') + flags(greedy)); + kv-parser( + prefix(".values.") + template("${.tmp.values}") + ); + }; + } elif { + parser { + csv-parser(delimiters(chars('') strings(';')) + columns('message', 'values') + prefix('.tmp.') + flags(greedy, drop-invalid)); + kv-parser( + prefix(".values.") + template("${.tmp.values}") + ); + }; + } else {}; + + if { + + parser { + regexp-parser( + prefix(".tmp.") + patterns('^(?\d\d\d\d-\w\w\w-\d\d \d\d:\d\d \w\w) \([^\d+-]+(?[-+]\d\d):(?\d\d)') + template("${.values.time}") + ); + date-parser-nofilter( + format( + '%Y-%b-%d %I:%M %p%z %S', + ) + template("${.tmp.timestampdate}${.tmp.timestamptza}${.tmp.timestamptzb} 00") + ); + }; + }; + + + rewrite { + r_set_splunk_dest_default( + index('print') + sourcetype('hpe:jetdirect') + vendor("hpe") + product("jetdirect") + ); + }; + + + }; +}; +application app-syslog-hpe_jetdirect[sc4s-syslog-pgm] { + filter { + program('^(printer|scanner)') + and message('[^;]+; '); + }; + + + parser { app-syslog-hpe_jetdirect(); }; +}; + diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-hpe_procurve_fmt1.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-hpe_procurve_fmt1.conf new file mode 100644 index 0000000000..a083bca32d --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-hpe_procurve_fmt1.conf @@ -0,0 +1,40 @@ +block parser app-syslog-hpe_procurve_fmt1() { + channel { + if { + filter { + "$PROGRAM" eq "(Stack)" + }; + rewrite { + set("true" value('.values.Stacked')); + subst('^(Stack) ', '', value('MESSAGE')); + }; + } elif { + filter { + "$PROGRAM" eq "" + }; + rewrite { + set("false" value('.values.Stacked')); + }; + }; + parser { + regexp-parser( + prefix(".tmp.") + patterns('^%%\d\d[A-Z]+\/\d\/(?[A-Z_\- ]+)\(\w\):\s') + ); + }; + rewrite { + r_set_splunk_dest_default( + index("netops") + source('hpe:procurve:$(lowercase ${.tmp.category})') + sourcetype('hpe:procurve') + vendor("hpe") + product("procurve") + class('$(lowercase ${.tmp.category})') + ); + }; + + }; +}; +application app-syslog-hpe_procurve_fmt1[sc4s-syslog] { + parser { app-syslog-hpe_procurve_fmt1(); }; +}; \ No newline at end of file diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-hpe_procurve_fmt2.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-hpe_procurve_fmt2.conf new file mode 100644 index 0000000000..a5d1714553 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-hpe_procurve_fmt2.conf @@ -0,0 +1,31 @@ +block parser app-syslog-hpe_procurve_fmt2() { + channel { + filter { + program('^\d{5}$'); + }; + parser { + regexp-parser( + prefix(".tmp.") + patterns('^(?802.1x|acl|addrmgr|arp-protect|auth|cdp|chassis|connfilt|console|cos|dca|dhcp|dhcp v6c|dhcpr|download|dhcp-snoop|dma|fault|fdr-log|ffi|garp|gvrp|hpesp|idm|igmp|inst-mon|ip|ipaddrmgr|iplock|ipx|kms|lacp|ldbal|lldp|loop_protect|macauth|maclock|mgr|mld|mtm|netinet|pagp|ports|radius|ratelim|sflow|snmp|sntp|ssh|ssl|stack|stp|system|tacacs|tcp|telnet|tftp|timep|udld|udpf|update|usb|vlan|xmodem):\s') + ); + }; + rewrite { + r_set_splunk_dest_default( + index("netops") + source('hpe:procurve:${tmp.category}') + sourcetype('hpe:procurve') + vendor("hpe") + product("procurve") + class('$(lowercase ${.tmp.category})') + ); + }; + rewrite { + set('$PROGRAM $MESSAGE', value('MESSAGE')); + unset(value('PROGRAM')); + }; + + }; +}; +application app-syslog-hpe_procurve_fmt2[sc4s-syslog] { + parser { app-syslog-hpe_procurve_fmt2(); }; +}; \ No newline at end of file diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-ibm_datapower.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-ibm_datapower.conf new file mode 100644 index 0000000000..a3839a58c3 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-ibm_datapower.conf @@ -0,0 +1,36 @@ +block parser app-syslog-ibm_datapower() { + channel { + rewrite { + r_set_splunk_dest_default( + index('infraops') + sourcetype('ibm:datapower:syslog') + vendor('ibm') + product('datapower') + template('t_msg_only') + ); + }; + + if { + parser { + regexp-parser( + prefix(".tmp.") + patterns('\[[^\]]*\]\[(?[^\]]*)\]\[[^\]]*\]\s') + ); + }; + + rewrite{ + r_set_splunk_dest_update_v2( + sourcetype('ibm:datapower:${.tmp.category}') + class("${.tmp.category}") + ); + }; + }; + }; +}; +application app-syslog-ibm_datapower[sc4s-syslog] { + filter { + message('^\[\w+\]\[0x[0-9abcdef]+\]\[\w+\]\[\w+\] '); + }; + parser { app-syslog-ibm_datapower(); }; +}; + diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-isc_bind.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-isc_bind.conf new file mode 100644 index 0000000000..4ef23b5fc4 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-isc_bind.conf @@ -0,0 +1,80 @@ +block parser app-syslog-isc_bind() { + channel { + rewrite { + r_set_splunk_dest_default( + index("netdns") + source("program:named") + sourcetype('isc:bind') + vendor("isc") + product("bind") + ); + }; + + if { + filter{ + (message(' query-errors: ')) + or (message('^client') and message('\s+query\s+failed\s+')) + }; + rewrite { + r_set_splunk_dest_update_v2( + index('netdns') + source('program:named') + sourcetype('isc:bind:queryerror') + ); + }; + } elif { + filter{ + message('^queries:') + or message('^client') + }; + rewrite { + r_set_splunk_dest_update_v2( + index('netdns') + source('program:named') + sourcetype('isc:bind:query') + ); + }; + } elif { + filter{ + message('^lame-servers:') + or message('^network unreachable resolving')}; + rewrite { + r_set_splunk_dest_update_v2( + index('netdns') + source('program:named') + sourcetype('isc:bind:lameserver') + ); + }; + } elif { + filter{ + message('^notify:') + or message('^zone') + }; + rewrite { + r_set_splunk_dest_update_v2( + index('netdns') + source('program:named') + sourcetype('isc:bind:transfer') + ); + }; + } else { + rewrite { + r_set_splunk_dest_update_v2( + index('netdns') + source('program:named') + sourcetype('isc:bind:network') + ); + }; + }; + + + }; +}; +application app-syslog-isc_bind[sc4s-syslog-pgm] { + filter { + program('named' type(string) flags(prefix)) + and not message('^CEF'); + }; + parser { app-syslog-isc_bind(); }; + +}; \ No newline at end of file diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-isc_dhcpd.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-isc_dhcpd.conf new file mode 100644 index 0000000000..b108ea3c51 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-isc_dhcpd.conf @@ -0,0 +1,21 @@ +block parser app-syslog-isc_dhcpd() { + channel { + rewrite { + r_set_splunk_dest_default( + index("netipam") + source("program:dhcpd") + sourcetype('isc:dhcpd') + vendor("isc") + product("dhcpd") + ); + set("1", value(".is_known_nix")); + }; + + }; +}; +application app-syslog-isc_dhcpd[sc4s-syslog-pgm] { + filter { + program('dhcpd' type(string) flags(prefix)); + }; + parser { app-syslog-isc_dhcpd(); }; +}; \ No newline at end of file diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-juniper_junos_structured.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-juniper_junos_structured.conf new file mode 100644 index 0000000000..c785cc4490 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-juniper_junos_structured.conf @@ -0,0 +1,82 @@ +block parser app-syslog-juniper_junos_structured() { + channel { + rewrite { + r_set_splunk_dest_default( + index("netops") + sourcetype('juniper:unknown') + vendor("juniper") + product("junos_structured") + template("t_5424_hdr_sdata_compact") + ); + }; + + rewrite { + set("structured", value(".juniper_type")); + r_set_splunk_dest_default( + index('netops') + sourcetype('juniper:structured') + ); + }; + + if (program('RT_IDP' type(string) flags(prefix))) { + rewrite { + r_set_splunk_dest_update_v2( + index('netids') + class('idp') + sourcetype('juniper:junos:idp:structured') + ); + }; + } elif (program('RT_FLOW' type(string) flags(prefix))) { + rewrite { + r_set_splunk_dest_update_v2( + index('netfw') + class('fw') + sourcetype('juniper:junos:firewall:structured') + ); + }; + } elif (program('RT_IDS' type(string) flags(prefix))) { + rewrite { + r_set_splunk_dest_update_v2( + index('netids') + class('ids') + sourcetype('juniper:junos:firewall:structured') + ); + }; + } elif (program('RT_UTM' type(string) flags(prefix))) { + rewrite { + r_set_splunk_dest_update_v2( + index('netfw') + class('utm') + sourcetype('juniper:junos:firewall:structured') + ); + }; + } elif (program('RT_AAMW' type(string) flags(prefix))) { + rewrite { + r_set_splunk_dest_update_v2( + index('netfw') + class('aamw') + sourcetype('juniper:junos:aamw:structured') + ); + }; + } elif (program('RT_SECINTEL' type(string) flags(prefix))) { + rewrite { + r_set_splunk_dest_update_v2( + index('netfw') + class('secintel') + sourcetype('juniper:junos:secintel:structured') + ); + }; + } else {}; + }; +}; +application app-syslog-juniper_junos_structured[sc4s-syslog-sdata] { + filter { + filter(f_is_rfc5424) + and match('\[junos@2636' value("SDATA")); + }; + parser { app-syslog-juniper_junos_structured(); }; +}; + + + + diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-juniper_junos_unstructured.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-juniper_junos_unstructured.conf new file mode 100644 index 0000000000..f1a71f5076 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-juniper_junos_unstructured.conf @@ -0,0 +1,101 @@ +block parser app-syslog-juniper_junos_unstructured() { + channel { + rewrite { + set("unstructured", value(".juniper_type")); + r_set_splunk_dest_default( + index('netops') + sourcetype('juniper:legacy') + vendor("juniper") + product("junos") + ); + }; + if (program('RT_IDP' type(string) flags(prefix))) { + rewrite { + r_set_splunk_dest_update_v2( + index('netids') + class('idp') + sourcetype('juniper:junos:idp') + ); + }; + } elif (program('RT_FLOW' type(string) flags(prefix)) or message('PFE_FW_|DFWD_|ESWD_')) { + rewrite { + r_set_splunk_dest_update_v2( + index('netfw') + class('firewall') + sourcetype('juniper:junos:firewall') + ); + }; + } elif (program('rpd' type(string) flags(prefix)) or message('rpd')) { + rewrite { + r_set_splunk_dest_update_v2( + index('netfw') + class('firewall') + sourcetype('juniper:junos:firewall') + ); + }; + } elif (program('RT_IDS' type(string) flags(prefix))) { + rewrite { + r_set_splunk_dest_update_v2( + index('netids') + class('ids') + sourcetype('juniper:junos:firewall') + ); + }; + } elif (program('RT_UTM' type(string) flags(prefix))) { + rewrite { + r_set_splunk_dest_update_v2( + index('netfw') + class('utm') + sourcetype('juniper:junos:firewall') + ); + }; + } elif (message('SNMP_TRAP_|SNMPD_|VRRPD_|CHASSISD_SNMP_')) { + rewrite { + r_set_splunk_dest_update_v2( + index('netops') + class('snmp') + sourcetype('juniper:junos:snmp') + ); + }; + } + else { + rewrite { + r_set_splunk_dest_update_v2( + class('legacy') + sourcetype('juniper:legacy') + ); + }; + }; + }; +}; +application app-syslog-juniper_junos_unstructured-pgm[sc4s-syslog-pgm] { + filter { + program('RT_IDP' type(string) flags(prefix)) + or program('RT_FLOW' type(string) flags(prefix)) + or program('RT_IDS' type(string) flags(prefix)) + or program('RT_UTM' type(string) flags(prefix)) + or program('RT_SYSTEM' type(string) flags(prefix)) + or program('Juniper' type(string) flags(prefix)) + or program('rpd' type(string) flags(prefix)) + or program('eswd' type(string) flags(prefix)) + or program('mib2d' type(string) flags(prefix)) + or program('ESWD_' type(string) flags(prefix)) + or program('mgd' type(string)) + or program('mcsnoopd' type(string)) + or (program('ifinfo' type(string)) and message('^PVIDB')) + ; + }; + parser { app-syslog-juniper_junos_unstructured(); }; +}; + +application app-syslog-juniper_junos_unstructured-msg[sc4s-syslog] { + filter { + message('^PFE_FW_SYSLOG_ETH_IP: ') + or message('^PFE_FW_SYSLOG_IP:') + or message('EX-BCM PIC') + ; + }; + parser { app-syslog-juniper_junos_unstructured(); }; +}; + + diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-juniper_netscreen.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-juniper_netscreen.conf new file mode 100644 index 0000000000..094949b1d3 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-juniper_netscreen.conf @@ -0,0 +1,23 @@ +block parser app-syslog-juniper_netscreen() { + channel { + rewrite { + r_set_splunk_dest_default( + index("netfw") + source("program:dhcpd") + sourcetype('netscreen:firewall') + vendor("juniper") + product("netscreen") + ); + }; + + + }; +}; +application app-syslog-juniper_netscreen[sc4s-syslog-pgm] { + filter { + program('^ns\d+') + and message('NetScreen' type(string) flags(prefix)); + }; + parser { app-syslog-juniper_netscreen(); }; +}; + diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-kaspersky_syslog.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-kaspersky_syslog.conf new file mode 100644 index 0000000000..b17f00deec --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-kaspersky_syslog.conf @@ -0,0 +1,36 @@ +block parser app-syslog-kaspersky_syslog() { + channel { + rewrite { + r_set_splunk_dest_default( + index("epav") + sourcetype('kaspersky:syslog') + vendor("kaspersky") + product("syslog") + template('t_5424_hdr_sdata_compact') + ); + }; + rewrite { + r_set_splunk_dest_update_v2( + sourcetype('kaspersky:es') + index('epav') + class('es') + condition(program('KES' type(string) flags(prefix))) + ) + }; + rewrite { + subst('\\r\\n', "\r\n", value('MESSAGE'), flags(global)); + }; + + }; +}; +filter f_k_sdid { + ("$(filter (match('@23668' template('$_'))) $(names .SDATA.*))" != '') +}; + +application app-syslog-kaspersky_syslog[sc4s-syslog-sdata] { + filter { + filter(f_k_sdid); +}; + parser { app-syslog-kaspersky_syslog(); }; +}; + diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-leef.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-leef.conf new file mode 100644 index 0000000000..b163bf7f1f --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-leef.conf @@ -0,0 +1,71 @@ +template t_leef_message { + template("${PROGRAM}:${MESSAGE}"); + }; + +template t_leef_event { + template("${.leef.event}"); +}; + +parser p_leef_kv { + python( + class("parser_leef.leef_kv") + ); +}; + + +parser p_leef_devTime { + date-parser-nofilter(format( + '%s.%f', + '%s', + '%b %d %H:%M:%S', + '%b %d %H:%M:%S.%f', + '%b %d %Y %H:%M:%S.%f' + ) + template("${.leef.devTime}") + ); +}; + + +block parser app-syslog-leef() { + channel { + rewrite { + r_set_splunk_dest_default( + sourcetype('leef') + vendor("leef") + product('generic') + ); + set("t_leef_event", value(".splunk.sc4s_template")); + set("$(template t_leef_message)" value("MSG")); + + }; + + # LEEF has two offical versions however in the wild we see "hybrid" + # Syntax where V2 is used but the separator is not provided and assumed + # to be \t + parser (p_leef_kv); + if { + filter { + match(".{4,}" value(".leef.devTime")); + }; + parser (p_leef_devTime); + }; + + + # leef TAs use the source as their bounds in props.conf + # leef source entries in splunk_metadata.csv should reflect the proper source as well as sourcetype + + rewrite { set("${.splunk.sourcetype}", value(".leef.sourcetype")); }; + + if { + parser { app-parser(topic(leef)); }; + }; + + }; +}; +application app-syslog-leef[sc4s-syslog-pgm] { + filter { + program('LEEF' type(string) flags(prefix)); + }; + parser { app-syslog-leef(); }; +}; + diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-liveaction_livenx.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-liveaction_livenx.conf new file mode 100644 index 0000000000..6183c343d8 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-liveaction_livenx.conf @@ -0,0 +1,18 @@ +block parser app-syslog-liveaction_livenx() { + channel { + rewrite { + r_set_splunk_dest_default( + index("netops") + sourcetype('liveaction:livenx') + vendor("liveaction") + product("livenx") + ) + }; + }; +}; +application app-syslog-liveaction_livenx[sc4s-syslog] { + filter { + message('%LIVEACTION' type(string) flags(prefix)); + }; + parser { app-syslog-liveaction_livenx(); }; +}; diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-mcafee_epo.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-mcafee_epo.conf new file mode 100644 index 0000000000..5e202ed6ab --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-mcafee_epo.conf @@ -0,0 +1,42 @@ +block parser app-syslog-mcafee_epo() { + channel { + rewrite { + r_set_splunk_dest_default( + index("epav") + source("epo") + sourcetype('mcafee:epo:syslog') + vendor("mcafee") + product("epo") + ); + }; + parser { + xml( + prefix('.values.') + drop-invalid(no) + ); + }; + if { + parser { + regexp-parser( + prefix(".tmp.") + patterns('ProductName="(?[^"]+)"') + ); + }; + rewrite { + set("$(lowercase ${.tmp.ProductName})" value(".tmp.productshort")); + subst('\s', '_', value(".tmp.productshort") flags("global")); + r_set_splunk_dest_update_v2( + source('${.tmp.productshort}') + ); + }; + }; + + }; +}; +application app-syslog-mcafee_epo[sc4s-syslog-pgm] { + filter { + program('EPOEvents' type(string) flags(prefix)); + }; + parser { app-syslog-mcafee_epo(); }; +}; + diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-mcafee_nsp.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-mcafee_nsp.conf new file mode 100644 index 0000000000..218b43e063 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-mcafee_nsp.conf @@ -0,0 +1,24 @@ +block parser app-syslog-mcafee_nsp() { + channel { + rewrite { + r_set_splunk_dest_default( + index("netids") + sourcetype('mcafee:nsp') + vendor("mcafee") + product("nsp") + template("t_msg_only") + ); + }; + + + }; +}; +application app-syslog-mcafee_nsp[sc4s-syslog-pgm] { + filter { + program('SyslogACLLogForwarder' type(string) flags(prefix)) + or program('SyslogAlertForwarder' type(string) flags(prefix)) + or program('SyslogAuditLogForwarder' type(string) flags(prefix)) + or program('SyslogFaultForwarder' type(string) flags(prefix)); + }; + parser { app-syslog-mcafee_nsp(); }; +}; diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-mcafee_wg.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-mcafee_wg.conf new file mode 100644 index 0000000000..b3a71698c0 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-mcafee_wg.conf @@ -0,0 +1,37 @@ +block parser app-syslog-mcafee_wg() { + channel { + rewrite { + r_set_splunk_dest_default( + index("netproxy") + source("mcafee:wg") + sourcetype('mcafee:wg:kv') + vendor("mcafee") + product("wg") + ); + }; + parser { + kv-parser( + prefix(".values.") + ); + }; + + if { + filter {"${.values.ProductName}" ne ""}; + rewrite { + set("$(lowercase ${.values.ProductName})" value(".tmp.ProductName")); + subst('\s', '_', value(".tmp.ProductName") flags("global")); + r_set_splunk_dest_update_v2( + source('mcafee:wg:${.tmp.ProductName}') + ); + }; + }; + + + }; +}; +application app-syslog-mcafee_wg[sc4s-syslog-pgm] { + filter { + program('mwg' type(string) flags(prefix)); + }; + parser { app-syslog-mcafee_wg(); }; +}; diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-netapp_grid.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-netapp_grid.conf new file mode 100644 index 0000000000..c703ab5d77 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-netapp_grid.conf @@ -0,0 +1,39 @@ +block parser app-syslog-netapp_grid() { + channel { + + if { + filter{ + program('Audit' type(string) flags(prefix)) + }; + rewrite { + r_set_splunk_dest_default( + index('infraops') + sourcetype('grid:auditlog') + vendor("netapp") + product("grid") + ); + }; + }; + + if { + filter { + program('NMS' type(string) flags(prefix)) + }; + rewrite { + r_set_splunk_dest_default( + index('infraops') + sourcetype('grid:rest:api') + vendor("netapp") + product("grid") + ); + }; + }; + }; +}; +application app-syslog-netapp_grid[sc4s-syslog] { + filter { + program('Audit' type(string) flags(prefix)) or + program('NMS' type(string) flags(prefix)); + }; + parser { app-syslog-netapp_grid(); }; +}; diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-netapp_ontap.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-netapp_ontap.conf new file mode 100644 index 0000000000..46b37a1b0b --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-netapp_ontap.conf @@ -0,0 +1,44 @@ +block parser app-syslog-netapp_ontap() { + channel { + parser { + regexp-parser( + prefix(".tmp.") + patterns('^[A-Za-z0-9\-\_\.]+: [0-9a-f]+\.[0-9a-f]+ [0-9a-f]+ [A-Z][a-z][a-z] (?[A-Z][a-z][a-z] \d\d \d\d\d\d \d\d:\d\d:\d\d [+-]?\d{1,2}:\d\d)') + ); + date-parser-nofilter( + format( + '%a %d %Y %H:%M:%S %z', + ) + template("${.tmp.timestamp}") + ); + }; + rewrite { + set('$PROGRAM: $MESSAGE', value(MESSAGE)); + set('$PROGRAM', value(HOST)); + unset(value(PROGRAM)); + }; + + rewrite { + r_set_splunk_dest_default( + index("infraops") + sourcetype('ontap:ems') + vendor("netapp") + product("ontap") + ); + }; + + rewrite { + r_set_splunk_dest_update_v2( + class('ems') + ); + }; + + }; +}; +application app-syslog-netapp_ontap[sc4s-syslog] { + filter { + program('^[A-Za-z0-9\-\_\.]+$') + and not "`SC4S_NETAPP_ONTAP_NEW_FORMAT`" eq "yes"; + }; + parser { app-syslog-netapp_ontap(); }; +}; diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-netmotion_syslog.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-netmotion_syslog.conf new file mode 100644 index 0000000000..afc029132a --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-netmotion_syslog.conf @@ -0,0 +1,43 @@ +block parser app-syslog-netmotion_syslog() { + channel { + if { + filter { + program('^nmreporting\.exe'); + }; + rewrite { + r_set_splunk_dest_default( + index("netops") + sourcetype('netmotion:reporting') + vendor("netmotion") + product("reporting") + ); + }; + } else { + parser { + regexp-parser( + prefix(".tmp.") + patterns('\[(?[^@]+)@11912') + template('$SDATA') + ); + }; + rewrite { + r_set_splunk_dest_default( + index("netops") + sourcetype('netmotion:mobilityserver:$(lowercase ${.tmp.sdid})') + vendor("netmotion") + product("mobility-server") + class('$(lowercase ${.tmp.sdid})') + template('t_5424_hdr_sdata_compact') + ); + }; + }; + }; +}; +application app-syslog-netmotion_syslog[sc4s-syslog-sdata] { + filter { + match('@11912' value('SDATA')) + + }; + parser { app-syslog-netmotion_syslog(); }; +}; + diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-nutanix.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-nutanix.conf new file mode 100644 index 0000000000..0182acc5db --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-nutanix.conf @@ -0,0 +1,27 @@ +block parser app-syslog-nutanix_syslog_new() { +channel { + # + filter { + match('ntnx-' value("MESSAGE") flags(ignore-case)) and + match('-cvm' value("MESSAGE") flags(ignore-case)); + }; + rewrite { + r_set_splunk_dest_default( + index('infraops') + sourcetype('nutanix:syslog') + vendor("nutanix") + product("syslog") + ); + }; + rewrite { + r_set_splunk_dest_update_v2( + sourcetype('nutanix:syslog:audit') + class('audit') + condition(program('audi' type(string) flags(substring, ignore-case))) + ) + }; + }; +}; +application app-syslog-nutanix_syslog_new[sc4s-syslog] { + parser { app-syslog-nutanix_syslog_new(); }; +}; \ No newline at end of file diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-ossec.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-ossec.conf new file mode 100644 index 0000000000..de1f48a197 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-ossec.conf @@ -0,0 +1,25 @@ +block parser app-syslog-ossec() { + channel { + rewrite { + r_set_splunk_dest_default( + index("main") + source("ossec:alerts") + sourcetype('ossec') + vendor("ossec") + product("agent") + ); + }; + + + + + }; +}; +application app-syslog-ossec[sc4s-syslog-pgm] { + filter { + program('ossec' type(string) flags(prefix)); + }; + parser { app-syslog-ossec(); }; +}; + + diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-pan_panos.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-pan_panos.conf new file mode 100644 index 0000000000..12034fc193 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-pan_panos.conf @@ -0,0 +1,222 @@ +block parser app-syslog-pan_panos() { + channel { + rewrite { + r_set_splunk_dest_default( + index("netops") + sourcetype('pan:log') + vendor("pan") + product('panos') + template('t_msg_only') + ); + set("$HOST", value("fields.pan_forwarder"), condition( program('logforwarder' type(string)))); + }; + + + if (message(',THREAT,' type(string) flags(substring))) { + parser { + csv-parser( + columns("future_use1","receive_time","serial_number","type","log_subtype","version","generated_time","src_ip","dest_ip","src_translated_ip","dest_translated_ip","rule","src_user","dest_user","app","vsys","src_zone","dest_zone","src_interface","dest_interface","log_forwarding_profile","future_use3","session_id","repeat_count","src_port","dest_port","src_translated_port","dest_translated_port","session_flags","transport","action","misc","threat","raw_category","severity","direction","sequence_number","action_flags","src_location","dest_location","future_use4","content_type","pcap_id","file_hash","cloud_address","url_index","user_agent","file_type","xff","referrer","sender","subject","recipient","report_id","devicegroup_level1","devicegroup_level2","devicegroup_level3","devicegroup_level4","vsys_name","dvc_name") + prefix(".values.") + delimiters(',') + quote-pairs('""') + flags(escape-double-char) + ); + }; + rewrite { + r_set_splunk_dest_update_v2( + index('netproxy') + class('threat') + sourcetype('pan:threat') + ); + }; + } elif (message(',TRAFFIC,' type(string) flags(substring))) { + parser { + csv-parser( + columns("future_use1","receive_time","serial_number","type","log_subtype","version","generated_time","src_ip","dest_ip","src_translated_ip","dest_translated_ip","rule","src_user","dest_user","app","vsys","src_zone","dest_zone","src_interface","dest_interface","log_forwarding_profile","future_use3","session_id","repeat_count","src_port","dest_port","src_translated_port","dest_translated_port","session_flags","transport","action","bytes","bytes_out","bytes_in","packets","start_time","duration","http_category","future_use4","sequence_number","action_flags","src_location","dest_location","future_use5","packets_out","packets_in","session_end_reason","devicegroup_level1","devicegroup_level2","devicegroup_level3","devicegroup_level4","vsys_name","dvc_name") + prefix(".values.") + delimiters(',') + quote-pairs('""') + flags(escape-double-char) + ); + }; + rewrite { + r_set_splunk_dest_update_v2( + index('netfw') + class('traffic') + sourcetype('pan:traffic') + ); + }; + } elif (message(',SYSTEM,' type(string) flags(substring))) { + parser { + csv-parser( + columns("future_use1","receive_time","serial_number","type","log_subtype","version","generated_time","vsys","event_id","object","future_use3","future_use4","module","severity","description","sequence_number","action_flags","devicegroup_level1","devicegroup_level2","devicegroup_level3","devicegroup_level4","vsys_name","dvc_name") + prefix(".values.") + delimiters(',') + quote-pairs('""') + flags(escape-double-char) + ); + }; + rewrite { + r_set_splunk_dest_update_v2( + index('netops') + class('system') + sourcetype('pan:system') + ); + }; + } elif (message(',CONFIG,' type(string) flags(substring))) { + parser { + csv-parser( + columns("future_use1","receive_time","serial_number","type","log_subtype","version","generated_time","host_name","vsys","command","admin","client","result","configuration_path","sequence_number","action_flags","before_change_detail","after_change_detail","devicegroup_level1","devicegroup_level2","devicegroup_level3","devicegroup_level4","vsys_name","dvc_name") + prefix(".values.") + delimiters(',') + quote-pairs('""') + flags(escape-double-char) + ); + }; + rewrite { + r_set_splunk_dest_update_v2( + index('netops') + class('config') + sourcetype('pan:config') + ); + }; + } elif (message(',HIPMATCH,' type(string) flags(substring))) { + parser { + csv-parser( + columns("future_use1","receive_time","serial_number","type","log_subtype","version","generated_time","src_user","vsys","host_name","os","src_ip","hip_name","hip_count","hip_type","future_use3","future_use4","sequence_number","action_flags","devicegroup_level1","devicegroup_level2","devicegroup_level3","devicegroup_level4","vsys_name","dvc_name") + prefix(".values.") + delimiters(',') + quote-pairs('""') + flags(escape-double-char) + ); + }; + rewrite { + r_set_splunk_dest_update_v2( + index('epintel') + class('hipmatch') + sourcetype('pan:hipmatch') + ); + }; + } elif (message(',CORRELATION,' type(string) flags(substring))) { + parser { + csv-parser( + columns("future_use1","receive_time","serial_number","type","log_subtype","version","generated_time","src_ip","src_user","vsys","category","severity","devicegroup_level1","devicegroup_level2","devicegroup_level3","devicegroup_level4","vsys_name","dvc_name","vsys_id","object","object_id","evidence") + prefix(".values.") + delimiters(',') + quote-pairs('""') + flags(escape-double-char) + ); + }; + rewrite{ + r_set_splunk_dest_update_v2( + class('correlation') + sourcetype('pan:correlation') + ); + }; + } elif (message(',USERID,' type(string) flags(substring))) { + parser { + csv-parser( + columns("future_use1","receive_time","serial_number","type","log_subtype","version","generated_time","vsys","src_ip","source_name","event_id","repeat_count","timeout_threshold","src_port","dest_port","source","source_type","sequence_number","action_flags","devicegroup_level1","devicegroup_level2","devicegroup_level3","devicegroup_level4","vsys_name","dvc_name") + prefix(".values.") + delimiters(',') + quote-pairs('""') + flags(escape-double-char) + ); + }; + rewrite { + r_set_splunk_dest_update_v2( + index('netauth') + class('userid') + sourcetype('pan:userid') + ); + }; + } elif (message(',GLOBALPROTECT,' type(string) flags(substring))) { + parser { + csv-parser( + columns("future_use1","receive_time","serial_number","log_type","future_use2","version","time_generated","vsys","event_id","stage","auth_method","tunnel_type","src_user","src_region","machine_name","public_ip","public_ipv6","private_ip","private_ipv6","host_id","serial_number","client_ver","client_os","client_os_ver","repeat_count","reason","error","opaque","status","location","login_duration","connect_method","error_code","portal","sequence_number","action_flags","event_time","selection_type","response_time","priority","attempted_gateways","gateway","devicegroup_level1","devicegroup_level2","devicegroup_level3","devicegroup_level4","vsys_name","dvc_name","vsys_id") + prefix(".values.") + delimiters(',') + quote-pairs('""') + flags(escape-double-char) + ); + }; + rewrite { + r_set_splunk_dest_update_v2( + index('netfw') + class('globalprotect') + sourcetype('pan:globalprotect') + ); + }; + } elif (message(',DECRYPTION,' type(string) flags(substring))) { + parser { + csv-parser( + columns("future_use1","receive_time","serial_number","log_type","log_subtype","version","generated_time","src_ip","dest_ip","src_translated_ip","dest_translated_ip","rule","src_user","dest_user","app","vsys","src_zone","dest_zone","src_interface","dest_interface","log_forwarding_profile","start_time","session_id","repeat_count","src_port","dest_port","src_translated_port","dest_translated_port","flags","IP_PROTOCOL","action","tunnel_id","future_use2","future_use3","src_vm_uuid","dest_vm_uuid","uuid_rule","stage_client_firewall","stage_firewall_client","tls_version","key_exchange_algorithm","encryption_algorithm","hash_algorithm","rule","elliptic_curve","error_index","root_status","chain_status","proxy_type","cert_serial_number","fingerprint","cert_start_time","cert_end_time","cert_version","cert_size","cn_length","issuer_cn_length","root_cn_length","sni_length","cert_flags","subject_cn","issuer_subject_cn","root_subject_cn","server_name","error","container_id","pod_namespace","pod_name","src_edl","dest_edl","src_dag","dest_dag","timestamp","src_dvc_category","src_dvc_profile","src_dvc_model","src_dvc_vendor","src_dvc_os","src_dvc_os_version","src_name","src_mac","dest_dvc_category","dest_dvc_profile","dest_dvc_model","dest_dvc_vendor","dest_dvc_os","dest_dvc_os_version","dest_name","dest_mac","sequence_number","action_flags") + prefix(".values.") + delimiters(',') + quote-pairs('""') + flags(escape-double-char) + ); + }; + rewrite { + r_set_splunk_dest_update_v2( + class('decryption') + sourcetype('pan:decryption') + ); + }; + } elif (message(',AUTH,' type(string) flags(substring))) { + parser { + csv-parser( + columns("future_use1","receive_time","serial_number","log_type","log_subtype","version","generated_time","vsys","src_ip","user","user_normalized","object","authentication_policy","repeast_count","authentication_id","pan_vendor","log_action","server_profile","description","client_type","event_type","factor_number","sequence_number","action_flags","device_group_hierarchy_1","device_group_hierarchy_2","device_group_hierarchy_3","device_group_hierarchy_4","vsys","dvc_name","vsys_id","authentication_protocol","rule","timestamp","src_host_category","src_host_profile","src_host_model","src_host_vendor","src_host_os_name","src_host_os_version","src_host","src_mac","region","future_use2","user_agent","session_id","cluster_name") + prefix(".values.") + delimiters(',') + quote-pairs('""') + flags(escape-double-char) + ); + }; + rewrite { + r_set_splunk_dest_update_v2( + index('netauth') + class('authentication') + sourcetype('pan:auth') + ); + }; + } else { }; + + # Palo IETF (5424) event is entirely contained in $MESSAGE; for BSD format event needs to be constructed from + # constituent parts. LEGACY_MSGHDR is null in IETF so concatenation is a no-op (so no test is needed). + + parser { + # Parse the date + # 2012/04/10 04:39:55 + date-parser-nofilter(format( + '%Y/%m/%d %H:%M:%S.%f', + '%Y/%m/%d %H:%M:%S', + '%Y-%m-%dT%H:%M:%S.%f%z',) + template("${.values.generated_time}") + ); + }; + + rewrite { + set("${.values.dvc_name}" value("HOST") + condition( match('^.' value('.values.dvc_name') )) ); + }; + + }; +}; +application app-syslog-pan_panos-pgm[sc4s-syslog-pgm] { + filter { + program('logforwarder' type(string)) + ; + }; + parser { app-syslog-pan_panos(); }; +}; + +application app-syslog-pan_panos[sc4s-syslog] { + filter { + "${PROGRAM}" eq "" + and message('1,' type(string) flags(prefix)) + and message('^1,[^,]+,[^,]+,[A-Z]+\,') + ; + }; + parser { app-syslog-pan_panos(); }; +}; diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-polycom_rprm.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-polycom_rprm.conf new file mode 100644 index 0000000000..292ab2e1f2 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-polycom_rprm.conf @@ -0,0 +1,22 @@ +block parser app-syslog-polycom_rprm() { + channel { + rewrite { + r_set_splunk_dest_default( + index('netops') + sourcetype('polycom:rprm:syslog') + vendor("polycom") + product("rprm") + template('t_5424_hdr_sdata_compact') + ); + }; + + + }; +}; +application app-syslog-polycom_rprm[sc4s-syslog-pgm] { + filter { + program('RPRM' type(string) flags(prefix)); + }; + parser { app-syslog-polycom_rprm(); }; +}; + diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-prisma_sd-wan.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-prisma_sd-wan.conf new file mode 100644 index 0000000000..28e0ca883f --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-prisma_sd-wan.conf @@ -0,0 +1,52 @@ +block parser app-syslog-prisma_sd-wan() { + channel { + + parser { + kv-parser( + prefix(".values.") + ); + }; + + rewrite { + set( + "flow", value(".tmp.class") condition(program('cgxFlowLogV1' type(string))) + ); + set( + "authentication", value(".tmp.class") condition( + "${.values.ION_HOST}" ne "" + and "${.values.FACILITY}" eq "auth" + ) + ); + set( + "event", value(".tmp.class") condition( + "${.values.ION_HOST}" ne "" + and "${.values.STATUS}" ne "" + ) + ); + }; + + filter { + "${.tmp.class}" ne "" + }; + + rewrite { + r_set_splunk_dest_default( + index('netwaf') + source('pan:prisma') + sourcetype('prisma:sd-wan:${.tmp.class}') + vendor("prisma") + product("sd-wan") + class("${.tmp.class}") + ); + }; + }; +}; + +application app-syslog-prisma_sd-wan[sc4s-syslog] { + filter { + program('log' type(string)) + or program('alarm' type(string)) + or program('cgxFlowLogV1' type(string)) + }; + parser { app-syslog-prisma_sd-wan() }; +}; \ No newline at end of file diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-pulse_connect_secure.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-pulse_connect_secure.conf new file mode 100644 index 0000000000..60d1f039ed --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-pulse_connect_secure.conf @@ -0,0 +1,30 @@ +block parser app-syslog-pulse_connect_secure() { + channel { + rewrite { + r_set_splunk_dest_default( + index("netfw") + sourcetype('pulse:connectsecure') + vendor("pulse_connect") + product("secure") + ); + }; + if { + filter{message("Access blocked" type(string) flags(substring)) or message("WebRequest" type(string) flags(substring))}; + rewrite { + r_set_splunk_dest_update_v2( + class('web') + index('netproxy') + sourcetype('pulse:connectsecure:web') + ); + }; + }; + + + }; +}; +application app-syslog-pulse_connect_secure[sc4s-syslog-pgm] { + filter { + program('PulseSecure:' type(string) flags(prefix)); + }; + parser { app-syslog-pulse_connect_secure(); }; +}; diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-pure_storage.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-pure_storage.conf new file mode 100644 index 0000000000..b3aeb2e56d --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-pure_storage.conf @@ -0,0 +1,35 @@ +block parser app-syslog-pure_storage() { + channel { + rewrite { + r_set_splunk_dest_default( + index("infraops") + sourcetype('purestorage:array') + vendor("purestorage") + product("array") + template('t_msg_only') + ); + }; + if { + parser { + regexp-parser( + prefix(".tmp.") + patterns('^purity\.(?[^:\s]+)') + template("$PROGRAM") + ); + }; + rewrite { + r_set_splunk_dest_update_v2( + sourcetype('purestorage:array:${.tmp.category}') + class('${.tmp.category}') + ); + }; + }; + }; +}; +application app-syslog-pure_storage[sc4s-syslog-pgm] { + filter { + program('^purity\.') + or program('space_utility_json.py'); + }; + parser { app-syslog-pure_storage(); }; +}; diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-qumulo.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-qumulo.conf new file mode 100644 index 0000000000..f960f16c70 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-qumulo.conf @@ -0,0 +1,21 @@ +block parser app-syslog-qumulo() { + channel { + rewrite { + r_set_splunk_dest_default( + index("infraops") + sourcetype('qumulo:storage') + vendor("qumulo") + product("storage") + template('t_msg_only') + ); + }; + + + }; +}; +application app-syslog-qumulo[sc4s-syslog-pgm] { + filter { + program('qumulo' type(string)); + }; + parser { app-syslog-qumulo(); }; +}; diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-rawdware_defensepro_a.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-rawdware_defensepro_a.conf new file mode 100644 index 0000000000..df5d8f0a29 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-rawdware_defensepro_a.conf @@ -0,0 +1,25 @@ +block parser app-syslog-rawdware_defensepro_a() { + channel { + # + filter { + program('^(M_\d+)'); + }; + + rewrite { + r_set_splunk_dest_default( + index('netops') + sourcetype('radware:defensepro') + vendor("radware") + product("defensepro") + ); + }; + + + }; +}; +application app-syslog-rawdware_defensepro_a[sc4s-syslog] { + parser { app-syslog-rawdware_defensepro_a(); }; +}; + + + diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-rawdware_defensepro_b.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-rawdware_defensepro_b.conf new file mode 100644 index 0000000000..4d31239398 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-rawdware_defensepro_b.conf @@ -0,0 +1,29 @@ +block parser app-syslog-rawdware_defensepro_b() { + channel { + # + parser { + regexp-parser( + prefix(".tmp.") + patterns('^(\[Device: (?[^ ]+) [^\]]+] M_\d+: .*)') + ); + }; + + rewrite { + r_set_splunk_dest_default( + index('netops') + sourcetype('radware:defensepro') + vendor("radware") + product("defensepro") + ); + set("${.tmp.host}", value('HOST')); + }; + + + }; +}; +application app-syslog-rawdware_defensepro_b[sc4s-syslog] { + parser { app-syslog-rawdware_defensepro_b(); }; +}; + + + diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-ricoh_syslog.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-ricoh_syslog.conf new file mode 100644 index 0000000000..36d37bccd0 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-ricoh_syslog.conf @@ -0,0 +1,22 @@ +block parser app-syslog-ricoh_syslog() { + channel { + rewrite { + r_set_splunk_dest_default( + index("print") + sourcetype('ricoh:mfp') + vendor("ricoh") + product("mfp") + ); + set("${PROGRAM}", value("HOST") condition("`SC4S_SOURCE_RICOH_SYSLOG_FIXHOST`" eq "yes")); + }; + + + }; +}; +application app-syslog-ricoh_syslog[sc4s-syslog-pgm] { + filter { + match('RICOH' type(string) flags(prefix) value("MSGID")); + }; + parser { app-syslog-ricoh_syslog(); }; +}; + diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-riverbed_syslog.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-riverbed_syslog.conf new file mode 100644 index 0000000000..04392edc4a --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-riverbed_syslog.conf @@ -0,0 +1,53 @@ +filter riverbed_syslog_stealconnect{ + program('scm_' type(string) flags(prefix)) + or match('^\SteelConnect@17163' value("SDATA")) +}; + +block parser app-syslog-riverbed_syslog() { + channel { + rewrite { + r_set_splunk_dest_default( + index('netops') + sourcetype('riverbed:syslog') + vendor("riverbed") + product("syslog") + ); + }; + rewrite { + r_set_splunk_dest_update_v2( + sourcetype('riverbed:steelconnect') + class("steelconnect") + condition(filter(riverbed_syslog_stealconnect)) + ); + }; + + rewrite { + r_set_splunk_dest_update_v2( + sourcetype('riverbed:steelhead') + class("steelhead") + condition( + program('cmcfc', type(string)) + ) + ); + }; + }; +}; + +application app-syslog-riverbed_syslog-sdata[sc4s-syslog-sdata] { + filter { + match('^\[[^@]@17163' value("SDATA")) + ; + }; + parser { app-syslog-f5_bigip_structured(); }; +}; + +application app-syslog-riverbed_syslog[sc4s-syslog-pgm] { + filter { + program('cmcfc', type(string)) + or program('sport', type(string)) + or program('scm_' type(string) flags(prefix)) + ; + }; + parser { app-syslog-riverbed_syslog(); }; +}; + diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-ruckus_smartzone.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-ruckus_smartzone.conf new file mode 100644 index 0000000000..126091499e --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-ruckus_smartzone.conf @@ -0,0 +1,48 @@ +parser parser_kvqf { + python( + class("parser_kvqf.kvqf_parse") + ); +}; + + +block parser app-syslog-ruckus_smartzone() { + channel { + rewrite { + r_set_splunk_dest_default( + index('netops') + sourcetype('ruckus:smartzone') + vendor("ruckus") + product("smartzone") + template('t_hdr_msg') + ); + }; + if { + parser { + csv-parser( + columns("id","name","pairs") + prefix(".tmp.") + delimiters(',') + flags(greedy) + ); + }; + parser(parser_kvqf); + rewrite{ + set('${.tmp.id}' value('.metadata.id')); + set('${.tmp.name}' value('.metadata.name')); + + r_set_splunk_dest_update_v2( + template('t_json_values') + ); + }; + + }; + }; +}; +application app-syslog-ruckus_smartzone[sc4s-syslog-pgm] { + filter { + program("Core", type(string)) + and message('@@', type(string), flags(prefix)) + }; + parser { app-syslog-ruckus_smartzone(); }; +}; + diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-secureauth_idp.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-secureauth_idp.conf new file mode 100644 index 0000000000..9ef49e3e6b --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-secureauth_idp.conf @@ -0,0 +1,21 @@ +block parser app-syslog-secureauth_idp() { + channel { + rewrite { + r_set_splunk_dest_default( + index("netops") + sourcetype('secureauth:idp') + vendor('secureauth') + product('idp') + template('t_5424_hdr_sdata_compact') + ); + }; + }; +}; +application app-syslog-secureauth_idp[sc4s-syslog-sdata] { + filter { + match('SecureAuth' value('SDATA')) + + }; + parser { app-syslog-secureauth_idp(); }; +}; + diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-semperis_dsp.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-semperis_dsp.conf new file mode 100644 index 0000000000..ed6e8b91db --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-semperis_dsp.conf @@ -0,0 +1,35 @@ +block parser app-syslog-semperis_dsp() { + channel { + rewrite { + r_set_splunk_dest_default( + index("netops") + sourcetype("semperis:dsp") + vendor("semperis") + product("dsp") + template("t_kv_values") + ); + }; + + parser { + regexp-parser( + template("${MESSAGE}") + prefix(".tmp.") + patterns('\[(?.*)') + ); + kv-parser( + prefix(".values.") + pair-separator("[") + value-separator("]") + template("${.tmp.message}") + ); + }; + }; +}; + + +application app-syslog-semperis_dsp[sc4s-syslog-pgm] { + filter { + program('Semperis.DSP' type(string) flags(ignore-case,prefix)); + }; + parser { app-syslog-semperis_dsp(); }; +}; diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-solace_eventbroker.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-solace_eventbroker.conf new file mode 100644 index 0000000000..a53d1a1d1c --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-solace_eventbroker.conf @@ -0,0 +1,27 @@ +block parser app-syslog-solace_eventbroker() { + channel { + # + rewrite { + r_set_splunk_dest_default( + index('main') + sourcetype('solace:eventbroker') + vendor("solace") + product("eventbroker") + ); + }; + + + }; +}; +application app-syslog-solace_eventbroker[sc4s-syslog-pgm] { + filter { + program('event' type(string) flags(prefix)) + and ( message('^SYSTEM: [A-Z]+') + or message('^CLIENT: [A-Z]') + or message('^VPN: [A-Z]') ); + }; + parser { app-syslog-solace_eventbroker(); }; +}; + + + diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-sophos_firewall_xg.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-sophos_firewall_xg.conf new file mode 100644 index 0000000000..c1c32eefa4 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-sophos_firewall_xg.conf @@ -0,0 +1,130 @@ +#app-syslog-sophos_firewall_xg +block parser app-syslog-sophos_firewall_xg() { + channel { + rewrite { + r_set_splunk_dest_default( + index('netdlp') + sourcetype('sophos:xg:logs') + vendor('sophos') + product('xg') + class('logs') + template('t_msg_only') + ); + }; + # ATP + rewrite { + r_set_splunk_dest_update_v2( + sourcetype('sophos:xg:atp') + class('atp') + condition(message('log_type="ATP"' type(string) flags(substring))) + ); + }; + # Anti Spam + rewrite { + r_set_splunk_dest_update_v2( + sourcetype('sophos:xg:anti_spam') + class('anti_spam') + condition(message('log_type="Anti-Spam"' type(string) flags(substring))) + ); + }; + # Anti Virus + rewrite { + r_set_splunk_dest_update_v2( + sourcetype('sophos:xg:anti_virus') + class('anti_virus') + condition(message('log_type="Anti-Virus"' type(string) flags(substring))) + ); + }; + # Content Filtering + rewrite { + r_set_splunk_dest_update_v2( + sourcetype('sophos:xg:content_filtering') + class('content_filtering') + condition(message('log_type="Content Filtering"' type(string) flags(substring))) + ); + }; + # Event + rewrite { + r_set_splunk_dest_update_v2( + sourcetype('sophos:xg:event') + class('event') + condition(message('log_type="Event"' type(string) flags(substring))) + ); + }; + # Firewall + rewrite { + r_set_splunk_dest_update_v2( + sourcetype('sophos:xg:firewall') + class('firewall') + condition(message('log_type="Firewall"' type(string) flags(substring))) + ); + }; + # SSL + rewrite { + r_set_splunk_dest_update_v2( + sourcetype('sophos:xg:ssl') + class('ssl') + condition(message('log_type="SSL"' type(string) flags(substring))) + ); + }; + # Sandbox + rewrite { + r_set_splunk_dest_update_v2( + sourcetype('sophos:xg:sandbox') + class('sandbox') + condition(message('log_type="Sandbox"' type(string) flags(substring))) + ); + }; + # System Health + rewrite { + r_set_splunk_dest_update_v2( + sourcetype('sophos:xg:system_health') + class('system_health') + condition(message('log_type="System Health"' type(string) flags(substring))) + ); + }; + # Heartbeat + rewrite { + r_set_splunk_dest_update_v2( + sourcetype('sophos:xg:heartbeat') + class('heartbeat') + condition(message('log_type="Heartbeat"' type(string) flags(substring))) + ); + }; + # WAF + rewrite { + r_set_splunk_dest_update_v2( + sourcetype('sophos:xg:waf') + class('waf') + condition(message('log_type="WAF"' type(string) flags(substring))) + ); + }; + # Wireless Protection + rewrite { + r_set_splunk_dest_update_v2( + sourcetype('sophos:xg:wireless_protection') + class('wireless_protection') + condition(message('log_type="Wireless Protection"' type(string) flags(substring))) + ); + }; + # IDP + rewrite { + r_set_splunk_dest_update_v2( + sourcetype('sophos:xg:idp') + class('idp') + condition(message('log_type="IDP"' type(string) flags(substring))) + ); + }; + }; +}; + +application app-syslog-sophos_firewall_xg[sc4s-syslog] { + filter { + ( + message("device=\"SFW\"" type(string) flags(substring)) or + message("device_name=\"SFW\"" type(string) flags(substring)) and + message("log_type=" type(string) flags(substring)) + ); + }; + parser { app-syslog-sophos_firewall_xg(); }; +}; diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-splunk_cooked.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-splunk_cooked.conf new file mode 100644 index 0000000000..3ccc9fb678 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-splunk_cooked.conf @@ -0,0 +1,81 @@ +block parser app-syslog-splunk_cooked() { + channel { + rewrite { + r_set_splunk_dest_default( + index("main") + sourcetype('splunk:cooked') + vendor("splunk") + product("cooked") + ); + }; + rewrite { + r_set_splunk_dest_update_v2( + index("${.SDATA.fields@274489.i}") + source('${.SDATA.fields@274489.s}') + sourcetype('${.SDATA.fields@274489.st}') + template("t_msg_only") + ); + }; + rewrite{ + set("${.SDATA.fields@274489.h}", value('HOST')); + }; + parser { + date-parser-nofilter(format('%s.%f','%s') + template("${.SDATA.fields@274489.t}") + ); + }; + if { + filter { + message('~~~SM~~~'); + }; + parser { + regexp-parser( + prefix(".tmp.") + patterns('^(?:~~~SM~~~(?.*)~~~EM~~~)?(?.*)') + ); + }; + rewrite{ + set("${.tmp.message}", value('MESSAGE')); + }; + rewrite{ + subst( + '::', + ':', value('.tmp.data'), + flags("global") + ); + subst( + ' data\.', + ' ', value('.tmp.data'), + flags("global") + ); + }; + parser { + kv-parser( + prefix("fields.") + pair-separator(" ") + value-separator(":") + template('${.tmp.data}') + ); + }; + rewrite{ + unset(value('fields._subsecond')); + unset(value('fields.date_hour')); + unset(value('fields.date_mday')); + unset(value('fields.date_minute')); + unset(value('fields.date_month')); + unset(value('fields.date_second')); + unset(value('fields.date_wday')); + unset(value('fields.date_year')); + unset(value('fields.date_zone')); + }; + }; + }; +}; +application app-syslog-splunk_cooked[sc4s-syslog-pgm] { + filter { + program('SPLUNK' type(string)) + and match('COOKED' type(string) value("MSGID")); + }; + parser { app-syslog-splunk_cooked(); }; +}; + diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-stealthbits_stealthintercept.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-stealthbits_stealthintercept.conf new file mode 100644 index 0000000000..914bfee97b --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-stealthbits_stealthintercept.conf @@ -0,0 +1,21 @@ +block parser app-syslog-stealthbits_stealthintercept() { + channel { + rewrite { + r_set_splunk_dest_default( + index('netids') + sourcetype('StealthINTERCEPT') + vendor("stealthbits") + product("stealthintercept") + template('t_hdr_msg') + ); + }; + + }; +}; +application app-syslog-stealthbits_stealthintercept[sc4s-syslog-pgm] { + filter { + "${PROGRAM}" eq "StealthINTERCEPT"; + }; + parser { app-syslog-stealthbits_stealthintercept(); }; +}; + diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-stealthbits_stealthintercept_alerts.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-stealthbits_stealthintercept_alerts.conf new file mode 100644 index 0000000000..57d2369637 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-stealthbits_stealthintercept_alerts.conf @@ -0,0 +1,46 @@ +parser sb_alertext_kv { + python( + class("parser_stealthbits.alerttext_kv") + ); +}; + +block parser app-syslog-stealthbits_stealthintercept_alerts() { + channel { + rewrite { + r_set_splunk_dest_default( + index('netids') + sourcetype('StealthINTERCEPT:alerts') + vendor("stealthbits") + product("stealthintercept") + class('alerts') + template('t_json_values') + ); + }; + parser { + kv-parser( + prefix(".values.") + pair-separator(" ") + template("t_hdr_msg") + ); + }; + if { + parser { + regexp-parser( + prefix(".values.") + patterns(' AlertText=(?.*)$') + ); + }; + parser(sb_alertext_kv); + rewrite{ + unset(value('.values.AlertText') condition("${.values.AlertText}" eq "")); + }; + }; + }; +}; +application app-syslog-stealthbits_stealthintercept_alerts[sc4s-syslog] { + filter { + message('AttackedHostIp=.* AlertText=') + }; + parser { app-syslog-stealthbits_stealthintercept_alerts(); }; +}; + diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-symantec_brightmail.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-symantec_brightmail.conf new file mode 100644 index 0000000000..5110ea053d --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-symantec_brightmail.conf @@ -0,0 +1,126 @@ +block parser app-syslog-symantec_brightmail() { + channel { + rewrite { + r_set_splunk_dest_default( + index("email") + source("program:${PROGRAM}") + sourcetype('symantec:smg') + vendor("symantec") + product("brightmail") + ); + }; + if { + + filter { + program('bmserver' type(string) flags(prefix)) and not message('[Brightmail]' type(string) flags(substring)) + }; + + parser { + csv-parser( + columns(seq, id, field, data) + prefix('.tmp.') + delimiters(chars("|")) + flags(greedy) + ); + kv-parser( + prefix(".values.") + pair-separator("=") + template("${.tmp.field}=${.tmp.data}") + ); + }; + rewrite { + set("${.tmp.id}", value(".metadata.id")); + set("${.tmp.seq}", value(".metadata.seq")); + set("${.tmp.field}|${.tmp.data}", value(".message")); + }; + } elif { + filter{ + program('ecelerity' type(string)); + }; + rewrite { + r_set_splunk_dest_update_v2( + sourcetype('symantec:smg:ecelerity') + template('t_msg_only') + ); + }; + }; + + }; +}; +application app-syslog-symantec_brightmail-pgm[sc4s-syslog-pgm] { + filter { + program('bmserver' type(string) flags(prefix)) + or program('ecelerity' type(string)) + ; + }; + parser { app-syslog-symantec_brightmail(); }; +}; +application app-syslog-symantec_brightmail[sc4s-syslog] { + filter { + message('[Brightmail]' type(string) flags(substring)) + ; + }; + parser { app-syslog-symantec_brightmail(); }; +}; + + + +#Destination type formatters +block parser app-syslog-symantec_brightmail-splunk_hec() { + channel { + rewrite{ + set("t_json_values", value(".splunk.sc4s_template")); + }; + }; +}; +application app-syslog-symantec_brightmail-splunk_hec[sc4s-lp-dest-format-hec_fmt] { + filter { + match('symantec', value('fields.sc4s_vendor') type(string)) + and match('brightmail', value('fields.sc4s_product') type(string)) + and match('merged', value('fields.sc4s_class') type(string)) + and ( + "`SC4S_DEST_SYMANTEC_BRIGHTMAIL_SPLUNK_HEC_FMT`" eq "JSON" + ) + }; + parser { app-syslog-symantec_brightmail-splunk_hec(); }; +}; + +block parser app-syslog-symantec_brightmail-syslog() { + channel { + parser { + map-value-pairs( + key('.values.*' rekey(shift-levels(2) add-prefix(".SDATA.sc4sfields@27389."))) + ); + map-value-pairs( + key('fields.*' rekey(shift-levels(2) add-prefix(".SDATA.sc4smeta@27389."))) + ); + map-value-pairs( + key('.metadata.*' rekey(shift-levels(2) add-prefix(".SDATA.sc4smeta@27389."))) + ); }; + rewrite{ + unset(value('MESSAGE')); + }; + }; +}; +application app-syslog-symantec_brightmail-syslog[sc4s-lp-dest-format-syslog] { + filter { + match('symantec', value('fields.sc4s_vendor') type(string)) + and match('brightmail', value('fields.sc4s_product') type(string)) + and match('merged', value('fields.sc4s_class') type(string)) + and ( + "`SC4S_DEST_SYMANTEC_BRIGHTMAIL_SYSLOG_FMT`" eq "SDATA" + ) + }; + parser { app-syslog-symantec_brightmail-syslog(); }; +}; + +block parser app-syslog-symantec_brightmail-bsd() { + channel { + rewrite{ + set('$(template ${.splunk.sc4s_template:-t_hdr_msg})' value('MESSAGE')); + }; + }; +}; +application app-syslog-symantec_brightmail-bsd[sc4s-lp-dest-format-bsd] { + parser { app-syslog-symantec_brightmail-bsd(); }; +}; diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-symantec_dlp.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-symantec_dlp.conf new file mode 100644 index 0000000000..660e85e320 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-symantec_dlp.conf @@ -0,0 +1,40 @@ +block parser app-syslog-symantec_dlp() { + channel { + rewrite { + r_set_splunk_dest_default( + index('netdlp') + sourcetype('symantec:dlp:system') + vendor("symantec") + product('dlp') + template('t_msg_only') + ); + }; + if { + filter { + program('SymantecDLPAlert', type(string) flags(prefix)) + }; + + rewrite { + r_set_splunk_dest_update_v2( + sourcetype('symantec:dlp:syslog') + class('alert') + ); + }; + if { + parser{ + kv-parser( + prefix(".values.") + pair-separator(" ") + ); + }; + }; + }; + + }; +}; +application app-syslog-symantec_dlp[sc4s-syslog-pgm] { + filter { + program('SymantecDLP', type(string) flags(prefix)) + }; + parser { app-syslog-symantec_dlp(); }; +}; diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-symantec_ep.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-symantec_ep.conf new file mode 100644 index 0000000000..62c63a618d --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-symantec_ep.conf @@ -0,0 +1,137 @@ +block parser app-syslog-symantec_ep() { + channel { + rewrite { + r_set_splunk_dest_default( + index("epav") + sourcetype('symantec:ep:syslog') + vendor("symantec") + product("ep") + ); + }; + + if { + filter { + message(',Detection\stype:') + }; + rewrite { + r_set_splunk_dest_update_v2( + sourcetype('symantec:ep:proactive:syslog') + ); + }; + } elif { + filter { + message(',Risk\sname:') + }; + rewrite { + r_set_splunk_dest_update_v2( + sourcetype('symantec:ep:risk:syslog') + ); + }; + } elif { + filter { + message(',Category:\s\d+,') + }; + rewrite { + r_set_splunk_dest_update_v2( + sourcetype('symantec:ep:agt:system:syslog') + ); + }; + } elif { + filter { + message(',(?:Inbound|Outbound|Unknown),Application:') + }; + rewrite { + r_set_splunk_dest_update_v2( + sourcetype('symantec:ep:packet:syslog') + ); + }; + } elif { + filter { + message(',(?:Inbound|Outbound|Unknown),Begin(?:\sTime)?:') + }; + rewrite { + r_set_splunk_dest_update_v2( + sourcetype('symantec:ep:traffic:syslog') + ); + }; + } elif { + filter { + message('CIDS\sSignature\sSubID:') + }; + rewrite { + r_set_splunk_dest_update_v2( + sourcetype('symantec:ep:security:syslog') + ); + }; + } elif { + filter { + message('Scan\sID:\s\d+') + }; + rewrite { + r_set_splunk_dest_update_v2( + sourcetype('symantec:ep:scan:syslog') + ); + }; + } elif { + filter { + message('Begin(?:\sTime)?:\s[^,]*,End(?:\sTime)?:') + }; + rewrite { + r_set_splunk_dest_update_v2( + sourcetype('symantec:ep:behavior:syslog') + ); + }; + } elif { + filter { + message('Admin:\s[^,]+,.*[Pp]olicy') + }; + rewrite { + r_set_splunk_dest_update_v2( + sourcetype('symantec:ep:policy:syslog') + ); + }; + } elif { + filter { + message('Domain(?:\sName)?:\s[^,]{0,25},Admin:') + }; + rewrite { + r_set_splunk_dest_update_v2( + sourcetype('symantec:ep:admin:syslog') + ); + }; + } elif { + filter { + message('(?:,The\smanagement\sserver|,The\sclient|issued Command)') + }; + rewrite { + r_set_splunk_dest_update_v2( + sourcetype('symantec:ep:agent:syslog') + ); + }; + } elif { + filter { + message('Site:\s[^,]+,Server(?:\sName)?:\s[^,]+,') + }; + rewrite { + r_set_splunk_dest_update_v2( + sourcetype('symantec:ep:scm:system:syslog') + ); + }; + } else { + rewrite { + r_set_splunk_dest_update_v2( + sourcetype('symantec:ep:syslog') + ); + }; + }; + + + }; +}; +application app-syslog-symantec_ep[sc4s-syslog-pgm] { + filter { + program('SymantecServer' type(string) flags(prefix)); + }; + parser { app-syslog-symantec_ep(); }; +}; + diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-symantec_proxysg.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-symantec_proxysg.conf new file mode 100644 index 0000000000..76076468bb --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-symantec_proxysg.conf @@ -0,0 +1,52 @@ +block parser app-syslog-symantec_proxysg() { + channel { + + rewrite { + r_set_splunk_dest_default( + index("netops") + sourcetype('bluecoat:proxysg:access:syslog') + vendor('bluecoat') + product('proxy') + class('syslog') + ); + subst( + "([-_a-zA-Z\(\)]+=(\"-\"|-| ))", + "", value(MESSAGE) + ); + }; + if { + filter { + match('splunk_format' value('MSGID') type(string)) + or message('SPLV' type(string) flags(prefix)) + }; + rewrite { + r_set_splunk_dest_default( + index("netproxy") + sourcetype('bluecoat:proxysg:access:kv') + vendor('bluecoat') + product('proxy') + class('splunkkv') + template('t_msg_only') + ); + subst( + "([-_a-zA-Z\(\)]+=(\"-\"|-| ))", + "", value(MESSAGE) + ); + subst( + "^SPL[^ ]+ ", + "", value(MESSAGE) + condition(message('SPLV' type(string) flags(prefix))) + ); + + }; + }; + }; +}; +application app-syslog-symantec_proxysg[sc4s-syslog-pgm] { + filter { + program('bluecoat' type(string) flags(prefix)) + or program('ProxySG' type(string) flags(prefix)); + }; + parser { app-syslog-symantec_proxysg(); }; +}; + diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-syslog-ng_error.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-syslog-ng_error.conf new file mode 100644 index 0000000000..3bfaa1027d --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-syslog-ng_error.conf @@ -0,0 +1,22 @@ +block parser app-syslog-syslog-ng_error() { + channel { + rewrite { + r_set_splunk_dest_default( + index("main") + sourcetype('sc4s:fallback') + vendor('splunk') + product('sc4s') + class('fallback') + ); + }; + + + }; +}; +application app-syslog-syslog-ng_error[sc4s-syslog-pgm] { + filter { + program('syslog-ng' type(string) flags(prefix)) + and message('Error processing log message: ' type(string) flags(prefix)); + }; + parser { app-syslog-syslog-ng_error(); }; +}; diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-syslog-ng_loggen.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-syslog-ng_loggen.conf new file mode 100644 index 0000000000..f34d5af187 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-syslog-ng_loggen.conf @@ -0,0 +1,24 @@ +block parser app-syslog-syslog-ng_loggen() { + channel { + rewrite { + r_set_splunk_dest_default( + index("main") + sourcetype('syslogng:loggen') + vendor("syslogng") + product("loggen") + template("t_json_values_msg") + ); + }; + + + + }; +}; +application app-syslog-syslog-ng_loggen[sc4s-syslog-pgm] { + filter { + program('prg00000' type(string) flags(prefix)); + }; + parser { app-syslog-syslog-ng_loggen(); }; +}; + + diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-tanium.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-tanium.conf new file mode 100644 index 0000000000..79eb5507be --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-tanium.conf @@ -0,0 +1,63 @@ +block parser app-syslog-tanium() { + channel { + if { + filter{ + "${.SDATA.tanium_droid@017472.Question}" ne "" + }; + }; + rewrite { + r_set_splunk_dest_default( + index("epintel") + sourcetype('tanium') + vendor("tanium") + product("syslog") + template('t_5424_hdr_sdata_compact') + ); + set("${.SDATA.tanium_droid@017472.Question}", value(".tmp.question")); + subst(" ", '', value(".tanium.question")); + set("tanium:question:${.SDATA.tanium_droid@017472.Question}", value(".splunk.source") condition("${.SDATA.tanium_droid@017472.Question}" ne "")); + + set("tanium:inventory", value(".splunk.sourcetype") condition("${.tanium.question}" eq "Splunk Basic Asset")); + set("tanium:hardware:cpu", value(".splunk.sourcetype") condition("${.tanium.question}" eq "Splunk Hardware CPU")); + set("tanium:endpoint:processes", value(".splunk.sourcetype") condition("${.tanium.question}" eq "Splunk Running Processes MD5")); + set("tanium:endpoint:processes:stream", value(".splunk.sourcetype") condition( match('\"event\"\:\"process_start\"\,\"tanium_computer_id\"' value('SDATA')))); + set("tanium:endpoint:services", value(".splunk.sourcetype") condition("${.tanium.question}" eq "Splunk Running Services")); + set("tanium:change:endpoint", value(".splunk.sourcetype") condition("${.tanium.question}" eq "Splunk Integrity Monitor")); + set("tanium:endpoint:netconn:stream", value(".splunk.sourcetype") condition( match('\"event\"\:\"network_connect\"\,\"tanium_computer_id\"' value('SDATA')))); + set("tanium:endpoint:netdisco:stream", value(".splunk.sourcetype") condition( match('\"event\"\:\"network_disconnect\"\,\"tanium_computer_id\"' value('SDATA')))); + set("tanium:endpoint:netaccept:stream", value(".splunk.sourcetype") condition( match('\"event\"\:\"network_accept\"\,\"tanium_computer_id\"' value('SDATA')))); + set("tanium:endpoint:filecreate:stream", value(".splunk.sourcetype") condition( match('\"event\"\:\"file_create\"\,\"tanium_computer_id\"' value('SDATA')))); + set("tanium:endpoint:filewrite:stream", value(".splunk.sourcetype") condition( match('\"event\"\:\"file_write\"\,\"tanium_computer_id\"' value('SDATA')))); + set("tanium:endpoint:fileread:stream", value(".splunk.sourcetype") condition( match('\"event\"\:\"file_read\"\,\"tanium_computer_id\"' value('SDATA')))); + set("tanium:endpoint:fileopen:stream", value(".splunk.sourcetype") condition( match('\"event\"\:\"file_open\"\,\"tanium_computer_id\"' value('SDATA')))); + set("tanium:endpoint:filemove:stream", value(".splunk.sourcetype") condition( match('\"event\"\:\"file_move\"\,\"tanium_computer_id\"' value('SDATA')))); + set("tanium:endpoint:filedelete:stream", value(".splunk.sourcetype") condition( match('\"event\"\:\"file_delete\"\,\"tanium_computer_id\"' value('SDATA')))); + set("tanium:endpoint:filepermchange:stream", value(".splunk.sourcetype") condition( match('\"event\"\:\"file_permission_change\"\,\"tanium_computer_id\"' value('SDATA')))); + set("tanium:endpoint:regcreate:stream", value(".splunk.sourcetype") condition( match('\"event\"\:\"registry_create\"\,\"tanium_computer_id\"' value('SDATA')))); + set("tanium:endpoint:regset:stream", value(".splunk.sourcetype") condition( match('\"event\"\:\"registry_set\"\,\"tanium_computer_id\"' value('SDATA')))); + set("tanium:updates", value(".splunk.sourcetype") condition("${.tanium.question}" eq "Splunk Patch List Applicability")); + set("tanium:endpoint:dns:stream", value(".splunk.sourcetype") condition( match('\"event\"\:\"dns_event\"\,\"tanium_computer_id\"' value('SDATA')))); + set("tanium:ids:netconns", value(".splunk.sourcetype") condition("${.tanium.question}" eq "Splunk Non Approved Established Connections")); + set("tanium:malware:attack", value(".splunk.sourcetype") condition("${.tanium.question}" eq "Tanium Detect")); + set("tanium:detect:signals", value(".splunk.sourcetype") condition( match('\"Intel Type\"\:\"tanium-signal\"' value('SDATA')))); + set("tanium:detect:openioc", value(".splunk.sourcetype") condition( match('\"Intel Type\"\:\"openioc\"' value('SDATA')))); + set("tanium:detect:yara", value(".splunk.sourcetype") condition( match('\"Intel Type\"\:\"yara\"' value('SDATA')))); + + #set-tag("log_path_known"); + #set-tag("tanium"); + + }; + parser { + date-parser-nofilter(format('%Y-%m-%d %H:%M:%S') + template("${.SDATA.tanium_droid@017472.Client-Time-UTC}")); + }; + + + }; +}; +application app-syslog-tanium[sc4s-syslog-pgm] { + filter { + program('Tanium' type(string) flags(prefix)); + }; + parser { app-syslog-tanium(); }; +}; diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-tenable_ad.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-tenable_ad.conf new file mode 100644 index 0000000000..7dfae9c160 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-tenable_ad.conf @@ -0,0 +1,20 @@ +block parser app-syslog-tenable_ad() { + channel { + rewrite { + r_set_splunk_dest_default( + index('oswinsec') + sourcetype('tenable:ad:alerts') + vendor("tenable") + product("ad") + class("alert") + ); + }; + + }; +}; +application app-syslog-tenable_ad[sc4s-syslog-pgm] { + filter { + "${PROGRAM}" eq "Tenable.ad"; + }; + parser { app-syslog-tenable_ad(); }; +}; diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-tenable_syslog.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-tenable_syslog.conf new file mode 100644 index 0000000000..ad3a24b0d9 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-tenable_syslog.conf @@ -0,0 +1,19 @@ +block parser app-syslog-tenable_syslog() { + channel { + rewrite { + r_set_splunk_dest_default( + index('netfw') + sourcetype('tenable:nnm:vuln') + vendor("tenable") + product("nnm") + ); + }; + + }; +}; +application app-syslog-tenable_syslog[sc4s-syslog-pgm] { + filter { + "${PROGRAM}" eq "nnm"; + }; + parser { app-syslog-tenable_syslog(); }; +}; diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-tintri.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-tintri.conf new file mode 100644 index 0000000000..d69dc2a5a6 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-tintri.conf @@ -0,0 +1,19 @@ +block parser app-syslog-tintri() { + channel { + rewrite { + r_set_splunk_dest_default( + index('infraops') + sourcetype('tintri') + vendor("tintri") + product("syslog") + ); + }; + }; +}; + +application app-syslog-tintri[sc4s-syslog] { + filter { + message(',com.tintri.log.LogBase] ', type(string), flags(substring)) + }; + parser { app-syslog-tintri(); }; +}; diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-ubiquiti_unifi.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-ubiquiti_unifi.conf new file mode 100644 index 0000000000..479e1cdd85 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-ubiquiti_unifi.conf @@ -0,0 +1,110 @@ +block parser app-syslog-ubiquiti_unifi() { + channel { + + parser { + regexp-parser( + prefix(".tmp.") + patterns('^(?:(?[^, ]+)(?: |,))?(([abcedf0-9]+),(?:v|.+-)\d+\.\d+\.\d+(?:\.\d+)?(?:\+\d+)?(?:: | ).*)') + ); + }; + + rewrite { + r_set_splunk_dest_default( + index('netops') + sourcetype('ubnt') + vendor("ubiquiti") + product("unifi") + ); + set("${.tmp.host}" value("HOST") condition("$1" ne "" and not match('^U[A-Z0-9\-a-z]+$', value("1")))); + }; + + if (message('^\S+\slinkcheck:')) { + rewrite { + r_set_splunk_dest_update_v2( + class('link') + sourcetype('ubnt:link') + ); + }; + } elif (message('^\S+\sdhcpd:\s\w+')) { + rewrite { + r_set_splunk_dest_update_v2( + class('dhcp') + sourcetype('ubnt:dhcp') + ); + }; + } elif (message('dnsmasq-dhcp\[\d+\]')) { + rewrite { + r_set_splunk_dest_update_v2( + class('dhcp') + sourcetype('ubnt:dhcp') + ); + }; + } elif (message('dnsmasq\[\d+\]')) { + rewrite { + r_set_splunk_dest_update_v2( + class('dns') + sourcetype('ubnt:dns') + ); + }; + } elif (message('kernel:\s\[\S+-[A|B|D]')) { + rewrite { + r_set_splunk_dest_update_v2( + class('fw') + sourcetype('ubnt:fw') + ); + }; + } elif (message('\d+:\d+:\d+\s\S+\smcad:')) { + rewrite { + r_set_splunk_dest_update_v2( + class('mcad') + sourcetype('ubnt:mcad') + ); + }; + } elif (message('\d+:\d+:\d+\s\S+\ssudo')) { + rewrite { + r_set_splunk_dest_update_v2( + class('sudo') + sourcetype('ubnt:sudo') + ); + }; + } elif (message('hostapd:\s+ath')) { + rewrite { + r_set_splunk_dest_update_v2( + class('hostapd') + sourcetype('ubnt:hostapd') + ); + }; + } elif (message('[^)]\s\S+\skernel:\s[^ll\sheader][^\[\d+.\d+\]]\S+\s\w+:')) { + rewrite { + r_set_splunk_dest_update_v2( + class('threat') + sourcetype('ubnt:threat') + ); + }; + } elif (message('EVT_AP_STA_ASSOC_TRACKER_DBG:')) { + rewrite { + r_set_splunk_dest_update_v2( + class('wireless') + sourcetype('ubnt:wireless') + ); + }; + } elif (message('wevent.ubnt_custom_event\(\):')) { + rewrite { + r_set_splunk_dest_update_v2( + class('wireless') + sourcetype('ubnt:wireless') + ); + }; + } elif (message('traputil.c\(696\) ')) { + rewrite { + r_set_splunk_dest_update_v2( + class('edgeswitch') + sourcetype('ubnt:edgeswitch') + ); + }; + }; + }; +}; +application app-syslog-ubiquiti_unifi[sc4s-syslog] { + parser { app-syslog-ubiquiti_unifi(); }; +}; diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-veeam_vbr.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-veeam_vbr.conf new file mode 100644 index 0000000000..c02a5ffb8c --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-veeam_vbr.conf @@ -0,0 +1,19 @@ +block parser app-syslog-veeam_vbr() { + channel { + rewrite { + r_set_splunk_dest_default( + index("infraops") + sourcetype('veeam:vbr:syslog') + vendor('veeam') + product('vbr') + class('syslog') + ); + }; + }; +}; +application app-syslog-veeam_vbr[sc4s-syslog-pgm] { + filter { + program('Veeam_MP' type(string) flags(prefix)); + }; + parser { app-syslog-veeam_vbr(); }; +}; \ No newline at end of file diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-vmware_cb-protect.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-vmware_cb-protect.conf new file mode 100644 index 0000000000..43479dc795 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-vmware_cb-protect.conf @@ -0,0 +1,47 @@ +block parser app-syslog-vmware_cb-protect() { + channel { + rewrite { + r_set_splunk_dest_default( + index("epintel") + sourcetype('vmware:cb:protect') + vendor("vmware") + product("cb-protect") + template("t_kv_values") + ); + }; + + if ("`SC4S_DISABLE_DROP_INVALID_VMWARE_CB_PROTECT`" eq "yes") { + parser { + csv-parser(delimiters(chars('') strings(': ')) + columns('header', 'message') + prefix('.tmp.') + flags(greedy)); + kv-parser( + prefix(".values.") + pair-separator(" ") + template('${.tmp.message}') + ); + }; + } elif { + parser { + csv-parser(delimiters(chars('') strings(': ')) + columns('header', 'message') + prefix('.tmp.') + flags(greedy, drop-invalid)); + kv-parser( + prefix(".values.") + pair-separator(" ") + template('${.tmp.message}') + ); + }; + } else {}; + }; +}; +application app-syslog-vmware_cb-protect[sc4s-syslog] { + filter { + message('Carbon Black App Control event: ' type(string) flags(prefix)); + }; + parser { app-syslog-vmware_cb-protect(); }; +}; + + diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-vmware_horizon.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-vmware_horizon.conf new file mode 100644 index 0000000000..4d5a98da90 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-vmware_horizon.conf @@ -0,0 +1,29 @@ +block parser app-syslog-vmware_horizon() { + channel { + rewrite { + set("$(lowercase ${PROGRAM})", value(".PROGRAM")); + subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM")); + r_set_splunk_dest_default( + index("infraops") + sourcetype('vmware:horizon') + vendor("vmware") + product("horizon") + template('t_5424_hdr_sdata_compact') + ); + }; + }; +}; +application app-syslog-vmware_horizon[sc4s-syslog-sdata] { + filter { + match('View@6876' value('SDATA')) + }; + parser { app-syslog-vmware_horizon(); }; +}; + +application app-syslog-vmware_horizon[sc4s-syslog-pgm] { + filter { + program("^view", flags(ignore-case)) + }; + parser { app-syslog-vmware_horizon(); }; +}; + diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-vmware_vsphere.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-vmware_vsphere.conf new file mode 100644 index 0000000000..4b48e01c35 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-vmware_vsphere.conf @@ -0,0 +1,165 @@ +filter syslog-vmware_vsphere-sdata{ + match('[[^\@]@6876' value('SDATA')) +}; + +filter syslog-vmware_vsphere-esx-pgm{ + program("vmk", type(string) flags(ignore-case, prefix)) + or program("lsud", type(string) flags(ignore-case, prefix)) + or program("vsan", type(string) flags(ignore-case, prefix)) + or program("iofiltervpd", type(string) flags(ignore-case,prefix)) + or program("hostd", type(string) flags(ignore-case, prefix)) + or program("cmmdstimemachine", type(string) flags(ignore-case, prefix)) + or program("vmware", type(string) flags(ignore-case, prefix)) + or program("vpxa", type(string) flags(prefix, ignore-case)) + or program("eam", type(string) flags(prefix, ignore-case)) + or program("rhttpproxy", type(string) flags(prefix, ignore-case)) + or program("sdrsInjector", type(string) flags( ignore-case)) + or program("fdm", type(string) flags( ignore-case)) + + #Requested by community https://github.com/splunk/splunk-connect-for-syslog/issues/1699 + or program("esxupdate", type(string) flags(ignore-case,prefix)) + or program("healthd", type(string) flags(ignore-case,prefix)) + or program("ConfigStore", type(string) flags(ignore-case,prefix)) + or program("kmxa", type(string) flags(ignore-case,prefix)) + or program("crx-cli", type(string) flags(ignore-case,prefix)) + or program("backup.sh", type(string) flags(ignore-case,prefix)) + or program("configStoreBackup", type(string) flags(ignore-case,prefix)) + or program("heartbeat", type(string) flags(ignore-case,prefix)) + + or program("Host", type(string) flags(ignore-case,prefix)) + or program("vmauthd", type(string) flags(ignore-case,prefix)) + or program("localcli", type(string) flags(ignore-case,prefix)) + or program("watchdog-vsanperfsvc", type(string) flags(ignore-case,prefix)) + or program("watchdog-iofiltervpd", type(string) flags(ignore-case,prefix)) + or program("apiForwarder", type(string) flags(ignore-case,prefix)) + or program("tmpwatch", type(string) flags(ignore-case,prefix)) + or program(".etc.init.d.vsanmgmtd", type(string) flags(ignore-case,prefix)) + or program("ComplianceManager", type(string) flags(ignore-case,prefix)) + or program("hostprofiletrace", type(string) flags(ignore-case,prefix)) + + or program("vobd", type(string) flags(ignore-case,prefix)) + or program("ucs-tool-esxi-inv", type(string) flags(ignore-case,prefix)) + or program("usbarb", type(string) flags(ignore-case,prefix)) + +}; + +filter syslog-vmware_vsphere-esx-autobackup-pgm{ + program("auto-backup.sh", type(string) flags(ignore-case,prefix)) +}; + +filter syslog-vmware_vsphere-nsx-pgm{ + program("nsx", type(string), flags(ignore-case,prefix)) + or program("dfwpktlogs", type(string), flags(ignore-case)) +}; + +filter syslog-vmware_vsphere-vsan-pgm{ + program("vsan", type(string), flags(ignore-case,prefix)) + or program("storagerm", type(string), flags(ignore-case,prefix)) +}; + +filter syslog-vmware_vsphere-vcenter-pgm{ + #Used by TA + program("vpxd", type(string) flags(ignore-case,prefix)) + or program("vws", type(string) flags(ignore-case,prefix)) + or program("stats", type(string) flags(ignore-case,prefix)) + or program("cim-diag", type(string) flags(ignore-case,prefix)) + or program("sms", type(string) flags(ignore-case,prefix)) + or program("vim", type(string) flags(ignore-case,prefix)) + + #Field identified + or program("cis-license", type(string) flags(ignore-case,prefix)) + or program("applmgmt-audit", type(string) flags(ignore-case,prefix)) + or program("updatemgr", type(string) flags(ignore-case,prefix)) + or program("vmafdd", type(string) flags(ignore-case,prefix)) + or program("vmcad", type(string) flags(ignore-case,prefix)) + or program("vmdird", type(string) flags(ignore-case,prefix)) + or program("vmon", type(string) flags(ignore-case,prefix)) + or program("osfsd", type(string) flags(ignore-case,prefix)) + or program("wcpxsvc", type(string) flags(ignore-case,prefix)) + or program("wcpsvc", type(string) flags(ignore-case,prefix)) + or program("mbcs", type(string) flags(ignore-case,prefix)) + or program("vmcam", type(string) flags(ignore-case,prefix)) + or program("vpostgres", type(string) flags(ignore-case,prefix)) + or program("vsphere", type(string) flags(ignore-case,prefix)) + or program("vcha", type(string) flags(ignore-case,prefix)) + + #Requested by community https://github.com/splunk/splunk-connect-for-syslog/issues/1699 + or program("vcenter-server", type(string) flags(ignore-case,prefix)) + + +}; +block parser app-syslog-vmware_vsphere() { + channel { + rewrite { + r_set_splunk_dest_default( + index("infraops") + source('vmware:esxlog:$(lowercase ${PROGRAM})') + sourcetype('vmware:esxlog:$(lowercase ${PROGRAM})') + vendor("vmware") + product("vsphere") + class("esx") + template('t_5424_hdr_sdata_compact') + ); + }; + + rewrite { + r_set_splunk_dest_update( + source('vmware:esxlog:$(lowercase ${PROGRAM})') + sourcetype("vmware:esxlog:auto-backup") + class("esx") + condition(filter(syslog-vmware_vsphere-esx-autobackup-pgm)) + ); + }; + + rewrite { + r_set_splunk_dest_update( + source('vmware:vclog:$(lowercase ${PROGRAM})') + sourcetype('vmware:vclog:$(lowercase ${PROGRAM})') + class("vc") + condition(filter(syslog-vmware_vsphere-vcenter-pgm)) + ); + }; + rewrite { + r_set_splunk_dest_update( + index("infraops") + source('vmware:nsxlog:$(lowercase ${PROGRAM})') + sourcetype('vmware:nsxlog:$(lowercase ${PROGRAM})') + class("nsx") + condition(filter(syslog-vmware_vsphere-nsx-pgm)) + ); + }; + rewrite { + r_set_splunk_dest_update( + index("netfw") + source('vmware:nsxlog:$(lowercase ${PROGRAM})') + sourcetype('vmware:nsxlog:$(lowercase ${PROGRAM})') + class("nsxfw") + condition(program("dfwpktlogs", type(string), flags(ignore-case))) + ); + }; + }; +}; +application app-syslog-vmware_vsphere-sdata[sc4s-syslog-sdata] { + filter { + filter(syslog-vmware_vsphere-sdata) + }; + parser { app-syslog-vmware_vsphere(); }; +}; + +application app-syslog-vmware_vsphere-pgm[sc4s-syslog-pgm] { + filter { + filter(syslog-vmware_vsphere-vcenter-pgm) + or filter(syslog-vmware_vsphere-esx-pgm) + or filter(syslog-vmware_vsphere-esx-autobackup-pgm) + or filter(syslog-vmware_vsphere-nsx-pgm) + or filter(syslog-vmware_vsphere-vsan-pgm) + ; + }; + parser { app-syslog-vmware_vsphere(); }; +}; +application app-syslog-vmware_vsphere-msg[sc4s-syslog] { + filter { + message('[Originator@6876 ' type(string) flags(substring)); + }; + parser { app-syslog-vmware_vsphere(); }; +}; diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-vormetric.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-vormetric.conf new file mode 100644 index 0000000000..7e9db38eb0 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-vormetric.conf @@ -0,0 +1,20 @@ +block parser app-syslog-thales_vormetric() { + channel { + rewrite { + r_set_splunk_dest_default( + index("netauth") + sourcetype('thales:vormetric') + vendor("thales") + product("vormetric") + template("t_5424_hdr_sdata_compact") + ); + }; + }; +}; + +application app-syslog-thales_vormetric[sc4s-syslog-sdata] { + filter { + match('@21513' value("SDATA")); + }; + parser { app-syslog-thales_vormetric(); }; +}; \ No newline at end of file diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-wallx_proxy.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-wallx_proxy.conf new file mode 100644 index 0000000000..f3fcf665a3 --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-wallx_proxy.conf @@ -0,0 +1,25 @@ +block parser app-syslog-wallx_proxy() { + channel { + rewrite { + r_set_splunk_dest_default( + index("infraops") + sourcetype('WB:syslog') + vendor("wallix") + product("bastion") + ); + }; + + + + }; +}; +application app-syslog-wallx_proxy[sc4s-syslog-pgm] { + filter { + program('wabengine' type(string)) + or program('rdpproxy' type(string)) + or program('sshproxy' type(string)); + }; + parser { app-syslog-wallx_proxy(); }; +}; + + diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-zscaler_nss_alerts.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-zscaler_nss_alerts.conf new file mode 100644 index 0000000000..dfec1b54ad --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-zscaler_nss_alerts.conf @@ -0,0 +1,25 @@ +block parser app-syslog-zscaler_nss_alerts() { + channel { + # + rewrite { + r_set_splunk_dest_default( + index('netops') + sourcetype('zscalernss-alerts') + vendor("zscaler") + product("nss") + class("alerts") + ); + }; + + }; +}; +application app-syslog-zscaler_nss_alerts[sc4s-syslog-pgm] { + filter { + program('ZscalerNSS' type(string)) + ; + }; + parser { app-syslog-zscaler_nss_alerts(); }; +}; + + + diff --git a/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-zscaler_nss_proxy.conf b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-zscaler_nss_proxy.conf new file mode 100644 index 0000000000..b498e0580f --- /dev/null +++ b/package/enterprise/etc/conf.d/conflib/syslog/app-syslog-zscaler_nss_proxy.conf @@ -0,0 +1,94 @@ +block parser app-syslog-zscaler_nss_proxy() { + channel { + parser { + #basic parsing + kv-parser( + prefix(".values.") + pair-separator("\t") + ); + }; + filter { + match("Zscaler" value(".values.vendor")); + }; + + if (match("dns" value(".values.product"))) { + rewrite { + r_set_splunk_dest_default( + index('netdns') + vendor("zscaler") + product("nss") + class('dns') + sourcetype('zscalernss-dns') + ); + }; + } elif (match("fw" value(".values.product"))) { + rewrite { + r_set_splunk_dest_default( + vendor("zscaler") + product("nss") + index('netfw') + sourcetype('zscalernss-fw') + class('fw') + ); + }; + } elif (match("NSS" value(".values.product"))) { + rewrite { + r_set_splunk_dest_default( + index('netproxy') + vendor("zscaler") + product("nss") + sourcetype('zscalernss-web') + class('web') + ); + }; + } elif (match("audit" value(".values.product"))) { + rewrite { + r_set_splunk_dest_default( + index('netops') + sourcetype('zscalernss-zia-audit') + product('zia') + class('audit') + ); + }; + } elif (match("sandbox" value(".values.product"))) { + rewrite { + r_set_splunk_dest_default( + index('main') + sourcetype('zscalernss-zia-sandbox') + product('zia') + class('sandbox') + ); + }; + } elif (match("tunnel.*|IKE.*" value(".values.product"))) { + rewrite { + r_set_splunk_dest_update( + index('netops') + vendor('zscaler') + sourcetype('zscalernss-tunnel') + product('nss') + class('tunnel') + ); + }; + } else { + rewrite { + r_set_splunk_dest_default( + index('netproxy') + vendor("zscaler") + product("nss") + sourcetype('zscalernss-web') + ); + }; + }; + + }; +}; +application app-syslog-zscaler_nss_proxy[sc4s-syslog] { + filter { + message('\t[^\t]+\t') + ; + }; + parser { app-syslog-zscaler_nss_proxy(); }; +}; + + + diff --git a/package/enterprise/etc/conf.d/destinations/archive_file.conf b/package/enterprise/etc/conf.d/destinations/archive_file.conf new file mode 100644 index 0000000000..7fb3cb22e3 --- /dev/null +++ b/package/enterprise/etc/conf.d/destinations/archive_file.conf @@ -0,0 +1,13 @@ +template t_archive_compliance { + template('`SC4S_VAR`/archive/${.splunk.sourcetype}/${HOST}/$YEAR-$MONTH-$DAY-archive.log'); +}; +template t_archive_diode { + template('`SC4S_VAR`/archive/$YEAR/$MONTH/$DAY/${fields.sc4s_vendor}_${fields.sc4s_product}_$YEAR$MONTH$DAY$HOUR$MIN.log'); +}; + +destination d_archive { + file( + '$(if ("`SC4S_GLOBAL_ARCHIVE_MODE`" ne "diode") "$(template t_archive_compliance)" "$(template t_archive_diode)")' + template("$(format-ewmm)") + ); +}; \ No newline at end of file diff --git a/package/enterprise/etc/conf.d/destinations/dest_bsd/plugin.conf b/package/enterprise/etc/conf.d/destinations/dest_bsd/plugin.conf new file mode 100644 index 0000000000..952e111493 --- /dev/null +++ b/package/enterprise/etc/conf.d/destinations/dest_bsd/plugin.conf @@ -0,0 +1,3 @@ + +@module confgen context(root) name(dest_bsd) exec("`SC4S_ETC`/conf.d/destinations/dest_bsd/plugin.py") +dest_bsd() diff --git a/package/enterprise/etc/conf.d/destinations/dest_bsd/plugin.jinja b/package/enterprise/etc/conf.d/destinations/dest_bsd/plugin.jinja new file mode 100644 index 0000000000..b754183ac0 --- /dev/null +++ b/package/enterprise/etc/conf.d/destinations/dest_bsd/plugin.jinja @@ -0,0 +1,36 @@ + +destination d_bsd{{ dest_mode }}{{ altname }}{ + network( + "{{ host }}" + log-fifo-size({{ log_fifo_size }}) + port({{ port }}) + transport("{{ transport }}") + persist-name("network{{ dest_mode }}{{ group }}") + + {%- if diskbuff_enable %} + disk-buffer( + compaction(yes) + {%- if diskbuff_reliable %} + mem-buf-size({{ mem_buf_size }}) + reliable(yes) + {%- else %} + mem-buf-length({{ mem_buf_length }}) + reliable(no) + {%- endif %} + disk-buf-size({{ disk_buf_size }}) + ) + {%- endif %} + {%- if tls %} + tls( + peer-verify({{ peer_verify }}) + {%- if cipher_suite %} + cipher-suite("{{ cipher_suite }}") + {%- endif %} + {%- if ssl_version %} + ssl-version("{{ ssl_version }}") + {%- endif %} + ) + {%- endif %} + + ); +}; diff --git a/package/enterprise/etc/conf.d/destinations/dest_bsd/plugin.py b/package/enterprise/etc/conf.d/destinations/dest_bsd/plugin.py new file mode 100755 index 0000000000..60e8dc7537 --- /dev/null +++ b/package/enterprise/etc/conf.d/destinations/dest_bsd/plugin.py @@ -0,0 +1,92 @@ +#! /usr/bin/env python3 +import os +import shutil +import jinja2 +import re + +plugin_path = os.path.dirname(os.path.abspath(__file__)) + +templateLoader = jinja2.FileSystemLoader(searchpath=plugin_path) +templateEnv = jinja2.Environment( + loader=templateLoader, + autoescape=jinja2.select_autoescape(default_for_string=False), +) +tm = templateEnv.get_template("plugin.jinja") + +msg_template = "$(template t_syslog)" +dests = [] + +regex = r"^SC4S_DEST_BSD_(.*)_HOST$" +for vn, vv in os.environ.items(): + m = re.search(regex, vn) + r = m.group(1) if m else "" + if r != "": + dests.append(r) + +for group in dests: + altname = f"_{ group }".lower() + + # print (mode) + if os.getenv(f"SC4S_DEST_BSD_{ group }_DISKBUFF_ENABLE", "yes").lower() in [ + "true", + "1", + "t", + "y", + "yes", + ]: + diskbuff_enable = True + else: + diskbuff_enable = False + + if os.getenv(f"SC4S_DEST_BSD_{ group }_DISKBUFF_RELIABLE", "no").lower() in [ + "true", + "1", + "t", + "y", + "yes", + ]: + diskbuff_reliable = True + else: + diskbuff_reliable = False + + # Used to calc disk space for buffer + disk_space, used, free = shutil.disk_usage(os.getenv("SC4S_VAR", "/")) + disk_space = disk_space - 5000000000 + + if disk_space < 0: + disk_space = 5000000000 + + port = os.getenv(f"SC4S_DEST_BSD_{ group }_PORT", 514) + transport = os.getenv(f"SC4S_DEST_BSD_{ group }_TRANSPORT", "tcp") + + #### if TLS is used as a transport type + tls = True if transport in ["tls", "TLS"] else False + + msg = tm.render( + tls=tls, + group=group, + altname=altname, + port=port, + transport=transport, + host=os.getenv(f"SC4S_DEST_BSD_{ group }_HOST"), + log_fifo_size=os.getenv(f"SC4S_DEST_BSD_{ group }_LOG_FIFO_SIZE", 180000000), + diskbuff_enable=diskbuff_enable, + diskbuff_reliable=diskbuff_reliable, + mem_buf_size=os.getenv( + f"SC4S_DEST_BSD_{ group }_DISKBUFF_MEMBUFSIZE", + int(163840000), + ), + mem_buf_length=os.getenv( + f"SC4S_DEST_BSD_{ group }_DISKBUFF_MEMBUFLENGTH", + int(60000), + ), + disk_buf_size=os.getenv( + f"SC4S_DEST_BSD_{ group }_DISKBUFF_DISKBUFSIZE", + int(disk_space), + ), + peer_verify=os.getenv(f"SC4S_DEST_BSD_{ group }_TLS_VERIFY", "yes"), + cipher_suite=os.getenv(f"SC4S_DEST_BSD_{ group }_CIPHER_SUITE"), + ssl_version=os.getenv(f"SC4S_DEST_BSD_{ group }_SSL_VERSION"), + ) + + print(msg) diff --git a/package/enterprise/etc/conf.d/destinations/dest_hec/plugin.conf b/package/enterprise/etc/conf.d/destinations/dest_hec/plugin.conf new file mode 100644 index 0000000000..81d15ffc09 --- /dev/null +++ b/package/enterprise/etc/conf.d/destinations/dest_hec/plugin.conf @@ -0,0 +1,3 @@ + +@module confgen context(root) name(dest_hec) exec("`SC4S_ETC`/conf.d/destinations/dest_hec/plugin.py") +dest_hec() diff --git a/package/enterprise/etc/conf.d/destinations/dest_hec/plugin.jinja b/package/enterprise/etc/conf.d/destinations/dest_hec/plugin.jinja new file mode 100644 index 0000000000..aa47b4b879 --- /dev/null +++ b/package/enterprise/etc/conf.d/destinations/dest_hec/plugin.jinja @@ -0,0 +1,56 @@ + +destination d_hec{{ dest_mode }}{{ altname }}{ + http( + url("{{ url }}") + method("POST") + log-fifo-size({{ log_fifo_size }}) + workers({{ workers }}) + batch-lines({{ batch_lines }}) + batch-bytes({{ batch_bytes }}) + batch-timeout({{ batch_timeout }}) + timeout({{ timeout }}) + user_agent({{ user_agent }}) + {%- if headers %} + headers({{ headers }}) + {%- endif %} + persist-name("splunk_hec{{ dest_mode }}{{ group }}") + response-action(400 => drop, 404 => retry) + + {%- if diskbuff_enable %} + disk-buffer( + compaction(yes) + {%- if diskbuff_reliable %} + mem-buf-size({{ mem_buf_size }}) + reliable(yes) + {%- else %} + mem-buf-length({{ mem_buf_length }}) + reliable(no) + {%- endif %} + disk-buf-size({{ disk_buf_size }}) + {%- if buff_dir_enable %} + dir("{{ buff_dir }}") + {%- endif %} + ) + {%- endif %} + tls( + peer-verify({{ peer_verify }}) + {% if tls_mount %} + key-file("{{ tls_mount }}/key.pem") + cert-file("{{ tls_mount }}/cert.pem") + ca-dir("{{ tls_mount }}") + {% endif %} + {%- if cipher_suite %} + cipher-suite("{{ cipher_suite }}") + {%- endif %} + {%- if ssl_version %} + ssl-version("{{ ssl_version }}") + {%- endif %} + ) + use-system-cert-store(yes) + body('{{ msg_template }}') + {%- if http_compression %} + content-compression("gzip") + accept-encoding("all") + {%- endif %} + ); +}; diff --git a/package/enterprise/etc/conf.d/destinations/dest_hec/plugin.py b/package/enterprise/etc/conf.d/destinations/dest_hec/plugin.py new file mode 100755 index 0000000000..e75f9492c8 --- /dev/null +++ b/package/enterprise/etc/conf.d/destinations/dest_hec/plugin.py @@ -0,0 +1,154 @@ +#! /usr/bin/env python3 +import os +import shutil +import jinja2 +import re + +def hec_endpoint_collector(hec_path, url_hec): + """the function is used to validate if the alternate destination url is correct""" + if hec_path in url_hec: + endpoint = url_hec + else: + endpoint = f"{url_hec}{hec_path}" + return endpoint + +plugin_path = os.path.dirname(os.path.abspath(__file__)) + +templateLoader = jinja2.FileSystemLoader(searchpath=plugin_path) +templateEnv = jinja2.Environment( + loader=templateLoader, + autoescape=jinja2.select_autoescape(default_for_string=False), +) +tm = templateEnv.get_template("plugin.jinja") + +msg_template = "$(template ${.splunk.sc4s_hec_template} $(template t_splunk_hec))" +dest_mode = "_fmt" +dests = [] + +regex = r"^SC4S_DEST_SPLUNK_HEC_(.*)_URL$" +for vn, vv in os.environ.items(): + m = re.search(regex, vn) + r = m.group(1) if m else "" + if r != "": + dests.append(r) + +for group in dests: + url = os.getenv(f"SC4S_DEST_SPLUNK_HEC_{group}_URL") + altname = "" + if group != "DEFAULT": + altname = f"_{group}".lower() + hec_endpoint_path = "/services/collector/event" + url = hec_endpoint_collector(hec_endpoint_path, url) + + # print (mode) + if os.getenv(f"SC4S_DEST_SPLUNK_HEC_{group}_DISKBUFF_ENABLE", "yes").lower() in [ + "true", + "1", + "t", + "y", + "yes", + ]: + diskbuff_enable = True + else: + diskbuff_enable = False + + if os.getenv(f"SC4S_DEST_SPLUNK_HEC_{group}_DISKBUFF_RELIABLE", "no").lower() in [ + "true", + "1", + "t", + "y", + "yes", + ]: + diskbuff_reliable = True + else: + diskbuff_reliable = False + +# Disk buffer directory for BYOE setup , don't use it for container solutions + buff_dir = os.getenv(f"SC4S_DEST_SPLUNK_HEC_{group}_DISKBUFF_DIR", "") + if buff_dir != "": + buff_dir_enable = True + else: + buff_dir_enable = False + + # Used to calc disk space for buffer + disk_space, used, free = shutil.disk_usage(os.getenv("SC4S_VAR", "/")) + disk_space = disk_space - 5000000000 + + if disk_space < 0: + disk_space = 5000000000 + + workers = int(os.getenv(f"SC4S_DEST_SPLUNK_HEC_{group}_WORKERS", 10)) + headers = [] + user_headers = os.getenv(f"SC4S_DEST_SPLUNK_HEC_{group}_HEADERS", "") + if user_headers != "": + headers += user_headers.split(",") + token = os.getenv(f"SC4S_DEST_SPLUNK_HEC_{group}_TOKEN") + headers.append(f"Authorization: Splunk {token}") + headers.append("__splunk_app_name: sc4syslog") + sc4s_version = os.getenv('SC4S_VERSION', "0.0.0") + headers.append(f"__splunk_app_version: {sc4s_version}") + + user_agent = f"sc4s/{sc4s_version}" + + if os.getenv(f"SC4S_DEST_SPLUNK_HEC_{group}_CONNECTION_CLOSE", "no").lower() in [ + "true", + "1", + "t", + "y", + "yes", + ]: + headers.append("Connection: close") + else: + headers.append("Connection: keep-alive") + + if os.getenv(f"SC4S_DEST_SPLUNK_HEC_{group}_HTTP_COMPRESSION", "no").lower() in [ + "true", + "1", + "t", + "y", + "yes", + ]: + http_compression = True + else: + http_compression = False + + msg = tm.render( + group=group, + altname=altname, + msg_template=msg_template, + dest_mode=dest_mode, + url=url, + log_fifo_size=os.getenv( + f"SC4S_DEST_SPLUNK_HEC_{group}_LOG_FIFO_SIZE", 180000000 + ), + workers=workers, + batch_lines=os.getenv(f"SC4S_DEST_SPLUNK_HEC_{group}_BATCH_LINES", 5000), + batch_bytes=os.getenv(f"SC4S_DEST_SPLUNK_HEC_{group}_BATCH_BYTES", "4096kb"), + batch_timeout=os.getenv(f"SC4S_DEST_SPLUNK_HEC_{group}_BATCH_TIMEOUT", 300), + timeout=os.getenv(f"SC4S_DEST_SPLUNK_HEC_{group}_TIMEOUT", 30), + user_agent=user_agent, + buff_dir=buff_dir, + buff_dir_enable=buff_dir_enable, + headers='"{0}"'.format('", "'.join(headers)), + diskbuff_enable=diskbuff_enable, + diskbuff_reliable=diskbuff_reliable, + mem_buf_size=os.getenv( + f"SC4S_DEST_SPLUNK_HEC_{group}_DISKBUFF_MEMBUFSIZE", + int(163840000 / workers), + ), + mem_buf_length=os.getenv( + f"SC4S_DEST_SPLUNK_HEC_{group}_DISKBUFF_MEMBUFLENGTH", + int(60000 / workers), + ), + disk_buf_size=os.getenv( + f"SC4S_DEST_SPLUNK_HEC_{group}_DISKBUFF_DISKBUFSIZE", + int(disk_space / workers), + ), + tls_mount=os.getenv(f"SC4S_DEST_SPLUNK_HEC_{group}_TLS_MOUNT"), + peer_verify=os.getenv(f"SC4S_DEST_SPLUNK_HEC_{group}_TLS_VERIFY", "yes"), + cipher_suite=os.getenv(f"SC4S_DEST_SPLUNK_HEC_{group}_CIPHER_SUITE"), + ssl_version=os.getenv(f"SC4S_DEST_SPLUNK_HEC_{group}_SSL_VERSION"), + http_compression=http_compression + ) + + print(msg) diff --git a/package/enterprise/etc/conf.d/destinations/dest_psc/plugin.conf b/package/enterprise/etc/conf.d/destinations/dest_psc/plugin.conf new file mode 100644 index 0000000000..e9f8f6f56b --- /dev/null +++ b/package/enterprise/etc/conf.d/destinations/dest_psc/plugin.conf @@ -0,0 +1,7 @@ +destination d_psc { + python( + class("parser_source_cache.psc_dest") + batch-timeout(3000) + batch-lines(100) + ); +}; \ No newline at end of file diff --git a/package/enterprise/etc/conf.d/destinations/dest_syslog/plugin.conf b/package/enterprise/etc/conf.d/destinations/dest_syslog/plugin.conf new file mode 100644 index 0000000000..e352417b52 --- /dev/null +++ b/package/enterprise/etc/conf.d/destinations/dest_syslog/plugin.conf @@ -0,0 +1,3 @@ + +@module confgen context(root) name(dest_syslog) exec("`SC4S_ETC`/conf.d/destinations/dest_syslog/plugin.py") +dest_syslog() diff --git a/package/enterprise/etc/conf.d/destinations/dest_syslog/plugin.jinja b/package/enterprise/etc/conf.d/destinations/dest_syslog/plugin.jinja new file mode 100644 index 0000000000..b296c95770 --- /dev/null +++ b/package/enterprise/etc/conf.d/destinations/dest_syslog/plugin.jinja @@ -0,0 +1,43 @@ + +destination d_syslog{{ dest_mode }}{{ altname }}{ + {%- if framed %} + syslog( + {%- else %} + network( + {%- endif %} + "{{ host }}" + log-fifo-size({{ log_fifo_size }}) + port({{ port }}) + transport("{{ transport }}") + persist-name("syslog{{ dest_mode }}{{ group }}") + + {%- if diskbuff_enable %} + disk-buffer( + compaction(yes) + {%- if diskbuff_reliable %} + mem-buf-size({{ mem_buf_size }}) + reliable(yes) + {%- else %} + mem-buf-length({{ mem_buf_length }}) + reliable(no) + {%- endif %} + disk-buf-size({{ disk_buf_size }}) + ) + {%- endif %} + {%- if tls %} + tls( + peer-verify({{ peer_verify }}) + {%- if cipher_suite %} + cipher-suite("{{ cipher_suite }}") + {%- endif %} + {%- if ssl_version %} + ssl-version("{{ ssl_version }}") + {%- endif %} + ) + {%- endif %} + {%- if not framed %} + flags(syslog-protocol) + {%- endif %} + + ); +}; diff --git a/package/enterprise/etc/conf.d/destinations/dest_syslog/plugin.py b/package/enterprise/etc/conf.d/destinations/dest_syslog/plugin.py new file mode 100755 index 0000000000..094543c3d2 --- /dev/null +++ b/package/enterprise/etc/conf.d/destinations/dest_syslog/plugin.py @@ -0,0 +1,104 @@ +#! /usr/bin/env python3 +import os +import shutil +import jinja2 +import re + +plugin_path = os.path.dirname(os.path.abspath(__file__)) + +templateLoader = jinja2.FileSystemLoader(searchpath=plugin_path) +templateEnv = jinja2.Environment( + loader=templateLoader, + autoescape=jinja2.select_autoescape(default_for_string=False), +) +tm = templateEnv.get_template("plugin.jinja") + +msg_template = "$(template t_syslog)" +dests = [] + +regex = r"^SC4S_DEST_SYSLOG_(.*)_HOST$" +for vn, vv in os.environ.items(): + m = re.search(regex, vn) + r = m.group(1) if m else "" + if r != "": + dests.append(r) + +for group in dests: + altname = f"_{ group }".lower() + + # print (mode) + if os.getenv(f"SC4S_DEST_SYSLOG_{ group }_DISKBUFF_ENABLE", "yes").lower() in [ + "true", + "1", + "t", + "y", + "yes", + ]: + diskbuff_enable = True + else: + diskbuff_enable = False + + if os.getenv(f"SC4S_DEST_SYSLOG_{ group }_DISKBUFF_RELIABLE", "no").lower() in [ + "true", + "1", + "t", + "y", + "yes", + ]: + diskbuff_reliable = True + else: + diskbuff_reliable = False + + # Used to calc disk space for buffer + disk_space, used, free = shutil.disk_usage(os.getenv("SC4S_VAR", "/")) + disk_space = disk_space - 5000000000 + + if disk_space < 0: + disk_space = 5000000000 + if os.getenv(f"SC4S_DEST_SYSLOG_{ group }_IETF", "yes") in [ + "true", + "1", + "t", + "y", + "yes", + ]: + port = os.getenv(f"SC4S_DEST_SYSLOG_{ group }_PORT", 601) + framed = True + else: + port = os.getenv(f"SC4S_DEST_SYSLOG_{ group }_PORT", 514) + framed = False + + transport = os.getenv(f"SC4S_DEST_SYSLOG_{ group }_TRANSPORT", "tcp") + + #### if TLS is used as a transport type + tls = True if transport in ["tls", "TLS"] else False + + msg = tm.render( + tls=tls, + group=group, + framed=framed, + altname=altname, + port=port, + transport=transport, + host=os.getenv(f"SC4S_DEST_SYSLOG_{ group }_HOST"), + log_fifo_size=os.getenv(f"SC4S_DEST_SYSLOG_{ group }_LOG_FIFO_SIZE", 180000000), + diskbuff_enable=diskbuff_enable, + diskbuff_reliable=diskbuff_reliable, + mem_buf_size=os.getenv( + f"SC4S_DEST_SYSLOG_{ group }_DISKBUFF_MEMBUFSIZE", + int(163840000), + ), + mem_buf_length=os.getenv( + f"SC4S_DEST_SYSLOG_{ group }_DISKBUFF_MEMBUFLENGTH", + int(60000), + ), + disk_buf_size=os.getenv( + f"SC4S_DEST_SYSLOG_{ group }_DISKBUFF_DISKBUFSIZE", + int(disk_space), + ), + peer_verify=os.getenv(f"SC4S_DEST_SYSLOG_{ group }_TLS_VERIFY", "yes"), + cipher_suite=os.getenv(f"SC4S_DEST_SYSLOG_{ group }_CIPHER_SUITE"), + ssl_version=os.getenv(f"SC4S_DEST_SYSLOG_{ group }_SSL_VERSION"), + ) + + print(msg) diff --git a/package/enterprise/etc/conf.d/destinations/dest_vpsc/plugin.conf b/package/enterprise/etc/conf.d/destinations/dest_vpsc/plugin.conf new file mode 100644 index 0000000000..8d44fdc368 --- /dev/null +++ b/package/enterprise/etc/conf.d/destinations/dest_vpsc/plugin.conf @@ -0,0 +1,7 @@ +destination d_vpsc { + python( + class("parser_vps_cache.vpsc_dest") + batch-timeout(3000) + batch-lines(100) + ); +}; \ No newline at end of file diff --git a/package/enterprise/etc/conf.d/destinations/rawmsg_file.conf b/package/enterprise/etc/conf.d/destinations/rawmsg_file.conf new file mode 100644 index 0000000000..b4e3e24446 --- /dev/null +++ b/package/enterprise/etc/conf.d/destinations/rawmsg_file.conf @@ -0,0 +1,11 @@ +destination d_rawmsg { + channel { + if ("${RAWMSG}" ne "") { + destination { + file("`SC4S_VAR`/archive/rawmsg/${.splunk.sourcetype}/${HOST}/$YEAR-$MONTH-$DAY-rawmsg.log" + template("${RAWMSG}\n") + ); + }; + }; + }; +}; \ No newline at end of file diff --git a/package/enterprise/etc/conf.d/destinations/splunk_hec_debug.conf b/package/enterprise/etc/conf.d/destinations/splunk_hec_debug.conf new file mode 100644 index 0000000000..8c52ab70d6 --- /dev/null +++ b/package/enterprise/etc/conf.d/destinations/splunk_hec_debug.conf @@ -0,0 +1,6 @@ +destination d_hec_debug { + file("`SC4S_VAR`/archive/debug/${.splunk.index}/${fields.sc4s_vendor}_${fields.sc4s_product}/$YEAR-$MONTH-$DAY-hec.log" + template("curl -k -u \"sc4s HEC debug:$$SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN\" \"`SC4S_DEST_SPLUNK_HEC_DEFAULT_URL`\" -d '$(template ${.splunk.sc4s_hec_template} $(template t_splunk_hec))'\n") + create_dirs(yes) + ); +}; diff --git a/package/enterprise/etc/conf.d/destinations/stdout.conf b/package/enterprise/etc/conf.d/destinations/stdout.conf new file mode 100644 index 0000000000..e2a0bbed08 --- /dev/null +++ b/package/enterprise/etc/conf.d/destinations/stdout.conf @@ -0,0 +1,3 @@ +destination d_stdout { + usertty("*"); +}; \ No newline at end of file diff --git a/package/enterprise/etc/conf.d/enrich/splunk_context.conf b/package/enterprise/etc/conf.d/enrich/splunk_context.conf new file mode 100644 index 0000000000..7771bb9d6c --- /dev/null +++ b/package/enterprise/etc/conf.d/enrich/splunk_context.conf @@ -0,0 +1,24 @@ +parser p_add_context_splunk{ + channel { + + parser { + add-contextual-data( + selector("${fields.sc4s_vendor}_${fields.sc4s_product}"), + database("conf.d/local/context/splunk_metadata.csv"), + prefix(".splunk."), + ignore-case(yes) + ); + } ; + if ("${fields.sc4s_class}" ne ""){ + parser { + add-contextual-data( + selector("${fields.sc4s_vendor}_${fields.sc4s_product}_${fields.sc4s_class}"), + database("conf.d/local/context/splunk_metadata.csv"), + prefix(".splunk."), + ignore-case(yes) + ); + } ; + }; + flags(final); + }; +}; diff --git a/package/enterprise/etc/conf.d/enrich/vendor_product_by_source.conf b/package/enterprise/etc/conf.d/enrich/vendor_product_by_source.conf new file mode 100644 index 0000000000..e188fd81d6 --- /dev/null +++ b/package/enterprise/etc/conf.d/enrich/vendor_product_by_source.conf @@ -0,0 +1,19 @@ +parser vendor_product_by_source { + channel { + if { + parser { + app-parser(topic(sc4s-vps)); + }; + } else { + parser { + add-contextual-data( + selector(filters("`syslog-ng-sysconfdir`/conf.d/local/context/vendor_product_by_source.conf")), + database("`syslog-ng-sysconfdir`/conf.d/local/context/vendor_product_by_source.csv") + ignore-case(yes) + prefix(".netsource.") + ); + }; + }; + }; +}; + diff --git a/package/enterprise/etc/conf.d/log_paths/0/lp_dest_filtered_alts_select/plugin.conf b/package/enterprise/etc/conf.d/log_paths/0/lp_dest_filtered_alts_select/plugin.conf new file mode 100644 index 0000000000..e50e75097d --- /dev/null +++ b/package/enterprise/etc/conf.d/log_paths/0/lp_dest_filtered_alts_select/plugin.conf @@ -0,0 +1,4 @@ + +@module confgen context(root) name(lp_dest_filtered_alts_select) exec("`SC4S_ETC`/conf.d/log_paths/0/lp_dest_filtered_alts_select/plugin.py") + +lp_dest_filtered_alts_select() diff --git a/package/enterprise/etc/conf.d/log_paths/0/lp_dest_filtered_alts_select/plugin.jinja b/package/enterprise/etc/conf.d/log_paths/0/lp_dest_filtered_alts_select/plugin.jinja new file mode 100644 index 0000000000..5f7a595ee3 --- /dev/null +++ b/package/enterprise/etc/conf.d/log_paths/0/lp_dest_filtered_alts_select/plugin.jinja @@ -0,0 +1,22 @@ + +# This supports https://splunk-connect-for-syslog.readthedocs.io/en/latest/configuration/#configuration-of-filtered-alternate-destinations-advanced +log{ + filter { + # Example given the following env vars + # SC4S_DEST_SPECTRACOM_XXX_ALT_FILTER="f_is_rfc3dfsfs" + # SC4S_DEST_SPECTRACOM_XXX_FILTERED_ALTERNATES="d_hec_debug" + + ('{{ dest_key }}' eq "${fields.sc4s_vendor}_${fields.sc4s_product}") or + ('{{ lower_dest_key }}' eq "${fields.sc4s_vendor}_${fields.sc4s_product}") + + }; + {% for f in filters %} + log { + filter({{ f }}); + {%- for destination in filters[f] %} + destination({{ destination }}); + {%- endfor %} + }; + {% endfor %} + flags(catchall,flow-control, final); +}; \ No newline at end of file diff --git a/package/enterprise/etc/conf.d/log_paths/0/lp_dest_filtered_alts_select/plugin.py b/package/enterprise/etc/conf.d/log_paths/0/lp_dest_filtered_alts_select/plugin.py new file mode 100755 index 0000000000..7ce41de60f --- /dev/null +++ b/package/enterprise/etc/conf.d/log_paths/0/lp_dest_filtered_alts_select/plugin.py @@ -0,0 +1,44 @@ +#! /usr/bin/env python3 +import os +import jinja2 +import pprint + +pp = pprint.PrettyPrinter(indent=4) + +plugin_path = os.path.dirname(os.path.abspath(__file__)) + +templateLoader = jinja2.FileSystemLoader(searchpath=plugin_path) +templateEnv = jinja2.Environment( + loader=templateLoader, + autoescape=jinja2.select_autoescape(default_for_string=False), +) +tm = templateEnv.get_template("plugin.jinja") + + +for var in os.environ: + routes = {} + filters = {} + if var.startswith("SC4S_DEST_") and var.endswith("_FILTERED_ALTERNATES"): + dest_key = var.replace("SC4S_DEST_", "").replace("_FILTERED_ALTERNATES", "") + lower_dest_key = dest_key.lower() + dest_key_dests = os.environ[var].split(",") + dest_filters = os.getenv( + f"SC4S_DEST_{ dest_key }_ALT_FILTER", "f_is_nevermatch" + ).split(",") + # create a list of all the dests + pairs = [] + filters = {} + for i in range(0, len(dest_key_dests)): + d = dest_key_dests[i] + # dest_key_dests[i] + if len(dest_filters) == 1: + f = dest_filters[0] + else: + f = dest_filters[i] + + if f in filters.keys(): + filters[f].append(d) + else: + filters[f] = [d] + msg = tm.render(dest_key=dest_key, filters=filters, lower_dest_key=lower_dest_key) + print(msg) \ No newline at end of file diff --git a/package/enterprise/etc/conf.d/log_paths/2/lp-dest-archive.conf b/package/enterprise/etc/conf.d/log_paths/2/lp-dest-archive.conf new file mode 100644 index 0000000000..952b47200c --- /dev/null +++ b/package/enterprise/etc/conf.d/log_paths/2/lp-dest-archive.conf @@ -0,0 +1,7 @@ +log { + parser { + app-parser(topic(sc4s-lp-archive)); + }; + destination(d_archive); + flags(catchall); +}; \ No newline at end of file diff --git a/package/enterprise/etc/conf.d/log_paths/2/lp-dest-hec_debug.conf b/package/enterprise/etc/conf.d/log_paths/2/lp-dest-hec_debug.conf new file mode 100644 index 0000000000..c0821d8f46 --- /dev/null +++ b/package/enterprise/etc/conf.d/log_paths/2/lp-dest-hec_debug.conf @@ -0,0 +1,7 @@ +log { + parser { + app-parser(topic(sc4s-hec-debug)); + }; + destination(d_hec_debug); + flags(catchall); +}; \ No newline at end of file diff --git a/package/enterprise/etc/conf.d/log_paths/2/lp-dest-psc.conf b/package/enterprise/etc/conf.d/log_paths/2/lp-dest-psc.conf new file mode 100644 index 0000000000..c05d22cc34 --- /dev/null +++ b/package/enterprise/etc/conf.d/log_paths/2/lp-dest-psc.conf @@ -0,0 +1,11 @@ +log { + if { + filter(f_host_is_nil_or_ip); + } elif { + filter { + "`SC4S_USE_NAME_CACHE`" eq "yes" + }; + destination(d_psc); + }; + flags(catchall); +}; \ No newline at end of file diff --git a/package/enterprise/etc/conf.d/log_paths/2/lp-dest-stdout.conf b/package/enterprise/etc/conf.d/log_paths/2/lp-dest-stdout.conf new file mode 100644 index 0000000000..aafc9145c7 --- /dev/null +++ b/package/enterprise/etc/conf.d/log_paths/2/lp-dest-stdout.conf @@ -0,0 +1,19 @@ +log { + source(s_internal); + + filter { + not filter(f_is_dest_splunk_null_queue) + and ("`SC4S_SEND_METRICS_TERMINAL`" eq "yes"); + }; + + destination(d_stdout); + +}; +log { + source(s_startup_out); + filter { + ("`SC4S_SEND_METRICS_TERMINAL`" eq "yes"); + }; + source(s_startup_err); + destination(d_stdout); +}; diff --git a/package/enterprise/etc/conf.d/log_paths/2/lp-dest-vpsc.conf b/package/enterprise/etc/conf.d/log_paths/2/lp-dest-vpsc.conf new file mode 100644 index 0000000000..b34ed1540f --- /dev/null +++ b/package/enterprise/etc/conf.d/log_paths/2/lp-dest-vpsc.conf @@ -0,0 +1,16 @@ +log { + if { + filter(f_host_is_nil_or_ip); + } elif { + filter{ + ("fields.sc4s_vendor" eq "nix" and "fields.sc4s_productr" eq "syslog" ) + or ("fields.sc4s_vendor" eq "splunk") + }; + } elif { + filter { + "`SC4S_USE_VPS_CACHE`" eq "yes" + }; + destination(d_vpsc); + }; + flags(catchall); +}; \ No newline at end of file diff --git a/package/enterprise/etc/conf.d/log_paths/2/lp_dest_alts_global/plugin.conf b/package/enterprise/etc/conf.d/log_paths/2/lp_dest_alts_global/plugin.conf new file mode 100644 index 0000000000..273dca0071 --- /dev/null +++ b/package/enterprise/etc/conf.d/log_paths/2/lp_dest_alts_global/plugin.conf @@ -0,0 +1,4 @@ + +@module confgen context(root) name(lp_dest_alts_global) exec("`SC4S_ETC`/conf.d/log_paths/2/lp_dest_alts_global/plugin.py") + +lp_dest_alts_global() \ No newline at end of file diff --git a/package/enterprise/etc/conf.d/log_paths/2/lp_dest_alts_global/plugin.jinja b/package/enterprise/etc/conf.d/log_paths/2/lp_dest_alts_global/plugin.jinja new file mode 100644 index 0000000000..fb4d0956e5 --- /dev/null +++ b/package/enterprise/etc/conf.d/log_paths/2/lp_dest_alts_global/plugin.jinja @@ -0,0 +1,62 @@ + +log{ + {%- if mode == "SELECT" %} + parser { + app-parser(topic(sc4s-lp-dest-select-{{ destination }})); + }; + {%- endif %} + {%- if mode == "GLOBAL" %} + if { + parser { + app-parser(topic(sc4s-lp-dest-select-{{ destination }})); + }; + } else {}; + {%- endif %} + {%- if filter !="" %} + filter { + {{ filter }} + }; + {%- endif %} + #These parsers can be used to make destination type or destination specific + #changes to the format of the event for example moving the kv pairs from a parsed + #MESSAGE to the sdata fields + if { + parser { + app-parser(topic(sc4s-lp-dest-select-drop-{{ destination }})); + }; + } else { + if { + parser { + app-parser(topic(sc4s-lp-dest-format-{{ destination }})); + }; + } elif { + parser { + app-parser(topic(sc4s-lp-dest-format-{{ dtype }})); + }; + } else {}; + + {%- if dtype == "hec_fmt" %} + rewrite(r_set_splunk_default); + if { + parser(p_add_context_splunk); + }; + if { + parser (compliance_meta_by_source); + }; + if { + parser (app-plugin-source-postprocess); + }; + if { + parser (app-plugin-source-finalprocess); + }; + if { + filter(f_is_dest_splunk_null_queue); + } else { + destination({{ destination }}); + }; + {%- else %} + destination({{ destination }}); + {%- endif %} + }; + flags(catchall); +}; diff --git a/package/enterprise/etc/conf.d/log_paths/2/lp_dest_alts_global/plugin.py b/package/enterprise/etc/conf.d/log_paths/2/lp_dest_alts_global/plugin.py new file mode 100755 index 0000000000..13bb14603d --- /dev/null +++ b/package/enterprise/etc/conf.d/log_paths/2/lp_dest_alts_global/plugin.py @@ -0,0 +1,91 @@ +#! /usr/bin/env python3 +import os +import jinja2 +import re + +plugin_path = os.path.dirname(os.path.abspath(__file__)) +templateLoader = jinja2.FileSystemLoader(searchpath=plugin_path) +templateEnv = jinja2.Environment( + loader=templateLoader, + autoescape=jinja2.select_autoescape(default_for_string=False), +) +tm = templateEnv.get_template("plugin.jinja") + +def normalize_env_variable_input(env_variable: str): + if os.getenv(env_variable, "no").lower() in [ + "true", + "1", + "t", + "y", + "yes", + ]: + normalized_value = True + else: + normalized_value = False + return normalized_value + + +regex_splunkhec = r"^SC4S_DEST_SPLUNK_HEC_(.*)_URL$" +regex_syslog = r"^SC4S_DEST_(SYSLOG|BSD)_(.*)_HOST$" +global_dests = {} +for vn, vv in os.environ.items(): + m = re.search(regex_splunkhec, vn) + r = m.group(1) if m else "" + # if r != "" and vv == "" and r not in ('DEFAULT','INTERNAL') and vv in ('GLOBAL', 'SELECT'): + if r != "": + modev = os.environ.get(f"SC4S_DEST_SPLUNK_HEC_{r}_MODE", "GLOBAL") + if ( + r == "DEFAULT" + and not os.environ.get("SC4S_DEST_SPLUNK_HEC_GLOBAL", "") == "" + ): + if os.environ.get("SC4S_DEST_SPLUNK_HEC_GLOBAL", "yes").lower() in [ + "true", + "1", + "t", + "y", + "yes", + ]: + modev = "GLOBAL" + else: + modev = "SELECT" + elif modev in ("GLOBAL", "SELECT"): + suffix = "" + if r != "DEFAULT": + suffix = f"_{r}" + + global_dests[r] = { + "destination": f"d_hec_fmt{suffix.lower()}", + "dtype": "hec_fmt", + "mode": modev, + "filter": "", + } + +for vn, vv in os.environ.items(): + m = re.search(regex_syslog, vn) + t = m.group(1) if m else "" + r = m.group(2) if m else "" + # if r != "" and vv == "" and r not in ('DEFAULT','INTERNAL') and vv in ('GLOBAL', 'SELECT'): + if r != "": + modev = os.environ.get(f"SC4S_DEST_{t}_{r}_MODE", "GLOBAL") + filter = os.environ.get(f"SC4S_DEST_{t}_{r}_FILTER", "") + if filter == "": + if t == "BSD": + filter = '"${MSG}" ne ""' + if modev.upper() in ("GLOBAL", "SELECT"): + global_dests[r] = { + "destination": f"d_{t.lower()}_{r.lower()}", + "dtype": t.lower(), + "mode": modev, + "filter": filter, + } + + +for d, m in global_dests.items(): + msg = tm.render( + id=d, + destination=m["destination"], + mode=m["mode"], + filter=m["filter"], + dtype=m["dtype"], + ) + print(msg) diff --git a/package/enterprise/etc/conf.d/plugin/app_parser_topics.conf b/package/enterprise/etc/conf.d/plugin/app_parser_topics.conf new file mode 100644 index 0000000000..3a72a8358e --- /dev/null +++ b/package/enterprise/etc/conf.d/plugin/app_parser_topics.conf @@ -0,0 +1,37 @@ + +parser app-group-sc4s-almost-syslog{ + app-parser(topic(sc4s-almost-syslog)); +}; + +parser app-group-sc4s-syslog-sdata{ + app-parser(topic(sc4s-syslog-sdata)); +}; +parser app-group-sc4s-syslog-pgm{ + app-parser(topic(sc4s-syslog-pgm)); +}; +parser app-group-sc4s-syslog{ + app-parser(topic(sc4s-syslog)); +}; + +parser app-group-sc4s-syslog-netsource{ + app-parser(topic(sc4s-network-source)); +}; + +parser app-group-sc4s-raw{ + app-parser(topic(sc4s-raw-syslog)); +}; +parser app-group-sc4s-fallback{ + app-parser(topic(fallback)); +}; + +parser app-plugin-syslog-fix-program{ + app-parser(topic(fix-invalid-program)); +}; + +parser app-plugin-source-postprocess{ + app-parser(topic(sc4s-postfilter)); +}; + +parser app-plugin-source-finalprocess{ + app-parser(topic(sc4s-finalfilter)); +}; \ No newline at end of file diff --git a/package/enterprise/etc/conf.d/sc4slib/app-lp-global-archive/plugin.conf b/package/enterprise/etc/conf.d/sc4slib/app-lp-global-archive/plugin.conf new file mode 100644 index 0000000000..20a7c2f9aa --- /dev/null +++ b/package/enterprise/etc/conf.d/sc4slib/app-lp-global-archive/plugin.conf @@ -0,0 +1,4 @@ + +@module confgen context(root) name(app-global-archive) exec("`SC4S_ETC`/conf.d/sc4slib/app-lp-global-archive/plugin.py") + +app-global-archive() \ No newline at end of file diff --git a/package/enterprise/etc/conf.d/sc4slib/app-lp-global-archive/plugin.jinja b/package/enterprise/etc/conf.d/sc4slib/app-lp-global-archive/plugin.jinja new file mode 100644 index 0000000000..1c602d6c1f --- /dev/null +++ b/package/enterprise/etc/conf.d/sc4slib/app-lp-global-archive/plugin.jinja @@ -0,0 +1,5 @@ +application app-lp-global_archive_{{ key }}[sc4s-lp-archive] { + filter { + match('{{ key }}' template('${fields.sc4s_vendor}_${fields.sc4s_product}') type(string) flags(ignore-case)) + }; +}; \ No newline at end of file diff --git a/package/enterprise/etc/conf.d/sc4slib/app-lp-global-archive/plugin.py b/package/enterprise/etc/conf.d/sc4slib/app-lp-global-archive/plugin.py new file mode 100755 index 0000000000..c8a73f2f35 --- /dev/null +++ b/package/enterprise/etc/conf.d/sc4slib/app-lp-global-archive/plugin.py @@ -0,0 +1,34 @@ +#! /usr/bin/env python3 + +import os +import shutil +import jinja2 +import re + +plugin_path = os.path.dirname(os.path.abspath(__file__)) + +templateLoader = jinja2.FileSystemLoader(searchpath=plugin_path) +templateEnv = jinja2.Environment( + loader=templateLoader, + autoescape=jinja2.select_autoescape(default_for_string=False), +) +tm = templateEnv.get_template("plugin.jinja") + +keys = [] +regex = r"^SC4S_DEST_(.*)_ARCHIVE$" +for vn, vv in os.environ.items(): + m = re.search(regex, vn) + r = m.group(1) if m else "" + if r != "" and vv.lower() in [ + "true", + "1", + "t", + "y", + "yes", + ]: + + msg = tm.render( + key=r, + ) + + print(msg) diff --git a/package/enterprise/etc/conf.d/sc4slib/app-lp-global-default_hec/plugin.conf b/package/enterprise/etc/conf.d/sc4slib/app-lp-global-default_hec/plugin.conf new file mode 100644 index 0000000000..2cf567e3c7 --- /dev/null +++ b/package/enterprise/etc/conf.d/sc4slib/app-lp-global-default_hec/plugin.conf @@ -0,0 +1,4 @@ + +@module confgen context(root) name(app-global-hec) exec("`SC4S_ETC`/conf.d/sc4slib/app-lp-global-default_hec/plugin.py") + +app-global-hec() \ No newline at end of file diff --git a/package/enterprise/etc/conf.d/sc4slib/app-lp-global-default_hec/plugin.jinja b/package/enterprise/etc/conf.d/sc4slib/app-lp-global-default_hec/plugin.jinja new file mode 100644 index 0000000000..3e02b5320e --- /dev/null +++ b/package/enterprise/etc/conf.d/sc4slib/app-lp-global-default_hec/plugin.jinja @@ -0,0 +1,5 @@ +application app-lp-global_default_hec_{{ key }}[sc4s-lp-default-hec] { + filter { + '{{ key }}' eq "${fields.sc4s_vendor}_${fields.sc4s_product}" + }; +}; \ No newline at end of file diff --git a/package/enterprise/etc/conf.d/sc4slib/app-lp-global-default_hec/plugin.py b/package/enterprise/etc/conf.d/sc4slib/app-lp-global-default_hec/plugin.py new file mode 100755 index 0000000000..34147ef948 --- /dev/null +++ b/package/enterprise/etc/conf.d/sc4slib/app-lp-global-default_hec/plugin.py @@ -0,0 +1,34 @@ +#! /usr/bin/env python3 + +import os +import shutil +import jinja2 +import re + +plugin_path = os.path.dirname(os.path.abspath(__file__)) + +templateLoader = jinja2.FileSystemLoader(searchpath=plugin_path) +templateEnv = jinja2.Environment( + loader=templateLoader, + autoescape=jinja2.select_autoescape(default_for_string=False), +) +tm = templateEnv.get_template("plugin.jinja") + +keys = [] +regex = r"^SC4S_DEST_(.*)_HEC$" +for vn, vv in os.environ.items(): + m = re.search(regex, vn) + r = m.group(1) if m else "" + if r != "" and vv.lower() in [ + "true", + "1", + "t", + "y", + "yes", + ]: + + msg = tm.render( + key=r, + ) + + print(msg) diff --git a/package/enterprise/etc/conf.d/sc4slib/global_options/plugin.jinja b/package/enterprise/etc/conf.d/sc4slib/global_options/plugin.jinja new file mode 100644 index 0000000000..fcf9bc771c --- /dev/null +++ b/package/enterprise/etc/conf.d/sc4slib/global_options/plugin.jinja @@ -0,0 +1,25 @@ +options { + log_msg_size (65536); + flush_lines (100); + time_reopen (10); + log_fifo_size ({{ log_fifo }}); + chain_hostnames (yes); + use_dns (no); + use_fqdn (no); + dns-cache(no); + create_dirs (no); + keep-hostname (yes); + create_dirs(yes); + dir_perm(0750); + stats ( + freq ({{ stats_freq }}) + level ({{ stats_level }}) + max-dynamics(2000) + ); + normalize-hostnames(yes); + on-error(fallback-to-string); + frac-digits(3); + recv-time-zone("`SC4S_DEFAULT_TIMEZONE`"); + send-time-zone("GMT"); + +}; diff --git a/package/enterprise/etc/conf.d/sc4slib/global_options/plugin.py b/package/enterprise/etc/conf.d/sc4slib/global_options/plugin.py new file mode 100755 index 0000000000..9fbeeed697 --- /dev/null +++ b/package/enterprise/etc/conf.d/sc4slib/global_options/plugin.py @@ -0,0 +1,22 @@ +#! /usr/bin/env python3 + +import os +import shutil +import jinja2 + +plugin_path = os.path.dirname(os.path.abspath(__file__)) + +templateLoader = jinja2.FileSystemLoader(searchpath=plugin_path) +templateEnv = jinja2.Environment( + loader=templateLoader, + autoescape=jinja2.select_autoescape(default_for_string=False), +) +tm = templateEnv.get_template("plugin.jinja") + +msg = tm.render( + stats_freq=os.getenv("SC4S_GLOBAL_OPTIONS_STATS_FREQ", 30), + stats_level=os.getenv("SC4S_GLOBAL_OPTIONS_STATS_LEVEL", 1), + log_fifo=os.getenv("SC4S_GLOBAL_OPTIONS_LOG_FIFO", 10000), +) + +print(msg) diff --git a/package/enterprise/etc/conf.d/sources/internal.conf b/package/enterprise/etc/conf.d/sources/internal.conf new file mode 100644 index 0000000000..af316aab30 --- /dev/null +++ b/package/enterprise/etc/conf.d/sources/internal.conf @@ -0,0 +1,172 @@ +# =============================================================================================== +# Internal sources +# =============================================================================================== + + +source s_internal { + channel { + source { + internal(); + }; + + rewrite { + subst('^.*@','' value('HOST')); + set("`SC4S_CONTAINER_HOST`", value("fields.sc4s_loghost") condition('`SC4S_CONTAINER_HOST`' ne "")); + set("`SC4S_VERSION`", value("fields.sc4s_version") condition('`SC4S_VERSION`' ne "")); + set($LOGHOST, value("fields.sc4s_container") condition(match('container' template('`SC4S_DEST_SPLUNK_INDEXED_FIELDS`') type(string) flags(substring)) )); + }; + + if (message("Log statistics; " type('string') flags(prefix))) { + + rewrite { + subst('.*Log statistics; ', '', value("MESSAGE"), flags("utf8" "global")); + subst('([^= ]+=\x27[^\(]+\(#anon[^,\)]+(?:,[^,]+,[^\)]+)?\)\=\d+\x27(?:, )?)', '', value("MESSAGE"), flags("utf8" "global")); + #Note special format for readability new line between metrics + subst('\d\d\d\d-\d\d-\d\d-', '', value("MESSAGE"), flags("utf8" "global")); + + }; + if { + filter { + "`SC4S_DEST_SPLUNK_SC4S_METRICS_HEC`" eq "event" + }; + rewrite { + r_set_splunk_dest_default( + index('main') + sourcetype("sc4s:metrics") + vendor('splunk') + product('sc4s') + class("metrics") + ); + }; + rewrite { + r_set_splunk_dest_update( + template_hec("t_splunk_hec_metric_event") + ); + }; + } elif { + filter { + "`SC4S_DEST_SPLUNK_SC4S_METRICS_HEC`" eq "single" or "`SC4S_DEST_SPLUNK_SC4S_METRICS_HEC`" eq "yes" + }; + rewrite { + r_set_splunk_dest_default( + index('_metrics') + sourcetype("sc4s:metrics") + vendor('splunk') + product('sc4s') + class("metrics") + ); + }; + rewrite { + r_set_splunk_dest_update( + template_hec("t_splunk_hec_metric_single") + ); + }; + } elif { + filter { + "`SC4S_DEST_SPLUNK_SC4S_METRICS_HEC`" eq "multi" + }; + rewrite { + r_set_splunk_dest_default( + index('_metrics') + sourcetype("sc4s:metrics") + vendor('splunk') + product('sc4s') + class("metrics") + ); + }; + rewrite { + r_set_splunk_dest_update( + template_hec("t_splunk_hec_metric_multi") + ); + }; + } else { + rewrite { + r_set_splunk_dest_default( + index('_metrics') + sourcetype("sc4s:metrics:v2") + vendor('splunk') + product('sc4s') + class("metrics") + ); + }; + rewrite { + r_set_splunk_dest_update( + template_hec("t_splunk_hec_metric_multi_v2") + ); + }; + }; + + } else { + rewrite { + r_set_splunk_dest_default( + index('main') + source("sc4s") + sourcetype("sc4s:events") + vendor('splunk') + product('sc4s') + class("events") + template('t_5424_hdr_sdata_compact') + ) + }; + if { + filter { + match("Input is valid utf8, but the log message is not tagged as such," value("MESSAGE")) + or match("Syslog connection closed; fd=" value("MESSAGE")) + or match("Syslog connection accepted; fd=" value("MESSAGE")) + or match("xml-parser failed; " value("MESSAGE")) + or match("Initializing PROXY protocol source driver" value("MESSAGE")) + }; + rewrite(r_set_dest_splunk_null_queue); + }; + + }; + }; +}; + + +source s_startup_out { + channel { + source { + file("`SC4S_VAR`/log/syslog-ng.out" + program-override("syslog-ng-config") + flags(no-hostname,no-parse,assume-utf8) + ); + }; + rewrite { + set("`SC4S_CONTAINER_HOST`", value("fields.sc4s_loghost") condition('`SC4S_CONTAINER_HOST`' ne "")); + set($LOGHOST, value("fields.sc4s_container") condition(match('container' template('`SC4S_DEST_SPLUNK_INDEXED_FIELDS`') type(string) flags(substring)) )); + + r_set_splunk_dest_default( + source("sc4s") + sourcetype("sc4s:events:startup:out") + vendor('splunk') + product('sc4s') + class("events") + ) + + }; + }; +}; + +source s_startup_err { + channel { + source { + file("`SC4S_VAR`/log/syslog-ng.err" + program-override("syslog-ng-config") + flags(no-hostname,no-parse,assume-utf8) + ); + }; + rewrite { + set("`SC4S_CONTAINER_HOST`", value("fields.sc4s_loghost") condition('`SC4S_CONTAINER_HOST`' ne "")); + set($LOGHOST, value("fields.sc4s_container") condition(match('container' template('`SC4S_DEST_SPLUNK_INDEXED_FIELDS`') type(string) flags(substring)) )); + + r_set_splunk_dest_default( + source("sc4s") + sourcetype("sc4s:events:startup:err") + vendor('splunk') + product('sc4s') + class("events") + ) + }; + }; +}; \ No newline at end of file diff --git a/package/enterprise/etc/conf.d/sources/source_syslog/plugin.conf b/package/enterprise/etc/conf.d/sources/source_syslog/plugin.conf new file mode 100644 index 0000000000..5a6a9910f0 --- /dev/null +++ b/package/enterprise/etc/conf.d/sources/source_syslog/plugin.conf @@ -0,0 +1,4 @@ + +@module confgen context(root) name(source_syslog) exec("`SC4S_ETC`/conf.d/sources/source_syslog/plugin.py") + +source_syslog() \ No newline at end of file diff --git a/package/enterprise/etc/conf.d/sources/source_syslog/plugin.jinja b/package/enterprise/etc/conf.d/sources/source_syslog/plugin.jinja new file mode 100644 index 0000000000..b382d11856 --- /dev/null +++ b/package/enterprise/etc/conf.d/sources/source_syslog/plugin.jinja @@ -0,0 +1,509 @@ + +source s_{{ port_id }} { + {%- if port_udp[0] != "disabled" or port_tcp[0] != "disabled" or port_tls[0] != "disabled" %} + channel { + source { + # List UDP for any format + {%- for port in port_udp %} + {%- if port != "disabled" %} + {%- for i in range(1, port_udp_sockets+1) %} + network ( + transport("udp") + so-reuseport(1) + persist-name("{{ port_id }}_{{ port }}_{{ i }}") + {%- if enable_ebpf == True %} + ebpf(reuseport(sockets({{ ebpf_no_sockets }}))) + {%- endif %} + port({{ port }}) + ip-protocol({{ enable_ipv6 }}) + {%- if port_udp_sorecvbuff >0 %} + so-rcvbuf({{ port_udp_sorecvbuff }}) + {%- endif %} + {%- if use_udp_log_iw == True %} + log-iw-size({{ port_udp_log_iw_size }}) + log-fetch-limit({{ port_udp_log_fetch_limit }}) + {%- endif %} + keep-hostname(yes) + keep-timestamp(yes) + use-dns(no) + use-fqdn(no) + chain-hostnames(off) + flags(validate-utf8, no-parse {%- if store_raw_message %} store-raw-message{% endif %}) + ); + {%- endfor %} + {%- endif %} + {%- endfor %} + + # List TCP for any format + {%- for port in port_tcp %} + {%- if port != "disabled" %} + {%- for i in range(1,port_tcp_sockets+1) %} + network ( + {%- if use_proxy_connect == True %} + transport("proxied-tcp") + {%- else %} + transport("tcp") + {%- endif %} + so-reuseport(1) + port({{ port }}) + persist-name("{{ port_id }}_tcp_{{ port }}_{{ i }}") + ip-protocol({{ enable_ipv6 }}) + max-connections({{ port_tcp_max_connections }}) + log-iw-size({{ port_tcp_log_iw_size }}) + log-fetch-limit({{ port_tcp_log_fetch_limit }}) + {%- if port_tcp_so_recvbuff >0 %} + so-rcvbuf({{ port_tcp_so_recvbuff }}) + {%- endif %} + keep-timestamp(yes) + use-dns(no) + use-fqdn(no) + chain-hostnames(off) + flags(validate-utf8, no-parse {%- if store_raw_message %} store-raw-message{% endif %}) + ); + {%- endfor %} + {%- endif %} + {%- endfor %} + + # List TLS for any format + {%- for port in port_tls %} + {%- if port != "disabled" and use_tls %} + {%- for i in range(1,port_tls_sockets+1) %} + network ( + {%- if use_proxy_connect == True %} + transport("proxied-tls") + {%- else %} + transport("tls") + {%- endif %} + so-reuseport(1) + port({{ port }}) + persist-name("{{ port_id }}_tls_{{ port }}_{{ i }}") + ip-protocol({{ enable_ipv6 }}) + max-connections({{ port_tcp_max_connections }}) + log-iw-size({{ port_tcp_log_iw_size }}) + log-fetch-limit({{ port_tcp_log_fetch_limit }}) + {%- if port_tcp_so_recvbuff >0 %} + so-rcvbuf({{ port_tcp_so_recvbuff }}) + {%- endif %} + + keep-timestamp(yes) + use-dns(no) + use-fqdn(no) + chain-hostnames(off) + flags(validate-utf8, no-parse {%- if store_raw_message %} store-raw-message{% endif %}) + tls( + allow-compress(yes) + key-file("{{ tls_dir }}/{{ key_file }}") + cert-file("{{ tls_dir }}/{{ cert_file }}") + ssl-options({{ port_tls_tls_options }}) + cipher-suite("{{ port_tls_cipher_suit }}") + peer-verify(no) + ) + ); + {%- endfor %} + {%- endif %} + {%- endfor %} + }; + {%- if enable_parallelize == True %} + parallelize(partitions({{ parallelize_no_partitions }})); + {%- endif %} + {%- if vendor and product %} + parser { + p_set_netsource_fields( + vendor('{{ vendor }}') + product('{{ product }}') + ); + }; + {%- endif %} + + {%- if use_proxy_connect == True %} + rewrite { + set("$PROXIED_SRCIP", value("HOST") condition(filter(f_is_proxy_ip)) ); + }; + {%- endif %} + + if { + if { + parser { + syslog-parser( + drop-invalid(yes) + flags(syslog-protocol, assume-utf8, no-rfc3164-fallback) + ); + }; + rewrite(set_rfc); + rewrite(set_rfc5424); + } elif { + #Because 3164 is not a standard we can't test for conformance. We need to first see if we + #Have anything that would be better this is not free the more we add here over time the worse perf will get + #Sadly there is little choice in the matter so pay attention to the regex and hand optimize + if { + filter{ + message('^\<\d+\>') + }; + if { + parser(app-group-sc4s-almost-syslog); + } elif { + #3164 has PRI + parser { + syslog-parser( + drop-invalid(yes) + flags(assume-utf8, guess-timezone)); + }; + rewrite(set_rfc); + rewrite(set_rfc3164); + }; + if { + filter{ + host('^slot\d+\/') + }; + if { + parser { + regexp-parser( + prefix(".tmp.") + template("$HOST") + patterns('^(?.*)\/(?.*)') + ); + }; + rewrite { + set('${.tmp.slot}' value('.metadata.header.slot')); + set('${.tmp.host}' value('HOST')); + }; + }; + }; + if { + filter(f_is_rfc3164); + filter{ + not host('^[a-zA-Z0-9_\-\.]+$') + }; + rewrite { + set("${HOST} $(template t_hdr_msg)" value("MSG")); + unset(value("LEGACY_MSGHDR")); + unset(value("PID")); + unset(value("PROGRAM")); + set('$SOURCEIP' value('HOST')); + }; + }; + if { + # If program is probably not valid cleanup MESSAGE so log paths don't have too + # This isn't great for performance but is reliable good reason to use 5424 + filter(f_is_rfc3164); + parser(app-plugin-syslog-fix-program); + rewrite { + set("$(template t_hdr_msg)" value("MSG")); + unset(value("LEGACY_MSGHDR")); + unset(value("PID")); + unset(value("PROGRAM")); + }; + }; + } elif { + #Try raw parsers remind customers these vendors need to improve + parser(app-group-sc4s-raw); + rewrite{ + groupunset(values('.tmp.*')); + }; + } else {}; + }; + + if { + filter(f_host_is_nil_or_ip); + if { + parser(p_add_context_host); + }; + {%- if use_namecache == True %} + if { + filter(f_host_is_nil_or_ip); + parser(p_host_cache); + }; + {%- endif %} + {%- if use_reverse_dns == True %} + if { + filter(f_host_is_nil_or_ip); + {%- if reverse_dns_keep_fqdn == True %} + parser(p_fix_fqdn_resolver); + {%- else %} + parser(p_fix_hostname_resolver); + {%- endif %} + }; + {%- endif %} + }; + rewrite { + set('$(lowercase "$HOST")' value(HOST)); + }; + {%- if not vendor or not product %} + {%- if use_vpscache == True %} + if { + parser(p_vpst_cache); + }; + {%- endif %} + if { + parser(vendor_product_by_source); + }; + {%- endif %} + if { + filter { + not filter(f_is_source_identified); + }; + if { + filter { + match('\[[^\@]+@\d+' value("SDATA")) + ; + }; + parser(app-group-sc4s-syslog-sdata); + rewrite(r_set_source_identified); + } elif { + filter { + "$PROGRAM" ne ""; + }; + parser(app-group-sc4s-syslog-pgm); + rewrite(r_set_source_identified); + } elif { + parser(app-group-sc4s-syslog); + rewrite(r_set_source_identified); + }; + rewrite{ + groupunset(values('.tmp.*')); + }; + }; + + if { + filter(f_is_rfc); + parser(app-group-sc4s-syslog-netsource); + rewrite(r_set_source_identified); + rewrite{ + groupunset(values('.tmp.*')); + }; + }; + + }; + rewrite { + groupunset(values('.tmp.*')); + }; + + if { + filter { + not filter(f_is_source_identified); + }; + parser(app-group-sc4s-fallback); + }; + + if { + filter { + match("." value(".netsource.sc4s_time_zone") ) + }; + rewrite { + fix-time-zone("${.netsource.sc4s_time_zone}"); + unset(value(".netsource.sc4s_time_zone")); + }; + }; + + {%- if set_source_sc4s == True %} + rewrite { + set("sc4s", value(".splunk.source")); + }; + {%- endif %} + + rewrite { + set($FACILITY, value("fields.sc4s_syslog_facility") condition(match('facility' template('`SC4S_DEST_SPLUNK_INDEXED_FIELDS`') type(string) flags(substring)))); + set($LEVEL, value("fields.sc4s_syslog_severity") condition(match('severity' template('`SC4S_DEST_SPLUNK_INDEXED_FIELDS`') type(string) flags(substring)) )); + }; + }; + {%- endif %} + + {%- if port_5426[0] != "disabled" or port_5425[0] != "disabled" or port_6587[0] != "disabled" %} + channel { + source { + # List UDP for RFC5426 + {%- for port in port_5426 %} + {%- if port != "disabled" %} + {%- for i in range(1,port_5426_sockets+1) %} + syslog ( + transport("udp") + so-reuseport(1) + persist-name("{{ port_id }}_5426_{{ port }}_{{ i }}") + port({{ port }}) + ip-protocol({{ enable_ipv6 }}) + {%- if port_5426_sorecvbuff >0 %} + so-rcvbuf({{ port_5426_sorecvbuff }}) + {%- endif %} + keep-hostname(yes) + keep-timestamp(yes) + use-dns(no) + use-fqdn(no) + chain-hostnames(off) + flags(validate-utf8) + ); + {%- endfor %} + {%- endif %} + {%- endfor %} + + # List 6587 for any format + {%- for port in port_6587 %} + {%- if port != "disabled" %} + {%- for i in range(1,port_6587_sockets+1) %} + syslog ( + {%- if use_proxy_connect == True %} + transport("proxied-tcp") + {%- else %} + transport("tcp") + {%- endif %} + so-reuseport(1) + port({{ port }}) + persist-name("{{ port_id }}_6587_{{ port }}_{{ i }}") + ip-protocol({{ enable_ipv6 }}) + max-connections({{ port_6587_max_connections }}) + log-iw-size({{ port_6587_log_iw_size }}) + log-fetch-limit({{ port_6587_log_fetch_limit }}) + {%- if port_6587_so_recvbuff >0 %} + so-rcvbuf({{ port_6587_so_recvbuff }}) + {%- endif %} + keep-timestamp(yes) + use-dns(no) + use-fqdn(no) + chain-hostnames(off) + flags(validate-utf8) + ); + {%- endfor %} + {%- endif %} + {%- endfor %} + + # List TLS for any format + {%- for port in port_5425 %} + {%- if port != "disabled" and use_tls %} + {%- for i in range(1,port_5425_sockets+1) %} + syslog ( + {%- if use_proxy_connect == True %} + transport("proxied-tls") + {%- else %} + transport("tls") + {%- endif %} + so-reuseport(1) + port({{ port }}) + persist-name("{{ port_id }}_5425_{{ port }}_{{ i }}") + ip-protocol({{ enable_ipv6 }}) + max-connections({{ port_5425_max_connections }}) + log-iw-size({{ port_5425_log_iw_size }}) + log-fetch-limit({{ port_5425_log_fetch_limit }}) + {%- if port_5425_so_recvbuff >0 %} + so-rcvbuf({{ port_5425_so_recvbuff }}) + {%- endif %} + keep-timestamp(yes) + use-dns(no) + use-fqdn(no) + chain-hostnames(off) + flags(validate-utf8) + tls( + allow-compress(yes) + key-file("{{ tls_dir }}/{{ key_file }}") + cert-file("{{ tls_dir }}/{{ cert_file }}") + ssl-options({{ port_5425_tls_options }}) + cipher-suite("{{ port_5425_cipher_suit }}") + peer-verify(no) + ) + ); + {%- endfor %} + {%- endif %} + {%- endfor %} + }; + + {%- if use_proxy_connect == True %} + rewrite { + set("$PROXIED_SRCIP", value("HOST") condition(filter(f_is_proxy_ip)) ); + }; + {%- endif %} + + {%- if vendor and product %} + parser { + p_set_netsource_fields( + vendor('{{ vendor }}') + product('product') + ); + }; + {%- endif %} + rewrite(set_rfc); + rewrite(set_rfc5424); + + if { + filter(f_host_is_nil_or_ip); + if { + parser(p_add_context_host); + }; + {%- if use_namecache == True %} + if { + filter(f_host_is_nil_or_ip); + parser(p_host_cache); + }; + {%- endif %} + {%- if use_reverse_dns == True %} + if { + filter(f_host_is_nil_or_ip); + {%- if reverse_dns_keep_fqdn == True %} + parser(p_fix_fqdn_resolver); + {%- else %} + parser(p_fix_hostname_resolver); + {%- endif %} + }; + {%- endif %} + }; + + rewrite { + set('$(lowercase "$HOST")' value(HOST)); + }; + {%- if not vendor or not product %} + {%- if use_vpscache == True %} + if { + parser(p_vpst_cache); + }; + {%- endif %} + if { + parser(vendor_product_by_source); + }; + {%- endif %} + + if { + filter { + match('\[[^\@]+@\d+' value("SDATA")) + ; + }; + parser(app-group-sc4s-syslog-sdata); + rewrite(r_set_source_identified); + } elif { + filter { + "$PROGRAM" ne ""; + }; + parser(app-group-sc4s-syslog-pgm); + rewrite(r_set_source_identified); + } elif { + parser(app-group-sc4s-syslog); + rewrite(r_set_source_identified); + }; + rewrite{ + groupunset(values('.tmp.*')); + }; + + + if { + parser(app-group-sc4s-syslog-netsource); + rewrite(r_set_source_identified); + }; + rewrite{ + groupunset(values('.tmp.*')); + }; + + if { + filter(f_is_source_identified); + } else { + parser(app-group-sc4s-fallback); + }; + + {%- if set_source_sc4s == True %} + rewrite { + set("sc4s", value(".splunk.source")); + }; + {%- endif %} + + rewrite { + set($FACILITY, value("fields.sc4s_syslog_facility") condition(match('facility' template('`SC4S_DEST_SPLUNK_INDEXED_FIELDS`') type(string) flags(substring)))); + set($LEVEL, value("fields.sc4s_syslog_severity") condition(match('severity' template('`SC4S_DEST_SPLUNK_INDEXED_FIELDS`') type(string) flags(substring)) )); + }; + }; + {%- endif %} +}; \ No newline at end of file diff --git a/package/enterprise/etc/conf.d/sources/source_syslog/plugin.py b/package/enterprise/etc/conf.d/sources/source_syslog/plugin.py new file mode 100755 index 0000000000..fdf790b0df --- /dev/null +++ b/package/enterprise/etc/conf.d/sources/source_syslog/plugin.py @@ -0,0 +1,138 @@ +#! /usr/bin/env python3 +import os +import jinja2 + +plugin_path = os.path.dirname(os.path.abspath(__file__)) + +templateLoader = jinja2.FileSystemLoader(searchpath=plugin_path) +templateEnv = jinja2.Environment(loader=templateLoader, autoescape=jinja2.select_autoescape(default_for_string=False)) +tm = templateEnv.get_template("plugin.jinja") + +ports = os.getenv("SOURCE_ALL_SET") + + +def normalize_env_variable_input(env_variable: str): + if os.getenv(env_variable, "no").lower() in [ + "true", + "1", + "t", + "y", + "yes", + ]: + normalized_value = True + else: + normalized_value = False + return normalized_value + + +if os.getenv("SC4S_IPV6_ENABLE", "no").lower() in [ + "true", + "1", + "t", + "y", + "yes", +]: + enable_ipv6 = "6" +else: + enable_ipv6 = "4" + + +if os.getenv("SC4S_RUNTIME_ENV", "unknown").lower() == "k8s": + cert_file = "tls.crt" + key_file = "tls.key" +else: + cert_file = "server.pem" + key_file = "server.key" + + +for port_id in ports.split(","): + vendor = None + product = None + if port_id != "DEFAULT": + port_parts = port_id.split("_", maxsplit=3) + if len(port_parts) == 2 or len(port_parts) == 3: + vendor = port_parts[0].lower() + product = port_parts[1].lower() + else: + pass + + outputText = tm.render( + vendor=vendor, + product=product, + enable_ipv6=enable_ipv6, + store_raw_message=normalize_env_variable_input("SC4S_SOURCE_STORE_RAWMSG"), + port_id=port_id, + use_reverse_dns=normalize_env_variable_input("SC4S_USE_REVERSE_DNS"), + reverse_dns_keep_fqdn=normalize_env_variable_input("SC4S_REVERSE_DNS_KEEP_FQDN"), + use_udp_log_iw=normalize_env_variable_input("SC4S_SOURCE_UDP_IW_USE"), + use_namecache=normalize_env_variable_input("SC4S_USE_NAME_CACHE"), + use_vpscache=normalize_env_variable_input("SC4S_USE_VPS_CACHE"), + use_tls=normalize_env_variable_input("SC4S_SOURCE_TLS_ENABLE"), + use_proxy_connect=normalize_env_variable_input("SC4S_SOURCE_PROXYCONNECT"), + tls_dir=os.getenv("SC4S_TLS", "/etc/syslog-ng/tls"), + cert_file=cert_file, + key_file=key_file, + enable_ebpf=normalize_env_variable_input("SC4S_ENABLE_EBPF"), + topic=os.getenv(f"SC4S_LISTEN_{ port_id }_TOPIC", "sc4s"), + port_udp=os.getenv(f"SC4S_LISTEN_{ port_id }_UDP_PORT", "disabled").split(","), + port_udp_sockets=int(os.getenv("SC4S_SOURCE_LISTEN_UDP_SOCKETS", 4)), + port_udp_sorecvbuff=int(os.getenv("SC4S_SOURCE_UDP_SO_RCVBUFF", -1)), + port_tcp=os.getenv(f"SC4S_LISTEN_{ port_id }_TCP_PORT", "disabled").split(","), + port_tcp_sockets=int(os.getenv("SC4S_SOURCE_LISTEN_TCP_SOCKETS", 1)), + port_tcp_max_connections=os.getenv("SC4S_SOURCE_TCP_MAX_CONNECTIONS", "2000"), + port_udp_log_iw_size=os.getenv("SC4S_SOURCE_UDP_IW_SIZE", "250000"), + port_udp_log_fetch_limit=os.getenv("SC4S_SOURCE_UDP_FETCH_LIMIT", "1000"), + port_tcp_log_iw_size=os.getenv("SC4S_SOURCE_TCP_IW_SIZE", "20000000"), + port_tcp_log_fetch_limit=os.getenv("SC4S_SOURCE_TCP_FETCH_LIMIT", "2000"), + port_tcp_so_recvbuff=int(os.getenv("SC4S_SOURCE_TCP_SO_RCVBUFF", -1)), + port_tls=os.getenv(f"SC4S_LISTEN_{ port_id }_TLS_PORT", "disabled").split(","), + port_tls_sockets=int(os.getenv("SC4S_SOURCE_LISTEN_TLS_SOCKETS", 1)), + port_tls_max_connections=os.getenv("SC4S_SOURCE_TLS_MAX_CONNECTIONS", "2000"), + port_tls_log_iw_size=os.getenv("SC4S_SOURCE_TCP_IW_SIZE", "20000000"), + port_tls_log_fetch_limit=os.getenv("SC4S_SOURCE_TCP_FETCH_LIMIT", "2000"), + port_tls_so_recvbuff=int(os.getenv("SC4S_SOURCE_TLS_SO_RCVBUFF", -1)), + port_tls_tls_options=os.getenv( + "SC4S_SOURCE_TLS_OPTIONS", "no-sslv2, no-sslv3, no-tlsv1" + ), + port_tls_cipher_suit=os.getenv( + "SC4S_SOURCE_TLS_CIPHER_SUITE", + "HIGH:!aNULL:!eNULL:!kECDH:!aDH:!RC4:!3DES:!CAMELLIA:!MD5:!PSK:!SRP:!KRB5:@STRENGTH", + ), + port_5426=os.getenv(f"SC4S_LISTEN_{ port_id }_RFC5426_PORT", "disabled").split( + "," + ), + port_5426_sockets=int(os.getenv("SC4S_SOURCE_LISTEN_RFC5426_SOCKETS", 1)), + port_5426_sorecvbuff=int(os.getenv("SC4S_SOURCE_RFC5426_SO_RCVBUFF", -1)), + port_6587=os.getenv(f"SC4S_LISTEN_{ port_id }_RFC6587_PORT", "disabled").split( + "," + ), + port_6587_sockets=int(os.getenv("SC4S_SOURCE_LISTEN_RFC6587_SOCKETS", 1)), + port_6587_max_connections=os.getenv( + "SC4S_SOURCE_RFC6587_MAX_CONNECTIONS", "2000" + ), + port_6587_log_iw_size=os.getenv("SC4S_SOURCE_RFC6587_IW_SIZE", "20000000"), + port_6587_log_fetch_limit=os.getenv("SC4S_SOURCE_RFC6587_FETCH_LIMIT", "2000"), + port_6587_so_recvbuff=int(os.getenv("SC4S_SOURCE_RFC6587_SO_RCVBUFF", -1)), + port_5425=os.getenv(f"SC4S_LISTEN_{ port_id }_RFC5425_PORT", "disabled").split( + "," + ), + port_5425_sockets=int(os.getenv("SC4S_SOURCE_LISTEN_RFC5425_SOCKETS", 1)), + port_5425_max_connections=os.getenv( + "SC4S_SOURCE_RFC5425_MAX_CONNECTIONS", "2000" + ), + port_5425_log_iw_size=os.getenv("SC4S_SOURCE_RFC5425_IW_SIZE", "20000000"), + port_5425_log_fetch_limit=os.getenv("SC4S_SOURCE_RFC5425_FETCH_LIMIT", "2000"), + port_5425_so_recvbuff=int(os.getenv("SC4S_SOURCE_RFC5425_SO_RCVBUFF", -1)), + port_5425_tls_options=os.getenv( + "SC4S_SOURCE_RFC5425_OPTIONS", "no-sslv2, no-sslv3, no-tlsv1" + ), + port_5425_cipher_suit=os.getenv( + "SC4S_SOURCE_RFC5425_CIPHER_SUITE", + "HIGH:!aNULL:!eNULL:!kECDH:!aDH:!RC4:!3DES:!CAMELLIA:!MD5:!PSK:!SRP:!KRB5:@STRENGTH", + ), + ebpf_no_sockets=int(os.getenv("SC4S_EBPF_NO_SOCKETS", 4)), + enable_parallelize=normalize_env_variable_input(f"SC4S_ENABLE_PARALLELIZE"), + parallelize_no_partitions=int(os.getenv(f"SC4S_PARALLELIZE_NO_PARTITION", 4)), + set_source_sc4s=normalize_env_variable_input("SC4S_SET_SOURCE_AS_SC4S"), + ) + print(outputText) diff --git a/package/enterprise/etc/context_templates/compliance_meta_by_source.conf.example b/package/enterprise/etc/context_templates/compliance_meta_by_source.conf.example new file mode 100644 index 0000000000..e74da8275e --- /dev/null +++ b/package/enterprise/etc/context_templates/compliance_meta_by_source.conf.example @@ -0,0 +1,5 @@ +filter f_test_test { +# host("something-*" type(glob)) or +# netmask(169.254.100.0/24) + host("cannot_ever_happen") +}; diff --git a/package/enterprise/etc/context_templates/compliance_meta_by_source.csv.example b/package/enterprise/etc/context_templates/compliance_meta_by_source.csv.example new file mode 100644 index 0000000000..4ecf53ab8b --- /dev/null +++ b/package/enterprise/etc/context_templates/compliance_meta_by_source.csv.example @@ -0,0 +1,2 @@ +f_test_test,.splunk.index,"will_never_happen_index" +f_test_test,fields.compliance,"pci" diff --git a/package/enterprise/etc/context_templates/host.csv.example b/package/enterprise/etc/context_templates/host.csv.example new file mode 100644 index 0000000000..3263644ddd --- /dev/null +++ b/package/enterprise/etc/context_templates/host.csv.example @@ -0,0 +1 @@ +169.254.0.2,HOST,foo.example \ No newline at end of file diff --git a/package/enterprise/etc/context_templates/splunk_metadata.csv.example b/package/enterprise/etc/context_templates/splunk_metadata.csv.example new file mode 100644 index 0000000000..4160c4c21a --- /dev/null +++ b/package/enterprise/etc/context_templates/splunk_metadata.csv.example @@ -0,0 +1,2 @@ +vmware_vsphere_nix_syslog,index,infraops +local_example,index,infraops \ No newline at end of file diff --git a/package/enterprise/etc/context_templates/vendor_product_by_source.conf.example b/package/enterprise/etc/context_templates/vendor_product_by_source.conf.example new file mode 100644 index 0000000000..23f6586969 --- /dev/null +++ b/package/enterprise/etc/context_templates/vendor_product_by_source.conf.example @@ -0,0 +1,12 @@ +filter f_tzfixny { + host("tzfny-*" type(glob)) + #or netmask(xxx.xxx.xxx.xxx/xx) +}; +#America/Chicago +filter f_tzfixchi { + host("tzfchi-*" type(glob)) + #or netmask(xxx.xxx.xxx.xxx/xx) +}; +filter f_f5{ + host('^test-f5') or "${HOST}" eq "f5_bigip" +}; \ No newline at end of file diff --git a/package/enterprise/etc/context_templates/vendor_product_by_source.csv.example b/package/enterprise/etc/context_templates/vendor_product_by_source.csv.example new file mode 100644 index 0000000000..723861336a --- /dev/null +++ b/package/enterprise/etc/context_templates/vendor_product_by_source.csv.example @@ -0,0 +1,3 @@ +f_tzfixny,sc4s_time_zone,"America/New_York" +f_tzfixchi,sc4s_time_zone,"America/Chicago" +f_f5,sc4s_vendor_product,f5_bigip diff --git a/package/enterprise/etc/local_config/app_parsers/syslog/app-nix_example.conf b/package/enterprise/etc/local_config/app_parsers/syslog/app-nix_example.conf new file mode 100644 index 0000000000..14278c7cb7 --- /dev/null +++ b/package/enterprise/etc/local_config/app_parsers/syslog/app-nix_example.conf @@ -0,0 +1,51 @@ +# the block parser is where the "parsing" of the event happens and enrichment of meta data +# sample: <111> Mar 24 10:45:00 osnixexample: this is a test +block parser nix_example-parser() { + channel { + rewrite { + #set defaults these values can be overidden at run time by splunk_metadata.csv + r_set_splunk_dest_default( + index("main") + source("os:nix:example") + sourcetype('os:nix:example') + #this value is used to lookup runtime settings such as index from splunk_metadata.csv + vendor("nix") + product("example") + #Common values are t_hdr_msg (BSD Style syslog without timestamp and host) and t_5424_hdr_sdata_compact RFC5424 with optional sdata and msg + #These values will be automatically selected based on the format of the source the specific value is only needed in special cases + #template("t_hdr_msg") + ); + }; + + #rewrite { + # r_set_splunk_dest_update_v2( + # index('notmain') + # source('newsource') + # sourcetype('somethingelse') + # #Using filter syntax identify an exception to the primary values used + # condition(match('audit\.runtime\.com\.rsa' value('.rsa.type'))) + # ); + #}; + + }; +}; +application nix_example[sc4s-network-source] { + filter { + + program('osnixexample' type(string) flags(prefix)) + # If using a custom port for this filter as in the example below the value after s_ will be used for env checks to determin the port number + # for example by using s_OS_NIX_EXAMPLE the following ENV vars will be checked to determine the port to be used if configured + # SC4S_LISTEN_OS_NIX_EXAMPLE_TCP_PORT + # SC4S_LISTEN_OS_NIX_EXAMPLE_UDP_PORT + # SC4S_LISTEN_OS_NIX_EXAMPLE_TLS_PORT + # SC4S_LISTEN_OS_NIX_EXAMPLE_RFC5426_PORT + # SC4S_LISTEN_OS_NIX_EXAMPLE_RFC6587_PORT + # SC4S_LISTEN_OS_NIX_EXAMPLE_RFC5425_PORT + + or tags(".source.s_OS_NIX_EXAMPLE"); + + }; + parser { nix_example-parser(); }; +}; + + diff --git a/package/enterprise/etc/local_config/destinations/README.md b/package/enterprise/etc/local_config/destinations/README.md new file mode 100644 index 0000000000..ee6571d565 --- /dev/null +++ b/package/enterprise/etc/local_config/destinations/README.md @@ -0,0 +1 @@ +This file exists to preserve the path for plugin use \ No newline at end of file diff --git a/package/enterprise/etc/local_config/filters/README.md b/package/enterprise/etc/local_config/filters/README.md new file mode 100644 index 0000000000..ee6571d565 --- /dev/null +++ b/package/enterprise/etc/local_config/filters/README.md @@ -0,0 +1 @@ +This file exists to preserve the path for plugin use \ No newline at end of file diff --git a/package/enterprise/etc/local_config/filters/example.conf b/package/enterprise/etc/local_config/filters/example.conf new file mode 100644 index 0000000000..eda8df9fa3 --- /dev/null +++ b/package/enterprise/etc/local_config/filters/example.conf @@ -0,0 +1 @@ +#This file intentionally blank to avoid upgrade issues, and can be removed in 2.0. diff --git a/package/enterprise/etc/local_config/log_paths/README.md b/package/enterprise/etc/local_config/log_paths/README.md new file mode 100644 index 0000000000..ee6571d565 --- /dev/null +++ b/package/enterprise/etc/local_config/log_paths/README.md @@ -0,0 +1 @@ +This file exists to preserve the path for plugin use \ No newline at end of file diff --git a/package/enterprise/etc/local_config/sources/README.md b/package/enterprise/etc/local_config/sources/README.md new file mode 100644 index 0000000000..ee6571d565 --- /dev/null +++ b/package/enterprise/etc/local_config/sources/README.md @@ -0,0 +1 @@ +This file exists to preserve the path for plugin use \ No newline at end of file diff --git a/package/enterprise/etc/pylib/config_generator/__init__.py b/package/enterprise/etc/pylib/config_generator/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/package/enterprise/etc/pylib/config_generator/__main__.py b/package/enterprise/etc/pylib/config_generator/__main__.py new file mode 100644 index 0000000000..856a3e53de --- /dev/null +++ b/package/enterprise/etc/pylib/config_generator/__main__.py @@ -0,0 +1,29 @@ +from argparse import ArgumentParser +from pathlib import Path + +from .addons import load_addons +from .config import load_addons_config +from .template_generator import template_generator + + +def parse_cli_args(): + cli_parser = ArgumentParser() + cli_parser.add_argument("--config", type=Path) + return cli_parser.parse_args() + + +def generate_syslogng_config() -> None: + cli_args = parse_cli_args() + config = load_addons_config(cli_args.config) + addons = load_addons(config.addons_path) + + syslogng_config = template_generator( + config.syslog_path, + config=config, + addons=sorted(addons, key=lambda addon: addon.path), + ) + print(syslogng_config) + + +if __name__ == "__main__": + generate_syslogng_config() diff --git a/package/enterprise/etc/pylib/config_generator/addons.py b/package/enterprise/etc/pylib/config_generator/addons.py new file mode 100644 index 0000000000..1da3ba7a3d --- /dev/null +++ b/package/enterprise/etc/pylib/config_generator/addons.py @@ -0,0 +1,50 @@ +from dataclasses import dataclass +from logging import getLogger +from pathlib import Path + +import yaml + + +logger = getLogger(__name__) + + +@dataclass +class AddonMetadata: + name: str + + +@dataclass +class Addon: + path: Path + metadata: AddonMetadata + + +def load_addons(addons_directory: Path) -> list[Addon]: + addons: list[Addon] = [] + + for potential_addon in addons_directory.iterdir(): + addon_full_path = addons_directory / potential_addon + + if ( + addon_full_path.is_dir() and + (addon_full_path / "addon_metadata.yaml").exists() + ): + try: + metadata = load_addon_metadata(addon_full_path) + addons.append(Addon(path=addon_full_path, metadata=metadata)) + except Exception as e: + logger.error(f"Skipping invalid addon {potential_addon}") + raise e + + return addons + + +def load_addon_metadata(addon_path: Path) -> AddonMetadata: + with open(addon_path / "addon_metadata.yaml", "r") as file_stream: + try: + metadata = yaml.safe_load(file_stream) + return AddonMetadata(name=metadata["name"]) + except yaml.YAMLError: + logger.error(f"Metadata file of {addon_path} should be valid yaml") + except KeyError: + logger.error(f"Missing metadata in {addon_path}") diff --git a/package/enterprise/etc/pylib/config_generator/config.py b/package/enterprise/etc/pylib/config_generator/config.py new file mode 100644 index 0000000000..d2c4ebb853 --- /dev/null +++ b/package/enterprise/etc/pylib/config_generator/config.py @@ -0,0 +1,26 @@ +from dataclasses import dataclass +from logging import getLogger +from os.path import expandvars +from pathlib import Path + +import yaml + +logger = getLogger(__name__) + + +@dataclass +class Config: + addons: list[str] + addons_path: Path = Path(expandvars("${SC4S_ETC}/addons")) + syslog_path: Path = Path(expandvars("${SC4S_ETC}/syslog-ng.conf.jinja")) + + +def load_addons_config(config_path: Path) -> Config: + with open(config_path, "r") as file_stream: + try: + raw_config = yaml.safe_load(file_stream) + return Config(**raw_config) + except yaml.YAMLError: + logger.error("Config should be correct yaml") + except KeyError: + logger.error("Field is missing in config") diff --git a/package/enterprise/etc/pylib/config_generator/template_generator.py b/package/enterprise/etc/pylib/config_generator/template_generator.py new file mode 100644 index 0000000000..7832f532d7 --- /dev/null +++ b/package/enterprise/etc/pylib/config_generator/template_generator.py @@ -0,0 +1,12 @@ +from pathlib import Path + +import jinja2 + + +def template_generator(template_path: Path, **kwargs) -> str: + env = jinja2.Environment( + loader=jinja2.FileSystemLoader(template_path.parent), + autoescape=False, + ) + template = env.get_template(template_path.name) + return template.render(**kwargs) diff --git a/package/enterprise/etc/pylib/parser_cef.py b/package/enterprise/etc/pylib/parser_cef.py new file mode 100644 index 0000000000..7d0673fd45 --- /dev/null +++ b/package/enterprise/etc/pylib/parser_cef.py @@ -0,0 +1,54 @@ +import re +import sys +import traceback + +try: + import syslogng + from syslogng import LogParser +except Exception: + + class LogParser: + pass + + +class cef_kv(LogParser): + def init(self, options): + self.logger = syslogng.Logger() + return True + + def parse(self, log_message): + + try: + data = log_message.get_as_str(".metadata.cef.ext", "") + + rpairs = re.findall(r"([^=\s]+)=((?:[\\]=|[^=])+)(?:\s|$)", data) + pairs = {} + keys = [] + for p in rpairs: + pairs[p[0]] = p[1] + keys.append(p[0]) + + cleanpairs = {} + for k in keys: + if k.endswith("Label"): + vk = k.rstrip("Label") + if k in pairs: + l = pairs[k] + if vk in pairs: + pairs[l] = pairs[vk] + del pairs[vk] + del pairs[k] + elif k == "rawEvent": + pairs[k] = pairs[k].replace(r"\=", "=").replace("&&", "\n") + + for k, v in pairs.items(): + kc = k.replace(" ", "_").replace(".", "_") + log_message[f".values.{kc}"] = v + + except Exception: + exc_type, exc_value, exc_traceback = sys.exc_info() + lines = traceback.format_exception(exc_type, exc_value, exc_traceback) + self.logger.debug("".join("!! " + line for line in lines)) + return False + self.logger.debug("kvqf_parse.parse complete") + return True \ No newline at end of file diff --git a/package/enterprise/etc/pylib/parser_fix_dns.py b/package/enterprise/etc/pylib/parser_fix_dns.py new file mode 100644 index 0000000000..de3bbd7d15 --- /dev/null +++ b/package/enterprise/etc/pylib/parser_fix_dns.py @@ -0,0 +1,64 @@ +""" +simple syslog-ng Python parser example +resolves IP to hostname +value pair names are hard-coded +""" +import re +import socket + +try: + import syslogng + from syslogng import LogParser +except Exception: + + class LogParser: + pass + + +class FixHostnameResolver(LogParser): + def parse(self, log_message): + """ + Resolves IP to hostname + """ + + # try to resolve the IP address + try: + ipaddr = log_message.get_as_str("SOURCEIP", "", repr="internal") + + hostname, aliaslist, ipaddrlist = socket.gethostbyaddr(ipaddr) + + if hostname == ipaddr: + return False + + parts = str(hostname).split(".") + name = parts[0] + if len(parts) > 1: + log_message["HOST"] = name + except Exception: + return False + + # return True, other way message is dropped + return True + + +class FixFQDNResolver(LogParser): + def parse(self, log_message): + """ + Resolves IP to FQDN + """ + + # try to resolve the IP address + try: + ipaddr = log_message.get_as_str("SOURCEIP", "", repr="internal") + + fqdn, aliaslist, ipaddrlist = socket.gethostbyaddr(ipaddr) + + if fqdn == ipaddr: + return False + + log_message["HOST"] = str(fqdn) + except Exception: + return False + + # return True, other way message is dropped + return True \ No newline at end of file diff --git a/package/enterprise/etc/pylib/parser_kvqf.py b/package/enterprise/etc/pylib/parser_kvqf.py new file mode 100644 index 0000000000..c5596c55f0 --- /dev/null +++ b/package/enterprise/etc/pylib/parser_kvqf.py @@ -0,0 +1,39 @@ +# coding=utf8 +# the above tag defines encoding for this document and is for Python 2.x compatibility +import sys +import traceback +import re + +try: + import syslogng + from syslogng import LogParser +except Exception: + + class LogParser: + pass + + +regex = r"\"([^\"]+)\"=\"([^\"]+)\"" + + +class kvqf_parse(LogParser): + def init(self, options): + self.logger = syslogng.Logger() + return True + + def parse(self, log_message): + try: + matches = re.finditer( + regex, log_message.get_as_str(".tmp.pairs", ""), re.MULTILINE + ) + for matchNum, match in enumerate(matches, start=1): + k = match.groups()[0] + v = match.groups()[1] + log_message[f".values.{k}"] = v + except Exception: + exc_type, exc_value, exc_traceback = sys.exc_info() + lines = traceback.format_exception(exc_type, exc_value, exc_traceback) + self.logger.debug("".join("!! " + line for line in lines)) + return False + self.logger.debug("kvqf_parse.parse complete") + return True \ No newline at end of file diff --git a/package/enterprise/etc/pylib/parser_leef.py b/package/enterprise/etc/pylib/parser_leef.py new file mode 100644 index 0000000000..1279701314 --- /dev/null +++ b/package/enterprise/etc/pylib/parser_leef.py @@ -0,0 +1,84 @@ +import re +import binascii +import sys +import traceback + +try: + import syslogng + from syslogng import LogParser +except Exception: + + class LogParser: + pass + + +class leef_kv(LogParser): + def init(self, options): + self.regex = r"( ?(?:[A-Z]{2,4}T|HAEC|IDLW|MSK|NT|UTC|THA))" + self.logger = syslogng.Logger() + return True + + def parse(self, log_message): + + try: + msg = log_message.get_as_str("MESSAGE", "") + # All LEEF message are | separated super structures + structure = msg.split("|") + # Indexed fields for Splunk + + log_message[".metadata.leef.version"] = structure[0][5:] + log_message[".metadata.leef.vendor"] = structure[1] + log_message[".metadata.leef.product"] = structure[2] + log_message[".metadata.leef.product_version"] = structure[3] + log_message[".metadata.leef.EventID"] = structure[4] + # We just want the event field + event = structure[len(structure) - 1] + log_message[".leef.event"] = event + # V1 will always use tab + if structure[0][5:].startswith("1"): + separator = "\t" + lv = "1" + pairs = event.split(separator) + if len(pairs) < 4: + separator = "|" + pairs = structure[5:] + event = "\t".join(pairs) + log_message[".leef.event"] = event + else: + lv = "2" + # V2 messages should always provide the sep but some fail do comply + # with the format spec if they don't assume tab + if len(structure) == 6 or not structure[5]: + separator = "\t" + pairs = event.split(separator) + else: + separator = structure[5] + if separator.startswith("0"): + separator = separator[1:] + pairs = event.split(separator) + + if separator.startswith("x"): + hex_sep = f"0{separator.lower()}" + else: + hex_sep = f'0x{binascii.b2a_hex(separator.encode("utf-8")).decode("utf-8").lower()}' + if structure[0][5:].startswith("1"): + log_message[".splunk.sourcetype"] = f"LEEF:{lv}" + else: + log_message[".splunk.sourcetype"] = f"LEEF:{lv}:{hex_sep}" + log_message[".splunk.source"] = f"{structure[1]}:{structure[2]}" + log_message["fields.sc4s_vendor"] = structure[1] + log_message["fields.sc4s_product"] = structure[2] + + for p in pairs: + f, v = p.split("=", 1) + if f == "devTime": + log_message[".leef." + f] = re.sub( + self.regex, "", v, 0, re.MULTILINE + ) + else: + log_message[".leef." + f] = v + except Exception as e: + log_message[".metadata.leef.exception"] = str(e) + + # return True, other way message is dropped + return True \ No newline at end of file diff --git a/package/enterprise/etc/pylib/parser_source_cache.py b/package/enterprise/etc/pylib/parser_source_cache.py new file mode 100644 index 0000000000..1a8bcbca2a --- /dev/null +++ b/package/enterprise/etc/pylib/parser_source_cache.py @@ -0,0 +1,129 @@ +import sys +import traceback +import socket +import struct +from sqlitedict import SqliteDict + +import time + +try: + import syslogng + from syslogng import LogParser, LogDestination +except Exception: + + class LogParser: + pass + + class LogDestination: + pass + + +def ip2int(addr): + ip4_to_int = lambda addr: struct.unpack("!I", socket.inet_aton(addr))[0] + + def ip6_to_int(addr): + ip6 = socket.inet_pton(socket.AF_INET6, addr) + a, b = struct.unpack(">QQ", ip6) + return (a << 64) | b + + try: + return ip4_to_int(addr) + except OSError: + return ip6_to_int(addr) + + +def int2ip(addr): + int_to_ip4 = lambda addr: socket.inet_ntoa(struct.pack("!I", addr)) + + def int_to_ip6(num): + a = (num >> 64) & 0xFFFFFFFFFFFFFFFF + b = num & 0xFFFFFFFFFFFFFFFF + ip6 = struct.pack(">QQ", a, b) + addr = socket.inet_ntop(socket.AF_INET6, ip6) + return addr + + try: + return int_to_ip4(addr) + except struct.error: + return int_to_ip6(addr) + + +hostdict = str("/var/lib/syslog-ng/hostip") + + +class psc_parse(LogParser): + def init(self, options): + self.logger = syslogng.Logger() + self.db = SqliteDict(f"{hostdict}.sqlite") + return True + + def deinit(self): + self.db.close() + + def parse(self, log_message): + try: + ipaddr = log_message.get_as_str("SOURCEIP", "", repr="internal") + ip_int = ip2int(ipaddr) + self.logger.debug(f"psc.parse sourceip={ipaddr} int={ip_int}") + name = self.db[ip_int] + self.logger.debug(f"psc.parse host={name}") + log_message["HOST"] = name + + except Exception: + exc_type, exc_value, exc_traceback = sys.exc_info() + lines = traceback.format_exception(exc_type, exc_value, exc_traceback) + self.logger.debug("".join("!! " + line for line in lines)) + return False + self.logger.debug("psc.parse complete") + return True + + +class psc_dest(LogDestination): + def init(self, options): + self.logger = syslogng.Logger() + try: + self.db = SqliteDict(f"{hostdict}.sqlite", autocommit=True) + except Exception: + exc_type, exc_value, exc_traceback = sys.exc_info() + lines = traceback.format_exception(exc_type, exc_value, exc_traceback) + self.logger.debug("".join("!! " + line for line in lines)) + return False + return True + + def deinit(self): + """Close the connection to the target service""" + self.db.commit() + self.db.close() + + def send(self, log_message): + try: + ipaddr = log_message.get_as_str("SOURCEIP", "", repr="internal") + ip_int = ip2int(ipaddr) + self.logger.debug( + f'psc.send sourceip={ipaddr} int={ip_int} host={log_message["HOST"]}' + ) + if ip_int in self.db: + current = self.db[ip_int] + if current != log_message["HOST"]: + self.db[ip_int] = log_message["HOST"] + else: + self.db[ip_int] = log_message["HOST"] + + except Exception: + exc_type, exc_value, exc_traceback = sys.exc_info() + lines = traceback.format_exception(exc_type, exc_value, exc_traceback) + self.logger.debug("".join("!! " + line for line in lines)) + return False + self.logger.debug("psc.send complete") + return True + + def flush(self): + self.db.commit() + return True + + +if __name__ == "__main__": + db = SqliteDict(f"{hostdict}.sqlite", autocommit=True) + db[0] = "seed" + db.commit() + db.close() diff --git a/package/enterprise/etc/pylib/parser_stealthbits.py b/package/enterprise/etc/pylib/parser_stealthbits.py new file mode 100644 index 0000000000..bdf0369816 --- /dev/null +++ b/package/enterprise/etc/pylib/parser_stealthbits.py @@ -0,0 +1,36 @@ +import re + +try: + import syslogng + from syslogng import LogParser +except Exception: + + class LogParser: + pass + + +regex = r"^(.*[\.\!\?])?(.*:.*)" + + +class alerttext_kv(LogParser): + def init(self, options): + return True + + def parse(self, log_message): + match = re.search(regex, log_message.get_as_str(".values.AlertText", "")) + if match: + log_message[".values.AlertText"] = match.groups()[0] + text = match.groups()[1] + else: + text = log_message.get_as_str(".values.AlertText", "") + log_message[".values.AlertText"] = "" + + pairs = text.split("; ") + + if len(pairs) == 0: + return False + for p in pairs: + k, v = p.split(": ") + cleank = k.replace(" ", "_").replace(".", "_") + log_message[f".values.AlertTextValues.{cleank}"] = v.strip() + return True \ No newline at end of file diff --git a/package/enterprise/etc/pylib/parser_vps_cache.py b/package/enterprise/etc/pylib/parser_vps_cache.py new file mode 100644 index 0000000000..4c8cf21250 --- /dev/null +++ b/package/enterprise/etc/pylib/parser_vps_cache.py @@ -0,0 +1,101 @@ +import sys +import traceback +import socket +import struct +from sqlitedict import SqliteDict + +import time + +try: + import syslogng + from syslogng import LogParser, LogDestination +except Exception: + + class LogParser: + pass + + class LogDestination: + pass + + +hostdict = str("/var/lib/syslog-ng/vps") + + +class vpsc_parse(LogParser): + def init(self, options): + self.logger = syslogng.Logger() + self.db = SqliteDict(f"{hostdict}.sqlite") + return True + + def deinit(self): + self.db.close() + + def parse(self, log_message): + try: + host = log_message.get_as_str("HOST", "") + self.logger.debug(f"vpsc.parse host={host}") + fields = self.db[host] + self.logger.debug(f"vpsc.parse host={host} fields={fields}") + for k, v in fields.items(): + log_message[k] = v + + except Exception: + exc_type, exc_value, exc_traceback = sys.exc_info() + lines = traceback.format_exception(exc_type, exc_value, exc_traceback) + self.logger.debug("".join("!! " + line for line in lines)) + return False + self.logger.debug("vpsc.parse complete") + return True + + +class vpsc_dest(LogDestination): + def init(self, options): + self.logger = syslogng.Logger() + try: + self.db = SqliteDict(f"{hostdict}.sqlite", autocommit=True) + except Exception: + exc_type, exc_value, exc_traceback = sys.exc_info() + lines = traceback.format_exception(exc_type, exc_value, exc_traceback) + self.logger.debug("".join("!! " + line for line in lines)) + return False + return True + + def deinit(self): + """Close the connection to the target service""" + self.db.commit() + self.db.close() + + def send(self, log_message): + try: + host = log_message.get_as_str("HOST", "") + fields = {} + fields[".netsource.sc4s_vendor"] = log_message.get_as_str( + "fields.sc4s_vendor" + ) + fields[".netsource.sc4s_product"] = log_message.get_as_str( + "fields.sc4s_product" + ) + + self.logger.debug(f"vpsc.send host={host} fields={fields}") + if host in self.db: + current = self.db[host] + if current != fields: + self.db[host] = fields + else: + self.db[host] = fields + + except Exception: + exc_type, exc_value, exc_traceback = sys.exc_info() + lines = traceback.format_exception(exc_type, exc_value, exc_traceback) + self.logger.debug("".join("!! " + line for line in lines)) + return False + self.logger.debug("psc.send complete") + return True + + def flush(self): + self.db.commit() + return True + + +if __name__ == "__main__": + pass diff --git a/package/enterprise/etc/pylib/psc_dump.py b/package/enterprise/etc/pylib/psc_dump.py new file mode 100644 index 0000000000..eccbe1e650 --- /dev/null +++ b/package/enterprise/etc/pylib/psc_dump.py @@ -0,0 +1,12 @@ +import sys +import traceback +import socket +import struct +from sqlitedict import SqliteDict + + +hostdict = str("/var/lib/syslog-ng/cache/hostip") +db = SqliteDict(f"{hostdict}.sqlite") + +for k, v in db.items(): + print(f"key={k}={v}") diff --git a/package/enterprise/etc/syslog-ng.conf b/package/enterprise/etc/syslog-ng.conf new file mode 100644 index 0000000000..4b5894e119 --- /dev/null +++ b/package/enterprise/etc/syslog-ng.conf @@ -0,0 +1,64 @@ +@version:4.9 + + +# syslog-ng configuration file. + +# Copyright © 2019,2020 Splunk Inc. +# Licensed under https://www.apache.org/licenses/LICENSE-2.0.txt + +# See syslog-ng(8) and syslog-ng.conf(5) for more information. + +# +# =============================================================================================== +# Global Options -- These options are generated via confgen function in sc4slib +# =============================================================================================== +@module confgen context(root) name(global_options) exec("`SC4S_ETC`/conf.d/sc4slib/global_options/plugin.py") +global_options(); +@include "scl.conf" +@include "conf.d/sc4slib/*/*.conf" + +# =============================================================================================== +# Default message template +# =============================================================================================== +@define splunk-template "t_standard" + +# =============================================================================================== +# Global modules and includes. All device-specific filters and destinations exist in conf.d +# Order is important; templates must be loaded first +# =============================================================================================== +@include "conf.d/templates/*.conf" +@include "conf.d/conflib/*.conf" +@include "conf.d/conflib/*/*.conf" +@include "conf.d/local/config/app_parsers/*.conf" +@include "conf.d/local/config/app_parsers/*/*.conf" + +@include "conf.d/filters/*/*.conf" +@include "conf.d/local/config/filters/*.conf" + +@include "conf.d/enrich/*.conf" +@include "conf.d/enrich/*/*.conf" + +@include "conf.d/plugin/*.conf" + +@include "conf.d/sources/*.conf" +@include "conf.d/sources/*/*.conf" +@include "conf.d/local/config/sources/*.conf" +@include "conf.d/local/config/sources/*/*.conf" + +@include "conf.d/destinations/*.conf" +@include "conf.d/destinations/*/*.conf" + +@include "conf.d/local/config/destinations/*.conf" +@include "conf.d/local/config/destinations/*/*.conf" + +# Group 0 log paths run before all others for high level activity like null queue +@include "conf.d/log_paths/0/*/*.conf" +@include "conf.d/log_paths/0/*.conf" + + +#Local Log paths run before stock +@include "conf.d/local/config/log_paths/*.conf" + +#Stock log paths +@include "conf.d/log_paths/2/*/*.conf" +@include "conf.d/log_paths/2/*.conf" diff --git a/poetry.lock b/poetry.lock index c379f0fc3b..118e63c2e7 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.9.0.dev0 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.5 and should not be changed by hand. [[package]] name = "arrow" @@ -1248,6 +1248,26 @@ files = [ {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, ] +[[package]] +name = "tornado" +version = "6.4.2" +description = "Tornado is a Python web framework and asynchronous networking library, originally developed at FriendFeed." +optional = false +python-versions = ">=3.8" +files = [ + {file = "tornado-6.4.2-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:e828cce1123e9e44ae2a50a9de3055497ab1d0aeb440c5ac23064d9e44880da1"}, + {file = "tornado-6.4.2-cp38-abi3-macosx_10_9_x86_64.whl", hash = "sha256:072ce12ada169c5b00b7d92a99ba089447ccc993ea2143c9ede887e0937aa803"}, + {file = "tornado-6.4.2-cp38-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1a017d239bd1bb0919f72af256a970624241f070496635784d9bf0db640d3fec"}, + {file = "tornado-6.4.2-cp38-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c36e62ce8f63409301537222faffcef7dfc5284f27eec227389f2ad11b09d946"}, + {file = "tornado-6.4.2-cp38-abi3-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bca9eb02196e789c9cb5c3c7c0f04fb447dc2adffd95265b2c7223a8a615ccbf"}, + {file = "tornado-6.4.2-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:304463bd0772442ff4d0f5149c6f1c2135a1fae045adf070821c6cdc76980634"}, + {file = "tornado-6.4.2-cp38-abi3-musllinux_1_2_i686.whl", hash = "sha256:c82c46813ba483a385ab2a99caeaedf92585a1f90defb5693351fa7e4ea0bf73"}, + {file = "tornado-6.4.2-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:932d195ca9015956fa502c6b56af9eb06106140d844a335590c1ec7f5277d10c"}, + {file = "tornado-6.4.2-cp38-abi3-win32.whl", hash = "sha256:2876cef82e6c5978fde1e0d5b1f919d756968d5b4282418f3146b79b58556482"}, + {file = "tornado-6.4.2-cp38-abi3-win_amd64.whl", hash = "sha256:908b71bf3ff37d81073356a5fadcc660eb10c1476ee6e2725588626ce7e5ca38"}, + {file = "tornado-6.4.2.tar.gz", hash = "sha256:92bad5b4746e9879fd7bf1eb21dce4e3fc5128d71601f80005afa39237ad620b"}, +] + [[package]] name = "types-python-dateutil" version = "2.9.0.20240906" @@ -1385,4 +1405,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "15a805f6379fa28a3b310fc492b3f9d468945660693413c9c0c7e54c86841475" +content-hash = "dd6bd807849cdae58d638b1ea4d7655c0b7acfcb8f75f8e61575b883a2bed861" diff --git a/pyproject.toml b/pyproject.toml index 5bd9a1cc5e..5ce243df98 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,6 +13,7 @@ shortuuid = "^1.0.11" pyyaml = "6.0.2" setuptools = "^73.0.1" restricted-sqlitedict = "^1.0.0" +tornado = "^6.4.2" [tool.poetry.group.dev.dependencies]