diff --git a/.dockerignore b/.dockerignore index b63f0cdfa..4172fe569 100644 --- a/.dockerignore +++ b/.dockerignore @@ -28,8 +28,8 @@ arkime-logs arkime-raw kubernetes malcolm-iso -sensor-iso -sensor-raspi +hedgehog-iso +hedgehog-raspi nginx/nginx_ldap*.conf pcap _site diff --git a/.github/workflows/sensor-iso-build-docker-wrap-push-ghcr.yml b/.github/workflows/hedgehog-iso-build-docker-wrap-push-ghcr.yml similarity index 87% rename from .github/workflows/sensor-iso-build-docker-wrap-push-ghcr.yml rename to .github/workflows/hedgehog-iso-build-docker-wrap-push-ghcr.yml index 14fe12261..76361ae13 100644 --- a/.github/workflows/sensor-iso-build-docker-wrap-push-ghcr.yml +++ b/.github/workflows/hedgehog-iso-build-docker-wrap-push-ghcr.yml @@ -1,4 +1,4 @@ -name: sensor-iso-build-docker-wrap-push-ghcr +name: hedgehog-iso-build-docker-wrap-push-ghcr on: push: @@ -6,10 +6,10 @@ on: - main - development paths: - - 'sensor-iso/**' + - 'hedgehog-iso/**' - 'shared/bin/*' - '.trigger_iso_workflow_build' - - '.github/workflows/sensor-iso-build-docker-wrap-push-ghcr.yml' + - '.github/workflows/hedgehog-iso-build-docker-wrap-push-ghcr.yml' workflow_dispatch: repository_dispatch: @@ -93,13 +93,13 @@ jobs: - name: Build image run: | - cp -r ./shared ./docs ./_config.yml ./_includes ./_layouts ./Gemfile ./README.md ./malcolm-iso/htpdate ./sensor-iso - cp ./scripts/malcolm_utils.py ./sensor-iso/shared/bin/ - cp ./scripts/documentation_build.sh ./sensor-iso/docs/ - cp -r ./arkime/patch ./sensor-iso/shared/arkime_patch - mkdir -p ./sensor-iso/suricata - cp -r ./suricata/rules-default ./sensor-iso/suricata/ - pushd ./sensor-iso + cp -r ./shared ./docs ./_config.yml ./_includes ./_layouts ./Gemfile ./README.md ./malcolm-iso/htpdate ./hedgehog-iso + cp ./scripts/malcolm_utils.py ./hedgehog-iso/shared/bin/ + cp ./scripts/documentation_build.sh ./hedgehog-iso/docs/ + cp -r ./arkime/patch ./hedgehog-iso/shared/arkime_patch + mkdir -p ./hedgehog-iso/suricata + cp -r ./suricata/rules-default ./hedgehog-iso/suricata/ + pushd ./hedgehog-iso echo "${{ steps.extract_malcolm_version.outputs.mversion }}" > ./shared/version.txt echo "${{ secrets.MAXMIND_GEOIP_DB_LICENSE_KEY }}" > ./shared/maxmind_license.txt echo "GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }}" > ./shared/environment.chroot @@ -115,7 +115,7 @@ jobs: uses: aquasecurity/trivy-action@master with: scan-type: 'fs' - scan-ref: './sensor-iso' + scan-ref: './hedgehog-iso' format: 'sarif' output: 'trivy-results.sarif' severity: 'HIGH,CRITICAL' @@ -140,6 +140,6 @@ jobs: name: Build and push ISO image uses: docker/build-push-action@v5 with: - context: ./sensor-iso + context: ./hedgehog-iso push: true tags: ghcr.io/${{ github.repository_owner }}/malcolm/hedgehog:${{ steps.extract_branch.outputs.branch }} diff --git a/.github/workflows/sensor-raspi-build-docker-wrap-push-ghcr.yml b/.github/workflows/hedgehog-raspi-build-docker-wrap-push-ghcr.yml similarity index 96% rename from .github/workflows/sensor-raspi-build-docker-wrap-push-ghcr.yml rename to .github/workflows/hedgehog-raspi-build-docker-wrap-push-ghcr.yml index 0d5faee04..952e850ab 100644 --- a/.github/workflows/sensor-raspi-build-docker-wrap-push-ghcr.yml +++ b/.github/workflows/hedgehog-raspi-build-docker-wrap-push-ghcr.yml @@ -1,4 +1,4 @@ -name: sensor-raspi-build-docker-wrap-push-ghcr +name: hedgehog-raspi-build-docker-wrap-push-ghcr on: # push: @@ -76,7 +76,7 @@ jobs: - name: Build image run: | - pushd ./sensor-raspi + pushd ./hedgehog-raspi mkdir -p ./shared echo "${{ steps.extract_malcolm_version.outputs.mversion }}" > ./shared/version.txt echo "${{ secrets.MAXMIND_GEOIP_DB_LICENSE_KEY }}" > ./shared/maxmind_license.txt @@ -97,6 +97,6 @@ jobs: name: Build and push IMG image uses: docker/build-push-action@v5 with: - context: ./sensor-raspi + context: ./hedgehog-raspi push: true tags: ghcr.io/${{ github.repository_owner }}/malcolm/hedgehog-raspi:${{ steps.extract_branch.outputs.branch }} diff --git a/.gitignore b/.gitignore index eadc2e2ff..1b3fe1317 100644 --- a/.gitignore +++ b/.gitignore @@ -37,13 +37,13 @@ malcolm_netbox_backup_*.gz *-build.log Gemfile.lock _site -sensor-iso/_config.yml -sensor-iso/_includes -sensor-iso/_layouts -sensor-iso/_site -sensor-iso/docs -sensor-iso/Gemfile -sensor-iso/README.md +hedgehog-iso/_config.yml +hedgehog-iso/_includes +hedgehog-iso/_layouts +hedgehog-iso/_site +hedgehog-iso/docs +hedgehog-iso/Gemfile +hedgehog-iso/README.md # Byte-compiled / optimized / DLL files __pycache__/ diff --git a/Dockerfiles/arkime.Dockerfile b/Dockerfiles/arkime.Dockerfile index cdc34e593..ac8d1f776 100644 --- a/Dockerfiles/arkime.Dockerfile +++ b/Dockerfiles/arkime.Dockerfile @@ -7,7 +7,7 @@ ENV TERM xterm ENV PYTHONDONTWRITEBYTECODE 1 ENV PYTHONUNBUFFERED 1 -ENV ARKIME_VERSION "v5.0.0" +ENV ARKIME_VERSION "v5.0.1" ENV ARKIME_DIR "/opt/arkime" ENV ARKIME_URL "https://github.com/arkime/arkime.git" ENV ARKIME_LOCALELASTICSEARCH no diff --git a/Dockerfiles/dashboards.Dockerfile b/Dockerfiles/dashboards.Dockerfile index b4e6e828d..3285435cd 100644 --- a/Dockerfiles/dashboards.Dockerfile +++ b/Dockerfiles/dashboards.Dockerfile @@ -1,4 +1,4 @@ -FROM opensearchproject/opensearch-dashboards:2.11.1 +FROM opensearchproject/opensearch-dashboards:2.12.0 LABEL maintainer="malcolm@inl.gov" LABEL org.opencontainers.image.authors='malcolm@inl.gov' @@ -20,7 +20,7 @@ ENV PUSER_PRIV_DROP true ENV TERM xterm ENV TINI_VERSION v0.19.0 -ENV OSD_TRANSFORM_VIS_VERSION 2.11.0 +ENV OSD_TRANSFORM_VIS_VERSION 2.12.0 ARG NODE_OPTIONS="--max_old_space_size=4096" ENV NODE_OPTIONS $NODE_OPTIONS @@ -39,10 +39,10 @@ RUN yum upgrade -y && \ # Malcolm manages authentication and encryption via NGINX reverse proxy /usr/share/opensearch-dashboards/bin/opensearch-dashboards-plugin remove securityDashboards --allow-root && \ cd /tmp && \ - unzip transformVis.zip opensearch-dashboards/transformVis/opensearch_dashboards.json opensearch-dashboards/transformVis/package.json && \ - sed -i "s/2\.11\.0/2\.11\.1/g" opensearch-dashboards/transformVis/opensearch_dashboards.json && \ - sed -i "s/2\.11\.0/2\.11\.1/g" opensearch-dashboards/transformVis/package.json && \ - zip transformVis.zip opensearch-dashboards/transformVis/opensearch_dashboards.json opensearch-dashboards/transformVis/package.json && \ + # unzip transformVis.zip opensearch-dashboards/transformVis/opensearch_dashboards.json opensearch-dashboards/transformVis/package.json && \ + # sed -i "s/2\.12\.0/2\.12\.0/g" opensearch-dashboards/transformVis/opensearch_dashboards.json && \ + # sed -i "s/2\.12\.0/2\.12\.0/g" opensearch-dashboards/transformVis/package.json && \ + # zip transformVis.zip opensearch-dashboards/transformVis/opensearch_dashboards.json opensearch-dashboards/transformVis/package.json && \ cd /usr/share/opensearch-dashboards/plugins && \ /usr/share/opensearch-dashboards/bin/opensearch-dashboards-plugin install file:///tmp/transformVis.zip --allow-root && \ rm -rf /tmp/transformVis /tmp/opensearch-dashboards && \ @@ -59,37 +59,6 @@ ADD dashboards/opensearch_dashboards.yml /usr/share/opensearch-dashboards/config ADD dashboards/scripts/docker_entrypoint.sh /usr/local/bin/ ADD scripts/malcolm_utils.py /usr/local/bin/ -# Yeah, I know about https://opensearch.org/docs/latest/dashboards/branding ... but I can't figure out a way -# to specify the entries in the opensearch_dashboards.yml such that they are valid BOTH from the -# internal opensearch code validating them AND the web browser retrieving them. So we're going scorched earth instead. - -COPY --chmod=644 docs/images/favicon/favicon192.png /usr/share/opensearch-dashboards/src/core/server/core_app/assets/favicons/android-chrome-192x192.png -COPY --chmod=644 docs/images/favicon/favicon512.png /usr/share/opensearch-dashboards/src/core/server/core_app/assets/favicons/android-chrome-512x512.png -COPY --chmod=644 docs/images/favicon/apple-touch-icon-precomposed.png /usr/share/opensearch-dashboards/src/core/server/core_app/assets/favicons/apple-touch-icon.png -COPY --chmod=644 docs/images/favicon/favicon16.png /usr/share/opensearch-dashboards/src/core/server/core_app/assets/favicons/favicon-16x16.png -COPY --chmod=644 docs/images/favicon/favicon32.png /usr/share/opensearch-dashboards/src/core/server/core_app/assets/favicons/favicon-32x32.png -COPY --chmod=644 docs/images/favicon/favicon.ico /usr/share/opensearch-dashboards/src/core/server/core_app/assets/favicons/favicon.ico -COPY --chmod=644 docs/images/favicon/favicon144.png /usr/share/opensearch-dashboards/src/core/server/core_app/assets/favicons/mstile-144x144.png -COPY --chmod=644 docs/images/favicon/favicon150.png /usr/share/opensearch-dashboards/src/core/server/core_app/assets/favicons/mstile-150x150.png -COPY --chmod=644 docs/images/favicon/favicon310.png /usr/share/opensearch-dashboards/src/core/server/core_app/assets/favicons/mstile-310x310.png -COPY --chmod=644 docs/images/favicon/favicon70.png /usr/share/opensearch-dashboards/src/core/server/core_app/assets/favicons/mstile-70x70.png -COPY --chmod=644 docs/images/logo/Malcolm.svg /usr/share/opensearch-dashboards/src/core/server/core_app/assets/logos/opensearch.svg -COPY --chmod=644 docs/images/icon/malcolm_mark_dashboards.svg /usr/share/opensearch-dashboards/src/core/server/core_app/assets/logos/opensearch_center_mark.svg -COPY --chmod=644 docs/images/icon/malcolm_mark_dashboards.svg /usr/share/opensearch-dashboards/src/core/server/core_app/assets/logos/opensearch_center_mark_on_dark.svg -COPY --chmod=644 docs/images/icon/malcolm_mark_dashboards.svg /usr/share/opensearch-dashboards/src/core/server/core_app/assets/logos/opensearch_center_mark_on_light.svg -COPY --chmod=644 docs/images/logo/Malcolm.svg /usr/share/opensearch-dashboards/src/core/server/core_app/assets/logos/opensearch_dashboards.svg -COPY --chmod=644 docs/images/logo/malcolm_logo.svg /usr/share/opensearch-dashboards/src/core/server/core_app/assets/logos/opensearch_dashboards_on_dark.svg -COPY --chmod=644 docs/images/logo/Malcolm.svg /usr/share/opensearch-dashboards/src/core/server/core_app/assets/logos/opensearch_dashboards_on_light.svg -COPY --chmod=644 docs/images/icon/malcolm_mark_dashboards.svg /usr/share/opensearch-dashboards/src/core/server/core_app/assets/logos/opensearch_mark.svg -COPY --chmod=644 docs/images/icon/malcolm_mark_dashboards.svg /usr/share/opensearch-dashboards/src/core/server/core_app/assets/logos/opensearch_mark_on_dark.svg -COPY --chmod=644 docs/images/icon/malcolm_mark_dashboards.svg /usr/share/opensearch-dashboards/src/core/server/core_app/assets/logos/opensearch_mark_on_light.svg -COPY --chmod=644 docs/images/logo/malcolm_logo.svg /usr/share/opensearch-dashboards/src/core/server/core_app/assets/logos/opensearch_on_dark.svg -COPY --chmod=644 docs/images/logo/Malcolm.svg /usr/share/opensearch-dashboards/src/core/server/core_app/assets/logos/opensearch_on_light.svg -COPY --chmod=644 docs/images/icon/malcolm_mark_dashboards.svg /usr/share/opensearch-dashboards/src/core/server/core_app/assets/logos/opensearch_spinner.svg -COPY --chmod=644 docs/images/icon/malcolm_mark_dashboards.svg /usr/share/opensearch-dashboards/src/core/server/core_app/assets/logos/opensearch_spinner_on_dark.svg -COPY --chmod=644 docs/images/icon/malcolm_mark_dashboards.svg /usr/share/opensearch-dashboards/src/core/server/core_app/assets/logos/opensearch_spinner_on_light.svg - - ENTRYPOINT ["/usr/bin/tini", \ "--", \ "/usr/local/bin/docker-uid-gid-setup.sh", \ diff --git a/Dockerfiles/netbox.Dockerfile b/Dockerfiles/netbox.Dockerfile index c513fe26a..e3cce9258 100644 --- a/Dockerfiles/netbox.Dockerfile +++ b/Dockerfiles/netbox.Dockerfile @@ -32,7 +32,7 @@ ENV SUPERCRONIC_CRONTAB "/etc/crontab" ENV NETBOX_INITIALIZERS_VERSION "ebf1f76" -ENV YQ_VERSION "4.33.3" +ENV YQ_VERSION "4.42.1" ENV YQ_URL "https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/yq_linux_amd64" ENV NETBOX_DEVICETYPE_LIBRARY_IMPORT_URL "https://codeload.github.com/netbox-community/Device-Type-Library-Import/tar.gz/develop" @@ -106,6 +106,13 @@ RUN apt-get -q update && \ mkdir -p ./repo && \ curl -sSL "${NETBOX_DEVICETYPE_LIBRARY_URL}" | tar xzf - -C ./repo --strip-components 1 && \ rm -rf ./repo/device-types/WatchGuard && \ + "${NETBOX_PATH}/venv/bin/python" -m pip install --break-system-packages --no-compile --no-cache-dir --upgrade \ + cryptography \ + GitPython \ + Jinja2 \ + "Django>=4.2.10,<5" \ + paramiko \ + pillow && \ mkdir -p "${NETBOX_PATH}/netbox/${BASE_PATH}" && \ mv "${NETBOX_PATH}/netbox/static" "${NETBOX_PATH}/netbox/${BASE_PATH}/static" && \ jq '. += { "settings": { "http": { "discard_unsafe_fields": false } } }' /etc/unit/nginx-unit.json | jq 'del(.listeners."[::]:8080")' | jq 'del(.listeners."[::]:8081")' | jq ".routes.main[0].match.uri = \"/${BASE_PATH}/static/*\"" > /etc/unit/nginx-unit-new.json && \ diff --git a/Dockerfiles/nginx.Dockerfile b/Dockerfiles/nginx.Dockerfile index 5b3bdfd2d..4dd01ec0a 100644 --- a/Dockerfiles/nginx.Dockerfile +++ b/Dockerfiles/nginx.Dockerfile @@ -232,7 +232,9 @@ RUN set -x ; \ rm -rf /usr/src/* /var/tmp/* /var/cache/apk/* /nginx.tar.gz /nginx-auth-ldap.tar.gz /ngx_http_substitutions_filter_module-master.tar.gz; \ touch /etc/nginx/nginx_ldap.conf /etc/nginx/nginx_blank.conf && \ find /usr/share/nginx/html/ -type d -exec chmod 755 "{}" \; && \ - find /usr/share/nginx/html/ -type f -exec chmod 644 "{}" \; + find /usr/share/nginx/html/ -type f -exec chmod 644 "{}" \; && \ + cd /usr/share/nginx/html/assets/img && \ + ln -s ./Malcolm_background.png ./bg-masthead.png COPY --from=docbuild /site/_site /usr/share/nginx/html/readme @@ -242,9 +244,13 @@ ADD nginx/scripts /usr/local/bin/ ADD nginx/*.conf /etc/nginx/ ADD nginx/templates /etc/nginx/templates/ ADD nginx/supervisord.conf /etc/ +COPY --chmod=644 docs/images/favicon/*.png /usr/share/nginx/html/assets/img/ +COPY --chmod=644 docs/images/icon/*.png /usr/share/nginx/html/assets/img/ +COPY --chmod=644 docs/images/icon/*.svg /usr/share/nginx/html/assets/img/ COPY --chmod=644 docs/images/icon/favicon.ico /usr/share/nginx/html/assets/favicon.ico COPY --chmod=644 docs/images/icon/favicon.ico /usr/share/nginx/html/favicon.ico -COPY --chmod=644 docs/images/logo/Malcolm_background.png /usr/share/nginx/html/assets/img/bg-masthead.png +COPY --chmod=644 docs/images/logo/*.png /usr/share/nginx/html/assets/img/ +COPY --chmod=644 docs/images/logo/*.svg /usr/share/nginx/html/assets/img/ VOLUME ["/etc/nginx/certs", "/etc/nginx/dhparam"] diff --git a/Dockerfiles/opensearch.Dockerfile b/Dockerfiles/opensearch.Dockerfile index cb9aa6a66..19694ba80 100644 --- a/Dockerfiles/opensearch.Dockerfile +++ b/Dockerfiles/opensearch.Dockerfile @@ -1,4 +1,4 @@ -FROM opensearchproject/opensearch:2.11.1 +FROM opensearchproject/opensearch:2.12.0 # Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. LABEL maintainer="malcolm@inl.gov" diff --git a/Dockerfiles/suricata.Dockerfile b/Dockerfiles/suricata.Dockerfile index e75aa2d30..6defb77fd 100644 --- a/Dockerfiles/suricata.Dockerfile +++ b/Dockerfiles/suricata.Dockerfile @@ -36,7 +36,7 @@ ENV SUPERCRONIC "supercronic-linux-amd64" ENV SUPERCRONIC_SHA1SUM "cd48d45c4b10f3f0bfdd3a57d054cd05ac96812b" ENV SUPERCRONIC_CRONTAB "/etc/crontab" -ENV YQ_VERSION "4.33.3" +ENV YQ_VERSION "4.42.1" ENV YQ_URL "https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/yq_linux_amd64" ENV SURICATA_CONFIG_DIR /etc/suricata diff --git a/Dockerfiles/zeek.Dockerfile b/Dockerfiles/zeek.Dockerfile index d85956cd8..986454342 100644 --- a/Dockerfiles/zeek.Dockerfile +++ b/Dockerfiles/zeek.Dockerfile @@ -160,8 +160,8 @@ ADD shared/bin/zeekdeploy.sh ${ZEEK_DIR}/bin/ # sanity checks to make sure the plugins installed and copied over correctly # these ENVs should match the number of third party scripts/plugins installed by zeek_install_plugins.sh -ENV ZEEK_THIRD_PARTY_PLUGINS_COUNT 22 -ENV ZEEK_THIRD_PARTY_PLUGINS_GREP "(Zeek::Spicy|ANALYZER_SPICY_DHCP|ANALYZER_SPICY_DNS|ANALYZER_SPICY_HTTP|ANALYZER_SPICY_OSPF|ANALYZER_SPICY_OPENVPN_UDP\b|ANALYZER_SPICY_IPSEC_UDP\b|ANALYZER_SPICY_TFTP|ANALYZER_SPICY_WIREGUARD|ANALYZER_SYNCHROPHASOR_TCP|ANALYZER_GENISYS_TCP|ANALYZER_S7COMM_TCP|Corelight::CommunityID|Corelight::PE_XOR|ICSNPP::BACnet|ICSNPP::BSAP|ICSNPP::ENIP|ICSNPP::ETHERCAT|ICSNPP::OPCUA_Binary|Salesforce::GQUIC|Zeek::PROFINET|Zeek::TDS)" +ENV ZEEK_THIRD_PARTY_PLUGINS_COUNT 23 +ENV ZEEK_THIRD_PARTY_PLUGINS_GREP "(Zeek::Spicy|ANALYZER_SPICY_DHCP|ANALYZER_SPICY_DNS|ANALYZER_SPICY_HTTP|ANALYZER_SPICY_OSPF|ANALYZER_SPICY_OPENVPN_UDP\b|ANALYZER_SPICY_IPSEC_UDP\b|ANALYZER_SPICY_TFTP|ANALYZER_SPICY_WIREGUARD|ANALYZER_SYNCHROPHASOR_TCP|ANALYZER_GENISYS_TCP|ANALYZER_SPICY_PROFINET_IO_CM|ANALYZER_S7COMM_TCP|Corelight::CommunityID|Corelight::PE_XOR|ICSNPP::BACnet|ICSNPP::BSAP|ICSNPP::ENIP|ICSNPP::ETHERCAT|ICSNPP::OPCUA_Binary|Salesforce::GQUIC|Zeek::PROFINET|Zeek::TDS)" ENV ZEEK_THIRD_PARTY_SCRIPTS_COUNT 25 ENV ZEEK_THIRD_PARTY_SCRIPTS_GREP "(bro-is-darknet/main|bro-simple-scan/scan|bzar/main|callstranger-detector/callstranger|cve-2020-0601/cve-2020-0601|cve-2020-13777/cve-2020-13777|CVE-2020-16898/CVE-2020-16898|CVE-2021-38647/omigod|CVE-2021-31166/detect|CVE-2021-41773/CVE_2021_41773|CVE-2021-42292/main|cve-2021-44228/CVE_2021_44228|cve-2022-22954/main|cve-2022-26809/main|CVE-2022-3602/__load__|hassh/hassh|http-more-files-names/main|ja3/ja3|pingback/detect|ripple20/ripple20|SIGRed/CVE-2020-1350|zeek-EternalSafety/main|zeek-httpattacks/main|zeek-sniffpass/__load__|zerologon/main)\.(zeek|bro)" @@ -199,6 +199,7 @@ ARG ZEEK_INTEL_ITEM_EXPIRATION=-1min ARG ZEEK_INTEL_REFRESH_CRON_EXPRESSION= ARG ZEEK_INTEL_REFRESH_THREADS=2 ARG ZEEK_INTEL_FEED_SINCE= +ARG ZEEK_INTEL_FEED_SSL_CERTIFICATE_VERIFICATION=false ARG ZEEK_EXTRACTOR_MODE=none ARG ZEEK_EXTRACTOR_PATH=/zeek/extract_files ARG ZEEK_INTEL_PATH=/opt/zeek/share/zeek/site/intel @@ -222,6 +223,7 @@ ENV ZEEK_INTEL_ITEM_EXPIRATION $ZEEK_INTEL_ITEM_EXPIRATION ENV ZEEK_INTEL_REFRESH_CRON_EXPRESSION $ZEEK_INTEL_REFRESH_CRON_EXPRESSION ENV ZEEK_INTEL_REFRESH_THREADS $ZEEK_INTEL_REFRESH_THREADS ENV ZEEK_INTEL_FEED_SINCE $ZEEK_INTEL_FEED_SINCE +eNV ZEEK_INTEL_FEED_SSL_CERTIFICATE_VERIFICATION $ZEEK_INTEL_FEED_SSL_CERTIFICATE_VERIFICATION ENV ZEEK_EXTRACTOR_MODE $ZEEK_EXTRACTOR_MODE ENV ZEEK_EXTRACTOR_PATH $ZEEK_EXTRACTOR_PATH ENV ZEEK_INTEL_PATH $ZEEK_INTEL_PATH diff --git a/_config.yml b/_config.yml index 0eecae127..86ec09832 100644 --- a/_config.yml +++ b/_config.yml @@ -84,7 +84,7 @@ exclude: - pcap-capture - pcap-monitor - scripts - - sensor-iso + - hedgehog-iso - shared - suricata - suricata-logs diff --git a/api/requirements.txt b/api/requirements.txt index 3f95576d4..462a38cc5 100644 --- a/api/requirements.txt +++ b/api/requirements.txt @@ -5,5 +5,5 @@ opensearch-py==2.4.2 requests==2.31.0 regex==2022.3.2 dateparser==1.1.1 -elasticsearch==8.12.0 +elasticsearch==8.12.1 elasticsearch-dsl==8.12.0 \ No newline at end of file diff --git a/arkime/arkime_regression_test_harness/docker-compose.yml b/arkime/arkime_regression_test_harness/docker-compose.yml index 1e112a7a6..8bd2d5f5b 100644 --- a/arkime/arkime_regression_test_harness/docker-compose.yml +++ b/arkime/arkime_regression_test_harness/docker-compose.yml @@ -7,7 +7,7 @@ services: environment: logger.level : 'INFO' bootstrap.memory_lock : 'true' - OPENSEARCH_JAVA_OPTS : '-Xms4g -Xmx4g -Xss256k -Djava.security.egd=file:/dev/./urandom' + OPENSEARCH_JAVA_OPTS : '-Xmx4g -Xms4g -Xss256k -Djava.security.egd=file:/dev/./urandom' discovery.type : 'single-node' cluster.routing.allocation.disk.threshold_enabled : 'false' cluster.routing.allocation.node_initial_primaries_recoveries : 8 diff --git a/arkime/etc/config.ini b/arkime/etc/config.ini index a34b194b3..e64928886 100644 --- a/arkime/etc/config.ini +++ b/arkime/etc/config.ini @@ -14,7 +14,7 @@ cronQueries=true dropGroup=arkime dropUser=arkime elasticsearch=http://opensearch:9200 -footerTemplate=_version_ | arkime.com 🦉 | Malc⦿lm 📄 | Dashboards 📊 | NetBox 💻 | _responseTime_ms ⏱️ +footerTemplate=_version_ | arkime.com 🦉 | Malc⦿lm 📄 | Dashboards 📊 | NetBox 💻 | _responseTime_ ⏱️ freeSpaceG=10% geoLite2ASN=/opt/arkime/etc/GeoLite2-ASN.mmdb geoLite2Country=/opt/arkime/etc/GeoLite2-Country.mmdb @@ -523,10 +523,10 @@ zeek.ftp.mime_type=db:zeek.ftp.mime_type;group:zeek_ftp;kind:termfield;viewerOnl zeek.ftp.file_size=db:zeek.ftp.file_size;group:zeek_ftp;kind:integer;viewerOnly:true;friendly:File Size;help:File Size zeek.ftp.reply_code=db:zeek.ftp.reply_code;group:zeek_ftp;kind:integer;viewerOnly:true;friendly:Reply Code;help:Reply Code zeek.ftp.reply_msg=db:zeek.ftp.reply_msg;group:zeek_ftp;kind:termfield;viewerOnly:true;friendly:Reply;help:Reply -zeek.ftp.data_channel_passive=db:zeek.ftp.data_channel_passive;group:zeek_ftp;kind:termfield;viewerOnly:true;friendly:Passive;help:Passive -zeek.ftp.data_channel_orig_h=db:zeek.ftp.data_channel_orig_h;group:zeek_ftp;kind:termfield;viewerOnly:true;friendly:Data Originating Host;help:Data Originating Host -zeek.ftp.data_channel_resp_h=db:zeek.ftp.data_channel_resp_h;group:zeek_ftp;kind:termfield;viewerOnly:true;friendly:Data Responding Host;help:Data Responding Host -zeek.ftp.data_channel_resp_p=db:zeek.ftp.data_channel_resp_p;group:zeek_ftp;kind:integer;viewerOnly:true;friendly:Data Responding Port;help:Data Responding Port +zeek.ftp.data_channel.passive=db:zeek.ftp.data_channel.passive;group:zeek_ftp;kind:termfield;viewerOnly:true;friendly:Passive;help:Passive +zeek.ftp.data_channel.orig_h=db:zeek.ftp.data_channel.orig_h;group:zeek_ftp;kind:termfield;viewerOnly:true;friendly:Data Originating Host;help:Data Originating Host +zeek.ftp.data_channel.resp_h=db:zeek.ftp.data_channel.resp_h;group:zeek_ftp;kind:termfield;viewerOnly:true;friendly:Data Responding Host;help:Data Responding Host +zeek.ftp.data_channel.resp_p=db:zeek.ftp.data_channel.resp_p;group:zeek_ftp;kind:integer;viewerOnly:true;friendly:Data Responding Port;help:Data Responding Port # genisys.log # https://github.com/cisagov/icsnpp-genisys @@ -1485,6 +1485,44 @@ zeek.profinet_dce_rpc.activity_uuid=db:zeek.profinet_dce_rpc.activity_uuid;group zeek.profinet_dce_rpc.server_boot_time=db:zeek.profinet_dce_rpc.server_boot_time;group:zeek_profinet;kind:integer;viewerOnly:true;friendly:Server Boot Time;help:Server Boot Time zeek.profinet_dce_rpc.operation=db:zeek.profinet_dce_rpc.operation;group:zeek_profinet;kind:termfield;viewerOnly:true;friendly:Operation;help:Operation +# profinet_io_cm.log +# https://github.com/cisagov/icsnpp-profinet-io-cm +zeek.profinet_io_cm.rpc_version=db:zeek.profinet_io_cm.rpc_version;group:zeek_profinet_io_cm;kind:integer;friendly:RPC version;help:RPC version +zeek.profinet_io_cm.packet_type=db:zeek.profinet_io_cm.packet_type;group:zeek_profinet_io_cm;kind:termfield;friendly:Packet type;help:Packet type +zeek.profinet_io_cm.reserved_for_impl_1=db:zeek.profinet_io_cm.reserved_for_impl_1;group:zeek_profinet_io_cm;kind:termfield;friendly:Implementation flag 0;help:Implementation flag 0 +zeek.profinet_io_cm.last_fragment=db:zeek.profinet_io_cm.last_fragment;group:zeek_profinet_io_cm;kind:termfield;friendly:Last fragment flag;help:Last fragment flag +zeek.profinet_io_cm.fragment=db:zeek.profinet_io_cm.fragment;group:zeek_profinet_io_cm;kind:termfield;friendly:Fragment flag;help:Fragment flag +zeek.profinet_io_cm.no_fragment_requested=db:zeek.profinet_io_cm.no_fragment_requested;group:zeek_profinet_io_cm;kind:termfield;friendly:No fragment ACK req flag;help:No fragment ACK req flag +zeek.profinet_io_cm.maybe=db:zeek.profinet_io_cm.maybe;group:zeek_profinet_io_cm;kind:termfield;friendly:Maybe flag;help:Maybe flag +zeek.profinet_io_cm.idempotent=db:zeek.profinet_io_cm.idempotent;group:zeek_profinet_io_cm;kind:termfield;friendly:Idempotent flag;help:Idempotent flag +zeek.profinet_io_cm.broadcast=db:zeek.profinet_io_cm.broadcast;group:zeek_profinet_io_cm;kind:termfield;friendly:Broadcast flag;help:Broadcast flag +zeek.profinet_io_cm.reserved_for_impl_2=db:zeek.profinet_io_cm.reserved_for_impl_2;group:zeek_profinet_io_cm;kind:termfield;friendly:Implementation flag 7;help:Implementation flag 7 +zeek.profinet_io_cm.cancel_was_pending_at_call_end=db:zeek.profinet_io_cm.cancel_was_pending_at_call_end;group:zeek_profinet_io_cm;kind:termfield;friendly:Cancel pending;help:Cancel pending +zeek.profinet_io_cm.integer_encoding=db:zeek.profinet_io_cm.integer_encoding;group:zeek_profinet_io_cm;kind:termfield;friendly:Integer encoding;help:Integer encoding +zeek.profinet_io_cm.character_encoding=db:zeek.profinet_io_cm.character_encoding;group:zeek_profinet_io_cm;kind:termfield;friendly:Character encoding;help:Character encoding +zeek.profinet_io_cm.floating_point_encoding=db:zeek.profinet_io_cm.floating_point_encoding;group:zeek_profinet_io_cm;kind:termfield;friendly:Floating point encoding;help:Floating point encoding +zeek.profinet_io_cm.serial_high=db:zeek.profinet_io_cm.serial_high;group:zeek_profinet_io_cm;kind:integer;friendly:Fragment number high octet;help:Fragment number high octet +zeek.profinet_io_cm.object_uuid=db:zeek.profinet_io_cm.object_uuid;group:zeek_profinet_io_cm;kind:termfield;friendly:Object UUID;help:Object UUID +zeek.profinet_io_cm.interface_uuid=db:zeek.profinet_io_cm.interface_uuid;group:zeek_profinet_io_cm;kind:termfield;friendly:Interface UUID;help:Interface UUID +zeek.profinet_io_cm.activity_uuid=db:zeek.profinet_io_cm.activity_uuid;group:zeek_profinet_io_cm;kind:termfield;friendly:Activity UUID;help:Activity UUID +zeek.profinet_io_cm.server_boot_time=db:zeek.profinet_io_cm.server_boot_time;group:zeek_profinet_io_cm;kind:integer;friendly:Server boot time;help:Server boot time +zeek.profinet_io_cm.uuid_version=db:zeek.profinet_io_cm.uuid_version;group:zeek_profinet_io_cm;kind:termfield;friendly:Interface version;help:UUID version +zeek.profinet_io_cm.sequence_num=db:zeek.profinet_io_cm.sequence_num;group:zeek_profinet_io_cm;kind:integer;friendly:Activity sequence number;help:Activity sequence number +zeek.profinet_io_cm.operation=db:zeek.profinet_io_cm.operation;group:zeek_profinet_io_cm;kind:termfield;friendly:PNIO operation;help:PNIO operation +zeek.profinet_io_cm.interface_hint=db:zeek.profinet_io_cm.interface_hint;group:zeek_profinet_io_cm;kind:integer;friendly:Interface hint;help:Interface hint +zeek.profinet_io_cm.activity_hint=db:zeek.profinet_io_cm.activity_hint;group:zeek_profinet_io_cm;kind:integer;friendly:Activity hint;help:Activity hint +zeek.profinet_io_cm.len_of_body=db:zeek.profinet_io_cm.len_of_body;group:zeek_profinet_io_cm;kind:integer;friendly:NDRD data length;help:NDRD data length +zeek.profinet_io_cm.fragment_num=db:zeek.profinet_io_cm.fragment_num;group:zeek_profinet_io_cm;kind:integer;friendly:Fragment number;help:Fragment number +zeek.profinet_io_cm.auth_protocol=db:zeek.profinet_io_cm.auth_protocol;group:zeek_profinet_io_cm;kind:integer;friendly:Auth protocol;help:Auth protocol +zeek.profinet_io_cm.serial_low=db:zeek.profinet_io_cm.serial_low;group:zeek_profinet_io_cm;kind:integer;friendly:Fragment number low octet;help:Fragment number low octet +zeek.profinet_io_cm.vers_fack=db:zeek.profinet_io_cm.vers_fack;group:zeek_profinet_io_cm;kind:termfield;friendly:Version FACK;help:Version FACK +zeek.profinet_io_cm.window_size=db:zeek.profinet_io_cm.window_size;group:zeek_profinet_io_cm;kind:integer;friendly:Window size;help:Window size +zeek.profinet_io_cm.max_tsdu=db:zeek.profinet_io_cm.max_tsdu;group:zeek_profinet_io_cm;kind:integer;friendly:Maximum TSDU;help:Maximum TSDU +zeek.profinet_io_cm.max_frag_size=db:zeek.profinet_io_cm.max_frag_size;group:zeek_profinet_io_cm;kind:integer;friendly:Maximum fragment size;help:Maximum fragment size +zeek.profinet_io_cm.serial_number=db:zeek.profinet_io_cm.serial_number;group:zeek_profinet_io_cm;kind:integer;friendly:Serial number;help:Serial number +zeek.profinet_io_cm.sel_ack_len=db:zeek.profinet_io_cm.sel_ack_len;group:zeek_profinet_io_cm;kind:integer;friendly:Selective ACKs length;help:Selective ACKs length +zeek.profinet_io_cm.sel_ack=db:zeek.profinet_io_cm.sel_ack;group:zeek_profinet_io_cm;kind:termfield;friendly:Selective ACKs;help:Selective ACKs + # radius.log # https://docs.zeek.org/en/stable/scripts/base/protocols/radius/main.zeek.html#type-RADIUS::Info zeek.radius.mac=db:zeek.radius.mac;group:zeek_radius;kind:termfield;viewerOnly:true;friendly:MAC Address;help:MAC Address @@ -1628,6 +1666,11 @@ zeek.smb_files.times_changed=db:zeek.smb_files.times_changed;group:zeek_smb;kind zeek.smb_files.data_offset_req=db:zeek.smb_files.data_offset_req;group:zeek_smb;kind:integer;viewerOnly:true;friendly:Data Offset Requested;help:Data Offset Requested zeek.smb_files.data_len_req=db:zeek.smb_files.data_len_req;group:zeek_smb;kind:integer;viewerOnly:true;friendly:Data Length Requested;help:Data Length Requested zeek.smb_files.data_len_rsp=db:zeek.smb_files.data_len_rsp;group:zeek_smb;kind:integer;viewerOnly:true;friendly:Data Length In Response;help:Data Length In Response +zeek.smb_files.ts=db:zeek.smb_files.ts;group:zeek_smb;kind:termfield;viewerOnly:true;friendly:Zeek Timestamp for Referenced File;help:Zeek Timestamp for Referenced File +zeek.smb_files.orig_h=db:zeek.smb_files.orig_h;group:zeek_smb;kind:termfield;viewerOnly:true;friendly:Originating IP for Referenced File;help:Originating IP for Referenced File +zeek.smb_files.orig_p=db:zeek.smb_files.orig_p;group:zeek_smb;kind:integer;viewerOnly:true;friendly:Originating Port for Referenced File;help:Originating Port for Referenced File +zeek.smb_files.resp_h=db:zeek.smb_files.resp_h;group:zeek_smb;kind:termfield;viewerOnly:true;friendly:Responding IP for Referenced File;help:Responding IP for Referenced File +zeek.smb_files.resp_p=db:zeek.smb_files.resp_p;group:zeek_smb;kind:integer;viewerOnly:true;friendly:Responding Port for Referenced File;help:Responding Port for Referenced File # smb_mapping.log # https://docs.zeek.org/en/stable/scripts/base/protocols/smb/main.zeek.html#type-SMB::TreeInfo @@ -1694,6 +1737,7 @@ zeek.software.version_minor2=db:zeek.software.version_minor2;group:zeek_software zeek.software.version_minor3=db:zeek.software.version_minor3;group:zeek_software;kind:integer;viewerOnly:true;friendly:Minor Patch;help:Minor Patch zeek.software.version_addl=db:zeek.software.version_addl;group:zeek_software;kind:termfield;viewerOnly:true;friendly:Additional Version;help:Additional Version zeek.software.unparsed_version=db:zeek.software.unparsed_version;group:zeek_software;kind:termfield;viewerOnly:true;friendly:Version;help:Version +zeek.software.url=db:zeek.software.url;group:zeek_software;kind:termfield;viewerOnly:true;friendly:URL;help:URL # ssh.log # https://docs.zeek.org/en/stable/scripts/base/protocols/ssh/main.zeek.html#type-SSH::Info @@ -1963,10 +2007,9 @@ zeek.tunnel.action=db:zeek.tunnel.action;group:zeek_tunnel;kind:termfield;viewer # weird.log # https://docs.zeek.org/en/stable/scripts/base/frameworks/notice/weird.zeek.html#type-Weird::Info -zeek.weird.name=db:zeek.weird.name;group:zeek_weird;kind:termfield;viewerOnly:true;friendly:Name;help:Name zeek.weird.addl=db:zeek.weird.addl;group:zeek_weird;kind:termfield;viewerOnly:true;friendly:Additional Info;help:Additional Info zeek.weird.notice=db:zeek.weird.notice;group:zeek_weird;kind:termfield;viewerOnly:true;friendly:Notice;help:Generated a notice -zeek.weird.peer=db:zeek.weird.peer;group:zeek_weird;kind:termfield;viewerOnly:true;friendly:Remote Peer;help:Remote Peer +zeek.weird.source=db:zeek.weird.source;group:zeek_weird;kind:termfield;viewerOnly:true;friendly:Weird Source;help:Weird Source # wireguard.log # https://github.com/corelight/zeek-spicy-wireguard/blob/master/analyzer/main.zeek @@ -2617,7 +2660,7 @@ o_zeek_ecat_registers=require:zeek.ecat_registers;title:Zeek ecat_registers.log; o_zeek_ecat_soe_info=require:zeek.ecat_soe_info;title:Zeek ecat_soe_info.log;fields:zeek.ecat_soe_info.opcode,zeek.ecat_soe_info.incomplete,zeek.ecat_soe_info.error,zeek.ecat_soe_info.drive_num,zeek.ecat_soe_info.element,zeek.ecat_soe_info.index o_zeek_enip=require:zeek.enip;title:Zeek enip.log;fields:zeek.enip.enip_command,zeek.enip.enip_command_code,zeek.enip.length,zeek.enip.session_handle,zeek.enip.enip_status,zeek.enip.sender_context,zeek.enip.options o_zeek_files=require:zeek.files;title:Zeek files.log;fields:zeek.files.tx_hosts,zeek.files.rx_hosts,zeek.files.conn_uids,zeek.files.source,zeek.files.depth,zeek.files.analyzers,zeek.files.mime_type,zeek.files.filename,zeek.files.ftime,zeek.files.duration,zeek.files.local_orig,zeek.files.seen_bytes,zeek.files.total_bytes,zeek.files.missing_bytes,zeek.files.overflow_bytes,zeek.files.timedout,zeek.files.parent_fuid,zeek.files.md5,zeek.files.sha1,zeek.files.sha256,zeek.files.extracted,zeek.files.extracted_cutoff,zeek.files.extracted_size -o_zeek_ftp=require:zeek.ftp;title:Zeek ftp.log;fields:zeek.ftp.command,zeek.ftp.arg,zeek.ftp.mime_type,zeek.ftp.file_size,zeek.ftp.reply_code,zeek.ftp.reply_msg,zeek.ftp.data_channel_passive,zeek.ftp.data_channel_orig_h,zeek.ftp.data_channel_resp_h,zeek.ftp.data_channel_resp_p +o_zeek_ftp=require:zeek.ftp;title:Zeek ftp.log;fields:zeek.ftp.command,zeek.ftp.arg,zeek.ftp.mime_type,zeek.ftp.file_size,zeek.ftp.reply_code,zeek.ftp.reply_msg,zeek.ftp.data_channel.passive,zeek.ftp.data_channel.orig_h,zeek.ftp.data_channel.resp_h,zeek.ftp.data_channel.resp_p o_zeek_genisys=require:zeek.genisys;title:Zeek genisys.log;fields:zeek.genisys.header,zeek.genisys.server,zeek.genisys.direction,zeek.genisys.crc_transmitted,zeek.genisys.crc_calculated,zeek.genisys.payload.address,zeek.genisys.payload.data o_zeek_gquic=require:zeek.gquic;title:Zeek gquic.log;fields:zeek.gquic.version,zeek.gquic.server_name,zeek.gquic.user_agent,zeek.gquic.tag_count,zeek.gquic.cyu,zeek.gquic.cyutags o_zeek_http=require:zeek.http;title:Zeek http.log;fields:zeek.http.trans_depth,zeek.http.method,zeek.http.host,zeek.http.uri,zeek.http.origin,zeek.http.post_password_plain,zeek.http.post_username,zeek.http.referrer,zeek.http.version,zeek.http.user_agent,zeek.http.request_body_len,zeek.http.response_body_len,zeek.http.status_code,zeek.http.status_msg,zeek.http.info_code,zeek.http.info_msg,zeek.http.tags,zeek.http.proxied,zeek.http.orig_fuids,zeek.http.orig_filenames,zeek.http.orig_mime_types,zeek.http.resp_fuids,zeek.http.resp_filenames,zeek.http.resp_mime_types @@ -2648,6 +2691,7 @@ o_zeek_ospf=require:zeek.ospf;title:Zeek ospf.log;fields:zeek.ospf.ospf_type,zee o_zeek_pe=require:zeek.pe;title:Zeek pe.log;fields:zeek.pe.machine,zeek.pe.compile_ts,zeek.pe.os,zeek.pe.subsystem,zeek.pe.is_exe,zeek.pe.is_64bit,zeek.pe.uses_aslr,zeek.pe.uses_dep,zeek.pe.uses_code_integrity,zeek.pe.uses_seh,zeek.pe.has_import_table,zeek.pe.has_export_table,zeek.pe.has_cert_table,zeek.pe.has_debug_data,zeek.pe.section_names o_zeek_profinet=require:zeek.profinet;title:Zeek profinet.log;fields:zeek.profinet.operation_type,zeek.profinet.block_version,zeek.profinet.slot_number,zeek.profinet.subslot_number,zeek.profinet.index o_zeek_profinet_dce_rpc=require:zeek.profinet_dce_rpc;title:Zeek profinet_dce_rpc.log;fields:zeek.profinet_dce_rpc.version,zeek.profinet_dce_rpc.packet_type,zeek.profinet_dce_rpc.object_uuid,zeek.profinet_dce_rpc.interface_uuid,zeek.profinet_dce_rpc.activity_uuid,zeek.profinet_dce_rpc.server_boot_time,zeek.profinet_dce_rpc.operation +o_zeek_profinet_io_cm=require:zeek.profinet_io_cm;title:Zeek profinet_io_cm.log;fields:zeek.profinet_io_cm.rpc_version,zeek.profinet_io_cm.packet_type,zeek.profinet_io_cm.reserved_for_impl_1,zeek.profinet_io_cm.last_fragment,zeek.profinet_io_cm.fragment,zeek.profinet_io_cm.no_fragment_requested,zeek.profinet_io_cm.maybe,zeek.profinet_io_cm.idempotent,zeek.profinet_io_cm.broadcast,zeek.profinet_io_cm.reserved_for_impl_2,zeek.profinet_io_cm.cancel_was_pending_at_call_end,zeek.profinet_io_cm.integer_encoding,zeek.profinet_io_cm.character_encoding,zeek.profinet_io_cm.floating_point_encoding,zeek.profinet_io_cm.serial_high,zeek.profinet_io_cm.object_uuid,zeek.profinet_io_cm.interface_uuid,zeek.profinet_io_cm.activity_uuid,zeek.profinet_io_cm.server_boot_time,zeek.profinet_io_cm.uuid_version,zeek.profinet_io_cm.sequence_num,zeek.profinet_io_cm.operation,zeek.profinet_io_cm.interface_hint,zeek.profinet_io_cm.activity_hint,zeek.profinet_io_cm.len_of_body,zeek.profinet_io_cm.fragment_num,zeek.profinet_io_cm.auth_protocol,zeek.profinet_io_cm.serial_low,zeek.profinet_io_cm.vers_fack,zeek.profinet_io_cm.window_size,zeek.profinet_io_cm.max_tsdu,zeek.profinet_io_cm.max_frag_size,zeek.profinet_io_cm.serial_number,zeek.profinet_io_cm.sel_ack_len,zeek.profinet_io_cm.sel_ack o_zeek_radius=require:zeek.radius;title:Zeek radius.log;fields:zeek.radius.mac,zeek.radius.framed_addr,zeek.radius.tunnel_client,zeek.radius.connect_info,zeek.radius.reply_msg,zeek.radius.result,zeek.radius.ttl o_zeek_rdp=require:zeek.rdp;title:Zeek rdp.log;fields:zeek.rdp.cookie,zeek.rdp.result,zeek.rdp.security_protocol,zeek.rdp.client_channels,zeek.rdp.keyboard_layout,zeek.rdp.client_build,zeek.rdp.client_name,zeek.rdp.client_dig_product_id,zeek.rdp.desktop_width,zeek.rdp.desktop_height,zeek.rdp.requested_color_depth,zeek.rdp.cert_type,zeek.rdp.cert_count,zeek.rdp.cert_permanent,zeek.rdp.encryption_level,zeek.rdp.encryption_method o_zeek_rfb=require:zeek.rfb;title:Zeek rfb.log;fields:zeek.rfb.client_major_version,zeek.rfb.client_minor_version,zeek.rfb.server_major_version,zeek.rfb.server_minor_version,zeek.rfb.authentication_method,zeek.rfb.auth,zeek.rfb.share_flag,zeek.rfb.desktop_name,zeek.rfb.width,zeek.rfb.height @@ -2658,12 +2702,12 @@ o_zeek_s7comm_upload_download=require:zeek.s7comm_upload_download;title:Zeek s7c o_zeek_signatures=require:zeek.signatures;title:Zeek signatures.log;fields:event.module,rule.category,rule.name,vulnerability.category,vulnerability.enumeration,vulnerability.id,zeek.signatures.sub_message,zeek.signatures.signature_count,zeek.signatures.host_count o_zeek_sip=require:zeek.sip;title:Zeek sip.log;fields:zeek.sip.trans_depth,zeek.sip.method,zeek.sip.uri,zeek.sip.date,zeek.sip.request_from,zeek.sip.request_to,zeek.sip.response_from,zeek.sip.response_to,zeek.sip.reply_to,zeek.sip.call_id,zeek.sip.seq,zeek.sip.subject,zeek.sip.request_path,zeek.sip.response_path,zeek.sip.user_agent,zeek.sip.status_code,zeek.sip.status_msg,zeek.sip.warning,zeek.sip.request_body_len,zeek.sip.response_body_len,zeek.sip.content_type,zeek.sip.version o_zeek_smb_cmd=require:zeek.smb_cmd;title:Zeek smb_cmd.log;fields:zeek.smb_cmd.command,zeek.smb_cmd.sub_command,zeek.smb_cmd.argument,zeek.smb_cmd.status,zeek.smb_cmd.rtt,zeek.smb_cmd.version,zeek.smb_cmd.user,zeek.smb_cmd.tree,zeek.smb_cmd.tree_service -o_zeek_smb_files=require:zeek.smb_files;title:Zeek smb_files.log;fields:zeek.smb_files.action,zeek.smb_files.path,zeek.smb_files.name,zeek.smb_files.size,zeek.smb_files.prev_name,zeek.smb_files.times_modified,zeek.smb_files.times_accessed,zeek.smb_files.times_created,zeek.smb_files.times_changed,zeek.smb_files.data_offset_req,zeek.smb_files.data_len_req,zeek.smb_files.data_len_rsp +o_zeek_smb_files=require:zeek.smb_files;title:Zeek smb_files.log;fields:zeek.smb_files.action,zeek.smb_files.path,zeek.smb_files.name,zeek.smb_files.size,zeek.smb_files.prev_name,zeek.smb_files.times_modified,zeek.smb_files.times_accessed,zeek.smb_files.times_created,zeek.smb_files.times_changed,zeek.smb_files.data_offset_req,zeek.smb_files.data_len_req,zeek.smb_files.data_len_rsp,zeek.smb_files.ts,zeek.smb_files.orig_h,zeek.smb_files.orig_p,zeek.smb_files.resp_h,zeek.smb_files.resp_p o_zeek_smb_mapping=require:zeek.smb_mapping;title:Zeek smb_mapping.log;fields:zeek.smb_mapping.path,zeek.smb_mapping.resource_type,zeek.smb_mapping.native_file_system,zeek.smb_mapping.share_type o_zeek_smtp=require:zeek.smtp;title:Zeek smtp.log;fields:zeek.smtp.trans_depth,zeek.smtp.helo,zeek.smtp.mailfrom,zeek.smtp.rcptto,zeek.smtp.date,zeek.smtp.from,zeek.smtp.to,zeek.smtp.cc,zeek.smtp.reply_to,zeek.smtp.msg_id,zeek.smtp.in_reply_to,zeek.smtp.subject,zeek.smtp.x_originating_ip,zeek.smtp.first_received,zeek.smtp.second_received,zeek.smtp.last_reply,zeek.smtp.last_reply_code,zeek.smtp.last_reply_msg,zeek.smtp.path,zeek.smtp.user_agent,zeek.smtp.tls,zeek.smtp.is_webmail o_zeek_snmp=require:zeek.snmp;title:Zeek snmp.log;fields:zeek.snmp.duration,zeek.snmp.version,zeek.snmp.community,zeek.snmp.get_requests,zeek.snmp.get_bulk_requests,zeek.snmp.get_responses,zeek.snmp.set_requests,zeek.snmp.display_string,zeek.snmp.up_since o_zeek_socks=require:zeek.socks;title:Zeek socks.log;fields:zeek.socks.version,zeek.socks.server_status,zeek.socks.request_host,zeek.socks.request_name,zeek.socks.request_port,zeek.socks.bound_host,zeek.socks.bound_name,zeek.socks.bound_port -o_zeek_software=require:zeek.software;title:Zeek software.log;fields:zeek.software.software_type,zeek.software.name,zeek.software.version_major,zeek.software.version_minor,zeek.software.version_minor2,zeek.software.version_minor3,zeek.software.version_addl,zeek.software.unparsed_version +o_zeek_software=require:zeek.software;title:Zeek software.log;fields:zeek.software.software_type,zeek.software.name,zeek.software.version_major,zeek.software.version_minor,zeek.software.version_minor2,zeek.software.version_minor3,zeek.software.version_addl,zeek.software.unparsed_version,zeek.software.url o_zeek_ssh=require:zeek.ssh;title:Zeek ssh.log;fields:zeek.ssh.version,zeek.ssh.auth_success,zeek.ssh.auth_attempts,zeek.ssh.direction,zeek.ssh.client,zeek.ssh.server,zeek.ssh.cipher_alg,zeek.ssh.mac_alg,zeek.ssh.compression_alg,zeek.ssh.kex_alg,zeek.ssh.host_key_alg,zeek.ssh.host_key,zeek.ssh.remote_location_country_code,zeek.ssh.remote_location_region,zeek.ssh.remote_location_city,zeek.ssh.remote_location_latitude,zeek.ssh.remote_location_longitude,zeek.ssh.hassh,zeek.ssh.hasshServer,zeek.ssh.hasshAlgorithms,zeek.ssh.hasshServerAlgorithms,zeek.ssh.cshka,zeek.ssh.sshka o_zeek_ssl=require:zeek.ssl;title:Zeek ssl.log;fields:zeek.ssl.ssl_version,zeek.ssl.ssl_history,zeek.ssl.sni_matches_cert,zeek.ssl.cipher,zeek.ssl.curve,zeek.ssl.server_name,zeek.ssl.resumed,zeek.ssl.last_alert,zeek.ssl.next_protocol,zeek.ssl.established,tls.client.ja3,tls.client.ja3_description,tls.server.ja3s,tls.server.ja3s_description,zeek.ssl.cert_chain_fuids,zeek.ssl.cert_chain_fps,zeek.ssl.client_cert_chain_fuids,zeek.ssl.client_cert_chain_fps,zeek.ssl.subject.CN,zeek.ssl.subject.C,zeek.ssl.subject.O,zeek.ssl.subject.OU,zeek.ssl.subject.ST,zeek.ssl.subject.SN,zeek.ssl.subject.L,zeek.ssl.subject.GN,zeek.ssl.subject.pseudonym,zeek.ssl.subject.serialNumber,zeek.ssl.subject.title,zeek.ssl.subject.initials,zeek.ssl.subject.emailAddress,zeek.ssl.subject.description,zeek.ssl.subject.postalCode,zeek.ssl.subject.street,zeek.ssl.client_subject.CN,zeek.ssl.client_subject.C,zeek.ssl.client_subject.O,zeek.ssl.client_subject.OU,zeek.ssl.client_subject.ST,zeek.ssl.client_subject.SN,zeek.ssl.client_subject.L,zeek.ssl.client_subject.GN,zeek.ssl.client_subject.pseudonym,zeek.ssl.client_subject.serialNumber,zeek.ssl.client_subject.title,zeek.ssl.client_subject.initials,zeek.ssl.client_subject.emailAddress,zeek.ssl.issuer.CN,zeek.ssl.issuer.C,zeek.ssl.issuer.O,zeek.ssl.issuer.OU,zeek.ssl.issuer.ST,zeek.ssl.issuer.SN,zeek.ssl.issuer.L,zeek.ssl.issuer.DC,zeek.ssl.issuer.GN,zeek.ssl.issuer.pseudonym,zeek.ssl.issuer.serialNumber,zeek.ssl.issuer.title,zeek.ssl.issuer.initials,zeek.ssl.issuer.emailAddress,zeek.ssl.client_issuer.CN,zeek.ssl.client_issuer.C,zeek.ssl.client_issuer.O,zeek.ssl.client_issuer.OU,zeek.ssl.client_issuer.ST,zeek.ssl.client_issuer.SN,zeek.ssl.client_issuer.L,zeek.ssl.client_issuer.DC,zeek.ssl.client_issuer.GN,zeek.ssl.client_issuer.pseudonym,zeek.ssl.client_issuer.serialNumber,zeek.ssl.client_issuer.title,zeek.ssl.client_issuer.initials,zeek.ssl.client_issuer.emailAddress,zeek.ssl.validation_status o_zeek_stun=require:zeek.stun;title:Zeek stun.log;fields:zeek.stun.trans_id,zeek.stun.method,zeek.stun.class,zeek.stun.attr_type,zeek.stun.attr_val @@ -2681,7 +2725,7 @@ o_zeek_tds_rpc=require:zeek.tds_rpc;title:Zeek tds_rpc.log;fields:zeek.tds_rpc.p o_zeek_tds_sql_batch=require:zeek.tds_sql_batch;title:Zeek tds_sql_batch.log;fields:zeek.tds_sql_batch.header_type,zeek.tds_sql_batch.query o_zeek_tftp=require:zeek.tftp;title:Zeek tftp.log;fields:zeek.tftp.block_acked,zeek.tftp.block_sent,zeek.tftp.error_code,zeek.tftp.error_msg,zeek.tftp.fname,zeek.tftp.mode,zeek.tftp.size,zeek.tftp.uid_data,zeek.tftp.wrq o_zeek_tunnel=require:zeek.tunnel;title:Zeek tunnel.log;fields:zeek.tunnel.tunnel_type,zeek.tunnel.action -o_zeek_weird=require:zeek.weird;title:Zeek weird.log;fields:rule.name,zeek.weird.addl,zeek.weird.notice +o_zeek_weird=require:zeek.weird;title:Zeek weird.log;fields:rule.name,zeek.weird.addl,zeek.weird.notice,zeek.weird.source o_zeek_wireguard=require:zeek.wireguard;title:Zeek wireguard.log;fields:zeek.wireguard.established,zeek.wireguard.initiations,zeek.wireguard.responses o_zeek_x509=require:zeek.x509;title:Zeek x509.log;fields:zeek.x509.certificate_version,zeek.x509.certificate_serial,zeek.x509.certificate_subject.CN,zeek.x509.certificate_subject.C,zeek.x509.certificate_subject.O,zeek.x509.certificate_subject.OU,zeek.x509.certificate_subject.ST,zeek.x509.certificate_subject.SN,zeek.x509.certificate_subject.L,zeek.x509.certificate_subject.DC,zeek.x509.certificate_subject.GN,zeek.x509.certificate_subject.pseudonym,zeek.x509.certificate_subject.serialNumber,zeek.x509.certificate_subject.title,zeek.x509.certificate_subject.initials,zeek.x509.certificate_subject.emailAddress,zeek.x509.certificate_subject.description,zeek.x509.certificate_subject.postalCode,zeek.x509.certificate_subject.street,zeek.x509.certificate_issuer.CN,zeek.x509.certificate_issuer.DC,zeek.x509.certificate_issuer.C,zeek.x509.certificate_issuer.O,zeek.x509.certificate_issuer.OU,zeek.x509.certificate_issuer.ST,zeek.x509.certificate_issuer.SN,zeek.x509.certificate_issuer.L,zeek.x509.certificate_issuer.GN,zeek.x509.certificate_issuer.pseudonym,zeek.x509.certificate_issuer.serialNumber,zeek.x509.certificate_issuer.title,zeek.x509.certificate_issuer.initials,zeek.x509.certificate_issuer.emailAddress,zeek.x509.certificate_not_valid_before,zeek.x509.certificate_not_valid_after,zeek.x509.certificate_key_alg,zeek.x509.certificate_sig_alg,zeek.x509.certificate_key_type,zeek.x509.certificate_key_length,zeek.x509.certificate_exponent,zeek.x509.certificate_curve,zeek.x509.client_cert,zeek.x509.fingerprint,zeek.x509.host_cert,zeek.x509.san_dns,zeek.x509.san_uri,zeek.x509.san_email,zeek.x509.san_ip,zeek.x509.basic_constraints_ca,zeek.x509.basic_constraints_path_len p_suricata=require:suricata.timestamp;title:Suricata Common Fields;fields:suricata.action,suricata.app_proto_expected,suricata.app_proto_orig,suricata.app_proto_tc,suricata.app_proto_ts,suricata.category,suricata.flow_id,suricata.pcap_cnt,suricata.pcap_filename,suricata.severity,suricata.signature,suricata.stream,suricata.timestamp,suricata.tx_id diff --git a/arkime/wise/source.zeeklogs.js b/arkime/wise/source.zeeklogs.js index 9e30b2a30..140ec922f 100644 --- a/arkime/wise/source.zeeklogs.js +++ b/arkime/wise/source.zeeklogs.js @@ -749,15 +749,15 @@ class MalcolmSource extends WISESource { "vulnerability.reference", "vulnerability.scanner.vendor", "zeek.bacnet.bvlc_function", - "zeek.bacnet.invoke_id", "zeek.bacnet.instance_number", + "zeek.bacnet.invoke_id", "zeek.bacnet.pdu_service", "zeek.bacnet.pdu_type", "zeek.bacnet.result_code", - "zeek.bacnet_device_control.time_duration", "zeek.bacnet_device_control.device_state", "zeek.bacnet_device_control.result", "zeek.bacnet_device_control.result_code", + "zeek.bacnet_device_control.time_duration", "zeek.bacnet_discovery.object_name", "zeek.bacnet_discovery.object_type", "zeek.bacnet_discovery.range", @@ -983,10 +983,10 @@ class MalcolmSource extends WISESource { "zeek.files.tx_hosts", "zeek.ftp.arg", "zeek.ftp.command", - "zeek.ftp.data_channel_orig_h", - "zeek.ftp.data_channel_passive", - "zeek.ftp.data_channel_resp_h", - "zeek.ftp.data_channel_resp_p", + "zeek.ftp.data_channel.orig_h", + "zeek.ftp.data_channel.passive", + "zeek.ftp.data_channel.resp_h", + "zeek.ftp.data_channel.resp_p", "zeek.ftp.file_size", "zeek.ftp.mime_type", "zeek.ftp.reply_code", @@ -1114,20 +1114,20 @@ class MalcolmSource extends WISESource { "zeek.login.success", "zeek.modbus.exception", "zeek.modbus.func", + "zeek.modbus.mei_type", "zeek.modbus.network_direction", "zeek.modbus.trans_id", "zeek.modbus.unit_id", - "zeek.modbus.mei_type", "zeek.modbus_detailed.address", "zeek.modbus_detailed.quantity", "zeek.modbus_detailed.values", "zeek.modbus_mask_write_register.and_mask", "zeek.modbus_mask_write_register.or_mask", - "zeek.modbus_read_device_identification.conformity_level_code", "zeek.modbus_read_device_identification.conformity_level", + "zeek.modbus_read_device_identification.conformity_level_code", "zeek.modbus_read_device_identification.device_id_code", - "zeek.modbus_read_device_identification.object_id_code", "zeek.modbus_read_device_identification.object_id", + "zeek.modbus_read_device_identification.object_id_code", "zeek.modbus_read_device_identification.object_value", "zeek.modbus_read_write_multiple_registers.read_quantity", "zeek.modbus_read_write_multiple_registers.read_registers", @@ -1719,6 +1719,41 @@ class MalcolmSource extends WISESource { "zeek.profinet_dce_rpc.packet_type", "zeek.profinet_dce_rpc.server_boot_time", "zeek.profinet_dce_rpc.version", + "zeek.profinet_io_cm.rpc_version", + "zeek.profinet_io_cm.packet_type", + "zeek.profinet_io_cm.reserved_for_impl_1", + "zeek.profinet_io_cm.last_fragment", + "zeek.profinet_io_cm.fragment", + "zeek.profinet_io_cm.no_fragment_requested", + "zeek.profinet_io_cm.maybe", + "zeek.profinet_io_cm.idempotent", + "zeek.profinet_io_cm.broadcast", + "zeek.profinet_io_cm.reserved_for_impl_2", + "zeek.profinet_io_cm.cancel_was_pending_at_call_end", + "zeek.profinet_io_cm.integer_encoding", + "zeek.profinet_io_cm.character_encoding", + "zeek.profinet_io_cm.floating_point_encoding", + "zeek.profinet_io_cm.serial_high", + "zeek.profinet_io_cm.object_uuid", + "zeek.profinet_io_cm.interface_uuid", + "zeek.profinet_io_cm.activity_uuid", + "zeek.profinet_io_cm.server_boot_time", + "zeek.profinet_io_cm.uuid_version", + "zeek.profinet_io_cm.sequence_num", + "zeek.profinet_io_cm.operation_num", + "zeek.profinet_io_cm.interface_hint", + "zeek.profinet_io_cm.activity_hint", + "zeek.profinet_io_cm.len_of_body", + "zeek.profinet_io_cm.fragment_num", + "zeek.profinet_io_cm.auth_protocol", + "zeek.profinet_io_cm.serial_low", + "zeek.profinet_io_cm.vers_fack", + "zeek.profinet_io_cm.window_size", + "zeek.profinet_io_cm.max_tsdu", + "zeek.profinet_io_cm.max_frag_size", + "zeek.profinet_io_cm.serial_number", + "zeek.profinet_io_cm.sel_ack_len", + "zeek.profinet_io_cm.sel_ack", "zeek.radius.connect_info", "zeek.radius.framed_addr", "zeek.radius.mac", @@ -1826,13 +1861,18 @@ class MalcolmSource extends WISESource { "zeek.smb_files.data_len_rsp", "zeek.smb_files.data_offset_req", "zeek.smb_files.name", + "zeek.smb_files.orig_h", + "zeek.smb_files.orig_p", "zeek.smb_files.path", "zeek.smb_files.prev_name", + "zeek.smb_files.resp_h", + "zeek.smb_files.resp_p", "zeek.smb_files.size", "zeek.smb_files.times_accessed", "zeek.smb_files.times_changed", "zeek.smb_files.times_created", "zeek.smb_files.times_modified", + "zeek.smb_files.ts", "zeek.smb_mapping.native_file_system", "zeek.smb_mapping.path", "zeek.smb_mapping.resource_type", @@ -2105,6 +2145,7 @@ class MalcolmSource extends WISESource { "zeek.uid", "zeek.weird.addl", "zeek.weird.notice", + "zeek.weird.source", "zeek.wireguard.established", "zeek.wireguard.initiations", "zeek.wireguard.receiver_index", @@ -2213,7 +2254,7 @@ class MalcolmSource extends WISESource { // add right-click for extracted files from zeek var carvedFieldsStr = allFields.filter(value => /^zeek\.files\.extracted$/i.test(value)).join(','); - this.api.addValueAction("malcolm_carved_file_quarantined", { name: "Download", url: "/dl-extracted-files/%TEXT%", fields: carvedFieldsStr }); + this.api.addValueAction("malcolm_carved_file_quarantined", { name: "Download", url: "/extracted-files/%TEXT%", fields: carvedFieldsStr }); // add right-clicks for pivoting into dashboards from Arkime (see nginx.conf) var filterLabel = "OpenSearch Dashboards %DBFIELD%"; diff --git a/config/logstash.env.example b/config/logstash.env.example index a9436ad51..f734bf17e 100644 --- a/config/logstash.env.example +++ b/config/logstash.env.example @@ -19,6 +19,6 @@ LOGSTASH_NETBOX_AUTO_POPULATE=false LOGSTASH_NETBOX_CACHE_SIZE=1000 LOGSTASH_NETBOX_CACHE_TTL=30 # Zeek log types that will be ignored (dropped) by LogStash -LOGSTASH_ZEEK_IGNORED_LOGS=analyzer,broker,bsap_ip_unknown,bsap_serial_unknown,capture_loss,cluster,config,ecat_arp_info,loaded_scripts,packet_filter,png,print,prof,reporter,stats,stderr,stdout +LOGSTASH_ZEEK_IGNORED_LOGS=analyzer,broker,capture_loss,cluster,config,loaded_scripts,packet_filter,png,print,prof,reporter,stats,stderr,stdout # Logstash memory allowance and other Java options -LS_JAVA_OPTS=-server -Xms2500m -Xmx2500m -Xss1536k -XX:-HeapDumpOnOutOfMemoryError -Djava.security.egd=file:/dev/./urandom -Dlog4j.formatMsgNoLookups=true \ No newline at end of file +LS_JAVA_OPTS=-server -Xmx2500m -Xms2500m -Xss1536k -XX:-HeapDumpOnOutOfMemoryError -Djava.security.egd=file:/dev/./urandom -Dlog4j.formatMsgNoLookups=true \ No newline at end of file diff --git a/config/opensearch.env.example b/config/opensearch.env.example index 95408c7f4..820854a40 100644 --- a/config/opensearch.env.example +++ b/config/opensearch.env.example @@ -39,7 +39,7 @@ OPENSEARCH_SECONDARY_CREDS_CONFIG_FILE=/var/local/curlrc/.opensearch.secondary.c # certificates). OPENSEARCH_SECONDARY_SSL_CERTIFICATE_VERIFICATION=false # OpenSearch memory allowance and other Java options -OPENSEARCH_JAVA_OPTS=-server -Xms10g -Xmx10g -Xss256k -XX:-HeapDumpOnOutOfMemoryError -Djava.security.egd=file:/dev/./urandom -Dlog4j.formatMsgNoLookups=true +OPENSEARCH_JAVA_OPTS=-server -Xmx10g -Xms10g -Xss256k -XX:-HeapDumpOnOutOfMemoryError -Djava.security.egd=file:/dev/./urandom -Dlog4j.formatMsgNoLookups=true # OpenSearch index patterns and timestamp fields # Index pattern for network traffic logs written via Logstash (e.g., Zeek logs, Suricata alerts) diff --git a/config/zeek-live.env.example b/config/zeek-live.env.example index 944833e85..abf8c7a4d 100644 --- a/config/zeek-live.env.example +++ b/config/zeek-live.env.example @@ -1,7 +1,7 @@ # Whether or not Zeek should monitor live traffic on a local # interface (PCAP_IFACE variable below specifies capture interfaces) ZEEK_LIVE_CAPTURE=false -# Set ZEEK_DISABLE_STATS to blank to generate stats.log and capture_loss.log +# Set ZEEK_DISABLE_STATS to false or blank to generate stats.log and capture_loss.log ZEEK_DISABLE_STATS=true ZEEK_PCAP_PROCESSOR=false diff --git a/config/zeek.env.example b/config/zeek.env.example index 966a1bfa2..400362120 100644 --- a/config/zeek.env.example +++ b/config/zeek.env.example @@ -3,12 +3,16 @@ # Note that by default, Zeek considers IANA-registered private address space # such as 10/8 and 192.168/16 site-local. ZEEK_LOCAL_NETS= +# Set to true to indicate that Zeek should output logs in JSON format +ZEEK_JSON= # Specifies the value for Zeek's Intel::item_expiration timeout (-1min to disable) ZEEK_INTEL_ITEM_EXPIRATION=-1min # When querying a TAXII or MISP feed, only process threat indicators that have # been created or modified since the time represented by this value; # it may be either a fixed date/time (01/01/2021) or relative interval (30 days ago) ZEEK_INTEL_FEED_SINCE= +# Whether or not to require SSL certificate verification when querying a TAXII or MISP feed +ZEEK_INTEL_FEED_SSL_CERTIFICATE_VERIFICATION=false # Specifies a cron expression indicating the refresh interval for generating the # Zeek Intelligence Framework files ('' disables automatic refresh) ZEEK_INTEL_REFRESH_CRON_EXPRESSION= @@ -56,7 +60,7 @@ EXTRACTED_FILE_HTTP_SERVER_MAGIC=false # HTTP server will look in subdirectories for requested filename (e.g., in "/quarantined" and "/preserved") EXTRACTED_FILE_HTTP_SERVER_RECURSIVE=true # Environment variables for tweaking Zeek at runtime (see local.zeek) -# Set to any non-blank value to disable the corresponding feature +# Set to true to disable the corresponding feature ZEEK_DISABLE_HASH_ALL_FILES= ZEEK_DISABLE_LOG_PASSWORDS= ZEEK_DISABLE_SSL_VALIDATE_CERTS= @@ -82,6 +86,7 @@ ZEEK_DISABLE_ICS_GENISYS=true ZEEK_DISABLE_ICS_OPCUA_BINARY= ZEEK_DISABLE_ICS_MODBUS= ZEEK_DISABLE_ICS_PROFINET= +ZEEK_DISABLE_ICS_PROFINET_IO_CM= ZEEK_DISABLE_ICS_S7COMM= ZEEK_DISABLE_ICS_SYNCHROPHASOR= ZEEK_SYNCHROPHASOR_PORTS= diff --git a/dashboards/dashboards/152f29dc-51a2-4f53-93e9-6e92765567b8.json b/dashboards/dashboards/152f29dc-51a2-4f53-93e9-6e92765567b8.json index 9ccb3d124..3fe16806b 100644 --- a/dashboards/dashboards/152f29dc-51a2-4f53-93e9-6e92765567b8.json +++ b/dashboards/dashboards/152f29dc-51a2-4f53-93e9-6e92765567b8.json @@ -710,7 +710,7 @@ ], "version": 1, "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"highlightAll\":true,\"version\":true,\"query\":{\"query\":\"event.dataset:modbus_read_device_identification\",\"language\":\"lucene\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" + "searchSourceJSON": "{\"highlightAll\":false,\"version\":true,\"query\":{\"query\":\"event.dataset:modbus_read_device_identification\",\"language\":\"lucene\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" } }, "references": [ diff --git a/dashboards/dashboards/a7514350-eba6-11e9-a384-0fcf32210194.json b/dashboards/dashboards/a7514350-eba6-11e9-a384-0fcf32210194.json index cafab2e32..c51cbcd3b 100644 --- a/dashboards/dashboards/a7514350-eba6-11e9-a384-0fcf32210194.json +++ b/dashboards/dashboards/a7514350-eba6-11e9-a384-0fcf32210194.json @@ -1,5 +1,5 @@ { - "version": "7.10.0", + "version": "2.12.0", "objects": [ { "id": "a7514350-eba6-11e9-a384-0fcf32210194", @@ -7,18 +7,18 @@ "namespaces": [ "default" ], - "updated_at": "2021-02-10T21:24:49.327Z", - "version": "WzY2MSwxXQ==", + "updated_at": "2024-02-27T18:15:37.621Z", + "version": "Wzk5MywxXQ==", "attributes": { "title": "PROFINET", "hits": 0, "description": "", - "panelsJSON": "[{\"version\":\"7.6.2\",\"gridData\":{\"x\":0,\"y\":0,\"w\":8,\"h\":69,\"i\":\"1\"},\"panelIndex\":\"1\",\"embeddableConfig\":{},\"panelRefName\":\"panel_0\"},{\"version\":\"7.6.2\",\"gridData\":{\"x\":8,\"y\":0,\"w\":8,\"h\":14,\"i\":\"2\"},\"panelIndex\":\"2\",\"embeddableConfig\":{},\"panelRefName\":\"panel_1\"},{\"version\":\"7.6.2\",\"gridData\":{\"x\":16,\"y\":0,\"w\":32,\"h\":14,\"i\":\"3\"},\"panelIndex\":\"3\",\"embeddableConfig\":{},\"panelRefName\":\"panel_2\"},{\"version\":\"7.6.2\",\"gridData\":{\"x\":0,\"y\":69,\"w\":48,\"h\":34,\"i\":\"4\"},\"panelIndex\":\"4\",\"embeddableConfig\":{},\"panelRefName\":\"panel_3\"},{\"version\":\"7.6.2\",\"gridData\":{\"x\":8,\"y\":50,\"w\":19,\"h\":19,\"i\":\"5\"},\"panelIndex\":\"5\",\"embeddableConfig\":{},\"panelRefName\":\"panel_4\"},{\"version\":\"7.6.2\",\"gridData\":{\"x\":27,\"y\":50,\"w\":21,\"h\":19,\"i\":\"6\"},\"panelIndex\":\"6\",\"embeddableConfig\":{},\"panelRefName\":\"panel_5\"},{\"version\":\"7.6.2\",\"gridData\":{\"x\":8,\"y\":14,\"w\":16,\"h\":19,\"i\":\"7\"},\"panelIndex\":\"7\",\"embeddableConfig\":{},\"panelRefName\":\"panel_6\"},{\"version\":\"7.6.2\",\"gridData\":{\"x\":24,\"y\":14,\"w\":24,\"h\":19,\"i\":\"8\"},\"panelIndex\":\"8\",\"embeddableConfig\":{},\"panelRefName\":\"panel_7\"},{\"version\":\"7.6.2\",\"gridData\":{\"x\":8,\"y\":33,\"w\":19,\"h\":17,\"i\":\"9\"},\"panelIndex\":\"9\",\"embeddableConfig\":{},\"panelRefName\":\"panel_8\"},{\"version\":\"7.6.2\",\"gridData\":{\"x\":27,\"y\":33,\"w\":21,\"h\":17,\"i\":\"10\"},\"panelIndex\":\"10\",\"embeddableConfig\":{},\"panelRefName\":\"panel_9\"}]", + "panelsJSON": "[{\"version\":\"2.12.0\",\"gridData\":{\"x\":0,\"y\":0,\"w\":8,\"h\":33,\"i\":\"1\"},\"panelIndex\":\"1\",\"embeddableConfig\":{},\"panelRefName\":\"panel_0\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":8,\"y\":0,\"w\":8,\"h\":14,\"i\":\"2\"},\"panelIndex\":\"2\",\"embeddableConfig\":{},\"panelRefName\":\"panel_1\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":16,\"y\":0,\"w\":32,\"h\":14,\"i\":\"3\"},\"panelIndex\":\"3\",\"embeddableConfig\":{},\"panelRefName\":\"panel_2\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":0,\"y\":33,\"w\":12,\"h\":34,\"i\":\"5\"},\"panelIndex\":\"5\",\"embeddableConfig\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":2,\"direction\":\"desc\"}},\"sortColumn\":{\"colIndex\":2,\"direction\":\"desc\"}}},\"panelRefName\":\"panel_3\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":12,\"y\":33,\"w\":12,\"h\":34,\"i\":\"6\"},\"panelIndex\":\"6\",\"embeddableConfig\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":2,\"direction\":\"desc\"}},\"sortColumn\":{\"colIndex\":2,\"direction\":\"desc\"}}},\"panelRefName\":\"panel_4\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":8,\"y\":14,\"w\":16,\"h\":19,\"i\":\"7\"},\"panelIndex\":\"7\",\"embeddableConfig\":{},\"panelRefName\":\"panel_5\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":24,\"y\":14,\"w\":24,\"h\":19,\"i\":\"8\"},\"panelIndex\":\"8\",\"embeddableConfig\":{},\"panelRefName\":\"panel_6\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":24,\"y\":33,\"w\":24,\"h\":34,\"i\":\"f4ece1af-6321-41fb-91fd-187f98ab9098\"},\"panelIndex\":\"f4ece1af-6321-41fb-91fd-187f98ab9098\",\"embeddableConfig\":{},\"panelRefName\":\"panel_7\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":0,\"y\":67,\"w\":48,\"h\":31,\"i\":\"76674c9f-4ea3-40cb-8c76-a8aa4d77677a\"},\"panelIndex\":\"76674c9f-4ea3-40cb-8c76-a8aa4d77677a\",\"embeddableConfig\":{},\"panelRefName\":\"panel_8\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":0,\"y\":98,\"w\":48,\"h\":28,\"i\":\"5827d746-068a-41b6-afa8-4c93ad914dae\"},\"panelIndex\":\"5827d746-068a-41b6-afa8-4c93ad914dae\",\"embeddableConfig\":{},\"panelRefName\":\"panel_9\"}]", "optionsJSON": "{\"useMargins\":true}", "version": 1, "timeRestore": false, "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"filter\":[],\"highlightAll\":false,\"version\":true,\"query\":{\"language\":\"lucene\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"default_field\":\"*\",\"query\":\"*\"}}}}" + "searchSourceJSON": "{\"highlightAll\":false,\"version\":true,\"query\":{\"language\":\"lucene\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"default_field\":\"*\",\"query\":\"*\",\"time_zone\":\"America/Denver\"}}},\"filter\":[]}" } }, "references": [ @@ -39,38 +39,38 @@ }, { "name": "panel_3", - "type": "search", - "id": "a0a10870-eba5-11e9-a384-0fcf32210194" + "type": "visualization", + "id": "ec42baa0-eba8-11e9-a384-0fcf32210194" }, { "name": "panel_4", "type": "visualization", - "id": "ec42baa0-eba8-11e9-a384-0fcf32210194" + "id": "0957f330-eba9-11e9-a384-0fcf32210194" }, { "name": "panel_5", "type": "visualization", - "id": "0957f330-eba9-11e9-a384-0fcf32210194" + "id": "41f36a70-ebaa-11e9-a384-0fcf32210194" }, { "name": "panel_6", "type": "visualization", - "id": "41f36a70-ebaa-11e9-a384-0fcf32210194" + "id": "9dccb5f0-eba9-11e9-a384-0fcf32210194" }, { "name": "panel_7", "type": "visualization", - "id": "9dccb5f0-eba9-11e9-a384-0fcf32210194" + "id": "dde62a00-d59b-11ee-becf-dfae5e789f77" }, { "name": "panel_8", - "type": "visualization", - "id": "17319090-ebab-11e9-a384-0fcf32210194" + "type": "search", + "id": "a0a10870-eba5-11e9-a384-0fcf32210194" }, { "name": "panel_9", - "type": "visualization", - "id": "8022cc90-ebab-11e9-a384-0fcf32210194" + "type": "search", + "id": "fde4f550-d598-11ee-becf-dfae5e789f77" } ], "migrationVersion": { @@ -83,8 +83,8 @@ "namespaces": [ "default" ], - "updated_at": "2021-02-10T21:25:09.616Z", - "version": "Wzg3MiwxXQ==", + "updated_at": "2024-02-27T17:29:26.729Z", + "version": "Wzg2MiwxXQ==", "attributes": { "title": "Network Logs", "visState": "{\"title\":\"Network Logs\",\"type\":\"markdown\",\"params\":{\"markdown\":\"### General\\n[Overview](#/dashboard/0ad3d7c2-3441-485e-9dfe-dbb22e84e576) \\n[Security Overview](#/dashboard/95479950-41f2-11ea-88fa-7151df485405) \\n[ICS/IoT Security Overview](#/dashboard/4a4bde20-4760-11ea-949c-bbb5a9feecbf) \\n[Severity](#/dashboard/d2dd0180-06b1-11ec-8c6b-353266ade330) \\n[Connections](#/dashboard/abdd7550-2c7c-40dc-947e-f6d186a158c4) \\n[Actions and Results](#/dashboard/a33e0a50-afcd-11ea-993f-b7d8522a8bed) \\n[Files](#/dashboard/9ee51f94-3316-4fc5-bd89-93a52af69714) \\n[Executables](#/dashboard/0a490422-0ce9-44bf-9a2d-19329ddde8c3) \\n[Software](#/dashboard/87d990cc-9e0b-41e5-b8fe-b10ae1da0c85) \\n[Zeek Known Summary](#/dashboard/89d1cc50-974c-11ed-bb6b-3fb06c879b11) \\n[Zeek Intelligence](#/dashboard/36ed695f-edcc-47c1-b0ec-50d20c93ce0f) \\n[Zeek Notices](#/dashboard/f1f09567-fc7f-450b-a341-19d2f2bb468b) \\n[Zeek Weird](#/dashboard/1fff49f6-0199-4a0f-820b-721aff9ff1f1) \\n[Signatures](#/dashboard/665d1610-523d-11e9-a30e-e3576242f3ed) \\n[Suricata Alerts](#/dashboard/5694ca60-cbdf-11ec-a50a-5fedd672f5c5) \\n[Asset Interaction Analysis](#/dashboard/677ee170-809e-11ed-8d5b-07069f823b6f) \\n[↪ NetBox](/netbox/) \\n[↪ Arkime](/arkime/) \\n\\n### Common Protocols\\n[DCE/RPC](#/dashboard/432af556-c5c0-4cc3-8166-b274b4e3a406) ● [DHCP](#/dashboard/2d98bb8e-214c-4374-837b-20e1bcd63a5e) ● [DNS](#/dashboard/2cf94cd0-ecab-40a5-95a7-8419f3a39cd9) ● [FTP](#/dashboard/078b9aa5-9bd4-4f02-ae5e-cf80fa6f887b) / [TFTP](#/dashboard/bf5efbb0-60f1-11eb-9d60-dbf0411cfc48) ● [HTTP](#/dashboard/37041ee1-79c0-4684-a436-3173b0e89876) ● [IRC](#/dashboard/76f2f912-80da-44cd-ab66-6a73c8344cc3) ● [Kerberos](#/dashboard/82da3101-2a9c-4ae2-bb61-d447a3fbe673) ● [LDAP](#/dashboard/05e3e000-f118-11e9-acda-83a8e29e1a24) ● [MQTT](#/dashboard/87a32f90-ef58-11e9-974e-9d600036d105) ● [MySQL](#/dashboard/50ced171-1b10-4c3f-8b67-2db9635661a6) ● [NTLM](#/dashboard/543118a9-02d7-43fe-b669-b8652177fc37) ● [NTP](#/dashboard/af5df620-eeb6-11e9-bdef-65a192b7f586) ● [OSPF](#/dashboard/1cc01ff0-5205-11ec-a62c-7bc80e88f3f0) ● [QUIC](#/dashboard/11ddd980-e388-11e9-b568-cf17de8e860c) ● [RADIUS](#/dashboard/ae79b7d1-4281-4095-b2f6-fa7eafda9970) ● [RDP](#/dashboard/7f41913f-cba8-43f5-82a8-241b7ead03e0) ● [RFB](#/dashboard/f77bf097-18a8-465c-b634-eb2acc7a4f26) ● [SIP](#/dashboard/0b2354ae-0fe9-4fd9-b156-1c3870e5c7aa) ● [SMB](#/dashboard/42e831b9-41a9-4f35-8b7d-e1566d368773) ● [SMTP](#/dashboard/bb827f8e-639e-468c-93c8-9f5bc132eb8f) ● [SNMP](#/dashboard/4e5f106e-c60a-4226-8f64-d534abb912ab) ● [SSH](#/dashboard/caef3ade-d289-4d05-a511-149f3e97f238) ● [SSL](#/dashboard/7f77b58a-df3e-4cc2-b782-fd7f8bad8ffb) / [X.509 Certificates](#/dashboard/024062a6-48d6-498f-a91a-3bf2da3a3cd3) ● [STUN](#/dashboard/fa477130-2b8a-11ec-a9f2-3911c8571bfd) ● [Syslog](#/dashboard/92985909-dc29-4533-9e80-d3182a0ecf1d) ● [TDS](#/dashboard/bed185a0-ef82-11e9-b38a-2db3ee640e88) / [TDS RPC](#/dashboard/32587740-ef88-11e9-b38a-2db3ee640e88) / [TDS SQL](#/dashboard/fa141950-ef89-11e9-b38a-2db3ee640e88) ● [Telnet / rlogin / rsh](#/dashboard/c2549e10-7f2e-11ea-9f8a-1fe1327e2cd2) ● [Tunnels](#/dashboard/11be6381-beef-40a7-bdce-88c5398392fc)\\n\\n### ICS/IoT Protocols\\n[BACnet](#/dashboard/2bec1490-eb94-11e9-a384-0fcf32210194) ● [BSAP](#/dashboard/ca5799a0-56b5-11eb-b749-576de068f8ad) ● [DNP3](#/dashboard/870a5862-6c26-4a08-99fd-0c06cda85ba3) ● [EtherCAT](#/dashboard/4a073440-b286-11eb-a4d4-09fa12a6ebd4) ● [EtherNet/IP](#/dashboard/29a1b290-eb98-11e9-a384-0fcf32210194) ● [GENISYS](#/dashboard/03207c00-d07e-11ec-b4a7-d1b4003706b7) ● [Modbus](#/dashboard/152f29dc-51a2-4f53-93e9-6e92765567b8) ● [OPCUA Binary](#/dashboard/dd87edd0-796a-11ec-9ce6-b395c1ff58f4) ● [PROFINET](#/dashboard/a7514350-eba6-11e9-a384-0fcf32210194) ● [S7comm](#/dashboard/e76d05c0-eb9f-11e9-a384-0fcf32210194) ● [Synchrophasor](#/dashboard/2cc56240-e460-11ed-a9d5-9f591c284cb4) ● [Best Guess](#/dashboard/12e3a130-d83b-11eb-a0b0-f328ce09b0b7)\",\"type\":\"markdown\",\"fontSize\":10,\"openLinksInNewTab\":false},\"aggs\":[]}", @@ -106,28 +106,28 @@ "namespaces": [ "default" ], - "updated_at": "2021-02-10T21:24:49.327Z", - "version": "WzY2MywxXQ==", + "updated_at": "2024-02-27T18:00:32.722Z", + "version": "Wzk2NiwxXQ==", "attributes": { "title": "PROFINET - Log Count", - "visState": "{\"title\":\"PROFINET - Log Count\",\"type\":\"metric\",\"params\":{\"addTooltip\":true,\"addLegend\":false,\"type\":\"metric\",\"metric\":{\"percentageMode\":false,\"useRanges\":false,\"colorSchema\":\"Green to Red\",\"metricColorMode\":\"None\",\"colorsRange\":[{\"from\":0,\"to\":10000}],\"labels\":{\"show\":true},\"invertColors\":false,\"style\":{\"bgFill\":\"#000\",\"bgColor\":false,\"labelColor\":false,\"subText\":\"\",\"fontSize\":36}}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{\"customLabel\":\"Log Count\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"event.dataset\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\",\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Log Type\"}}]}", + "visState": "{\"title\":\"PROFINET - Log Count\",\"type\":\"metric\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{\"customLabel\":\"Log Count\"},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"event.dataset\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":5,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Log Type\"},\"schema\":\"group\"}],\"params\":{\"addTooltip\":true,\"addLegend\":false,\"type\":\"metric\",\"metric\":{\"percentageMode\":false,\"useRanges\":false,\"colorSchema\":\"Green to Red\",\"metricColorMode\":\"None\",\"colorsRange\":[{\"from\":0,\"to\":10000}],\"labels\":{\"show\":true},\"invertColors\":false,\"style\":{\"bgFill\":\"#000\",\"bgColor\":false,\"labelColor\":false,\"subText\":\"\",\"fontSize\":36}}}}", "uiStateJSON": "{}", "description": "", "version": 1, "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[{\"meta\":{\"type\":\"phrases\",\"key\":\"event.dataset\",\"value\":\"profinet, profinet_dce_rpc\",\"params\":[\"profinet\",\"profinet_dce_rpc\"],\"negate\":false,\"disabled\":false,\"alias\":\"Zeek Log Type\",\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index\"},\"query\":{\"bool\":{\"should\":[{\"match_phrase\":{\"event.dataset\":\"profinet\"}},{\"match_phrase\":{\"event.dataset\":\"profinet_dce_rpc\"}}],\"minimum_should_match\":1}},\"$state\":{\"store\":\"appState\"}}],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" + "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[{\"meta\":{\"type\":\"phrases\",\"key\":\"event.dataset\",\"value\":\"profinet, profinet_dce_rpc, profinet_io_cm\",\"params\":[\"profinet\",\"profinet_dce_rpc\",\"profinet_io_cm\"],\"alias\":\"Zeek Log Type\",\"negate\":false,\"disabled\":false,\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index\"},\"query\":{\"bool\":{\"should\":[{\"match_phrase\":{\"event.dataset\":\"profinet\"}},{\"match_phrase\":{\"event.dataset\":\"profinet_dce_rpc\"}},{\"match_phrase\":{\"event.dataset\":\"profinet_io_cm\"}}],\"minimum_should_match\":1}},\"$state\":{\"store\":\"appState\"}}],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" } }, "references": [ { "name": "kibanaSavedObjectMeta.searchSourceJSON.index", "type": "index-pattern", - "id": "MALCOLM_NETWORK_INDEX_PATTERN_REPLACER" + "id": "arkime_sessions3-*" }, { "name": "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index", "type": "index-pattern", - "id": "MALCOLM_NETWORK_INDEX_PATTERN_REPLACER" + "id": "arkime_sessions3-*" } ], "migrationVersion": { @@ -140,89 +140,45 @@ "namespaces": [ "default" ], - "updated_at": "2021-02-10T21:24:49.327Z", - "version": "WzY2NCwxXQ==", + "updated_at": "2024-02-27T17:59:35.087Z", + "version": "Wzk2NSwxXQ==", "attributes": { "title": "PROFINET - Logs Over Time", - "visState": "{\"title\":\"PROFINET - Logs Over Time\",\"type\":\"histogram\",\"params\":{\"type\":\"histogram\",\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"truncate\":100},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"\"}}],\"seriesParams\":[{\"show\":\"true\",\"type\":\"histogram\",\"mode\":\"stacked\",\"data\":{\"label\":\"Count\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"MALCOLM_NETWORK_INDEX_TIME_FIELD_REPLACER\",\"timeRange\":{\"from\":\"now-25y\",\"to\":\"now\",\"mode\":\"relative\"},\"useNormalizedEsInterval\":true,\"interval\":\"auto\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{},\"customLabel\":\" \"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"event.dataset\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Log Type\"}}]}", - "uiStateJSON": "{\"vis\":{\"legendOpen\":false}}", + "visState": "{\"title\":\"PROFINET - Logs Over Time\",\"type\":\"histogram\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"params\":{\"field\":\"firstPacket\",\"timeRange\":{\"from\":\"now-15y\",\"to\":\"now\"},\"useNormalizedOpenSearchInterval\":true,\"scaleMetricValues\":false,\"interval\":\"auto\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{},\"customLabel\":\" \"},\"schema\":\"segment\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"event.dataset\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":5,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Log Type\"},\"schema\":\"group\"}],\"params\":{\"type\":\"histogram\",\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"truncate\":100},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"\"}}],\"seriesParams\":[{\"show\":\"true\",\"type\":\"histogram\",\"mode\":\"stacked\",\"data\":{\"label\":\"Count\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"bottom\",\"times\":[],\"addTimeMarker\":false,\"labels\":{\"show\":false},\"thresholdLine\":{\"show\":false,\"value\":10,\"width\":1,\"style\":\"full\",\"color\":\"#E7664C\"}}}", + "uiStateJSON": "{\"vis\":{\"legendOpen\":true}}", "description": "", "version": 1, "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[{\"meta\":{\"type\":\"phrases\",\"key\":\"event.dataset\",\"value\":\"profinet, profinet_dce_rpc\",\"params\":[\"profinet\",\"profinet_dce_rpc\"],\"negate\":false,\"disabled\":false,\"alias\":\"Zeek Log Type\",\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index\"},\"query\":{\"bool\":{\"should\":[{\"match_phrase\":{\"event.dataset\":\"profinet\"}},{\"match_phrase\":{\"event.dataset\":\"profinet_dce_rpc\"}}],\"minimum_should_match\":1}},\"$state\":{\"store\":\"appState\"}}],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" + "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[{\"meta\":{\"type\":\"phrases\",\"key\":\"event.dataset\",\"value\":\"profinet, profinet_dce_rpc, profinet_io_cm\",\"params\":[\"profinet\",\"profinet_dce_rpc\",\"profinet_io_cm\"],\"alias\":\"Zeek Log Type\",\"negate\":false,\"disabled\":false,\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index\"},\"query\":{\"bool\":{\"should\":[{\"match_phrase\":{\"event.dataset\":\"profinet\"}},{\"match_phrase\":{\"event.dataset\":\"profinet_dce_rpc\"}},{\"match_phrase\":{\"event.dataset\":\"profinet_io_cm\"}}],\"minimum_should_match\":1}},\"$state\":{\"store\":\"appState\"}}],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" } }, "references": [ { "name": "kibanaSavedObjectMeta.searchSourceJSON.index", "type": "index-pattern", - "id": "MALCOLM_NETWORK_INDEX_PATTERN_REPLACER" + "id": "arkime_sessions3-*" }, { "name": "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index", "type": "index-pattern", - "id": "MALCOLM_NETWORK_INDEX_PATTERN_REPLACER" + "id": "arkime_sessions3-*" } ], "migrationVersion": { "visualization": "7.10.0" } }, - { - "id": "a0a10870-eba5-11e9-a384-0fcf32210194", - "type": "search", - "namespaces": [ - "default" - ], - "updated_at": "2021-02-10T21:24:49.327Z", - "version": "WzY2NSwxXQ==", - "attributes": { - "title": "PROFINET and Related - Logs", - "description": "", - "hits": 0, - "columns": [ - "source.ip", - "source.port", - "destination.ip", - "destination.port", - "zeek.profinet.operation_type", - "zeek.profinet.index", - "zeek.profinet_dce_rpc.operation", - "event.id" - ], - "sort": [ - [ - "MALCOLM_NETWORK_INDEX_TIME_FIELD_REPLACER", - "desc" - ] - ], - "version": 1, - "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"highlightAll\":false,\"version\":true,\"query\":{\"query\":\"event.dataset:(profinet OR profinet_dce_rpc)\",\"language\":\"lucene\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" - } - }, - "references": [ - { - "name": "kibanaSavedObjectMeta.searchSourceJSON.index", - "type": "index-pattern", - "id": "MALCOLM_NETWORK_INDEX_PATTERN_REPLACER" - } - ], - "migrationVersion": { - "search": "7.9.3" - } - }, { "id": "ec42baa0-eba8-11e9-a384-0fcf32210194", "type": "visualization", "namespaces": [ "default" ], - "updated_at": "2021-02-10T21:24:49.327Z", - "version": "WzY2NiwxXQ==", + "updated_at": "2024-02-27T18:13:58.684Z", + "version": "Wzk5MSwxXQ==", "attributes": { "title": "PROFINET - Source IP", - "visState": "{\"title\":\"PROFINET - Source IP\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"source.ip\",\"size\":250,\"order\":\"desc\",\"orderBy\":\"1\",\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Source IP\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"source.port\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\",\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Source Port\"}}]}", + "visState": "{\"title\":\"PROFINET - Source IP\",\"type\":\"table\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"source.ip\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":250,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Source IP\"},\"schema\":\"bucket\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"source.port\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":10,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Source Port\"},\"schema\":\"bucket\"}],\"params\":{\"perPage\":20,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\"}}", "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":2,\"direction\":\"desc\"}}}}", "description": "", "version": 1, @@ -233,8 +189,8 @@ }, "references": [ { - "type": "search", "name": "search_0", + "type": "search", "id": "a0a10870-eba5-11e9-a384-0fcf32210194" } ], @@ -248,11 +204,11 @@ "namespaces": [ "default" ], - "updated_at": "2021-02-10T21:24:49.327Z", - "version": "WzY2NywxXQ==", + "updated_at": "2024-02-27T18:14:38.259Z", + "version": "Wzk5MiwxXQ==", "attributes": { "title": "PROFINET - Destination IP", - "visState": "{\"title\":\"PROFINET - Destination IP\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":2,\"direction\":\"desc\"},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"destination.ip\",\"size\":250,\"order\":\"desc\",\"orderBy\":\"1\",\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Destination IP\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"destination.port\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\",\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Destination Port\"}}]}", + "visState": "{\"title\":\"PROFINET - Destination IP\",\"type\":\"table\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"destination.ip\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":250,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Destination IP\"},\"schema\":\"bucket\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"destination.port\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":10,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Destination Port\"},\"schema\":\"bucket\"}],\"params\":{\"perPage\":20,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":2,\"direction\":\"desc\"},\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\"}}", "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":2,\"direction\":\"desc\"}}}}", "description": "", "version": 1, @@ -263,8 +219,8 @@ }, "references": [ { - "type": "search", "name": "search_0", + "type": "search", "id": "a0a10870-eba5-11e9-a384-0fcf32210194" } ], @@ -278,8 +234,8 @@ "namespaces": [ "default" ], - "updated_at": "2021-02-10T21:24:49.327Z", - "version": "WzY2OCwxXQ==", + "updated_at": "2024-02-27T17:29:03.431Z", + "version": "WzYxMCwxXQ==", "attributes": { "title": "PROFINET - Operation", "visState": "{\"title\":\"PROFINET - Operation\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":4,\"direction\":\"desc\"},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"zeek.profinet.operation_type\",\"size\":250,\"order\":\"desc\",\"orderBy\":\"1\",\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"Unknown\",\"customLabel\":\"Operation\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"zeek.profinet.index\",\"size\":30,\"order\":\"desc\",\"orderBy\":\"1\",\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"Unknown\",\"customLabel\":\"Index\"}}]}", @@ -308,8 +264,8 @@ "namespaces": [ "default" ], - "updated_at": "2021-02-10T21:24:49.327Z", - "version": "WzY2OSwxXQ==", + "updated_at": "2024-02-27T17:29:03.431Z", + "version": "WzYxMSwxXQ==", "attributes": { "title": "PROFINET - Operation Details", "visState": "{\"title\":\"PROFINET - Operation Details\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":4,\"direction\":\"desc\"},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"zeek.profinet.operation_type\",\"size\":250,\"order\":\"desc\",\"orderBy\":\"1\",\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"Unknown\",\"customLabel\":\"Operation\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"zeek.profinet.index\",\"size\":30,\"order\":\"desc\",\"orderBy\":\"1\",\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"Unknown\",\"customLabel\":\"Index\"}},{\"id\":\"4\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"zeek.profinet.slot_number\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\",\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"Unknown\",\"customLabel\":\"Slot\"}},{\"id\":\"5\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"zeek.profinet.subslot_number\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\",\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"Unknown\",\"customLabel\":\"Subslot\"}}]}", @@ -333,29 +289,29 @@ } }, { - "id": "17319090-ebab-11e9-a384-0fcf32210194", + "id": "dde62a00-d59b-11ee-becf-dfae5e789f77", "type": "visualization", "namespaces": [ "default" ], - "updated_at": "2021-02-10T21:24:49.327Z", - "version": "WzY3MCwxXQ==", + "updated_at": "2024-02-27T18:13:11.455Z", + "version": "Wzk5MCwxXQ==", "attributes": { - "title": "PROFINET DCE/RPC - Operation", - "visState": "{\"title\":\"PROFINET DCE/RPC - Operation\",\"type\":\"pie\",\"params\":{\"type\":\"pie\",\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"isDonut\":true,\"labels\":{\"show\":false,\"values\":true,\"last_level\":true,\"truncate\":100}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"zeek.profinet_dce_rpc.operation\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\",\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Operation\"}}]}", - "uiStateJSON": "{}", + "title": "PROFINET I/O CM - Operation", + "visState": "{\"title\":\"PROFINET I/O CM - Operation\",\"type\":\"horizontal_bar\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"zeek.profinet_io_cm.operation\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":25,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"Unknown\",\"customLabel\":\"Operation\"},\"schema\":\"segment\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"zeek.profinet_io_cm.packet_type\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":5,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"Other\",\"customLabel\":\"Packet Type\"},\"schema\":\"group\"}],\"params\":{\"type\":\"histogram\",\"grid\":{\"categoryLines\":false},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":200},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"square root\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":75,\"filter\":true,\"truncate\":100},\"title\":{\"text\":\"Count\"}}],\"seriesParams\":[{\"show\":true,\"type\":\"histogram\",\"mode\":\"stacked\",\"data\":{\"label\":\"Count\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"lineWidth\":2,\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"bottom\",\"times\":[],\"addTimeMarker\":false,\"labels\":{},\"thresholdLine\":{\"show\":false,\"value\":10,\"width\":1,\"style\":\"full\",\"color\":\"#E7664C\"}}}", + "uiStateJSON": "{\"vis\":{\"legendOpen\":true}}", "description": "", "version": 1, "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[]}" + "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}" }, "savedSearchRefName": "search_0" }, "references": [ { - "type": "search", "name": "search_0", - "id": "96d31d60-eba5-11e9-a384-0fcf32210194" + "type": "search", + "id": "fde4f550-d598-11ee-becf-dfae5e789f77" } ], "migrationVersion": { @@ -363,45 +319,60 @@ } }, { - "id": "8022cc90-ebab-11e9-a384-0fcf32210194", - "type": "visualization", + "id": "a0a10870-eba5-11e9-a384-0fcf32210194", + "type": "search", "namespaces": [ "default" ], - "updated_at": "2021-02-10T21:24:49.327Z", - "version": "WzY3MSwxXQ==", + "updated_at": "2024-02-27T18:05:08.977Z", + "version": "Wzk4OCwxXQ==", "attributes": { - "title": "PROFINET DCE/RPC - Packet Type", - "visState": "{\"title\":\"PROFINET DCE/RPC - Packet Type\",\"type\":\"pie\",\"params\":{\"type\":\"pie\",\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"isDonut\":true,\"labels\":{\"show\":false,\"values\":true,\"last_level\":true,\"truncate\":100}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"zeek.profinet_dce_rpc.packet_type\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\",\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Packet Type\"}}]}", - "uiStateJSON": "{}", + "title": "PROFINET and Related - Logs", "description": "", + "hits": 0, + "columns": [ + "event.dataset", + "source.ip", + "source.port", + "destination.ip", + "destination.port", + "network.protocol_version", + "event.action", + "zeek.profinet.index", + "event.id" + ], + "sort": [ + [ + "firstPacket", + "desc" + ] + ], "version": 1, "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[]}" - }, - "savedSearchRefName": "search_0" + "searchSourceJSON": "{\"query\":{\"query\":\"event.dataset:(profinet OR profinet_dce_rpc OR profinet_io_cm)\",\"language\":\"lucene\"},\"highlightAll\":false,\"version\":true,\"aggs\":{\"2\":{\"date_histogram\":{\"field\":\"firstPacket\",\"fixed_interval\":\"30d\",\"time_zone\":\"America/Denver\",\"min_doc_count\":1}}},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" + } }, "references": [ { - "type": "search", - "name": "search_0", - "id": "96d31d60-eba5-11e9-a384-0fcf32210194" + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern", + "id": "arkime_sessions3-*" } ], "migrationVersion": { - "visualization": "7.10.0" + "search": "7.9.3" } }, { - "id": "8524e670-eba5-11e9-a384-0fcf32210194", + "id": "fde4f550-d598-11ee-becf-dfae5e789f77", "type": "search", "namespaces": [ "default" ], - "updated_at": "2021-02-10T21:24:49.327Z", - "version": "WzY3MiwxXQ==", + "updated_at": "2024-02-27T17:52:46.344Z", + "version": "Wzk2MywxXQ==", "attributes": { - "title": "PROFINET - Logs", + "title": "PROFINET I/O CM - Logs", "description": "", "hits": 0, "columns": [ @@ -409,27 +380,27 @@ "source.port", "destination.ip", "destination.port", - "zeek.profinet.block_version", - "zeek.profinet.operation_type", - "zeek.profinet.index", + "event.action", + "network.protocol_version", + "zeek.profinet_io_cm.sequence_num", "event.id" ], "sort": [ [ - "MALCOLM_NETWORK_INDEX_TIME_FIELD_REPLACER", + "firstPacket", "desc" ] ], "version": 1, "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"highlightAll\":false,\"version\":true,\"query\":{\"query\":\"event.dataset:profinet\",\"language\":\"lucene\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" + "searchSourceJSON": "{\"query\":{\"query\":\"event.dataset:profinet_io_cm\",\"language\":\"kuery\"},\"highlightAll\":false,\"version\":true,\"aggs\":{\"2\":{\"date_histogram\":{\"field\":\"firstPacket\",\"fixed_interval\":\"30d\",\"time_zone\":\"America/Denver\",\"min_doc_count\":1}}},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" } }, "references": [ { "name": "kibanaSavedObjectMeta.searchSourceJSON.index", "type": "index-pattern", - "id": "MALCOLM_NETWORK_INDEX_PATTERN_REPLACER" + "id": "arkime_sessions3-*" } ], "migrationVersion": { @@ -437,15 +408,15 @@ } }, { - "id": "96d31d60-eba5-11e9-a384-0fcf32210194", + "id": "8524e670-eba5-11e9-a384-0fcf32210194", "type": "search", "namespaces": [ "default" ], - "updated_at": "2021-02-10T21:24:49.327Z", - "version": "WzY3MywxXQ==", + "updated_at": "2024-02-27T17:29:03.431Z", + "version": "WzYxNCwxXQ==", "attributes": { - "title": "PROFINET DCE/RPC - Logs", + "title": "PROFINET - Logs", "description": "", "hits": 0, "columns": [ @@ -453,26 +424,27 @@ "source.port", "destination.ip", "destination.port", - "zeek.profinet_dce_rpc.version", - "zeek.profinet_dce_rpc.operation", + "zeek.profinet.block_version", + "zeek.profinet.operation_type", + "zeek.profinet.index", "event.id" ], "sort": [ [ - "MALCOLM_NETWORK_INDEX_TIME_FIELD_REPLACER", + "firstPacket", "desc" ] ], "version": 1, "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"highlightAll\":false,\"version\":true,\"query\":{\"query\":\"event.dataset:profinet_dce_rpc\",\"language\":\"lucene\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" + "searchSourceJSON": "{\"highlightAll\":false,\"version\":true,\"query\":{\"query\":\"event.dataset:profinet\",\"language\":\"lucene\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" } }, "references": [ { "name": "kibanaSavedObjectMeta.searchSourceJSON.index", "type": "index-pattern", - "id": "MALCOLM_NETWORK_INDEX_PATTERN_REPLACER" + "id": "arkime_sessions3-*" } ], "migrationVersion": { diff --git a/dashboards/dashboards/beats/0d4955f0-eb25-11ec-a6d4-b3526526c2c7.json b/dashboards/dashboards/beats/0d4955f0-eb25-11ec-a6d4-b3526526c2c7.json index acd0455b2..983bfdaa0 100644 --- a/dashboards/dashboards/beats/0d4955f0-eb25-11ec-a6d4-b3526526c2c7.json +++ b/dashboards/dashboards/beats/0d4955f0-eb25-11ec-a6d4-b3526526c2c7.json @@ -10,9 +10,9 @@ "updated_at": "2022-06-13T14:30:49.985Z", "version": "WzkxMSwyXQ==", "attributes": { - "title": "Malcolm Sensor Temperature", + "title": "Hardware Temperature", "hits": 0, - "description": "Malcolm sensor hardware component temperature summary", + "description": "Hardware component temperature summary, including from Malcolm sensors and aggregators", "panelsJSON": "[{\"version\":\"2.0.0\",\"gridData\":{\"x\":0,\"y\":0,\"w\":13,\"h\":9,\"i\":\"0a9ba334-b226-4245-b00e-ad8fcabf0e00\"},\"panelIndex\":\"0a9ba334-b226-4245-b00e-ad8fcabf0e00\",\"embeddableConfig\":{\"hidePanelTitles\":true},\"panelRefName\":\"panel_0\"},{\"version\":\"2.0.0\",\"gridData\":{\"x\":13,\"y\":0,\"w\":35,\"h\":18,\"i\":\"47ef4d93-d7d5-41c3-a3f4-d3b543ad22bc\"},\"panelIndex\":\"47ef4d93-d7d5-41c3-a3f4-d3b543ad22bc\",\"embeddableConfig\":{},\"panelRefName\":\"panel_1\"},{\"version\":\"2.0.0\",\"gridData\":{\"x\":0,\"y\":9,\"w\":13,\"h\":9,\"i\":\"970e9ca1-f3a4-4660-8f3e-8879e53cc63e\"},\"panelIndex\":\"970e9ca1-f3a4-4660-8f3e-8879e53cc63e\",\"embeddableConfig\":{\"hidePanelTitles\":true},\"panelRefName\":\"panel_2\"},{\"version\":\"2.0.0\",\"gridData\":{\"x\":0,\"y\":18,\"w\":48,\"h\":15,\"i\":\"594e7e58-e32d-4a2d-b492-a97541a2a2d4\"},\"panelIndex\":\"594e7e58-e32d-4a2d-b492-a97541a2a2d4\",\"embeddableConfig\":{\"hidePanelTitles\":false,\"vis\":null},\"panelRefName\":\"panel_3\"},{\"version\":\"2.0.0\",\"gridData\":{\"x\":0,\"y\":33,\"w\":48,\"h\":35,\"i\":\"aae9ec90-dea9-4a53-b4a7-99c5a6c91b55\"},\"panelIndex\":\"aae9ec90-dea9-4a53-b4a7-99c5a6c91b55\",\"embeddableConfig\":{},\"panelRefName\":\"panel_4\"}]", "optionsJSON": "{\"hidePanelTitles\":false,\"useMargins\":false}", "version": 1, @@ -61,8 +61,8 @@ "updated_at": "2022-06-13T13:57:55.163Z", "version": "Wzg5NiwyXQ==", "attributes": { - "title": "Malcolm Sensor Temperatures - Host Chooser", - "visState": "{\"title\":\"Malcolm Sensor Temperatures - Host Chooser\",\"type\":\"input_control_vis\",\"aggs\":[],\"params\":{\"controls\":[{\"id\":\"1655127648591\",\"fieldName\":\"host.name\",\"parent\":\"\",\"label\":\"Host Chooser\",\"type\":\"list\",\"options\":{\"type\":\"terms\",\"multiselect\":true,\"dynamicOptions\":true,\"size\":5,\"order\":\"desc\"},\"indexPatternRefName\":\"control_0_index_pattern\"}],\"updateFiltersOnChange\":false,\"useTimeFilter\":false,\"pinFilters\":false}}", + "title": "Hardware Temperature - Host Chooser", + "visState": "{\"title\":\"Hardware Temperature - Host Chooser\",\"type\":\"input_control_vis\",\"aggs\":[],\"params\":{\"controls\":[{\"id\":\"1655127648591\",\"fieldName\":\"host.name\",\"parent\":\"\",\"label\":\"Host Chooser\",\"type\":\"list\",\"options\":{\"type\":\"terms\",\"multiselect\":true,\"dynamicOptions\":true,\"size\":5,\"order\":\"desc\"},\"indexPatternRefName\":\"control_0_index_pattern\"}],\"updateFiltersOnChange\":false,\"useTimeFilter\":false,\"pinFilters\":false}}", "uiStateJSON": "{}", "description": "", "version": 1, @@ -90,8 +90,8 @@ "updated_at": "2022-06-13T14:26:58.711Z", "version": "WzkwOCwyXQ==", "attributes": { - "title": "Malcolm Sensor Temperature - Over Time", - "visState": "{\"title\":\"Malcolm Sensor Temperature - Over Time\",\"type\":\"line\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"max\",\"params\":{\"field\":\"miscbeat.thermal.temp\",\"customLabel\":\"High °C\"},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"params\":{\"field\":\"@timestamp\",\"timeRange\":{\"from\":\"now-30m\",\"to\":\"now\"},\"useNormalizedOpenSearchInterval\":true,\"scaleMetricValues\":false,\"interval\":\"auto\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{}},\"schema\":\"segment\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"host.name\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":8,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Host\"},\"schema\":\"group\"}],\"params\":{\"addLegend\":true,\"addTimeMarker\":false,\"addTooltip\":true,\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"labels\":{\"filter\":true,\"show\":true,\"truncate\":100},\"position\":\"bottom\",\"scale\":{\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{},\"type\":\"category\"}],\"grid\":{\"categoryLines\":true,\"valueAxis\":\"ValueAxis-1\"},\"labels\":{},\"legendPosition\":\"bottom\",\"seriesParams\":[{\"data\":{\"id\":\"1\",\"label\":\"High °C\"},\"drawLinesBetweenPoints\":true,\"interpolate\":\"cardinal\",\"lineWidth\":1.5,\"mode\":\"normal\",\"show\":true,\"showCircles\":false,\"type\":\"line\",\"valueAxis\":\"ValueAxis-1\"}],\"thresholdLine\":{\"color\":\"#E7664C\",\"show\":true,\"style\":\"full\",\"value\":95,\"width\":1},\"times\":[],\"type\":\"line\",\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"labels\":{\"filter\":false,\"rotate\":0,\"show\":true,\"truncate\":100},\"name\":\"LeftAxis-1\",\"position\":\"left\",\"scale\":{\"defaultYExtents\":true,\"mode\":\"normal\",\"type\":\"linear\",\"setYExtents\":true,\"min\":35,\"max\":115},\"show\":true,\"style\":{},\"title\":{\"text\":\"High °C\"},\"type\":\"value\"}]}}", + "title": "Hardware Temperature - Over Time", + "visState": "{\"title\":\"Hardware Temperature - Over Time\",\"type\":\"line\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"max\",\"params\":{\"field\":\"miscbeat.thermal.temp\",\"customLabel\":\"High °C\"},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"params\":{\"field\":\"@timestamp\",\"timeRange\":{\"from\":\"now-30m\",\"to\":\"now\"},\"useNormalizedOpenSearchInterval\":true,\"scaleMetricValues\":false,\"interval\":\"auto\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{}},\"schema\":\"segment\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"host.name\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":8,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Host\"},\"schema\":\"group\"}],\"params\":{\"addLegend\":true,\"addTimeMarker\":false,\"addTooltip\":true,\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"labels\":{\"filter\":true,\"show\":true,\"truncate\":100},\"position\":\"bottom\",\"scale\":{\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{},\"type\":\"category\"}],\"grid\":{\"categoryLines\":true,\"valueAxis\":\"ValueAxis-1\"},\"labels\":{},\"legendPosition\":\"bottom\",\"seriesParams\":[{\"data\":{\"id\":\"1\",\"label\":\"High °C\"},\"drawLinesBetweenPoints\":true,\"interpolate\":\"cardinal\",\"lineWidth\":1.5,\"mode\":\"normal\",\"show\":true,\"showCircles\":false,\"type\":\"line\",\"valueAxis\":\"ValueAxis-1\"}],\"thresholdLine\":{\"color\":\"#E7664C\",\"show\":true,\"style\":\"full\",\"value\":95,\"width\":1},\"times\":[],\"type\":\"line\",\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"labels\":{\"filter\":false,\"rotate\":0,\"show\":true,\"truncate\":100},\"name\":\"LeftAxis-1\",\"position\":\"left\",\"scale\":{\"defaultYExtents\":true,\"mode\":\"normal\",\"type\":\"linear\",\"setYExtents\":true,\"min\":35,\"max\":115},\"show\":true,\"style\":{},\"title\":{\"text\":\"High °C\"},\"type\":\"value\"}]}}", "uiStateJSON": "{\"vis\":{\"legendOpen\":true}}", "description": "", "version": 1, @@ -120,8 +120,8 @@ "updated_at": "2022-06-13T14:13:28.297Z", "version": "WzkwNCwyXQ==", "attributes": { - "title": "Malcolm Sensor Temprature - Sensor Chooser", - "visState": "{\"title\":\"Malcolm Sensor Temprature - Sensor Chooser\",\"type\":\"input_control_vis\",\"aggs\":[],\"params\":{\"controls\":[{\"id\":\"1655129580950\",\"fieldName\":\"miscbeat.thermal.type\",\"parent\":\"\",\"label\":\"Sensor Type\",\"type\":\"list\",\"options\":{\"type\":\"terms\",\"multiselect\":true,\"dynamicOptions\":true,\"size\":5,\"order\":\"desc\"},\"indexPatternRefName\":\"control_0_index_pattern\"}],\"updateFiltersOnChange\":false,\"useTimeFilter\":false,\"pinFilters\":false}}", + "title": "Hardware Temperature - Sensor Chooser", + "visState": "{\"title\":\"Hardware Temperature - Sensor Chooser\",\"type\":\"input_control_vis\",\"aggs\":[],\"params\":{\"controls\":[{\"id\":\"1655129580950\",\"fieldName\":\"miscbeat.thermal.type\",\"parent\":\"\",\"label\":\"Sensor Type\",\"type\":\"list\",\"options\":{\"type\":\"terms\",\"multiselect\":true,\"dynamicOptions\":true,\"size\":5,\"order\":\"desc\"},\"indexPatternRefName\":\"control_0_index_pattern\"}],\"updateFiltersOnChange\":false,\"useTimeFilter\":false,\"pinFilters\":false}}", "uiStateJSON": "{}", "description": "", "version": 1, @@ -149,8 +149,8 @@ "updated_at": "2022-06-13T14:24:35.386Z", "version": "WzkwNiwyXQ==", "attributes": { - "title": "Malcolm Sensor Temperature - Sensor Type", - "visState": "{\"title\":\"Malcolm Sensor Temperature - Sensor Type\",\"type\":\"gauge\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"max\",\"params\":{\"field\":\"miscbeat.thermal.temp\",\"customLabel\":\" \"},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"miscbeat.thermal.type\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":6,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Sensor Type\"},\"schema\":\"group\"}],\"params\":{\"type\":\"gauge\",\"addTooltip\":true,\"addLegend\":false,\"isDisplayWarning\":false,\"gauge\":{\"alignment\":\"automatic\",\"extendRange\":true,\"percentageMode\":false,\"gaugeType\":\"Arc\",\"gaugeStyle\":\"Full\",\"backStyle\":\"Full\",\"orientation\":\"vertical\",\"colorSchema\":\"Green to Red\",\"gaugeColorMode\":\"Labels\",\"colorsRange\":[{\"from\":0,\"to\":45},{\"from\":45,\"to\":60},{\"from\":60,\"to\":80},{\"from\":80,\"to\":90},{\"from\":90,\"to\":110}],\"invertColors\":false,\"labels\":{\"show\":true,\"color\":\"black\"},\"scale\":{\"show\":false,\"labels\":false,\"color\":\"rgba(105,112,125,0.2)\"},\"type\":\"meter\",\"style\":{\"bgWidth\":0.9,\"width\":0.9,\"mask\":false,\"bgMask\":false,\"maskBars\":50,\"bgFill\":\"rgba(105,112,125,0.2)\",\"bgColor\":true,\"subText\":\"\",\"fontSize\":60}}}}", + "title": "Hardware Temperature - Sensor Type", + "visState": "{\"title\":\"Hardware Temperature - Sensor Type\",\"type\":\"gauge\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"max\",\"params\":{\"field\":\"miscbeat.thermal.temp\",\"customLabel\":\" \"},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"miscbeat.thermal.type\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":6,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Sensor Type\"},\"schema\":\"group\"}],\"params\":{\"type\":\"gauge\",\"addTooltip\":true,\"addLegend\":false,\"isDisplayWarning\":false,\"gauge\":{\"alignment\":\"automatic\",\"extendRange\":true,\"percentageMode\":false,\"gaugeType\":\"Arc\",\"gaugeStyle\":\"Full\",\"backStyle\":\"Full\",\"orientation\":\"vertical\",\"colorSchema\":\"Green to Red\",\"gaugeColorMode\":\"Labels\",\"colorsRange\":[{\"from\":0,\"to\":45},{\"from\":45,\"to\":60},{\"from\":60,\"to\":80},{\"from\":80,\"to\":90},{\"from\":90,\"to\":110}],\"invertColors\":false,\"labels\":{\"show\":true,\"color\":\"black\"},\"scale\":{\"show\":false,\"labels\":false,\"color\":\"rgba(105,112,125,0.2)\"},\"type\":\"meter\",\"style\":{\"bgWidth\":0.9,\"width\":0.9,\"mask\":false,\"bgMask\":false,\"maskBars\":50,\"bgFill\":\"rgba(105,112,125,0.2)\",\"bgColor\":true,\"subText\":\"\",\"fontSize\":60}}}}", "uiStateJSON": "{}", "description": "", "version": 1, @@ -179,7 +179,7 @@ "updated_at": "2022-06-13T14:01:12.279Z", "version": "WzkwMSwyXQ==", "attributes": { - "title": "Malcolm Sensor Temperature - Logs", + "title": "Hardware Temperature - Logs", "description": "", "hits": 0, "columns": [ @@ -191,7 +191,7 @@ "sort": [], "version": 1, "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"highlightAll\":true,\"version\":true,\"query\":{\"query\":\"event.module:thermal\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" + "searchSourceJSON": "{\"highlightAll\":false,\"version\":true,\"query\":{\"query\":\"event.module:thermal\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" } }, "references": [ diff --git a/dashboards/dashboards/beats/3768ef70-d819-11ee-820d-dd9fd73a3921.json b/dashboards/dashboards/beats/3768ef70-d819-11ee-820d-dd9fd73a3921.json new file mode 100644 index 000000000..d7c991a76 --- /dev/null +++ b/dashboards/dashboards/beats/3768ef70-d819-11ee-820d-dd9fd73a3921.json @@ -0,0 +1,141 @@ +{ + "version": "2.12.0", + "objects": [ + { + "id": "3768ef70-d819-11ee-820d-dd9fd73a3921", + "type": "dashboard", + "namespaces": [ + "default" + ], + "updated_at": "2024-03-01T22:15:31.047Z", + "version": "WzExMTEsMV0=", + "attributes": { + "title": "Linux Kernel Messages", + "hits": 0, + "description": "Linux kernel messages (kmsg), including from Malcolm sensors and aggregators", + "panelsJSON": "[{\"embeddableConfig\":{},\"gridData\":{\"h\":16,\"i\":\"1c0ec894-1fd1-4a1a-8aa1-730e651ecca8\",\"w\":14,\"x\":0,\"y\":0},\"panelIndex\":\"1c0ec894-1fd1-4a1a-8aa1-730e651ecca8\",\"version\":\"2.12.0\",\"panelRefName\":\"panel_0\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":16,\"i\":\"d395e5d6-8064-4226-b4bd-7c21db0d6cf5\",\"w\":34,\"x\":14,\"y\":0},\"panelIndex\":\"d395e5d6-8064-4226-b4bd-7c21db0d6cf5\",\"version\":\"2.12.0\",\"panelRefName\":\"panel_1\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":19,\"i\":\"4178095b-48cb-4ce8-b79c-eea288285dd5\",\"w\":48,\"x\":0,\"y\":16},\"panelIndex\":\"4178095b-48cb-4ce8-b79c-eea288285dd5\",\"version\":\"2.12.0\",\"panelRefName\":\"panel_2\"}]", + "optionsJSON": "{\"hidePanelTitles\":false,\"useMargins\":true}", + "version": 1, + "timeRestore": false, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"language\":\"kuery\",\"query\":\"\"},\"filter\":[]}" + } + }, + "references": [ + { + "name": "panel_0", + "type": "visualization", + "id": "822595a0-d818-11ee-820d-dd9fd73a3921" + }, + { + "name": "panel_1", + "type": "visualization", + "id": "ab235be0-d818-11ee-820d-dd9fd73a3921" + }, + { + "name": "panel_2", + "type": "search", + "id": "27ac44c0-d818-11ee-820d-dd9fd73a3921" + } + ], + "migrationVersion": { + "dashboard": "7.9.3" + } + }, + { + "id": "822595a0-d818-11ee-820d-dd9fd73a3921", + "type": "visualization", + "namespaces": [ + "default" + ], + "updated_at": "2024-03-01T22:10:26.938Z", + "version": "WzExMDgsMV0=", + "attributes": { + "title": "Linux Kernel Messages by Host", + "visState": "{\"title\":\"Linux Kernel Messages by Host\",\"type\":\"table\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"host.name\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":50,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Host\"},\"schema\":\"bucket\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"top_hits\",\"params\":{\"field\":\"@timestamp\",\"aggregate\":\"concat\",\"size\":1,\"sortField\":\"@timestamp\",\"sortOrder\":\"desc\",\"customLabel\":\"Last Kernel Message\"},\"schema\":\"metric\"}],\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\"}}", + "uiStateJSON": "{}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}" + }, + "savedSearchRefName": "search_0" + }, + "references": [ + { + "name": "search_0", + "type": "search", + "id": "27ac44c0-d818-11ee-820d-dd9fd73a3921" + } + ], + "migrationVersion": { + "visualization": "7.10.0" + } + }, + { + "id": "ab235be0-d818-11ee-820d-dd9fd73a3921", + "type": "visualization", + "namespaces": [ + "default" + ], + "updated_at": "2024-03-01T22:14:59.550Z", + "version": "WzExMTAsMV0=", + "attributes": { + "title": "Linux Kernel Messages over Time", + "visState": "{\"title\":\"Linux Kernel Messages over Time\",\"type\":\"histogram\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{\"customLabel\":\"Message Count\"},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"params\":{\"field\":\"@timestamp\",\"timeRange\":{\"from\":\"now/d\",\"to\":\"now/d\"},\"useNormalizedOpenSearchInterval\":true,\"scaleMetricValues\":false,\"interval\":\"auto\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{},\"customLabel\":\"Messages over Time by Priority\"},\"schema\":\"segment\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"miscbeat.kmsg.priority\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":10,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Priority\"},\"schema\":\"group\"}],\"params\":{\"type\":\"histogram\",\"grid\":{\"categoryLines\":false},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"filter\":true,\"truncate\":100},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"square root\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Message Count\"}}],\"seriesParams\":[{\"show\":true,\"type\":\"histogram\",\"mode\":\"stacked\",\"data\":{\"label\":\"Message Count\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"lineWidth\":2,\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"bottom\",\"times\":[],\"addTimeMarker\":false,\"labels\":{\"show\":false},\"thresholdLine\":{\"show\":false,\"value\":10,\"width\":1,\"style\":\"full\",\"color\":\"#E7664C\"}}}", + "uiStateJSON": "{\"vis\":{\"legendOpen\":true}}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}" + }, + "savedSearchRefName": "search_0" + }, + "references": [ + { + "name": "search_0", + "type": "search", + "id": "27ac44c0-d818-11ee-820d-dd9fd73a3921" + } + ], + "migrationVersion": { + "visualization": "7.10.0" + } + }, + { + "id": "27ac44c0-d818-11ee-820d-dd9fd73a3921", + "type": "search", + "namespaces": [ + "default" + ], + "updated_at": "2024-03-01T22:07:55.148Z", + "version": "WzExMDUsMV0=", + "attributes": { + "title": "Linux Kernel Messages", + "description": "", + "hits": 0, + "columns": [ + "host.name", + "miscbeat.kmsg.sequence", + "miscbeat.kmsg.priority", + "event.original" + ], + "sort": [], + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query\":\"event.module:kmsg\",\"language\":\"kuery\"},\"highlightAll\":false,\"version\":true,\"aggs\":{\"2\":{\"date_histogram\":{\"field\":\"@timestamp\",\"fixed_interval\":\"30m\",\"time_zone\":\"America/Denver\",\"min_doc_count\":1}}},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" + } + }, + "references": [ + { + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern", + "id": "malcolm_beats_*" + } + ], + "migrationVersion": { + "search": "7.9.3" + } + } + ] +} \ No newline at end of file diff --git a/dashboards/dashboards/beats/4ca94c70-d7da-11ee-9ed3-e7afff29e59a.json b/dashboards/dashboards/beats/4ca94c70-d7da-11ee-9ed3-e7afff29e59a.json new file mode 100644 index 000000000..e17788646 --- /dev/null +++ b/dashboards/dashboards/beats/4ca94c70-d7da-11ee-9ed3-e7afff29e59a.json @@ -0,0 +1,550 @@ +{ + "version": "2.12.0", + "objects": [ + { + "id": "4ca94c70-d7da-11ee-9ed3-e7afff29e59a", + "type": "dashboard", + "namespaces": [ + "default" + ], + "updated_at": "2024-03-04T21:05:53.644Z", + "version": "Wzg4NCwxXQ==", + "attributes": { + "title": "Packet Capture Statistics", + "hits": 0, + "description": "Statistics and diagnostics for packet capture from Zeek and Suricata", + "panelsJSON": "[{\"version\":\"2.12.0\",\"gridData\":{\"x\":0,\"y\":0,\"w\":13,\"h\":15,\"i\":\"0c179e97-9bcf-4f72-b717-b7a93667c1a0\"},\"panelIndex\":\"0c179e97-9bcf-4f72-b717-b7a93667c1a0\",\"embeddableConfig\":{},\"panelRefName\":\"panel_0\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":13,\"y\":0,\"w\":35,\"h\":15,\"i\":\"b483d809-a528-4280-b79e-aa7ada17d275\"},\"panelIndex\":\"b483d809-a528-4280-b79e-aa7ada17d275\",\"embeddableConfig\":{\"hidePanelTitles\":false},\"panelRefName\":\"panel_1\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":0,\"y\":15,\"w\":13,\"h\":10,\"i\":\"e10dc0a6-f197-4cbc-a1ad-e67194f95a63\"},\"panelIndex\":\"e10dc0a6-f197-4cbc-a1ad-e67194f95a63\",\"embeddableConfig\":{\"hidePanelTitles\":false},\"panelRefName\":\"panel_2\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":13,\"y\":15,\"w\":13,\"h\":10,\"i\":\"01b20859-4d95-47e0-a536-6b1e9932c35b\"},\"panelIndex\":\"01b20859-4d95-47e0-a536-6b1e9932c35b\",\"embeddableConfig\":{\"hidePanelTitles\":false},\"panelRefName\":\"panel_3\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":26,\"y\":15,\"w\":22,\"h\":20,\"i\":\"8e013ce7-3205-4d06-a805-6285826c1c5d\"},\"panelIndex\":\"8e013ce7-3205-4d06-a805-6285826c1c5d\",\"embeddableConfig\":{\"columns\":[\"@timestamp\",\"host.name\",\"zeek.capture_loss.peer\",\"zeek.capture_loss.acks\",\"zeek.capture_loss.gaps\",\"zeek.capture_loss.percent_lost\"]},\"panelRefName\":\"panel_4\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":0,\"y\":25,\"w\":13,\"h\":10,\"i\":\"147b45ae-804b-4d9e-a9a9-806772ad3b35\"},\"panelIndex\":\"147b45ae-804b-4d9e-a9a9-806772ad3b35\",\"embeddableConfig\":{\"hidePanelTitles\":false},\"panelRefName\":\"panel_5\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":13,\"y\":25,\"w\":13,\"h\":10,\"i\":\"687597e3-4848-4629-8b85-45c0773efb79\"},\"panelIndex\":\"687597e3-4848-4629-8b85-45c0773efb79\",\"embeddableConfig\":{\"hidePanelTitles\":false},\"panelRefName\":\"panel_6\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":0,\"y\":35,\"w\":24,\"h\":15,\"i\":\"0174654c-2010-463a-b49e-fa5759b61b9c\"},\"panelIndex\":\"0174654c-2010-463a-b49e-fa5759b61b9c\",\"embeddableConfig\":{},\"panelRefName\":\"panel_7\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":0,\"y\":50,\"w\":48,\"h\":21,\"i\":\"36e03a4a-e017-42b8-82cf-205d26b2ed6b\"},\"panelIndex\":\"36e03a4a-e017-42b8-82cf-205d26b2ed6b\",\"embeddableConfig\":{},\"panelRefName\":\"panel_8\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":0,\"y\":71,\"w\":48,\"h\":21,\"i\":\"e1c0f1e0-de36-4527-bafa-a297fe9452a2\"},\"panelIndex\":\"e1c0f1e0-de36-4527-bafa-a297fe9452a2\",\"embeddableConfig\":{},\"panelRefName\":\"panel_9\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":0,\"y\":92,\"w\":13,\"h\":20,\"i\":\"74a841b8-2ffc-4f6d-8b5a-ca7960eb6b10\"},\"panelIndex\":\"74a841b8-2ffc-4f6d-8b5a-ca7960eb6b10\",\"embeddableConfig\":{},\"panelRefName\":\"panel_10\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":13,\"y\":92,\"w\":35,\"h\":20,\"i\":\"f15e46fe-040f-4602-ad13-01aab36b372a\"},\"panelIndex\":\"f15e46fe-040f-4602-ad13-01aab36b372a\",\"embeddableConfig\":{},\"panelRefName\":\"panel_11\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":0,\"y\":112,\"w\":16,\"h\":17,\"i\":\"bfdc6d50-66f1-4f9a-9ea5-cd30bc01099d\"},\"panelIndex\":\"bfdc6d50-66f1-4f9a-9ea5-cd30bc01099d\",\"embeddableConfig\":{},\"panelRefName\":\"panel_12\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":16,\"y\":112,\"w\":32,\"h\":17,\"i\":\"efbd7f15-5af7-4e39-9889-c1c944a40dc2\"},\"panelIndex\":\"efbd7f15-5af7-4e39-9889-c1c944a40dc2\",\"embeddableConfig\":{\"columns\":[\"@timestamp\",\"host.name\",\"zeek.reporter.level\",\"zeek.reporter.msg\",\"zeek.reporter.location\"]},\"panelRefName\":\"panel_13\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":24,\"y\":35,\"w\":24,\"h\":15,\"i\":\"2ecc4ac3-d694-46ab-a6b1-9c86e5e9d394\"},\"panelIndex\":\"2ecc4ac3-d694-46ab-a6b1-9c86e5e9d394\",\"embeddableConfig\":{},\"panelRefName\":\"panel_14\"}]", + "optionsJSON": "{\"hidePanelTitles\":false,\"useMargins\":true}", + "version": 1, + "timeRestore": false, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"language\":\"kuery\",\"query\":\"\"},\"filter\":[]}" + } + }, + "references": [ + { + "name": "panel_0", + "type": "visualization", + "id": "a1480cb0-d7ff-11ee-b25e-e793ed358448" + }, + { + "name": "panel_1", + "type": "visualization", + "id": "611a3e20-d7e0-11ee-b25e-e793ed358448" + }, + { + "name": "panel_2", + "type": "visualization", + "id": "13facbf0-d7e5-11ee-b25e-e793ed358448" + }, + { + "name": "panel_3", + "type": "visualization", + "id": "8051d2c0-d7e6-11ee-b25e-e793ed358448" + }, + { + "name": "panel_4", + "type": "search", + "id": "991158b0-d7d9-11ee-9ed3-e7afff29e59a" + }, + { + "name": "panel_5", + "type": "visualization", + "id": "cc976b70-d7fc-11ee-b25e-e793ed358448" + }, + { + "name": "panel_6", + "type": "visualization", + "id": "9833e6a0-d7fd-11ee-b25e-e793ed358448" + }, + { + "name": "panel_7", + "type": "visualization", + "id": "6b7b9a40-faa1-11e6-86b1-cd7735ff7e23-ecs" + }, + { + "name": "panel_8", + "type": "search", + "id": "9c6a1610-d7d8-11ee-9ed3-e7afff29e59a" + }, + { + "name": "panel_9", + "type": "search", + "id": "b0aabf40-d7fa-11ee-b25e-e793ed358448" + }, + { + "name": "panel_10", + "type": "visualization", + "id": "abcfca50-d7dd-11ee-b25e-e793ed358448" + }, + { + "name": "panel_11", + "type": "search", + "id": "351ef380-d7d9-11ee-9ed3-e7afff29e59a" + }, + { + "name": "panel_12", + "type": "visualization", + "id": "c75b6df0-d7e3-11ee-b25e-e793ed358448" + }, + { + "name": "panel_13", + "type": "search", + "id": "17619ea0-d7da-11ee-9ed3-e7afff29e59a" + }, + { + "name": "panel_14", + "type": "visualization", + "id": "089b85d0-1b16-11e7-b09e-037021c4f8df-ecs" + } + ], + "migrationVersion": { + "dashboard": "7.9.3" + } + }, + { + "id": "a1480cb0-d7ff-11ee-b25e-e793ed358448", + "type": "visualization", + "namespaces": [ + "default" + ], + "updated_at": "2024-03-04T21:05:53.644Z", + "version": "Wzg4NSwxXQ==", + "attributes": { + "title": "Last Capture Metric Timestamp by Host", + "visState": "{\"title\":\"Last Capture Metric Timestamp by Host\",\"type\":\"table\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"top_hits\",\"params\":{\"field\":\"@timestamp\",\"aggregate\":\"concat\",\"size\":1,\"sortField\":\"@timestamp\",\"sortOrder\":\"desc\",\"customLabel\":\"Last Metric Timestamp\"},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"host.name\",\"orderBy\":\"_key\",\"order\":\"desc\",\"size\":100,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Capture Host\"},\"schema\":\"bucket\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"event.provider\",\"orderBy\":\"_key\",\"order\":\"desc\",\"size\":5,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Other\"},\"schema\":\"bucket\"}],\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\"}}", + "uiStateJSON": "{\"vis\":{\"sortColumn\":{\"colIndex\":2,\"direction\":\"desc\"}}}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query\":\"event.kind:metric\",\"language\":\"lucene\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" + } + }, + "references": [ + { + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern", + "id": "malcolm_beats_*" + } + ], + "migrationVersion": { + "visualization": "7.10.0" + } + }, + { + "id": "611a3e20-d7e0-11ee-b25e-e793ed358448", + "type": "visualization", + "namespaces": [ + "default" + ], + "updated_at": "2024-03-04T21:07:41.024Z", + "version": "Wzk5OSwxXQ==", + "attributes": { + "title": "Zeek and Suricata Capture Measurements ", + "visState": "{\"title\":\"Zeek and Suricata Capture Measurements \",\"type\":\"metrics\",\"aggs\":[],\"params\":{\"id\":\"61ca57f0-469d-11e7-af02-69e470af7417\",\"type\":\"timeseries\",\"series\":[{\"id\":\"32d1fca0-d7e1-11ee-ad81-217e54128a4b\",\"color\":\"rgba(33,150,243,1)\",\"split_mode\":\"everything\",\"metrics\":[{\"id\":\"32d1fca1-d7e1-11ee-ad81-217e54128a4b\",\"type\":\"sum\",\"field\":\"zeek.stats.pkts_link\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"line\",\"line_width\":\"0\",\"point_size\":\"0\",\"fill\":0.5,\"stacked\":\"none\",\"label\":\"Zeek: packets seen\",\"type\":\"timeseries\",\"terms_field\":\"host.name\",\"terms_size\":\"25\",\"terms_order_by\":\"32d1fca1-d7e1-11ee-ad81-217e54128a4b\",\"split_color_mode\":\"opensearchDashboards\"},{\"id\":\"02bbf6a0-d7fb-11ee-a5f1-9ff9da698a18\",\"color\":\"rgba(84,179,153,1)\",\"split_mode\":\"everything\",\"metrics\":[{\"unit\":\"\",\"id\":\"02bbf6a1-d7fb-11ee-a5f1-9ff9da698a18\",\"type\":\"positive_rate\",\"field\":\"suricata.stats.capture.kernel_packets\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"line\",\"line_width\":\"0\",\"point_size\":\"0\",\"fill\":0.5,\"stacked\":\"none\",\"label\":\"Suricata: packets seen\",\"type\":\"timeseries\"},{\"id\":\"e4143600-d7e0-11ee-ad81-217e54128a4b\",\"color\":\"rgba(229,115,115,1)\",\"split_mode\":\"everything\",\"metrics\":[{\"id\":\"e4143601-d7e0-11ee-ad81-217e54128a4b\",\"type\":\"sum\",\"field\":\"zeek.stats.pkts_dropped\"},{\"id\":\"f6df2790-d7e0-11ee-ad81-217e54128a4b\",\"type\":\"math\",\"variables\":[{\"id\":\"f8ee0a60-d7e0-11ee-ad81-217e54128a4b\",\"name\":\"packets\",\"field\":\"e4143601-d7e0-11ee-ad81-217e54128a4b\"}],\"script\":\"params.packets*-1\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"line\",\"line_width\":\"0\",\"point_size\":\"0\",\"fill\":0.5,\"stacked\":\"none\",\"label\":\"Zeek: packets dropped\",\"type\":\"timeseries\",\"terms_field\":\"host.name\",\"terms_size\":\"25\",\"terms_order_by\":\"e4143601-d7e0-11ee-ad81-217e54128a4b\",\"split_color_mode\":\"opensearchDashboards\"},{\"id\":\"20b9a420-d7df-11ee-ad81-217e54128a4b\",\"color\":\"rgba(211,96,134,1)\",\"split_mode\":\"everything\",\"metrics\":[{\"id\":\"20b9a421-d7df-11ee-ad81-217e54128a4b\",\"type\":\"sum\",\"field\":\"zeek.capture_loss.gaps\"},{\"id\":\"9a3afce0-d7df-11ee-ad81-217e54128a4b\",\"type\":\"math\",\"variables\":[{\"id\":\"9dece150-d7df-11ee-ad81-217e54128a4b\",\"name\":\"gaps\",\"field\":\"20b9a421-d7df-11ee-ad81-217e54128a4b\"}],\"script\":\"params.gaps*-1\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"line\",\"line_width\":\"0\",\"point_size\":\"0\",\"fill\":0.5,\"stacked\":\"none\",\"label\":\"Zeek: ACKS missed\",\"type\":\"timeseries\",\"terms_field\":\"host.name\",\"terms_size\":\"25\",\"terms_order_by\":\"20b9a421-d7df-11ee-ad81-217e54128a4b\",\"split_color_mode\":\"opensearchDashboards\"},{\"id\":\"cad40600-d7fb-11ee-a5f1-9ff9da698a18\",\"color\":\"rgba(255,171,145,1)\",\"split_mode\":\"everything\",\"metrics\":[{\"unit\":\"\",\"id\":\"cad40601-d7fb-11ee-a5f1-9ff9da698a18\",\"type\":\"positive_rate\",\"field\":\"suricata.stats.pkts_dropped\"},{\"id\":\"f5352cd0-d7fb-11ee-a5f1-9ff9da698a18\",\"type\":\"math\",\"variables\":[{\"id\":\"f79def70-d7fb-11ee-a5f1-9ff9da698a18\",\"name\":\"packets\",\"field\":\"cad40601-d7fb-11ee-a5f1-9ff9da698a18\"}],\"script\":\"params.packets*-1\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"line\",\"line_width\":\"0\",\"point_size\":\"0\",\"fill\":0.5,\"stacked\":\"none\",\"label\":\"Suricata: packets dropped\",\"type\":\"timeseries\"}],\"time_field\":\"@timestamp\",\"index_pattern\":\"malcolm_beats_*\",\"interval\":\"\",\"axis_position\":\"left\",\"axis_formatter\":\"number\",\"axis_scale\":\"normal\",\"show_legend\":1,\"show_grid\":1,\"tooltip_mode\":\"show_all\",\"default_index_pattern\":\"arkime_sessions3-*\",\"default_timefield\":\"firstPacket\",\"isModelInvalid\":false,\"filter\":{\"query\":\"(event.provider:zeek OR event.provider:suricata) AND event.kind:metric\",\"language\":\"kuery\"},\"legend_position\":\"right\",\"background_color\":null}}", + "uiStateJSON": "{}", + "description": "Positive values on the y-axis represent observed packets while negative values represent missing dropped packets and missing ACKs.\n\nThis data is logged by Zeek in stats.log (https://docs.zeek.org/en/master/scripts/policy/misc/stats.zeek.html#type-Stats::Info) and capture_loss.log (https://docs.zeek.org/en/master/scripts/policy/misc/capture-loss.zeek.html#type-CaptureLoss::Info), and by Suricata (https://docs.suricata.io/en/suricata-6.0.0/performance/statistics.html).", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[]}" + } + }, + "references": [], + "migrationVersion": { + "visualization": "7.10.0" + } + }, + { + "id": "13facbf0-d7e5-11ee-b25e-e793ed358448", + "type": "visualization", + "namespaces": [ + "default" + ], + "updated_at": "2024-03-04T21:05:53.644Z", + "version": "Wzg4NywxXQ==", + "attributes": { + "title": "Zeek Stats - Packets and Bytes", + "visState": "{\"title\":\"Zeek Stats - Packets and Bytes\",\"type\":\"metrics\",\"aggs\":[],\"params\":{\"id\":\"61ca57f0-469d-11e7-af02-69e470af7417\",\"type\":\"metric\",\"series\":[{\"id\":\"61ca57f1-469d-11e7-af02-69e470af7417\",\"color\":\"#54B399\",\"split_mode\":\"everything\",\"split_color_mode\":\"opensearchDashboards\",\"metrics\":[{\"id\":\"61ca57f2-469d-11e7-af02-69e470af7417\",\"type\":\"sum\",\"field\":\"zeek.stats.pkts_link\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"line\",\"line_width\":1,\"point_size\":1,\"fill\":0.5,\"stacked\":\"none\",\"label\":\"Total Packets Seen\"},{\"id\":\"bd4560e0-d7e4-11ee-ad81-217e54128a4b\",\"color\":\"#68BC00\",\"split_mode\":\"everything\",\"metrics\":[{\"id\":\"bd4560e1-d7e4-11ee-ad81-217e54128a4b\",\"type\":\"sum\",\"field\":\"zeek.stats.bytes_recv\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"bytes\",\"chart_type\":\"line\",\"line_width\":1,\"point_size\":1,\"fill\":0.5,\"stacked\":\"none\",\"label\":\"Total Bytes Seen\"}],\"time_field\":\"@timestamp\",\"index_pattern\":\"malcolm_beats_*\",\"interval\":\"\",\"axis_position\":\"left\",\"axis_formatter\":\"number\",\"axis_scale\":\"normal\",\"show_legend\":1,\"show_grid\":1,\"tooltip_mode\":\"show_all\",\"default_index_pattern\":\"arkime_sessions3-*\",\"default_timefield\":\"firstPacket\",\"isModelInvalid\":false,\"background_color_rules\":[{\"id\":\"28bcc800-d7e4-11ee-ad81-217e54128a4b\"}],\"filter\":{\"query\":\"event.provider:zeek AND event.dataset:stats\",\"language\":\"kuery\"},\"time_range_mode\":\"entire_time_range\"}}", + "uiStateJSON": "{}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[]}" + } + }, + "references": [], + "migrationVersion": { + "visualization": "7.10.0" + } + }, + { + "id": "8051d2c0-d7e6-11ee-b25e-e793ed358448", + "type": "visualization", + "namespaces": [ + "default" + ], + "updated_at": "2024-03-04T21:05:53.644Z", + "version": "Wzg4OCwxXQ==", + "attributes": { + "title": "Zeek Stats - Capture Loss", + "visState": "{\"title\":\"Zeek Stats - Capture Loss\",\"type\":\"metrics\",\"aggs\":[],\"params\":{\"id\":\"61ca57f0-469d-11e7-af02-69e470af7417\",\"type\":\"metric\",\"series\":[{\"id\":\"61ca57f1-469d-11e7-af02-69e470af7417\",\"color\":\"#54B399\",\"split_mode\":\"everything\",\"split_color_mode\":\"opensearchDashboards\",\"metrics\":[{\"id\":\"61ca57f2-469d-11e7-af02-69e470af7417\",\"type\":\"sum\",\"field\":\"zeek.stats.pkts_dropped\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"line\",\"line_width\":1,\"point_size\":1,\"fill\":0.5,\"stacked\":\"none\",\"label\":\"Packets Dropped\"},{\"id\":\"bd4560e0-d7e4-11ee-ad81-217e54128a4b\",\"color\":\"#68BC00\",\"split_mode\":\"everything\",\"metrics\":[{\"id\":\"bd4560e1-d7e4-11ee-ad81-217e54128a4b\",\"type\":\"sum\",\"field\":\"zeek.capture_loss.gaps\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"line\",\"line_width\":1,\"point_size\":1,\"fill\":0.5,\"stacked\":\"none\",\"label\":\"ACKs Missed\"}],\"time_field\":\"@timestamp\",\"index_pattern\":\"malcolm_beats_*\",\"interval\":\"\",\"axis_position\":\"left\",\"axis_formatter\":\"number\",\"axis_scale\":\"normal\",\"show_legend\":1,\"show_grid\":1,\"tooltip_mode\":\"show_all\",\"default_index_pattern\":\"arkime_sessions3-*\",\"default_timefield\":\"firstPacket\",\"isModelInvalid\":false,\"background_color_rules\":[{\"id\":\"28bcc800-d7e4-11ee-ad81-217e54128a4b\"}],\"filter\":{\"query\":\"event.provider:zeek AND event.dataset:(stats OR capture_loss)\",\"language\":\"kuery\"},\"time_range_mode\":\"entire_time_range\"}}", + "uiStateJSON": "{}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[]}" + } + }, + "references": [], + "migrationVersion": { + "visualization": "7.10.0" + } + }, + { + "id": "991158b0-d7d9-11ee-9ed3-e7afff29e59a", + "type": "search", + "namespaces": [ + "default" + ], + "updated_at": "2024-03-04T21:05:53.644Z", + "version": "Wzg4OSwxXQ==", + "attributes": { + "title": "Packet Capture - Zeek capture_loss.log", + "description": "", + "hits": 0, + "columns": [ + "zeek.capture_loss.ts_delta", + "host.name", + "zeek.capture_loss.peer", + "zeek.capture_loss.acks", + "zeek.capture_loss.gaps", + "zeek.capture_loss.percent_lost" + ], + "sort": [], + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query\":\"event.provider:zeek and event.dataset:capture_loss\",\"language\":\"kuery\"},\"highlightAll\":false,\"version\":true,\"aggs\":{\"2\":{\"date_histogram\":{\"field\":\"@timestamp\",\"fixed_interval\":\"30s\",\"time_zone\":\"America/Denver\",\"min_doc_count\":1}}},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" + } + }, + "references": [ + { + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern", + "id": "malcolm_beats_*" + } + ], + "migrationVersion": { + "search": "7.9.3" + } + }, + { + "id": "cc976b70-d7fc-11ee-b25e-e793ed358448", + "type": "visualization", + "namespaces": [ + "default" + ], + "updated_at": "2024-03-04T21:05:53.644Z", + "version": "Wzg5MCwxXQ==", + "attributes": { + "title": "Suricata Stats - Packets and Bytes", + "visState": "{\"title\":\"Suricata Stats - Packets and Bytes\",\"type\":\"metrics\",\"aggs\":[],\"params\":{\"id\":\"61ca57f0-469d-11e7-af02-69e470af7417\",\"type\":\"metric\",\"series\":[{\"id\":\"61ca57f1-469d-11e7-af02-69e470af7417\",\"color\":\"#54B399\",\"split_mode\":\"everything\",\"split_color_mode\":\"opensearchDashboards\",\"metrics\":[{\"size\":1,\"agg_with\":\"max\",\"order\":\"desc\",\"unit\":\"\",\"id\":\"61ca57f2-469d-11e7-af02-69e470af7417\",\"type\":\"top_hit\",\"field\":\"suricata.stats.capture.kernel_packets\",\"order_by\":\"@timestamp\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"line\",\"line_width\":1,\"point_size\":1,\"fill\":0.5,\"stacked\":\"none\",\"label\":\"Total Packets Seen\"},{\"id\":\"bd4560e0-d7e4-11ee-ad81-217e54128a4b\",\"color\":\"#68BC00\",\"split_mode\":\"everything\",\"metrics\":[{\"size\":1,\"agg_with\":\"max\",\"order\":\"desc\",\"id\":\"bd4560e1-d7e4-11ee-ad81-217e54128a4b\",\"type\":\"top_hit\",\"field\":\"suricata.stats.decoder.bytes\",\"order_by\":\"@timestamp\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"bytes\",\"chart_type\":\"line\",\"line_width\":1,\"point_size\":1,\"fill\":0.5,\"stacked\":\"none\",\"label\":\"Total Bytes Seen\"}],\"time_field\":\"@timestamp\",\"index_pattern\":\"malcolm_beats_*\",\"interval\":\"\",\"axis_position\":\"left\",\"axis_formatter\":\"number\",\"axis_scale\":\"normal\",\"show_legend\":1,\"show_grid\":1,\"tooltip_mode\":\"show_all\",\"default_index_pattern\":\"arkime_sessions3-*\",\"default_timefield\":\"firstPacket\",\"isModelInvalid\":false,\"background_color_rules\":[{\"id\":\"28bcc800-d7e4-11ee-ad81-217e54128a4b\"}],\"filter\":{\"query\":\"event.provider:suricata AND event.kind:metric\",\"language\":\"kuery\"},\"time_range_mode\":\"entire_time_range\"}}", + "uiStateJSON": "{}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[]}" + } + }, + "references": [], + "migrationVersion": { + "visualization": "7.10.0" + } + }, + { + "id": "9833e6a0-d7fd-11ee-b25e-e793ed358448", + "type": "visualization", + "namespaces": [ + "default" + ], + "updated_at": "2024-03-04T21:07:12.009Z", + "version": "Wzk5NiwxXQ==", + "attributes": { + "title": "Suricata Stats - Capture Loss", + "visState": "{\"title\":\"Suricata Stats - Capture Loss\",\"type\":\"metrics\",\"aggs\":[],\"params\":{\"id\":\"61ca57f0-469d-11e7-af02-69e470af7417\",\"type\":\"metric\",\"series\":[{\"id\":\"61ca57f1-469d-11e7-af02-69e470af7417\",\"color\":\"#54B399\",\"split_mode\":\"everything\",\"split_color_mode\":\"opensearchDashboards\",\"metrics\":[{\"size\":1,\"agg_with\":\"max\",\"order\":\"desc\",\"id\":\"61ca57f2-469d-11e7-af02-69e470af7417\",\"type\":\"top_hit\",\"field\":\"suricata.stats.pkts_dropped\",\"order_by\":\"@timestamp\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"line\",\"line_width\":1,\"point_size\":1,\"fill\":0.5,\"stacked\":\"none\",\"label\":\"Packets Dropped\"}],\"time_field\":\"@timestamp\",\"index_pattern\":\"malcolm_beats_*\",\"interval\":\"\",\"axis_position\":\"left\",\"axis_formatter\":\"number\",\"axis_scale\":\"normal\",\"show_legend\":1,\"show_grid\":1,\"tooltip_mode\":\"show_all\",\"default_index_pattern\":\"arkime_sessions3-*\",\"default_timefield\":\"firstPacket\",\"isModelInvalid\":false,\"background_color_rules\":[{\"id\":\"28bcc800-d7e4-11ee-ad81-217e54128a4b\"}],\"filter\":{\"query\":\"event.provider:suricata AND event.kind:metric\",\"language\":\"kuery\"},\"time_range_mode\":\"entire_time_range\"}}", + "uiStateJSON": "{}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[]}" + } + }, + "references": [], + "migrationVersion": { + "visualization": "7.10.0" + } + }, + { + "id": "6b7b9a40-faa1-11e6-86b1-cd7735ff7e23-ecs", + "type": "visualization", + "namespaces": [ + "default" + ], + "updated_at": "2024-03-04T21:05:59.817Z", + "version": "Wzk0MCwxXQ==", + "attributes": { + "title": "Network Traffic (Packets)", + "visState": "{\"title\":\"Network Traffic (Packets)\",\"type\":\"metrics\",\"aggs\":[],\"params\":{\"axis_formatter\":\"number\",\"axis_position\":\"left\",\"axis_scale\":\"normal\",\"default_index_pattern\":\"arkime_sessions3-*\",\"default_timefield\":\"firstPacket\",\"filter\":{\"language\":\"lucene\",\"query\":\"\"},\"id\":\"da1046f0-faa0-11e6-86b1-cd7735ff7e23\",\"index_pattern\":\"malcolm_beats_*\",\"interval\":\"auto\",\"isModelInvalid\":false,\"series\":[{\"axis_position\":\"right\",\"chart_type\":\"line\",\"color\":\"rgba(0,133,255,1)\",\"fill\":\"1\",\"formatter\":\"'0a'\",\"id\":\"49931900-ebf3-11ec-a401-f5db2d59e6af\",\"label\":\"Inbound\",\"line_width\":1,\"metrics\":[{\"unit\":\"1s\",\"id\":\"49931901-ebf3-11ec-a401-f5db2d59e6af\",\"type\":\"positive_rate\",\"field\":\"miscbeat.network.packets.rx\"}],\"point_size\":1,\"separate_axis\":0,\"split_mode\":\"terms\",\"stacked\":\"none\",\"type\":\"timeseries\",\"terms_field\":\"miscbeat.network.interface\",\"terms_size\":\"3\",\"terms_order_by\":\"_count\",\"value_template\":\"{{value}}/s\",\"split_color_mode\":\"gradient\"},{\"id\":\"75fba890-ebf3-11ec-a401-f5db2d59e6af\",\"color\":\"rgba(13,212,26,1)\",\"split_mode\":\"terms\",\"metrics\":[{\"unit\":\"1s\",\"id\":\"75fba891-ebf3-11ec-a401-f5db2d59e6af\",\"type\":\"positive_rate\",\"field\":\"miscbeat.network.packets.tx\"},{\"id\":\"96daba60-ebf3-11ec-a401-f5db2d59e6af\",\"type\":\"math\",\"variables\":[{\"id\":\"98e138c0-ebf3-11ec-a401-f5db2d59e6af\",\"name\":\"rate\",\"field\":\"75fba891-ebf3-11ec-a401-f5db2d59e6af\"}],\"script\":\"params.rate*-1\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"'0a'\",\"chart_type\":\"line\",\"line_width\":1,\"point_size\":1,\"fill\":\"1\",\"stacked\":\"none\",\"label\":\"Outbound\",\"type\":\"timeseries\",\"terms_size\":\"3\",\"terms_field\":\"miscbeat.network.interface\",\"terms_order_by\":\"_count\",\"split_color_mode\":\"gradient\",\"value_template\":\"{{value}}/s\"}],\"show_grid\":1,\"show_legend\":1,\"time_field\":\"@timestamp\",\"tooltip_mode\":\"show_all\",\"type\":\"timeseries\"}}", + "uiStateJSON": "{}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}" + } + }, + "references": [], + "migrationVersion": { + "visualization": "7.10.0" + } + }, + { + "id": "9c6a1610-d7d8-11ee-9ed3-e7afff29e59a", + "type": "search", + "namespaces": [ + "default" + ], + "updated_at": "2024-03-04T21:05:53.644Z", + "version": "Wzg5MywxXQ==", + "attributes": { + "title": "Packet Capture - Zeek stats.log", + "description": "", + "hits": 0, + "columns": [ + "host.name", + "zeek.stats.peer", + "zeek.stats.mem", + "zeek.stats.pkts_link", + "zeek.stats.pkts_proc", + "zeek.stats.pkts_dropped", + "zeek.stats.bytes_recv", + "zeek.stats.tcp_conns", + "zeek.stats.udp_conns", + "zeek.stats.icmp_conns", + "zeek.stats.files" + ], + "sort": [], + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query\":\"event.provider:zeek and event.dataset:stats\",\"language\":\"kuery\"},\"highlightAll\":false,\"version\":true,\"aggs\":{\"2\":{\"date_histogram\":{\"field\":\"@timestamp\",\"fixed_interval\":\"30s\",\"time_zone\":\"America/Denver\",\"min_doc_count\":1}}},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" + } + }, + "references": [ + { + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern", + "id": "malcolm_beats_*" + } + ], + "migrationVersion": { + "search": "7.9.3" + } + }, + { + "id": "b0aabf40-d7fa-11ee-b25e-e793ed358448", + "type": "search", + "namespaces": [ + "default" + ], + "updated_at": "2024-03-04T21:05:53.644Z", + "version": "Wzg5NCwxXQ==", + "attributes": { + "title": "Packet Capture - Suricata Stats", + "description": "", + "hits": 0, + "columns": [ + "host.name", + "suricata.stats.capture.kernel_packets", + "suricata.stats.pkts_dropped", + "suricata.stats.capture.errors", + "suricata.stats.decoder.bytes", + "suricata.stats.decoder.ethernet", + "suricata.stats.decoder.ipv4", + "suricata.stats.decoder.ipv6", + "suricata.stats.detect.engines.rules_loaded", + "suricata.stats.detect.alert" + ], + "sort": [], + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query\":\"event.provider:suricata AND event.kind:metric\",\"language\":\"lucene\"},\"highlightAll\":false,\"version\":true,\"aggs\":{\"2\":{\"date_histogram\":{\"field\":\"@timestamp\",\"fixed_interval\":\"30m\",\"time_zone\":\"America/Denver\",\"min_doc_count\":1}}},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" + } + }, + "references": [ + { + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern", + "id": "malcolm_beats_*" + } + ], + "migrationVersion": { + "search": "7.9.3" + } + }, + { + "id": "abcfca50-d7dd-11ee-b25e-e793ed358448", + "type": "visualization", + "namespaces": [ + "default" + ], + "updated_at": "2024-03-04T21:05:53.644Z", + "version": "Wzg5NSwxXQ==", + "attributes": { + "title": "Zeek Analyzer Messages", + "visState": "{\"title\":\"Zeek Analyzer Messages\",\"type\":\"table\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"4\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"zeek.analyzer.cause\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":50,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"-\",\"customLabel\":\"Cause\"},\"schema\":\"bucket\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"zeek.analyzer.analyzer_kind\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":25,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"-\",\"customLabel\":\"Class\"},\"schema\":\"bucket\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"zeek.analyzer.analyzer_name\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"-\",\"customLabel\":\"Analyzer\"},\"schema\":\"bucket\"}],\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\"}}", + "uiStateJSON": "{}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[]}" + }, + "savedSearchRefName": "search_0" + }, + "references": [ + { + "name": "search_0", + "type": "search", + "id": "351ef380-d7d9-11ee-9ed3-e7afff29e59a" + } + ], + "migrationVersion": { + "visualization": "7.10.0" + } + }, + { + "id": "351ef380-d7d9-11ee-9ed3-e7afff29e59a", + "type": "search", + "namespaces": [ + "default" + ], + "updated_at": "2024-03-04T21:05:53.644Z", + "version": "Wzg5NiwxXQ==", + "attributes": { + "title": "Packet Capture - Zeek analyzer.log", + "description": "", + "hits": 0, + "columns": [ + "host.name", + "zeek.analyzer.analyzer_kind", + "zeek.analyzer.analyzer_name", + "zeek.analyzer.cause", + "zeek.analyzer.failure_reason", + "source.ip", + "destination.ip", + "event.id" + ], + "sort": [], + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query\":\"event.provider:zeek and event.dataset:analyzer\",\"language\":\"kuery\"},\"highlightAll\":false,\"version\":true,\"aggs\":{\"2\":{\"date_histogram\":{\"field\":\"firstPacket\",\"fixed_interval\":\"30s\",\"time_zone\":\"America/Denver\",\"min_doc_count\":1}}},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" + } + }, + "references": [ + { + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern", + "id": "arkime_sessions3-*" + } + ], + "migrationVersion": { + "search": "7.9.3" + } + }, + { + "id": "c75b6df0-d7e3-11ee-b25e-e793ed358448", + "type": "visualization", + "namespaces": [ + "default" + ], + "updated_at": "2024-03-04T21:05:53.644Z", + "version": "Wzg5NywxXQ==", + "attributes": { + "title": "Zeek - Reporter Categories", + "visState": "{\"title\":\"Zeek - Reporter Categories\",\"type\":\"pie\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"zeek.reporter.level\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":10,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"},\"schema\":\"segment\"}],\"params\":{\"type\":\"pie\",\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"isDonut\":false,\"labels\":{\"show\":true,\"values\":true,\"last_level\":true,\"truncate\":100}}}", + "uiStateJSON": "{\"vis\":{\"legendOpen\":false}}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[]}" + }, + "savedSearchRefName": "search_0" + }, + "references": [ + { + "name": "search_0", + "type": "search", + "id": "17619ea0-d7da-11ee-9ed3-e7afff29e59a" + } + ], + "migrationVersion": { + "visualization": "7.10.0" + } + }, + { + "id": "17619ea0-d7da-11ee-9ed3-e7afff29e59a", + "type": "search", + "namespaces": [ + "default" + ], + "updated_at": "2024-03-04T21:05:53.644Z", + "version": "Wzg5OCwxXQ==", + "attributes": { + "title": "Packet Capture - Zeek reporter.log", + "description": "", + "hits": 0, + "columns": [ + "host.name", + "zeek.reporter.level", + "zeek.reporter.location", + "zeek.reporter.msg" + ], + "sort": [], + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query\":\"event.provider:zeek and event.dataset:reporter\",\"language\":\"kuery\"},\"highlightAll\":false,\"version\":true,\"aggs\":{\"2\":{\"date_histogram\":{\"field\":\"@timestamp\",\"fixed_interval\":\"30d\",\"time_zone\":\"America/Denver\",\"min_doc_count\":1}}},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" + } + }, + "references": [ + { + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern", + "id": "malcolm_beats_*" + } + ], + "migrationVersion": { + "search": "7.9.3" + } + }, + { + "id": "089b85d0-1b16-11e7-b09e-037021c4f8df-ecs", + "type": "visualization", + "namespaces": [ + "default" + ], + "updated_at": "2024-03-04T21:05:59.817Z", + "version": "Wzk0MSwxXQ==", + "attributes": { + "title": "Network Traffic (Bytes)", + "visState": "{\"title\":\"Network Traffic (Bytes)\",\"type\":\"metrics\",\"aggs\":[],\"params\":{\"axis_formatter\":\"number\",\"axis_position\":\"left\",\"axis_scale\":\"normal\",\"default_index_pattern\":\"arkime_sessions3-*\",\"default_timefield\":\"firstPacket\",\"filter\":{\"language\":\"lucene\",\"query\":\"\"},\"id\":\"da1046f0-faa0-11e6-86b1-cd7735ff7e23\",\"index_pattern\":\"malcolm_beats_*\",\"interval\":\"auto\",\"isModelInvalid\":false,\"series\":[{\"axis_position\":\"right\",\"chart_type\":\"line\",\"color\":\"rgba(0,133,255,1)\",\"fill\":\"1\",\"formatter\":\"bytes\",\"id\":\"6d8b8ab0-ebf1-11ec-a401-f5db2d59e6af\",\"line_width\":1,\"metrics\":[{\"unit\":\"1s\",\"id\":\"6d8b8ab1-ebf1-11ec-a401-f5db2d59e6af\",\"type\":\"positive_rate\",\"field\":\"miscbeat.network.bytes.rx\"}],\"point_size\":1,\"separate_axis\":0,\"split_mode\":\"terms\",\"stacked\":\"none\",\"label\":\"Inbound\",\"type\":\"timeseries\",\"terms_field\":\"miscbeat.network.interface\",\"terms_size\":\"3\",\"terms_order_by\":\"_key\",\"value_template\":\"{{value}}/s\",\"split_color_mode\":\"gradient\"},{\"id\":\"b5977de0-ebf2-11ec-a401-f5db2d59e6af\",\"color\":\"rgba(13,212,26,1)\",\"split_mode\":\"terms\",\"metrics\":[{\"unit\":\"1s\",\"id\":\"b5977de1-ebf2-11ec-a401-f5db2d59e6af\",\"type\":\"positive_rate\",\"field\":\"miscbeat.network.bytes.tx\"},{\"id\":\"cdfb1540-ebf2-11ec-a401-f5db2d59e6af\",\"type\":\"math\",\"variables\":[{\"id\":\"d1b9caf0-ebf2-11ec-a401-f5db2d59e6af\",\"name\":\"rate\",\"field\":\"b5977de1-ebf2-11ec-a401-f5db2d59e6af\"}],\"script\":\"params.rate*-1\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"bytes\",\"chart_type\":\"line\",\"line_width\":1,\"point_size\":1,\"fill\":\"1\",\"stacked\":\"none\",\"label\":\"Outbound\",\"split_color_mode\":\"gradient\",\"type\":\"timeseries\",\"terms_size\":\"3\",\"terms_order_by\":\"_key\",\"terms_field\":\"miscbeat.network.interface\",\"value_template\":\"{{value}}/s\"}],\"show_grid\":1,\"show_legend\":1,\"time_field\":\"@timestamp\",\"tooltip_mode\":\"show_all\",\"type\":\"timeseries\"}}", + "uiStateJSON": "{}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}" + } + }, + "references": [], + "migrationVersion": { + "visualization": "7.10.0" + } + } + ] +} \ No newline at end of file diff --git a/dashboards/dashboards/beats/79202ee0-d811-11ee-820d-dd9fd73a3921.json b/dashboards/dashboards/beats/79202ee0-d811-11ee-820d-dd9fd73a3921.json new file mode 100644 index 000000000..d714f6d1f --- /dev/null +++ b/dashboards/dashboards/beats/79202ee0-d811-11ee-820d-dd9fd73a3921.json @@ -0,0 +1,349 @@ +{ + "version": "2.12.0", + "objects": [ + { + "id": "79202ee0-d811-11ee-820d-dd9fd73a3921", + "type": "dashboard", + "namespaces": [ + "default" + ], + "updated_at": "2024-03-01T22:03:46.831Z", + "version": "WzEwOTgsMV0=", + "attributes": { + "title": "Windows Events", + "hits": 0, + "description": "Windows events and resource statistics sent via Fluent Bit", + "panelsJSON": "[{\"version\":\"2.12.0\",\"gridData\":{\"h\":15,\"i\":\"346bb696-5fa2-4504-a1d8-5a6f51244c7b\",\"w\":18,\"x\":0,\"y\":0},\"panelIndex\":\"346bb696-5fa2-4504-a1d8-5a6f51244c7b\",\"embeddableConfig\":{\"hidePanelTitles\":true,\"vis\":{\"sortColumn\":{\"colIndex\":2,\"direction\":\"desc\"}}},\"panelRefName\":\"panel_0\"},{\"version\":\"2.12.0\",\"gridData\":{\"h\":15,\"i\":\"9c39d8b3-ad8c-4247-b97f-9736e469c988\",\"w\":30,\"x\":18,\"y\":0},\"panelIndex\":\"9c39d8b3-ad8c-4247-b97f-9736e469c988\",\"embeddableConfig\":{\"hidePanelTitles\":true},\"panelRefName\":\"panel_1\"},{\"version\":\"2.12.0\",\"gridData\":{\"h\":36,\"i\":\"bccfb126-a864-4c11-a8c7-a9a1286c8f0f\",\"w\":30,\"x\":0,\"y\":15},\"panelIndex\":\"bccfb126-a864-4c11-a8c7-a9a1286c8f0f\",\"embeddableConfig\":{},\"panelRefName\":\"panel_2\"},{\"version\":\"2.12.0\",\"gridData\":{\"h\":15,\"i\":\"bc116b54-f251-4e77-833c-c557b5d5c1d7\",\"w\":18,\"x\":30,\"y\":15},\"panelIndex\":\"bc116b54-f251-4e77-833c-c557b5d5c1d7\",\"embeddableConfig\":{\"hidePanelTitles\":true,\"vis\":{\"sortColumn\":{\"colIndex\":1,\"direction\":\"desc\"}},\"table\":null},\"panelRefName\":\"panel_3\"},{\"version\":\"2.12.0\",\"gridData\":{\"h\":21,\"i\":\"5566d8f7-5c04-4f92-acf3-5af407b28b0d\",\"w\":18,\"x\":30,\"y\":30},\"panelIndex\":\"5566d8f7-5c04-4f92-acf3-5af407b28b0d\",\"embeddableConfig\":{\"hidePanelTitles\":true,\"vis\":{\"sortColumn\":{\"colIndex\":1,\"direction\":\"desc\"}}},\"panelRefName\":\"panel_4\"},{\"version\":\"2.12.0\",\"gridData\":{\"h\":30,\"i\":\"49f38efc-1ab3-4e38-96e5-b0458c026491\",\"w\":48,\"x\":0,\"y\":66},\"panelIndex\":\"49f38efc-1ab3-4e38-96e5-b0458c026491\",\"embeddableConfig\":{},\"panelRefName\":\"panel_5\"},{\"version\":\"2.12.0\",\"gridData\":{\"h\":25,\"i\":\"0249a576-f162-4bf9-aa2d-bf85552665c7\",\"w\":48,\"x\":0,\"y\":96},\"panelIndex\":\"0249a576-f162-4bf9-aa2d-bf85552665c7\",\"embeddableConfig\":{},\"panelRefName\":\"panel_6\"},{\"version\":\"2.12.0\",\"gridData\":{\"h\":15,\"i\":\"65476182-297e-4bbf-a7af-b0ff0584eeb0\",\"w\":24,\"x\":0,\"y\":51},\"panelIndex\":\"65476182-297e-4bbf-a7af-b0ff0584eeb0\",\"embeddableConfig\":{\"hidePanelTitles\":true},\"panelRefName\":\"panel_7\"},{\"version\":\"2.12.0\",\"gridData\":{\"x\":24,\"y\":51,\"w\":24,\"h\":15,\"i\":\"0bf80b0d-91f3-47ca-b2a3-ea6844c9727a\"},\"panelIndex\":\"0bf80b0d-91f3-47ca-b2a3-ea6844c9727a\",\"embeddableConfig\":{\"hidePanelTitles\":true},\"panelRefName\":\"panel_8\"}]", + "optionsJSON": "{\"hidePanelTitles\":false,\"useMargins\":true}", + "version": 1, + "timeRestore": false, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"language\":\"kuery\",\"query\":\"\"},\"filter\":[]}" + } + }, + "references": [ + { + "name": "panel_0", + "type": "visualization", + "id": "0100b010-d811-11ee-820d-dd9fd73a3921" + }, + { + "name": "panel_1", + "type": "visualization", + "id": "6a4710f0-d811-11ee-820d-dd9fd73a3921" + }, + { + "name": "panel_2", + "type": "visualization", + "id": "db80a970-d811-11ee-820d-dd9fd73a3921" + }, + { + "name": "panel_3", + "type": "visualization", + "id": "4d51c700-d812-11ee-820d-dd9fd73a3921" + }, + { + "name": "panel_4", + "type": "visualization", + "id": "f7fcb9d0-d812-11ee-820d-dd9fd73a3921" + }, + { + "name": "panel_5", + "type": "search", + "id": "be2f24d0-d809-11ee-820d-dd9fd73a3921" + }, + { + "name": "panel_6", + "type": "search", + "id": "3770db80-d815-11ee-820d-dd9fd73a3921" + }, + { + "name": "panel_7", + "type": "visualization", + "id": "7e9a8500-d816-11ee-820d-dd9fd73a3921" + }, + { + "name": "panel_8", + "type": "visualization", + "id": "008fb710-d817-11ee-820d-dd9fd73a3921" + } + ], + "migrationVersion": { + "dashboard": "7.9.3" + } + }, + { + "id": "0100b010-d811-11ee-820d-dd9fd73a3921", + "type": "visualization", + "namespaces": [ + "default" + ], + "updated_at": "2024-03-01T21:16:43.792Z", + "version": "Wzk4MCwxXQ==", + "attributes": { + "title": "Windows Events by Host", + "visState": "{\"title\":\"Windows Events by Host\",\"type\":\"table\",\"aggs\":[{\"id\":\"4\",\"enabled\":true,\"type\":\"top_hits\",\"params\":{\"field\":\"@timestamp\",\"aggregate\":\"concat\",\"size\":1,\"sortField\":\"@timestamp\",\"sortOrder\":\"desc\",\"customLabel\":\"Last Log\"},\"schema\":\"metric\"},{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"host.name\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":50,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"-\",\"customLabel\":\"Host Forwarder\"},\"schema\":\"bucket\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"miscbeat.winlog.Computer\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":50,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"-\",\"customLabel\":\"Computer Name\"},\"schema\":\"bucket\"}],\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\"}}", + "uiStateJSON": "{}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[]}" + }, + "savedSearchRefName": "search_0" + }, + "references": [ + { + "name": "search_0", + "type": "search", + "id": "be2f24d0-d809-11ee-820d-dd9fd73a3921" + } + ], + "migrationVersion": { + "visualization": "7.10.0" + } + }, + { + "id": "6a4710f0-d811-11ee-820d-dd9fd73a3921", + "type": "visualization", + "namespaces": [ + "default" + ], + "updated_at": "2024-03-01T21:20:23.112Z", + "version": "Wzk4MywxXQ==", + "attributes": { + "title": "Windows Events over Time", + "visState": "{\"title\":\"Windows Events over Time\",\"type\":\"histogram\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"params\":{\"field\":\"@timestamp\",\"timeRange\":{\"from\":\"now/d\",\"to\":\"now/d\"},\"useNormalizedOpenSearchInterval\":true,\"scaleMetricValues\":false,\"interval\":\"auto\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{}},\"schema\":\"segment\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"event.dataset\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":5,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"},\"schema\":\"group\"}],\"params\":{\"type\":\"histogram\",\"grid\":{\"categoryLines\":false},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"filter\":true,\"truncate\":100},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"square root\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Count\"}}],\"seriesParams\":[{\"show\":true,\"type\":\"histogram\",\"mode\":\"stacked\",\"data\":{\"label\":\"Count\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"lineWidth\":2,\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"bottom\",\"times\":[],\"addTimeMarker\":false,\"labels\":{\"show\":false},\"thresholdLine\":{\"show\":false,\"value\":10,\"width\":1,\"style\":\"full\",\"color\":\"#E7664C\"}}}", + "uiStateJSON": "{\"vis\":{\"legendOpen\":true}}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[]}" + }, + "savedSearchRefName": "search_0" + }, + "references": [ + { + "name": "search_0", + "type": "search", + "id": "be2f24d0-d809-11ee-820d-dd9fd73a3921" + } + ], + "migrationVersion": { + "visualization": "7.10.0" + } + }, + { + "id": "db80a970-d811-11ee-820d-dd9fd73a3921", + "type": "visualization", + "namespaces": [ + "default" + ], + "updated_at": "2024-03-01T21:28:23.050Z", + "version": "Wzk4OCwxXQ==", + "attributes": { + "title": "Windows Event Provider", + "visState": "{\"title\":\"Windows Event Provider\",\"type\":\"horizontal_bar\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"miscbeat.winlog.ProviderName\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":25,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Event Provider\"},\"schema\":\"segment\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"miscbeat.winlog.ProviderName\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":25,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Event Provider\"},\"schema\":\"group\"}],\"params\":{\"type\":\"histogram\",\"grid\":{\"categoryLines\":false},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":50},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"square root\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":75,\"filter\":true,\"truncate\":100},\"title\":{\"text\":\"Count\"}}],\"seriesParams\":[{\"show\":true,\"type\":\"histogram\",\"mode\":\"stacked\",\"data\":{\"label\":\"Count\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"lineWidth\":2,\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false,\"labels\":{\"show\":true},\"thresholdLine\":{\"show\":false,\"value\":10,\"width\":1,\"style\":\"full\",\"color\":\"#E7664C\"}}}", + "uiStateJSON": "{\"vis\":{\"legendOpen\":false}}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[]}" + }, + "savedSearchRefName": "search_0" + }, + "references": [ + { + "name": "search_0", + "type": "search", + "id": "be2f24d0-d809-11ee-820d-dd9fd73a3921" + } + ], + "migrationVersion": { + "visualization": "7.10.0" + } + }, + { + "id": "4d51c700-d812-11ee-820d-dd9fd73a3921", + "type": "visualization", + "namespaces": [ + "default" + ], + "updated_at": "2024-03-01T21:26:30.332Z", + "version": "Wzk4NywxXQ==", + "attributes": { + "title": "Windows Event Results", + "visState": "{\"title\":\"Windows Event Results\",\"type\":\"table\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"event.result\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Event Result\"},\"schema\":\"bucket\"}],\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\"}}", + "uiStateJSON": "{}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[]}" + }, + "savedSearchRefName": "search_0" + }, + "references": [ + { + "name": "search_0", + "type": "search", + "id": "be2f24d0-d809-11ee-820d-dd9fd73a3921" + } + ], + "migrationVersion": { + "visualization": "7.10.0" + } + }, + { + "id": "f7fcb9d0-d812-11ee-820d-dd9fd73a3921", + "type": "visualization", + "namespaces": [ + "default" + ], + "updated_at": "2024-03-01T21:30:47.661Z", + "version": "Wzk4OSwxXQ==", + "attributes": { + "title": "Windows Event Insertion Strings", + "visState": "{\"title\":\"Windows Event Insertion Strings\",\"type\":\"table\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"miscbeat.winlog.StringInserts\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":250,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Insertion Strings\"},\"schema\":\"bucket\"}],\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\"}}", + "uiStateJSON": "{\"vis\":{\"sortColumn\":{\"colIndex\":1,\"direction\":\"desc\"}}}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[]}" + }, + "savedSearchRefName": "search_0" + }, + "references": [ + { + "name": "search_0", + "type": "search", + "id": "be2f24d0-d809-11ee-820d-dd9fd73a3921" + } + ], + "migrationVersion": { + "visualization": "7.10.0" + } + }, + { + "id": "be2f24d0-d809-11ee-820d-dd9fd73a3921", + "type": "search", + "namespaces": [ + "default" + ], + "updated_at": "2024-03-01T20:24:45.212Z", + "version": "Wzk3OCwxXQ==", + "attributes": { + "title": "Windows Event Logs", + "description": "", + "hits": 0, + "columns": [ + "host.name", + "miscbeat.winlog.Computer", + "event.dataset", + "event.id", + "miscbeat.winlog.ProviderName", + "event.result", + "event.original" + ], + "sort": [], + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query\":\"event.module:(winlog OR winevtlog)\",\"language\":\"lucene\"},\"highlightAll\":false,\"version\":true,\"aggs\":{\"2\":{\"date_histogram\":{\"field\":\"@timestamp\",\"fixed_interval\":\"30s\",\"time_zone\":\"America/Denver\",\"min_doc_count\":1}}},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" + } + }, + "references": [ + { + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern", + "id": "malcolm_beats_*" + } + ], + "migrationVersion": { + "search": "7.9.3" + } + }, + { + "id": "3770db80-d815-11ee-820d-dd9fd73a3921", + "type": "search", + "namespaces": [ + "default" + ], + "updated_at": "2024-03-01T21:46:53.112Z", + "version": "WzEwMDQsMV0=", + "attributes": { + "title": "Windows Resource Utilization", + "description": "", + "hits": 0, + "columns": [ + "host.name", + "miscbeat.winstat.cpu_utilization", + "miscbeat.winstat.physical_total", + "miscbeat.winstat.physical_used", + "miscbeat.winstat.physical_available", + "miscbeat.winstat.processes", + "miscbeat.winstat.threads", + "miscbeat.winstat.handles" + ], + "sort": [], + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query\":\"event.module:winstat\",\"language\":\"kuery\"},\"highlightAll\":false,\"version\":true,\"aggs\":{\"2\":{\"date_histogram\":{\"field\":\"@timestamp\",\"fixed_interval\":\"30s\",\"time_zone\":\"America/Denver\",\"min_doc_count\":1}}},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" + } + }, + "references": [ + { + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern", + "id": "malcolm_beats_*" + } + ], + "migrationVersion": { + "search": "7.9.3" + } + }, + { + "id": "7e9a8500-d816-11ee-820d-dd9fd73a3921", + "type": "visualization", + "namespaces": [ + "default" + ], + "updated_at": "2024-03-01T21:56:02.000Z", + "version": "WzEwNjEsMV0=", + "attributes": { + "title": "Windows RAM Usage", + "visState": "{\"title\":\"Windows RAM Usage\",\"type\":\"metrics\",\"aggs\":[],\"params\":{\"id\":\"61ca57f0-469d-11e7-af02-69e470af7417\",\"type\":\"gauge\",\"series\":[{\"id\":\"61ca57f1-469d-11e7-af02-69e470af7417\",\"color\":\"#54B399\",\"split_mode\":\"terms\",\"split_color_mode\":\"opensearchDashboards\",\"metrics\":[{\"size\":1,\"agg_with\":\"max\",\"order\":\"desc\",\"id\":\"61ca57f2-469d-11e7-af02-69e470af7417\",\"type\":\"top_hit\",\"field\":\"miscbeat.winstat.physical_used\",\"order_by\":\"@timestamp\"},{\"size\":1,\"agg_with\":\"max\",\"order\":\"desc\",\"id\":\"e4b6eb90-d815-11ee-a6da-0f56a6d37163\",\"type\":\"top_hit\",\"field\":\"miscbeat.winstat.physical_total\",\"order_by\":\"@timestamp\"},{\"id\":\"fe21e0d0-d815-11ee-a6da-0f56a6d37163\",\"type\":\"math\",\"variables\":[{\"id\":\"01e57880-d816-11ee-a6da-0f56a6d37163\",\"name\":\"used\",\"field\":\"61ca57f2-469d-11e7-af02-69e470af7417\"},{\"id\":\"082756a0-d816-11ee-a6da-0f56a6d37163\",\"name\":\"total\",\"field\":\"e4b6eb90-d815-11ee-a6da-0f56a6d37163\"}],\"script\":\"(params.used/params.total)*100\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"00.\",\"chart_type\":\"line\",\"line_width\":1,\"point_size\":1,\"fill\":0.5,\"stacked\":\"none\",\"terms_field\":\"host.name\",\"label\":\"RAM Usage\",\"terms_order_by\":\"_count\",\"value_template\":\"{{value}}%\"}],\"time_field\":\"@timestamp\",\"index_pattern\":\"malcolm_beats_*\",\"interval\":\"\",\"axis_position\":\"left\",\"axis_formatter\":\"number\",\"axis_scale\":\"normal\",\"show_legend\":1,\"show_grid\":1,\"tooltip_mode\":\"show_all\",\"default_index_pattern\":\"arkime_sessions3-*\",\"default_timefield\":\"firstPacket\",\"isModelInvalid\":false,\"gauge_color_rules\":[{\"id\":\"8db33970-d815-11ee-a6da-0f56a6d37163\"}],\"gauge_width\":10,\"gauge_inner_width\":10,\"gauge_style\":\"half\",\"filter\":{\"query\":\"event.module:winstat\",\"language\":\"kuery\"},\"time_range_mode\":\"entire_time_range\",\"gauge_max\":\"100\"}}", + "uiStateJSON": "{}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}" + } + }, + "references": [], + "migrationVersion": { + "visualization": "7.10.0" + } + }, + { + "id": "008fb710-d817-11ee-820d-dd9fd73a3921", + "type": "visualization", + "namespaces": [ + "default" + ], + "updated_at": "2024-03-01T21:59:40.033Z", + "version": "WzEwOTIsMV0=", + "attributes": { + "title": "Windows CPU Usage", + "visState": "{\"title\":\"Windows CPU Usage\",\"type\":\"metrics\",\"aggs\":[],\"params\":{\"id\":\"61ca57f0-469d-11e7-af02-69e470af7417\",\"type\":\"gauge\",\"series\":[{\"id\":\"61ca57f1-469d-11e7-af02-69e470af7417\",\"color\":\"#54B399\",\"split_mode\":\"terms\",\"split_color_mode\":\"opensearchDashboards\",\"metrics\":[{\"size\":1,\"agg_with\":\"noop\",\"order\":\"desc\",\"id\":\"61ca57f2-469d-11e7-af02-69e470af7417\",\"type\":\"top_hit\",\"field\":\"miscbeat.winstat.cpu_utilization\",\"order_by\":\"@timestamp\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"00\",\"chart_type\":\"line\",\"line_width\":1,\"point_size\":1,\"fill\":0.5,\"stacked\":\"none\",\"label\":\"CPU Usage\",\"value_template\":\"{{value}}%\",\"filter\":{\"query\":\"event.module:winstat\",\"language\":\"kuery\"},\"terms_field\":\"host.name\"}],\"time_field\":\"@timestamp\",\"index_pattern\":\"malcolm_beats_*\",\"interval\":\"\",\"axis_position\":\"left\",\"axis_formatter\":\"number\",\"axis_scale\":\"normal\",\"show_legend\":1,\"show_grid\":1,\"tooltip_mode\":\"show_all\",\"default_index_pattern\":\"arkime_sessions3-*\",\"default_timefield\":\"firstPacket\",\"isModelInvalid\":false,\"gauge_color_rules\":[{\"id\":\"a50173c0-d816-11ee-a6da-0f56a6d37163\"}],\"gauge_width\":10,\"gauge_inner_width\":10,\"gauge_style\":\"half\",\"gauge_max\":\"100\",\"time_range_mode\":\"entire_time_range\"}}", + "uiStateJSON": "{}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}" + } + }, + "references": [], + "migrationVersion": { + "visualization": "7.10.0" + } + } + ] +} \ No newline at end of file diff --git a/dashboards/dashboards/beats/7a7e0a60-e8e8-11ec-b9d4-4569bb965430.json b/dashboards/dashboards/beats/7a7e0a60-e8e8-11ec-b9d4-4569bb965430.json index dcdf9e469..f815ffdc7 100644 --- a/dashboards/dashboards/beats/7a7e0a60-e8e8-11ec-b9d4-4569bb965430.json +++ b/dashboards/dashboards/beats/7a7e0a60-e8e8-11ec-b9d4-4569bb965430.json @@ -260,7 +260,7 @@ "sort": [], "version": 1, "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"highlightAll\":true,\"version\":true,\"query\":{\"language\":\"kuery\",\"query\":\"event.module:auditlog AND miscbeat.auditlog:*\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" + "searchSourceJSON": "{\"highlightAll\":false,\"version\":true,\"query\":{\"language\":\"kuery\",\"query\":\"event.module:auditlog AND miscbeat.auditlog:*\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" } }, "references": [ diff --git a/dashboards/dashboards/beats/903f42c0-f634-11ec-828d-2fb7a4a26e1f.json b/dashboards/dashboards/beats/903f42c0-f634-11ec-828d-2fb7a4a26e1f.json index d9ffc162c..34cfd2492 100644 --- a/dashboards/dashboards/beats/903f42c0-f634-11ec-828d-2fb7a4a26e1f.json +++ b/dashboards/dashboards/beats/903f42c0-f634-11ec-828d-2fb7a4a26e1f.json @@ -12,7 +12,7 @@ "attributes": { "title": "Malcolm Sensor File/Directory Integrity", "hits": 0, - "description": "", + "description": "AIDE file/directory integrity report for Malcolm sensors and aggregators", "panelsJSON": "[{\"version\":\"2.0.0\",\"gridData\":{\"x\":0,\"y\":0,\"w\":18,\"h\":7,\"i\":\"1b520d1b-b30e-4216-8c83-3eff88564503\"},\"panelIndex\":\"1b520d1b-b30e-4216-8c83-3eff88564503\",\"embeddableConfig\":{\"hidePanelTitles\":true},\"panelRefName\":\"panel_0\"},{\"version\":\"2.0.0\",\"gridData\":{\"x\":18,\"y\":0,\"w\":30,\"h\":14,\"i\":\"1a09a091-d27c-48df-a145-5a33ecc33ffb\"},\"panelIndex\":\"1a09a091-d27c-48df-a145-5a33ecc33ffb\",\"embeddableConfig\":{},\"panelRefName\":\"panel_1\"},{\"version\":\"2.0.0\",\"gridData\":{\"x\":0,\"y\":7,\"w\":18,\"h\":28,\"i\":\"4c570390-7394-4129-b637-81e58e3fa066\"},\"panelIndex\":\"4c570390-7394-4129-b637-81e58e3fa066\",\"embeddableConfig\":{},\"panelRefName\":\"panel_2\"},{\"version\":\"2.0.0\",\"gridData\":{\"x\":18,\"y\":14,\"w\":30,\"h\":21,\"i\":\"3e3af3d3-f0e6-4472-b27a-a0cc71db516b\"},\"panelIndex\":\"3e3af3d3-f0e6-4472-b27a-a0cc71db516b\",\"embeddableConfig\":{},\"panelRefName\":\"panel_3\"},{\"version\":\"2.0.0\",\"gridData\":{\"x\":0,\"y\":35,\"w\":48,\"h\":22,\"i\":\"93ccff9d-7037-4a11-9478-ee6f5341831e\"},\"panelIndex\":\"93ccff9d-7037-4a11-9478-ee6f5341831e\",\"embeddableConfig\":{},\"panelRefName\":\"panel_4\"}]", "optionsJSON": "{\"hidePanelTitles\":false,\"useMargins\":true}", "version": 1, @@ -195,7 +195,7 @@ "sort": [], "version": 1, "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"highlightAll\":true,\"version\":true,\"query\":{\"query\":\"event.module:aide\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" + "searchSourceJSON": "{\"highlightAll\":false,\"version\":true,\"query\":{\"query\":\"event.module:aide\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" } }, "references": [ diff --git a/dashboards/dashboards/beats/Filebeat-nginx-logs.json b/dashboards/dashboards/beats/Filebeat-nginx-logs.json index 46f35dcdd..879a0ae00 100644 --- a/dashboards/dashboards/beats/Filebeat-nginx-logs.json +++ b/dashboards/dashboards/beats/Filebeat-nginx-logs.json @@ -10,8 +10,8 @@ "updated_at": "2022-06-01T19:53:27.884Z", "version": "WzEwMTgsMV0=", "attributes": { - "title": "Malcolm nginx Access and Error Logs", - "description": "Logs from Malcolm's own nginx instance", + "title": "nginx Access and Error Logs", + "description": "nginx Access and Error logs, including from Malcolm's own nginx instance", "hits": 0, "description": "", "panelsJSON": "[{\"version\":\"2.0.0\",\"gridData\":{\"x\":0,\"y\":16,\"w\":48,\"h\":15,\"i\":\"11\"},\"panelIndex\":\"11\",\"embeddableConfig\":{\"columns\":[\"log.level\",\"error.message\"],\"sort\":[\"@timestamp\",\"desc\"]},\"panelRefName\":\"panel_0\"},{\"version\":\"2.0.0\",\"gridData\":{\"x\":0,\"y\":31,\"w\":48,\"h\":23,\"i\":\"16\"},\"panelIndex\":\"16\",\"embeddableConfig\":{\"columns\":[\"url.original\",\"http.request.method\",\"http.response.status_code\",\"http.response.body.bytes\"],\"sort\":[\"@timestamp\",\"desc\"]},\"panelRefName\":\"panel_1\"},{\"version\":\"2.0.0\",\"gridData\":{\"x\":0,\"y\":4,\"w\":48,\"h\":12,\"i\":\"18\"},\"panelIndex\":\"18\",\"embeddableConfig\":{},\"panelRefName\":\"panel_2\"},{\"version\":\"2.0.0\",\"gridData\":{\"x\":0,\"y\":0,\"w\":48,\"h\":4,\"i\":\"19\"},\"panelIndex\":\"19\",\"embeddableConfig\":{},\"panelRefName\":\"panel_3\"}]", @@ -19,7 +19,7 @@ "version": 1, "timeRestore": false, "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"version\":true,\"highlightAll\":true,\"filter\":[]}" + "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"version\":true,\"highlightAll\":false,\"filter\":[]}" } }, "references": [ @@ -64,7 +64,7 @@ "description": "", "hits": 0, "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"filter\":[],\"highlightAll\":true,\"query\":{\"query\":\"event.module:nginx AND error.message:*\",\"language\":\"kuery\"},\"version\":true,\"highlight\":{\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fields\":{\"*\":{}},\"pre_tags\":[\"@kibana-highlighted-field@\"],\"require_field_match\":false,\"fragment_size\":2147483647},\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" + "searchSourceJSON": "{\"filter\":[],\"highlightAll\":false,\"query\":{\"query\":\"event.module:nginx AND error.message:*\",\"language\":\"kuery\"},\"version\":true,\"highlight\":{\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fields\":{\"*\":{}},\"pre_tags\":[\"@kibana-highlighted-field@\"],\"require_field_match\":false,\"fragment_size\":2147483647},\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" }, "sort": [ [ @@ -104,7 +104,7 @@ "description": "", "hits": 0, "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"filter\":[],\"highlightAll\":true,\"query\":{\"query\":\"event.module:nginx AND url.original:*\",\"language\":\"kuery\"},\"version\":true,\"highlight\":{\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fields\":{\"*\":{}},\"pre_tags\":[\"@kibana-highlighted-field@\"],\"require_field_match\":false,\"fragment_size\":2147483647},\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" + "searchSourceJSON": "{\"filter\":[],\"highlightAll\":false,\"query\":{\"query\":\"event.module:nginx AND url.original:*\",\"language\":\"kuery\"},\"version\":true,\"highlight\":{\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fields\":{\"*\":{}},\"pre_tags\":[\"@kibana-highlighted-field@\"],\"require_field_match\":false,\"fragment_size\":2147483647},\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" }, "sort": [ [ diff --git a/dashboards/dashboards/beats/Filebeat-nginx-overview.json b/dashboards/dashboards/beats/Filebeat-nginx-overview.json index 4df0fb11c..73e3fe9c5 100644 --- a/dashboards/dashboards/beats/Filebeat-nginx-overview.json +++ b/dashboards/dashboards/beats/Filebeat-nginx-overview.json @@ -10,16 +10,15 @@ "updated_at": "2022-06-01T19:41:23.453Z", "version": "Wzg0NywxXQ==", "attributes": { - "description": "Overview of logs from Malcolm's own nginx instance", + "description": "nginx logs, including from Malcolm's own nginx instance", "hits": 0, "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"version\":true,\"highlightAll\":true}" + "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"version\":true,\"highlightAll\":false}" }, "optionsJSON": "{\"darkTheme\": false}", "panelsJSON": "[{\"panelIndex\":\"3\",\"panelRefName\":\"panel_0\",\"version\":\"7.3.0\",\"gridData\":{\"x\":36,\"y\":44,\"w\":12,\"h\":12,\"i\":\"3\"},\"embeddableConfig\":{}},{\"panelIndex\":\"4\",\"panelRefName\":\"panel_1\",\"version\":\"7.3.0\",\"gridData\":{\"x\":24,\"y\":44,\"w\":12,\"h\":12,\"i\":\"4\"},\"embeddableConfig\":{\"vis\":{\"legendOpen\":true}}},{\"panelIndex\":\"8\",\"panelRefName\":\"panel_2\",\"version\":\"7.3.0\",\"gridData\":{\"x\":0,\"y\":4,\"w\":48,\"h\":16,\"i\":\"8\"},\"embeddableConfig\":{\"mapZoom\":2,\"mapBounds\":{\"bottom_right\":{\"lat\":-7.362466865535738,\"lon\":245.39062500000003},\"top_left\":{\"lat\":77.07878389624943,\"lon\":-245.74218750000003}},\"mapCenter\":[50.51342652633956,-0.17578125],\"mapCollar\":{\"bottom_right\":{\"lat\":-49.583095,\"lon\":180},\"zoom\":2,\"top_left\":{\"lat\":90,\"lon\":-180}}}},{\"panelIndex\":\"13\",\"panelRefName\":\"panel_3\",\"version\":\"7.3.0\",\"gridData\":{\"x\":0,\"y\":20,\"w\":48,\"h\":12,\"i\":\"13\"},\"embeddableConfig\":{}},{\"panelIndex\":\"14\",\"panelRefName\":\"panel_4\",\"version\":\"7.3.0\",\"gridData\":{\"x\":24,\"y\":32,\"w\":24,\"h\":12,\"i\":\"14\"},\"embeddableConfig\":{}},{\"panelIndex\":\"15\",\"panelRefName\":\"panel_5\",\"version\":\"7.3.0\",\"gridData\":{\"x\":0,\"y\":32,\"w\":24,\"h\":12,\"i\":\"15\"},\"embeddableConfig\":{}},{\"panelIndex\":\"16\",\"panelRefName\":\"panel_6\",\"version\":\"7.3.0\",\"gridData\":{\"x\":0,\"y\":44,\"w\":24,\"h\":12,\"i\":\"16\"},\"embeddableConfig\":{}},{\"panelIndex\":\"17\",\"panelRefName\":\"panel_7\",\"version\":\"7.3.0\",\"gridData\":{\"x\":0,\"y\":0,\"w\":48,\"h\":4,\"i\":\"17\"},\"embeddableConfig\":{}}]", "timeRestore": false, - "title": "Malcolm nginx Overview", - "description": "Overview of Malcolm's own nginx instance", + "title": "nginx Overview", "version": 1 }, "references": [ @@ -290,7 +289,7 @@ "description": "", "hits": 0, "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"filter\":[],\"highlightAll\":true,\"query\":{\"query\":\"event.module:nginx\",\"language\":\"kuery\"},\"version\":true,\"highlight\":{\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fields\":{\"*\":{}},\"pre_tags\":[\"@kibana-highlighted-field@\"],\"require_field_match\":false,\"fragment_size\":2147483647},\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" + "searchSourceJSON": "{\"filter\":[],\"highlightAll\":false,\"query\":{\"query\":\"event.module:nginx\",\"language\":\"kuery\"},\"version\":true,\"highlight\":{\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fields\":{\"*\":{}},\"pre_tags\":[\"@kibana-highlighted-field@\"],\"require_field_match\":false,\"fragment_size\":2147483647},\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" }, "sort": [ [ diff --git a/dashboards/dashboards/beats/Metricbeat-host-overview.json b/dashboards/dashboards/beats/Metricbeat-host-overview.json index fd140606b..b0e401a3f 100644 --- a/dashboards/dashboards/beats/Metricbeat-host-overview.json +++ b/dashboards/dashboards/beats/Metricbeat-host-overview.json @@ -10,15 +10,15 @@ "updated_at": "2022-06-30T17:54:04.824Z", "version": "WzEwMzgsMV0=", "attributes": { - "title": "Malcolm Sensor Resources - Hosts Overview", + "title": "Resources - Hosts Overview", "hits": 0, - "description": "Malcolm sensor hosts' resources overview", + "description": "System resources hosts' level overview, including Malcolm sensors and aggregators", "panelsJSON": "[{\"embeddableConfig\":{},\"gridData\":{\"h\":18,\"i\":\"1\",\"w\":24,\"x\":24,\"y\":57},\"panelIndex\":\"1\",\"version\":\"2.0.0\",\"panelRefName\":\"panel_0\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":18,\"i\":\"3\",\"w\":24,\"x\":0,\"y\":57},\"panelIndex\":\"3\",\"version\":\"2.0.0\",\"panelRefName\":\"panel_1\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":17,\"i\":\"4\",\"w\":24,\"x\":24,\"y\":22},\"panelIndex\":\"4\",\"version\":\"2.0.0\",\"panelRefName\":\"panel_2\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":17,\"i\":\"7\",\"w\":24,\"x\":24,\"y\":5},\"panelIndex\":\"7\",\"version\":\"2.0.0\",\"panelRefName\":\"panel_3\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":18,\"i\":\"8\",\"w\":24,\"x\":24,\"y\":39},\"panelIndex\":\"8\",\"version\":\"2.0.0\",\"panelRefName\":\"panel_4\"},{\"embeddableConfig\":{\"hidePanelTitles\":true},\"gridData\":{\"h\":13,\"i\":\"10\",\"w\":12,\"x\":0,\"y\":5},\"panelIndex\":\"10\",\"version\":\"2.0.0\",\"panelRefName\":\"panel_5\"},{\"embeddableConfig\":{\"hidePanelTitles\":true},\"gridData\":{\"h\":13,\"i\":\"11\",\"w\":12,\"x\":12,\"y\":5},\"panelIndex\":\"11\",\"version\":\"2.0.0\",\"panelRefName\":\"panel_6\"},{\"embeddableConfig\":{\"hidePanelTitles\":true},\"gridData\":{\"h\":13,\"i\":\"12\",\"w\":12,\"x\":0,\"y\":30},\"panelIndex\":\"12\",\"version\":\"2.0.0\",\"panelRefName\":\"panel_7\"},{\"embeddableConfig\":{\"hidePanelTitles\":true},\"gridData\":{\"h\":13,\"i\":\"13\",\"w\":12,\"x\":12,\"y\":30},\"panelIndex\":\"13\",\"version\":\"2.0.0\",\"panelRefName\":\"panel_8\"},{\"embeddableConfig\":{\"hidePanelTitles\":true},\"gridData\":{\"h\":5,\"i\":\"16\",\"w\":24,\"x\":0,\"y\":0},\"panelIndex\":\"16\",\"version\":\"2.0.0\",\"panelRefName\":\"panel_9\"},{\"embeddableConfig\":{\"hidePanelTitles\":true},\"gridData\":{\"h\":12,\"i\":\"21\",\"w\":8,\"x\":0,\"y\":18},\"panelIndex\":\"21\",\"version\":\"2.0.0\",\"panelRefName\":\"panel_10\"},{\"embeddableConfig\":{\"hidePanelTitles\":true},\"gridData\":{\"h\":12,\"i\":\"22\",\"w\":8,\"x\":8,\"y\":18},\"panelIndex\":\"22\",\"version\":\"2.0.0\",\"panelRefName\":\"panel_11\"},{\"embeddableConfig\":{\"hidePanelTitles\":true},\"gridData\":{\"h\":12,\"i\":\"23\",\"w\":8,\"x\":16,\"y\":18},\"panelIndex\":\"23\",\"version\":\"2.0.0\",\"panelRefName\":\"panel_12\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":18,\"i\":\"27\",\"w\":24,\"x\":0,\"y\":75},\"panelIndex\":\"27\",\"version\":\"2.0.0\",\"panelRefName\":\"panel_13\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":18,\"i\":\"28\",\"w\":24,\"x\":24,\"y\":75},\"panelIndex\":\"28\",\"version\":\"2.0.0\",\"panelRefName\":\"panel_14\"},{\"embeddableConfig\":{\"hidePanelTitles\":true},\"gridData\":{\"h\":5,\"i\":\"30\",\"w\":24,\"x\":24,\"y\":0},\"panelIndex\":\"30\",\"version\":\"2.0.0\",\"panelRefName\":\"panel_15\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":14,\"i\":\"b751999d-6577-4995-95e6-b7276ab6388d\",\"w\":24,\"x\":0,\"y\":43},\"panelIndex\":\"b751999d-6577-4995-95e6-b7276ab6388d\",\"version\":\"2.0.0\",\"panelRefName\":\"panel_16\"}]", "optionsJSON": "{\"darkTheme\":false}", "version": 1, "timeRestore": false, "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"query\":{\"language\":\"kuery\",\"query\":\"host.name:\\\"hedgehogvm\\\"\"},\"version\":true,\"highlightAll\":true,\"filter\":[]}" + "searchSourceJSON": "{\"query\":{\"language\":\"kuery\",\"query\":\"host.name:\\\"hedgehogvm\\\"\"},\"version\":true,\"highlightAll\":false,\"filter\":[]}" } }, "references": [ diff --git a/dashboards/dashboards/beats/Metricbeat-system-overview.json b/dashboards/dashboards/beats/Metricbeat-system-overview.json index e8bbf3ba9..eea8dd158 100644 --- a/dashboards/dashboards/beats/Metricbeat-system-overview.json +++ b/dashboards/dashboards/beats/Metricbeat-system-overview.json @@ -10,15 +10,15 @@ "updated_at": "2022-06-30T17:45:03.314Z", "version": "Wzk2NCwxXQ==", "attributes": { - "title": "Malcolm Sensor Resources - System Overview", + "title": "Resources - System Overview", "hits": 0, - "description": "Malcolm sensor system resources overview", + "description": "System resources overview, including Malcolm sensors and aggregators", "panelsJSON": "[{\"embeddableConfig\":{\"hidePanelTitles\":true},\"gridData\":{\"h\":5,\"i\":\"9\",\"w\":8,\"x\":0,\"y\":0},\"panelIndex\":\"9\",\"version\":\"2.0.0\",\"panelRefName\":\"panel_0\"},{\"embeddableConfig\":{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"}}},\"gridData\":{\"h\":8,\"i\":\"11\",\"w\":8,\"x\":0,\"y\":5},\"panelIndex\":\"11\",\"version\":\"2.0.0\",\"panelRefName\":\"panel_1\"},{\"embeddableConfig\":{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"}}},\"gridData\":{\"h\":20,\"i\":\"12\",\"w\":24,\"x\":24,\"y\":13},\"panelIndex\":\"12\",\"version\":\"2.0.0\",\"panelRefName\":\"panel_2\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":20,\"i\":\"13\",\"w\":24,\"x\":0,\"y\":13},\"panelIndex\":\"13\",\"version\":\"2.0.0\",\"panelRefName\":\"panel_3\"},{\"embeddableConfig\":{\"vis\":{\"defaultColors\":{\"0% - 15%\":\"rgb(247,252,245)\",\"15% - 30%\":\"rgb(199,233,192)\",\"30% - 45%\":\"rgb(116,196,118)\",\"45% - 60%\":\"rgb(35,139,69)\"}}},\"gridData\":{\"h\":24,\"i\":\"14\",\"w\":48,\"x\":0,\"y\":33},\"panelIndex\":\"14\",\"version\":\"2.0.0\",\"panelRefName\":\"panel_4\"},{\"embeddableConfig\":{\"hidePanelTitles\":true,\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"}}},\"gridData\":{\"h\":13,\"i\":\"16\",\"w\":8,\"x\":32,\"y\":0},\"panelIndex\":\"16\",\"version\":\"2.0.0\",\"panelRefName\":\"panel_5\"},{\"embeddableConfig\":{\"hidePanelTitles\":true},\"gridData\":{\"h\":13,\"i\":\"17\",\"w\":8,\"x\":40,\"y\":0},\"panelIndex\":\"17\",\"version\":\"2.0.0\",\"panelRefName\":\"panel_6\"},{\"embeddableConfig\":{\"hidePanelTitles\":true},\"gridData\":{\"h\":13,\"i\":\"18\",\"w\":8,\"x\":24,\"y\":0},\"panelIndex\":\"18\",\"version\":\"2.0.0\",\"panelRefName\":\"panel_7\"},{\"embeddableConfig\":{\"hidePanelTitles\":true},\"gridData\":{\"h\":13,\"i\":\"19\",\"w\":8,\"x\":16,\"y\":0},\"panelIndex\":\"19\",\"version\":\"2.0.0\",\"panelRefName\":\"panel_8\"},{\"embeddableConfig\":{\"hidePanelTitles\":true},\"gridData\":{\"h\":13,\"i\":\"20\",\"w\":8,\"x\":8,\"y\":0},\"panelIndex\":\"20\",\"version\":\"2.0.0\",\"panelRefName\":\"panel_9\"}]", "optionsJSON": "{\"darkTheme\":false}", "version": 1, "timeRestore": false, "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"query\":{\"language\":\"kuery\",\"query\":\"\"},\"version\":true,\"highlightAll\":true,\"filter\":[]}" + "searchSourceJSON": "{\"query\":{\"language\":\"kuery\",\"query\":\"\"},\"version\":true,\"highlightAll\":false,\"filter\":[]}" } }, "references": [ diff --git a/dashboards/dashboards/beats/f6600310-9943-11ee-a029-e973f4774355.json b/dashboards/dashboards/beats/f6600310-9943-11ee-a029-e973f4774355.json index f465091b6..0ce72ec85 100644 --- a/dashboards/dashboards/beats/f6600310-9943-11ee-a029-e973f4774355.json +++ b/dashboards/dashboards/beats/f6600310-9943-11ee-a029-e973f4774355.json @@ -10,9 +10,9 @@ "updated_at": "2023-12-14T22:33:38.334Z", "version": "WzkzOCwxXQ==", "attributes": { - "title": "Malcolm Sensor Journald Logs", + "title": "Journald Logs", "hits": 0, - "description": "", + "description": "Messages from Systemd's Journald daemon on Linux environments, including Malcolm sensors and aggregators", "panelsJSON": "[{\"embeddableConfig\":{\"vis\":{\"legendOpen\":true}},\"gridData\":{\"h\":19,\"i\":\"b514b4e8-689b-465e-8335-ca20c20d46fe\",\"w\":14,\"x\":0,\"y\":0},\"panelIndex\":\"b514b4e8-689b-465e-8335-ca20c20d46fe\",\"version\":\"2.11.1\",\"panelRefName\":\"panel_0\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":19,\"i\":\"7a7cfec2-8688-45a7-9790-66b3f0e9fd7e\",\"w\":11,\"x\":14,\"y\":0},\"panelIndex\":\"7a7cfec2-8688-45a7-9790-66b3f0e9fd7e\",\"version\":\"2.11.1\",\"panelRefName\":\"panel_1\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":19,\"i\":\"2143906d-7adb-4de7-8484-2f87c8c98332\",\"w\":23,\"x\":25,\"y\":0},\"panelIndex\":\"2143906d-7adb-4de7-8484-2f87c8c98332\",\"version\":\"2.11.1\",\"panelRefName\":\"panel_2\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":18,\"i\":\"bb4d56fd-b110-4d58-b6aa-e4189bdba918\",\"w\":24,\"x\":0,\"y\":19},\"panelIndex\":\"bb4d56fd-b110-4d58-b6aa-e4189bdba918\",\"version\":\"2.11.1\",\"panelRefName\":\"panel_3\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":18,\"i\":\"4e4780cd-add6-4dbe-95ac-d11afbcd630c\",\"w\":24,\"x\":24,\"y\":19},\"panelIndex\":\"4e4780cd-add6-4dbe-95ac-d11afbcd630c\",\"version\":\"2.11.1\",\"panelRefName\":\"panel_4\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":32,\"i\":\"77bbae8a-66cd-4e30-9b90-c7ab7c271320\",\"w\":48,\"x\":0,\"y\":37},\"panelIndex\":\"77bbae8a-66cd-4e30-9b90-c7ab7c271320\",\"version\":\"2.11.1\",\"panelRefName\":\"panel_5\"}]", "optionsJSON": "{\"hidePanelTitles\":false,\"useMargins\":true}", "version": 1, @@ -66,8 +66,8 @@ "updated_at": "2023-12-14T22:33:38.334Z", "version": "WzkzOSwxXQ==", "attributes": { - "title": "Malcolm Sensor Journald - Logger", - "visState": "{\"title\":\"Malcolm Sensor Journald - Logger\",\"type\":\"pie\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"log.logger\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":10,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"},\"schema\":\"segment\"}],\"params\":{\"type\":\"pie\",\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"bottom\",\"isDonut\":true,\"labels\":{\"show\":true,\"values\":true,\"last_level\":true,\"truncate\":100}}}", + "title": "Journald - Logger", + "visState": "{\"title\":\"Journald - Logger\",\"type\":\"pie\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"log.logger\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":10,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"},\"schema\":\"segment\"}],\"params\":{\"type\":\"pie\",\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"bottom\",\"isDonut\":true,\"labels\":{\"show\":true,\"values\":true,\"last_level\":true,\"truncate\":100}}}", "uiStateJSON": "{\"vis\":{\"legendOpen\":false}}", "description": "", "version": 1, @@ -96,8 +96,8 @@ "updated_at": "2023-12-14T22:50:36.981Z", "version": "Wzk1NSwxXQ==", "attributes": { - "title": "Malcolm Sensor Journald - Process UID", - "visState": "{\"title\":\"Malcolm Sensor Journald - Process UID\",\"type\":\"table\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"miscbeat.systemd.hostname\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"-\",\"customLabel\":\"Systemd Host\"},\"schema\":\"bucket\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"process.user.id\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Process UID\"},\"schema\":\"bucket\"}],\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\"}}", + "title": "Journald - Process UID", + "visState": "{\"title\":\"Journald - Process UID\",\"type\":\"table\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"miscbeat.systemd.hostname\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"-\",\"customLabel\":\"Systemd Host\"},\"schema\":\"bucket\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"process.user.id\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Process UID\"},\"schema\":\"bucket\"}],\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\"}}", "uiStateJSON": "{\"vis\":{\"sortColumn\":{\"colIndex\":1,\"direction\":\"desc\"}}}", "description": "", "version": 1, @@ -126,8 +126,8 @@ "updated_at": "2023-12-14T22:33:38.334Z", "version": "Wzk0MSwxXQ==", "attributes": { - "title": "Malcolm Sensor Journald - Logs by Host Over Time", - "visState": "{\"title\":\"Malcolm Sensor Journald - Logs by Host Over Time\",\"type\":\"histogram\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"params\":{\"field\":\"@timestamp\",\"timeRange\":{\"from\":\"now-15m\",\"to\":\"now\"},\"useNormalizedOpenSearchInterval\":true,\"scaleMetricValues\":false,\"interval\":\"auto\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{}},\"schema\":\"segment\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"miscbeat.systemd.hostname\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":10,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"Unknown\",\"customLabel\":\"Journald Host\"},\"schema\":\"group\"}],\"params\":{\"type\":\"histogram\",\"grid\":{\"categoryLines\":false},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"filter\":true,\"truncate\":100},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"square root\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Count\"}}],\"seriesParams\":[{\"show\":true,\"type\":\"histogram\",\"mode\":\"stacked\",\"data\":{\"label\":\"Count\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"lineWidth\":2,\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"bottom\",\"times\":[],\"addTimeMarker\":true,\"labels\":{\"show\":false},\"thresholdLine\":{\"show\":false,\"value\":10,\"width\":1,\"style\":\"full\",\"color\":\"#E7664C\"}}}", + "title": "Journald - Logs by Host Over Time", + "visState": "{\"title\":\"Journald - Logs by Host Over Time\",\"type\":\"histogram\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"params\":{\"field\":\"@timestamp\",\"timeRange\":{\"from\":\"now-15m\",\"to\":\"now\"},\"useNormalizedOpenSearchInterval\":true,\"scaleMetricValues\":false,\"interval\":\"auto\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{}},\"schema\":\"segment\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"miscbeat.systemd.hostname\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":10,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"Unknown\",\"customLabel\":\"Journald Host\"},\"schema\":\"group\"}],\"params\":{\"type\":\"histogram\",\"grid\":{\"categoryLines\":false},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"filter\":true,\"truncate\":100},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"square root\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Count\"}}],\"seriesParams\":[{\"show\":true,\"type\":\"histogram\",\"mode\":\"stacked\",\"data\":{\"label\":\"Count\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"lineWidth\":2,\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"bottom\",\"times\":[],\"addTimeMarker\":true,\"labels\":{\"show\":false},\"thresholdLine\":{\"show\":false,\"value\":10,\"width\":1,\"style\":\"full\",\"color\":\"#E7664C\"}}}", "uiStateJSON": "{}", "description": "", "version": 1, @@ -156,8 +156,8 @@ "updated_at": "2023-12-14T22:33:38.334Z", "version": "Wzk0MiwxXQ==", "attributes": { - "title": "Malcolm Sensor Journald - Systemd Unit", - "visState": "{\"title\":\"Malcolm Sensor Journald - Systemd Unit\",\"type\":\"table\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"miscbeat.systemd.systemd_unit\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"-\",\"customLabel\":\"Systemd Unit\"},\"schema\":\"bucket\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"miscbeat.systemd.systemd_user_unit\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"-\",\"customLabel\":\"Systemd User Unit\"},\"schema\":\"bucket\"},{\"id\":\"4\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"miscbeat.systemd.user_unit\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"-\",\"customLabel\":\"User Unit\"},\"schema\":\"bucket\"}],\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\"}}", + "title": "Journald - Systemd Unit", + "visState": "{\"title\":\"Journald - Systemd Unit\",\"type\":\"table\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"miscbeat.systemd.systemd_unit\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"-\",\"customLabel\":\"Systemd Unit\"},\"schema\":\"bucket\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"miscbeat.systemd.systemd_user_unit\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"-\",\"customLabel\":\"Systemd User Unit\"},\"schema\":\"bucket\"},{\"id\":\"4\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"miscbeat.systemd.user_unit\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"-\",\"customLabel\":\"User Unit\"},\"schema\":\"bucket\"}],\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\"}}", "uiStateJSON": "{\"vis\":{\"sortColumn\":{\"colIndex\":3,\"direction\":\"desc\"}}}", "description": "", "version": 1, @@ -186,8 +186,8 @@ "updated_at": "2023-12-14T22:33:38.334Z", "version": "Wzk0MywxXQ==", "attributes": { - "title": "Malcolm Sensor Journald - Process Name", - "visState": "{\"title\":\"Malcolm Sensor Journald - Process Name\",\"type\":\"tagcloud\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"process.name\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":50,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"},\"schema\":\"segment\"}],\"params\":{\"scale\":\"square root\",\"orientation\":\"single\",\"minFontSize\":18,\"maxFontSize\":72,\"showLabel\":false}}", + "title": "Journald - Process Name", + "visState": "{\"title\":\"Journald - Process Name\",\"type\":\"tagcloud\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"process.name\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":50,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"},\"schema\":\"segment\"}],\"params\":{\"scale\":\"square root\",\"orientation\":\"single\",\"minFontSize\":18,\"maxFontSize\":72,\"showLabel\":false}}", "uiStateJSON": "{}", "description": "", "version": 1, @@ -216,7 +216,7 @@ "updated_at": "2023-12-14T22:33:38.334Z", "version": "Wzk0NCwxXQ==", "attributes": { - "title": "Malcolm Sensor Journald - Logs", + "title": "Journald - Logs", "description": "", "hits": 0, "columns": [ @@ -233,7 +233,7 @@ "sort": [], "version": 1, "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"query\":{\"query\":\"miscbeat.systemd:*\",\"language\":\"kuery\"},\"highlightAll\":true,\"version\":true,\"aggs\":{\"2\":{\"date_histogram\":{\"field\":\"@timestamp\",\"calendar_interval\":\"1w\",\"time_zone\":\"America/Denver\",\"min_doc_count\":1}}},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" + "searchSourceJSON": "{\"query\":{\"query\":\"miscbeat.systemd:*\",\"language\":\"kuery\"},\"highlightAll\":false,\"version\":true,\"aggs\":{\"2\":{\"date_histogram\":{\"field\":\"@timestamp\",\"calendar_interval\":\"1w\",\"time_zone\":\"America/Denver\",\"min_doc_count\":1}}},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" } }, "references": [ diff --git a/dashboards/opensearch_dashboards.yml b/dashboards/opensearch_dashboards.yml index 67d9f4f65..e8159076b 100644 --- a/dashboards/opensearch_dashboards.yml +++ b/dashboards/opensearch_dashboards.yml @@ -17,21 +17,16 @@ data_source.enabled: false opensearchDashboards.branding: applicationTitle: "Malcolm Dashboards" useExpandedHeader: false - # Yeah, I know about https://opensearch.org/docs/latest/dashboards/branding ... but I can't figure out a way - # to specify the entries in the opensearch_dashboards.yml such that they are valid BOTH from the - # internal opensearch code validating them AND the web browser retrieving them. So we're going scorched earth instead - # by just overwriting the originals in our Dockerconfig. - # - # logo: - # defaultUrl: "http://dashboards:5601/dashboards/ui/assets/Malcolm.svg" - # darkModeUrl: "http://dashboards:5601/dashboards/ui/assets/malcolm_logo.svg" - # mark: - # defaultUrl: "http://dashboards:5601/dashboards/ui/assets/icon.png" - # darkModeUrl: "http://dashboards:5601/dashboards/ui/assets/icon_dark.png" - # loadingLogo: - # defaultUrl: "http://dashboards:5601/dashboards/ui/assets/icon.png" - # darkModeUrl: "http://dashboards:5601/dashboards/ui/assets/icon_dark.png" - # faviconUrl: "http://dashboards:5601/dashboards/ui/assets/favicon.ico" + logo: + defaultUrl: "/assets/img/Malcolm_outline_banner_dark.png" + darkModeUrl: "/assets/img/Malcolm_outline_banner_dark.png" + mark: + defaultUrl: "/assets/img/icon.png" + darkModeUrl: "/assets/img/icon_dark.png" + loadingLogo: + defaultUrl: "/assets/img/icon.png" + darkModeUrl: "/assets/img/icon_dark.png" + faviconUrl: "/assets/img/favicon32.png" map.regionmap: includeOpenSearchMapsService: false diff --git a/dashboards/scripts/index-refresh.py b/dashboards/scripts/index-refresh.py index 4e2ea7458..028b574af 100755 --- a/dashboards/scripts/index-refresh.py +++ b/dashboards/scripts/index-refresh.py @@ -416,7 +416,7 @@ def main(): elif re.search(r'(^zeek\.files\.extracted$)', field['name'], re.IGNORECASE) is not None: # add download for extracted zeek files drilldownInfoParamsUrlTemplateValues = {} - drilldownInfoParamsUrlTemplateValues['url'] = '/dl-extracted-files/{{value}}' + drilldownInfoParamsUrlTemplateValues['url'] = '/extracted-files/{{value}}' drilldownInfoParamsUrlTemplateValues['label'] = 'Download' drilldownInfoParamsUrlTemplates.append(drilldownInfoParamsUrlTemplateValues) diff --git a/dashboards/templates/composable/component/miscbeat.json b/dashboards/templates/composable/component/miscbeat.json index d3072b3c4..0e309d6a5 100644 --- a/dashboards/templates/composable/component/miscbeat.json +++ b/dashboards/templates/composable/component/miscbeat.json @@ -93,6 +93,33 @@ "properties": { } }, + "kmsg": { + "properties": { + "priority": { "type": "integer" }, + "sequence": { "type": "long" }, + "sec": { "type": "long" }, + "usec": { "type": "integer" } + } + }, + "proc": { + "properties": { + "alive": { "type": "keyword" }, + "proc_name": { "type": "keyword" }, + "pid": { "type": "long" }, + "mem.VmPeak": { "type": "long" }, + "mem.VmSize": { "type": "long" }, + "mem.VmLck": { "type": "long" }, + "mem.VmHWM": { "type": "long" }, + "mem.VmRSS": { "type": "long" }, + "mem.VmData": { "type": "long" }, + "mem.VmStk": { "type": "long" }, + "mem.VmExe": { "type": "long" }, + "mem.VmLib": { "type": "long" }, + "mem.VmPTE": { "type": "long" }, + "mem.VmSwap": { "type": "long" }, + "fd": { "type": "integer" } + } + }, "systemd": { "properties": { "audit_loginuid": { "type": "integer" }, diff --git a/dashboards/templates/composable/component/suricata_stats.json b/dashboards/templates/composable/component/suricata_stats.json index 03fb77b2c..3aeb82a91 100644 --- a/dashboards/templates/composable/component/suricata_stats.json +++ b/dashboards/templates/composable/component/suricata_stats.json @@ -169,6 +169,7 @@ "ftp.memuse": { "type": "long" }, "http.memcap": { "type": "long" }, "http.memuse": { "type": "long" }, + "pkts_dropped": { "type": "long" }, "tcp.insert_data_normal_fail": { "type": "long" }, "tcp.insert_data_overlap_fail": { "type": "long" }, "tcp.insert_list_fail": { "type": "long" }, diff --git a/dashboards/templates/composable/component/zeek.json b/dashboards/templates/composable/component/zeek.json index 30f8d0f4e..76b20aafd 100644 --- a/dashboards/templates/composable/component/zeek.json +++ b/dashboards/templates/composable/component/zeek.json @@ -84,10 +84,10 @@ "zeek.files.tx_hosts": { "type": "ip" }, "zeek.ftp.arg": { "type": "keyword" }, "zeek.ftp.command": { "type": "keyword" }, - "zeek.ftp.data_channel_orig_h": { "type": "ip" }, - "zeek.ftp.data_channel_passive": { "type": "keyword" }, - "zeek.ftp.data_channel_resp_h": { "type": "ip" }, - "zeek.ftp.data_channel_resp_p": { "type": "integer" }, + "zeek.ftp.data_channel.orig_h": { "type": "ip" }, + "zeek.ftp.data_channel.passive": { "type": "keyword" }, + "zeek.ftp.data_channel.resp_h": { "type": "ip" }, + "zeek.ftp.data_channel.resp_p": { "type": "integer" }, "zeek.ftp.file_size": { "type": "long" }, "zeek.ftp.mime_type": { "type": "keyword" }, "zeek.ftp.reply_code": { "type": "short" }, @@ -407,6 +407,10 @@ "zeek.smb_files.times_created": { "type": "date" }, "zeek.smb_files.times_modified": { "type": "date" }, "zeek.smb_files.ts": { "type": "date" }, + "zeek.smb_files.orig_h": { "type": "ip" }, + "zeek.smb_files.orig_p": { "type": "integer" }, + "zeek.smb_files.resp_h": { "type": "ip" }, + "zeek.smb_files.resp_p": { "type": "integer" }, "zeek.smb_mapping.native_file_system": { "type": "keyword" }, "zeek.smb_mapping.path": { "type": "keyword", "ignore_above": 1024, "fields": { "text": { "type": "text" } } }, "zeek.smb_mapping.resource_type": { "type": "keyword" }, @@ -453,6 +457,7 @@ "zeek.software.name": { "type": "keyword", "ignore_above": 256, "fields": { "text": { "type": "text" } } }, "zeek.software.software_type": { "type": "keyword" }, "zeek.software.unparsed_version": { "type": "keyword", "ignore_above": 1024 }, + "zeek.software.url": { "type": "keyword", "ignore_above": 16384, "fields": { "text": { "type": "text" } } }, "zeek.software.version_addl": { "type": "keyword", "ignore_above": 1024 }, "zeek.software.version_major": { "type": "integer" }, "zeek.software.version_minor": { "type": "integer" }, @@ -592,6 +597,7 @@ "zeek.uid": { "type": "keyword" }, "zeek.weird.addl": { "type": "keyword", "doc_values": false, "ignore_above": 16384, "fields": { "text": { "type": "text", "norms": false } } }, "zeek.weird.notice": { "type": "keyword" }, + "zeek.weird.source": { "type": "keyword" }, "zeek.wireguard.established": { "type": "keyword" }, "zeek.wireguard.initiations": { "type": "integer" }, "zeek.wireguard.receiver_index": { "type": "keyword" }, diff --git a/dashboards/templates/composable/component/zeek_diagnostic.json b/dashboards/templates/composable/component/zeek_diagnostic.json index eaa65e284..f4b8f0420 100644 --- a/dashboards/templates/composable/component/zeek_diagnostic.json +++ b/dashboards/templates/composable/component/zeek_diagnostic.json @@ -26,11 +26,9 @@ "packet_filter.success": { "type": "keyword" }, "packet_filter.failure_reason": { "type": "keyword", "ignore_above": 1024, "fields": { "text": { "type": "text" } } }, "print.vals": { "type": "keyword", "ignore_above": 1024, "fields": { "text": { "type": "text" } } }, - "reporter.node": { "type": "keyword" }, - "reporter.filter": { "type": "keyword", "ignore_above": 1024, "fields": { "text": { "type": "text" } } }, - "reporter.init": { "type": "keyword" }, - "reporter.success": { "type": "keyword" }, - "reporter.failure_reason": { "type": "keyword", "ignore_above": 1024, "fields": { "text": { "type": "text" } } }, + "reporter.level": { "type": "keyword" }, + "reporter.msg": { "type": "keyword", "ignore_above": 1024, "fields": { "text": { "type": "text" } } }, + "reporter.location": { "type": "keyword" }, "stats.peer": { "type": "keyword" }, "stats.mem": { "type": "long" }, "stats.pkts_proc": { "type": "long" }, diff --git a/dashboards/templates/composable/component/zeek_ot.json b/dashboards/templates/composable/component/zeek_ot.json index 2ed2174ef..fb09b02e0 100644 --- a/dashboards/templates/composable/component/zeek_ot.json +++ b/dashboards/templates/composable/component/zeek_ot.json @@ -691,6 +691,41 @@ "zeek.profinet_dce_rpc.packet_type": { "type": "keyword" }, "zeek.profinet_dce_rpc.server_boot_time": { "type": "integer" }, "zeek.profinet_dce_rpc.version": { "type": "integer" }, + "zeek.profinet_io_cm.rpc_version": { "type": "long" }, + "zeek.profinet_io_cm.packet_type": { "type": "keyword" }, + "zeek.profinet_io_cm.reserved_for_impl_1": { "type": "keyword" }, + "zeek.profinet_io_cm.last_fragment": { "type": "keyword" }, + "zeek.profinet_io_cm.fragment": { "type": "keyword" }, + "zeek.profinet_io_cm.no_fragment_requested": { "type": "keyword" }, + "zeek.profinet_io_cm.maybe": { "type": "keyword" }, + "zeek.profinet_io_cm.idempotent": { "type": "keyword" }, + "zeek.profinet_io_cm.broadcast": { "type": "keyword" }, + "zeek.profinet_io_cm.reserved_for_impl_2": { "type": "keyword" }, + "zeek.profinet_io_cm.cancel_was_pending_at_call_end": { "type": "keyword" }, + "zeek.profinet_io_cm.integer_encoding": { "type": "keyword" }, + "zeek.profinet_io_cm.character_encoding": { "type": "keyword" }, + "zeek.profinet_io_cm.floating_point_encoding": { "type": "keyword" }, + "zeek.profinet_io_cm.serial_high": { "type": "long" }, + "zeek.profinet_io_cm.object_uuid": { "type": "keyword" }, + "zeek.profinet_io_cm.interface_uuid": { "type": "keyword" }, + "zeek.profinet_io_cm.activity_uuid": { "type": "keyword" }, + "zeek.profinet_io_cm.server_boot_time": { "type": "long" }, + "zeek.profinet_io_cm.uuid_version": { "type": "keyword" }, + "zeek.profinet_io_cm.sequence_num": { "type": "long" }, + "zeek.profinet_io_cm.operation": { "type": "keyword" }, + "zeek.profinet_io_cm.interface_hint": { "type": "long" }, + "zeek.profinet_io_cm.activity_hint": { "type": "long" }, + "zeek.profinet_io_cm.len_of_body": { "type": "long" }, + "zeek.profinet_io_cm.fragment_num": { "type": "long" }, + "zeek.profinet_io_cm.auth_protocol": { "type": "long" }, + "zeek.profinet_io_cm.serial_low": { "type": "long" }, + "zeek.profinet_io_cm.vers_fack": { "type": "keyword" }, + "zeek.profinet_io_cm.window_size": { "type": "long" }, + "zeek.profinet_io_cm.max_tsdu": { "type": "long" }, + "zeek.profinet_io_cm.max_frag_size": { "type": "long" }, + "zeek.profinet_io_cm.serial_number": { "type": "long" }, + "zeek.profinet_io_cm.sel_ack_len": { "type": "long" }, + "zeek.profinet_io_cm.sel_ack": { "type": "keyword" }, "zeek.s7comm.error_class": { "type": "keyword" }, "zeek.s7comm.error_code": { "type": "keyword" }, "zeek.s7comm.function_code": { "type": "keyword" }, diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index 0639b407a..f2fb41ff7 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -15,7 +15,7 @@ services: build: context: . dockerfile: Dockerfiles/opensearch.Dockerfile - image: ghcr.io/idaholab/malcolm/opensearch:24.02.0 + image: ghcr.io/idaholab/malcolm/opensearch:24.03.0 # Technically the "hedgehog" profile doesn't have OpenSearch, but in that case # OPENSEARCH_PRIMARY will be set to remote, which means the container will # start but not actually run OpenSearch. It's included in both profiles to @@ -60,7 +60,7 @@ services: build: context: . dockerfile: Dockerfiles/dashboards-helper.Dockerfile - image: ghcr.io/idaholab/malcolm/dashboards-helper:24.02.0 + image: ghcr.io/idaholab/malcolm/dashboards-helper:24.03.0 profiles: ["malcolm"] logging: *default-logging restart: "no" @@ -93,7 +93,7 @@ services: build: context: . dockerfile: Dockerfiles/dashboards.Dockerfile - image: ghcr.io/idaholab/malcolm/dashboards:24.02.0 + image: ghcr.io/idaholab/malcolm/dashboards:24.03.0 profiles: ["malcolm"] logging: *default-logging restart: "no" @@ -124,7 +124,7 @@ services: build: context: . dockerfile: Dockerfiles/logstash.Dockerfile - image: ghcr.io/idaholab/malcolm/logstash-oss:24.02.0 + image: ghcr.io/idaholab/malcolm/logstash-oss:24.03.0 profiles: ["malcolm"] logging: *default-logging restart: "no" @@ -178,7 +178,7 @@ services: build: context: . dockerfile: Dockerfiles/filebeat.Dockerfile - image: ghcr.io/idaholab/malcolm/filebeat-oss:24.02.0 + image: ghcr.io/idaholab/malcolm/filebeat-oss:24.03.0 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -216,7 +216,7 @@ services: build: context: . dockerfile: Dockerfiles/arkime.Dockerfile - image: ghcr.io/idaholab/malcolm/arkime:24.02.0 + image: ghcr.io/idaholab/malcolm/arkime:24.03.0 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -255,7 +255,7 @@ services: build: context: . dockerfile: Dockerfiles/arkime.Dockerfile - image: ghcr.io/idaholab/malcolm/arkime:24.02.0 + image: ghcr.io/idaholab/malcolm/arkime:24.03.0 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -294,7 +294,7 @@ services: build: context: . dockerfile: Dockerfiles/zeek.Dockerfile - image: ghcr.io/idaholab/malcolm/zeek:24.02.0 + image: ghcr.io/idaholab/malcolm/zeek:24.03.0 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -330,7 +330,7 @@ services: build: context: . dockerfile: Dockerfiles/zeek.Dockerfile - image: ghcr.io/idaholab/malcolm/zeek:24.02.0 + image: ghcr.io/idaholab/malcolm/zeek:24.03.0 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -362,7 +362,7 @@ services: build: context: . dockerfile: Dockerfiles/suricata.Dockerfile - image: ghcr.io/idaholab/malcolm/suricata:24.02.0 + image: ghcr.io/idaholab/malcolm/suricata:24.03.0 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -393,7 +393,7 @@ services: build: context: . dockerfile: Dockerfiles/suricata.Dockerfile - image: ghcr.io/idaholab/malcolm/suricata:24.02.0 + image: ghcr.io/idaholab/malcolm/suricata:24.03.0 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -426,7 +426,7 @@ services: build: context: . dockerfile: Dockerfiles/file-monitor.Dockerfile - image: ghcr.io/idaholab/malcolm/file-monitor:24.02.0 + image: ghcr.io/idaholab/malcolm/file-monitor:24.03.0 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -458,7 +458,7 @@ services: build: context: . dockerfile: Dockerfiles/pcap-capture.Dockerfile - image: ghcr.io/idaholab/malcolm/pcap-capture:24.02.0 + image: ghcr.io/idaholab/malcolm/pcap-capture:24.03.0 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -490,7 +490,7 @@ services: build: context: . dockerfile: Dockerfiles/pcap-monitor.Dockerfile - image: ghcr.io/idaholab/malcolm/pcap-monitor:24.02.0 + image: ghcr.io/idaholab/malcolm/pcap-monitor:24.03.0 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -521,7 +521,7 @@ services: build: context: . dockerfile: Dockerfiles/file-upload.Dockerfile - image: ghcr.io/idaholab/malcolm/file-upload:24.02.0 + image: ghcr.io/idaholab/malcolm/file-upload:24.03.0 profiles: ["malcolm"] logging: *default-logging restart: "no" @@ -548,7 +548,7 @@ services: retries: 3 start_period: 60s htadmin: - image: ghcr.io/idaholab/malcolm/htadmin:24.02.0 + image: ghcr.io/idaholab/malcolm/htadmin:24.03.0 profiles: ["malcolm"] logging: *default-logging build: @@ -578,7 +578,7 @@ services: retries: 3 start_period: 60s freq: - image: ghcr.io/idaholab/malcolm/freq:24.02.0 + image: ghcr.io/idaholab/malcolm/freq:24.03.0 profiles: ["malcolm"] logging: *default-logging build: @@ -605,7 +605,7 @@ services: retries: 3 start_period: 60s netbox: - image: ghcr.io/idaholab/malcolm/netbox:24.02.0 + image: ghcr.io/idaholab/malcolm/netbox:24.03.0 profiles: ["malcolm"] logging: *default-logging build: @@ -642,7 +642,7 @@ services: retries: 3 start_period: 120s netbox-postgres: - image: ghcr.io/idaholab/malcolm/postgresql:24.02.0 + image: ghcr.io/idaholab/malcolm/postgresql:24.03.0 profiles: ["malcolm"] logging: *default-logging build: @@ -671,7 +671,7 @@ services: retries: 3 start_period: 45s netbox-redis: - image: ghcr.io/idaholab/malcolm/redis:24.02.0 + image: ghcr.io/idaholab/malcolm/redis:24.03.0 profiles: ["malcolm"] logging: *default-logging build: @@ -704,7 +704,7 @@ services: retries: 3 start_period: 45s netbox-redis-cache: - image: ghcr.io/idaholab/malcolm/redis:24.02.0 + image: ghcr.io/idaholab/malcolm/redis:24.03.0 profiles: ["malcolm"] logging: *default-logging build: @@ -736,7 +736,7 @@ services: retries: 3 start_period: 45s api: - image: ghcr.io/idaholab/malcolm/api:24.02.0 + image: ghcr.io/idaholab/malcolm/api:24.03.0 profiles: ["malcolm"] logging: *default-logging build: @@ -769,7 +769,7 @@ services: build: context: . dockerfile: Dockerfiles/nginx.Dockerfile - image: ghcr.io/idaholab/malcolm/nginx-proxy:24.02.0 + image: ghcr.io/idaholab/malcolm/nginx-proxy:24.03.0 profiles: ["malcolm"] logging: *default-logging restart: "no" diff --git a/docker-compose.yml b/docker-compose.yml index ae9444ecc..51077bea9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -12,7 +12,7 @@ x-logging: services: opensearch: - image: ghcr.io/idaholab/malcolm/opensearch:24.02.0 + image: ghcr.io/idaholab/malcolm/opensearch:24.03.0 # Technically the "hedgehog" profile doesn't have OpenSearch, but in that case # OPENSEARCH_PRIMARY will be set to remote, which means the container will # start but not actually run OpenSearch. It's included in both profiles to @@ -54,7 +54,7 @@ services: retries: 3 start_period: 180s dashboards-helper: - image: ghcr.io/idaholab/malcolm/dashboards-helper:24.02.0 + image: ghcr.io/idaholab/malcolm/dashboards-helper:24.03.0 profiles: ["malcolm"] logging: *default-logging restart: "no" @@ -84,7 +84,7 @@ services: retries: 3 start_period: 30s dashboards: - image: ghcr.io/idaholab/malcolm/dashboards:24.02.0 + image: ghcr.io/idaholab/malcolm/dashboards:24.03.0 profiles: ["malcolm"] logging: *default-logging restart: "no" @@ -112,7 +112,7 @@ services: retries: 3 start_period: 210s logstash: - image: ghcr.io/idaholab/malcolm/logstash-oss:24.02.0 + image: ghcr.io/idaholab/malcolm/logstash-oss:24.03.0 profiles: ["malcolm"] logging: *default-logging restart: "no" @@ -159,7 +159,7 @@ services: retries: 3 start_period: 600s filebeat: - image: ghcr.io/idaholab/malcolm/filebeat-oss:24.02.0 + image: ghcr.io/idaholab/malcolm/filebeat-oss:24.03.0 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -194,7 +194,7 @@ services: retries: 3 start_period: 60s arkime: - image: ghcr.io/idaholab/malcolm/arkime:24.02.0 + image: ghcr.io/idaholab/malcolm/arkime:24.03.0 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -228,7 +228,7 @@ services: retries: 3 start_period: 210s arkime-live: - image: ghcr.io/idaholab/malcolm/arkime:24.02.0 + image: ghcr.io/idaholab/malcolm/arkime:24.03.0 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -261,7 +261,7 @@ services: - ./arkime/rules:/opt/arkime/rules:ro - ./pcap:/data/pcap zeek: - image: ghcr.io/idaholab/malcolm/zeek:24.02.0 + image: ghcr.io/idaholab/malcolm/zeek:24.03.0 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -293,7 +293,7 @@ services: retries: 3 start_period: 60s zeek-live: - image: ghcr.io/idaholab/malcolm/zeek:24.02.0 + image: ghcr.io/idaholab/malcolm/zeek:24.03.0 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -321,7 +321,7 @@ services: - ./zeek/intel:/opt/zeek/share/zeek/site/intel - ./zeek/custom:/opt/zeek/share/zeek/site/custom:ro suricata: - image: ghcr.io/idaholab/malcolm/suricata:24.02.0 + image: ghcr.io/idaholab/malcolm/suricata:24.03.0 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -349,7 +349,7 @@ services: retries: 3 start_period: 120s suricata-live: - image: ghcr.io/idaholab/malcolm/suricata:24.02.0 + image: ghcr.io/idaholab/malcolm/suricata:24.03.0 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -379,7 +379,7 @@ services: - ./suricata/rules:/opt/suricata/rules:ro - ./suricata/include-configs:/opt/suricata/include-configs:ro file-monitor: - image: ghcr.io/idaholab/malcolm/file-monitor:24.02.0 + image: ghcr.io/idaholab/malcolm/file-monitor:24.03.0 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -408,7 +408,7 @@ services: retries: 3 start_period: 60s pcap-capture: - image: ghcr.io/idaholab/malcolm/pcap-capture:24.02.0 + image: ghcr.io/idaholab/malcolm/pcap-capture:24.03.0 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -437,7 +437,7 @@ services: - ./nginx/ca-trust:/var/local/ca-trust:ro - ./pcap/upload:/pcap pcap-monitor: - image: ghcr.io/idaholab/malcolm/pcap-monitor:24.02.0 + image: ghcr.io/idaholab/malcolm/pcap-monitor:24.03.0 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -465,7 +465,7 @@ services: retries: 3 start_period: 90s upload: - image: ghcr.io/idaholab/malcolm/file-upload:24.02.0 + image: ghcr.io/idaholab/malcolm/file-upload:24.03.0 profiles: ["malcolm"] logging: *default-logging restart: "no" @@ -492,7 +492,7 @@ services: retries: 3 start_period: 60s htadmin: - image: ghcr.io/idaholab/malcolm/htadmin:24.02.0 + image: ghcr.io/idaholab/malcolm/htadmin:24.03.0 profiles: ["malcolm"] logging: *default-logging restart: "no" @@ -519,7 +519,7 @@ services: retries: 3 start_period: 60s freq: - image: ghcr.io/idaholab/malcolm/freq:24.02.0 + image: ghcr.io/idaholab/malcolm/freq:24.03.0 profiles: ["malcolm"] logging: *default-logging restart: "no" @@ -543,7 +543,7 @@ services: retries: 3 start_period: 60s netbox: - image: ghcr.io/idaholab/malcolm/netbox:24.02.0 + image: ghcr.io/idaholab/malcolm/netbox:24.03.0 profiles: ["malcolm"] logging: *default-logging restart: "no" @@ -577,7 +577,7 @@ services: retries: 3 start_period: 120s netbox-postgres: - image: ghcr.io/idaholab/malcolm/postgresql:24.02.0 + image: ghcr.io/idaholab/malcolm/postgresql:24.03.0 profiles: ["malcolm"] logging: *default-logging restart: "no" @@ -603,7 +603,7 @@ services: retries: 3 start_period: 45s netbox-redis: - image: ghcr.io/idaholab/malcolm/redis:24.02.0 + image: ghcr.io/idaholab/malcolm/redis:24.03.0 profiles: ["malcolm"] logging: *default-logging restart: "no" @@ -633,7 +633,7 @@ services: retries: 3 start_period: 45s netbox-redis-cache: - image: ghcr.io/idaholab/malcolm/redis:24.02.0 + image: ghcr.io/idaholab/malcolm/redis:24.03.0 profiles: ["malcolm"] logging: *default-logging restart: "no" @@ -662,7 +662,7 @@ services: retries: 3 start_period: 45s api: - image: ghcr.io/idaholab/malcolm/api:24.02.0 + image: ghcr.io/idaholab/malcolm/api:24.03.0 profiles: ["malcolm"] logging: *default-logging command: gunicorn --bind 0:5000 manage:app @@ -689,7 +689,7 @@ services: retries: 3 start_period: 60s nginx-proxy: - image: ghcr.io/idaholab/malcolm/nginx-proxy:24.02.0 + image: ghcr.io/idaholab/malcolm/nginx-proxy:24.03.0 profiles: ["malcolm"] logging: *default-logging restart: "no" diff --git a/docs/api-version.md b/docs/api-version.md index 76364f95c..7e4c08db5 100644 --- a/docs/api-version.md +++ b/docs/api-version.md @@ -47,6 +47,6 @@ Returns version information about Malcolm and version/[health](https://opensearc } }, "sha": "77574975", - "version": "24.02.0" + "version": "24.03.0" } ``` diff --git a/docs/contributing-logstash.md b/docs/contributing-logstash.md index b37d5650d..c9cfa9d5f 100644 --- a/docs/contributing-logstash.md +++ b/docs/contributing-logstash.md @@ -38,8 +38,8 @@ The following modifications must be made in order for Malcolm to parse new Zeek * Follow patterns for existing log files as an example * For common Zeek fields such as the `id` four-tuple, timestamp, etc., use the same convention used by existing Zeek logs in that file (e.g., `ts`, `uid`, `orig_h`, `orig_p`, `resp_h`, `resp_p`) * Take care, especially when copy-pasting filter code, the Zeek delimiter isn't modified from a tab character to a space character (see "*zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP*" warnings in that file) -1. If necessary, perform log normalization in [`logstash/pipelines/zeek/12_zeek_normalize.conf`]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/logstash/pipelines/zeek/12_zeek_normalize.conf) for values such as action (`event.action`), result (`event.result`), application protocol version (`network.protocol_version`), etc. -1. If necessary, define conversions for floating point or integer values in [`logstash/pipelines/zeek/11_zeek_parse.conf`]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/logstash/pipelines/zeek/14_zeek_convert.conf) +1. If necessary, perform log normalization in [`logstash/pipelines/zeek/13_zeek_normalize.conf`]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/logstash/pipelines/zeek/13_zeek_normalize.conf) for values such as action (`event.action`), result (`event.result`), application protocol version (`network.protocol_version`), etc. +1. If necessary, define conversions for floating point or integer values in [`logstash/pipelines/zeek/14_zeek_convert.conf`]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/logstash/pipelines/zeek/14_zeek_convert.conf) 1. Identify the new fields and add them as described in [Adding new log fields](contributing-new-log-fields.md#NewFields) The script [`scripts/zeek_script_to_malcolm_boilerplate.py`]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/scripts/zeek_script_to_malcolm_boilerplate.py) may help by autogenerating these filters. diff --git a/docs/contributing-pcap.md b/docs/contributing-pcap.md index 167235639..adac63fb4 100644 --- a/docs/contributing-pcap.md +++ b/docs/contributing-pcap.md @@ -1,6 +1,6 @@ # PCAP processors -When a PCAP is uploaded (either through Malcolm's [upload web interface](upload.md#Upload) or just copied manually into the `./pcap/upload` directory), the `pcap-monitor` container has a script that picks up those PCAP files and publishes to a [ZeroMQ](https://zeromq.org/) topic that can be subscribed to by any other process that wants to analyze that PCAP. In Malcolm (at the time of the [v24.02.0 release]({{ site.github.repository_url }}/releases/tag/v24.02.0)), there are three such ZeroMQ topics: the `zeek`, `suricata` and `arkime` containers. These actually share the [same script]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/shared/bin/pcap_processor.py) to run the PCAP through Zeek, Suricata, and Arkime, respectively. For an example to follow, the `zeek` container is the less complicated of the two. To integrate a new PCAP processing tool into Malcolm (named `cooltool` for this example) the process would entail: +When a PCAP is uploaded (either through Malcolm's [upload web interface](upload.md#Upload) or just copied manually into the `./pcap/upload` directory), the `pcap-monitor` container has a script that picks up those PCAP files and publishes to a [ZeroMQ](https://zeromq.org/) topic that can be subscribed to by any other process that wants to analyze that PCAP. In Malcolm (at the time of the [v24.03.0 release]({{ site.github.repository_url }}/releases/tag/v24.03.0)), there are three such ZeroMQ topics: the `zeek`, `suricata` and `arkime` containers. These actually share the [same script]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/shared/bin/pcap_processor.py) to run the PCAP through Zeek, Suricata, and Arkime, respectively. For an example to follow, the `zeek` container is the less complicated of the two. To integrate a new PCAP processing tool into Malcolm (named `cooltool` for this example) the process would entail: 1. Define the service as instructed in the [Adding a new service](contributing-new-image.md#NewImage) section * Note how the existing `zeek` and `arkime` services use [bind mounts](contributing-local-modifications.md#Bind) to access the local `./pcap` directory diff --git a/docs/development.md b/docs/development.md index b59bc3b49..c8efe2d94 100644 --- a/docs/development.md +++ b/docs/development.md @@ -16,6 +16,8 @@ Checking out the [Malcolm source code]({{ site.github.repository_url }}/tree/{{ * `file-monitor` - code and configuration for the `file-monitor` container that can scan files extracted by Zeek * `file-upload` - code and configuration for the `upload` container that serves a web browser-based upload form for uploading PCAP files and Zeek logs, and serves an SFTP share as an alternate upload * `freq-server` - code and configuration for the `freq` container used for calculating entropy of strings +* `hedgehog-iso` - code and configuration for building a [Hedgehog Linux](live-analysis.md#Hedgehog) ISO +* `hedgehog-raspi` - code and configuration for building a [Hedgehog Linux](live-analysis.md#Hedgehog) [Raspberry Pi image](hedgehog-raspi-build.md#HedgehogRaspiBuild) * `htadmin` - configuration for the `htadmin` user account management container * `logstash` - code and configuration for the `logstash` container that parses Zeek logs and forwards them to the `opensearch` container * `malcolm-iso` - code and configuration for building an [installer ISO](malcolm-iso.md#ISO) for a minimal Debian-based Linux installation for running Malcolm @@ -27,7 +29,6 @@ Checking out the [Malcolm source code]({{ site.github.repository_url }}/tree/{{ * `pcap-capture` - code and configuration for the `pcap-capture` container that can capture network traffic * `pcap-monitor` - code and configuration for the `pcap-monitor` container that watches for new or uploaded PCAP files and notifies the other services to process them * `scripts` - control scripts for starting, stopping, restarting, etc., Malcolm -* `sensor-iso` - code and configuration for building a [Hedgehog Linux](live-analysis.md#Hedgehog) ISO * `shared` - miscellaneous code used by various Malcolm components * `suricata` - code and configuration for the `suricata` container that handles PCAP processing using Suricata * `suricata-logs` - an initially empty directory for Suricata logs to be uploaded, processed, and stored diff --git a/docs/download.md b/docs/download.md index 083f84936..0a0a01a42 100644 --- a/docs/download.md +++ b/docs/download.md @@ -16,7 +16,7 @@ While official downloads of the Malcolm installer ISO are not provided, an **uno | ISO | SHA256 | |---|---| -| [malcolm-24.02.0.iso](/iso/malcolm-24.02.0.iso) (5.1GiB) | [`xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx`](/iso/malcolm-24.02.0.iso.sha256.txt) | +| [malcolm-24.03.0.iso](/iso/malcolm-24.03.0.iso) (5.1GiB) | [`f2e15167a3bed28cc6ec5af727d2eecdc5e7b58ef5325f4e1db84b8d8a2da2b5`](/iso/malcolm-24.03.0.iso.sha256.txt) | ## Hedgehog Linux @@ -26,7 +26,7 @@ While official downloads of the Malcolm installer ISO are not provided, an **uno | ISO | SHA256 | |---|---| -| [hedgehog-24.02.0.iso](/iso/hedgehog-24.02.0.iso) (2.5GiB) | [`xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx`](/iso/hedgehog-24.02.0.iso.sha256.txt) | +| [hedgehog-24.03.0.iso](/iso/hedgehog-24.03.0.iso) (2.5GiB) | [`d849ab533ea1f3c37c87f6b4064ddff31e1ac40a060d36f120b9fefe229577ae`](/iso/hedgehog-24.03.0.iso.sha256.txt) | ### Raspberry Pi 4 Image @@ -34,7 +34,7 @@ While official downloads of the Malcolm installer ISO are not provided, an **uno | Image | SHA256 | |---|---| -| [hedgehog-24.02.0_raspi_4.img.xz](/iso/hedgehog-24.02.0_raspi_4.img.xz) (1.4GiB) | [`xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx`](/iso/hedgehog-24.02.0_raspi_4.img.xz.sha256.txt) | +| [hedgehog-24.03.0_raspi_4.img.xz](/iso/hedgehog-24.03.0_raspi_4.img.xz) (1.4GiB) | [`xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx`](/iso/hedgehog-24.03.0_raspi_4.img.xz.sha256.txt) | ## Warning diff --git a/docs/hedgehog-iso-build.md b/docs/hedgehog-iso-build.md index 1c4e7faa0..4de1fe858 100644 --- a/docs/hedgehog-iso-build.md +++ b/docs/hedgehog-iso-build.md @@ -18,7 +18,7 @@ The build should work with either the [VirtualBox](https://www.virtualbox.org/) To perform a clean build the Hedgehog Linux installer ISO, navigate to your local [Malcolm]({{ site.github.repository_url }}/) working copy and run: ``` -$ ./sensor-iso/build_via_vagrant.sh -f +$ ./hedgehog-iso/build_via_vagrant.sh -f … Starting build machine... Bringing machine 'default' up with 'virtualbox' provider... @@ -29,8 +29,8 @@ Building the ISO may take 90 minutes or more depending on your system. As the bu ``` … -Finished, created "/sensor-build/hedgehog-24.02.0.iso" +Finished, created "/sensor-build/hedgehog-24.03.0.iso" … ``` -Alternately, if you have forked Malcolm on GitHub, [workflow files]({{ site.github.repository_url }}/tree/{{ site.github.build_revision }}/.github/workflows/) are provided which contain instructions for GitHub to build the docker images and Hedgehog and [Malcolm]({{ site.github.repository_url }}) installer ISOs, specifically [`sensor-iso-build-docker-wrap-push-ghcr.yml`]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/.github/workflows/sensor-iso-build-docker-wrap-push-ghcr.yml) for the Hedgehog ISO. The resulting ISO file is wrapped in a Docker image that provides an HTTP server from which the ISO may be downloaded. \ No newline at end of file +Alternately, if you have forked Malcolm on GitHub, [workflow files]({{ site.github.repository_url }}/tree/{{ site.github.build_revision }}/.github/workflows/) are provided which contain instructions for GitHub to build the docker images and Hedgehog and [Malcolm]({{ site.github.repository_url }}) installer ISOs, specifically [`hedgehog-iso-build-docker-wrap-push-ghcr.yml`]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/.github/workflows/hedgehog-iso-build-docker-wrap-push-ghcr.yml) for the Hedgehog ISO. The resulting ISO file is wrapped in a Docker image that provides an HTTP server from which the ISO may be downloaded. \ No newline at end of file diff --git a/docs/hedgehog-raspi-build.md b/docs/hedgehog-raspi-build.md index 82e4dc188..6d1f0cf8d 100644 --- a/docs/hedgehog-raspi-build.md +++ b/docs/hedgehog-raspi-build.md @@ -1,6 +1,6 @@ # Appendix B - Generating a Raspberry Pi Image -Hedgehog Linux can [also be built]({{ site.github.repository_url }}/tree/{{ site.github.build_revision }}/sensor-raspi) for the Raspberry Pi platform, although this capability is still considered experimental. +Hedgehog Linux can [also be built]({{ site.github.repository_url }}/tree/{{ site.github.build_revision }}/hedgehog-raspi) for the Raspberry Pi platform, although this capability is still considered experimental. * [Building the Image](#HedgehogRaspiBuild) * [Writing the Image to Flash Media](#HedgehogRaspiBurn) @@ -27,7 +27,7 @@ The build should work with either the [VirtualBox](https://www.virtualbox.org/) To perform a clean build the Hedgehog Linux Raspberry Pi image, navigate to your local [Malcolm]({{ site.github.repository_url }}/) working copy and run: ``` -$ ./sensor-raspi/build_via_vagrant.sh -f -z +$ ./hedgehog-raspi/build_via_vagrant.sh -f -z … Starting build machine... Bringing machine 'vagrant-hedgehog-raspi-build' up with 'virtualbox' provider... diff --git a/docs/hedgehog-upgrade.md b/docs/hedgehog-upgrade.md index 37d8a1a53..3b3cec849 100644 --- a/docs/hedgehog-upgrade.md +++ b/docs/hedgehog-upgrade.md @@ -47,9 +47,9 @@ deb https://XXXXXX:443/debian buster-backports main contrib non-free 5. Update underlying system packages with `apt-get` - `apt-get update && apt-get dist-upgrade` -6. If there were [new system deb packages added]({{ site.github.repository_url }}/tree/{{ site.github.build_revision }}/sensor-iso/config/package-lists) to this release of Hedgehog Linux (you might have to [manually compare]({{ site.github.repository_url }}/commits/main/sensor-iso/config/package-lists) on GitHub), install them. If you're not sure, of course, you could just install everything, like this (although you may have to tweak some version numbers or something if the base distribution of your Hedgehog branch is different than `main`; in this example I'm not jumping between Debian releases, just upgrading within a release): +6. If there were [new system deb packages added]({{ site.github.repository_url }}/tree/{{ site.github.build_revision }}/hedgehog-iso/config/package-lists) to this release of Hedgehog Linux (you might have to [manually compare]({{ site.github.repository_url }}/commits/main/hedgehog-iso/config/package-lists) on GitHub), install them. If you're not sure, of course, you could just install everything, like this (although you may have to tweak some version numbers or something if the base distribution of your Hedgehog branch is different than `main`; in this example I'm not jumping between Debian releases, just upgrading within a release): ``` -$ for LIST in apps desktopmanager net system; do curl -L -J -O {{ site.github.repository_url }}/main/sensor-iso/config/package-lists/$LIST.list.chroot; done +$ for LIST in apps desktopmanager net system; do curl -L -J -O {{ site.github.repository_url }}/main/hedgehog-iso/config/package-lists/$LIST.list.chroot; done ... $ apt-get install $(cat *.list.chroot) ``` @@ -59,7 +59,7 @@ $ apt-get install $(cat *.list.chroot) * `python3 -m pip list --outdated --format=freeze | grep -v '^\-e' | cut -d = -f 1 | xargs -r -n1 python3 -m pip install -U` - if this fails for some reason, you may need to reinstall pip first with `python3 -m pip install --force -U pip` - some *very* old builds of Hedgehog Linux had separate Python 3.5 and 3.7 installations: in this case, you'd need to do this for both `python3 -m pip` and `python3.7 -m pip` (or whatever `python3.x` you have) - * If there were [new python packages](https://raw.githubusercontent.com/{{ site.github.repository_nwo }}/master/sensor-iso/config/hooks/normal/0169-pip-installs.hook.chroot) added to this release of Hedgehog Linux (you might have to [manually compare]({{ site.github.repository_url }}/blame/main/sensor-iso/config/hooks/normal/0169-pip-installs.hook.chroot) on GitHub), install them. If you are using a PyPI mirror, replace `XXXXXX` here with your mirror's IP. The `colorama` package is used here as an example, your package list might vary. + * If there were [new python packages](https://raw.githubusercontent.com/{{ site.github.repository_nwo }}/master/hedgehog-iso/config/hooks/normal/0169-pip-installs.hook.chroot) added to this release of Hedgehog Linux (you might have to [manually compare]({{ site.github.repository_url }}/blame/main/hedgehog-iso/config/hooks/normal/0169-pip-installs.hook.chroot) on GitHub), install them. If you are using a PyPI mirror, replace `XXXXXX` here with your mirror's IP. The `colorama` package is used here as an example, your package list might vary. - `python3 -m pip install --no-compile --no-cache-dir --force-reinstall --upgrade --index-url=https://XXXXXX:443/pypi/simple --trusted-host=XXXXXX:443 colorama` 8. Okay, **now** things start to get a little bit ugly. You're going to need access to the ISO of the release of Hedgehog Linux you're upgrading to, as we're going to grab some packages off of it. On another Linux system, [build it](hedgehog-iso-build.md#HedgehogISOBuild). diff --git a/docs/hedgehog.md b/docs/hedgehog.md index f3399d5e3..94880fa51 100644 --- a/docs/hedgehog.md +++ b/docs/hedgehog.md @@ -11,7 +11,7 @@ Hedgehog Linux is a Debian-based operating system built to * detect file transfers in network traffic and extract and scan those files for threats * generate and forward Zeek logs, Arkime sessions, and other information to [Malcolm]({{ site.github.repository_url }}) -![sensor-iso-build-docker-wrap-push-ghcr]({{ site.github.repository_url }}/workflows/sensor-iso-build-docker-wrap-push-ghcr/badge.svg) +![hedgehog-iso-build-docker-wrap-push-ghcr]({{ site.github.repository_url }}/workflows/hedgehog-iso-build-docker-wrap-push-ghcr/badge.svg) * [Sensor installation](hedgehog-installation.md#HedgehogInstallation) diff --git a/docs/kubernetes.md b/docs/kubernetes.md index 3b41df9a5..c74fb5d94 100644 --- a/docs/kubernetes.md +++ b/docs/kubernetes.md @@ -272,28 +272,28 @@ agent2 | agent2 | 192.168.56.12 | agent2 | k3s | 6000m | agent1 | agent1 | 192.168.56.11 | agent1 | k3s | 6000m | 861.34m | 14.36% | 19.55Gi | 9.29Gi | 61.28Gi | 11 | Pod Name | State | Pod IP | Pod Kind | Worker Node | CPU Usage | Memory Usage | Container Name:Restarts | Container Image | -api-deployment-6f4686cf59-bn286 | Running | 10.42.2.14 | ReplicaSet | agent1 | 0.11m | 59.62Mi | api-container:0 | api:24.02.0 | -file-monitor-deployment-855646bd75-vk7st | Running | 10.42.2.16 | ReplicaSet | agent1 | 8.47m | 1.46Gi | file-monitor-container:0 | file-monitor:24.02.0 | -zeek-live-deployment-64b69d4b6f-947vr | Running | 10.42.2.17 | ReplicaSet | agent1 | 0.02m | 12.44Mi | zeek-live-container:0 | zeek:24.02.0 | -dashboards-helper-deployment-69dc54f6b6-ln4sq | Running | 10.42.2.15 | ReplicaSet | agent1 | 10.77m | 38.43Mi | dashboards-helper-container:0 | dashboards-helper:24.02.0 | -upload-deployment-586568844b-4jnk9 | Running | 10.42.2.18 | ReplicaSet | agent1 | 0.15m | 29.78Mi | upload-container:0 | file-upload:24.02.0 | -filebeat-deployment-6ff8bc444f-t7h49 | Running | 10.42.2.20 | ReplicaSet | agent1 | 2.84m | 70.71Mi | filebeat-container:0 | filebeat-oss:24.02.0 | -zeek-offline-deployment-844f4865bd-g2sdm | Running | 10.42.2.21 | ReplicaSet | agent1 | 0.17m | 41.92Mi | zeek-offline-container:0 | zeek:24.02.0 | -logstash-deployment-6fbc9fdcd5-hwx8s | Running | 10.42.2.22 | ReplicaSet | agent1 | 85.55m | 2.91Gi | logstash-container:0 | logstash-oss:24.02.0 | -netbox-deployment-cdcff4977-hbbw5 | Running | 10.42.2.23 | ReplicaSet | agent1 | 807.64m | 702.86Mi | netbox-container:0 | netbox:24.02.0 | -suricata-offline-deployment-6ccdb89478-z5696 | Running | 10.42.2.19 | ReplicaSet | agent1 | 0.22m | 34.88Mi | suricata-offline-container:0 | suricata:24.02.0 | -dashboards-deployment-69b5465db-vz88g | Running | 10.42.1.14 | ReplicaSet | agent2 | 0.94m | 100.12Mi | dashboards-container:0 | dashboards:24.02.0 | -netbox-redis-cache-deployment-5f77d47b8b-z7t2z | Running | 10.42.1.15 | ReplicaSet | agent2 | 3.57m | 7.36Mi | netbox-redis-cache-container:0 | redis:24.02.0 | -suricata-live-deployment-6494c77759-9rlnt | Running | 10.42.1.16 | ReplicaSet | agent2 | 0.02m | 9.69Mi | suricata-live-container:0 | suricata:24.02.0 | -freq-deployment-cfd84fd97-dnngf | Running | 10.42.1.17 | ReplicaSet | agent2 | 0.2m | 26.36Mi | freq-container:0 | freq:24.02.0 | -arkime-deployment-56999cdd66-s98pp | Running | 10.42.1.18 | ReplicaSet | agent2 | 4.15m | 113.07Mi | arkime-container:0 | arkime:24.02.0 | -pcap-monitor-deployment-594ff674c4-fsm7m | Running | 10.42.1.19 | ReplicaSet | agent2 | 1.24m | 48.44Mi | pcap-monitor-container:0 | pcap-monitor:24.02.0 | -pcap-capture-deployment-7c8bf6957-jzpzn | Running | 10.42.1.20 | ReplicaSet | agent2 | 0.02m | 9.64Mi | pcap-capture-container:0 | pcap-capture:24.02.0 | -netbox-postgres-deployment-5879b8dffc-kkt56 | Running | 10.42.1.21 | ReplicaSet | agent2 | 70.91m | 33.02Mi | netbox-postgres-container:0 | postgresql:24.02.0 | -htadmin-deployment-6fc46888b9-sq6ln | Running | 10.42.1.23 | ReplicaSet | agent2 | 0.14m | 30.53Mi | htadmin-container:0 | htadmin:24.02.0 | -netbox-redis-deployment-5bcd8f6c96-j5xpf | Running | 10.42.1.24 | ReplicaSet | agent2 | 1.46m | 7.34Mi | netbox-redis-container:0 | redis:24.02.0 | -nginx-proxy-deployment-69fcc4968d-f68tq | Running | 10.42.1.22 | ReplicaSet | agent2 | 0.31m | 22.63Mi | nginx-proxy-container:0 | nginx-proxy:24.02.0 | -opensearch-deployment-75498799f6-4zmwd | Running | 10.42.1.25 | ReplicaSet | agent2 | 89.8m | 11.03Gi | opensearch-container:0 | opensearch:24.02.0 | +api-deployment-6f4686cf59-bn286 | Running | 10.42.2.14 | ReplicaSet | agent1 | 0.11m | 59.62Mi | api-container:0 | api:24.03.0 | +file-monitor-deployment-855646bd75-vk7st | Running | 10.42.2.16 | ReplicaSet | agent1 | 8.47m | 1.46Gi | file-monitor-container:0 | file-monitor:24.03.0 | +zeek-live-deployment-64b69d4b6f-947vr | Running | 10.42.2.17 | ReplicaSet | agent1 | 0.02m | 12.44Mi | zeek-live-container:0 | zeek:24.03.0 | +dashboards-helper-deployment-69dc54f6b6-ln4sq | Running | 10.42.2.15 | ReplicaSet | agent1 | 10.77m | 38.43Mi | dashboards-helper-container:0 | dashboards-helper:24.03.0 | +upload-deployment-586568844b-4jnk9 | Running | 10.42.2.18 | ReplicaSet | agent1 | 0.15m | 29.78Mi | upload-container:0 | file-upload:24.03.0 | +filebeat-deployment-6ff8bc444f-t7h49 | Running | 10.42.2.20 | ReplicaSet | agent1 | 2.84m | 70.71Mi | filebeat-container:0 | filebeat-oss:24.03.0 | +zeek-offline-deployment-844f4865bd-g2sdm | Running | 10.42.2.21 | ReplicaSet | agent1 | 0.17m | 41.92Mi | zeek-offline-container:0 | zeek:24.03.0 | +logstash-deployment-6fbc9fdcd5-hwx8s | Running | 10.42.2.22 | ReplicaSet | agent1 | 85.55m | 2.91Gi | logstash-container:0 | logstash-oss:24.03.0 | +netbox-deployment-cdcff4977-hbbw5 | Running | 10.42.2.23 | ReplicaSet | agent1 | 807.64m | 702.86Mi | netbox-container:0 | netbox:24.03.0 | +suricata-offline-deployment-6ccdb89478-z5696 | Running | 10.42.2.19 | ReplicaSet | agent1 | 0.22m | 34.88Mi | suricata-offline-container:0 | suricata:24.03.0 | +dashboards-deployment-69b5465db-vz88g | Running | 10.42.1.14 | ReplicaSet | agent2 | 0.94m | 100.12Mi | dashboards-container:0 | dashboards:24.03.0 | +netbox-redis-cache-deployment-5f77d47b8b-z7t2z | Running | 10.42.1.15 | ReplicaSet | agent2 | 3.57m | 7.36Mi | netbox-redis-cache-container:0 | redis:24.03.0 | +suricata-live-deployment-6494c77759-9rlnt | Running | 10.42.1.16 | ReplicaSet | agent2 | 0.02m | 9.69Mi | suricata-live-container:0 | suricata:24.03.0 | +freq-deployment-cfd84fd97-dnngf | Running | 10.42.1.17 | ReplicaSet | agent2 | 0.2m | 26.36Mi | freq-container:0 | freq:24.03.0 | +arkime-deployment-56999cdd66-s98pp | Running | 10.42.1.18 | ReplicaSet | agent2 | 4.15m | 113.07Mi | arkime-container:0 | arkime:24.03.0 | +pcap-monitor-deployment-594ff674c4-fsm7m | Running | 10.42.1.19 | ReplicaSet | agent2 | 1.24m | 48.44Mi | pcap-monitor-container:0 | pcap-monitor:24.03.0 | +pcap-capture-deployment-7c8bf6957-jzpzn | Running | 10.42.1.20 | ReplicaSet | agent2 | 0.02m | 9.64Mi | pcap-capture-container:0 | pcap-capture:24.03.0 | +netbox-postgres-deployment-5879b8dffc-kkt56 | Running | 10.42.1.21 | ReplicaSet | agent2 | 70.91m | 33.02Mi | netbox-postgres-container:0 | postgresql:24.03.0 | +htadmin-deployment-6fc46888b9-sq6ln | Running | 10.42.1.23 | ReplicaSet | agent2 | 0.14m | 30.53Mi | htadmin-container:0 | htadmin:24.03.0 | +netbox-redis-deployment-5bcd8f6c96-j5xpf | Running | 10.42.1.24 | ReplicaSet | agent2 | 1.46m | 7.34Mi | netbox-redis-container:0 | redis:24.03.0 | +nginx-proxy-deployment-69fcc4968d-f68tq | Running | 10.42.1.22 | ReplicaSet | agent2 | 0.31m | 22.63Mi | nginx-proxy-container:0 | nginx-proxy:24.03.0 | +opensearch-deployment-75498799f6-4zmwd | Running | 10.42.1.25 | ReplicaSet | agent2 | 89.8m | 11.03Gi | opensearch-container:0 | opensearch:24.03.0 | ``` The other control scripts (`stop`, `restart`, `logs`, etc.) work in a similar manner as in a Docker-based deployment. One notable difference is the `wipe` script: data on PersistentVolume storage cannot be deleted by `wipe`. It must be deleted manually on the storage media underlying the PersistentVolumes. @@ -553,28 +553,28 @@ agent1 | agent1 | 192.168.56.11 | agent1 | k3s | 6000m | agent2 | agent2 | 192.168.56.12 | agent2 | k3s | 6000m | 552.71m | 9.21% | 19.55Gi | 13.27Gi | 61.28Gi | 12 | Pod Name | State | Pod IP | Pod Kind | Worker Node | CPU Usage | Memory Usage | Container Name:Restarts | Container Image | -netbox-redis-cache-deployment-5f77d47b8b-jr9nt | Running | 10.42.2.6 | ReplicaSet | agent2 | 1.89m | 7.24Mi | netbox-redis-cache-container:0 | redis:24.02.0 | -netbox-redis-deployment-5bcd8f6c96-bkzmh | Running | 10.42.2.5 | ReplicaSet | agent2 | 1.62m | 7.52Mi | netbox-redis-container:0 | redis:24.02.0 | -dashboards-helper-deployment-69dc54f6b6-ks7ps | Running | 10.42.2.4 | ReplicaSet | agent2 | 12.95m | 40.75Mi | dashboards-helper-container:0 | dashboards-helper:24.02.0 | -freq-deployment-cfd84fd97-5bwp6 | Running | 10.42.2.8 | ReplicaSet | agent2 | 0.11m | 26.33Mi | freq-container:0 | freq:24.02.0 | -pcap-capture-deployment-7c8bf6957-hkvkn | Running | 10.42.2.12 | ReplicaSet | agent2 | 0.02m | 9.21Mi | pcap-capture-container:0 | pcap-capture:24.02.0 | -nginx-proxy-deployment-69fcc4968d-m57rz | Running | 10.42.2.10 | ReplicaSet | agent2 | 0.91m | 22.72Mi | nginx-proxy-container:0 | nginx-proxy:24.02.0 | -htadmin-deployment-6fc46888b9-vpt7l | Running | 10.42.2.7 | ReplicaSet | agent2 | 0.16m | 30.21Mi | htadmin-container:0 | htadmin:24.02.0 | -opensearch-deployment-75498799f6-5v92w | Running | 10.42.2.13 | ReplicaSet | agent2 | 139.2m | 10.86Gi | opensearch-container:0 | opensearch:24.02.0 | -zeek-live-deployment-64b69d4b6f-fcb6n | Running | 10.42.2.9 | ReplicaSet | agent2 | 0.02m | 109.55Mi | zeek-live-container:0 | zeek:24.02.0 | -dashboards-deployment-69b5465db-kgsqk | Running | 10.42.2.3 | ReplicaSet | agent2 | 14.98m | 108.85Mi | dashboards-container:0 | dashboards:24.02.0 | -arkime-deployment-56999cdd66-xxpw9 | Running | 10.42.2.11 | ReplicaSet | agent2 | 208.95m | 78.42Mi | arkime-container:0 | arkime:24.02.0 | -api-deployment-6f4686cf59-xt9md | Running | 10.42.1.3 | ReplicaSet | agent1 | 0.14m | 56.88Mi | api-container:0 | api:24.02.0 | -netbox-postgres-deployment-5879b8dffc-lb4qm | Running | 10.42.1.6 | ReplicaSet | agent1 | 141.2m | 48.02Mi | netbox-postgres-container:0 | postgresql:24.02.0 | -pcap-monitor-deployment-594ff674c4-fwq7g | Running | 10.42.1.12 | ReplicaSet | agent1 | 3.93m | 46.44Mi | pcap-monitor-container:0 | pcap-monitor:24.02.0 | -suricata-offline-deployment-6ccdb89478-j5fgj | Running | 10.42.1.10 | ReplicaSet | agent1 | 10.42m | 35.12Mi | suricata-offline-container:0 | suricata:24.02.0 | -suricata-live-deployment-6494c77759-rpt48 | Running | 10.42.1.8 | ReplicaSet | agent1 | 0.01m | 9.62Mi | suricata-live-container:0 | suricata:24.02.0 | -netbox-deployment-cdcff4977-7ns2q | Running | 10.42.1.7 | ReplicaSet | agent1 | 830.47m | 530.7Mi | netbox-container:0 | netbox:24.02.0 | -zeek-offline-deployment-844f4865bd-7x68b | Running | 10.42.1.9 | ReplicaSet | agent1 | 1.44m | 43.66Mi | zeek-offline-container:0 | zeek:24.02.0 | -filebeat-deployment-6ff8bc444f-pdgzj | Running | 10.42.1.11 | ReplicaSet | agent1 | 0.78m | 75.25Mi | filebeat-container:0 | filebeat-oss:24.02.0 | -file-monitor-deployment-855646bd75-nbngq | Running | 10.42.1.4 | ReplicaSet | agent1 | 1.69m | 1.46Gi | file-monitor-container:0 | file-monitor:24.02.0 | -upload-deployment-586568844b-9s7f5 | Running | 10.42.1.13 | ReplicaSet | agent1 | 0.14m | 29.62Mi | upload-container:0 | file-upload:24.02.0 | -logstash-deployment-6fbc9fdcd5-2hhx8 | Running | 10.42.1.5 | ReplicaSet | agent1 | 3236.29m | 357.36Mi | logstash-container:0 | logstash-oss:24.02.0 | +netbox-redis-cache-deployment-5f77d47b8b-jr9nt | Running | 10.42.2.6 | ReplicaSet | agent2 | 1.89m | 7.24Mi | netbox-redis-cache-container:0 | redis:24.03.0 | +netbox-redis-deployment-5bcd8f6c96-bkzmh | Running | 10.42.2.5 | ReplicaSet | agent2 | 1.62m | 7.52Mi | netbox-redis-container:0 | redis:24.03.0 | +dashboards-helper-deployment-69dc54f6b6-ks7ps | Running | 10.42.2.4 | ReplicaSet | agent2 | 12.95m | 40.75Mi | dashboards-helper-container:0 | dashboards-helper:24.03.0 | +freq-deployment-cfd84fd97-5bwp6 | Running | 10.42.2.8 | ReplicaSet | agent2 | 0.11m | 26.33Mi | freq-container:0 | freq:24.03.0 | +pcap-capture-deployment-7c8bf6957-hkvkn | Running | 10.42.2.12 | ReplicaSet | agent2 | 0.02m | 9.21Mi | pcap-capture-container:0 | pcap-capture:24.03.0 | +nginx-proxy-deployment-69fcc4968d-m57rz | Running | 10.42.2.10 | ReplicaSet | agent2 | 0.91m | 22.72Mi | nginx-proxy-container:0 | nginx-proxy:24.03.0 | +htadmin-deployment-6fc46888b9-vpt7l | Running | 10.42.2.7 | ReplicaSet | agent2 | 0.16m | 30.21Mi | htadmin-container:0 | htadmin:24.03.0 | +opensearch-deployment-75498799f6-5v92w | Running | 10.42.2.13 | ReplicaSet | agent2 | 139.2m | 10.86Gi | opensearch-container:0 | opensearch:24.03.0 | +zeek-live-deployment-64b69d4b6f-fcb6n | Running | 10.42.2.9 | ReplicaSet | agent2 | 0.02m | 109.55Mi | zeek-live-container:0 | zeek:24.03.0 | +dashboards-deployment-69b5465db-kgsqk | Running | 10.42.2.3 | ReplicaSet | agent2 | 14.98m | 108.85Mi | dashboards-container:0 | dashboards:24.03.0 | +arkime-deployment-56999cdd66-xxpw9 | Running | 10.42.2.11 | ReplicaSet | agent2 | 208.95m | 78.42Mi | arkime-container:0 | arkime:24.03.0 | +api-deployment-6f4686cf59-xt9md | Running | 10.42.1.3 | ReplicaSet | agent1 | 0.14m | 56.88Mi | api-container:0 | api:24.03.0 | +netbox-postgres-deployment-5879b8dffc-lb4qm | Running | 10.42.1.6 | ReplicaSet | agent1 | 141.2m | 48.02Mi | netbox-postgres-container:0 | postgresql:24.03.0 | +pcap-monitor-deployment-594ff674c4-fwq7g | Running | 10.42.1.12 | ReplicaSet | agent1 | 3.93m | 46.44Mi | pcap-monitor-container:0 | pcap-monitor:24.03.0 | +suricata-offline-deployment-6ccdb89478-j5fgj | Running | 10.42.1.10 | ReplicaSet | agent1 | 10.42m | 35.12Mi | suricata-offline-container:0 | suricata:24.03.0 | +suricata-live-deployment-6494c77759-rpt48 | Running | 10.42.1.8 | ReplicaSet | agent1 | 0.01m | 9.62Mi | suricata-live-container:0 | suricata:24.03.0 | +netbox-deployment-cdcff4977-7ns2q | Running | 10.42.1.7 | ReplicaSet | agent1 | 830.47m | 530.7Mi | netbox-container:0 | netbox:24.03.0 | +zeek-offline-deployment-844f4865bd-7x68b | Running | 10.42.1.9 | ReplicaSet | agent1 | 1.44m | 43.66Mi | zeek-offline-container:0 | zeek:24.03.0 | +filebeat-deployment-6ff8bc444f-pdgzj | Running | 10.42.1.11 | ReplicaSet | agent1 | 0.78m | 75.25Mi | filebeat-container:0 | filebeat-oss:24.03.0 | +file-monitor-deployment-855646bd75-nbngq | Running | 10.42.1.4 | ReplicaSet | agent1 | 1.69m | 1.46Gi | file-monitor-container:0 | file-monitor:24.03.0 | +upload-deployment-586568844b-9s7f5 | Running | 10.42.1.13 | ReplicaSet | agent1 | 0.14m | 29.62Mi | upload-container:0 | file-upload:24.03.0 | +logstash-deployment-6fbc9fdcd5-2hhx8 | Running | 10.42.1.5 | ReplicaSet | agent1 | 3236.29m | 357.36Mi | logstash-container:0 | logstash-oss:24.03.0 | ``` View container logs for the Malcolm deployment with `./scripts/logs` (if **[stern](https://github.com/stern/stern)** present in `$PATH`): diff --git a/docs/malcolm-config.md b/docs/malcolm-config.md index 7caac0274..a1d1bb6f7 100644 --- a/docs/malcolm-config.md +++ b/docs/malcolm-config.md @@ -38,7 +38,7 @@ Although the configuration script automates many of the following configuration - `LOGSTASH_SEVERITY_SCORING` - if set to `true`, Logstash will perform [severity scoring](severity.md#Severity) when analyzing Zeek logs (default `true`) - `LOGSTASH_NETBOX_ENRICHMENT` - if set to `true`, Logstash will [enrich network traffic metadata](asset-interaction-analysis.md#NetBoxEnrichment) via NetBox API calls - `LOGSTASH_NETBOX_AUTO_POPULATE` - if set to `true`, Logstash will [populate the NetBox inventory](asset-interaction-analysis.md#NetBoxPopPassive) based on observed network traffic - - `LS_JAVA_OPTS` - part of LogStash's [JVM settings](https://www.elastic.co/guide/en/logstash/current/jvm-settings.html), the `-Xms` and `-Xmx` values set the size of LogStash's Java heap (we recommend somewhere between `1500m` and `4g`) + - `LS_JAVA_OPTS` - part of LogStash's [JVM settings](https://www.elastic.co/guide/en/logstash/current/jvm-settings.html), the `-Xmx` and `-Xms` values set the size of LogStash's Java heap (we recommend somewhere between `1500m` and `4g`) * `pipeline.workers`, `pipeline.batch.size` and `pipeline.batch.delay` - these settings are used to tune the performance and resource utilization of the the `logstash` container; see [Tuning and Profiling Logstash Performance](https://www.elastic.co/guide/en/logstash/current/tuning-logstash.html), [`logstash.yml`](https://www.elastic.co/guide/en/logstash/current/logstash-settings-file.html) and [Multiple Pipelines](https://www.elastic.co/guide/en/logstash/current/multiple-pipelines.html) * **`lookup-common.env`** - settings for enrichment lookups, including those used for [customizing event severity scoring](severity.md#SeverityConfig) - `CONNECTION_SECONDS_SEVERITY_THRESHOLD` - when [severity scoring](severity.md#Severity) is enabled, this variable indicates the duration threshold (in seconds) for assigning severity to long connections (default `3600`) @@ -56,7 +56,7 @@ Although the configuration script automates many of the following configuration - `NGINX_LOG_ACCESS_AND_ERRORS` - if set to `true`, all access to Malcolm via its [web interfaces](quickstart.md#UserInterfaceURLs) will be logged to OpenSearch (default `false`) - `NGINX_SSL` - if set to `true`, require HTTPS connections to Malcolm's `nginx-proxy` container (default); if set to `false`, use unencrypted HTTP connections (using unsecured HTTP connections is **NOT** recommended unless you are running Malcolm behind another reverse proxy such as Traefik, Caddy, etc.) * **`opensearch.env`** - settings specific to [OpenSearch](https://opensearch.org/) - - `OPENSEARCH_JAVA_OPTS` - one of OpenSearch's most [important settings](https://opensearch.org/docs/latest/install-and-configure/install-opensearch/index/#important-settings), the `-Xms` and `-Xmx` values set the size of OpenSearch's Java heap (we recommend setting this value to half of system RAM, up to 32 gigabytes) + - `OPENSEARCH_JAVA_OPTS` - one of OpenSearch's most [important settings](https://opensearch.org/docs/latest/install-and-configure/install-opensearch/index/#important-settings), the `-Xmx` and `-Xms` values set the size of OpenSearch's Java heap (we recommend setting this value to half of system RAM, up to 32 gigabytes) - `OPENSEARCH_PRIMARY` - one of `opensearch-local`, `opensearch-remote`, or `elasticsearch-remote`, to determine the [OpenSearch or Elasticsearch instance](opensearch-instances.md#OpenSearchInstance) Malcolm will use (default `opensearch-local`) - `OPENSEARCH_URL` - when using Malcolm's internal OpenSearch instance (i.e., `OPENSEARCH_PRIMARY` is `opensearch-local`) this should be `http://opensearch:9200`, otherwise this value specifies the primary remote instance URL in the format `protocol://host:port` (default `http://opensearch:9200`) - `OPENSEARCH_SSL_CERTIFICATE_VERIFICATION` - if set to `true`, connections to the primary remote OpenSearch instance will require full TLS certificate validation (this may fail if using self-signed certificates) (default `false`) @@ -111,9 +111,10 @@ Although the configuration script automates many of the following configuration - `VTOT_API2_KEY` – used to specify a [VirusTotal Public API v.20](https://www.virustotal.com/en/documentation/public-api/) key, which, if specified, will be used to submit hashes of [Zeek-extracted files](file-scanning.md#ZeekFileExtraction) to VirusTotal - `ZEEK_AUTO_ANALYZE_PCAP_FILES` – if set to `true`, all PCAP files imported into Malcolm will automatically be analyzed by Zeek, and the resulting logs will also be imported (default `false`) - `ZEEK_AUTO_ANALYZE_PCAP_THREADS` – the number of threads available to Malcolm for analyzing Zeek logs (default `1`) - - `ZEEK_DISABLE_…` - if set to any non-blank value, each of these variables can be used to disable a certain Zeek function when it analyzes PCAP files (for example, setting `ZEEK_DISABLE_LOG_PASSWORDS` to `true` to disable logging of cleartext passwords) + - `ZEEK_JSON` - whether Zeek should generate [JSON format logs](https://docs.zeek.org/en/master/log-formats.html#zeek-json-format-logs) (`true`) or [TSV format logs](https://docs.zeek.org/en/master/log-formats.html#zeek-tsv-format-logs) (`false`) + - `ZEEK_DISABLE_…` - if set to `true`, each of these variables can be used to disable a certain Zeek function when it analyzes PCAP files (for example, setting `ZEEK_DISABLE_LOG_PASSWORDS` to `true` to disable logging of cleartext passwords) - `ZEEK_…_PORTS` - used to specify non-default ports to register certain Zeek analyzers (e.g., `ZEEK_SYNCHROPHASOR_PORTS` for the [ICSNPP-Synchrophasor analyzer](https://github.com/cisagov/icsnpp-synchrophasor/), `ZEEK_GENISYS_PORTS` for the [ICSNPP-Genisys analyzer](https://github.com/cisagov/icsnpp-genisys/), and `ZEEK_ENIP_PORTS` for the [ICSNPP-Ethernet/IP analyzer](https://github.com/cisagov/icsnpp-enip/)) formatted as a comma-separated list of [Zeek ports](https://docs.zeek.org/en/master/scripting/basics.html#port) (e.g., `12345/tcp` or `4041/tcp,4042/udp`) - - `ZEEK_DISABLE_ICS_ALL` and `ZEEK_DISABLE_ICS_…` - if set to any non-blank value, these variables can be used to disable Zeek's protocol analyzers for Operational Technology/Industrial Control Systems (OT/ICS) protocols + - `ZEEK_DISABLE_ICS_ALL` and `ZEEK_DISABLE_ICS_…` - if set to `true`, these variables can be used to disable Zeek's protocol analyzers for Operational Technology/Industrial Control Systems (OT/ICS) protocols - `ZEEK_DISABLE_BEST_GUESS_ICS` - see ["Best Guess" Fingerprinting for ICS Protocols](ics-best-guess.md#ICSBestGuess) - `ZEEK_EXTRACTOR_MODE` – determines the file extraction behavior for file transfers detected by Zeek; see [Automatic file extraction and scanning](file-scanning.md#ZeekFileExtraction) for more details - `ZEEK_INTEL_FEED_SINCE` - when querying a [TAXII](zeek-intel.md#ZeekIntelSTIX) or [MISP](zeek-intel.md#ZeekIntelMISP) feed, only process threat indicators created or modified since the time represented by this value; it may be either a fixed date/time (`01/01/2021`) or relative interval (`30 days ago`) diff --git a/docs/malcolm-hedgehog-e2e-iso-install.md b/docs/malcolm-hedgehog-e2e-iso-install.md index 666bfc743..71a303031 100644 --- a/docs/malcolm-hedgehog-e2e-iso-install.md +++ b/docs/malcolm-hedgehog-e2e-iso-install.md @@ -463,7 +463,7 @@ To specify which files should be extracted, specify the Zeek file carving mode: ![Zeek file carving mode](./images/hedgehog/images/zeek_file_carve_mode.png) -If unsure what mode to choose, both **mapped (except common plain text files)** (to carve and scan almost all files) and **interesting** (to only carve and scan files with [mime types of common attack vectors]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/sensor-iso/interface/sensor_ctl/zeek/extractor_override.interesting.zeek)) are probably good choices. +If unsure what mode to choose, both **mapped (except common plain text files)** (to carve and scan almost all files) and **interesting** (to only carve and scan files with [mime types of common attack vectors]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/hedgehog-iso/interface/sensor_ctl/zeek/extractor_override.interesting.zeek)) are probably good choices. Next, specify which carved files to preserve (saved on the sensor under `/capture/bro/capture/extract_files/quarantine` by default). In order to not consume all the sensor's available storage space, the oldest preserved files will be pruned along with the oldest Zeek logs as described below with **AUTOSTART_PRUNE_ZEEK** in the [autostart services](#HedgehogConfigAutostart) section. diff --git a/docs/malcolm-iso.md b/docs/malcolm-iso.md index da5e35fe0..21ec5f9eb 100644 --- a/docs/malcolm-iso.md +++ b/docs/malcolm-iso.md @@ -41,7 +41,7 @@ Building the ISO may take 30 minutes or more depending on the system. As the bui ``` … -Finished, created "/malcolm-build/malcolm-iso/malcolm-24.02.0.iso" +Finished, created "/malcolm-build/malcolm-iso/malcolm-24.03.0.iso" … ``` diff --git a/docs/protocols.md b/docs/protocols.md index e40e11084..abda373b3 100644 --- a/docs/protocols.md +++ b/docs/protocols.md @@ -33,6 +33,7 @@ Malcolm uses [Zeek](https://docs.zeek.org/en/stable/script-reference/proto-analy |OpenVPN|[🔗](https://en.wikipedia.org/wiki/OpenVPN)|[🔗](https://openvpn.net/community-resources/openvpn-protocol/)[🔗](https://zeek.org/2021/03/16/a-zeek-openvpn-protocol-analyzer/)||[✓](https://github.com/corelight/zeek-spicy-openvpn)| |PostgreSQL|[🔗](https://en.wikipedia.org/wiki/PostgreSQL)|[🔗](https://www.postgresql.org/)|[✓](https://github.com/arkime/arkime/blob/master/capture/parsers/postgresql.c)|| |Process Field Net (PROFINET)|[🔗](https://en.wikipedia.org/wiki/PROFINET)|[🔗](https://us.profinet.com/technology/profinet/)||[✓](https://github.com/amzn/zeek-plugin-profinet/blob/master/scripts/main.zeek)| +|PROFINET IO CM (Input/Output Context Manager)|[🔗](https://wiki.wireshark.org/PROFINET/IO)|[🔗](https://us.profinet.com/technology/profinet/)[🔗](https://webstore.iec.ch/publication/83418)||[✓](https://github.com/cisagov/icsnpp-profinet-io-cm/blob/main/analyzer/types.zeek)| |Remote Authentication Dial-In User Service (RADIUS)|[🔗](https://en.wikipedia.org/wiki/RADIUS)|[🔗](https://tools.ietf.org/html/rfc2865)|[✓](https://github.com/arkime/arkime/blob/master/capture/parsers/radius.c)|[✓](https://docs.zeek.org/en/stable/scripts/base/protocols/radius/main.zeek.html#type-RADIUS::Info)| |Remote Desktop Protocol (RDP)|[🔗](https://en.wikipedia.org/wiki/Remote_Desktop_Protocol)|[🔗](https://docs.microsoft.com/en-us/windows/win32/termserv/remote-desktop-protocol?redirectedfrom=MSDN)||[✓](https://docs.zeek.org/en/stable/scripts/base/protocols/rdp/main.zeek.html#type-RDP::Info)| |Remote Framebuffer (RFB)|[🔗](https://en.wikipedia.org/wiki/RFB_protocol)|[🔗](https://tools.ietf.org/html/rfc6143)||[✓](https://docs.zeek.org/en/stable/scripts/base/protocols/rfb/main.zeek.html#type-RFB::Info)| diff --git a/docs/quickstart.md b/docs/quickstart.md index de66729d6..8d4e5a8a9 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -54,25 +54,25 @@ You can then observe the images have been retrieved by running `docker images`: ``` $ docker images REPOSITORY TAG IMAGE ID CREATED SIZE -ghcr.io/idaholab/malcolm/api 24.02.0 xxxxxxxxxxxx 3 days ago 158MB -ghcr.io/idaholab/malcolm/arkime 24.02.0 xxxxxxxxxxxx 3 days ago 816MB -ghcr.io/idaholab/malcolm/dashboards 24.02.0 xxxxxxxxxxxx 3 days ago 1.02GB -ghcr.io/idaholab/malcolm/dashboards-helper 24.02.0 xxxxxxxxxxxx 3 days ago 184MB -ghcr.io/idaholab/malcolm/file-monitor 24.02.0 xxxxxxxxxxxx 3 days ago 588MB -ghcr.io/idaholab/malcolm/file-upload 24.02.0 xxxxxxxxxxxx 3 days ago 259MB -ghcr.io/idaholab/malcolm/filebeat-oss 24.02.0 xxxxxxxxxxxx 3 days ago 624MB -ghcr.io/idaholab/malcolm/freq 24.02.0 xxxxxxxxxxxx 3 days ago 132MB -ghcr.io/idaholab/malcolm/htadmin 24.02.0 xxxxxxxxxxxx 3 days ago 242MB -ghcr.io/idaholab/malcolm/logstash-oss 24.02.0 xxxxxxxxxxxx 3 days ago 1.35GB -ghcr.io/idaholab/malcolm/netbox 24.02.0 xxxxxxxxxxxx 3 days ago 1.01GB -ghcr.io/idaholab/malcolm/nginx-proxy 24.02.0 xxxxxxxxxxxx 3 days ago 121MB -ghcr.io/idaholab/malcolm/opensearch 24.02.0 xxxxxxxxxxxx 3 days ago 1.17GB -ghcr.io/idaholab/malcolm/pcap-capture 24.02.0 xxxxxxxxxxxx 3 days ago 121MB -ghcr.io/idaholab/malcolm/pcap-monitor 24.02.0 xxxxxxxxxxxx 3 days ago 213MB -ghcr.io/idaholab/malcolm/postgresql 24.02.0 xxxxxxxxxxxx 3 days ago 268MB -ghcr.io/idaholab/malcolm/redis 24.02.0 xxxxxxxxxxxx 3 days ago 34.2MB -ghcr.io/idaholab/malcolm/suricata 24.02.0 xxxxxxxxxxxx 3 days ago 278MB -ghcr.io/idaholab/malcolm/zeek 24.02.0 xxxxxxxxxxxx 3 days ago 1GB +ghcr.io/idaholab/malcolm/api 24.03.0 xxxxxxxxxxxx 3 days ago 158MB +ghcr.io/idaholab/malcolm/arkime 24.03.0 xxxxxxxxxxxx 3 days ago 816MB +ghcr.io/idaholab/malcolm/dashboards 24.03.0 xxxxxxxxxxxx 3 days ago 1.02GB +ghcr.io/idaholab/malcolm/dashboards-helper 24.03.0 xxxxxxxxxxxx 3 days ago 184MB +ghcr.io/idaholab/malcolm/file-monitor 24.03.0 xxxxxxxxxxxx 3 days ago 588MB +ghcr.io/idaholab/malcolm/file-upload 24.03.0 xxxxxxxxxxxx 3 days ago 259MB +ghcr.io/idaholab/malcolm/filebeat-oss 24.03.0 xxxxxxxxxxxx 3 days ago 624MB +ghcr.io/idaholab/malcolm/freq 24.03.0 xxxxxxxxxxxx 3 days ago 132MB +ghcr.io/idaholab/malcolm/htadmin 24.03.0 xxxxxxxxxxxx 3 days ago 242MB +ghcr.io/idaholab/malcolm/logstash-oss 24.03.0 xxxxxxxxxxxx 3 days ago 1.35GB +ghcr.io/idaholab/malcolm/netbox 24.03.0 xxxxxxxxxxxx 3 days ago 1.01GB +ghcr.io/idaholab/malcolm/nginx-proxy 24.03.0 xxxxxxxxxxxx 3 days ago 121MB +ghcr.io/idaholab/malcolm/opensearch 24.03.0 xxxxxxxxxxxx 3 days ago 1.17GB +ghcr.io/idaholab/malcolm/pcap-capture 24.03.0 xxxxxxxxxxxx 3 days ago 121MB +ghcr.io/idaholab/malcolm/pcap-monitor 24.03.0 xxxxxxxxxxxx 3 days ago 213MB +ghcr.io/idaholab/malcolm/postgresql 24.03.0 xxxxxxxxxxxx 3 days ago 268MB +ghcr.io/idaholab/malcolm/redis 24.03.0 xxxxxxxxxxxx 3 days ago 34.2MB +ghcr.io/idaholab/malcolm/suricata 24.03.0 xxxxxxxxxxxx 3 days ago 278MB +ghcr.io/idaholab/malcolm/zeek 24.03.0 xxxxxxxxxxxx 3 days ago 1GB ``` ### Import from pre-packaged tarballs diff --git a/docs/ubuntu-install-example.md b/docs/ubuntu-install-example.md index 179901b8f..ff4f0d4b0 100644 --- a/docs/ubuntu-install-example.md +++ b/docs/ubuntu-install-example.md @@ -257,25 +257,25 @@ Pulling zeek ... done user@host:~/Malcolm$ docker images REPOSITORY TAG IMAGE ID CREATED SIZE -ghcr.io/idaholab/malcolm/api 24.02.0 xxxxxxxxxxxx 3 days ago 158MB -ghcr.io/idaholab/malcolm/arkime 24.02.0 xxxxxxxxxxxx 3 days ago 816MB -ghcr.io/idaholab/malcolm/dashboards 24.02.0 xxxxxxxxxxxx 3 days ago 1.02GB -ghcr.io/idaholab/malcolm/dashboards-helper 24.02.0 xxxxxxxxxxxx 3 days ago 184MB -ghcr.io/idaholab/malcolm/file-monitor 24.02.0 xxxxxxxxxxxx 3 days ago 588MB -ghcr.io/idaholab/malcolm/file-upload 24.02.0 xxxxxxxxxxxx 3 days ago 259MB -ghcr.io/idaholab/malcolm/filebeat-oss 24.02.0 xxxxxxxxxxxx 3 days ago 624MB -ghcr.io/idaholab/malcolm/freq 24.02.0 xxxxxxxxxxxx 3 days ago 132MB -ghcr.io/idaholab/malcolm/htadmin 24.02.0 xxxxxxxxxxxx 3 days ago 242MB -ghcr.io/idaholab/malcolm/logstash-oss 24.02.0 xxxxxxxxxxxx 3 days ago 1.35GB -ghcr.io/idaholab/malcolm/netbox 24.02.0 xxxxxxxxxxxx 3 days ago 1.01GB -ghcr.io/idaholab/malcolm/nginx-proxy 24.02.0 xxxxxxxxxxxx 3 days ago 121MB -ghcr.io/idaholab/malcolm/opensearch 24.02.0 xxxxxxxxxxxx 3 days ago 1.17GB -ghcr.io/idaholab/malcolm/pcap-capture 24.02.0 xxxxxxxxxxxx 3 days ago 121MB -ghcr.io/idaholab/malcolm/pcap-monitor 24.02.0 xxxxxxxxxxxx 3 days ago 213MB -ghcr.io/idaholab/malcolm/postgresql 24.02.0 xxxxxxxxxxxx 3 days ago 268MB -ghcr.io/idaholab/malcolm/redis 24.02.0 xxxxxxxxxxxx 3 days ago 34.2MB -ghcr.io/idaholab/malcolm/suricata 24.02.0 xxxxxxxxxxxx 3 days ago 278MB -ghcr.io/idaholab/malcolm/zeek 24.02.0 xxxxxxxxxxxx 3 days ago 1GB +ghcr.io/idaholab/malcolm/api 24.03.0 xxxxxxxxxxxx 3 days ago 158MB +ghcr.io/idaholab/malcolm/arkime 24.03.0 xxxxxxxxxxxx 3 days ago 816MB +ghcr.io/idaholab/malcolm/dashboards 24.03.0 xxxxxxxxxxxx 3 days ago 1.02GB +ghcr.io/idaholab/malcolm/dashboards-helper 24.03.0 xxxxxxxxxxxx 3 days ago 184MB +ghcr.io/idaholab/malcolm/file-monitor 24.03.0 xxxxxxxxxxxx 3 days ago 588MB +ghcr.io/idaholab/malcolm/file-upload 24.03.0 xxxxxxxxxxxx 3 days ago 259MB +ghcr.io/idaholab/malcolm/filebeat-oss 24.03.0 xxxxxxxxxxxx 3 days ago 624MB +ghcr.io/idaholab/malcolm/freq 24.03.0 xxxxxxxxxxxx 3 days ago 132MB +ghcr.io/idaholab/malcolm/htadmin 24.03.0 xxxxxxxxxxxx 3 days ago 242MB +ghcr.io/idaholab/malcolm/logstash-oss 24.03.0 xxxxxxxxxxxx 3 days ago 1.35GB +ghcr.io/idaholab/malcolm/netbox 24.03.0 xxxxxxxxxxxx 3 days ago 1.01GB +ghcr.io/idaholab/malcolm/nginx-proxy 24.03.0 xxxxxxxxxxxx 3 days ago 121MB +ghcr.io/idaholab/malcolm/opensearch 24.03.0 xxxxxxxxxxxx 3 days ago 1.17GB +ghcr.io/idaholab/malcolm/pcap-capture 24.03.0 xxxxxxxxxxxx 3 days ago 121MB +ghcr.io/idaholab/malcolm/pcap-monitor 24.03.0 xxxxxxxxxxxx 3 days ago 213MB +ghcr.io/idaholab/malcolm/postgresql 24.03.0 xxxxxxxxxxxx 3 days ago 268MB +ghcr.io/idaholab/malcolm/redis 24.03.0 xxxxxxxxxxxx 3 days ago 34.2MB +ghcr.io/idaholab/malcolm/suricata 24.03.0 xxxxxxxxxxxx 3 days ago 278MB +ghcr.io/idaholab/malcolm/zeek 24.03.0 xxxxxxxxxxxx 3 days ago 1GB ``` Finally, start Malcolm. When Malcolm starts it will stream informational and debug messages to the console until it has completed initializing. diff --git a/docs/upload.md b/docs/upload.md index 4c3b8bb6f..4210402e7 100644 --- a/docs/upload.md +++ b/docs/upload.md @@ -15,6 +15,7 @@ The types of files supported are: * PCAP files (of mime type `application/vnd.tcpdump.pcap` or `application/x-pcapng`) - PCAPNG files are *partially* supported: Zeek is able to process PCAPNG files, but not all of Arkime's packet examination features work correctly * Zeek logs in archive files (`application/gzip`, `application/x-gzip`, `application/x-7z-compressed`, `application/x-bzip2`, `application/x-cpio`, `application/x-lzip`, `application/x-lzma`, `application/x-rar-compressed`, `application/x-tar`, `application/x-xz`, or `application/zip`) + - because log fields may differ depending on Zeek's configuration, users are recommended to use [Zeek JSON format logs](https://docs.zeek.org/en/master/log-formats.html#zeek-json-format-logs) when generating Zeek logs outside of Malcolm to later be uploaded to Malcolm for procesing - where the Zeek logs are found in the internal directory structure in the archive file does not matter Files uploaded via these methods are monitored and moved automatically to other directories for processing, generally within 1 minute of completion of the upload. diff --git a/docs/zeek-intel.md b/docs/zeek-intel.md index 20cd52427..1d853f66a 100644 --- a/docs/zeek-intel.md +++ b/docs/zeek-intel.md @@ -8,9 +8,9 @@ To quote Zeek's [Intelligence Framework](https://docs.zeek.org/en/master/framewo Malcolm doesn't come bundled with intelligence files from any particular feed, but they can be easily included into a local instance. On [startup]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/shared/bin/zeek_intel_setup.sh), Malcolm's `ghcr.io/idaholab/malcolm/zeek` docker container enumerates the subdirectories under `./zeek/intel` (which is [bind mounted](https://docs.docker.com/storage/bind-mounts/) into the container's runtime) and configures Zeek so those intelligence files will be automatically included in its local policy. Subdirectories under `./zeek/intel` that contain their own `__load__.zeek` file will be `@load`-ed as-is, while subdirectories containing "loose" intelligence files will be [loaded](https://docs.zeek.org/en/master/frameworks/intel.html#loading-intelligence) automatically with a `redef Intel::read_files` directive. -Note that Malcolm does not manage updates for these intelligence files. You should use the update mechanism suggested by the feeds' maintainers to keep intelligence files up to date, or use a [TAXII](#ZeekIntelSTIX) or [MISP](#ZeekIntelMISP) feed as described below. +Note that Malcolm does not manage updates for these intelligence files. Users use the update mechanism suggested by the feeds' maintainers to keep intelligence files up to date, or use a [TAXII](#ZeekIntelSTIX) or [MISP](#ZeekIntelMISP) feed as described below. -Adding and deleting intelligence files under this directory will take effect upon [restarting Malcolm](running.md#StopAndRestart). Alternately, you can use the `ZEEK_INTEL_REFRESH_CRON_EXPRESSION` environment variable containing a [cron expression](https://en.wikipedia.org/wiki/Cron#CRON_expression) to specify the interval at which the intel files should be refreshed. This can also be done manually without restarting Malcolm by running the following command from the Malcolm installation directory: +Adding and deleting intelligence files under this directory will take effect upon [restarting Malcolm](running.md#StopAndRestart). Alternately, users can use the `ZEEK_INTEL_REFRESH_CRON_EXPRESSION` environment variable containing a [cron expression](https://en.wikipedia.org/wiki/Cron#CRON_expression) to specify the interval at which the intel files should be refreshed. This can also be done manually without restarting Malcolm by running the following command from the Malcolm installation directory: ``` docker compose exec --user $(id -u) zeek /usr/local/bin/entrypoint.sh true @@ -36,7 +36,7 @@ taxii|2.1|https://example.com/taxii/api2/|URL Blocklist … ``` -Malcolm will attempt to query the TAXII feed(s) for `indicator` STIX objects and convert them to the Zeek intelligence format as described above. There are publicly available TAXII 2.x-compatible services provided by a number of organizations including [Anomali Labs](https://www.anomali.com/resources/limo) and [MITRE](https://www.mitre.org/capabilities/cybersecurity/overview/cybersecurity-blog/attck%E2%84%A2-content-available-in-stix%E2%84%A2-20-via); or you may choose from several open-source offerings to roll your own TAXII 2 server (e.g., [oasis-open/cti-taxii-server](https://github.com/oasis-open/cti-taxii-server), [freetaxii/server](https://github.com/freetaxii/server), [StephenOTT/TAXII-Server](https://github.com/StephenOTT/TAXII-Server), etc.). +Malcolm will attempt to query the TAXII feed(s) for `indicator` STIX objects and convert them to the Zeek intelligence format as described above. There are publicly available TAXII 2.x-compatible services provided by a number of organizations including [Anomali Labs](https://www.anomali.com/resources/limo) and [MITRE](https://www.mitre.org/capabilities/cybersecurity/overview/cybersecurity-blog/attck%E2%84%A2-content-available-in-stix%E2%84%A2-20-via); or users may choose from several open-source offerings to roll their own TAXII 2 server (e.g., [oasis-open/cti-taxii-server](https://github.com/oasis-open/cti-taxii-server), [freetaxii/server](https://github.com/freetaxii/server), [StephenOTT/TAXII-Server](https://github.com/StephenOTT/TAXII-Server), etc.). Note that only **indicators** of [**cyber-observable objects**](https://docs.oasis-open.org/cti/stix/v2.1/cs01/stix-v2.1-cs01.html#_mlbmudhl16lr) matched with the **equals (`=`)** [comparison operator](https://docs.oasis-open.org/cti/stix/v2.1/cs01/stix-v2.1-cs01.html#_t11hn314cr7w) against a **single value** can be expressed as Zeek intelligence items. More complex STIX indicators will be silently ignored. @@ -44,19 +44,30 @@ Note that only **indicators** of [**cyber-observable objects**](https://docs.oas In addition to loading Zeek intelligence files on startup, Malcolm will [automatically generate]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/shared/bin/zeek_intel_from_threat_feed.py) a Zeek intelligence file for all [Malware Information Sharing Platform (MISP)](https://www.misp-project.org/datamodels/) JSON files found under `./zeek/intel/MISP`. -Additionally, if a special text file named `.misp_input.txt` is found in `./zeek/intel/MISP`, that file will be read and processed as a list of [MISP feed](https://misp.gitbooks.io/misp-book/content/managing-feeds/#feeds) URLs, one per line, according to the following format (the authentication key is optional): +Additionally, if a special text file named `.misp_input.txt` is found in `./zeek/intel/MISP`, that file will be read and processed as a list of [MISP feed](https://misp.gitbooks.io/misp-book/content/managing-feeds/#feeds) URLs, one per line, according to the following format: ``` -misp|manifest_url|auth_key +misp|misp_url|auth_key (optional) ``` For example: ``` misp|https://example.com/data/feed-osint/manifest.json|df97338db644c64fbfd90f3e03ba8870 +misp|https://example.com/doc/misp/| +misp|https://example.com/attributes|a943f5ff506ee6198e996333e0b672b1 +misp|https://example.com/events|a943f5ff506ee6198e996333e0b672b1 … ``` -Malcolm will attempt to connect to the MISP feed(s) and retrieve [`Attribute`](https://www.misp-standard.org/rfc/misp-standard-core.html#name-attribute) objects of MISP events and convert them to the Zeek intelligence format as described above. There are publicly available [MISP feeds](https://www.misp-project.org/feeds/) and [communities](https://www.misp-project.org/communities/), or you may [run your own MISP instance](https://www.misp-project.org/2019/09/25/hostev-vs-own-misp.html/). +Malcolm will attempt to connect to the MISP feed(s) and retrieve [`Attribute`](https://www.misp-standard.org/rfc/misp-standard-core.html#name-attribute) objects of MISP events and convert them to the Zeek intelligence format as described above. There are publicly available [MISP feeds](https://www.misp-project.org/feeds/) and [communities](https://www.misp-project.org/communities/), or users [may run](https://github.com/MISP/misp-docker) their [own MISP instance](https://www.misp-project.org/2019/09/25/hostev-vs-own-misp.html/). + +Upon Malcolm connects to the URLs for the MISP feeds in `.misp_input.txt`, it will attempt to determine the format of the data served and process it accordingly. This could be presented as: + +* a manifest JSON file +* a directory listing containing a file named `manifest.json` +* a directory listing of JSON files without a `manifest.json` file +* a list of [Events](https://www.misp-project.org/openapi/#tag/Events) returned for a request via the [MISP Automation API](https://www.misp-project.org/openapi/) made to a MISP platform's [`/events` endpoint](https://www.misp-project.org/openapi/#tag/Events/operation/restSearchEvents) +* a list of [Attributes](https://www.misp-project.org/openapi/#tag/Attributes) returned for a request via the [MISP Automation API](https://www.misp-project.org/openapi/) made to a MISP platform's [`/attributes` endpoint](https://www.misp-project.org/openapi/#tag/Attributes/operation/restSearchAttributes) Note that only a subset of MISP [attribute types](https://www.misp-project.org/datamodels/#attribute-categories-vs-types) can be expressed with the Zeek intelligence [indicator types](https://docs.zeek.org/en/master/scripts/base/frameworks/intel/main.zeek.html#type-Intel::Type). MISP attributes with other types will be silently ignored. \ No newline at end of file diff --git a/file-monitor/scripts/extracted_files_http_server.py b/file-monitor/scripts/extracted_files_http_server.py index 48ef609e7..e1f2e0743 100755 --- a/file-monitor/scripts/extracted_files_http_server.py +++ b/file-monitor/scripts/extracted_files_http_server.py @@ -242,6 +242,7 @@ def do_GET(self): a( fileinfo, href=f'https://www.iana.org/assignments/media-types/{fileinfo}', + target="_blank", ), ) if args.magic @@ -263,6 +264,7 @@ def do_GET(self): a( fid, href=f'/arkime/idark2dash/filter?start={timestampStartFilterStr}&stop={tomorrowStr}&field=event.id&value={fid}', + target="_blank", ) for fid in fids ], @@ -297,19 +299,24 @@ def do_GET(self): ) with div(cls="col-lg-6 h-100 text-center text-lg-end my-auto").add(ul(cls="list-inline mb-0")): - li(cls="list-inline-item").add(a(href=f'/')).add(i(cls="bi bi-house fs-3", title="Malcolm")) - li(cls="list-inline-item").add(a(href=f'/readme/')).add( + li(cls="list-inline-item").add(a(href=f'/', target="_blank")).add( + i(cls="bi bi-house fs-3", title="Malcolm") + ) + li(cls="list-inline-item").add(a(href=f'/readme/', target="_blank")).add( i(cls="bi bi-question-circle fs-3", title="Documentation") ) li(cls="list-inline-item").add( - a(href=f'/dashboards/app/dashboards#/view/9ee51f94-3316-4fc5-bd89-93a52af69714') + a( + href=f'/dashboards/app/dashboards#/view/9ee51f94-3316-4fc5-bd89-93a52af69714', + target="_blank", + ) ).add(i(cls="bi bi-bar-chart-line fs-3", title="Dashboards")) - li(cls="list-inline-item").add(a(href=f'/arkime/sessions/')).add( + li(cls="list-inline-item").add(a(href=f'/arkime/sessions/', target="_blank")).add( i(cls="bi bi-table fs-3", title="Arkime") ) - li(cls="list-inline-item").add(a(href=f'https://github.com/idaholab/Malcolm/')).add( - i(cls="bi-github fs-3", title="GitHub") - ) + li(cls="list-inline-item").add( + a(href=f'https://github.com/idaholab/Malcolm/', target="_blank") + ).add(i(cls="bi-github fs-3", title="GitHub")) script(type="text/javascript", src=f"{args.assetsDirRespReplacer}js/bootstrap.bundle.min.js") script(type="text/javascript", src=f"{args.assetsDirRespReplacer}js/scripts.js") @@ -337,15 +344,14 @@ def do_GET(self): ): # serve the asset file satisfied = True - ctype = self.guess_type(fullpath) - with open(fullpath, 'rb') as fhandle: - fs = os.fstat(fhandle.fileno()) + with open(fullpath, 'rb') as f: + fs = os.fstat(f.fileno()) self.send_response(200) self.send_header('Content-type', self.guess_type(fullpath)) self.send_header("Content-Length", str(fs[6])) self.send_header("Last-Modified", self.date_time_string(fs.st_mtime)) self.end_headers() - while chunk := fhandle.read(1024): + while chunk := f.read(1024): self.wfile.write(chunk) # handle regular file downloads @@ -395,7 +401,15 @@ def do_GET(self): else: # original file, unencrypted - SimpleHTTPRequestHandler.do_GET(self) + with open(fullpath, 'rb') as f: + fs = os.fstat(f.fileno()) + self.send_response(200) + self.send_header('Content-type', self.guess_type(fullpath)) + self.send_header("Content-Length", str(fs[6])) + self.send_header("Last-Modified", self.date_time_string(fs.st_mtime)) + self.end_headers() + while chunk := f.read(1024): + self.wfile.write(chunk) else: self.send_error(404, "Not Found") diff --git a/filebeat/scripts/filebeat-process-zeek-folder.sh b/filebeat/scripts/filebeat-process-zeek-folder.sh index 825e26609..5fc0a024e 100755 --- a/filebeat/scripts/filebeat-process-zeek-folder.sh +++ b/filebeat/scripts/filebeat-process-zeek-folder.sh @@ -18,8 +18,6 @@ LOCKDIR="/tmp/zeek-beats-process-folder" export SCRIPT_DIR="$( cd -P "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -export ZEEK_LOG_FIELD_BITMAP_SCRIPT="$SCRIPT_DIR/zeek-log-field-bitmap.py" - export ZEEK_LOG_AUTO_TAG=${AUTO_TAG:-"true"} ZEEK_LOGS_DIR=${FILEBEAT_ZEEK_DIR:-/zeek/} @@ -81,13 +79,8 @@ if mkdir $LOCKDIR; then do PROCESS_TIME=$(date +%s%N) TAGS_JOINED=$(printf "%s," "${TAGS[@]}")${PROCESS_TIME} - FIELDS_BITMAP="$($ZEEK_LOG_FIELD_BITMAP_SCRIPT "$LOGFILE" | head -n 1)" LINKNAME_BASE="$(basename "$LOGFILE" .log)" - if [[ -n $FIELDS_BITMAP ]]; then - LINKNAME="${LINKNAME_BASE}(${TAGS_JOINED},${FIELDS_BITMAP}).log" - else - LINKNAME="${LINKNAME_BASE}(${TAGS_JOINED}).log" - fi + LINKNAME="${LINKNAME_BASE}(${TAGS_JOINED}).log" touch "$LOGFILE" ln -sfr "$LOGFILE" "$LINKDIR/$LINKNAME" done diff --git a/filebeat/scripts/zeek-log-field-bitmap.py b/filebeat/scripts/zeek-log-field-bitmap.py deleted file mode 100755 index 6ee960c40..000000000 --- a/filebeat/scripts/zeek-log-field-bitmap.py +++ /dev/null @@ -1,160 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - -# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. - -################################################################################################### -# parse the fields names from the header of of the log file and compare them to the -# known list of total fields. if this zeek log has is a subset of the known fields, -# create a bitmap of the included fields to be included as a special tag -# which can help the logstash parser know on a line-by-line basis which fields are included. -# when logstash-filter-dissect gets this implemented, we may not have to do this: -# - https://github.com/logstash-plugins/logstash-filter-dissect/issues/56 -# - https://github.com/logstash-plugins/logstash-filter-dissect/issues/62 -# -# arguments: accepts one argument, the name of a zeek log file -# output: returns a string suitable for use as a tag indicating the field bitset., eg., ZEEKFLDx00x01FFFFFF -# -# ZEEKFLDx00x01FFFFFF -# | └ bitmap of included fields within field list -# └ index into zeekLogFields list indicating (to support legacy field configurations, see below) -# -# example: -# $ ./zeek-log-field-bitmap.py /path/to/conn.log -# ZEEKFLDx00x01FFFFFF -# -# there are two cases we're trying to cover here by indicating the field types: -# 1. certain fields can be turned on/off in config (for example, enabling/disabling MACs or VLANs for conn.log) -# 2. a Zeek version upgrade changed the field list (see notes about DHCP.log in -# https://docs.zeek.org/en/latest/install/release-notes.html#bro-2-6) -# -# The first case is pretty simple, because in that case the fields in the zeek log will be some subset of -# the list of all known fields for that type. -# -# The second case is more complicated because the field list could be completely different. Because of this case -# each of the entries in zeekLogFields is itself a list, with older configurations occuring earlier in the list -# -# $ zeek-log-field-bitmap.py ./bro2.5/dhcp.log -# ZEEKFLDx00x000003FF -# -# $ zeek-log-field-bitmap.py ./bro2.6/dhcp.log -# ZEEKFLDx01x00007FFF -# - -import sys -import os -import json -from collections import defaultdict -from ordered_set import OrderedSet - -# lists of all known fields for each type of zeek log we're concerned with mapping (ordered as in the .log file header) -# are stored in zeek-log-fields.json -FIELDS_JSON_FILE = os.path.join(os.path.dirname(os.path.realpath(__file__)), "zeek-log-fields.json") - -ZEEK_LOG_DELIMITER = '\t' # zeek log file field delimiter -ZEEK_LOG_HEADER_LOGTYPE = 'path' # header value for zeek log type (conn, weird, etc.) -ZEEK_LOG_HEADER_FIELDS = 'fields' # header value for zeek log fields list - -# file prefix for bitmap to stdout, eg., ZEEKFLDx00x01FFFFFF -ZEEK_LOG_BITMAP_PREFIX = 'ZEEKFLD' - - -################################################################################################### -# print to stderr -def eprint(*args, **kwargs): - print(*args, file=sys.stderr, **kwargs) - - -################################################################################################### -# Set the index'th bit of v to 1 if x is truthy, else to 0, and return the new value -def set_bit(v, index, x): - mask = 1 << index # Compute mask, an integer with just bit 'index' set. - v &= ~mask # Clear the bit indicated by the mask (if x is False) - if x: - v |= mask # If x was True, set the bit indicated by the mask. - return v - - -################################################################################################### -# main -def main(): - errCode = os.EX_DATAERR - - dataError = False - zeekLogFields = defaultdict(list) - - # load from json canonical list of known zeek log fields we're concerned with mapping - zeekLogFieldsTmp = json.load(open(FIELDS_JSON_FILE, 'r')) - if isinstance(zeekLogFieldsTmp, dict): - for logType, listOfFieldLists in zeekLogFieldsTmp.items(): - if isinstance(logType, str) and isinstance(listOfFieldLists, list): - zeekLogFields[str(logType)] = [OrderedSet(fieldList) for fieldList in listOfFieldLists] - else: - dataError = True - break - else: - dataError = True - - if dataError: - # something is wrong with the json file - eprint("Error loading {} (not found or incorrectly formatted)".format(FIELDS_JSON_FILE)) - - else: - if (len(sys.argv) == 2) and os.path.isfile(sys.argv[1]): - - fieldsBitmap = 0 - - # loop over header lines in zeek log file (beginning with '#') and extract the header values - # into a dictionary containing, among other things: - # - the "path" which is the zeek log type (eg., conn, weird, etc.) - # - the "fields" list of field names - headers = {} - try: - with open(sys.argv[1], "r", encoding='utf-8') as zeekLogFile: - for line in zeekLogFile: - if line.startswith('#'): - values = line.strip().split(ZEEK_LOG_DELIMITER) - key = values.pop(0)[1:] - if len(values) == 1: - headers[key] = values[0] - else: - headers[key] = values - else: - break - except Exception as e: - eprint("{} for '{}': {}".format(type(e).__name__, sys.argv[1], e)) - - if ( - (ZEEK_LOG_HEADER_LOGTYPE in headers) - and (ZEEK_LOG_HEADER_FIELDS in headers) # the "path" header exists - and (headers[ZEEK_LOG_HEADER_LOGTYPE] in zeekLogFields) # the "fields" header exists - ): # this zeek log type is one we're concerned with mapping - - # the set of field names in *this* log file - logFieldNames = OrderedSet(headers[ZEEK_LOG_HEADER_FIELDS]) - - for versionIdx, allFieldNames in reversed( - list(enumerate(zeekLogFields[headers[ZEEK_LOG_HEADER_LOGTYPE]])) - ): - - # are this logfile's fields a subset of the complete list? - if logFieldNames.issubset(allFieldNames): - - # determine which fields in the complete list are included in this log file - for i, fName in enumerate(allFieldNames): - fieldsBitmap = set_bit(fieldsBitmap, i, fName in logFieldNames) - - # eprint(fieldsBitmap) - print('{0}x{1:02X}x{2:08X}'.format(ZEEK_LOG_BITMAP_PREFIX, versionIdx, fieldsBitmap)) - errCode = os.EX_OK - - else: - # invalid command-line arguments - eprint("{} ".format(sys.argv[0])) - errCode = os.EX_USAGE - - return errCode - - -if __name__ == '__main__': - sys.exit(main()) diff --git a/filebeat/scripts/zeek-log-fields.json b/filebeat/scripts/zeek-log-fields.json deleted file mode 100644 index 290ba4bde..000000000 --- a/filebeat/scripts/zeek-log-fields.json +++ /dev/null @@ -1,262 +0,0 @@ -{ - "conn": [ - [ - "ts", - "uid", - "id.orig_h", - "id.orig_p", - "id.resp_h", - "id.resp_p", - "proto", - "service", - "duration", - "orig_bytes", - "resp_bytes", - "conn_state", - "local_orig", - "local_resp", - "missed_bytes", - "history", - "orig_pkts", - "orig_ip_bytes", - "resp_pkts", - "resp_ip_bytes", - "tunnel_parents", - "vlan", - "inner_vlan", - "orig_l2_addr", - "resp_l2_addr", - "community_id" - ] - ], - "dhcp": [ - [ - "ts", - "uids", - "client_addr", - "server_addr", - "mac", - "host_name", - "client_fqdn", - "domain", - "requested_addr", - "assigned_addr", - "lease_time", - "client_message", - "server_message", - "msg_types", - "duration", - "client_software", - "server_software" - ] - ], - "files": [ - [ - "ts", - "fuid", - "tx_hosts", - "rx_hosts", - "conn_uids", - "source", - "depth", - "analyzers", - "mime_type", - "filename", - "duration", - "local_orig", - "is_orig", - "seen_bytes", - "total_bytes", - "missing_bytes", - "overflow_bytes", - "timedout", - "parent_fuid", - "md5", - "sha1", - "sha256", - "extracted", - "extracted_cutoff", - "extracted_size", - "ftime" - ], - [ - "ts", - "fuid", - "uid", - "id.orig_h", - "id.orig_p", - "id.resp_h", - "id.resp_p", - "source", - "depth", - "analyzers", - "mime_type", - "filename", - "duration", - "local_orig", - "is_orig", - "seen_bytes", - "total_bytes", - "missing_bytes", - "overflow_bytes", - "timedout", - "parent_fuid", - "md5", - "sha1", - "sha256", - "extracted", - "extracted_cutoff", - "extracted_size", - "ftime" - ] - ], - "http": [ - [ - "ts", - "uid", - "id.orig_h", - "id.orig_p", - "id.resp_h", - "id.resp_p", - "trans_depth", - "method", - "host", - "uri", - "referrer", - "version", - "user_agent", - "origin", - "request_body_len", - "response_body_len", - "status_code", - "status_msg", - "info_code", - "info_msg", - "tags", - "username", - "password", - "proxied", - "orig_fuids", - "orig_filenames", - "orig_mime_types", - "resp_fuids", - "resp_filenames", - "resp_mime_types", - "post_username", - "post_password_plain", - "post_password_md5", - "post_password_sha1", - "post_password_sha256" - ] - ], - "rdp": [ - [ - "ts", - "uid", - "id.orig_h", - "id.orig_p", - "id.resp_h", - "id.resp_p", - "cookie", - "result", - "security_protocol", - "client_channels", - "keyboard_layout", - "client_build", - "client_name", - "client_dig_product_id", - "desktop_width", - "desktop_height", - "requested_color_depth", - "cert_type", - "cert_count", - "cert_permanent", - "encryption_level", - "encryption_method" - ] - ], - "smb_files": [ - [ - "ts", - "uid", - "id.orig_h", - "id.orig_p", - "id.resp_h", - "id.resp_p", - "fuid", - "action", - "path", - "name", - "size", - "prev_name", - "times.modified", - "times.accessed", - "times.created", - "times.changed", - "data_offset_req", - "data_len_req", - "data_len_rsp" - ] - ], - "ssh": [ - [ - "ts", - "uid", - "id.orig_h", - "id.orig_p", - "id.resp_h", - "id.resp_p", - "version", - "auth_success", - "auth_attempts", - "direction", - "client", - "server", - "cipher_alg", - "mac_alg", - "compression_alg", - "kex_alg", - "host_key_alg", - "host_key", - "remote_location.country_code", - "remote_location.region", - "remote_location.city", - "remote_location.latitude", - "remote_location.longitude", - "hasshVersion", - "hassh", - "hasshServer", - "cshka", - "hasshAlgorithms", - "sshka", - "hasshServerAlgorithms" - ] - ], - "ssl": [ - [ - "ts", - "uid", - "id.orig_h", - "id.orig_p", - "id.resp_h", - "id.resp_p", - "version", - "cipher", - "curve", - "server_name", - "resumed", - "last_alert", - "next_protocol", - "established", - "ssl_history", - "cert_chain_fps", - "client_cert_chain_fps", - "sni_matches_cert", - "validation_status", - "ja3", - "ja3s" - ] - ] -} - - diff --git a/sensor-iso/.dockerignore b/hedgehog-iso/.dockerignore similarity index 100% rename from sensor-iso/.dockerignore rename to hedgehog-iso/.dockerignore diff --git a/sensor-iso/.gitignore b/hedgehog-iso/.gitignore similarity index 100% rename from sensor-iso/.gitignore rename to hedgehog-iso/.gitignore diff --git a/sensor-iso/Dockerfile b/hedgehog-iso/Dockerfile similarity index 95% rename from sensor-iso/Dockerfile rename to hedgehog-iso/Dockerfile index 774c62213..c7a9970b2 100644 --- a/sensor-iso/Dockerfile +++ b/hedgehog-iso/Dockerfile @@ -5,7 +5,7 @@ FROM ghcr.io/mmguero/qemu-live-iso:latest LABEL maintainer="malcolm@inl.gov" LABEL org.opencontainers.image.authors='malcolm@inl.gov' LABEL org.opencontainers.image.url='https://github.com/idaholab/Malcolm' -LABEL org.opencontainers.image.documentation='https://github.com/idaholab/Malcolm/blob/main/sensor-iso/README.md' +LABEL org.opencontainers.image.documentation='https://github.com/idaholab/Malcolm/blob/main/hedgehog-iso/README.md' LABEL org.opencontainers.image.source='https://github.com/idaholab/Malcolm' LABEL org.opencontainers.image.vendor='Idaho National Laboratory' LABEL org.opencontainers.image.title='ghcr.io/idaholab/malcolm/hedgehog' diff --git a/sensor-iso/arkime/Dockerfile b/hedgehog-iso/arkime/Dockerfile similarity index 97% rename from sensor-iso/arkime/Dockerfile rename to hedgehog-iso/arkime/Dockerfile index 53a312ff0..4ef89f0da 100644 --- a/sensor-iso/arkime/Dockerfile +++ b/hedgehog-iso/arkime/Dockerfile @@ -6,7 +6,7 @@ LABEL maintainer="malcolm@inl.gov" ENV DEBIAN_FRONTEND noninteractive -ENV ARKIME_VERSION "5.0.0" +ENV ARKIME_VERSION "5.0.1" ENV ARKIME_DIR "/opt/arkime" RUN sed -i "s/main$/main contrib non-free/g" /etc/apt/sources.list.d/debian.sources && \ diff --git a/sensor-iso/arkime/build-arkime-deb.sh b/hedgehog-iso/arkime/build-arkime-deb.sh similarity index 100% rename from sensor-iso/arkime/build-arkime-deb.sh rename to hedgehog-iso/arkime/build-arkime-deb.sh diff --git a/sensor-iso/arkime/build-docker-image.sh b/hedgehog-iso/arkime/build-docker-image.sh similarity index 100% rename from sensor-iso/arkime/build-docker-image.sh rename to hedgehog-iso/arkime/build-docker-image.sh diff --git a/sensor-iso/arkime/etc/.gitignore b/hedgehog-iso/arkime/etc/.gitignore similarity index 100% rename from sensor-iso/arkime/etc/.gitignore rename to hedgehog-iso/arkime/etc/.gitignore diff --git a/sensor-iso/build.sh b/hedgehog-iso/build.sh similarity index 100% rename from sensor-iso/build.sh rename to hedgehog-iso/build.sh diff --git a/sensor-iso/build_via_vagrant.sh b/hedgehog-iso/build_via_vagrant.sh similarity index 100% rename from sensor-iso/build_via_vagrant.sh rename to hedgehog-iso/build_via_vagrant.sh diff --git a/sensor-iso/config/archives/beats.key.binary b/hedgehog-iso/config/archives/beats.key.binary similarity index 100% rename from sensor-iso/config/archives/beats.key.binary rename to hedgehog-iso/config/archives/beats.key.binary diff --git a/sensor-iso/config/archives/beats.key.chroot b/hedgehog-iso/config/archives/beats.key.chroot similarity index 100% rename from sensor-iso/config/archives/beats.key.chroot rename to hedgehog-iso/config/archives/beats.key.chroot diff --git a/sensor-iso/config/archives/beats.list.binary b/hedgehog-iso/config/archives/beats.list.binary similarity index 100% rename from sensor-iso/config/archives/beats.list.binary rename to hedgehog-iso/config/archives/beats.list.binary diff --git a/sensor-iso/config/archives/beats.list.chroot b/hedgehog-iso/config/archives/beats.list.chroot similarity index 100% rename from sensor-iso/config/archives/beats.list.chroot rename to hedgehog-iso/config/archives/beats.list.chroot diff --git a/sensor-iso/config/archives/fluentbit.key.binary b/hedgehog-iso/config/archives/fluentbit.key.binary similarity index 100% rename from sensor-iso/config/archives/fluentbit.key.binary rename to hedgehog-iso/config/archives/fluentbit.key.binary diff --git a/sensor-iso/config/archives/fluentbit.key.chroot b/hedgehog-iso/config/archives/fluentbit.key.chroot similarity index 100% rename from sensor-iso/config/archives/fluentbit.key.chroot rename to hedgehog-iso/config/archives/fluentbit.key.chroot diff --git a/sensor-iso/config/archives/fluentbit.list.binary b/hedgehog-iso/config/archives/fluentbit.list.binary similarity index 100% rename from sensor-iso/config/archives/fluentbit.list.binary rename to hedgehog-iso/config/archives/fluentbit.list.binary diff --git a/sensor-iso/config/archives/fluentbit.list.chroot b/hedgehog-iso/config/archives/fluentbit.list.chroot similarity index 100% rename from sensor-iso/config/archives/fluentbit.list.chroot rename to hedgehog-iso/config/archives/fluentbit.list.chroot diff --git a/sensor-iso/config/archives/mozilla.key.binary b/hedgehog-iso/config/archives/mozilla.key.binary similarity index 100% rename from sensor-iso/config/archives/mozilla.key.binary rename to hedgehog-iso/config/archives/mozilla.key.binary diff --git a/sensor-iso/config/archives/mozilla.key.chroot b/hedgehog-iso/config/archives/mozilla.key.chroot similarity index 100% rename from sensor-iso/config/archives/mozilla.key.chroot rename to hedgehog-iso/config/archives/mozilla.key.chroot diff --git a/sensor-iso/config/archives/mozilla.list.binary b/hedgehog-iso/config/archives/mozilla.list.binary similarity index 100% rename from sensor-iso/config/archives/mozilla.list.binary rename to hedgehog-iso/config/archives/mozilla.list.binary diff --git a/sensor-iso/config/archives/mozilla.list.chroot b/hedgehog-iso/config/archives/mozilla.list.chroot similarity index 100% rename from sensor-iso/config/archives/mozilla.list.chroot rename to hedgehog-iso/config/archives/mozilla.list.chroot diff --git a/sensor-iso/config/archives/mozilla.pref.binary b/hedgehog-iso/config/archives/mozilla.pref.binary similarity index 100% rename from sensor-iso/config/archives/mozilla.pref.binary rename to hedgehog-iso/config/archives/mozilla.pref.binary diff --git a/sensor-iso/config/archives/mozilla.pref.chroot b/hedgehog-iso/config/archives/mozilla.pref.chroot similarity index 100% rename from sensor-iso/config/archives/mozilla.pref.chroot rename to hedgehog-iso/config/archives/mozilla.pref.chroot diff --git a/sensor-iso/config/bootloaders/extlinux/extlinux.conf b/hedgehog-iso/config/bootloaders/extlinux/extlinux.conf similarity index 100% rename from sensor-iso/config/bootloaders/extlinux/extlinux.conf rename to hedgehog-iso/config/bootloaders/extlinux/extlinux.conf diff --git a/sensor-iso/config/bootloaders/grub-legacy/menu.lst b/hedgehog-iso/config/bootloaders/grub-legacy/menu.lst similarity index 100% rename from sensor-iso/config/bootloaders/grub-legacy/menu.lst rename to hedgehog-iso/config/bootloaders/grub-legacy/menu.lst diff --git a/sensor-iso/config/bootloaders/grub-legacy/splash.xpm.gz b/hedgehog-iso/config/bootloaders/grub-legacy/splash.xpm.gz similarity index 100% rename from sensor-iso/config/bootloaders/grub-legacy/splash.xpm.gz rename to hedgehog-iso/config/bootloaders/grub-legacy/splash.xpm.gz diff --git a/sensor-iso/config/bootloaders/grub-pc/config.cfg b/hedgehog-iso/config/bootloaders/grub-pc/config.cfg similarity index 100% rename from sensor-iso/config/bootloaders/grub-pc/config.cfg rename to hedgehog-iso/config/bootloaders/grub-pc/config.cfg diff --git a/sensor-iso/config/bootloaders/grub-pc/dejavu-bold-14.pf2 b/hedgehog-iso/config/bootloaders/grub-pc/dejavu-bold-14.pf2 similarity index 100% rename from sensor-iso/config/bootloaders/grub-pc/dejavu-bold-14.pf2 rename to hedgehog-iso/config/bootloaders/grub-pc/dejavu-bold-14.pf2 diff --git a/sensor-iso/config/bootloaders/grub-pc/dejavu-bold-16.pf2 b/hedgehog-iso/config/bootloaders/grub-pc/dejavu-bold-16.pf2 similarity index 100% rename from sensor-iso/config/bootloaders/grub-pc/dejavu-bold-16.pf2 rename to hedgehog-iso/config/bootloaders/grub-pc/dejavu-bold-16.pf2 diff --git a/sensor-iso/config/bootloaders/grub-pc/grub.cfg b/hedgehog-iso/config/bootloaders/grub-pc/grub.cfg similarity index 100% rename from sensor-iso/config/bootloaders/grub-pc/grub.cfg rename to hedgehog-iso/config/bootloaders/grub-pc/grub.cfg diff --git a/sensor-iso/config/bootloaders/grub-pc/hedgehog-color.png b/hedgehog-iso/config/bootloaders/grub-pc/hedgehog-color.png similarity index 100% rename from sensor-iso/config/bootloaders/grub-pc/hedgehog-color.png rename to hedgehog-iso/config/bootloaders/grub-pc/hedgehog-color.png diff --git a/sensor-iso/config/bootloaders/grub-pc/install_start_text.cfg b/hedgehog-iso/config/bootloaders/grub-pc/install_start_text.cfg similarity index 100% rename from sensor-iso/config/bootloaders/grub-pc/install_start_text.cfg rename to hedgehog-iso/config/bootloaders/grub-pc/install_start_text.cfg diff --git a/sensor-iso/config/bootloaders/grub-pc/install_text.cfg b/hedgehog-iso/config/bootloaders/grub-pc/install_text.cfg similarity index 100% rename from sensor-iso/config/bootloaders/grub-pc/install_text.cfg rename to hedgehog-iso/config/bootloaders/grub-pc/install_text.cfg diff --git a/sensor-iso/config/bootloaders/grub-pc/live-theme/theme.txt b/hedgehog-iso/config/bootloaders/grub-pc/live-theme/theme.txt similarity index 100% rename from sensor-iso/config/bootloaders/grub-pc/live-theme/theme.txt rename to hedgehog-iso/config/bootloaders/grub-pc/live-theme/theme.txt diff --git a/sensor-iso/config/bootloaders/grub-pc/memtest.cfg b/hedgehog-iso/config/bootloaders/grub-pc/memtest.cfg similarity index 100% rename from sensor-iso/config/bootloaders/grub-pc/memtest.cfg rename to hedgehog-iso/config/bootloaders/grub-pc/memtest.cfg diff --git a/sensor-iso/config/bootloaders/grub-pc/splash.png b/hedgehog-iso/config/bootloaders/grub-pc/splash.png similarity index 100% rename from sensor-iso/config/bootloaders/grub-pc/splash.png rename to hedgehog-iso/config/bootloaders/grub-pc/splash.png diff --git a/sensor-iso/config/bootloaders/grub-pc/theme.cfg b/hedgehog-iso/config/bootloaders/grub-pc/theme.cfg similarity index 100% rename from sensor-iso/config/bootloaders/grub-pc/theme.cfg rename to hedgehog-iso/config/bootloaders/grub-pc/theme.cfg diff --git a/sensor-iso/config/bootloaders/isolinux/isolinux.cfg b/hedgehog-iso/config/bootloaders/isolinux/isolinux.cfg similarity index 100% rename from sensor-iso/config/bootloaders/isolinux/isolinux.cfg rename to hedgehog-iso/config/bootloaders/isolinux/isolinux.cfg diff --git a/sensor-iso/config/bootloaders/pxelinux/pxelinux.cfg/default b/hedgehog-iso/config/bootloaders/pxelinux/pxelinux.cfg/default similarity index 100% rename from sensor-iso/config/bootloaders/pxelinux/pxelinux.cfg/default rename to hedgehog-iso/config/bootloaders/pxelinux/pxelinux.cfg/default diff --git a/sensor-iso/config/bootloaders/syslinux/syslinux.cfg b/hedgehog-iso/config/bootloaders/syslinux/syslinux.cfg similarity index 100% rename from sensor-iso/config/bootloaders/syslinux/syslinux.cfg rename to hedgehog-iso/config/bootloaders/syslinux/syslinux.cfg diff --git a/sensor-iso/config/bootloaders/syslinux_common/install_text.cfg b/hedgehog-iso/config/bootloaders/syslinux_common/install_text.cfg similarity index 100% rename from sensor-iso/config/bootloaders/syslinux_common/install_text.cfg rename to hedgehog-iso/config/bootloaders/syslinux_common/install_text.cfg diff --git a/sensor-iso/config/bootloaders/syslinux_common/live.cfg.in b/hedgehog-iso/config/bootloaders/syslinux_common/live.cfg.in similarity index 100% rename from sensor-iso/config/bootloaders/syslinux_common/live.cfg.in rename to hedgehog-iso/config/bootloaders/syslinux_common/live.cfg.in diff --git a/sensor-iso/config/bootloaders/syslinux_common/memtest.cfg b/hedgehog-iso/config/bootloaders/syslinux_common/memtest.cfg similarity index 100% rename from sensor-iso/config/bootloaders/syslinux_common/memtest.cfg rename to hedgehog-iso/config/bootloaders/syslinux_common/memtest.cfg diff --git a/sensor-iso/config/bootloaders/syslinux_common/menu.cfg b/hedgehog-iso/config/bootloaders/syslinux_common/menu.cfg similarity index 100% rename from sensor-iso/config/bootloaders/syslinux_common/menu.cfg rename to hedgehog-iso/config/bootloaders/syslinux_common/menu.cfg diff --git a/sensor-iso/config/bootloaders/syslinux_common/splash.svg b/hedgehog-iso/config/bootloaders/syslinux_common/splash.svg similarity index 100% rename from sensor-iso/config/bootloaders/syslinux_common/splash.svg rename to hedgehog-iso/config/bootloaders/syslinux_common/splash.svg diff --git a/sensor-iso/config/bootloaders/syslinux_common/stdmenu.cfg b/hedgehog-iso/config/bootloaders/syslinux_common/stdmenu.cfg similarity index 100% rename from sensor-iso/config/bootloaders/syslinux_common/stdmenu.cfg rename to hedgehog-iso/config/bootloaders/syslinux_common/stdmenu.cfg diff --git a/sensor-iso/config/bootloaders/syslinux_common/utilities.cfg b/hedgehog-iso/config/bootloaders/syslinux_common/utilities.cfg similarity index 100% rename from sensor-iso/config/bootloaders/syslinux_common/utilities.cfg rename to hedgehog-iso/config/bootloaders/syslinux_common/utilities.cfg diff --git a/sensor-iso/config/hooks/normal/0169-pip-installs.hook.chroot b/hedgehog-iso/config/hooks/normal/0169-pip-installs.hook.chroot similarity index 100% rename from sensor-iso/config/hooks/normal/0169-pip-installs.hook.chroot rename to hedgehog-iso/config/hooks/normal/0169-pip-installs.hook.chroot diff --git a/sensor-iso/config/hooks/normal/0900-setup-rc-local.hook.chroot b/hedgehog-iso/config/hooks/normal/0900-setup-rc-local.hook.chroot similarity index 100% rename from sensor-iso/config/hooks/normal/0900-setup-rc-local.hook.chroot rename to hedgehog-iso/config/hooks/normal/0900-setup-rc-local.hook.chroot diff --git a/sensor-iso/config/hooks/normal/0910-sensor-build.hook.chroot b/hedgehog-iso/config/hooks/normal/0910-sensor-build.hook.chroot similarity index 100% rename from sensor-iso/config/hooks/normal/0910-sensor-build.hook.chroot rename to hedgehog-iso/config/hooks/normal/0910-sensor-build.hook.chroot diff --git a/sensor-iso/config/hooks/normal/0911-get-stig-scripts.hook.chroot b/hedgehog-iso/config/hooks/normal/0911-get-stig-scripts.hook.chroot similarity index 100% rename from sensor-iso/config/hooks/normal/0911-get-stig-scripts.hook.chroot rename to hedgehog-iso/config/hooks/normal/0911-get-stig-scripts.hook.chroot diff --git a/sensor-iso/config/hooks/normal/0990-remove-unwanted-pkg.hook.chroot b/hedgehog-iso/config/hooks/normal/0990-remove-unwanted-pkg.hook.chroot similarity index 100% rename from sensor-iso/config/hooks/normal/0990-remove-unwanted-pkg.hook.chroot rename to hedgehog-iso/config/hooks/normal/0990-remove-unwanted-pkg.hook.chroot diff --git a/sensor-iso/config/hooks/normal/0991-security-performance.hook.chroot b/hedgehog-iso/config/hooks/normal/0991-security-performance.hook.chroot similarity index 100% rename from sensor-iso/config/hooks/normal/0991-security-performance.hook.chroot rename to hedgehog-iso/config/hooks/normal/0991-security-performance.hook.chroot diff --git a/sensor-iso/config/hooks/normal/0992-localepurge.hook.chroot b/hedgehog-iso/config/hooks/normal/0992-localepurge.hook.chroot similarity index 100% rename from sensor-iso/config/hooks/normal/0992-localepurge.hook.chroot rename to hedgehog-iso/config/hooks/normal/0992-localepurge.hook.chroot diff --git a/sensor-iso/config/hooks/normal/0992-login.hook.chroot b/hedgehog-iso/config/hooks/normal/0992-login.hook.chroot similarity index 100% rename from sensor-iso/config/hooks/normal/0992-login.hook.chroot rename to hedgehog-iso/config/hooks/normal/0992-login.hook.chroot diff --git a/sensor-iso/config/includes.binary/install/preseed_base.cfg b/hedgehog-iso/config/includes.binary/install/preseed_base.cfg similarity index 100% rename from sensor-iso/config/includes.binary/install/preseed_base.cfg rename to hedgehog-iso/config/includes.binary/install/preseed_base.cfg diff --git a/sensor-iso/config/includes.binary/install/preseed_multipar.cfg b/hedgehog-iso/config/includes.binary/install/preseed_multipar.cfg similarity index 100% rename from sensor-iso/config/includes.binary/install/preseed_multipar.cfg rename to hedgehog-iso/config/includes.binary/install/preseed_multipar.cfg diff --git a/sensor-iso/config/includes.binary/install/preseed_vmware.cfg b/hedgehog-iso/config/includes.binary/install/preseed_vmware.cfg similarity index 100% rename from sensor-iso/config/includes.binary/install/preseed_vmware.cfg rename to hedgehog-iso/config/includes.binary/install/preseed_vmware.cfg diff --git a/sensor-iso/config/includes.binary/live/filesystem.packages-remove b/hedgehog-iso/config/includes.binary/live/filesystem.packages-remove similarity index 100% rename from sensor-iso/config/includes.binary/live/filesystem.packages-remove rename to hedgehog-iso/config/includes.binary/live/filesystem.packages-remove diff --git a/sensor-iso/config/includes.chroot/etc/audit/rules.d/audit.rules b/hedgehog-iso/config/includes.chroot/etc/audit/rules.d/audit.rules similarity index 100% rename from sensor-iso/config/includes.chroot/etc/audit/rules.d/audit.rules rename to hedgehog-iso/config/includes.chroot/etc/audit/rules.d/audit.rules diff --git a/sensor-iso/config/includes.chroot/etc/bash.bash_aliases b/hedgehog-iso/config/includes.chroot/etc/bash.bash_aliases similarity index 100% rename from sensor-iso/config/includes.chroot/etc/bash.bash_aliases rename to hedgehog-iso/config/includes.chroot/etc/bash.bash_aliases diff --git a/sensor-iso/config/includes.chroot/etc/bash.bash_functions b/hedgehog-iso/config/includes.chroot/etc/bash.bash_functions similarity index 100% rename from sensor-iso/config/includes.chroot/etc/bash.bash_functions rename to hedgehog-iso/config/includes.chroot/etc/bash.bash_functions diff --git a/sensor-iso/config/includes.chroot/etc/installer b/hedgehog-iso/config/includes.chroot/etc/installer similarity index 100% rename from sensor-iso/config/includes.chroot/etc/installer rename to hedgehog-iso/config/includes.chroot/etc/installer diff --git a/sensor-iso/config/includes.chroot/etc/lightdm/slick-greeter.conf b/hedgehog-iso/config/includes.chroot/etc/lightdm/slick-greeter.conf similarity index 100% rename from sensor-iso/config/includes.chroot/etc/lightdm/slick-greeter.conf rename to hedgehog-iso/config/includes.chroot/etc/lightdm/slick-greeter.conf diff --git a/sensor-iso/config/includes.chroot/etc/live/config.conf b/hedgehog-iso/config/includes.chroot/etc/live/config.conf similarity index 100% rename from sensor-iso/config/includes.chroot/etc/live/config.conf rename to hedgehog-iso/config/includes.chroot/etc/live/config.conf diff --git a/sensor-iso/config/includes.chroot/etc/login.defs b/hedgehog-iso/config/includes.chroot/etc/login.defs similarity index 100% rename from sensor-iso/config/includes.chroot/etc/login.defs rename to hedgehog-iso/config/includes.chroot/etc/login.defs diff --git a/sensor-iso/config/includes.chroot/etc/security/limits.d/limits.conf b/hedgehog-iso/config/includes.chroot/etc/security/limits.d/limits.conf similarity index 100% rename from sensor-iso/config/includes.chroot/etc/security/limits.d/limits.conf rename to hedgehog-iso/config/includes.chroot/etc/security/limits.d/limits.conf diff --git a/sensor-iso/config/includes.chroot/etc/skel/.bashrc b/hedgehog-iso/config/includes.chroot/etc/skel/.bashrc similarity index 100% rename from sensor-iso/config/includes.chroot/etc/skel/.bashrc rename to hedgehog-iso/config/includes.chroot/etc/skel/.bashrc diff --git a/sensor-iso/config/includes.chroot/etc/skel/.config/autostart/capture-format-wait.desktop b/hedgehog-iso/config/includes.chroot/etc/skel/.config/autostart/capture-format-wait.desktop similarity index 100% rename from sensor-iso/config/includes.chroot/etc/skel/.config/autostart/capture-format-wait.desktop rename to hedgehog-iso/config/includes.chroot/etc/skel/.config/autostart/capture-format-wait.desktop diff --git a/sensor-iso/config/includes.chroot/etc/skel/.config/autostart/dod-login-banner.desktop b/hedgehog-iso/config/includes.chroot/etc/skel/.config/autostart/dod-login-banner.desktop similarity index 100% rename from sensor-iso/config/includes.chroot/etc/skel/.config/autostart/dod-login-banner.desktop rename to hedgehog-iso/config/includes.chroot/etc/skel/.config/autostart/dod-login-banner.desktop diff --git a/sensor-iso/config/includes.chroot/etc/skel/.config/autostart/hedgehog-kiosk.desktop b/hedgehog-iso/config/includes.chroot/etc/skel/.config/autostart/hedgehog-kiosk.desktop similarity index 100% rename from sensor-iso/config/includes.chroot/etc/skel/.config/autostart/hedgehog-kiosk.desktop rename to hedgehog-iso/config/includes.chroot/etc/skel/.config/autostart/hedgehog-kiosk.desktop diff --git a/sensor-iso/config/includes.chroot/etc/skel/.config/autostart/set-dconf-screen-lock-defaults.desktop b/hedgehog-iso/config/includes.chroot/etc/skel/.config/autostart/set-dconf-screen-lock-defaults.desktop similarity index 100% rename from sensor-iso/config/includes.chroot/etc/skel/.config/autostart/set-dconf-screen-lock-defaults.desktop rename to hedgehog-iso/config/includes.chroot/etc/skel/.config/autostart/set-dconf-screen-lock-defaults.desktop diff --git a/sensor-iso/config/includes.chroot/etc/skel/.config/dconf/user b/hedgehog-iso/config/includes.chroot/etc/skel/.config/dconf/user similarity index 100% rename from sensor-iso/config/includes.chroot/etc/skel/.config/dconf/user rename to hedgehog-iso/config/includes.chroot/etc/skel/.config/dconf/user diff --git a/sensor-iso/config/includes.chroot/etc/skel/.config/gtk-3.0/bookmarks b/hedgehog-iso/config/includes.chroot/etc/skel/.config/gtk-3.0/bookmarks similarity index 100% rename from sensor-iso/config/includes.chroot/etc/skel/.config/gtk-3.0/bookmarks rename to hedgehog-iso/config/includes.chroot/etc/skel/.config/gtk-3.0/bookmarks diff --git a/sensor-iso/config/includes.chroot/etc/skel/.config/gtk-3.0/settings.ini b/hedgehog-iso/config/includes.chroot/etc/skel/.config/gtk-3.0/settings.ini similarity index 100% rename from sensor-iso/config/includes.chroot/etc/skel/.config/gtk-3.0/settings.ini rename to hedgehog-iso/config/includes.chroot/etc/skel/.config/gtk-3.0/settings.ini diff --git a/sensor-iso/config/includes.chroot/etc/skel/.config/light-locker-dconf-defaults.conf b/hedgehog-iso/config/includes.chroot/etc/skel/.config/light-locker-dconf-defaults.conf similarity index 100% rename from sensor-iso/config/includes.chroot/etc/skel/.config/light-locker-dconf-defaults.conf rename to hedgehog-iso/config/includes.chroot/etc/skel/.config/light-locker-dconf-defaults.conf diff --git a/sensor-iso/config/includes.chroot/etc/skel/.config/mimeapps.list b/hedgehog-iso/config/includes.chroot/etc/skel/.config/mimeapps.list similarity index 100% rename from sensor-iso/config/includes.chroot/etc/skel/.config/mimeapps.list rename to hedgehog-iso/config/includes.chroot/etc/skel/.config/mimeapps.list diff --git a/sensor-iso/config/includes.chroot/etc/skel/.config/xfce4/helpers.rc b/hedgehog-iso/config/includes.chroot/etc/skel/.config/xfce4/helpers.rc similarity index 100% rename from sensor-iso/config/includes.chroot/etc/skel/.config/xfce4/helpers.rc rename to hedgehog-iso/config/includes.chroot/etc/skel/.config/xfce4/helpers.rc diff --git a/sensor-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-15/16346759461.desktop b/hedgehog-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-15/16346759461.desktop similarity index 100% rename from sensor-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-15/16346759461.desktop rename to hedgehog-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-15/16346759461.desktop diff --git a/sensor-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-16/16346759562.desktop b/hedgehog-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-16/16346759562.desktop similarity index 100% rename from sensor-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-16/16346759562.desktop rename to hedgehog-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-16/16346759562.desktop diff --git a/sensor-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-17/16346759653.desktop b/hedgehog-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-17/16346759653.desktop similarity index 100% rename from sensor-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-17/16346759653.desktop rename to hedgehog-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-17/16346759653.desktop diff --git a/sensor-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-18/16346759724.desktop b/hedgehog-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-18/16346759724.desktop similarity index 100% rename from sensor-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-18/16346759724.desktop rename to hedgehog-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-18/16346759724.desktop diff --git a/sensor-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-19/16346759855.desktop b/hedgehog-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-19/16346759855.desktop similarity index 100% rename from sensor-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-19/16346759855.desktop rename to hedgehog-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-19/16346759855.desktop diff --git a/sensor-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-20/16346759956.desktop b/hedgehog-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-20/16346759956.desktop similarity index 100% rename from sensor-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-20/16346759956.desktop rename to hedgehog-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-20/16346759956.desktop diff --git a/sensor-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-21/16346760027.desktop b/hedgehog-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-21/16346760027.desktop similarity index 100% rename from sensor-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-21/16346760027.desktop rename to hedgehog-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-21/16346760027.desktop diff --git a/sensor-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-22/16346760098.desktop b/hedgehog-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-22/16346760098.desktop similarity index 100% rename from sensor-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-22/16346760098.desktop rename to hedgehog-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-22/16346760098.desktop diff --git a/sensor-iso/config/includes.chroot/etc/skel/.config/xfce4/terminal/terminalrc b/hedgehog-iso/config/includes.chroot/etc/skel/.config/xfce4/terminal/terminalrc similarity index 100% rename from sensor-iso/config/includes.chroot/etc/skel/.config/xfce4/terminal/terminalrc rename to hedgehog-iso/config/includes.chroot/etc/skel/.config/xfce4/terminal/terminalrc diff --git a/sensor-iso/config/includes.chroot/etc/skel/.config/xfce4/xfconf/xfce-perchannel-xml/xfce4-desktop.xml b/hedgehog-iso/config/includes.chroot/etc/skel/.config/xfce4/xfconf/xfce-perchannel-xml/xfce4-desktop.xml similarity index 100% rename from sensor-iso/config/includes.chroot/etc/skel/.config/xfce4/xfconf/xfce-perchannel-xml/xfce4-desktop.xml rename to hedgehog-iso/config/includes.chroot/etc/skel/.config/xfce4/xfconf/xfce-perchannel-xml/xfce4-desktop.xml diff --git a/sensor-iso/config/includes.chroot/etc/skel/.config/xfce4/xfconf/xfce-perchannel-xml/xfce4-panel.xml b/hedgehog-iso/config/includes.chroot/etc/skel/.config/xfce4/xfconf/xfce-perchannel-xml/xfce4-panel.xml similarity index 100% rename from sensor-iso/config/includes.chroot/etc/skel/.config/xfce4/xfconf/xfce-perchannel-xml/xfce4-panel.xml rename to hedgehog-iso/config/includes.chroot/etc/skel/.config/xfce4/xfconf/xfce-perchannel-xml/xfce4-panel.xml diff --git a/sensor-iso/config/includes.chroot/etc/skel/.config/xfce4/xfconf/xfce-perchannel-xml/xfce4-power-manager.xml b/hedgehog-iso/config/includes.chroot/etc/skel/.config/xfce4/xfconf/xfce-perchannel-xml/xfce4-power-manager.xml similarity index 100% rename from sensor-iso/config/includes.chroot/etc/skel/.config/xfce4/xfconf/xfce-perchannel-xml/xfce4-power-manager.xml rename to hedgehog-iso/config/includes.chroot/etc/skel/.config/xfce4/xfconf/xfce-perchannel-xml/xfce4-power-manager.xml diff --git a/sensor-iso/config/includes.chroot/etc/skel/.config/xfce4/xfconf/xfce-perchannel-xml/xfce4-session.xml b/hedgehog-iso/config/includes.chroot/etc/skel/.config/xfce4/xfconf/xfce-perchannel-xml/xfce4-session.xml similarity index 100% rename from sensor-iso/config/includes.chroot/etc/skel/.config/xfce4/xfconf/xfce-perchannel-xml/xfce4-session.xml rename to hedgehog-iso/config/includes.chroot/etc/skel/.config/xfce4/xfconf/xfce-perchannel-xml/xfce4-session.xml diff --git a/sensor-iso/config/includes.chroot/etc/skel/.config/xfce4/xfconf/xfce-perchannel-xml/xfwm4.xml b/hedgehog-iso/config/includes.chroot/etc/skel/.config/xfce4/xfconf/xfce-perchannel-xml/xfwm4.xml similarity index 100% rename from sensor-iso/config/includes.chroot/etc/skel/.config/xfce4/xfconf/xfce-perchannel-xml/xfwm4.xml rename to hedgehog-iso/config/includes.chroot/etc/skel/.config/xfce4/xfconf/xfce-perchannel-xml/xfwm4.xml diff --git a/sensor-iso/config/includes.chroot/etc/skel/.config/xfce4/xfconf/xfce-perchannel-xml/xsettings.xml b/hedgehog-iso/config/includes.chroot/etc/skel/.config/xfce4/xfconf/xfce-perchannel-xml/xsettings.xml similarity index 100% rename from sensor-iso/config/includes.chroot/etc/skel/.config/xfce4/xfconf/xfce-perchannel-xml/xsettings.xml rename to hedgehog-iso/config/includes.chroot/etc/skel/.config/xfce4/xfconf/xfce-perchannel-xml/xsettings.xml diff --git a/sensor-iso/config/includes.chroot/etc/skel/.hushlogin b/hedgehog-iso/config/includes.chroot/etc/skel/.hushlogin similarity index 100% rename from sensor-iso/config/includes.chroot/etc/skel/.hushlogin rename to hedgehog-iso/config/includes.chroot/etc/skel/.hushlogin diff --git a/sensor-iso/config/includes.chroot/etc/skel/.local/share/xfce4/helpers/custom-TerminalEmulator.desktop b/hedgehog-iso/config/includes.chroot/etc/skel/.local/share/xfce4/helpers/custom-TerminalEmulator.desktop similarity index 100% rename from sensor-iso/config/includes.chroot/etc/skel/.local/share/xfce4/helpers/custom-TerminalEmulator.desktop rename to hedgehog-iso/config/includes.chroot/etc/skel/.local/share/xfce4/helpers/custom-TerminalEmulator.desktop diff --git a/sensor-iso/config/includes.chroot/etc/skel/.selected_editor b/hedgehog-iso/config/includes.chroot/etc/skel/.selected_editor similarity index 100% rename from sensor-iso/config/includes.chroot/etc/skel/.selected_editor rename to hedgehog-iso/config/includes.chroot/etc/skel/.selected_editor diff --git a/sensor-iso/config/includes.chroot/etc/skel/.tmux.conf b/hedgehog-iso/config/includes.chroot/etc/skel/.tmux.conf similarity index 100% rename from sensor-iso/config/includes.chroot/etc/skel/.tmux.conf rename to hedgehog-iso/config/includes.chroot/etc/skel/.tmux.conf diff --git a/sensor-iso/config/includes.chroot/etc/skel/.vimrc b/hedgehog-iso/config/includes.chroot/etc/skel/.vimrc similarity index 100% rename from sensor-iso/config/includes.chroot/etc/skel/.vimrc rename to hedgehog-iso/config/includes.chroot/etc/skel/.vimrc diff --git a/sensor-iso/config/includes.chroot/etc/ssh/sshd_config b/hedgehog-iso/config/includes.chroot/etc/ssh/sshd_config similarity index 100% rename from sensor-iso/config/includes.chroot/etc/ssh/sshd_config rename to hedgehog-iso/config/includes.chroot/etc/ssh/sshd_config diff --git a/sensor-iso/config/includes.chroot/etc/sudoers.d/aide_integrity_check b/hedgehog-iso/config/includes.chroot/etc/sudoers.d/aide_integrity_check similarity index 100% rename from sensor-iso/config/includes.chroot/etc/sudoers.d/aide_integrity_check rename to hedgehog-iso/config/includes.chroot/etc/sudoers.d/aide_integrity_check diff --git a/sensor-iso/config/includes.chroot/etc/sudoers.d/nic_capture_setup b/hedgehog-iso/config/includes.chroot/etc/sudoers.d/nic_capture_setup similarity index 100% rename from sensor-iso/config/includes.chroot/etc/sudoers.d/nic_capture_setup rename to hedgehog-iso/config/includes.chroot/etc/sudoers.d/nic_capture_setup diff --git a/sensor-iso/config/includes.chroot/etc/sudoers.d/ufw_arkime_viewer b/hedgehog-iso/config/includes.chroot/etc/sudoers.d/ufw_arkime_viewer similarity index 100% rename from sensor-iso/config/includes.chroot/etc/sudoers.d/ufw_arkime_viewer rename to hedgehog-iso/config/includes.chroot/etc/sudoers.d/ufw_arkime_viewer diff --git a/sensor-iso/config/includes.chroot/etc/systemd/system/sensor-shutdown.service b/hedgehog-iso/config/includes.chroot/etc/systemd/system/sensor-shutdown.service similarity index 100% rename from sensor-iso/config/includes.chroot/etc/systemd/system/sensor-shutdown.service rename to hedgehog-iso/config/includes.chroot/etc/systemd/system/sensor-shutdown.service diff --git a/sensor-iso/config/includes.chroot/opt/zeek/share/zeek/site/extractor.zeek b/hedgehog-iso/config/includes.chroot/opt/zeek/share/zeek/site/extractor.zeek similarity index 100% rename from sensor-iso/config/includes.chroot/opt/zeek/share/zeek/site/extractor.zeek rename to hedgehog-iso/config/includes.chroot/opt/zeek/share/zeek/site/extractor.zeek diff --git a/sensor-iso/config/includes.chroot/opt/zeek/share/zeek/site/extractor_params.zeek b/hedgehog-iso/config/includes.chroot/opt/zeek/share/zeek/site/extractor_params.zeek similarity index 100% rename from sensor-iso/config/includes.chroot/opt/zeek/share/zeek/site/extractor_params.zeek rename to hedgehog-iso/config/includes.chroot/opt/zeek/share/zeek/site/extractor_params.zeek diff --git a/sensor-iso/config/includes.chroot/root/.selected_editor b/hedgehog-iso/config/includes.chroot/root/.selected_editor similarity index 100% rename from sensor-iso/config/includes.chroot/root/.selected_editor rename to hedgehog-iso/config/includes.chroot/root/.selected_editor diff --git a/sensor-iso/config/includes.chroot/root/.vimrc b/hedgehog-iso/config/includes.chroot/root/.vimrc similarity index 100% rename from sensor-iso/config/includes.chroot/root/.vimrc rename to hedgehog-iso/config/includes.chroot/root/.vimrc diff --git a/sensor-iso/config/includes.chroot/usr/local/bin/hedgehog-kiosk.sh b/hedgehog-iso/config/includes.chroot/usr/local/bin/hedgehog-kiosk.sh similarity index 100% rename from sensor-iso/config/includes.chroot/usr/local/bin/hedgehog-kiosk.sh rename to hedgehog-iso/config/includes.chroot/usr/local/bin/hedgehog-kiosk.sh diff --git a/sensor-iso/config/includes.chroot/usr/local/bin/sensor-shutdown b/hedgehog-iso/config/includes.chroot/usr/local/bin/sensor-shutdown similarity index 100% rename from sensor-iso/config/includes.chroot/usr/local/bin/sensor-shutdown rename to hedgehog-iso/config/includes.chroot/usr/local/bin/sensor-shutdown diff --git a/sensor-iso/config/includes.chroot/usr/local/etc/zeek/guess.zeek b/hedgehog-iso/config/includes.chroot/usr/local/etc/zeek/guess.zeek similarity index 100% rename from sensor-iso/config/includes.chroot/usr/local/etc/zeek/guess.zeek rename to hedgehog-iso/config/includes.chroot/usr/local/etc/zeek/guess.zeek diff --git a/sensor-iso/config/includes.chroot/usr/local/etc/zeek/guess_ics_map.txt b/hedgehog-iso/config/includes.chroot/usr/local/etc/zeek/guess_ics_map.txt similarity index 100% rename from sensor-iso/config/includes.chroot/usr/local/etc/zeek/guess_ics_map.txt rename to hedgehog-iso/config/includes.chroot/usr/local/etc/zeek/guess_ics_map.txt diff --git a/sensor-iso/config/includes.chroot/usr/local/etc/zeek/local.zeek b/hedgehog-iso/config/includes.chroot/usr/local/etc/zeek/local.zeek similarity index 82% rename from sensor-iso/config/includes.chroot/usr/local/etc/zeek/local.zeek rename to hedgehog-iso/config/includes.chroot/usr/local/etc/zeek/local.zeek index 7d3b0ccee..c1c305f94 100644 --- a/sensor-iso/config/includes.chroot/usr/local/etc/zeek/local.zeek +++ b/hedgehog-iso/config/includes.chroot/usr/local/etc/zeek/local.zeek @@ -4,46 +4,54 @@ ##! https://docs.zeek.org/en/stable/script-reference/scripts.html ##! https://github.com/zeek/zeek/blob/master/scripts/site/local.zeek -global disable_stats = (getenv("ZEEK_DISABLE_STATS") == "") ? F : T; -global disable_hash_all_files = (getenv("ZEEK_DISABLE_HASH_ALL_FILES") == "") ? F : T; -global disable_log_passwords = (getenv("ZEEK_DISABLE_LOG_PASSWORDS") == "") ? F : T; -global disable_ssl_validate_certs = (getenv("ZEEK_DISABLE_SSL_VALIDATE_CERTS") == "") ? F : T; -global disable_track_all_assets = (getenv("ZEEK_DISABLE_TRACK_ALL_ASSETS") == "") ? F : T; -global disable_best_guess_ics = (getenv("ZEEK_DISABLE_BEST_GUESS_ICS") == "") ? F : T; -global synchrophasor_detailed = (getenv("ZEEK_SYNCHROPHASOR_DETAILED") == "") ? F : T; +global true_regex: pattern = /^\s*(?i:t(rue)?|y(es)?|on|1)\s*$/; + +global disable_stats = (getenv("ZEEK_DISABLE_STATS") == true_regex) ? T : F; +global disable_hash_all_files = (getenv("ZEEK_DISABLE_HASH_ALL_FILES") == true_regex) ? T : F; +global disable_log_passwords = (getenv("ZEEK_DISABLE_LOG_PASSWORDS") == true_regex) ? T : F; +global disable_ssl_validate_certs = (getenv("ZEEK_DISABLE_SSL_VALIDATE_CERTS") == true_regex) ? T : F; +global disable_track_all_assets = (getenv("ZEEK_DISABLE_TRACK_ALL_ASSETS") == true_regex) ? T : F; +global disable_best_guess_ics = (getenv("ZEEK_DISABLE_BEST_GUESS_ICS") == true_regex) ? T : F; +global synchrophasor_detailed = (getenv("ZEEK_SYNCHROPHASOR_DETAILED") == true_regex) ? T : F; global synchrophasor_ports_str = getenv("ZEEK_SYNCHROPHASOR_PORTS"); global genisys_ports_str = getenv("ZEEK_GENISYS_PORTS"); global enip_ports_str = getenv("ZEEK_ENIP_PORTS"); global zeek_local_nets_str = getenv("ZEEK_LOCAL_NETS"); -global disable_spicy_dhcp = (getenv("ZEEK_DISABLE_SPICY_DHCP") == "") ? F : T; -global disable_spicy_dns = (getenv("ZEEK_DISABLE_SPICY_DNS") == "") ? F : T; -global disable_spicy_http = (getenv("ZEEK_DISABLE_SPICY_HTTP") == "") ? F : T; -global disable_spicy_ipsec = (getenv("ZEEK_DISABLE_SPICY_IPSEC") == "") ? F : T; -global disable_spicy_ldap = (getenv("ZEEK_DISABLE_SPICY_LDAP") == "") ? F : T; -global disable_spicy_openvpn = (getenv("ZEEK_DISABLE_SPICY_OPENVPN") == "") ? F : T; -global disable_spicy_quic = (getenv("ZEEK_DISABLE_SPICY_QUIC") == "") ? F : T; -global disable_spicy_stun = (getenv("ZEEK_DISABLE_SPICY_STUN") == "") ? F : T; -global disable_spicy_tailscale = (getenv("ZEEK_DISABLE_SPICY_TAILSCALE") == "") ? F : T; -global disable_spicy_tftp = (getenv("ZEEK_DISABLE_SPICY_TFTP") == "") ? F : T; -global disable_spicy_wireguard = (getenv("ZEEK_DISABLE_SPICY_WIREGUARD") == "") ? F : T; +global disable_spicy_dhcp = (getenv("ZEEK_DISABLE_SPICY_DHCP") == true_regex) ? T : F; +global disable_spicy_dns = (getenv("ZEEK_DISABLE_SPICY_DNS") == true_regex) ? T : F; +global disable_spicy_http = (getenv("ZEEK_DISABLE_SPICY_HTTP") == true_regex) ? T : F; +global disable_spicy_ipsec = (getenv("ZEEK_DISABLE_SPICY_IPSEC") == true_regex) ? T : F; +global disable_spicy_ldap = (getenv("ZEEK_DISABLE_SPICY_LDAP") == true_regex) ? T : F; +global disable_spicy_openvpn = (getenv("ZEEK_DISABLE_SPICY_OPENVPN") == true_regex) ? T : F; +global disable_spicy_quic = (getenv("ZEEK_DISABLE_SPICY_QUIC") == true_regex) ? T : F; +global disable_spicy_stun = (getenv("ZEEK_DISABLE_SPICY_STUN") == true_regex) ? T : F; +global disable_spicy_tailscale = (getenv("ZEEK_DISABLE_SPICY_TAILSCALE") == true_regex) ? T : F; +global disable_spicy_tftp = (getenv("ZEEK_DISABLE_SPICY_TFTP") == true_regex) ? T : F; +global disable_spicy_wireguard = (getenv("ZEEK_DISABLE_SPICY_WIREGUARD") == true_regex) ? T : F; -global disable_ics_all = (getenv("ZEEK_DISABLE_ICS_ALL") == "") ? F : T; -global disable_ics_bacnet = (getenv("ZEEK_DISABLE_ICS_BACNET") == "") ? F : T; -global disable_ics_bsap = (getenv("ZEEK_DISABLE_ICS_BSAP") == "") ? F : T; -global disable_ics_dnp3 = (getenv("ZEEK_DISABLE_ICS_DNP3") == "") ? F : T; -global disable_ics_enip = (getenv("ZEEK_DISABLE_ICS_ENIP") == "") ? F : T; -global disable_ics_ethercat = (getenv("ZEEK_DISABLE_ICS_ETHERCAT") == "") ? F : T; -global disable_ics_genisys = (getenv("ZEEK_DISABLE_ICS_GENISYS") == "") ? F : T; -global disable_ics_opcua_binary = (getenv("ZEEK_DISABLE_ICS_OPCUA_BINARY") == "") ? F : T; -global disable_ics_modbus = (getenv("ZEEK_DISABLE_ICS_MODBUS") == "") ? F : T; -global disable_ics_profinet = (getenv("ZEEK_DISABLE_ICS_PROFINET") == "") ? F : T; -global disable_ics_s7comm = (getenv("ZEEK_DISABLE_ICS_S7COMM") == "") ? F : T; -global disable_ics_synchrophasor = (getenv("ZEEK_DISABLE_ICS_SYNCHROPHASOR") == "") ? F : T; +global disable_ics_all = (getenv("ZEEK_DISABLE_ICS_ALL") == true_regex) ? T : F; +global disable_ics_bacnet = (getenv("ZEEK_DISABLE_ICS_BACNET") == true_regex) ? T : F; +global disable_ics_bsap = (getenv("ZEEK_DISABLE_ICS_BSAP") == true_regex) ? T : F; +global disable_ics_dnp3 = (getenv("ZEEK_DISABLE_ICS_DNP3") == true_regex) ? T : F; +global disable_ics_enip = (getenv("ZEEK_DISABLE_ICS_ENIP") == true_regex) ? T : F; +global disable_ics_ethercat = (getenv("ZEEK_DISABLE_ICS_ETHERCAT") == true_regex) ? T : F; +global disable_ics_genisys = (getenv("ZEEK_DISABLE_ICS_GENISYS") == true_regex) ? T : F; +global disable_ics_opcua_binary = (getenv("ZEEK_DISABLE_ICS_OPCUA_BINARY") == true_regex) ? T : F; +global disable_ics_modbus = (getenv("ZEEK_DISABLE_ICS_MODBUS") == true_regex) ? T : F; +global disable_ics_profinet = (getenv("ZEEK_DISABLE_ICS_PROFINET") == true_regex) ? T : F; +global disable_ics_profinet_io_cm = (getenv("ZEEK_DISABLE_ICS_PROFINET_IO_CM") == true_regex) ? T : F; +global disable_ics_s7comm = (getenv("ZEEK_DISABLE_ICS_S7COMM") == true_regex) ? T : F; +global disable_ics_synchrophasor = (getenv("ZEEK_DISABLE_ICS_SYNCHROPHASOR") == true_regex) ? T : F; redef Broker::default_listen_address = "127.0.0.1"; redef ignore_checksums = T; +global json_format = (getenv("ZEEK_JSON") == true_regex) ? T : F; +@if (json_format) + redef LogAscii::use_json = T; +@endif + @load tuning/defaults @load frameworks/software/vulnerable @load frameworks/software/version-changes @@ -139,6 +147,9 @@ event zeek_init() &priority=-5 { if (disable_ics_all || disable_ics_profinet) { Analyzer::disable_analyzer(Analyzer::ANALYZER_PROFINET); } + if (disable_ics_all || disable_ics_profinet_io_cm) { + Spicy::disable_protocol_analyzer(Analyzer::ANALYZER_SPICY_PROFINET_IO_CM); + } if (disable_ics_all || disable_ics_s7comm) { Analyzer::disable_analyzer(Analyzer::ANALYZER_S7COMM_TCP); } @@ -290,3 +301,36 @@ redef CVE_2021_44228::log = F; break; } @endif + +##! Other logs we're just disabling unilaterally + +# amzn/zeek-plugin-profinet's profinet_dce_rpc.log is covered by cisagov/icsnpp-profinet-io-cm +hook Profinet::log_policy_profinet_dce_rpc( + rec: Profinet::Profinet_DCE_RPC, + id: Log::ID, + filter: Log::Filter) { + break; +} + +# we're not tracking the BSAP "unknown" logs +hook Bsap::log_policy_bsap_ip_unknown( + rec: Bsap::BSAP_IP_UNKNOWN, + id: Log::ID, + filter: Log::Filter) { + break; +} + +hook Bsap::log_policy_bsap_serial_unknown( + rec: Bsap::BSAP_SERIAL_UNKNOWN, + id: Log::ID, + filter: Log::Filter) { + break; +} + +# all ARP traffic is logged by the ethercat parser, which is overkill +hook PacketAnalyzer::ECAT::log_policy_ecat_arp( + rec: PacketAnalyzer::ECAT::ECAT_ARP_INFO, + id: Log::ID, + filter: Log::Filter) { + break; +} diff --git a/sensor-iso/config/includes.chroot/usr/local/etc/zeek/login.zeek b/hedgehog-iso/config/includes.chroot/usr/local/etc/zeek/login.zeek similarity index 100% rename from sensor-iso/config/includes.chroot/usr/local/etc/zeek/login.zeek rename to hedgehog-iso/config/includes.chroot/usr/local/etc/zeek/login.zeek diff --git a/sensor-iso/config/includes.chroot/usr/share/applications/configure-capture.desktop b/hedgehog-iso/config/includes.chroot/usr/share/applications/configure-capture.desktop similarity index 100% rename from sensor-iso/config/includes.chroot/usr/share/applications/configure-capture.desktop rename to hedgehog-iso/config/includes.chroot/usr/share/applications/configure-capture.desktop diff --git a/sensor-iso/config/includes.chroot/usr/share/applications/configure-interfaces.desktop b/hedgehog-iso/config/includes.chroot/usr/share/applications/configure-interfaces.desktop similarity index 100% rename from sensor-iso/config/includes.chroot/usr/share/applications/configure-interfaces.desktop rename to hedgehog-iso/config/includes.chroot/usr/share/applications/configure-interfaces.desktop diff --git a/sensor-iso/config/includes.chroot/usr/share/applications/hedgehog-kiosk.desktop b/hedgehog-iso/config/includes.chroot/usr/share/applications/hedgehog-kiosk.desktop similarity index 100% rename from sensor-iso/config/includes.chroot/usr/share/applications/hedgehog-kiosk.desktop rename to hedgehog-iso/config/includes.chroot/usr/share/applications/hedgehog-kiosk.desktop diff --git a/sensor-iso/config/includes.chroot/usr/share/applications/hedgehog-readme.desktop b/hedgehog-iso/config/includes.chroot/usr/share/applications/hedgehog-readme.desktop similarity index 100% rename from sensor-iso/config/includes.chroot/usr/share/applications/hedgehog-readme.desktop rename to hedgehog-iso/config/includes.chroot/usr/share/applications/hedgehog-readme.desktop diff --git a/sensor-iso/config/includes.chroot/usr/share/applications/sensor-services-full-restart.desktop b/hedgehog-iso/config/includes.chroot/usr/share/applications/sensor-services-full-restart.desktop similarity index 100% rename from sensor-iso/config/includes.chroot/usr/share/applications/sensor-services-full-restart.desktop rename to hedgehog-iso/config/includes.chroot/usr/share/applications/sensor-services-full-restart.desktop diff --git a/sensor-iso/config/includes.chroot/usr/share/applications/sensor-services-status.desktop b/hedgehog-iso/config/includes.chroot/usr/share/applications/sensor-services-status.desktop similarity index 100% rename from sensor-iso/config/includes.chroot/usr/share/applications/sensor-services-status.desktop rename to hedgehog-iso/config/includes.chroot/usr/share/applications/sensor-services-status.desktop diff --git a/sensor-iso/config/package-lists/build.list.chroot b/hedgehog-iso/config/package-lists/build.list.chroot similarity index 100% rename from sensor-iso/config/package-lists/build.list.chroot rename to hedgehog-iso/config/package-lists/build.list.chroot diff --git a/sensor-iso/config/package-lists/desktopmanager.list.chroot b/hedgehog-iso/config/package-lists/desktopmanager.list.chroot similarity index 100% rename from sensor-iso/config/package-lists/desktopmanager.list.chroot rename to hedgehog-iso/config/package-lists/desktopmanager.list.chroot diff --git a/sensor-iso/config/package-lists/grub.list.binary b/hedgehog-iso/config/package-lists/grub.list.binary similarity index 100% rename from sensor-iso/config/package-lists/grub.list.binary rename to hedgehog-iso/config/package-lists/grub.list.binary diff --git a/sensor-iso/config/package-lists/live.list.chroot b/hedgehog-iso/config/package-lists/live.list.chroot similarity index 100% rename from sensor-iso/config/package-lists/live.list.chroot rename to hedgehog-iso/config/package-lists/live.list.chroot diff --git a/sensor-iso/config/package-lists/net.list.chroot b/hedgehog-iso/config/package-lists/net.list.chroot similarity index 100% rename from sensor-iso/config/package-lists/net.list.chroot rename to hedgehog-iso/config/package-lists/net.list.chroot diff --git a/sensor-iso/config/package-lists/python.list.chroot b/hedgehog-iso/config/package-lists/python.list.chroot similarity index 100% rename from sensor-iso/config/package-lists/python.list.chroot rename to hedgehog-iso/config/package-lists/python.list.chroot diff --git a/sensor-iso/config/package-lists/sensor.list.chroot b/hedgehog-iso/config/package-lists/sensor.list.chroot similarity index 100% rename from sensor-iso/config/package-lists/sensor.list.chroot rename to hedgehog-iso/config/package-lists/sensor.list.chroot diff --git a/sensor-iso/config/package-lists/system.list.chroot b/hedgehog-iso/config/package-lists/system.list.chroot similarity index 100% rename from sensor-iso/config/package-lists/system.list.chroot rename to hedgehog-iso/config/package-lists/system.list.chroot diff --git a/sensor-iso/config/package-lists/virtualguest.list.chroot b/hedgehog-iso/config/package-lists/virtualguest.list.chroot similarity index 100% rename from sensor-iso/config/package-lists/virtualguest.list.chroot rename to hedgehog-iso/config/package-lists/virtualguest.list.chroot diff --git a/sensor-iso/interface/.gitignore b/hedgehog-iso/interface/.gitignore similarity index 100% rename from sensor-iso/interface/.gitignore rename to hedgehog-iso/interface/.gitignore diff --git a/sensor-iso/interface/init.sh b/hedgehog-iso/interface/init.sh similarity index 100% rename from sensor-iso/interface/init.sh rename to hedgehog-iso/interface/init.sh diff --git a/sensor-iso/interface/kiosk.service b/hedgehog-iso/interface/kiosk.service similarity index 100% rename from sensor-iso/interface/kiosk.service rename to hedgehog-iso/interface/kiosk.service diff --git a/sensor-iso/interface/requirements.txt b/hedgehog-iso/interface/requirements.txt similarity index 100% rename from sensor-iso/interface/requirements.txt rename to hedgehog-iso/interface/requirements.txt diff --git a/sensor-iso/interface/sensor_ctl/arkime/config.ini b/hedgehog-iso/interface/sensor_ctl/arkime/config.ini similarity index 100% rename from sensor-iso/interface/sensor_ctl/arkime/config.ini rename to hedgehog-iso/interface/sensor_ctl/arkime/config.ini diff --git a/sensor-iso/interface/sensor_ctl/arkime/rules.yml b/hedgehog-iso/interface/sensor_ctl/arkime/rules.yml similarity index 100% rename from sensor-iso/interface/sensor_ctl/arkime/rules.yml rename to hedgehog-iso/interface/sensor_ctl/arkime/rules.yml diff --git a/sensor-iso/interface/sensor_ctl/clean b/hedgehog-iso/interface/sensor_ctl/clean similarity index 100% rename from sensor-iso/interface/sensor_ctl/clean rename to hedgehog-iso/interface/sensor_ctl/clean diff --git a/sensor-iso/interface/sensor_ctl/clean.sh b/hedgehog-iso/interface/sensor_ctl/clean.sh similarity index 100% rename from sensor-iso/interface/sensor_ctl/clean.sh rename to hedgehog-iso/interface/sensor_ctl/clean.sh diff --git a/sensor-iso/interface/sensor_ctl/control.sh b/hedgehog-iso/interface/sensor_ctl/control.sh similarity index 100% rename from sensor-iso/interface/sensor_ctl/control.sh rename to hedgehog-iso/interface/sensor_ctl/control.sh diff --git a/sensor-iso/interface/sensor_ctl/control_vars.conf b/hedgehog-iso/interface/sensor_ctl/control_vars.conf similarity index 98% rename from sensor-iso/interface/sensor_ctl/control_vars.conf rename to hedgehog-iso/interface/sensor_ctl/control_vars.conf index 956d6daa5..56d4b8985 100644 --- a/sensor-iso/interface/sensor_ctl/control_vars.conf +++ b/hedgehog-iso/interface/sensor_ctl/control_vars.conf @@ -47,6 +47,7 @@ export ZEEK_LB_METHOD=custom export ZEEK_AF_PACKET_BUFFER_SIZE=67108864 export ZEEK_LOCAL_NETS= +export ZEEK_JSON= export ZEEK_RULESET=local export ZEEK_INTEL_ITEM_EXPIRATION=-1min export ZEEK_INTEL_FEED_SINCE= @@ -81,6 +82,7 @@ export ZEEK_DISABLE_ICS_GENISYS=true export ZEEK_DISABLE_ICS_OPCUA_BINARY= export ZEEK_DISABLE_ICS_MODBUS= export ZEEK_DISABLE_ICS_PROFINET= +export ZEEK_DISABLE_ICS_PROFINET_IO_CM= export ZEEK_DISABLE_ICS_S7COMM= export ZEEK_DISABLE_ICS_SYNCHROPHASOR= export ZEEK_SYNCHROPHASOR_PORTS= diff --git a/sensor-iso/interface/sensor_ctl/filebeat/filebeat.yml b/hedgehog-iso/interface/sensor_ctl/filebeat/filebeat.yml similarity index 100% rename from sensor-iso/interface/sensor_ctl/filebeat/filebeat.yml rename to hedgehog-iso/interface/sensor_ctl/filebeat/filebeat.yml diff --git a/sensor-iso/interface/sensor_ctl/filebeat/sensor_filebeat_local.sh b/hedgehog-iso/interface/sensor_ctl/filebeat/sensor_filebeat_local.sh similarity index 100% rename from sensor-iso/interface/sensor_ctl/filebeat/sensor_filebeat_local.sh rename to hedgehog-iso/interface/sensor_ctl/filebeat/sensor_filebeat_local.sh diff --git a/sensor-iso/interface/sensor_ctl/logstash-client-certificates/.gitignore b/hedgehog-iso/interface/sensor_ctl/logstash-client-certificates/.gitignore similarity index 100% rename from sensor-iso/interface/sensor_ctl/logstash-client-certificates/.gitignore rename to hedgehog-iso/interface/sensor_ctl/logstash-client-certificates/.gitignore diff --git a/sensor-iso/interface/sensor_ctl/miscbeat/filebeat.yml b/hedgehog-iso/interface/sensor_ctl/miscbeat/filebeat.yml similarity index 100% rename from sensor-iso/interface/sensor_ctl/miscbeat/filebeat.yml rename to hedgehog-iso/interface/sensor_ctl/miscbeat/filebeat.yml diff --git a/sensor-iso/interface/sensor_ctl/miscbeat/sensor_miscbeat_local.sh b/hedgehog-iso/interface/sensor_ctl/miscbeat/sensor_miscbeat_local.sh similarity index 100% rename from sensor-iso/interface/sensor_ctl/miscbeat/sensor_miscbeat_local.sh rename to hedgehog-iso/interface/sensor_ctl/miscbeat/sensor_miscbeat_local.sh diff --git a/sensor-iso/interface/sensor_ctl/reload b/hedgehog-iso/interface/sensor_ctl/reload similarity index 100% rename from sensor-iso/interface/sensor_ctl/reload rename to hedgehog-iso/interface/sensor_ctl/reload diff --git a/sensor-iso/interface/sensor_ctl/restart b/hedgehog-iso/interface/sensor_ctl/restart similarity index 100% rename from sensor-iso/interface/sensor_ctl/restart rename to hedgehog-iso/interface/sensor_ctl/restart diff --git a/sensor-iso/interface/sensor_ctl/scripts/log_disk_space.sh b/hedgehog-iso/interface/sensor_ctl/scripts/log_disk_space.sh similarity index 100% rename from sensor-iso/interface/sensor_ctl/scripts/log_disk_space.sh rename to hedgehog-iso/interface/sensor_ctl/scripts/log_disk_space.sh diff --git a/sensor-iso/interface/sensor_ctl/shutdown b/hedgehog-iso/interface/sensor_ctl/shutdown similarity index 100% rename from sensor-iso/interface/sensor_ctl/shutdown rename to hedgehog-iso/interface/sensor_ctl/shutdown diff --git a/sensor-iso/interface/sensor_ctl/start b/hedgehog-iso/interface/sensor_ctl/start similarity index 100% rename from sensor-iso/interface/sensor_ctl/start rename to hedgehog-iso/interface/sensor_ctl/start diff --git a/sensor-iso/interface/sensor_ctl/status b/hedgehog-iso/interface/sensor_ctl/status similarity index 100% rename from sensor-iso/interface/sensor_ctl/status rename to hedgehog-iso/interface/sensor_ctl/status diff --git a/sensor-iso/interface/sensor_ctl/stop b/hedgehog-iso/interface/sensor_ctl/stop similarity index 100% rename from sensor-iso/interface/sensor_ctl/stop rename to hedgehog-iso/interface/sensor_ctl/stop diff --git a/sensor-iso/interface/sensor_ctl/supervisor.d/arkime.conf b/hedgehog-iso/interface/sensor_ctl/supervisor.d/arkime.conf similarity index 100% rename from sensor-iso/interface/sensor_ctl/supervisor.d/arkime.conf rename to hedgehog-iso/interface/sensor_ctl/supervisor.d/arkime.conf diff --git a/sensor-iso/interface/sensor_ctl/supervisor.d/beats.conf b/hedgehog-iso/interface/sensor_ctl/supervisor.d/beats.conf similarity index 100% rename from sensor-iso/interface/sensor_ctl/supervisor.d/beats.conf rename to hedgehog-iso/interface/sensor_ctl/supervisor.d/beats.conf diff --git a/sensor-iso/interface/sensor_ctl/supervisor.d/clamav.conf b/hedgehog-iso/interface/sensor_ctl/supervisor.d/clamav.conf similarity index 100% rename from sensor-iso/interface/sensor_ctl/supervisor.d/clamav.conf rename to hedgehog-iso/interface/sensor_ctl/supervisor.d/clamav.conf diff --git a/sensor-iso/interface/sensor_ctl/supervisor.d/documentation.conf b/hedgehog-iso/interface/sensor_ctl/supervisor.d/documentation.conf similarity index 100% rename from sensor-iso/interface/sensor_ctl/supervisor.d/documentation.conf rename to hedgehog-iso/interface/sensor_ctl/supervisor.d/documentation.conf diff --git a/sensor-iso/interface/sensor_ctl/supervisor.d/fluentbit-aide.conf b/hedgehog-iso/interface/sensor_ctl/supervisor.d/fluentbit-aide.conf similarity index 100% rename from sensor-iso/interface/sensor_ctl/supervisor.d/fluentbit-aide.conf rename to hedgehog-iso/interface/sensor_ctl/supervisor.d/fluentbit-aide.conf diff --git a/sensor-iso/interface/sensor_ctl/supervisor.d/fluentbit-auditlog.conf b/hedgehog-iso/interface/sensor_ctl/supervisor.d/fluentbit-auditlog.conf similarity index 100% rename from sensor-iso/interface/sensor_ctl/supervisor.d/fluentbit-auditlog.conf rename to hedgehog-iso/interface/sensor_ctl/supervisor.d/fluentbit-auditlog.conf diff --git a/sensor-iso/interface/sensor_ctl/supervisor.d/fluentbit-kmsg.conf b/hedgehog-iso/interface/sensor_ctl/supervisor.d/fluentbit-kmsg.conf similarity index 100% rename from sensor-iso/interface/sensor_ctl/supervisor.d/fluentbit-kmsg.conf rename to hedgehog-iso/interface/sensor_ctl/supervisor.d/fluentbit-kmsg.conf diff --git a/sensor-iso/interface/sensor_ctl/supervisor.d/fluentbit-metrics.conf b/hedgehog-iso/interface/sensor_ctl/supervisor.d/fluentbit-metrics.conf similarity index 100% rename from sensor-iso/interface/sensor_ctl/supervisor.d/fluentbit-metrics.conf rename to hedgehog-iso/interface/sensor_ctl/supervisor.d/fluentbit-metrics.conf diff --git a/sensor-iso/interface/sensor_ctl/supervisor.d/fluentbit-systemd.conf b/hedgehog-iso/interface/sensor_ctl/supervisor.d/fluentbit-systemd.conf similarity index 100% rename from sensor-iso/interface/sensor_ctl/supervisor.d/fluentbit-systemd.conf rename to hedgehog-iso/interface/sensor_ctl/supervisor.d/fluentbit-systemd.conf diff --git a/sensor-iso/interface/sensor_ctl/supervisor.d/fluentbit-thermal.conf b/hedgehog-iso/interface/sensor_ctl/supervisor.d/fluentbit-thermal.conf similarity index 100% rename from sensor-iso/interface/sensor_ctl/supervisor.d/fluentbit-thermal.conf rename to hedgehog-iso/interface/sensor_ctl/supervisor.d/fluentbit-thermal.conf diff --git a/sensor-iso/interface/sensor_ctl/supervisor.d/netsniff.template b/hedgehog-iso/interface/sensor_ctl/supervisor.d/netsniff.template similarity index 100% rename from sensor-iso/interface/sensor_ctl/supervisor.d/netsniff.template rename to hedgehog-iso/interface/sensor_ctl/supervisor.d/netsniff.template diff --git a/sensor-iso/interface/sensor_ctl/supervisor.d/prune.conf b/hedgehog-iso/interface/sensor_ctl/supervisor.d/prune.conf similarity index 100% rename from sensor-iso/interface/sensor_ctl/supervisor.d/prune.conf rename to hedgehog-iso/interface/sensor_ctl/supervisor.d/prune.conf diff --git a/sensor-iso/interface/sensor_ctl/supervisor.d/supercronic.conf b/hedgehog-iso/interface/sensor_ctl/supervisor.d/supercronic.conf similarity index 100% rename from sensor-iso/interface/sensor_ctl/supervisor.d/supercronic.conf rename to hedgehog-iso/interface/sensor_ctl/supervisor.d/supercronic.conf diff --git a/sensor-iso/interface/sensor_ctl/supervisor.d/suricata.conf b/hedgehog-iso/interface/sensor_ctl/supervisor.d/suricata.conf similarity index 100% rename from sensor-iso/interface/sensor_ctl/supervisor.d/suricata.conf rename to hedgehog-iso/interface/sensor_ctl/supervisor.d/suricata.conf diff --git a/sensor-iso/interface/sensor_ctl/supervisor.d/tcpdump.template b/hedgehog-iso/interface/sensor_ctl/supervisor.d/tcpdump.template similarity index 100% rename from sensor-iso/interface/sensor_ctl/supervisor.d/tcpdump.template rename to hedgehog-iso/interface/sensor_ctl/supervisor.d/tcpdump.template diff --git a/sensor-iso/interface/sensor_ctl/supervisor.d/zeek.conf b/hedgehog-iso/interface/sensor_ctl/supervisor.d/zeek.conf similarity index 100% rename from sensor-iso/interface/sensor_ctl/supervisor.d/zeek.conf rename to hedgehog-iso/interface/sensor_ctl/supervisor.d/zeek.conf diff --git a/sensor-iso/interface/sensor_ctl/supervisor.init/arkime_config_populate.sh b/hedgehog-iso/interface/sensor_ctl/supervisor.init/arkime_config_populate.sh similarity index 100% rename from sensor-iso/interface/sensor_ctl/supervisor.init/arkime_config_populate.sh rename to hedgehog-iso/interface/sensor_ctl/supervisor.init/arkime_config_populate.sh diff --git a/sensor-iso/interface/sensor_ctl/supervisor.init/supercronic_populate.sh b/hedgehog-iso/interface/sensor_ctl/supervisor.init/supercronic_populate.sh similarity index 100% rename from sensor-iso/interface/sensor_ctl/supervisor.init/supercronic_populate.sh rename to hedgehog-iso/interface/sensor_ctl/supervisor.init/supercronic_populate.sh diff --git a/sensor-iso/interface/sensor_ctl/supervisor.init/suricata_config_populate.sh b/hedgehog-iso/interface/sensor_ctl/supervisor.init/suricata_config_populate.sh similarity index 100% rename from sensor-iso/interface/sensor_ctl/supervisor.init/suricata_config_populate.sh rename to hedgehog-iso/interface/sensor_ctl/supervisor.init/suricata_config_populate.sh diff --git a/sensor-iso/interface/sensor_ctl/supervisor.sh b/hedgehog-iso/interface/sensor_ctl/supervisor.sh similarity index 100% rename from sensor-iso/interface/sensor_ctl/supervisor.sh rename to hedgehog-iso/interface/sensor_ctl/supervisor.sh diff --git a/sensor-iso/interface/sensor_ctl/supervisord.conf b/hedgehog-iso/interface/sensor_ctl/supervisord.conf similarity index 100% rename from sensor-iso/interface/sensor_ctl/supervisord.conf rename to hedgehog-iso/interface/sensor_ctl/supervisord.conf diff --git a/sensor-iso/interface/sensor_ctl/zeek/extractor_override.interesting.zeek b/hedgehog-iso/interface/sensor_ctl/zeek/extractor_override.interesting.zeek similarity index 100% rename from sensor-iso/interface/sensor_ctl/zeek/extractor_override.interesting.zeek rename to hedgehog-iso/interface/sensor_ctl/zeek/extractor_override.interesting.zeek diff --git a/sensor-iso/interface/sensor_ctl/zeek/intel/.gitignore b/hedgehog-iso/interface/sensor_ctl/zeek/intel/.gitignore similarity index 100% rename from sensor-iso/interface/sensor_ctl/zeek/intel/.gitignore rename to hedgehog-iso/interface/sensor_ctl/zeek/intel/.gitignore diff --git a/sensor-iso/interface/sensor_interface/__init__.py b/hedgehog-iso/interface/sensor_interface/__init__.py similarity index 100% rename from sensor-iso/interface/sensor_interface/__init__.py rename to hedgehog-iso/interface/sensor_interface/__init__.py diff --git a/sensor-iso/interface/sensor_interface/__main__.py b/hedgehog-iso/interface/sensor_interface/__main__.py similarity index 100% rename from sensor-iso/interface/sensor_interface/__main__.py rename to hedgehog-iso/interface/sensor_interface/__main__.py diff --git a/sensor-iso/interface/sensor_interface/routes.py b/hedgehog-iso/interface/sensor_interface/routes.py similarity index 100% rename from sensor-iso/interface/sensor_interface/routes.py rename to hedgehog-iso/interface/sensor_interface/routes.py diff --git a/sensor-iso/interface/sensor_interface/static/css/custom.css b/hedgehog-iso/interface/sensor_interface/static/css/custom.css similarity index 100% rename from sensor-iso/interface/sensor_interface/static/css/custom.css rename to hedgehog-iso/interface/sensor_interface/static/css/custom.css diff --git a/sensor-iso/interface/sensor_interface/static/css/freeboard.min.css b/hedgehog-iso/interface/sensor_interface/static/css/freeboard.min.css similarity index 100% rename from sensor-iso/interface/sensor_interface/static/css/freeboard.min.css rename to hedgehog-iso/interface/sensor_interface/static/css/freeboard.min.css diff --git a/sensor-iso/interface/sensor_interface/static/css/material.css b/hedgehog-iso/interface/sensor_interface/static/css/material.css similarity index 100% rename from sensor-iso/interface/sensor_interface/static/css/material.css rename to hedgehog-iso/interface/sensor_interface/static/css/material.css diff --git a/sensor-iso/interface/sensor_interface/static/dashboard/dashboard.json b/hedgehog-iso/interface/sensor_interface/static/dashboard/dashboard.json similarity index 100% rename from sensor-iso/interface/sensor_interface/static/dashboard/dashboard.json rename to hedgehog-iso/interface/sensor_interface/static/dashboard/dashboard.json diff --git a/sensor-iso/interface/sensor_interface/static/icons/icons.woff2 b/hedgehog-iso/interface/sensor_interface/static/icons/icons.woff2 similarity index 100% rename from sensor-iso/interface/sensor_interface/static/icons/icons.woff2 rename to hedgehog-iso/interface/sensor_interface/static/icons/icons.woff2 diff --git a/sensor-iso/interface/sensor_interface/static/img/dropdown-arrow.png b/hedgehog-iso/interface/sensor_interface/static/img/dropdown-arrow.png similarity index 100% rename from sensor-iso/interface/sensor_interface/static/img/dropdown-arrow.png rename to hedgehog-iso/interface/sensor_interface/static/img/dropdown-arrow.png diff --git a/sensor-iso/interface/sensor_interface/static/img/glyphicons-blackboard.png b/hedgehog-iso/interface/sensor_interface/static/img/glyphicons-blackboard.png similarity index 100% rename from sensor-iso/interface/sensor_interface/static/img/glyphicons-blackboard.png rename to hedgehog-iso/interface/sensor_interface/static/img/glyphicons-blackboard.png diff --git a/sensor-iso/interface/sensor_interface/static/img/glyphicons-halflings-white.png b/hedgehog-iso/interface/sensor_interface/static/img/glyphicons-halflings-white.png similarity index 100% rename from sensor-iso/interface/sensor_interface/static/img/glyphicons-halflings-white.png rename to hedgehog-iso/interface/sensor_interface/static/img/glyphicons-halflings-white.png diff --git a/sensor-iso/interface/sensor_interface/static/img/glyphicons-halflings.png b/hedgehog-iso/interface/sensor_interface/static/img/glyphicons-halflings.png similarity index 100% rename from sensor-iso/interface/sensor_interface/static/img/glyphicons-halflings.png rename to hedgehog-iso/interface/sensor_interface/static/img/glyphicons-halflings.png diff --git a/sensor-iso/interface/sensor_interface/static/img/glyphicons-log-in.png b/hedgehog-iso/interface/sensor_interface/static/img/glyphicons-log-in.png similarity index 100% rename from sensor-iso/interface/sensor_interface/static/img/glyphicons-log-in.png rename to hedgehog-iso/interface/sensor_interface/static/img/glyphicons-log-in.png diff --git a/sensor-iso/interface/sensor_interface/static/img/glyphicons-log-out.png b/hedgehog-iso/interface/sensor_interface/static/img/glyphicons-log-out.png similarity index 100% rename from sensor-iso/interface/sensor_interface/static/img/glyphicons-log-out.png rename to hedgehog-iso/interface/sensor_interface/static/img/glyphicons-log-out.png diff --git a/sensor-iso/interface/sensor_interface/static/js/custom.js b/hedgehog-iso/interface/sensor_interface/static/js/custom.js similarity index 100% rename from sensor-iso/interface/sensor_interface/static/js/custom.js rename to hedgehog-iso/interface/sensor_interface/static/js/custom.js diff --git a/sensor-iso/interface/sensor_interface/static/js/freeboard.min.js b/hedgehog-iso/interface/sensor_interface/static/js/freeboard.min.js similarity index 100% rename from sensor-iso/interface/sensor_interface/static/js/freeboard.min.js rename to hedgehog-iso/interface/sensor_interface/static/js/freeboard.min.js diff --git a/sensor-iso/interface/sensor_interface/static/js/freeboard.plugins.min.js b/hedgehog-iso/interface/sensor_interface/static/js/freeboard.plugins.min.js similarity index 100% rename from sensor-iso/interface/sensor_interface/static/js/freeboard.plugins.min.js rename to hedgehog-iso/interface/sensor_interface/static/js/freeboard.plugins.min.js diff --git a/sensor-iso/interface/sensor_interface/static/js/freeboard.thirdparty.min.js b/hedgehog-iso/interface/sensor_interface/static/js/freeboard.thirdparty.min.js similarity index 100% rename from sensor-iso/interface/sensor_interface/static/js/freeboard.thirdparty.min.js rename to hedgehog-iso/interface/sensor_interface/static/js/freeboard.thirdparty.min.js diff --git a/sensor-iso/interface/sensor_interface/static/js/freeboard_plugins.min.js b/hedgehog-iso/interface/sensor_interface/static/js/freeboard_plugins.min.js similarity index 100% rename from sensor-iso/interface/sensor_interface/static/js/freeboard_plugins.min.js rename to hedgehog-iso/interface/sensor_interface/static/js/freeboard_plugins.min.js diff --git a/sensor-iso/interface/sensor_interface/static/js/jquery.sparkline.min.js b/hedgehog-iso/interface/sensor_interface/static/js/jquery.sparkline.min.js similarity index 100% rename from sensor-iso/interface/sensor_interface/static/js/jquery.sparkline.min.js rename to hedgehog-iso/interface/sensor_interface/static/js/jquery.sparkline.min.js diff --git a/sensor-iso/interface/sensor_interface/static/js/material.min.js b/hedgehog-iso/interface/sensor_interface/static/js/material.min.js similarity index 100% rename from sensor-iso/interface/sensor_interface/static/js/material.min.js rename to hedgehog-iso/interface/sensor_interface/static/js/material.min.js diff --git a/sensor-iso/interface/sensor_interface/sysquery/__init__.py b/hedgehog-iso/interface/sensor_interface/sysquery/__init__.py similarity index 100% rename from sensor-iso/interface/sensor_interface/sysquery/__init__.py rename to hedgehog-iso/interface/sensor_interface/sysquery/__init__.py diff --git a/sensor-iso/interface/sensor_interface/sysquery/sys_service.py b/hedgehog-iso/interface/sensor_interface/sysquery/sys_service.py similarity index 100% rename from sensor-iso/interface/sensor_interface/sysquery/sys_service.py rename to hedgehog-iso/interface/sensor_interface/sysquery/sys_service.py diff --git a/sensor-iso/interface/sensor_interface/templates/base.html b/hedgehog-iso/interface/sensor_interface/templates/base.html similarity index 100% rename from sensor-iso/interface/sensor_interface/templates/base.html rename to hedgehog-iso/interface/sensor_interface/templates/base.html diff --git a/sensor-iso/interface/sensor_interface/templates/buttons.html b/hedgehog-iso/interface/sensor_interface/templates/buttons.html similarity index 100% rename from sensor-iso/interface/sensor_interface/templates/buttons.html rename to hedgehog-iso/interface/sensor_interface/templates/buttons.html diff --git a/sensor-iso/interface/sensor_interface/templates/system_block.html b/hedgehog-iso/interface/sensor_interface/templates/system_block.html similarity index 100% rename from sensor-iso/interface/sensor_interface/templates/system_block.html rename to hedgehog-iso/interface/sensor_interface/templates/system_block.html diff --git a/sensor-iso/vagrant/Vagrantfile b/hedgehog-iso/vagrant/Vagrantfile similarity index 100% rename from sensor-iso/vagrant/Vagrantfile rename to hedgehog-iso/vagrant/Vagrantfile diff --git a/sensor-iso/vagrant/ssh_config b/hedgehog-iso/vagrant/ssh_config similarity index 100% rename from sensor-iso/vagrant/ssh_config rename to hedgehog-iso/vagrant/ssh_config diff --git a/sensor-iso/yara/Dockerfile b/hedgehog-iso/yara/Dockerfile similarity index 100% rename from sensor-iso/yara/Dockerfile rename to hedgehog-iso/yara/Dockerfile diff --git a/sensor-iso/yara/build-docker-image.sh b/hedgehog-iso/yara/build-docker-image.sh similarity index 100% rename from sensor-iso/yara/build-docker-image.sh rename to hedgehog-iso/yara/build-docker-image.sh diff --git a/sensor-iso/yara/build-yara-deb.sh b/hedgehog-iso/yara/build-yara-deb.sh similarity index 100% rename from sensor-iso/yara/build-yara-deb.sh rename to hedgehog-iso/yara/build-yara-deb.sh diff --git a/sensor-raspi/.gitignore b/hedgehog-raspi/.gitignore similarity index 100% rename from sensor-raspi/.gitignore rename to hedgehog-raspi/.gitignore diff --git a/sensor-raspi/Dockerfile b/hedgehog-raspi/Dockerfile similarity index 95% rename from sensor-raspi/Dockerfile rename to hedgehog-raspi/Dockerfile index 7338c21a3..dcdcb9468 100644 --- a/sensor-raspi/Dockerfile +++ b/hedgehog-raspi/Dockerfile @@ -5,7 +5,7 @@ FROM ghcr.io/mmguero/qemu-live-iso:latest LABEL maintainer="malcolm@inl.gov" LABEL org.opencontainers.image.authors='malcolm@inl.gov' LABEL org.opencontainers.image.url='https://github.com/idaholab/Malcolm' -LABEL org.opencontainers.image.documentation='https://github.com/idaholab/Malcolm/blob/main/sensor-iso/README.md' +LABEL org.opencontainers.image.documentation='https://github.com/idaholab/Malcolm/blob/main/hedgehog-iso/README.md' LABEL org.opencontainers.image.source='https://github.com/idaholab/Malcolm' LABEL org.opencontainers.image.vendor='Idaho National Laboratory' LABEL org.opencontainers.image.title='ghcr.io/idaholab/malcolm/hedgehog-raspi' diff --git a/sensor-raspi/Makefile b/hedgehog-raspi/Makefile similarity index 100% rename from sensor-raspi/Makefile rename to hedgehog-raspi/Makefile diff --git a/sensor-raspi/README.md b/hedgehog-raspi/README.md similarity index 99% rename from sensor-raspi/README.md rename to hedgehog-raspi/README.md index 9b2bc1c42..fe63ddbee 100644 --- a/sensor-raspi/README.md +++ b/hedgehog-raspi/README.md @@ -11,7 +11,7 @@ clone this repository: ```shell git clone https://github.com/idaholab/Malcolm.git -cd Malcolm/sensor-raspi +cd Malcolm/hedgehog-raspi ``` For this you will first need to install the following packages on a diff --git a/sensor-raspi/attribution.txt b/hedgehog-raspi/attribution.txt similarity index 100% rename from sensor-raspi/attribution.txt rename to hedgehog-raspi/attribution.txt diff --git a/sensor-raspi/build_via_vagrant.sh b/hedgehog-raspi/build_via_vagrant.sh similarity index 96% rename from sensor-raspi/build_via_vagrant.sh rename to hedgehog-raspi/build_via_vagrant.sh index 6141a6aa3..396f51d88 100755 --- a/sensor-raspi/build_via_vagrant.sh +++ b/hedgehog-raspi/build_via_vagrant.sh @@ -102,7 +102,7 @@ YML_IMAGE_VERSION="$(grep -P "^\s+image:.*/malcolm/" "$SCRIPT_PATH"/../docker-co echo "VCS_REVSION=$( git rev-parse --short HEAD 2>/dev/null || echo main )" >> "$SCRIPT_PATH"/shared/environment.chroot trap cleanup_shared_and_docs EXIT -vm_execute "sudo bash -c \"whoami && cd /Malcolm/sensor-raspi && pwd && make ${IMAGE}${XZ_EXT}\"" +vm_execute "sudo bash -c \"whoami && cd /Malcolm/hedgehog-raspi && pwd && make ${IMAGE}${XZ_EXT}\"" if [[ -n $NEED_SHUTDOWN ]]; then echo "Shutting down $VM_NAME..." >&2 diff --git a/sensor-raspi/debian/salsa-ci.yml b/hedgehog-raspi/debian/salsa-ci.yml similarity index 100% rename from sensor-raspi/debian/salsa-ci.yml rename to hedgehog-raspi/debian/salsa-ci.yml diff --git a/sensor-raspi/debian/yamllint.yml b/hedgehog-raspi/debian/yamllint.yml similarity index 100% rename from sensor-raspi/debian/yamllint.yml rename to hedgehog-raspi/debian/yamllint.yml diff --git a/sensor-raspi/generate-recipe.py b/hedgehog-raspi/generate-recipe.py similarity index 99% rename from sensor-raspi/generate-recipe.py rename to hedgehog-raspi/generate-recipe.py index e40b8a638..4ad1535c7 100755 --- a/sensor-raspi/generate-recipe.py +++ b/hedgehog-raspi/generate-recipe.py @@ -7,7 +7,7 @@ SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) MALCOLM_DIR = os.path.dirname(SCRIPT_DIR) -SENSOR_DIR = os.path.join(MALCOLM_DIR, 'sensor-iso') +SENSOR_DIR = os.path.join(MALCOLM_DIR, 'hedgehog-iso') # pylint: disable=invalid-name diff --git a/sensor-raspi/raspi_master.yaml b/hedgehog-raspi/raspi_master.yaml similarity index 100% rename from sensor-raspi/raspi_master.yaml rename to hedgehog-raspi/raspi_master.yaml diff --git a/sensor-raspi/rootfs/boot/firmware/sysconf.txt b/hedgehog-raspi/rootfs/boot/firmware/sysconf.txt similarity index 100% rename from sensor-raspi/rootfs/boot/firmware/sysconf.txt rename to hedgehog-raspi/rootfs/boot/firmware/sysconf.txt diff --git a/sensor-raspi/rootfs/etc/fstab b/hedgehog-raspi/rootfs/etc/fstab similarity index 100% rename from sensor-raspi/rootfs/etc/fstab rename to hedgehog-raspi/rootfs/etc/fstab diff --git a/sensor-raspi/rootfs/etc/initramfs-tools/hooks/rpi-resizerootfs b/hedgehog-raspi/rootfs/etc/initramfs-tools/hooks/rpi-resizerootfs similarity index 100% rename from sensor-raspi/rootfs/etc/initramfs-tools/hooks/rpi-resizerootfs rename to hedgehog-raspi/rootfs/etc/initramfs-tools/hooks/rpi-resizerootfs diff --git a/sensor-raspi/rootfs/etc/initramfs-tools/scripts/local-bottom/rpi-resizerootfs b/hedgehog-raspi/rootfs/etc/initramfs-tools/scripts/local-bottom/rpi-resizerootfs similarity index 100% rename from sensor-raspi/rootfs/etc/initramfs-tools/scripts/local-bottom/rpi-resizerootfs rename to hedgehog-raspi/rootfs/etc/initramfs-tools/scripts/local-bottom/rpi-resizerootfs diff --git a/sensor-raspi/rootfs/etc/network/interfaces.d/eth0 b/hedgehog-raspi/rootfs/etc/network/interfaces.d/eth0 similarity index 100% rename from sensor-raspi/rootfs/etc/network/interfaces.d/eth0 rename to hedgehog-raspi/rootfs/etc/network/interfaces.d/eth0 diff --git a/sensor-raspi/rootfs/etc/network/interfaces.d/wlan0 b/hedgehog-raspi/rootfs/etc/network/interfaces.d/wlan0 similarity index 100% rename from sensor-raspi/rootfs/etc/network/interfaces.d/wlan0 rename to hedgehog-raspi/rootfs/etc/network/interfaces.d/wlan0 diff --git a/sensor-raspi/rootfs/etc/systemd/system/rpi-generate-ssh-host-keys.service b/hedgehog-raspi/rootfs/etc/systemd/system/rpi-generate-ssh-host-keys.service similarity index 100% rename from sensor-raspi/rootfs/etc/systemd/system/rpi-generate-ssh-host-keys.service rename to hedgehog-raspi/rootfs/etc/systemd/system/rpi-generate-ssh-host-keys.service diff --git a/sensor-raspi/rootfs/etc/systemd/system/rpi-reconfigure-raspi-firmware.service b/hedgehog-raspi/rootfs/etc/systemd/system/rpi-reconfigure-raspi-firmware.service similarity index 100% rename from sensor-raspi/rootfs/etc/systemd/system/rpi-reconfigure-raspi-firmware.service rename to hedgehog-raspi/rootfs/etc/systemd/system/rpi-reconfigure-raspi-firmware.service diff --git a/sensor-raspi/rootfs/etc/systemd/system/rpi-set-sysconf.service b/hedgehog-raspi/rootfs/etc/systemd/system/rpi-set-sysconf.service similarity index 100% rename from sensor-raspi/rootfs/etc/systemd/system/rpi-set-sysconf.service rename to hedgehog-raspi/rootfs/etc/systemd/system/rpi-set-sysconf.service diff --git a/sensor-raspi/rootfs/etc/wpa_supplicant/wpa_supplicant.conf b/hedgehog-raspi/rootfs/etc/wpa_supplicant/wpa_supplicant.conf similarity index 100% rename from sensor-raspi/rootfs/etc/wpa_supplicant/wpa_supplicant.conf rename to hedgehog-raspi/rootfs/etc/wpa_supplicant/wpa_supplicant.conf diff --git a/sensor-raspi/rootfs/usr/local/sbin/rpi-set-sysconf b/hedgehog-raspi/rootfs/usr/local/sbin/rpi-set-sysconf similarity index 100% rename from sensor-raspi/rootfs/usr/local/sbin/rpi-set-sysconf rename to hedgehog-raspi/rootfs/usr/local/sbin/rpi-set-sysconf diff --git a/sensor-raspi/sensor_install.sh b/hedgehog-raspi/sensor_install.sh similarity index 98% rename from sensor-raspi/sensor_install.sh rename to hedgehog-raspi/sensor_install.sh index d060e2ba7..5e607bdbf 100644 --- a/sensor-raspi/sensor_install.sh +++ b/hedgehog-raspi/sensor_install.sh @@ -58,7 +58,7 @@ BUILD_ERROR_CODE=1 build_arkime(){ mkdir -p /tmp/arkime-deb - arkime_ver='5.0.0-1' + arkime_ver='5.0.1-1' curl -sSL -o /tmp/arkime-deb/arkime.deb "https://github.com/arkime/arkime/releases/download/v5.0.0/arkime_${arkime_ver}.ubuntu2204_arm64.deb" dpkg -i /tmp/arkime-deb/*.deb || apt-get -f install -y --no-install-suggests } @@ -66,7 +66,7 @@ build_arkime(){ build_arkime_src(){ arkime_repo='https://github.com/arkime/arkime.git' - arkime_ver='5.0.0' + arkime_ver='5.0.1' arkime_dir='/opt/arkime' build_jobs=$((PROC_CNT/2)) @@ -270,8 +270,12 @@ clean_up() { /opt/hedgehog_install_artifacts \ /opt/hooks \ /opt/patches \ - /root/.bash_history \ - /root/.wget-hsts \ + /root/.bash_history \ + /root/.wget-hsts \ + /root/.cache \ + /root/.local/share/gem \ + /root/.npm \ + "${DEBS_DIR}" \ /tmp/* find /var/log/ -type f -print0 2>/dev/null | \ xargs -0 -r -I XXX bash -c "file 'XXX' | grep -q text && > 'XXX'" @@ -329,7 +333,7 @@ install_deps() { deps+=$(tr '\n' ' ' < "$file") done - # Remove Sensor-ISO packages not relevant to RPI + # Remove hedgehog-iso packages not relevant to RPI # Rar is excluded because Debian doesn't have an ARM package # htpdate removed because repo version doesn't support https # aide is removed as we're not applying the same hardening requirements ot the rpi image diff --git a/sensor-raspi/vagrant/Vagrantfile b/hedgehog-raspi/vagrant/Vagrantfile similarity index 100% rename from sensor-raspi/vagrant/Vagrantfile rename to hedgehog-raspi/vagrant/Vagrantfile diff --git a/sensor-raspi/vagrant/ssh_config b/hedgehog-raspi/vagrant/ssh_config similarity index 100% rename from sensor-raspi/vagrant/ssh_config rename to hedgehog-raspi/vagrant/ssh_config diff --git a/kubernetes/03-opensearch.yml b/kubernetes/03-opensearch.yml index 8f82de56e..ec9b8dd56 100644 --- a/kubernetes/03-opensearch.yml +++ b/kubernetes/03-opensearch.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: opensearch-container - image: ghcr.io/idaholab/malcolm/opensearch:24.02.0 + image: ghcr.io/idaholab/malcolm/opensearch:24.03.0 imagePullPolicy: Always stdin: false tty: true @@ -71,7 +71,7 @@ spec: subPath: "opensearch" initContainers: - name: opensearch-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.02.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.03.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/04-dashboards.yml b/kubernetes/04-dashboards.yml index dc9fe4aca..f70cb83aa 100644 --- a/kubernetes/04-dashboards.yml +++ b/kubernetes/04-dashboards.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: dashboards-container - image: ghcr.io/idaholab/malcolm/dashboards:24.02.0 + image: ghcr.io/idaholab/malcolm/dashboards:24.03.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/05-upload.yml b/kubernetes/05-upload.yml index 54ac4699e..e0aeb3745 100644 --- a/kubernetes/05-upload.yml +++ b/kubernetes/05-upload.yml @@ -34,7 +34,7 @@ spec: spec: containers: - name: upload-container - image: ghcr.io/idaholab/malcolm/file-upload:24.02.0 + image: ghcr.io/idaholab/malcolm/file-upload:24.03.0 imagePullPolicy: Always stdin: false tty: true @@ -73,7 +73,7 @@ spec: subPath: "upload" initContainers: - name: upload-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.02.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.03.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/06-pcap-monitor.yml b/kubernetes/06-pcap-monitor.yml index f3fe9213a..427bb4d7c 100644 --- a/kubernetes/06-pcap-monitor.yml +++ b/kubernetes/06-pcap-monitor.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: pcap-monitor-container - image: ghcr.io/idaholab/malcolm/pcap-monitor:24.02.0 + image: ghcr.io/idaholab/malcolm/pcap-monitor:24.03.0 imagePullPolicy: Always stdin: false tty: true @@ -70,7 +70,7 @@ spec: name: pcap-monitor-zeek-volume initContainers: - name: pcap-monitor-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.02.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.03.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/07-arkime.yml b/kubernetes/07-arkime.yml index 9358a7980..9085b877d 100644 --- a/kubernetes/07-arkime.yml +++ b/kubernetes/07-arkime.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: arkime-container - image: ghcr.io/idaholab/malcolm/arkime:24.02.0 + image: ghcr.io/idaholab/malcolm/arkime:24.03.0 imagePullPolicy: Always stdin: false tty: true @@ -79,7 +79,7 @@ spec: name: arkime-pcap-volume initContainers: - name: arkime-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.02.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.03.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/08-api.yml b/kubernetes/08-api.yml index cd8462b80..584799bd5 100644 --- a/kubernetes/08-api.yml +++ b/kubernetes/08-api.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: api-container - image: ghcr.io/idaholab/malcolm/api:24.02.0 + image: ghcr.io/idaholab/malcolm/api:24.03.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/09-dashboards-helper.yml b/kubernetes/09-dashboards-helper.yml index 4f8598940..3ea236ed1 100644 --- a/kubernetes/09-dashboards-helper.yml +++ b/kubernetes/09-dashboards-helper.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: dashboards-helper-container - image: ghcr.io/idaholab/malcolm/dashboards-helper:24.02.0 + image: ghcr.io/idaholab/malcolm/dashboards-helper:24.03.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/10-zeek.yml b/kubernetes/10-zeek.yml index 34352453c..10827bb57 100644 --- a/kubernetes/10-zeek.yml +++ b/kubernetes/10-zeek.yml @@ -16,7 +16,7 @@ spec: spec: containers: - name: zeek-offline-container - image: ghcr.io/idaholab/malcolm/zeek:24.02.0 + image: ghcr.io/idaholab/malcolm/zeek:24.03.0 imagePullPolicy: Always stdin: false tty: true @@ -64,7 +64,7 @@ spec: subPath: "zeek/intel" initContainers: - name: zeek-offline-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.02.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.03.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/11-suricata.yml b/kubernetes/11-suricata.yml index c19818116..80de0fed8 100644 --- a/kubernetes/11-suricata.yml +++ b/kubernetes/11-suricata.yml @@ -16,7 +16,7 @@ spec: spec: containers: - name: suricata-offline-container - image: ghcr.io/idaholab/malcolm/suricata:24.02.0 + image: ghcr.io/idaholab/malcolm/suricata:24.03.0 imagePullPolicy: Always stdin: false tty: true @@ -55,7 +55,7 @@ spec: name: suricata-offline-custom-configs-volume initContainers: - name: suricata-offline-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.02.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.03.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/12-file-monitor.yml b/kubernetes/12-file-monitor.yml index ed4ee049c..647cddedd 100644 --- a/kubernetes/12-file-monitor.yml +++ b/kubernetes/12-file-monitor.yml @@ -33,7 +33,7 @@ spec: spec: containers: - name: file-monitor-container - image: ghcr.io/idaholab/malcolm/file-monitor:24.02.0 + image: ghcr.io/idaholab/malcolm/file-monitor:24.03.0 imagePullPolicy: Always stdin: false tty: true @@ -83,7 +83,7 @@ spec: name: file-monitor-yara-rules-custom-volume initContainers: - name: file-monitor-live-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.02.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.03.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/13-filebeat.yml b/kubernetes/13-filebeat.yml index 553f04d46..edb237b3f 100644 --- a/kubernetes/13-filebeat.yml +++ b/kubernetes/13-filebeat.yml @@ -33,7 +33,7 @@ spec: spec: containers: - name: filebeat-container - image: ghcr.io/idaholab/malcolm/filebeat-oss:24.02.0 + image: ghcr.io/idaholab/malcolm/filebeat-oss:24.03.0 imagePullPolicy: Always stdin: false tty: true @@ -83,7 +83,7 @@ spec: subPath: "nginx" initContainers: - name: filebeat-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.02.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.03.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/14-logstash.yml b/kubernetes/14-logstash.yml index 47b4943a4..cfe42b9b2 100644 --- a/kubernetes/14-logstash.yml +++ b/kubernetes/14-logstash.yml @@ -49,7 +49,7 @@ spec: # topologyKey: "kubernetes.io/hostname" containers: - name: logstash-container - image: ghcr.io/idaholab/malcolm/logstash-oss:24.02.0 + image: ghcr.io/idaholab/malcolm/logstash-oss:24.03.0 imagePullPolicy: Always stdin: false tty: true @@ -115,7 +115,7 @@ spec: subPath: "logstash" initContainers: - name: logstash-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.02.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.03.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/15-netbox-redis.yml b/kubernetes/15-netbox-redis.yml index 27e54661d..206a486a9 100644 --- a/kubernetes/15-netbox-redis.yml +++ b/kubernetes/15-netbox-redis.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: netbox-redis-container - image: ghcr.io/idaholab/malcolm/redis:24.02.0 + image: ghcr.io/idaholab/malcolm/redis:24.03.0 imagePullPolicy: Always stdin: false tty: true @@ -83,7 +83,7 @@ spec: subPath: netbox/redis initContainers: - name: netbox-redis-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.02.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.03.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/16-netbox-redis-cache.yml b/kubernetes/16-netbox-redis-cache.yml index 011f7baf7..a7985dfb1 100644 --- a/kubernetes/16-netbox-redis-cache.yml +++ b/kubernetes/16-netbox-redis-cache.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: netbox-redis-cache-container - image: ghcr.io/idaholab/malcolm/redis:24.02.0 + image: ghcr.io/idaholab/malcolm/redis:24.03.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/17-netbox-postgres.yml b/kubernetes/17-netbox-postgres.yml index 2b345240f..a9e2cab83 100644 --- a/kubernetes/17-netbox-postgres.yml +++ b/kubernetes/17-netbox-postgres.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: netbox-postgres-container - image: ghcr.io/idaholab/malcolm/postgresql:24.02.0 + image: ghcr.io/idaholab/malcolm/postgresql:24.03.0 imagePullPolicy: Always stdin: false tty: true @@ -74,7 +74,7 @@ spec: subPath: netbox/postgres initContainers: - name: netbox-postgres-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.02.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.03.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/18-netbox.yml b/kubernetes/18-netbox.yml index 37577f687..f2a3469c5 100644 --- a/kubernetes/18-netbox.yml +++ b/kubernetes/18-netbox.yml @@ -36,7 +36,7 @@ spec: spec: containers: - name: netbox-container - image: ghcr.io/idaholab/malcolm/netbox:24.02.0 + image: ghcr.io/idaholab/malcolm/netbox:24.03.0 imagePullPolicy: Always stdin: false tty: true @@ -88,7 +88,7 @@ spec: subPath: netbox/media initContainers: - name: netbox-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.02.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.03.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/19-htadmin.yml b/kubernetes/19-htadmin.yml index f39cb01d7..9bb3f0736 100644 --- a/kubernetes/19-htadmin.yml +++ b/kubernetes/19-htadmin.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: htadmin-container - image: ghcr.io/idaholab/malcolm/htadmin:24.02.0 + image: ghcr.io/idaholab/malcolm/htadmin:24.03.0 imagePullPolicy: Always stdin: false tty: true @@ -63,7 +63,7 @@ spec: subPath: "htadmin" initContainers: - name: htadmin-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.02.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.03.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/20-pcap-capture.yml b/kubernetes/20-pcap-capture.yml index f0ddd2ff8..7656623d9 100644 --- a/kubernetes/20-pcap-capture.yml +++ b/kubernetes/20-pcap-capture.yml @@ -16,7 +16,7 @@ spec: spec: containers: - name: pcap-capture-container - image: ghcr.io/idaholab/malcolm/pcap-capture:24.02.0 + image: ghcr.io/idaholab/malcolm/pcap-capture:24.03.0 imagePullPolicy: Always stdin: false tty: true @@ -50,7 +50,7 @@ spec: subPath: "upload" initContainers: - name: pcap-capture-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.02.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.03.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/21-zeek-live.yml b/kubernetes/21-zeek-live.yml index 1af8ac204..b4e220e20 100644 --- a/kubernetes/21-zeek-live.yml +++ b/kubernetes/21-zeek-live.yml @@ -16,7 +16,7 @@ spec: spec: containers: - name: zeek-live-container - image: ghcr.io/idaholab/malcolm/zeek:24.02.0 + image: ghcr.io/idaholab/malcolm/zeek:24.03.0 imagePullPolicy: Always stdin: false tty: true @@ -61,7 +61,7 @@ spec: subPath: "zeek/intel" initContainers: - name: zeek-live-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.02.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.03.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/22-suricata-live.yml b/kubernetes/22-suricata-live.yml index bd1ca6e31..3490b1dbe 100644 --- a/kubernetes/22-suricata-live.yml +++ b/kubernetes/22-suricata-live.yml @@ -16,7 +16,7 @@ spec: spec: containers: - name: suricata-live-container - image: ghcr.io/idaholab/malcolm/suricata:24.02.0 + image: ghcr.io/idaholab/malcolm/suricata:24.03.0 imagePullPolicy: Always stdin: false tty: true @@ -56,7 +56,7 @@ spec: name: suricata-live-custom-configs-volume initContainers: - name: suricata-live-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.02.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.03.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/23-arkime-live.yml b/kubernetes/23-arkime-live.yml index a84945410..0ce28c8a9 100644 --- a/kubernetes/23-arkime-live.yml +++ b/kubernetes/23-arkime-live.yml @@ -16,7 +16,7 @@ spec: spec: containers: - name: arkime-live-container - image: ghcr.io/idaholab/malcolm/arkime:24.02.0 + image: ghcr.io/idaholab/malcolm/arkime:24.03.0 imagePullPolicy: Always stdin: false tty: true @@ -62,7 +62,7 @@ spec: name: arkime-live-pcap-volume initContainers: - name: arkime-live-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.02.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.03.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/24-freq.yml b/kubernetes/24-freq.yml index 6fcd077cd..930bfacb7 100644 --- a/kubernetes/24-freq.yml +++ b/kubernetes/24-freq.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: freq-container - image: ghcr.io/idaholab/malcolm/freq:24.02.0 + image: ghcr.io/idaholab/malcolm/freq:24.03.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/98-nginx-proxy.yml b/kubernetes/98-nginx-proxy.yml index 35dbe4986..7b3ec0ae0 100644 --- a/kubernetes/98-nginx-proxy.yml +++ b/kubernetes/98-nginx-proxy.yml @@ -39,7 +39,7 @@ spec: spec: containers: - name: nginx-proxy-container - image: ghcr.io/idaholab/malcolm/nginx-proxy:24.02.0 + image: ghcr.io/idaholab/malcolm/nginx-proxy:24.03.0 imagePullPolicy: Always stdin: false tty: true @@ -99,7 +99,7 @@ spec: subPath: "nginx" initContainers: - name: nginx-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.02.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.03.0 imagePullPolicy: Always stdin: false tty: true diff --git a/logstash/maps/zeek_log_ecs_categories.yaml b/logstash/maps/zeek_log_ecs_categories.yaml index 5a27fc270..6f7e94d18 100644 --- a/logstash/maps/zeek_log_ecs_categories.yaml +++ b/logstash/maps/zeek_log_ecs_categories.yaml @@ -61,6 +61,7 @@ "pe": ["file"] "profinet": ["ot", "network"] "profinet_dce_rpc": ["ot", "network"] +"profinet_io_cm": ["ot", "network"] "radius": ["authentication", "iam", "network"] "rdp": ["network"] "rfb": ["network"] diff --git a/logstash/pipelines/beats/11_beats_logs.conf b/logstash/pipelines/beats/11_beats_logs.conf index 7f883c163..2d9febd02 100644 --- a/logstash/pipelines/beats/11_beats_logs.conf +++ b/logstash/pipelines/beats/11_beats_logs.conf @@ -492,6 +492,29 @@ filter { } # if ([miscbeat][network]) + if ([miscbeat][proc]) { + #------------------------------------------------- + # proc - https://docs.fluentbit.io/manual/pipeline/inputs/process + + if (![event][hash]) { + fingerprint { + id => "fingerprint_malcolm_miscbeat_proc" + source => [ "[host][name]", + "[event][module]", + "[miscbeat][proc][proc_name]", + "[miscbeat][proc][pid]", + "[miscbeat][proc][fd]", + "[@timestamp]" ] + concatenate_sources => true + # uses event.hash + ecs_compatibility => "v8" + method => "MURMUR3_128" + base64encode => true + } + } + + } # if ([miscbeat][proc]) + if ([miscbeat][syslog]) { #------------------------------------------------- # syslog - https://docs.fluentbit.io/manual/pipeline/inputs/syslog @@ -814,6 +837,17 @@ filter { # Suricata statistics # https://docs.suricata.io/en/suricata-6.0.2/configuration/suricata-yaml.html#stats + # sum packet drops into suricata.stats.pkts_dropped + ruby { + id => "ruby_miscbeat_suricata_stats_pkts_dropped" + code => " + pkts_dropped = event.get('[suricata][stats][capture][kernel_drops]').to_i + + event.get('[suricata][stats][tcp][segment_memcap_drop]').to_i + + event.get('[suricata][stats][tcp][ssn_memcap_drop]').to_i + event.set('[suricata][stats][pkts_dropped]', pkts_dropped) unless (pkts_dropped == 0) + " + } + if ([suricata][stats]) { # remove zero values from suricata stats ruby { diff --git a/logstash/pipelines/output/98_finalize.conf b/logstash/pipelines/output/98_finalize.conf index 6cf43e49e..4a2580c64 100644 --- a/logstash/pipelines/output/98_finalize.conf +++ b/logstash/pipelines/output/98_finalize.conf @@ -20,6 +20,7 @@ filter { "_geoip_lookup_failure", "_grokparsefailure", "_jsonparsefailure", + "_jsonparsesuccess", "_malcolm_miscbeat", "_malcolm_beats", "_ouilookupfailure", diff --git a/logstash/pipelines/zeek/10_zeek_prep.conf b/logstash/pipelines/zeek/10_zeek_prep.conf index 5201f75a7..da3ebf627 100644 --- a/logstash/pipelines/zeek/10_zeek_prep.conf +++ b/logstash/pipelines/zeek/10_zeek_prep.conf @@ -38,31 +38,9 @@ filter { id => "ruby_zeek_prune_tags" code => " filenameTags = event.get('[@metadata][zeek_log_tags]').split(',') - zeekFieldsTag = filenameTags.select { |str| str.start_with?('ZEEKFLDx') } - if (zeekFieldsTag.size > 0) then - zeekFieldsTags = zeekFieldsTag.first.split('x', 3) - if (zeekFieldsTags.size == 3) then - event.set('[@metadata][zeek_fields_bitmap_version]', zeekFieldsTags[1].to_i(16)) - event.set('[@metadata][zeek_fields_bitmap]', zeekFieldsTags[2].to_i(16)) - end - end - filenameTags.delete_if{|v| ((v == nil) or (v == '') or (v !~ /\D/) or (v =~ /\A\s*(ZEEKFLDx|autocarve)/i) or (v =~ /\A\s*(pcap|dmp|log|bro|zeek|suricata|m?tcpdump|m?netsniff|autozeek|autosuricata)s?\s*\z/i) or (v == event.get('[log_source]')))} + filenameTags.delete_if{|v| ((v == nil) or (v == '') or (v !~ /\D/) or (v =~ /\A\s*(autocarve)/i) or (v =~ /\A\s*(pcap|dmp|log|bro|zeek|suricata|m?tcpdump|m?netsniff|autozeek|autosuricata)s?\s*\z/i) or (v == event.get('[log_source]')))} event.set('[@metadata][zeek_log_tags]', filenameTags.uniq) unless (filenameTags.length == 0) " - # - # ZEEKFLDx00x01FFFFFF - # | | └ bitmap of included fields within field list - # | └ index into zeek-log-field-bitmap.py:ZEEK_LOG_FIELDS list indicating field configuration within differing Zeek versions - # └ indicates that the field list has been pre-processed by zeek-process-pcap.py - # - # when logstash-filter-dissect gets this implemented, we *may* not have to do this - # - see zeek-process-pcap.py for the format of the bitmap number - # - https://github.com/logstash-plugins/logstash-filter-dissect/issues/56 - # - https://github.com/logstash-plugins/logstash-filter-dissect/issues/62 - # - # todo: right now rather than using the bitmap intelligently to build the field list, I'm just looking for - # known preconfigurations. look into doing it "smart". - # } if ([@metadata][zeek_log_tags]) { mutate { id => "mutate_merge_zeek_log_tags" merge => { "[tags]" => "[@metadata][zeek_log_tags]" } } } diff --git a/logstash/pipelines/zeek/11_zeek_parse.conf b/logstash/pipelines/zeek/11_zeek_parse.conf index 6cec9e898..c539aca21 100644 --- a/logstash/pipelines/zeek/11_zeek_parse.conf +++ b/logstash/pipelines/zeek/11_zeek_parse.conf @@ -1,4 +1,5 @@ -######################## # zeek -> arkime session creation and enrichment +######################## +# zeek -> arkime session creation and enrichment # # see https://docs.zeek.org/en/stable/script-reference/log-files.html for Zeek logfile documentation # @@ -15,6 +16,57 @@ filter { + # handle JSON-formatted Zeek logs right out of the gate, we'll do the field renaming below + if ([message] =~ /^{.*}$/) { json { + id => "json_zeek_message_parse" + source => "[message]" + target => "[zeek_cols]" + add_tag => [ "_jsonparsesuccess" ] + } } + + + # in JSON, do some global renaming of common fields to make them match the names we'd + # be assigning to them if we were reading TSV + if ("_jsonparsesuccess" in [tags]) { + + # some of the ICSNPP parsers do an interesting thing to handle source and destination fields + # (see https://github.com/cisagov/icsnpp-bacnet/?tab=readme-ov-file#source-and-destination-fields) + # so check for and handle those first + ruby { + id => "ruby_zeek_json_determine_source_destination_fields" + code => " + if ![event.get('[zeek_cols][source_h]').to_s, + event.get('[zeek_cols][source_p]').to_s, + event.get('[zeek_cols][destination_h]').to_s, + event.get('[zeek_cols][destination_p]').to_s].reject{ |e| e.nil? || e.empty? || (e == '0') }.empty? then + event.set('[@metadata][icsnpp_source_dest_fields]', 'true') + end + " + } + if ([@metadata][icsnpp_source_dest_fields]) { + mutate { + id => "mutate_rename_zeek_json_common_reversed_direction_fields" + rename => { "[zeek_cols][id.orig_h]" => "[zeek_cols][drop_orig_h]" } + rename => { "[zeek_cols][id.orig_p]" => "[zeek_cols][drop_orig_p]" } + rename => { "[zeek_cols][id.resp_h]" => "[zeek_cols][drop_resp_h]" } + rename => { "[zeek_cols][id.resp_p]" => "[zeek_cols][drop_resp_p]" } + rename => { "[zeek_cols][source_h]" => "[zeek_cols][orig_h]" } + rename => { "[zeek_cols][source_p]" => "[zeek_cols][orig_p]" } + rename => { "[zeek_cols][destination_h]" => "[zeek_cols][resp_h]" } + rename => { "[zeek_cols][destination_p]" => "[zeek_cols][resp_p]" } + } + + } else { + mutate { + id => "mutate_rename_zeek_json_common_fields" + rename => { "[zeek_cols][id.orig_h]" => "[zeek_cols][orig_h]" } + rename => { "[zeek_cols][id.orig_p]" => "[zeek_cols][orig_p]" } + rename => { "[zeek_cols][id.resp_h]" => "[zeek_cols][resp_h]" } + rename => { "[zeek_cols][id.resp_p]" => "[zeek_cols][resp_p]" } + } + } # icsnpp_source_dest_fields or not + } # _jsonparsesuccess in tags + # The Dissect is WAY faster than CSV, and quite a bit faster than mutate.split. However, it # is not as flexible when it comes to missing or extra columns # (See https://github.com/logstash-plugins/logstash-filter-dissect/issues/62) @@ -31,148 +83,46 @@ filter { # conn.log # https://docs.zeek.org/en/stable/scripts/base/protocols/conn/main.zeek.html#type-Conn::Info - if ([@metadata][zeek_fields_bitmap] and [@metadata][zeek_fields_bitmap_version]) { - - # bitmap conn.log field configuration version 0 - # - # Todo: I need to just bite the bullet and code this up in a Ruby block rather than hardcoding values. - # - # vlan + mac + community_id: 0x3ffffff / 67108863 - # vlan + mac: 0x1ffffff / 33554431 - # mac + community_id: 0x39fffff / 60817407 - # mac only: 0x19fffff / 27262975 - # vlan + community_id: 0x27fffff / 41943039 - # vlan only: 0x7fffff / 8388607 - # community_id only: 0x1fffff / 2097151 - # no custom fields: - - if ([@metadata][zeek_fields_bitmap_version] == 0) { - - if ([@metadata][zeek_fields_bitmap] == 67108863) { - # conn.log has vlans + macs + community_id - dissect { - id => "dissect_zeek_conn_with_all_fields" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][proto]} %{[zeek_cols][service]} %{[zeek_cols][duration]} %{[zeek_cols][orig_bytes]} %{[zeek_cols][resp_bytes]} %{[zeek_cols][conn_state]} %{[zeek_cols][local_orig]} %{[zeek_cols][local_resp]} %{[zeek_cols][missed_bytes]} %{[zeek_cols][history]} %{[zeek_cols][orig_pkts]} %{[zeek_cols][orig_ip_bytes]} %{[zeek_cols][resp_pkts]} %{[zeek_cols][resp_ip_bytes]} %{[zeek_cols][tunnel_parents]} %{[zeek_cols][vlan]} %{[zeek_cols][inner_vlan]} %{[zeek_cols][orig_l2_addr]} %{[zeek_cols][resp_l2_addr]} %{[zeek_cols][community_id]}" - } - } - - } else if ([@metadata][zeek_fields_bitmap] == 33554431) { - # conn.log has vlans + macs - dissect { - id => "dissect_zeek_conn_with_vlan_and_mac_fields" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][proto]} %{[zeek_cols][service]} %{[zeek_cols][duration]} %{[zeek_cols][orig_bytes]} %{[zeek_cols][resp_bytes]} %{[zeek_cols][conn_state]} %{[zeek_cols][local_orig]} %{[zeek_cols][local_resp]} %{[zeek_cols][missed_bytes]} %{[zeek_cols][history]} %{[zeek_cols][orig_pkts]} %{[zeek_cols][orig_ip_bytes]} %{[zeek_cols][resp_pkts]} %{[zeek_cols][resp_ip_bytes]} %{[zeek_cols][tunnel_parents]} %{[zeek_cols][vlan]} %{[zeek_cols][inner_vlan]} %{[zeek_cols][orig_l2_addr]} %{[zeek_cols][resp_l2_addr]}" - } - } - - } else if ([@metadata][zeek_fields_bitmap] == 60817407) { - # conn.log has macs + community_id - dissect { - id => "dissect_zeek_conn_with_mac_and_community_fields" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][proto]} %{[zeek_cols][service]} %{[zeek_cols][duration]} %{[zeek_cols][orig_bytes]} %{[zeek_cols][resp_bytes]} %{[zeek_cols][conn_state]} %{[zeek_cols][local_orig]} %{[zeek_cols][local_resp]} %{[zeek_cols][missed_bytes]} %{[zeek_cols][history]} %{[zeek_cols][orig_pkts]} %{[zeek_cols][orig_ip_bytes]} %{[zeek_cols][resp_pkts]} %{[zeek_cols][resp_ip_bytes]} %{[zeek_cols][tunnel_parents]} %{[zeek_cols][orig_l2_addr]} %{[zeek_cols][resp_l2_addr]} %{[zeek_cols][community_id]}" - } - } - - } else if ([@metadata][zeek_fields_bitmap] == 27262975) { - # conn.log has macs - dissect { - id => "dissect_zeek_conn_with_mac_fields" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][proto]} %{[zeek_cols][service]} %{[zeek_cols][duration]} %{[zeek_cols][orig_bytes]} %{[zeek_cols][resp_bytes]} %{[zeek_cols][conn_state]} %{[zeek_cols][local_orig]} %{[zeek_cols][local_resp]} %{[zeek_cols][missed_bytes]} %{[zeek_cols][history]} %{[zeek_cols][orig_pkts]} %{[zeek_cols][orig_ip_bytes]} %{[zeek_cols][resp_pkts]} %{[zeek_cols][resp_ip_bytes]} %{[zeek_cols][tunnel_parents]} %{[zeek_cols][orig_l2_addr]} %{[zeek_cols][resp_l2_addr]}" - } - } - - } else if ([@metadata][zeek_fields_bitmap] == 41943039) { - # conn.log has vlans + community_id - dissect { - id => "dissect_zeek_conn_with_vlan_and_community_fields" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][proto]} %{[zeek_cols][service]} %{[zeek_cols][duration]} %{[zeek_cols][orig_bytes]} %{[zeek_cols][resp_bytes]} %{[zeek_cols][conn_state]} %{[zeek_cols][local_orig]} %{[zeek_cols][local_resp]} %{[zeek_cols][missed_bytes]} %{[zeek_cols][history]} %{[zeek_cols][orig_pkts]} %{[zeek_cols][orig_ip_bytes]} %{[zeek_cols][resp_pkts]} %{[zeek_cols][resp_ip_bytes]} %{[zeek_cols][tunnel_parents]} %{[zeek_cols][vlan]} %{[zeek_cols][inner_vlan]} %{[zeek_cols][community_id]}" - } - } - - } else if ([@metadata][zeek_fields_bitmap] == 8388607) { - # conn.log has vlans - dissect { - id => "dissect_zeek_conn_with_vlan_fields" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][proto]} %{[zeek_cols][service]} %{[zeek_cols][duration]} %{[zeek_cols][orig_bytes]} %{[zeek_cols][resp_bytes]} %{[zeek_cols][conn_state]} %{[zeek_cols][local_orig]} %{[zeek_cols][local_resp]} %{[zeek_cols][missed_bytes]} %{[zeek_cols][history]} %{[zeek_cols][orig_pkts]} %{[zeek_cols][orig_ip_bytes]} %{[zeek_cols][resp_pkts]} %{[zeek_cols][resp_ip_bytes]} %{[zeek_cols][tunnel_parents]} %{[zeek_cols][vlan]} %{[zeek_cols][inner_vlan]}" - } - } - - } else if ([@metadata][zeek_fields_bitmap] == 2097151) { - # conn.log has community_id - dissect { - id => "dissect_zeek_conn_with_community_fields" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][proto]} %{[zeek_cols][service]} %{[zeek_cols][duration]} %{[zeek_cols][orig_bytes]} %{[zeek_cols][resp_bytes]} %{[zeek_cols][conn_state]} %{[zeek_cols][local_orig]} %{[zeek_cols][local_resp]} %{[zeek_cols][missed_bytes]} %{[zeek_cols][history]} %{[zeek_cols][orig_pkts]} %{[zeek_cols][orig_ip_bytes]} %{[zeek_cols][resp_pkts]} %{[zeek_cols][resp_ip_bytes]} %{[zeek_cols][tunnel_parents]} %{[zeek_cols][community_id]}" - } - } - - } else if ([@metadata][zeek_fields_bitmap] == 2097151) { - # conn.log has no custom fields - dissect { - id => "dissect_zeek_conn_with_minimal_fields" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][proto]} %{[zeek_cols][service]} %{[zeek_cols][duration]} %{[zeek_cols][orig_bytes]} %{[zeek_cols][resp_bytes]} %{[zeek_cols][conn_state]} %{[zeek_cols][local_orig]} %{[zeek_cols][local_resp]} %{[zeek_cols][missed_bytes]} %{[zeek_cols][history]} %{[zeek_cols][orig_pkts]} %{[zeek_cols][orig_ip_bytes]} %{[zeek_cols][resp_pkts]} %{[zeek_cols][resp_ip_bytes]} %{[zeek_cols][tunnel_parents]}" - } - } - - } else { - # who knows? the conn.log preprocessed bitmap is not one we're expecting, we've got to guess and cannot use dissect - mutate { id => "mutate_add_tag_dissect_failure_unknown_conn_bitmap" - add_tag => [ "_dissectfailure" ] } - } - - } else { - # who knows? the conn.log preprocessed bitmap field list version is not one we're expecting, we've got to guess and cannot use dissect - mutate { id => "mutate_add_tag_dissect_failure_unknown_bitmap_conn_version" - add_tag => [ "_dissectfailure" ] } - } - - } else { - # who knows? the conn.log was not preprocessed to determine fields, we've got to guess and cannot use dissect - mutate { id => "mutate_add_tag_dissect_failure_missing_conn_bitmap" - add_tag => [ "_dissectfailure" ] } - } - - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_conn" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_conn_with_all_fields" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][proto]} %{[zeek_cols][service]} %{[zeek_cols][duration]} %{[zeek_cols][orig_bytes]} %{[zeek_cols][resp_bytes]} %{[zeek_cols][conn_state]} %{[zeek_cols][local_orig]} %{[zeek_cols][local_resp]} %{[zeek_cols][missed_bytes]} %{[zeek_cols][history]} %{[zeek_cols][orig_pkts]} %{[zeek_cols][orig_ip_bytes]} %{[zeek_cols][resp_pkts]} %{[zeek_cols][resp_ip_bytes]} %{[zeek_cols][tunnel_parents]} %{[zeek_cols][vlan]} %{[zeek_cols][inner_vlan]} %{[zeek_cols][orig_l2_addr]} %{[zeek_cols][resp_l2_addr]} %{[zeek_cols][community_id]}" + } } - ruby { - id => "ruby_zip_zeek_conn" - init => "@zeek_conn_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto', 'service', 'duration', 'orig_bytes', 'resp_bytes', 'conn_state', 'local_orig', 'local_resp', 'missed_bytes', 'history', 'orig_pkts', 'orig_ip_bytes', 'resp_pkts', 'resp_ip_bytes', 'tunnel_parents', 'vlan', 'inner_vlan', 'orig_l2_addr', 'resp_l2_addr', 'community_id' ]" - code => "event.set('[zeek_cols]', @zeek_conn_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_conn" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_conn" + init => "@zeek_conn_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto', 'service', 'duration', 'orig_bytes', 'resp_bytes', 'conn_state', 'local_orig', 'local_resp', 'missed_bytes', 'history', 'orig_pkts', 'orig_ip_bytes', 'resp_pkts', 'resp_ip_bytes', 'tunnel_parents', 'vlan', 'inner_vlan', 'orig_l2_addr', 'resp_l2_addr', 'community_id' ]" + code => "event.set('[zeek_cols]', @zeek_conn_field_names.zip(event.get('[message]')).to_h)" + } } } # normalize service string(s) + # For some reason, even in JSON, I have service strings like: + # ...,"proto":"udp","service":"profinet,profinet_dce_rpc,spicy_profinet_io_cm,profinet",... + # so whatever reason it's not already an array. Split it here. + mutate { id => "mutate_split_zeek_conn_commas" + split => { "[zeek_cols][service]" => "," } } + # some services are named like blah_udp/blah_tcp/blah_data, and we don't care about the suffix mutate { id => "mutate_gsub_field_zeek_conn_service_protocol_suffix" gsub => [ "[zeek_cols][service]", "[_-](tcp|udp|data)", "" ] } - if ([zeek_cols][service] =~ /spicy_/) { - # if it's coming from spicy, we don't care to have that in the service name - mutate { id => "mutate_gsub_field_zeek_conn_service_spicy_prefix" - gsub => [ "[zeek_cols][service]", "spicy_", "" ] } + # if it's coming from spicy, we don't care to have that in the service name + mutate { id => "mutate_gsub_field_zeek_conn_service_spicy_prefix" + gsub => [ "[zeek_cols][service]", "spicy_", "" ] } - mutate { id => "mutate_gsub_field_zeek_conn_service_spicy_cipher_suffix" - gsub => [ "[zeek_cols][service]", "(_hmac)?(_(sha|md)\d+)?$", "" ] } - } + mutate { id => "mutate_gsub_field_zeek_conn_service_spicy_cipher_suffix" + gsub => [ "[zeek_cols][service]", "(_hmac)?(_(sha|md)\d+)?$", "" ] } if ([zeek_cols][orig_ip_bytes]) and ([zeek_cols][orig_ip_bytes] != '-') and ([zeek_cols][orig_ip_bytes] != '(empty)') and ([zeek_cols][orig_ip_bytes] != '') { mutate { id => "mutate_add_field_zeek_srcBytes" @@ -199,8 +149,8 @@ filter { add_field => { "[server][bytes]" => "%{[zeek_cols][resp_bytes]}" } } } if ([zeek_cols][tunnel_parents]) and ([zeek_cols][tunnel_parents] != '(empty)') and ([zeek_cols][tunnel_parents] != '-') and ([zeek_cols][tunnel_parents] != '') { - mutate { id => "mutate_split_zeek_tunnel_parents" - split => { "[zeek_cols][tunnel_parents]" => "," } } + if ("_jsonparsesuccess" not in [tags]) { mutate { id => "mutate_split_zeek_tunnel_parents" + split => { "[zeek_cols][tunnel_parents]" => "," } } } mutate { id => "mutate_add_field_zeek_conn_rootId" add_field => { "[rootId]" => "%{[zeek_cols][tunnel_parents][0]}" } } } @@ -210,23 +160,25 @@ filter { # bacnet.log # https://github.com/cisagov/ICSNPP - dissect { - id => "dissect_zeek_bacnet" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][bvlc_function]} %{[zeek_cols][pdu_type]} %{[zeek_cols][pdu_service]} %{[zeek_cols][invoke_id]} %{[zeek_cols][result_code]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_bacnet" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_bacnet" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][bvlc_function]} %{[zeek_cols][pdu_type]} %{[zeek_cols][pdu_service]} %{[zeek_cols][invoke_id]} %{[zeek_cols][result_code]}" + } } - ruby { - id => "ruby_zip_zeek_bacnet" - init => "@zeek_bacnet_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'bvlc_function', 'pdu_type', 'pdu_service', 'invoke_id', 'result_code' ]" - code => "event.set('[zeek_cols]', @zeek_bacnet_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_bacnet" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_bacnet" + init => "@zeek_bacnet_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'bvlc_function', 'pdu_type', 'pdu_service', 'invoke_id', 'result_code' ]" + code => "event.set('[zeek_cols]', @zeek_bacnet_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -243,23 +195,25 @@ filter { ############################################################################################################################# # bestguess.log - dissect { - id => "dissect_zeek_bestguess" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][proto]} %{[zeek_cols][name]} %{[zeek_cols][category]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_bestguess" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_bestguess" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][proto]} %{[zeek_cols][name]} %{[zeek_cols][category]}" + } } - ruby { - id => "ruby_zip_zeek_bestguess" - init => "@zeek_bestguess_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto', 'name', 'category' ]" - code => "event.set('[zeek_cols]', @zeek_bestguess_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_bestguess" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_bestguess" + init => "@zeek_bestguess_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto', 'name', 'category' ]" + code => "event.set('[zeek_cols]', @zeek_bestguess_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -271,23 +225,25 @@ filter { # bsap_ip_header.log # https://github.com/cisagov/ICSNPP - dissect { - id => "dissect_zeek_bsap_ip_header" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][num_msg]} %{[zeek_cols][type_name]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_bsap_ip_header" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_bsap_ip_header" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][num_msg]} %{[zeek_cols][type_name]}" + } } - ruby { - id => "ruby_zip_zeek_bsap_ip_header" - init => "@zeek_bsap_ip_header_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'num_msg', 'type_name' ]" - code => "event.set('[zeek_cols]', @zeek_bsap_ip_header_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_bsap_ip_header" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_bsap_ip_header" + init => "@zeek_bsap_ip_header_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'num_msg', 'type_name' ]" + code => "event.set('[zeek_cols]', @zeek_bsap_ip_header_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -305,23 +261,25 @@ filter { # bsap_ip_rdb.log # https://github.com/cisagov/ICSNPP - dissect { - id => "dissect_zeek_bsap_ip_rdb" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][header_size]} %{[zeek_cols][mes_seq]} %{[zeek_cols][res_seq]} %{[zeek_cols][data_len]} %{[zeek_cols][sequence]} %{[zeek_cols][app_func_code]} %{[zeek_cols][node_status]} %{[zeek_cols][func_code]} %{[zeek_cols][variable_count]} %{[zeek_cols][variables]} %{[zeek_cols][variable_value]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_bsap_ip_rdb" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_bsap_ip_rdb" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][header_size]} %{[zeek_cols][mes_seq]} %{[zeek_cols][res_seq]} %{[zeek_cols][data_len]} %{[zeek_cols][sequence]} %{[zeek_cols][app_func_code]} %{[zeek_cols][node_status]} %{[zeek_cols][func_code]} %{[zeek_cols][variable_count]} %{[zeek_cols][variables]} %{[zeek_cols][variable_value]}" + } } - ruby { - id => "ruby_zip_zeek_bsap_ip_rdb" - init => "@zeek_bsap_ip_rdb_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'header_size', 'mes_seq', 'res_seq', 'data_len', 'sequence', 'app_func_code', 'node_status', 'func_code', 'variable_count', 'variables', 'variable_value' ]" - code => "event.set('[zeek_cols]', @zeek_bsap_ip_rdb_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_bsap_ip_rdb" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_bsap_ip_rdb" + init => "@zeek_bsap_ip_rdb_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'header_size', 'mes_seq', 'res_seq', 'data_len', 'sequence', 'app_func_code', 'node_status', 'func_code', 'variable_count', 'variables', 'variable_value' ]" + code => "event.set('[zeek_cols]', @zeek_bsap_ip_rdb_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -339,23 +297,25 @@ filter { # bsap_serial_header.log # https://github.com/cisagov/ICSNPP - dissect { - id => "dissect_zeek_bsap_serial_header" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][ser]} %{[zeek_cols][dadd]} %{[zeek_cols][sadd]} %{[zeek_cols][ctl]} %{[zeek_cols][dfun]} %{[zeek_cols][seq]} %{[zeek_cols][sfun]} %{[zeek_cols][nsb]} %{[zeek_cols][type_name]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_bsap_serial_header" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_bsap_serial_header" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][ser]} %{[zeek_cols][dadd]} %{[zeek_cols][sadd]} %{[zeek_cols][ctl]} %{[zeek_cols][dfun]} %{[zeek_cols][seq]} %{[zeek_cols][sfun]} %{[zeek_cols][nsb]} %{[zeek_cols][type_name]}" + } } - ruby { - id => "ruby_zip_zeek_bsap_serial_header" - init => "@zeek_bsap_serial_header_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'ser', 'dadd', 'sadd', 'ctl', 'dfun', 'seq', 'sfun', 'nsb', 'type_name' ]" - code => "event.set('[zeek_cols]', @zeek_bsap_serial_header_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_bsap_serial_header" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_bsap_serial_header" + init => "@zeek_bsap_serial_header_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'ser', 'dadd', 'sadd', 'ctl', 'dfun', 'seq', 'sfun', 'nsb', 'type_name' ]" + code => "event.set('[zeek_cols]', @zeek_bsap_serial_header_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -373,23 +333,25 @@ filter { # bsap_serial_rdb.log # https://github.com/cisagov/ICSNPP - dissect { - id => "dissect_zeek_bsap_serial_rdb" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][func_code]} %{[zeek_cols][variable_count]} %{[zeek_cols][variables]} %{[zeek_cols][variable_value]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_bsap_serial_rdb" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_bsap_serial_rdb" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][func_code]} %{[zeek_cols][variable_count]} %{[zeek_cols][variables]} %{[zeek_cols][variable_value]}" + } } - ruby { - id => "ruby_zip_zeek_bsap_serial_rdb" - init => "@zeek_bsap_serial_rdb_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'func_code', 'variable_count', 'variables', 'variable_value' ]" - code => "event.set('[zeek_cols]', @zeek_bsap_serial_rdb_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_bsap_serial_rdb" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_bsap_serial_rdb" + init => "@zeek_bsap_serial_rdb_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'func_code', 'variable_count', 'variables', 'variable_value' ]" + code => "event.set('[zeek_cols]', @zeek_bsap_serial_rdb_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -407,23 +369,25 @@ filter { # bsap_serial_rdb_ext.log # https://github.com/cisagov/ICSNPP - dissect { - id => "dissect_zeek_bsap_serial_rdb_ext" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][dfun]} %{[zeek_cols][seq]} %{[zeek_cols][sfun]} %{[zeek_cols][nsb]} %{[zeek_cols][extfun]} %{[zeek_cols][data]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_bsap_serial_rdb_ext" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_bsap_serial_rdb_ext" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][dfun]} %{[zeek_cols][seq]} %{[zeek_cols][sfun]} %{[zeek_cols][nsb]} %{[zeek_cols][extfun]} %{[zeek_cols][data]}" + } } - ruby { - id => "ruby_zip_zeek_bsap_serial_rdb_ext" - init => "@zeek_bsap_serial_rdb_ext_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'dfun', 'seq', 'sfun', 'nsb', 'extfun', 'data' ]" - code => "event.set('[zeek_cols]', @zeek_bsap_serial_rdb_ext_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_bsap_serial_rdb_ext" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_bsap_serial_rdb_ext" + init => "@zeek_bsap_serial_rdb_ext_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'dfun', 'seq', 'sfun', 'nsb', 'extfun', 'data' ]" + code => "event.set('[zeek_cols]', @zeek_bsap_serial_rdb_ext_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -447,23 +411,25 @@ filter { # bacnet_device_control.log # https://github.com/cisagov/ICSNPP - dissect { - id => "dissect_zeek_bacnet_device_control" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][invoke_id]} %{[zeek_cols][pdu_service]} %{[zeek_cols][time_duration]} %{[zeek_cols][device_state]} %{[zeek_cols][password]} %{[zeek_cols][result]} %{[zeek_cols][result_code]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_bacnet_device_control" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_bacnet_device_control" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][invoke_id]} %{[zeek_cols][pdu_service]} %{[zeek_cols][time_duration]} %{[zeek_cols][device_state]} %{[zeek_cols][password]} %{[zeek_cols][result]} %{[zeek_cols][result_code]}" + } } - ruby { - id => "ruby_zip_zeek_bacnet_device_control" - init => "@zeek_bacnet_device_control_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'invoke_id', 'pdu_service', 'time_duration', 'device_state', 'password', 'result', 'result_code' ]" - code => "event.set('[zeek_cols]', @zeek_bacnet_device_control_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_bacnet_device_control" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_bacnet_device_control" + init => "@zeek_bacnet_device_control_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'invoke_id', 'pdu_service', 'time_duration', 'device_state', 'password', 'result', 'result_code' ]" + code => "event.set('[zeek_cols]', @zeek_bacnet_device_control_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -481,23 +447,25 @@ filter { # bacnet_discovery.log # https://github.com/cisagov/ICSNPP - dissect { - id => "dissect_zeek_bacnet_discovery" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][pdu_service]} %{[zeek_cols][object_type]} %{[zeek_cols][instance_number]} %{[zeek_cols][vendor]} %{[zeek_cols][range]} %{[zeek_cols][object_name]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_bacnet_discovery" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_bacnet_discovery" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][pdu_service]} %{[zeek_cols][object_type]} %{[zeek_cols][instance_number]} %{[zeek_cols][vendor]} %{[zeek_cols][range]} %{[zeek_cols][object_name]}" + } } - ruby { - id => "ruby_zip_zeek_bacnet_discovery" - init => "@zeek_bacnet_discovery_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'pdu_service', 'object_type', 'instance_number', 'vendor', 'range', 'object_name' ]" - code => "event.set('[zeek_cols]', @zeek_bacnet_discovery_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_bacnet_discovery" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_bacnet_discovery" + init => "@zeek_bacnet_discovery_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'pdu_service', 'object_type', 'instance_number', 'vendor', 'range', 'object_name' ]" + code => "event.set('[zeek_cols]', @zeek_bacnet_discovery_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -515,23 +483,25 @@ filter { # bacnet_property.log # https://github.com/cisagov/ICSNPP - dissect { - id => "dissect_zeek_bacnet_property" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][invoke_id]} %{[zeek_cols][pdu_service]} %{[zeek_cols][object_type]} %{[zeek_cols][instance_number]} %{[zeek_cols][property]} %{[zeek_cols][array_index]} %{[zeek_cols][value]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_bacnet_property" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_bacnet_property" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][invoke_id]} %{[zeek_cols][pdu_service]} %{[zeek_cols][object_type]} %{[zeek_cols][instance_number]} %{[zeek_cols][property]} %{[zeek_cols][array_index]} %{[zeek_cols][value]}" + } } - ruby { - id => "ruby_zip_zeek_bacnet_property" - init => "@zeek_bacnet_property_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'invoke_id', 'pdu_service', 'object_type', 'instance_number', 'property', 'array_index', 'value' ]" - code => "event.set('[zeek_cols]', @zeek_bacnet_property_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_bacnet_property" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_bacnet_property" + init => "@zeek_bacnet_property_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'invoke_id', 'pdu_service', 'object_type', 'instance_number', 'property', 'array_index', 'value' ]" + code => "event.set('[zeek_cols]', @zeek_bacnet_property_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -551,23 +521,25 @@ filter { # # todo: class_id, instance_id is a hex integer, should it be converted to an integer? - dissect { - id => "dissect_zeek_cip" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][cip_sequence_count]} %{[zeek_cols][direction]} %{[zeek_cols][cip_service_code]} %{[zeek_cols][cip_service]} %{[zeek_cols][cip_status_code]} %{[zeek_cols][cip_status]} %{[zeek_cols][cip_extended_status_code]} %{[zeek_cols][cip_extended_status]} %{[zeek_cols][class_id]} %{[zeek_cols][class_name]} %{[zeek_cols][instance_id]} %{[zeek_cols][attribute_id]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_cip" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_cip" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][cip_sequence_count]} %{[zeek_cols][direction]} %{[zeek_cols][cip_service_code]} %{[zeek_cols][cip_service]} %{[zeek_cols][cip_status_code]} %{[zeek_cols][cip_status]} %{[zeek_cols][cip_extended_status_code]} %{[zeek_cols][cip_extended_status]} %{[zeek_cols][class_id]} %{[zeek_cols][class_name]} %{[zeek_cols][instance_id]} %{[zeek_cols][attribute_id]}" + } } - ruby { - id => "ruby_zip_zeek_cip" - init => "@zeek_cip_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'cip_sequence_count', 'direction', 'cip_service_code', 'cip_service', 'cip_status_code', 'cip_status', 'cip_extended_status_code', 'cip_extended_status', 'class_id', 'class_name', 'instance_id', 'attribute_id' ]" - code => "event.set('[zeek_cols]', @zeek_cip_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_cip" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_cip" + init => "@zeek_cip_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'cip_sequence_count', 'direction', 'cip_service_code', 'cip_service', 'cip_status_code', 'cip_status', 'cip_extended_status_code', 'cip_extended_status', 'class_id', 'class_name', 'instance_id', 'attribute_id' ]" + code => "event.set('[zeek_cols]', @zeek_cip_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -584,23 +556,25 @@ filter { # # TODO: device_status and device_state are a hex int string, convert to int? - dissect { - id => "dissect_zeek_cip_identity" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][encapsulation_version]} %{[zeek_cols][socket_address]} %{[zeek_cols][socket_port]} %{[zeek_cols][vendor_id]} %{[zeek_cols][vendor_name]} %{[zeek_cols][device_type_id]} %{[zeek_cols][device_type_name]} %{[zeek_cols][product_code]} %{[zeek_cols][revision]} %{[zeek_cols][device_status]} %{[zeek_cols][serial_number]} %{[zeek_cols][product_name]} %{[zeek_cols][device_state]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_cip_identity" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_cip_identity" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][encapsulation_version]} %{[zeek_cols][socket_address]} %{[zeek_cols][socket_port]} %{[zeek_cols][vendor_id]} %{[zeek_cols][vendor_name]} %{[zeek_cols][device_type_id]} %{[zeek_cols][device_type_name]} %{[zeek_cols][product_code]} %{[zeek_cols][revision]} %{[zeek_cols][device_status]} %{[zeek_cols][serial_number]} %{[zeek_cols][product_name]} %{[zeek_cols][device_state]}" + } } - ruby { - id => "ruby_zip_zeek_cip_identity" - init => "@zeek_cip_identity_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'encapsulation_version', 'socket_address', 'socket_port', 'vendor_id', 'vendor_name', 'device_type_id', 'device_type_name', 'product_code', 'device_status', 'serial_number', 'product_name', 'device_state' ]" - code => "event.set('[zeek_cols]', @zeek_cip_identity_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_cip_identity" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_cip_identity" + init => "@zeek_cip_identity_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'encapsulation_version', 'socket_address', 'socket_port', 'vendor_id', 'vendor_name', 'device_type_id', 'device_type_name', 'product_code', 'revision', 'device_status', 'serial_number', 'product_name', 'device_state' ]" + code => "event.set('[zeek_cols]', @zeek_cip_identity_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -616,23 +590,25 @@ filter { # https://github.com/cisagov/ICSNPP # - dissect { - id => "dissect_zeek_cip_io" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][connection_id]} %{[zeek_cols][sequence_number]} %{[zeek_cols][data_length]} %{[zeek_cols][io_data]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_cip_io" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_cip_io" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][connection_id]} %{[zeek_cols][sequence_number]} %{[zeek_cols][data_length]} %{[zeek_cols][io_data]}" + } } - ruby { - id => "ruby_zip_zeek_cip_io" - init => "@zeek_cip_io_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'connection_id', 'sequence_number', 'data_length', 'io_data' ]" - code => "event.set('[zeek_cols]', @zeek_cip_io_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_cip_io" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_cip_io" + init => "@zeek_cip_io_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'connection_id', 'sequence_number', 'data_length', 'io_data' ]" + code => "event.set('[zeek_cols]', @zeek_cip_io_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -647,23 +623,25 @@ filter { # dce_rpc.log # https://docs.zeek.org/en/stable/scripts/base/protocols/dce-rpc/main.zeek.html#type-DCE_RPC::Info - dissect { - id => "dissect_zeek_dce_rpc" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][rtt]} %{[zeek_cols][named_pipe]} %{[zeek_cols][endpoint]} %{[zeek_cols][operation]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_dce_rpc" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_dce_rpc" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][rtt]} %{[zeek_cols][named_pipe]} %{[zeek_cols][endpoint]} %{[zeek_cols][operation]}" + } } - ruby { - id => "ruby_zip_zeek_dce_rpc" - init => "@zeek_dce_rpc_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'rtt', 'named_pipe', 'endpoint', 'operation' ]" - code => "event.set('[zeek_cols]', @zeek_dce_rpc_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_dce_rpc" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_dce_rpc" + init => "@zeek_dce_rpc_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'rtt', 'named_pipe', 'endpoint', 'operation' ]" + code => "event.set('[zeek_cols]', @zeek_dce_rpc_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -680,68 +658,47 @@ filter { # dhcp.log # https://docs.zeek.org/en/stable/scripts/base/protocols/dhcp/main.zeek.html#type-DHCP::Info - if ([@metadata][zeek_fields_bitmap] and [@metadata][zeek_fields_bitmap_version]) { - - # bitmap dhcp.log field configuration version 0 - # - # all fields : 0x0001FFFF / 131071 - # all fields minus software: 0x00007FFF / 32767 - - if ([@metadata][zeek_fields_bitmap_version] == 0) { - - if ([@metadata][zeek_fields_bitmap] == 131071) { - dissect { - id => "dissect_zeek_dhcp_with_all_fields" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][resp_h]} %{[zeek_cols][orig_l2_addr]} %{[zeek_cols][host_name]} %{[zeek_cols][client_fqdn]} %{[zeek_cols][domain]} %{[zeek_cols][requested_ip]} %{[zeek_cols][assigned_ip]} %{[zeek_cols][lease_time]} %{[zeek_cols][client_message]} %{[zeek_cols][server_message]} %{[zeek_cols][msg_types]} %{[zeek_cols][duration]} %{[zeek_cols][client_software]} %{[zeek_cols][server_software]}" - } - } - - } else if ([@metadata][zeek_fields_bitmap] == 32767) { - dissect { - id => "dissect_zeek_dhcp_with_all_fields_minus_software" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][resp_h]} %{[zeek_cols][orig_l2_addr]} %{[zeek_cols][host_name]} %{[zeek_cols][client_fqdn]} %{[zeek_cols][domain]} %{[zeek_cols][requested_ip]} %{[zeek_cols][assigned_ip]} %{[zeek_cols][lease_time]} %{[zeek_cols][client_message]} %{[zeek_cols][server_message]} %{[zeek_cols][msg_types]} %{[zeek_cols][duration]}" - } - } - - } else { - # who knows? the dhcp.log preprocessed bitmap is not one we're expecting, we've got to guess and cannot use dissect - mutate { id => "mutate_add_tag_dissect_failure_unknown_dhcp_bitmap" - add_tag => [ "_dissectfailure" ] } - } - - } else { - # who knows? the dhcp.log preprocessed bitmap field list version is not one we're expecting, we've got to guess and cannot use dissect - mutate { id => "mutate_add_tag_dissect_failure_unknown_bitmap_dhcp_version" - add_tag => [ "_dissectfailure" ] } + if ("_jsonparsesuccess" in [tags]) { + mutate { + id => "mutate_rename_zeek_json_dhcp_fields" + rename => { "[zeek_cols][uids]" => "[zeek_cols][uid]" } + rename => { "[zeek_cols][client_addr]" => "[zeek_cols][orig_h]" } + rename => { "[zeek_cols][server_addr]" => "[zeek_cols][orig_p]" } + rename => { "[zeek_cols][mac]" => "[zeek_cols][orig_l2_addr]" } + rename => { "[zeek_cols][requested_addr]" => "[zeek_cols][requested_ip]" } + rename => { "[zeek_cols][assigned_addr]" => "[zeek_cols][assigned_ip]" } } } else { - # who knows? the dhcp.log was not preprocessed to determine fields, we've got to guess and cannot use dissect - mutate { id => "mutate_add_tag_dissect_failure_missing_dhcp_bitmap" - add_tag => [ "_dissectfailure" ] } - } - - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_dhcp" + dissect { + id => "dissect_zeek_dhcp_with_all_fields" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } - } - ruby { - id => "ruby_zip_zeek_dhcp" - init => "@zeek_dhcp_field_names = [ 'ts', 'uid', 'orig_h', 'resp_h', 'orig_l2_addr', 'host_name', 'client_fqdn', 'domain', 'requested_ip', 'assigned_ip', 'lease_time', 'client_message', 'server_message', 'msg_types', 'duration', 'client_software', 'server_software' ]" - code => "event.set('[zeek_cols]', @zeek_dhcp_field_names.zip(event.get('[message]')).to_h)" + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][resp_h]} %{[zeek_cols][orig_l2_addr]} %{[zeek_cols][host_name]} %{[zeek_cols][client_fqdn]} %{[zeek_cols][domain]} %{[zeek_cols][requested_ip]} %{[zeek_cols][assigned_ip]} %{[zeek_cols][lease_time]} %{[zeek_cols][client_message]} %{[zeek_cols][server_message]} %{[zeek_cols][msg_types]} %{[zeek_cols][duration]} %{[zeek_cols][client_software]} %{[zeek_cols][server_software]}" + } } - } - - if ([zeek_cols][uid]) and ([zeek_cols][uid] != '(empty)') and ([zeek_cols][uid] != '') { - mutate { id => "mutate_split_zeek_dhcp_uids" - split => { "[zeek_cols][uid]" => "," } } - } + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_dhcp" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_dhcp" + init => "@zeek_dhcp_field_names = [ 'ts', 'uid', 'orig_h', 'resp_h', 'orig_l2_addr', 'host_name', 'client_fqdn', 'domain', 'requested_ip', 'assigned_ip', 'lease_time', 'client_message', 'server_message', 'msg_types', 'duration', 'client_software', 'server_software' ]" + code => "event.set('[zeek_cols]', @zeek_dhcp_field_names.zip(event.get('[message]')).to_h)" + } + } + mutate { id => "mutate_split_zeek_dhcp_msg_types" + split => { "[zeek_cols][msg_types]" => "," } } + if ([zeek_cols][uid]) and + ([zeek_cols][uid] != '(empty)') and + ([zeek_cols][uid] != '') { + mutate { id => "mutate_split_zeek_dhcp_uids" + split => { "[zeek_cols][uid]" => "," } } + } + } + if ([zeek_cols][uid] and [zeek_cols][uid][0]) { mutate { id => "mutate_add_field_zeek_dhcp_uids" add_field => { "[rootId]" => "%{[zeek_cols][uid][0]}" } } @@ -770,23 +727,25 @@ filter { # dnp3.log # https://docs.zeek.org/en/stable/scripts/base/protocols/dnp3/main.zeek.html#type-DNP3::Info - dissect { - id => "dissect_zeek_dnp3" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][fc_request]} %{[zeek_cols][fc_reply]} %{[zeek_cols][iin]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_dnp3" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_dnp3" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][fc_request]} %{[zeek_cols][fc_reply]} %{[zeek_cols][iin]}" + } } - ruby { - id => "ruby_zip_zeek_dnp3" - init => "@zeek_dnp3_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'fc_request', 'fc_reply', 'iin' ]" - code => "event.set('[zeek_cols]', @zeek_dnp3_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_dnp3" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_dnp3" + init => "@zeek_dnp3_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'fc_request', 'fc_reply', 'iin' ]" + code => "event.set('[zeek_cols]', @zeek_dnp3_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -801,23 +760,25 @@ filter { # dnp3_control.log # https://github.com/cisagov/ICSNPP - dissect { - id => "dissect_zeek_dnp3_control" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][block_type]} %{[zeek_cols][function_code]} %{[zeek_cols][index_number]} %{[zeek_cols][trip_control_code]} %{[zeek_cols][operation_type]} %{[zeek_cols][execute_count]} %{[zeek_cols][on_time]} %{[zeek_cols][off_time]} %{[zeek_cols][status_code]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_dnp3_control" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_dnp3_control" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][block_type]} %{[zeek_cols][function_code]} %{[zeek_cols][index_number]} %{[zeek_cols][trip_control_code]} %{[zeek_cols][operation_type]} %{[zeek_cols][execute_count]} %{[zeek_cols][on_time]} %{[zeek_cols][off_time]} %{[zeek_cols][status_code]}" + } } - ruby { - id => "ruby_zip_zeek_dnp3_control" - init => "@zeek_dnp3_control_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'block_type', 'function_code', 'index_number', 'trip_control_code', 'operation_type', 'execute_count', 'on_time', 'off_time', 'status_code' ]" - code => "event.set('[zeek_cols]', @zeek_dnp3_control_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_dnp3_control" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_dnp3_control" + init => "@zeek_dnp3_control_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'block_type', 'function_code', 'index_number', 'trip_control_code', 'operation_type', 'execute_count', 'on_time', 'off_time', 'status_code' ]" + code => "event.set('[zeek_cols]', @zeek_dnp3_control_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -832,23 +793,25 @@ filter { # dnp3_objects.log # https://github.com/cisagov/ICSNPP - dissect { - id => "dissect_zeek_dnp3_objects" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][function_code]} %{[zeek_cols][object_type]} %{[zeek_cols][object_count]} %{[zeek_cols][range_low]} %{[zeek_cols][range_high]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_dnp3_objects" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_dnp3_objects" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][function_code]} %{[zeek_cols][object_type]} %{[zeek_cols][object_count]} %{[zeek_cols][range_low]} %{[zeek_cols][range_high]}" + } } - ruby { - id => "ruby_zip_zeek_dnp3_objects" - init => "@zeek_dnp3_objects_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'function_code', 'object_type', 'object_count', 'range_low', 'range_high' ]" - code => "event.set('[zeek_cols]', @zeek_dnp3_objects_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_dnp3_objects" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_dnp3_objects" + init => "@zeek_dnp3_objects_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'function_code', 'object_type', 'object_count', 'range_low', 'range_high' ]" + code => "event.set('[zeek_cols]', @zeek_dnp3_objects_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -863,24 +826,29 @@ filter { # dns.log # https://docs.zeek.org/en/stable/scripts/base/protocols/dns/main.zeek.html#type-DNS::Info - dissect { - id => "dissect_zeek_dns" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][proto]} %{[zeek_cols][trans_id]} %{[zeek_cols][rtt]} %{[zeek_cols][query]} %{[zeek_cols][qclass]} %{[zeek_cols][qclass_name]} %{[zeek_cols][qtype]} %{[zeek_cols][qtype_name]} %{[zeek_cols][rcode]} %{[zeek_cols][rcode_name]} %{[zeek_cols][AA]} %{[zeek_cols][TC]} %{[zeek_cols][RD]} %{[zeek_cols][RA]} %{[zeek_cols][Z]} %{[zeek_cols][answers]} %{[zeek_cols][TTLs]} %{[zeek_cols][rejected]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_dns" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_dns" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][proto]} %{[zeek_cols][trans_id]} %{[zeek_cols][rtt]} %{[zeek_cols][query]} %{[zeek_cols][qclass]} %{[zeek_cols][qclass_name]} %{[zeek_cols][qtype]} %{[zeek_cols][qtype_name]} %{[zeek_cols][rcode]} %{[zeek_cols][rcode_name]} %{[zeek_cols][AA]} %{[zeek_cols][TC]} %{[zeek_cols][RD]} %{[zeek_cols][RA]} %{[zeek_cols][Z]} %{[zeek_cols][answers]} %{[zeek_cols][TTLs]} %{[zeek_cols][rejected]}" + } } - ruby { - id => "ruby_zip_zeek_dns" - init => "@zeek_dns_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto', 'trans_id', 'rtt', 'query', 'qclass', 'qclass_name', 'qtype', 'qtype_name', 'rcode', 'rcode_name', 'AA', 'TC', 'RD', 'RA', 'Z', 'answers', 'TTLs', 'rejected' ]" - code => "event.set('[zeek_cols]', @zeek_dns_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_dns" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_dns" + init => "@zeek_dns_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto', 'trans_id', 'rtt', 'query', 'qclass', 'qclass_name', 'qtype', 'qtype_name', 'rcode', 'rcode_name', 'AA', 'TC', 'RD', 'RA', 'Z', 'answers', 'TTLs', 'rejected' ]" + code => "event.set('[zeek_cols]', @zeek_dns_field_names.zip(event.get('[message]')).to_h)" + } } + mutate { id => "mutate_split_zeek_dns_commas" + split => { "[zeek_cols][TTLs]" => "," + "[zeek_cols][answers]" => "," } } } # remove C_ prefix from qclass_name @@ -897,23 +865,31 @@ filter { # dpd.log # https://docs.zeek.org/en/stable/scripts/base/frameworks/dpd/main.zeek.html#type-DPD::Info - dissect { - id => "dissect_zeek_dpd" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][proto]} %{[zeek_cols][service]} %{[zeek_cols][failure_reason]}" - } - } - if ("_dissectfailure" in [tags]) { + if ("_jsonparsesuccess" in [tags]) { mutate { - id => "mutate_split_zeek_dpd" + id => "mutate_rename_zeek_json_dpd_fields" + rename => { "[zeek_cols][analyzer]" => "[zeek_cols][service]" } + } + + } else { + dissect { + id => "dissect_zeek_dpd" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][proto]} %{[zeek_cols][service]} %{[zeek_cols][failure_reason]}" + } } - ruby { - id => "ruby_zip_zeek_dpd" - init => "@zeek_dpd_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto', 'service', 'failure_reason' ]" - code => "event.set('[zeek_cols]', @zeek_dpd_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_dpd" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_dpd" + init => "@zeek_dpd_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto', 'service', 'failure_reason' ]" + code => "event.set('[zeek_cols]', @zeek_dpd_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -938,23 +914,25 @@ filter { # https://github.com/cisagov/ICSNPP # - dissect { - id => "dissect_zeek_enip" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][enip_command_code]} %{[zeek_cols][enip_command]} %{[zeek_cols][length]} %{[zeek_cols][session_handle]} %{[zeek_cols][enip_status]} %{[zeek_cols][sender_context]} %{[zeek_cols][options]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_enip" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_enip" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][enip_command_code]} %{[zeek_cols][enip_command]} %{[zeek_cols][length]} %{[zeek_cols][session_handle]} %{[zeek_cols][enip_status]} %{[zeek_cols][sender_context]} %{[zeek_cols][options]}" + } } - ruby { - id => "ruby_zip_zeek_enip" - init => "@zeek_enip_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'enip_command', 'length', 'session_handle', 'enip_status', 'sender_context', 'options' ]" - code => "event.set('[zeek_cols]', @zeek_enip_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_enip" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_enip" + init => "@zeek_enip_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'enip_command_code', 'enip_command', 'length', 'session_handle', 'enip_status', 'sender_context', 'options' ]" + code => "event.set('[zeek_cols]', @zeek_enip_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -969,23 +947,36 @@ filter { # ecat_registers.log # https://github.com/cisagov/icsnpp-ethercat - dissect { - id => "dissect_zeek_ecat_registers" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][orig_l2_addr]} %{[zeek_cols][resp_l2_addr]} %{[zeek_cols][command]} %{[zeek_cols][server_addr]} %{[zeek_cols][register_type]} %{[zeek_cols][register_addr]} %{[zeek_cols][data]}" - } - } - if ("_dissectfailure" in [tags]) { + if ("_jsonparsesuccess" in [tags]) { mutate { - id => "mutate_split_zeek_ecat_registers" + id => "mutate_rename_zeek_json_ecat_registers_fields" + rename => { "[zeek_cols][srcmac]" => "[zeek_cols][orig_l2_addr]" } + rename => { "[zeek_cols][dstmac]" => "[zeek_cols][resp_l2_addr]" } + rename => { "[zeek_cols][Command]" => "[zeek_cols][command]" } + rename => { "[zeek_cols][Slave_Addr]" => "[zeek_cols][server_addr]" } + rename => { "[zeek_cols][Register_Type]" => "[zeek_cols][register_type]" } + rename => { "[zeek_cols][Register_Addr]" => "[zeek_cols][register_addr]" } + } + + } else { + dissect { + id => "dissect_zeek_ecat_registers" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][orig_l2_addr]} %{[zeek_cols][resp_l2_addr]} %{[zeek_cols][command]} %{[zeek_cols][server_addr]} %{[zeek_cols][register_type]} %{[zeek_cols][register_addr]} %{[zeek_cols][data]}" + } } - ruby { - id => "ruby_zip_zeek_ecat_registers" - init => "@zeek_ecat_registers_field_names = [ 'ts', 'orig_l2_addr', 'resp_l2_addr', 'command', 'server_addr', 'register_type', 'register_addr', 'data' ]" - code => "event.set('[zeek_cols]', @zeek_ecat_registers_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_ecat_registers" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_ecat_registers" + init => "@zeek_ecat_registers_field_names = [ 'ts', 'orig_l2_addr', 'resp_l2_addr', 'command', 'server_addr', 'register_type', 'register_addr', 'data' ]" + code => "event.set('[zeek_cols]', @zeek_ecat_registers_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -1002,23 +993,35 @@ filter { # ecat_log_address.log # https://github.com/cisagov/icsnpp-ethercat - dissect { - id => "dissect_zeek_ecat_log_address" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][orig_l2_addr]} %{[zeek_cols][resp_l2_addr]} %{[zeek_cols][log_addr]} %{[zeek_cols][length]} %{[zeek_cols][command]} %{[zeek_cols][data]}" - } - } - if ("_dissectfailure" in [tags]) { + if ("_jsonparsesuccess" in [tags]) { mutate { - id => "mutate_split_zeek_ecat_log_address" + id => "mutate_rename_zeek_json_ecat_log_address_fields" + rename => { "[zeek_cols][srcmac]" => "[zeek_cols][orig_l2_addr]" } + rename => { "[zeek_cols][dstmac]" => "[zeek_cols][resp_l2_addr]" } + rename => { "[zeek_cols][Log_Addr]" => "[zeek_cols][log_addr]" } + rename => { "[zeek_cols][Length]" => "[zeek_cols][length]" } + rename => { "[zeek_cols][Command]" => "[zeek_cols][command]" } + } + + } else { + dissect { + id => "dissect_zeek_ecat_log_address" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][orig_l2_addr]} %{[zeek_cols][resp_l2_addr]} %{[zeek_cols][log_addr]} %{[zeek_cols][length]} %{[zeek_cols][command]} %{[zeek_cols][data]}" + } } - ruby { - id => "ruby_zip_zeek_ecat_log_address" - init => "@zeek_ecat_log_address_field_names = [ 'ts', 'orig_l2_addr', 'resp_l2_addr', 'log_addr', 'length', 'command', 'data' ]" - code => "event.set('[zeek_cols]', @zeek_ecat_log_address_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_ecat_log_address" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_ecat_log_address" + init => "@zeek_ecat_log_address_field_names = [ 'ts', 'orig_l2_addr', 'resp_l2_addr', 'log_addr', 'length', 'command', 'data' ]" + code => "event.set('[zeek_cols]', @zeek_ecat_log_address_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -1035,23 +1038,31 @@ filter { # ecat_dev_info.log # https://github.com/cisagov/icsnpp-ethercat - dissect { - id => "dissect_zeek_ecat_dev_info" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][server_id]} %{[zeek_cols][revision]} %{[zeek_cols][dev_type]} %{[zeek_cols][build]} %{[zeek_cols][fmmucnt]} %{[zeek_cols][smcount]} %{[zeek_cols][ports]} %{[zeek_cols][dpram]} %{[zeek_cols][features]}" - } - } - if ("_dissectfailure" in [tags]) { + if ("_jsonparsesuccess" in [tags]) { mutate { - id => "mutate_split_zeek_ecat_dev_info" + id => "mutate_rename_zeek_json_ecat_dev_info_fields" + rename => { "[zeek_cols][slave_id]" => "[zeek_cols][server_id]" } + } + + } else { + dissect { + id => "dissect_zeek_ecat_dev_info" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][server_id]} %{[zeek_cols][revision]} %{[zeek_cols][dev_type]} %{[zeek_cols][build]} %{[zeek_cols][fmmucnt]} %{[zeek_cols][smcount]} %{[zeek_cols][ports]} %{[zeek_cols][dpram]} %{[zeek_cols][features]}" + } } - ruby { - id => "ruby_zip_zeek_ecat_dev_info" - init => "@zeek_ecat_dev_info_field_names = [ 'ts', 'server_id', 'revision', 'dev_type', 'build', 'fmmucnt', 'smcount', 'ports', 'dpram', 'features' ]" - code => "event.set('[zeek_cols]', @zeek_ecat_dev_info_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_ecat_dev_info" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_ecat_dev_info" + init => "@zeek_ecat_dev_info_field_names = [ 'ts', 'server_id', 'revision', 'dev_type', 'build', 'fmmucnt', 'smcount', 'ports', 'dpram', 'features' ]" + code => "event.set('[zeek_cols]', @zeek_ecat_dev_info_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -1068,23 +1079,36 @@ filter { # ecat_aoe_info.log # https://github.com/cisagov/icsnpp-ethercat - dissect { - id => "dissect_zeek_ecat_aoe_info" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][resp_l2_addr]} %{[zeek_cols][resp_port]} %{[zeek_cols][orig_l2_addr]} %{[zeek_cols][orig_port]} %{[zeek_cols][command]} %{[zeek_cols][state]} %{[zeek_cols][data]}" - } - } - if ("_dissectfailure" in [tags]) { + if ("_jsonparsesuccess" in [tags]) { mutate { - id => "mutate_split_zeek_ecat_aoe_info" + id => "mutate_rename_zeek_json_ecat_aoe_info_fields" + rename => { "[zeek_cols][targetid]" => "[zeek_cols][resp_l2_addr]" } + rename => { "[zeek_cols][targetport]" => "[zeek_cols][resp_port]" } + rename => { "[zeek_cols][senderid]" => "[zeek_cols][orig_l2_addr]" } + rename => { "[zeek_cols][senderport]" => "[zeek_cols][orig_port]" } + rename => { "[zeek_cols][cmd]" => "[zeek_cols][command]" } + rename => { "[zeek_cols][stateflags]" => "[zeek_cols][state]" } + } + + } else { + dissect { + id => "dissect_zeek_ecat_aoe_info" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][resp_l2_addr]} %{[zeek_cols][resp_port]} %{[zeek_cols][orig_l2_addr]} %{[zeek_cols][orig_port]} %{[zeek_cols][command]} %{[zeek_cols][state]} %{[zeek_cols][data]}" + } } - ruby { - id => "ruby_zip_zeek_ecat_aoe_info" - init => "@zeek_ecat_aoe_info_field_names = [ 'ts', 'resp_l2_addr', 'resp_port', 'orig_l2_addr', 'orig_port', 'command', 'state', 'data' ]" - code => "event.set('[zeek_cols]', @zeek_ecat_aoe_info_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_ecat_aoe_info" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_ecat_aoe_info" + init => "@zeek_ecat_aoe_info_field_names = [ 'ts', 'resp_l2_addr', 'resp_port', 'orig_l2_addr', 'orig_port', 'command', 'state', 'data' ]" + code => "event.set('[zeek_cols]', @zeek_ecat_aoe_info_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -1101,23 +1125,31 @@ filter { # ecat_coe_info.log # https://github.com/cisagov/icsnpp-ethercat - dissect { - id => "dissect_zeek_ecat_coe_info" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][number]} %{[zeek_cols][type]} %{[zeek_cols][req_resp]} %{[zeek_cols][index]} %{[zeek_cols][subindex]} %{[zeek_cols][dataoffset]}" - } - } - if ("_dissectfailure" in [tags]) { + if ("_jsonparsesuccess" in [tags]) { mutate { - id => "mutate_split_zeek_ecat_coe_info" + id => "mutate_rename_zeek_json_ecat_coe_info_fields" + rename => { "[zeek_cols][Type]" => "[zeek_cols][type]" } + } + + } else { + dissect { + id => "dissect_zeek_ecat_coe_info" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][number]} %{[zeek_cols][type]} %{[zeek_cols][req_resp]} %{[zeek_cols][index]} %{[zeek_cols][subindex]} %{[zeek_cols][dataoffset]}" + } } - ruby { - id => "ruby_zip_zeek_ecat_coe_info" - init => "@zeek_ecat_coe_info_field_names = [ 'ts', 'number', 'type', 'req_resp', 'index', 'subindex', 'dataoffset' ]" - code => "event.set('[zeek_cols]', @zeek_ecat_coe_info_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_ecat_coe_info" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_ecat_coe_info" + init => "@zeek_ecat_coe_info_field_names = [ 'ts', 'number', 'type', 'req_resp', 'index', 'subindex', 'dataoffset' ]" + code => "event.set('[zeek_cols]', @zeek_ecat_coe_info_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -1134,23 +1166,31 @@ filter { # ecat_foe_info.log # https://github.com/cisagov/icsnpp-ethercat - dissect { - id => "dissect_zeek_ecat_foe_info" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][opcode]} %{[zeek_cols][reserved]} %{[zeek_cols][packet_num]} %{[zeek_cols][error_code]} %{[zeek_cols][filename]} %{[zeek_cols][data]}" - } - } - if ("_dissectfailure" in [tags]) { + if ("_jsonparsesuccess" in [tags]) { mutate { - id => "mutate_split_zeek_ecat_foe_info" + id => "mutate_rename_zeek_json_ecat_foe_info_fields" + rename => { "[zeek_cols][opCode]" => "[zeek_cols][opcode]" } + } + + } else { + dissect { + id => "dissect_zeek_ecat_foe_info" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][opcode]} %{[zeek_cols][reserved]} %{[zeek_cols][packet_num]} %{[zeek_cols][error_code]} %{[zeek_cols][filename]} %{[zeek_cols][data]}" + } } - ruby { - id => "ruby_zip_zeek_ecat_foe_info" - init => "@zeek_ecat_foe_info_field_names = [ 'ts', 'opcode', 'reserved', 'packet_num', 'error_code', 'filename', 'data' ]" - code => "event.set('[zeek_cols]', @zeek_ecat_foe_info_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_ecat_foe_info" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_ecat_foe_info" + init => "@zeek_ecat_foe_info_field_names = [ 'ts', 'opcode', 'reserved', 'packet_num', 'error_code', 'filename', 'data' ]" + code => "event.set('[zeek_cols]', @zeek_ecat_foe_info_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -1167,23 +1207,32 @@ filter { # ecat_soe_info.log # https://github.com/cisagov/icsnpp-ethercat - dissect { - id => "dissect_zeek_ecat_soe_info" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][opcode]} %{[zeek_cols][incomplete]} %{[zeek_cols][error]} %{[zeek_cols][drive_num]} %{[zeek_cols][element]} %{[zeek_cols][index]}" - } - } - if ("_dissectfailure" in [tags]) { + if ("_jsonparsesuccess" in [tags]) { mutate { - id => "mutate_split_zeek_ecat_soe_info" + id => "mutate_rename_zeek_json_ecat_soe_info_fields" + rename => { "[zeek_cols][opCode]" => "[zeek_cols][opcode]" } + rename => { "[zeek_cols][element_flags]" => "[zeek_cols][element]" } + } + + } else { + dissect { + id => "dissect_zeek_ecat_soe_info" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][opcode]} %{[zeek_cols][incomplete]} %{[zeek_cols][error]} %{[zeek_cols][drive_num]} %{[zeek_cols][element]} %{[zeek_cols][index]}" + } } - ruby { - id => "ruby_zip_zeek_ecat_soe_info" - init => "@zeek_ecat_soe_info_field_names = [ 'ts', 'opcode', 'incomplete', 'error', 'drive_num', 'element', 'index' ]" - code => "event.set('[zeek_cols]', @zeek_ecat_soe_info_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_ecat_soe_info" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_ecat_soe_info" + init => "@zeek_ecat_soe_info_field_names = [ 'ts', 'opcode', 'incomplete', 'error', 'drive_num', 'element', 'index' ]" + code => "event.set('[zeek_cols]', @zeek_ecat_soe_info_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -1200,27 +1249,39 @@ filter { # ecat_arp_info.log # https://github.com/cisagov/icsnpp-ethercat # - # NOTE: I currently have this disabled by including ecat_arp_info in drop_zeek_ignored_source - # above, as it is including ALL arps and not just those from ethercat traffic which can - # be misleading (i.e., indicating ecat traffic where there is none) + # NOTE: I currently have this disabled via policy hook in local.zeek, as it is including ALL arps and + # not just those from ethercat traffic which can be misleading (i.e., indicating ecat traffic where there is none) - dissect { - id => "dissect_zeek_ecat_arp_info" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][arp_type]} %{[zeek_cols][orig_l2_addr]} %{[zeek_cols][resp_l2_addr]} %{[zeek_cols][orig_proto_addr]} %{[zeek_cols][orig_hw_addr]} %{[zeek_cols][resp_proto_addr]} %{[zeek_cols][resp_hw_addr]}" - } - } - if ("_dissectfailure" in [tags]) { + if ("_jsonparsesuccess" in [tags]) { mutate { - id => "mutate_split_zeek_ecat_arp_info" + id => "mutate_rename_zeek_json_ecat_arp_info_fields" + rename => { "[zeek_cols][mac_src]" => "[zeek_cols][orig_l2_addr]" } + rename => { "[zeek_cols][mac_dst]" => "[zeek_cols][resp_l2_addr]" } + rename => { "[zeek_cols][SPA]" => "[zeek_cols][orig_proto_addr]" } + rename => { "[zeek_cols][SHA]" => "[zeek_cols][orig_hw_addr]" } + rename => { "[zeek_cols][TPA]" => "[zeek_cols][resp_proto_addr]" } + rename => { "[zeek_cols][THA]" => "[zeek_cols][resp_hw_addr]" } + } + + } else { + dissect { + id => "dissect_zeek_ecat_arp_info" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][arp_type]} %{[zeek_cols][orig_l2_addr]} %{[zeek_cols][resp_l2_addr]} %{[zeek_cols][orig_proto_addr]} %{[zeek_cols][orig_hw_addr]} %{[zeek_cols][resp_proto_addr]} %{[zeek_cols][resp_hw_addr]}" + } } - ruby { - id => "ruby_zip_zeek_ecat_arp_info" - init => "@zeek_ecat_arp_info_field_names = [ 'ts', 'arp_type', 'orig_l2_addr', 'resp_l2_addr', 'orig_proto_addr', 'orig_hw_addr', 'resp_proto_addr', 'resp_hw_addr' ]" - code => "event.set('[zeek_cols]', @zeek_ecat_arp_info_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_ecat_arp_info" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_ecat_arp_info" + init => "@zeek_ecat_arp_info_field_names = [ 'ts', 'arp_type', 'orig_l2_addr', 'resp_l2_addr', 'orig_proto_addr', 'orig_hw_addr', 'resp_proto_addr', 'resp_hw_addr' ]" + code => "event.set('[zeek_cols]', @zeek_ecat_arp_info_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -1238,87 +1299,41 @@ filter { # files.log # https://docs.zeek.org/en/stable/scripts/base/frameworks/files/main.zeek.html#type-Files::Info - if ([@metadata][zeek_fields_bitmap] and [@metadata][zeek_fields_bitmap_version]) { - - # bitmap files.log field configuration version 1 - # - # all fields (v5.1+, with extracted file size info and spicy-zip's ftime) : 0x0FFFFFFF / 268435455 - - # bitmap files.log field configuration version 0 - # all fields (< v5.1+, with extracted file size info and spicy-zip's ftime) : 0x03FFFFFF / 67108863 - - if ([@metadata][zeek_fields_bitmap_version] == 1) { - - if ([@metadata][zeek_fields_bitmap] == 268435455) { - - dissect { - id => "dissect_zeek_v51_files_with_all_fields" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][fuid]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][source]} %{[zeek_cols][depth]} %{[zeek_cols][analyzers]} %{[zeek_cols][mime_type]} %{[zeek_cols][filename]} %{[zeek_cols][duration]} %{[zeek_cols][local_orig]} %{[zeek_cols][is_orig]} %{[zeek_cols][seen_bytes]} %{[zeek_cols][total_bytes]} %{[zeek_cols][missing_bytes]} %{[zeek_cols][overflow_bytes]} %{[zeek_cols][timedout]} %{[zeek_cols][parent_fuid]} %{[zeek_cols][md5]} %{[zeek_cols][sha1]} %{[zeek_cols][sha256]} %{[zeek_cols][extracted]} %{[zeek_cols][extracted_cutoff]} %{[zeek_cols][extracted_size]} %{[zeek_cols][ftime]}" - } - } - - } else { - # who knows? the files.log preprocessed bitmap is not one we're expecting, we've got to guess and cannot use dissect - mutate { id => "mutate_add_tag_dissect_failure_unknown_files_v51_bitmap" - add_tag => [ "_dissectfailure" ] } + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_v51_files_with_all_fields" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][fuid]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][source]} %{[zeek_cols][depth]} %{[zeek_cols][analyzers]} %{[zeek_cols][mime_type]} %{[zeek_cols][filename]} %{[zeek_cols][duration]} %{[zeek_cols][local_orig]} %{[zeek_cols][is_orig]} %{[zeek_cols][seen_bytes]} %{[zeek_cols][total_bytes]} %{[zeek_cols][missing_bytes]} %{[zeek_cols][overflow_bytes]} %{[zeek_cols][timedout]} %{[zeek_cols][parent_fuid]} %{[zeek_cols][md5]} %{[zeek_cols][sha1]} %{[zeek_cols][sha256]} %{[zeek_cols][extracted]} %{[zeek_cols][extracted_cutoff]} %{[zeek_cols][extracted_size]} %{[zeek_cols][ftime]}" } - - } else if ([@metadata][zeek_fields_bitmap_version] == 0) { - - if ([@metadata][zeek_fields_bitmap] == 67108863) { - - dissect { - id => "dissect_zeek_files_with_all_fields" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][fuid]} %{[zeek_cols][tx_hosts]} %{[zeek_cols][rx_hosts]} %{[zeek_cols][conn_uids]} %{[zeek_cols][source]} %{[zeek_cols][depth]} %{[zeek_cols][analyzers]} %{[zeek_cols][mime_type]} %{[zeek_cols][filename]} %{[zeek_cols][duration]} %{[zeek_cols][local_orig]} %{[zeek_cols][is_orig]} %{[zeek_cols][seen_bytes]} %{[zeek_cols][total_bytes]} %{[zeek_cols][missing_bytes]} %{[zeek_cols][overflow_bytes]} %{[zeek_cols][timedout]} %{[zeek_cols][parent_fuid]} %{[zeek_cols][md5]} %{[zeek_cols][sha1]} %{[zeek_cols][sha256]} %{[zeek_cols][extracted]} %{[zeek_cols][extracted_cutoff]} %{[zeek_cols][extracted_size]} %{[zeek_cols][ftime]}" - } - } - - } else { - # who knows? the files.log preprocessed bitmap is not one we're expecting, we've got to guess and cannot use dissect - mutate { id => "mutate_add_tag_dissect_failure_unknown_files_bitmap" - add_tag => [ "_dissectfailure" ] } + } + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_files" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_files" + init => "@zeek_files_field_names = [ 'ts', 'fuid', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'source', 'depth', 'analyzers', 'mime_type', 'filename', 'duration', 'local_orig', 'is_orig', 'seen_bytes', 'total_bytes', 'missing_bytes', 'overflow_bytes', 'timedout', 'parent_fuid', 'md5', 'sha1', 'sha256', 'extracted', 'extracted_cutoff', 'extracted_size', 'ftime' ]" + code => "event.set('[zeek_cols]', @zeek_files_field_names.zip(event.get('[message]')).to_h)" } - - } else { - # who knows? the files.log preprocessed bitmap field list version is not one we're expecting, we've got to guess and cannot use dissect - mutate { id => "mutate_add_tag_dissect_failure_unknown_bitmap_files_version" - add_tag => [ "_dissectfailure" ] } } - - } else { - # who knows? the files.log was not preprocessed to determine fields, we've got to guess and cannot use dissect - mutate { id => "mutate_add_tag_dissect_failure_missing_files_bitmap" - add_tag => [ "_dissectfailure" ] } - } - - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_files" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + if ([zeek_cols][conn_uids]) and ([zeek_cols][conn_uids] != '(empty)') and ([zeek_cols][conn_uids] != '') { + mutate { id => "mutate_split_zeek_files_conn_uids" + split => { "[zeek_cols][conn_uids]" => "," } } } - ruby { - id => "ruby_zip_zeek_files" - init => "@zeek_files_field_names = [ 'ts', 'fuid', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'source', 'depth', 'analyzers', 'mime_type', 'filename', 'duration', 'local_orig', 'is_orig', 'seen_bytes', 'total_bytes', 'missing_bytes', 'overflow_bytes', 'timedout', 'parent_fuid', 'md5', 'sha1', 'sha256', 'extracted', 'extracted_cutoff', 'extracted_size', 'ftime' ]" - code => "event.set('[zeek_cols]', @zeek_files_field_names.zip(event.get('[message]')).to_h)" + if ([zeek_cols][tx_hosts]) and ([zeek_cols][tx_hosts] != '(empty)') and ([zeek_cols][tx_hosts] != '') { + mutate { id => "mutate_split_zeek_files_tx_hosts" + split => { "[zeek_cols][tx_hosts]" => "," } } } - } - - if ([zeek_cols][conn_uids]) and ([zeek_cols][conn_uids] != '(empty)') and ([zeek_cols][conn_uids] != '') { - mutate { id => "mutate_split_zeek_files_conn_uids" - split => { "[zeek_cols][conn_uids]" => "," } } - } - if ([zeek_cols][tx_hosts]) and ([zeek_cols][tx_hosts] != '(empty)') and ([zeek_cols][tx_hosts] != '') { - mutate { id => "mutate_split_zeek_files_tx_hosts" - split => { "[zeek_cols][tx_hosts]" => "," } } - } - if ([zeek_cols][rx_hosts]) and ([zeek_cols][rx_hosts] != '(empty)') and ([zeek_cols][rx_hosts] != '') { - mutate { id => "mutate_split_zeek_files_rx_hosts" - split => { "[zeek_cols][rx_hosts]" => "," } } + if ([zeek_cols][rx_hosts]) and ([zeek_cols][rx_hosts] != '(empty)') and ([zeek_cols][rx_hosts] != '') { + mutate { id => "mutate_split_zeek_files_rx_hosts" + split => { "[zeek_cols][rx_hosts]" => "," } } + } + mutate { id => "mutate_split_zeek_files_parent_fuid_and_analyzers" + split => { "[zeek_cols][parent_fuid]" => "," + "[zeek_cols][analyzers]" => "," } } } if ([zeek_cols][conn_uids] and [zeek_cols][conn_uids][0]) { @@ -1350,26 +1365,28 @@ filter { # ftp.log # https://docs.zeek.org/en/stable/scripts/base/protocols/ftp/info.zeek.html#type-FTP::Info - dissect { - id => "dissect_zeek_ftp" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][user]} %{[zeek_cols][password]} %{[zeek_cols][command]} %{[zeek_cols][arg]} %{[zeek_cols][mime_type]} %{[zeek_cols][file_size]} %{[zeek_cols][reply_code]} %{[zeek_cols][reply_msg]} %{[zeek_cols][data_channel_passive]} %{[zeek_cols][data_channel_orig_h]} %{[zeek_cols][data_channel_resp_h]} %{[zeek_cols][data_channel_resp_p]} %{[zeek_cols][fuid]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_ftp" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_ftp" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } - } - ruby { - id => "ruby_zip_zeek_ftp" - init => "@zeek_ftp_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'user', 'password', 'command', 'arg', 'mime_type', 'file_size', 'reply_code', 'reply_msg', 'data_channel_passive', 'data_channel_orig_h', 'data_channel_resp_h', 'data_channel_resp_p', 'fuid' ]" - code => "event.set('[zeek_cols]', @zeek_ftp_field_names.zip(event.get('[message]')).to_h)" + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][user]} %{[zeek_cols][password]} %{[zeek_cols][command]} %{[zeek_cols][arg]} %{[zeek_cols][mime_type]} %{[zeek_cols][file_size]} %{[zeek_cols][reply_code]} %{[zeek_cols][reply_msg]} %{[zeek_cols][data_channel][passive]} %{[zeek_cols][data_channel][orig_h]} %{[zeek_cols][data_channel][resp_h]} %{[zeek_cols][data_channel][resp_p]} %{[zeek_cols][fuid]}" + } } - } - + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_ftp" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_ftp" + init => "@zeek_ftp_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'user', 'password', 'command', 'arg', 'mime_type', 'file_size', 'reply_code', 'reply_msg', 'data_channel.passive', 'data_channel.orig_h', 'data_channel.resp_h', 'data_channel.resp_p', 'fuid' ]" + code => "event.set('[zeek_cols]', @zeek_ftp_field_names.zip(event.get('[message]')).to_h)" + } + } + } + mutate { id => "mutate_add_fields_zeek_ftp" add_field => { @@ -1383,23 +1400,31 @@ filter { # genisys.log # https://github.com/cisagov/icsnpp-genisys - dissect { - id => "dissect_zeek_genisys" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][proto]} %{[zeek_cols][header]} %{[zeek_cols][server]} %{[zeek_cols][direction]} %{[zeek_cols][crc_transmitted]} %{[zeek_cols][crc_calculated]} %{[zeek_cols][payload_raw]}" - } - } - if ("_dissectfailure" in [tags]) { + if ("_jsonparsesuccess" in [tags]) { mutate { - id => "mutate_split_zeek_genisys" + id => "mutate_rename_zeek_json_genisys_fields" + rename => { "[zeek_cols][payload]" => "[zeek_cols][payload_raw]" } + } + + } else { + dissect { + id => "dissect_zeek_genisys" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][proto]} %{[zeek_cols][header]} %{[zeek_cols][server]} %{[zeek_cols][direction]} %{[zeek_cols][crc_transmitted]} %{[zeek_cols][crc_calculated]} %{[zeek_cols][payload_raw]}" + } } - ruby { - id => "ruby_zip_zeek_genisys" - init => "@zeek_genisys_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto', 'header', 'server', 'direction', 'crc_transmitted', 'crc_calculated', 'payload_raw' ]" - code => "event.set('[zeek_cols]', @zeek_genisys_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_genisys" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_genisys" + init => "@zeek_genisys_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto', 'header', 'server', 'direction', 'crc_transmitted', 'crc_calculated', 'payload_raw' ]" + code => "event.set('[zeek_cols]', @zeek_genisys_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -1416,23 +1441,25 @@ filter { # gquic.log # https://github.com/salesforce/GQUIC_Protocol_Analyzer/blob/master/scripts/Salesforce/GQUIC/main.bro - dissect { - id => "dissect_zeek_gquic" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][version]} %{[zeek_cols][server_name]} %{[zeek_cols][user_agent]} %{[zeek_cols][tag_count]} %{[zeek_cols][cyu]} %{[zeek_cols][cyutags]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_gquic" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_gquic" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][version]} %{[zeek_cols][server_name]} %{[zeek_cols][user_agent]} %{[zeek_cols][tag_count]} %{[zeek_cols][cyu]} %{[zeek_cols][cyutags]}" + } } - ruby { - id => "ruby_zip_zeek_gquic" - init => "@zeek_gquic_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'version', 'server_name', 'user_agent', 'tag_count', 'cyu', 'cyutags' ]" - code => "event.set('[zeek_cols]', @zeek_gquic_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_gquic" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_gquic" + init => "@zeek_gquic_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'version', 'server_name', 'user_agent', 'tag_count', 'cyu', 'cyutags' ]" + code => "event.set('[zeek_cols]', @zeek_gquic_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -1449,71 +1476,41 @@ filter { # http.log # https://docs.zeek.org/en/stable/scripts/base/protocols/http/main.zeek.html#type-HTTP::Info - if ([@metadata][zeek_fields_bitmap] and [@metadata][zeek_fields_bitmap_version]) { - - # bitmap http.log field configuration version 0 - # all fields: 0x7FFFFFFFF / 34359738367 - # minus fields added by zeek-sniffpass: 0x03FFFFFFF / 1073741823 - # minus origin and fields added by zeek-sniffpass: 0x03FFFDFFF / 1073733631 - - if ([@metadata][zeek_fields_bitmap_version] == 0) { - - if ([@metadata][zeek_fields_bitmap] == 34359738367) { - dissect { - id => "dissect_zeek_http_with_all_fields" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][trans_depth]} %{[zeek_cols][method]} %{[zeek_cols][host]} %{[zeek_cols][uri]} %{[zeek_cols][referrer]} %{[zeek_cols][version]} %{[zeek_cols][user_agent]} %{[zeek_cols][origin]} %{[zeek_cols][request_body_len]} %{[zeek_cols][response_body_len]} %{[zeek_cols][status_code]} %{[zeek_cols][status_msg]} %{[zeek_cols][info_code]} %{[zeek_cols][info_msg]} %{[zeek_cols][tags]} %{[zeek_cols][user]} %{[zeek_cols][password]} %{[zeek_cols][proxied]} %{[zeek_cols][orig_fuids]} %{[zeek_cols][orig_filenames]} %{[zeek_cols][orig_mime_types]} %{[zeek_cols][resp_fuids]} %{[zeek_cols][resp_filenames]} %{[zeek_cols][resp_mime_types]} %{[zeek_cols][post_username]} %{[zeek_cols][post_password_plain]} %{[zeek_cols][post_password_md5]} %{[zeek_cols][post_password_sha1]} %{[zeek_cols][post_password_sha256]}" - } - } - - } else if ([@metadata][zeek_fields_bitmap] == 1073741823) { - dissect { - id => "dissect_zeek_http_with_all_fields_minus_sniffpass" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][trans_depth]} %{[zeek_cols][method]} %{[zeek_cols][host]} %{[zeek_cols][uri]} %{[zeek_cols][referrer]} %{[zeek_cols][version]} %{[zeek_cols][user_agent]} %{[zeek_cols][origin]} %{[zeek_cols][request_body_len]} %{[zeek_cols][response_body_len]} %{[zeek_cols][status_code]} %{[zeek_cols][status_msg]} %{[zeek_cols][info_code]} %{[zeek_cols][info_msg]} %{[zeek_cols][tags]} %{[zeek_cols][user]} %{[zeek_cols][password]} %{[zeek_cols][proxied]} %{[zeek_cols][orig_fuids]} %{[zeek_cols][orig_filenames]} %{[zeek_cols][orig_mime_types]} %{[zeek_cols][resp_fuids]} %{[zeek_cols][resp_filenames]} %{[zeek_cols][resp_mime_types]}" - } - } - - } else if ([@metadata][zeek_fields_bitmap] == 1073733631) { - dissect { - id => "dissect_zeek_http_with_all_fields_minus_origin" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][trans_depth]} %{[zeek_cols][method]} %{[zeek_cols][host]} %{[zeek_cols][uri]} %{[zeek_cols][referrer]} %{[zeek_cols][version]} %{[zeek_cols][user_agent]} %{[zeek_cols][request_body_len]} %{[zeek_cols][response_body_len]} %{[zeek_cols][status_code]} %{[zeek_cols][status_msg]} %{[zeek_cols][info_code]} %{[zeek_cols][info_msg]} %{[zeek_cols][tags]} %{[zeek_cols][user]} %{[zeek_cols][password]} %{[zeek_cols][proxied]} %{[zeek_cols][orig_fuids]} %{[zeek_cols][orig_filenames]} %{[zeek_cols][orig_mime_types]} %{[zeek_cols][resp_fuids]} %{[zeek_cols][resp_filenames]} %{[zeek_cols][resp_mime_types]}" - } - } - - } else { - # who knows? the http.log preprocessed bitmap is not one we're expecting, we've got to guess and cannot use dissect - mutate { id => "mutate_add_tag_dissect_failure_unknown_http_bitmap" - add_tag => [ "_dissectfailure" ] } - } - - } else { - # who knows? the http.log preprocessed bitmap field list version is not one we're expecting, we've got to guess and cannot use dissect - mutate { id => "mutate_add_tag_dissect_failure_unknown_bitmap_http_version" - add_tag => [ "_dissectfailure" ] } + if ("_jsonparsesuccess" in [tags]) { + mutate { + id => "mutate_rename_zeek_json_http_fields" + rename => { "[zeek_cols][username]" => "[zeek_cols][user]" } } } else { - # who knows? the http.log was not preprocessed to determine fields, we've got to guess and cannot use dissect - mutate { id => "mutate_add_tag_dissect_failure_missing_http_bitmap" - add_tag => [ "_dissectfailure" ] } - } - - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_http" + dissect { + id => "dissect_zeek_http_with_all_fields" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][trans_depth]} %{[zeek_cols][method]} %{[zeek_cols][host]} %{[zeek_cols][uri]} %{[zeek_cols][referrer]} %{[zeek_cols][version]} %{[zeek_cols][user_agent]} %{[zeek_cols][origin]} %{[zeek_cols][request_body_len]} %{[zeek_cols][response_body_len]} %{[zeek_cols][status_code]} %{[zeek_cols][status_msg]} %{[zeek_cols][info_code]} %{[zeek_cols][info_msg]} %{[zeek_cols][tags]} %{[zeek_cols][user]} %{[zeek_cols][password]} %{[zeek_cols][proxied]} %{[zeek_cols][orig_fuids]} %{[zeek_cols][orig_filenames]} %{[zeek_cols][orig_mime_types]} %{[zeek_cols][resp_fuids]} %{[zeek_cols][resp_filenames]} %{[zeek_cols][resp_mime_types]} %{[zeek_cols][post_username]} %{[zeek_cols][post_password_plain]} %{[zeek_cols][post_password_md5]} %{[zeek_cols][post_password_sha1]} %{[zeek_cols][post_password_sha256]}" + } } - ruby { - id => "ruby_zip_zeek_http" - init => "@zeek_http_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'trans_depth', 'method', 'host', 'uri', 'referrer', 'version', 'user_agent', 'origin', 'request_body_len', 'response_body_len', 'status_code', 'status_msg', 'info_code', 'info_msg', 'tags', 'user', 'password', 'proxied', 'orig_fuids', 'orig_filenames', 'orig_mime_types', 'resp_fuids', 'resp_filenames', 'resp_mime_types', 'post_username', 'post_password_plain', 'post_password_md5', 'post_password_sha1', 'post_password_sha256' ]" - code => "event.set('[zeek_cols]', @zeek_http_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_http" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_http" + init => "@zeek_http_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'trans_depth', 'method', 'host', 'uri', 'referrer', 'version', 'user_agent', 'origin', 'request_body_len', 'response_body_len', 'status_code', 'status_msg', 'info_code', 'info_msg', 'tags', 'user', 'password', 'proxied', 'orig_fuids', 'orig_filenames', 'orig_mime_types', 'resp_fuids', 'resp_filenames', 'resp_mime_types', 'post_username', 'post_password_plain', 'post_password_md5', 'post_password_sha1', 'post_password_sha256' ]" + code => "event.set('[zeek_cols]', @zeek_http_field_names.zip(event.get('[message]')).to_h)" + } } + mutate { id => "mutate_split_zeek_http_commas" + split => { "[zeek_cols][orig_fuids]" => "," + "[zeek_cols][orig_filenames]" => "," + "[zeek_cols][orig_mime_types]" => "," + "[zeek_cols][resp_fuids]" => "," + "[zeek_cols][resp_filenames]" => "," + "[zeek_cols][resp_mime_types]" => "," + "[zeek_cols][proxied]" => "," + "[zeek_cols][tags]" => "," } } } mutate { @@ -1529,49 +1526,82 @@ filter { # intel.log # https://docs.zeek.org/en/stable/scripts/base/frameworks/intel/main.zeek.html#type-Intel::Info - dissect { - id => "dissect_zeek_intel" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][seen_indicator]} %{[zeek_cols][seen_indicator_type]} %{[zeek_cols][seen_where]} %{[zeek_cols][seen_node]} %{[zeek_cols][matched]} %{[zeek_cols][sources]} %{[zeek_cols][fuid]} %{[zeek_cols][file_mime_type]} %{[zeek_cols][file_desc]} %{[zeek_cols][cif_tags]} %{[zeek_cols][cif_confidence]} %{[zeek_cols][cif_source]} %{[zeek_cols][cif_description]} %{[zeek_cols][cif_firstseen]} %{[zeek_cols][cif_lastseen]}" - } - } - if ("_dissectfailure" in [tags]) { + if ("_jsonparsesuccess" in [tags]) { mutate { - id => "mutate_split_zeek_intel" + id => "mutate_rename_zeek_json_intel_fields" + rename => { "[zeek_cols][cif.firstseen]" => "[zeek_cols][cif_firstseen]" } + rename => { "[zeek_cols][cif.lastseen]" => "[zeek_cols][cif_lastseen]" } + rename => { "[zeek_cols][cif.tags]" => "[zeek_cols][cif_tags]" } + rename => { "[zeek_cols][seen.indicator]" => "[zeek_cols][seen_indicator]" } + rename => { "[zeek_cols][seen.indicator_type]" => "[zeek_cols][seen_indicator_type]" } + rename => { "[zeek_cols][seen.node]" => "[zeek_cols][seen_node]" } + rename => { "[zeek_cols][seen.where]" => "[zeek_cols][seen_where]" } + } + + } else { + dissect { + id => "dissect_zeek_intel" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][seen_indicator]} %{[zeek_cols][seen_indicator_type]} %{[zeek_cols][seen_where]} %{[zeek_cols][seen_node]} %{[zeek_cols][matched]} %{[zeek_cols][sources]} %{[zeek_cols][fuid]} %{[zeek_cols][file_mime_type]} %{[zeek_cols][file_desc]} %{[zeek_cols][cif_tags]} %{[zeek_cols][cif_confidence]} %{[zeek_cols][cif_source]} %{[zeek_cols][cif_description]} %{[zeek_cols][cif_firstseen]} %{[zeek_cols][cif_lastseen]}" + } } - ruby { - id => "ruby_zip_zeek_intel" - init => "@zeek_intel_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'seen_indicator', 'seen_indicator_type', 'seen_where', 'seen_node', 'matched', 'sources', 'fuid', 'file_mime_type', 'file_desc', 'cif_tags', 'cif_confidence', 'cif_source', 'cif_description', 'cif_firstseen', 'cif_lastseen' ]" - code => "event.set('[zeek_cols]', @zeek_intel_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_intel" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_intel" + init => "@zeek_intel_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'seen_indicator', 'seen_indicator_type', 'seen_where', 'seen_node', 'matched', 'sources', 'fuid', 'file_mime_type', 'file_desc', 'cif_tags', 'cif_confidence', 'cif_source', 'cif_description', 'cif_firstseen', 'cif_lastseen' ]" + code => "event.set('[zeek_cols]', @zeek_intel_field_names.zip(event.get('[message]')).to_h)" + } } + mutate { id => "mutate_split_zeek_intel_commas" + split => { "[zeek_cols][sources]" => "," + "[zeek_cols][matched]" => "," } } } + # For some reason, even in JSON, I have cif_tags strings like: + # Network activity,osint:source-type=\"block-or-filter-list\" + # so whatever reason it's not already an array. Split it here. + mutate { id => "mutate_split_zeek_intel_cif_tags" + split => { "[zeek_cols][cif_tags]" => "," } } + } else if ([log_source] == "ipsec") { ############################################################################################################################# # ipsec.log # https://github.com/corelight/zeek-spicy-ipsec/blob/master/analyzer/main.zeek - dissect { - id => "dissect_zeek_ipsec" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][initiator_spi]} %{[zeek_cols][responder_spi]} %{[zeek_cols][maj_ver]} %{[zeek_cols][min_ver]} %{[zeek_cols][exchange_type]} %{[zeek_cols][flag_e]} %{[zeek_cols][flag_c]} %{[zeek_cols][flag_a]} %{[zeek_cols][flag_i]} %{[zeek_cols][flag_v]} %{[zeek_cols][flag_r]} %{[zeek_cols][message_id]} %{[zeek_cols][vendor_ids]} %{[zeek_cols][notify_messages]} %{[zeek_cols][transforms]} %{[zeek_cols][ke_dh_groups]} %{[zeek_cols][proposals]} %{[zeek_cols][protocol_id]} %{[zeek_cols][certificates]} %{[zeek_cols][transform_attributes]} %{[zeek_cols][length]} %{[zeek_cols][hash]} %{[zeek_cols][doi]} %{[zeek_cols][situation]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_ipsec" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_ipsec" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][initiator_spi]} %{[zeek_cols][responder_spi]} %{[zeek_cols][maj_ver]} %{[zeek_cols][min_ver]} %{[zeek_cols][exchange_type]} %{[zeek_cols][flag_e]} %{[zeek_cols][flag_c]} %{[zeek_cols][flag_a]} %{[zeek_cols][flag_i]} %{[zeek_cols][flag_v]} %{[zeek_cols][flag_r]} %{[zeek_cols][message_id]} %{[zeek_cols][vendor_ids]} %{[zeek_cols][notify_messages]} %{[zeek_cols][transforms]} %{[zeek_cols][ke_dh_groups]} %{[zeek_cols][proposals]} %{[zeek_cols][protocol_id]} %{[zeek_cols][certificates]} %{[zeek_cols][transform_attributes]} %{[zeek_cols][length]} %{[zeek_cols][hash]} %{[zeek_cols][doi]} %{[zeek_cols][situation]}" + } } - ruby { - id => "ruby_zip_zeek_ipsec" - init => "@zeek_ipsec_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'is_orig', 'initiator_spi', 'responder_spi', 'maj_ver', 'min_ver', 'exchange_type', 'flag_e', 'flag_c', 'flag_a', 'flag_i', 'flag_v', 'flag_r', 'message_id', 'vendor_ids', 'notify_messages', 'transforms', 'ke_dh_groups', 'proposals', 'protocol_id', 'certificates', 'transform_attributes', 'length', 'hash', 'doi', 'situation' ]" - code => "event.set('[zeek_cols]', @zeek_ipsec_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_ipsec" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_ipsec" + init => "@zeek_ipsec_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'is_orig', 'initiator_spi', 'responder_spi', 'maj_ver', 'min_ver', 'exchange_type', 'flag_e', 'flag_c', 'flag_a', 'flag_i', 'flag_v', 'flag_r', 'message_id', 'vendor_ids', 'notify_messages', 'transforms', 'ke_dh_groups', 'proposals', 'protocol_id', 'certificates', 'transform_attributes', 'length', 'hash', 'doi', 'situation' ]" + code => "event.set('[zeek_cols]', @zeek_ipsec_field_names.zip(event.get('[message]')).to_h)" + } } + mutate { id => "mutate_split_zeek_ipsec_commas" + split => { "[zeek_cols][vendor_ids]" => "," + "[zeek_cols][notify_messages]" => "," + "[zeek_cols][transforms]" => "," + "[zeek_cols][ke_dh_groups]" => "," + "[zeek_cols][proposals]" => "," + "[zeek_cols][certificates]" => "," + "[zeek_cols][transform_attributes]" => "," } } } mutate { @@ -1586,23 +1616,25 @@ filter { # irc.log # https://docs.zeek.org/en/stable/scripts/base/protocols/irc/main.zeek.html#type-IRC::Info - dissect { - id => "dissect_zeek_irc" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][nick]} %{[zeek_cols][user]} %{[zeek_cols][command]} %{[zeek_cols][value]} %{[zeek_cols][addl]} %{[zeek_cols][dcc_file_name]} %{[zeek_cols][dcc_file_size]} %{[zeek_cols][dcc_mime_type]} %{[zeek_cols][fuid]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_irc" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_irc" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][nick]} %{[zeek_cols][user]} %{[zeek_cols][command]} %{[zeek_cols][value]} %{[zeek_cols][addl]} %{[zeek_cols][dcc_file_name]} %{[zeek_cols][dcc_file_size]} %{[zeek_cols][dcc_mime_type]} %{[zeek_cols][fuid]}" + } } - ruby { - id => "ruby_zip_zeek_irc" - init => "@zeek_irc_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'nick', 'user', 'command', 'value', 'addl', 'dcc_file_name', 'dcc_file_size', 'dcc_mime_type', 'fuid' ]" - code => "event.set('[zeek_cols]', @zeek_irc_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_irc" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_irc" + init => "@zeek_irc_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'nick', 'user', 'command', 'value', 'addl', 'dcc_file_name', 'dcc_file_size', 'dcc_mime_type', 'fuid' ]" + code => "event.set('[zeek_cols]', @zeek_irc_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -1619,23 +1651,25 @@ filter { # cotp.log # https://github.com/cisagov/icsnpp-s7comm - dissect { - id => "dissect_zeek_cotp" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][pdu_code]} %{[zeek_cols][pdu_name]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_cotp" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_cotp" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][pdu_code]} %{[zeek_cols][pdu_name]}" + } } - ruby { - id => "ruby_zip_zeek_cotp" - init => "@zeek_cotp_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'pdu_code', 'pdu_name' ]" - code => "event.set('[zeek_cols]', @zeek_cotp_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_cotp" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_cotp" + init => "@zeek_cotp_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'pdu_code', 'pdu_name' ]" + code => "event.set('[zeek_cols]', @zeek_cotp_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -1653,24 +1687,36 @@ filter { # kerberos.log # https://docs.zeek.org/en/stable/scripts/base/protocols/krb/main.zeek.html#type-KRB::Info - dissect { - id => "dissect_zeek_kerberos" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][request_type]} %{[zeek_cols][cname]} %{[zeek_cols][sname]} %{[zeek_cols][success]} %{[zeek_cols][error_msg]} %{[zeek_cols][from]} %{[zeek_cols][till]} %{[zeek_cols][cipher]} %{[zeek_cols][forwardable]} %{[zeek_cols][renewable]} %{[zeek_cols][client_cert_subject]} %{[zeek_cols][client_cert_fuid]} %{[zeek_cols][server_cert_subject]} %{[zeek_cols][server_cert_fuid]}" - } - } - if ("_dissectfailure" in [tags]) { + if ("_jsonparsesuccess" in [tags]) { mutate { - id => "mutate_split_zeek_kerberos" + id => "mutate_rename_zeek_json_kerberos_fields" + rename => { "[zeek_cols][client]" => "[zeek_cols][cname]" } + rename => { "[zeek_cols][service]" => "[zeek_cols][sname]" } + } + + } else { + dissect { + id => "dissect_zeek_kerberos" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][request_type]} %{[zeek_cols][cname]} %{[zeek_cols][sname]} %{[zeek_cols][success]} %{[zeek_cols][error_msg]} %{[zeek_cols][from]} %{[zeek_cols][till]} %{[zeek_cols][cipher]} %{[zeek_cols][forwardable]} %{[zeek_cols][renewable]} %{[zeek_cols][client_cert_subject]} %{[zeek_cols][client_cert_fuid]} %{[zeek_cols][server_cert_subject]} %{[zeek_cols][server_cert_fuid]}" + } } - ruby { - id => "ruby_zip_zeek_kerberos" - init => "@zeek_kerberos_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'request_type', 'cname', 'sname', 'success', 'error_msg', 'from', 'till', 'cipher', 'forwardable', 'renewable', 'client_cert_subject', 'client_cert_fuid', 'server_cert_subject', 'server_cert_fuid' ]" - code => "event.set('[zeek_cols]', @zeek_kerberos_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_kerberos" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_kerberos" + init => "@zeek_kerberos_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'request_type', 'cname', 'sname', 'success', 'error_msg', 'from', 'till', 'cipher', 'forwardable', 'renewable', 'client_cert_subject', 'client_cert_fuid', 'server_cert_subject', 'server_cert_fuid' ]" + code => "event.set('[zeek_cols]', @zeek_kerberos_field_names.zip(event.get('[message]')).to_h)" + } } + mutate { id => "mutate_split_zeek_kerberos_commas" + split => { "[zeek_cols][client_cert_fuid]" => "," + "[zeek_cols][server_cert_fuid]" => "," } } } mutate { @@ -1683,23 +1729,32 @@ filter { # known_certs.log # https://docs.zeek.org/en/stable/scripts/policy/protocols/ssl/known-certs.zeek.html#type-Known::CertsInfo - dissect { - id => "dissect_zeek_known_certs" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][subject]} %{[zeek_cols][issuer_subject]} %{[zeek_cols][serial]}" - } - } - if ("_dissectfailure" in [tags]) { + if ("_jsonparsesuccess" in [tags]) { mutate { - id => "mutate_split_zeek_known_certs" + id => "mutate_rename_zeek_json_known_certs_fields" + rename => { "[zeek_cols][host]" => "[zeek_cols][orig_h]" } + rename => { "[zeek_cols][port_num]" => "[zeek_cols][orig_p]" } + } + + } else { + dissect { + id => "dissect_zeek_known_certs" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][subject]} %{[zeek_cols][issuer_subject]} %{[zeek_cols][serial]}" + } } - ruby { - id => "ruby_zip_zeek_known_certs" - init => "@zeek_known_certs_field_names = [ 'ts', 'orig_h', 'orig_p', 'subject', 'resp_h', 'issuer_subject', 'serial' ]" - code => "event.set('[zeek_cols]', @zeek_known_certs_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_known_certs" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_known_certs" + init => "@zeek_known_certs_field_names = [ 'ts', 'orig_h', 'orig_p', 'subject', 'resp_h', 'issuer_subject', 'serial' ]" + code => "event.set('[zeek_cols]', @zeek_known_certs_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -1716,23 +1771,31 @@ filter { # known_hosts.log # https://docs.zeek.org/en/stable/scripts/policy/protocols/conn/known-hosts.zeek.html#type-Known::HostsInfo - dissect { - id => "dissect_zeek_known_hosts" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][orig_h]}" - } - } - if ("_dissectfailure" in [tags]) { + if ("_jsonparsesuccess" in [tags]) { mutate { - id => "mutate_split_zeek_known_hosts" + id => "mutate_rename_zeek_json_known_hosts_fields" + rename => { "[zeek_cols][host]" => "[zeek_cols][orig_h]" } + } + + } else { + dissect { + id => "dissect_zeek_known_hosts" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][orig_h]}" + } } - ruby { - id => "ruby_zip_zeek_known_hosts" - init => "@zeek_known_hosts_field_names = [ 'ts', 'orig_h' ]" - code => "event.set('[zeek_cols]', @zeek_known_hosts_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_known_hosts" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_known_hosts" + init => "@zeek_known_hosts_field_names = [ 'ts', 'orig_h' ]" + code => "event.set('[zeek_cols]', @zeek_known_hosts_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -1741,23 +1804,31 @@ filter { # known_modbus.log # https://docs.zeek.org/en/stable/scripts/policy/protocols/modbus/known-masters-slaves.zeek.html#type-Known::ModbusInfo - dissect { - id => "dissect_zeek_known_modbus" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][orig_h]} %{[zeek_cols][device_type]}" - } - } - if ("_dissectfailure" in [tags]) { + if ("_jsonparsesuccess" in [tags]) { mutate { - id => "mutate_split_zeek_known_modbus" + id => "mutate_rename_zeek_json_known_modbus_fields" + rename => { "[zeek_cols][host]" => "[zeek_cols][orig_h]" } + } + + } else { + dissect { + id => "dissect_zeek_known_modbus" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][orig_h]} %{[zeek_cols][device_type]}" + } } - ruby { - id => "ruby_zip_zeek_known_modbus" - init => "@zeek_known_modbus_field_names = [ 'ts', 'orig_h', 'device_type' ]" - code => "event.set('[zeek_cols]', @zeek_known_modbus_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_known_modbus" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_known_modbus" + init => "@zeek_known_modbus_field_names = [ 'ts', 'orig_h', 'device_type' ]" + code => "event.set('[zeek_cols]', @zeek_known_modbus_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -1778,23 +1849,33 @@ filter { # known_services.log # https://docs.zeek.org/en/stable/scripts/policy/protocols/conn/known-services.zeek.html#type-Known::ServicesInfo - dissect { - id => "dissect_zeek_known_services" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][proto]} %{[zeek_cols][service]}" - } - } - if ("_dissectfailure" in [tags]) { + if ("_jsonparsesuccess" in [tags]) { mutate { - id => "mutate_split_zeek_known_services" + id => "mutate_rename_zeek_json_known_services_fields" + rename => { "[zeek_cols][host]" => "[zeek_cols][resp_h]" } + rename => { "[zeek_cols][port_num]" => "[zeek_cols][resp_p]" } + rename => { "[zeek_cols][port_proto]" => "[zeek_cols][proto]" } + } + + } else { + dissect { + id => "dissect_zeek_known_services" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][proto]} %{[zeek_cols][service]}" + } } - ruby { - id => "ruby_zip_zeek_known_services" - init => "@zeek_known_services_field_names = [ 'ts', 'resp_h', 'resp_p', 'proto', 'service' ]" - code => "event.set('[zeek_cols]', @zeek_known_services_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_known_services" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_known_services" + init => "@zeek_known_services_field_names = [ 'ts', 'resp_h', 'resp_p', 'proto', 'service' ]" + code => "event.set('[zeek_cols]', @zeek_known_services_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -1824,25 +1905,41 @@ filter { # ldap.log # main.zeek (https://docs.zeek.org/en/master/scripts/base/protocols/ldap/main.zeek.html) - dissect { - id => "dissect_zeek_ldap" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][message_id]} %{[zeek_cols][version]} %{[zeek_cols][operation]} %{[zeek_cols][result_code]} %{[zeek_cols][result_message]} %{[zeek_cols][object]} %{[zeek_cols][argument]}" + if ("_jsonparsesuccess" in [tags]) { + mutate { + id => "mutate_rename_zeek_json_ldap_fields" + rename => { "[zeek_cols][arguments]" => "[zeek_cols][argument]" } + rename => { "[zeek_cols][opcodes]" => "[zeek_cols][operation]" } + rename => { "[zeek_cols][results]" => "[zeek_cols][result_code]" } + rename => { "[zeek_cols][diagnostic_message]" => "[zeek_cols][result_message]" } } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_ldap" + } else { + dissect { + id => "dissect_zeek_ldap" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][message_id]} %{[zeek_cols][version]} %{[zeek_cols][operation]} %{[zeek_cols][result_code]} %{[zeek_cols][result_message]} %{[zeek_cols][object]} %{[zeek_cols][argument]}" + } } - ruby { - id => "ruby_zip_zeek_ldap" - init => "@zeek_ldap_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'message_id', 'version', 'operation', 'result_code', 'result_message', 'object', 'argument' ]" - code => "event.set('[zeek_cols]', @zeek_ldap_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_ldap" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_ldap" + init => "@zeek_ldap_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'message_id', 'version', 'operation', 'result_code', 'result_message', 'object', 'argument' ]" + code => "event.set('[zeek_cols]', @zeek_ldap_field_names.zip(event.get('[message]')).to_h)" + } } + mutate { id => "mutate_split_zeek_ldap_commas" + split => { "[zeek_cols][operation]" => "," + "[zeek_cols][result_code]" => "," + "[zeek_cols][result_message]" => "," + "[zeek_cols][object]" => "," + "[zeek_cols][argument]" => "," } } } mutate { @@ -1858,25 +1955,44 @@ filter { # ldap_search.log # main.zeek (https://docs.zeek.org/en/master/scripts/base/protocols/ldap/main.zeek.html) - dissect { - id => "dissect_zeek_ldap_search" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][message_id]} %{[zeek_cols][scope]} %{[zeek_cols][deref]} %{[zeek_cols][base_object]} %{[zeek_cols][result_count]} %{[zeek_cols][result_code]} %{[zeek_cols][result_message]} %{[zeek_cols][filter]} %{[zeek_cols][attributes]}" + if ("_jsonparsesuccess" in [tags]) { + mutate { + id => "mutate_rename_zeek_json_ldap_search_fields" + rename => { "[zeek_cols][deref_aliases]" => "[zeek_cols][deref]" } + rename => { "[zeek_cols][derefs]" => "[zeek_cols][deref]" } + rename => { "[zeek_cols][base_objects]" => "[zeek_cols][base_object]" } + rename => { "[zeek_cols][results]" => "[zeek_cols][result_code]" } + rename => { "[zeek_cols][result]" => "[zeek_cols][result_code]" } + rename => { "[zeek_cols][diagnostic_message]" => "[zeek_cols][result_message]" } } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_ldap_search" + } else { + dissect { + id => "dissect_zeek_ldap_search" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][message_id]} %{[zeek_cols][scope]} %{[zeek_cols][deref]} %{[zeek_cols][base_object]} %{[zeek_cols][result_count]} %{[zeek_cols][result_code]} %{[zeek_cols][result_message]} %{[zeek_cols][filter]} %{[zeek_cols][attributes]}" + } } - ruby { - id => "ruby_zip_zeek_ldap_search" - init => "@zeek_ldap_search_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'message_id', 'scope', 'deref', 'base_object', 'result_count', 'result_code', 'result_message', 'filter', 'attributes' ]" - code => "event.set('[zeek_cols]', @zeek_ldap_search_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_ldap_search" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_ldap_search" + init => "@zeek_ldap_search_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'message_id', 'scope', 'deref', 'base_object', 'result_count', 'result_code', 'result_message', 'filter', 'attributes' ]" + code => "event.set('[zeek_cols]', @zeek_ldap_search_field_names.zip(event.get('[message]')).to_h)" + } } + mutate { id => "mutate_split_zeek_ldap_search_commas" + split => { "[zeek_cols][scope]" => "," + "[zeek_cols][deref]" => "," + "[zeek_cols][attributes]" => "," + "[zeek_cols][base_object]" => "," + "[zeek_cols][result_code]" => "," + "[zeek_cols][result_message]" => "," } } } mutate { @@ -1892,23 +2008,25 @@ filter { # login.log # custom login.log module (rudimentary, telnet/rlogin/rsh analyzers are old and not the greatest) - dissect { - id => "dissect_zeek_login" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][service]} %{[zeek_cols][success]} %{[zeek_cols][confused]} %{[zeek_cols][user]} %{[zeek_cols][client_user]} %{[zeek_cols][password]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_login" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_login" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][service]} %{[zeek_cols][success]} %{[zeek_cols][confused]} %{[zeek_cols][user]} %{[zeek_cols][client_user]} %{[zeek_cols][password]}" + } } - ruby { - id => "ruby_zip_zeek_login" - init => "@zeek_login_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'service', 'success', 'confused', 'user', 'client_user', 'password' ]" - code => "event.set('[zeek_cols]', @zeek_login_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_login" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_login" + init => "@zeek_login_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'service', 'success', 'confused', 'user', 'client_user', 'password' ]" + code => "event.set('[zeek_cols]', @zeek_login_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -1917,23 +2035,25 @@ filter { # modbus.log # https://docs.zeek.org/en/stable/scripts/base/protocols/modbus/main.zeek.html#type-Modbus::Info - dissect { - id => "dissect_zeek_modbus" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][trans_id]} %{[zeek_cols][unit_id]} %{[zeek_cols][func]} %{[zeek_cols][network_direction]} %{[zeek_cols][exception]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_modbus" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_modbus" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][trans_id]} %{[zeek_cols][unit_id]} %{[zeek_cols][func]} %{[zeek_cols][network_direction]} %{[zeek_cols][exception]}" + } } - ruby { - id => "ruby_zip_zeek_modbus" - init => "@zeek_modbus_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'trans_id', 'unit_id', 'func', 'network_direction', 'exception' ]" - code => "event.set('[zeek_cols]', @zeek_modbus_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_modbus" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_modbus" + init => "@zeek_modbus_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'trans_id', 'unit_id', 'func', 'network_direction', 'exception' ]" + code => "event.set('[zeek_cols]', @zeek_modbus_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -1948,26 +2068,37 @@ filter { # modbus_detailed.log # main.zeek (https://github.com/cisagov/icsnpp-modbus) - dissect { - id => "dissect_zeek_modbus_detailed" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][trans_id]} %{[zeek_cols][unit_id]} %{[zeek_cols][func]} %{[zeek_cols][network_direction]} %{[zeek_cols][address]} %{[zeek_cols][quantity]} %{[zeek_cols][values]}" - } - } - - if ("_dissectfailure" in [tags]) { + if ("_jsonparsesuccess" in [tags]) { mutate { - id => "mutate_split_zeek_modbus_detailed" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + id => "mutate_rename_zeek_json_modbus_detailed_fields" + rename => { "[zeek_cols][tid]" => "[zeek_cols][trans_id]" } + rename => { "[zeek_cols][unit]" => "[zeek_cols][unit_id]" } + rename => { "[zeek_cols][request_response]" => "[zeek_cols][network_direction]" } } - ruby { - id => "ruby_zip_zeek_modbus_detailed" - init => "@zeek_modbus_detailed_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'trans_id', 'unit_id', 'func', 'network_direction', 'address', 'quantity', 'values' ]" - code => "event.set('[zeek_cols]', @zeek_modbus_detailed_field_names.zip(event.get('[message]')).to_h)" - } - } + + } else { + dissect { + id => "dissect_zeek_modbus_detailed" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][trans_id]} %{[zeek_cols][unit_id]} %{[zeek_cols][func]} %{[zeek_cols][network_direction]} %{[zeek_cols][address]} %{[zeek_cols][quantity]} %{[zeek_cols][values]}" + } + } + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_modbus_detailed" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_modbus_detailed" + init => "@zeek_modbus_detailed_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'trans_id', 'unit_id', 'func', 'network_direction', 'address', 'quantity', 'values' ]" + code => "event.set('[zeek_cols]', @zeek_modbus_detailed_field_names.zip(event.get('[message]')).to_h)" + } + } + mutate { id => "mutate_split_zeek_modbus_detailed_values" + split => { "[zeek_cols][values]" => "," } } + } mutate { id => "mutate_add_fields_zeek_modbus_detailed" @@ -1982,24 +2113,33 @@ filter { # modbus_mask_write_register.log # main.zeek (https://github.com/cisagov/icsnpp-modbus) - dissect { - id => "dissect_zeek_modbus_mask_write_register" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][trans_id]} %{[zeek_cols][unit_id]} %{[zeek_cols][func]} %{[zeek_cols][network_direction]} %{[zeek_cols][address]} %{[zeek_cols][and_mask]} %{[zeek_cols][or_mask]}" + if ("_jsonparsesuccess" in [tags]) { + mutate { + id => "mutate_rename_zeek_json_modbus_mask_write_register_fields" + rename => { "[zeek_cols][tid]" => "[zeek_cols][trans_id]" } + rename => { "[zeek_cols][unit]" => "[zeek_cols][unit_id]" } + rename => { "[zeek_cols][request_response]" => "[zeek_cols][network_direction]" } } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_modbus_mask_write_register" + } else { + dissect { + id => "dissect_zeek_modbus_mask_write_register" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][trans_id]} %{[zeek_cols][unit_id]} %{[zeek_cols][func]} %{[zeek_cols][network_direction]} %{[zeek_cols][address]} %{[zeek_cols][and_mask]} %{[zeek_cols][or_mask]}" + } } - ruby { - id => "ruby_zip_zeek_modbus_mask_write_register" - init => "@zeek_modbus_mask_write_register_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'trans_id', 'unit_id', 'func', 'network_direction', 'address', 'and_mask', 'or_mask' ]" - code => "event.set('[zeek_cols]', @zeek_modbus_mask_write_register_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_modbus_mask_write_register" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_modbus_mask_write_register" + init => "@zeek_modbus_mask_write_register_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'trans_id', 'unit_id', 'func', 'network_direction', 'address', 'and_mask', 'or_mask' ]" + code => "event.set('[zeek_cols]', @zeek_modbus_mask_write_register_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -2016,24 +2156,33 @@ filter { # modbus_read_device_identification.log # main.zeek (https://github.com/cisagov/icsnpp-modbus) - dissect { - id => "dissect_zeek_modbus_read_device_identification" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][trans_id]} %{[zeek_cols][unit_id]} %{[zeek_cols][func]} %{[zeek_cols][network_direction]} %{[zeek_cols][mei_type]} %{[zeek_cols][conformity_level_code]} %{[zeek_cols][conformity_level]} %{[zeek_cols][device_id_code]} %{[zeek_cols][object_id_code]} %{[zeek_cols][object_id]} %{[zeek_cols][object_value]}" + if ("_jsonparsesuccess" in [tags]) { + mutate { + id => "mutate_rename_zeek_json_modbus_read_device_identification_fields" + rename => { "[zeek_cols][tid]" => "[zeek_cols][trans_id]" } + rename => { "[zeek_cols][unit]" => "[zeek_cols][unit_id]" } + rename => { "[zeek_cols][request_response]" => "[zeek_cols][network_direction]" } } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_modbus_read_device_identification" + } else { + dissect { + id => "dissect_zeek_modbus_read_device_identification" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][trans_id]} %{[zeek_cols][unit_id]} %{[zeek_cols][func]} %{[zeek_cols][network_direction]} %{[zeek_cols][mei_type]} %{[zeek_cols][conformity_level_code]} %{[zeek_cols][conformity_level]} %{[zeek_cols][device_id_code]} %{[zeek_cols][object_id_code]} %{[zeek_cols][object_id]} %{[zeek_cols][object_value]}" + } } - ruby { - id => "ruby_zip_zeek_modbus_read_device_identification" - init => "@zeek_modbus_read_device_identification_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'trans_id', 'unit_id', 'func', 'network_direction', 'mei_type', 'conformity_level_code', 'conformity_level', 'device_id_code', 'object_id_code', 'object_id', 'object_value' ]" - code => "event.set('[zeek_cols]', @zeek_modbus_read_device_identification_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_modbus_read_device_identification" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_modbus_read_device_identification" + init => "@zeek_modbus_read_device_identification_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'trans_id', 'unit_id', 'func', 'network_direction', 'mei_type', 'conformity_level_code', 'conformity_level', 'device_id_code', 'object_id_code', 'object_id', 'object_value' ]" + code => "event.set('[zeek_cols]', @zeek_modbus_read_device_identification_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -2050,25 +2199,37 @@ filter { # modbus_read_write_multiple_registers.log # main.zeek (https://github.com/cisagov/icsnpp-modbus) - dissect { - id => "dissect_zeek_modbus_read_write_multiple_registers" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][`]} %{[zeek_cols][unit_id]} %{[zeek_cols][func]} %{[zeek_cols][network_direction]} %{[zeek_cols][write_start_address]} %{[zeek_cols][write_registers]} %{[zeek_cols][read_start_address]} %{[zeek_cols][read_quantity]} %{[zeek_cols][read_registers]}" + if ("_jsonparsesuccess" in [tags]) { + mutate { + id => "mutate_rename_zeek_json_modbus_read_write_multiple_registers_fields" + rename => { "[zeek_cols][tid]" => "[zeek_cols][trans_id]" } + rename => { "[zeek_cols][unit]" => "[zeek_cols][unit_id]" } + rename => { "[zeek_cols][request_response]" => "[zeek_cols][network_direction]" } } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_modbus_read_write_multiple_registers" + } else { + dissect { + id => "dissect_zeek_modbus_read_write_multiple_registers" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][trans_id]} %{[zeek_cols][unit_id]} %{[zeek_cols][func]} %{[zeek_cols][network_direction]} %{[zeek_cols][write_start_address]} %{[zeek_cols][write_registers]} %{[zeek_cols][read_start_address]} %{[zeek_cols][read_quantity]} %{[zeek_cols][read_registers]}" + } } - ruby { - id => "ruby_zip_zeek_modbus_read_write_multiple_registers" - init => "@zeek_modbus_read_write_multiple_registers_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'trans_id', 'unit_id', 'func', 'network_direction', 'write_start_address', 'write_registers', 'read_start_address', 'read_quantity', 'read_registers' ]" - code => "event.set('[zeek_cols]', @zeek_modbus_read_write_multiple_registers_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_modbus_read_write_multiple_registers" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_modbus_read_write_multiple_registers" + init => "@zeek_modbus_read_write_multiple_registers_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'trans_id', 'unit_id', 'func', 'network_direction', 'write_start_address', 'write_registers', 'read_start_address', 'read_quantity', 'read_registers' ]" + code => "event.set('[zeek_cols]', @zeek_modbus_read_write_multiple_registers_field_names.zip(event.get('[message]')).to_h)" + } } + mutate { id => "mutate_split_zeek_modbus_read_write_multiple_registers_read_commas" + split => { "[zeek_cols][read_registers]" => "," + "[zeek_cols][write_registers]" => "," } } } mutate { @@ -2084,23 +2245,25 @@ filter { # mqtt_connect.log # https://docs.zeek.org/en/stable/scripts/policy/protocols/mqtt/main.zeek.html#type-MQTT::ConnectInfo - dissect { - id => "dissect_zeek_mqtt_connect" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][proto_name]} %{[zeek_cols][proto_version]} %{[zeek_cols][client_id]} %{[zeek_cols][connect_status]} %{[zeek_cols][will_topic]} %{[zeek_cols][will_payload]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_mqtt_connect" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_mqtt_connect" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][proto_name]} %{[zeek_cols][proto_version]} %{[zeek_cols][client_id]} %{[zeek_cols][connect_status]} %{[zeek_cols][will_topic]} %{[zeek_cols][will_payload]}" + } } - ruby { - id => "ruby_zip_zeek_mqtt_connect" - init => "@zeek_mqtt_connect_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto_name', 'proto_version', 'client_id', 'connect_status', 'will_topic', 'will_payload' ]" - code => "event.set('[zeek_cols]', @zeek_mqtt_connect_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_mqtt_connect" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_mqtt_connect" + init => "@zeek_mqtt_connect_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto_name', 'proto_version', 'client_id', 'connect_status', 'will_topic', 'will_payload' ]" + code => "event.set('[zeek_cols]', @zeek_mqtt_connect_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -2117,23 +2280,25 @@ filter { # mqtt_publish.log # https://docs.zeek.org/en/stable/scripts/policy/protocols/mqtt/main.zeek.html#type-MQTT::PublishInfo - dissect { - id => "dissect_zeek_mqtt_publish" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][from_client]} %{[zeek_cols][retain]} %{[zeek_cols][qos]} %{[zeek_cols][status]} %{[zeek_cols][topic]} %{[zeek_cols][payload]} %{[zeek_cols][payload_len]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_mqtt_publish" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_mqtt_publish" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][from_client]} %{[zeek_cols][retain]} %{[zeek_cols][qos]} %{[zeek_cols][status]} %{[zeek_cols][topic]} %{[zeek_cols][payload]} %{[zeek_cols][payload_len]}" + } } - ruby { - id => "ruby_zip_zeek_mqtt_publish" - init => "@zeek_mqtt_publish_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'from_client', 'retain', 'qos', 'status', 'topic', 'payload', 'payload_len' ]" - code => "event.set('[zeek_cols]', @zeek_mqtt_publish_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_mqtt_publish" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_mqtt_publish" + init => "@zeek_mqtt_publish_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'from_client', 'retain', 'qos', 'status', 'topic', 'payload', 'payload_len' ]" + code => "event.set('[zeek_cols]', @zeek_mqtt_publish_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -2150,24 +2315,29 @@ filter { # mqtt_subscribe.log # https://docs.zeek.org/en/stable/scripts/policy/protocols/mqtt/main.zeek.html#type-MQTT::SubscribeInfo - dissect { - id => "dissect_zeek_mqtt_subscribe" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][action]} %{[zeek_cols][topics]} %{[zeek_cols][qos_levels]} %{[zeek_cols][granted_qos_level]} %{[zeek_cols][ack]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_mqtt_subscribe" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_mqtt_subscribe" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][action]} %{[zeek_cols][topics]} %{[zeek_cols][qos_levels]} %{[zeek_cols][granted_qos_level]} %{[zeek_cols][ack]}" + } } - ruby { - id => "ruby_zip_zeek_mqtt_subscribe" - init => "@zeek_mqtt_subscribe_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'action', 'topics', 'qos_levels', 'granted_qos_level', 'ack' ]" - code => "event.set('[zeek_cols]', @zeek_mqtt_subscribe_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_mqtt_subscribe" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_mqtt_subscribe" + init => "@zeek_mqtt_subscribe_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'action', 'topics', 'qos_levels', 'granted_qos_level', 'ack' ]" + code => "event.set('[zeek_cols]', @zeek_mqtt_subscribe_field_names.zip(event.get('[message]')).to_h)" + } } + mutate { id => "mutate_split_zeek_mqtt_subscribe_commas" + split => { "[zeek_cols][topics]" => "," + "[zeek_cols][qos_levels]" => "," } } } mutate { @@ -2186,23 +2356,25 @@ filter { # mysql.log # https://docs.zeek.org/en/stable/scripts/base/protocols/mysql/main.zeek.html#type-MySQL::Info - dissect { - id => "dissect_zeek_mysql" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][cmd]} %{[zeek_cols][arg]} %{[zeek_cols][success]} %{[zeek_cols][rows]} %{[zeek_cols][response]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_mysql" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_mysql" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][cmd]} %{[zeek_cols][arg]} %{[zeek_cols][success]} %{[zeek_cols][rows]} %{[zeek_cols][response]}" + } } - ruby { - id => "ruby_zip_zeek_mysql" - init => "@zeek_mysql_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'cmd', 'arg', 'success', 'rows', 'response' ]" - code => "event.set('[zeek_cols]', @zeek_mysql_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_mysql" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_mysql" + init => "@zeek_mysql_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'cmd', 'arg', 'success', 'rows', 'response' ]" + code => "event.set('[zeek_cols]', @zeek_mysql_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -2216,39 +2388,42 @@ filter { # notice.log # https://docs.zeek.org/en/stable/scripts/base/frameworks/notice/main.zeek.html#type-Notice::Info - dissect { - id => "dissect_zeek_notice_with_all_fields" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][fuid]} %{[zeek_cols][file_mime_type]} %{[zeek_cols][file_desc]} %{[zeek_cols][proto]} %{[zeek_cols][note]} %{[zeek_cols][msg]} %{[zeek_cols][sub]} %{[zeek_cols][src]} %{[zeek_cols][dst]} %{[zeek_cols][p]} %{[zeek_cols][n]} %{[zeek_cols][peer_descr]} %{[zeek_cols][actions]} %{[zeek_cols][email_dest]} %{[zeek_cols][suppress_for]} %{[zeek_cols][remote_location_country_code]} %{[zeek_cols][remote_location_region]} %{[zeek_cols][remote_location_city]} %{[zeek_cols][remote_location_latitude]} %{[zeek_cols][remote_location_longitude]}" - } - } - - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_notice" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_notice_with_all_fields" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][fuid]} %{[zeek_cols][file_mime_type]} %{[zeek_cols][file_desc]} %{[zeek_cols][proto]} %{[zeek_cols][note]} %{[zeek_cols][msg]} %{[zeek_cols][sub]} %{[zeek_cols][src]} %{[zeek_cols][dst]} %{[zeek_cols][p]} %{[zeek_cols][n]} %{[zeek_cols][peer_descr]} %{[zeek_cols][actions]} %{[zeek_cols][email_dest]} %{[zeek_cols][suppress_for]} %{[zeek_cols][remote_location_country_code]} %{[zeek_cols][remote_location_region]} %{[zeek_cols][remote_location_city]} %{[zeek_cols][remote_location_latitude]} %{[zeek_cols][remote_location_longitude]}" + } } - ruby { - id => "ruby_zip_zeek_notice" - init => "@zeek_notice_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'fuid', 'file_mime_type', 'file_desc', 'proto', 'note', 'msg', 'sub', 'src', 'dst', 'p', 'n', 'peer_descr', 'actions', 'email_dest', 'suppress_for', 'remote_location_country_code', 'remote_location_region', 'remote_location_city', 'remote_location_latitude', 'remote_location_longitude' ]" - code => "event.set('[zeek_cols]', @zeek_notice_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_notice" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_notice" + init => "@zeek_notice_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'fuid', 'file_mime_type', 'file_desc', 'proto', 'note', 'msg', 'sub', 'src', 'dst', 'p', 'n', 'peer_descr', 'actions', 'email_dest', 'suppress_for', 'remote_location_country_code', 'remote_location_region', 'remote_location_city', 'remote_location_latitude', 'remote_location_longitude' ]" + code => "event.set('[zeek_cols]', @zeek_notice_field_names.zip(event.get('[message]')).to_h)" + } } + mutate { id => "mutate_split_zeek_notice_actions" + split => { "[zeek_cols][actions]" => "," } } } - if (![zeek_cols][orig_h]) or ([zeek_cols][orig_h] == '(empty)') or - ([zeek_cols][orig_h] == '-') or ([zeek_cols][orig_h] == '') { + if ([zeek_cols][src]) and ((![zeek_cols][orig_h]) or ([zeek_cols][orig_h] == '(empty)') or + ([zeek_cols][orig_h] == '-') or ([zeek_cols][orig_h] == '')) { mutate { id => "mutate_replace_zeek_notice_orig_h" replace => { "[zeek_cols][orig_h]" => "%{[zeek_cols][src]}" } } } - if (![zeek_cols][resp_h]) or ([zeek_cols][resp_h] == '(empty)') or - ([zeek_cols][resp_h] == '-') or ([zeek_cols][resp_h] == '') { + if ([zeek_cols][dst]) and ((![zeek_cols][resp_h]) or ([zeek_cols][resp_h] == '(empty)') or + ([zeek_cols][resp_h] == '-') or ([zeek_cols][resp_h] == '')) { mutate { id => "mutate_replace_zeek_notice_resp_h" replace => { "[zeek_cols][resp_h]" => "%{[zeek_cols][dst]}" } } } - if (![zeek_cols][resp_p]) or ([zeek_cols][resp_p] == '(empty)') or - ([zeek_cols][resp_p] == '-') or ([zeek_cols][resp_p] == '') { + if [zeek_cols][p] and ((![zeek_cols][resp_p]) or ([zeek_cols][resp_p] == '(empty)') or + ([zeek_cols][resp_p] == '-') or ([zeek_cols][resp_p] == '')) { mutate { id => "mutate_replace_zeek_resp_p" replace => { "[zeek_cols][resp_p]" => "%{[zeek_cols][p]}" } } } @@ -2258,24 +2433,36 @@ filter { # ntlm.log # https://docs.zeek.org/en/stable/scripts/base/protocols/ntlm/main.zeek.html#type-NTLM::Info - dissect { - id => "dissect_zeek_ntlm_with_all_fields" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][user]} %{[zeek_cols][host]} %{[zeek_cols][domain]} %{[zeek_cols][server_nb_computer]} %{[zeek_cols][server_dns_computer]} %{[zeek_cols][server_tree]} %{[zeek_cols][success]}" + if ("_jsonparsesuccess" in [tags]) { + mutate { + id => "mutate_rename_zeek_json_ntlm_fields" + rename => { "[zeek_cols][hostname]" => "[zeek_cols][host]" } + rename => { "[zeek_cols][domainname]" => "[zeek_cols][domain]" } + rename => { "[zeek_cols][server_nb_computer_name]" => "[zeek_cols][server_nb_computer]" } + rename => { "[zeek_cols][server_dns_computer_name]" => "[zeek_cols][server_dns_computer]" } + rename => { "[zeek_cols][server_tree_name]" => "[zeek_cols][server_tree]" } + rename => { "[zeek_cols][username]" => "[zeek_cols][user]" } } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_ntlm" + } else { + dissect { + id => "dissect_zeek_ntlm_with_all_fields" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][user]} %{[zeek_cols][host]} %{[zeek_cols][domain]} %{[zeek_cols][server_nb_computer]} %{[zeek_cols][server_dns_computer]} %{[zeek_cols][server_tree]} %{[zeek_cols][success]}" + } } - ruby { - id => "ruby_zip_zeek_ntlm" - init => "@zeek_ntlm_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'user', 'host', 'domain', 'server_nb_computer', 'server_dns_computer', 'server_tree', 'success' ]" - code => "event.set('[zeek_cols]', @zeek_ntlm_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_ntlm" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_ntlm" + init => "@zeek_ntlm_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'user', 'host', 'domain', 'server_nb_computer', 'server_dns_computer', 'server_tree', 'success' ]" + code => "event.set('[zeek_cols]', @zeek_ntlm_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -2289,23 +2476,25 @@ filter { # ntp.log # https://docs.zeek.org/en/latest/scripts/base/protocols/ntp/main.zeek.html#type-NTP::Info - dissect { - id => "dissect_zeek_ntp" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][version]} %{[zeek_cols][mode]} %{[zeek_cols][stratum]} %{[zeek_cols][poll]} %{[zeek_cols][precision]} %{[zeek_cols][root_delay]} %{[zeek_cols][root_disp]} %{[zeek_cols][ref_id]} %{[zeek_cols][ref_time]} %{[zeek_cols][org_time]} %{[zeek_cols][rec_time]} %{[zeek_cols][xmt_time]} %{[zeek_cols][num_exts]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_ntp" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_ntp" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][version]} %{[zeek_cols][mode]} %{[zeek_cols][stratum]} %{[zeek_cols][poll]} %{[zeek_cols][precision]} %{[zeek_cols][root_delay]} %{[zeek_cols][root_disp]} %{[zeek_cols][ref_id]} %{[zeek_cols][ref_time]} %{[zeek_cols][org_time]} %{[zeek_cols][rec_time]} %{[zeek_cols][xmt_time]} %{[zeek_cols][num_exts]}" + } } - ruby { - id => "ruby_zip_zeek_ntp" - init => "@zeek_ntp_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'version', 'mode', 'stratum', 'poll', 'precision', 'root_delay', 'root_disp', 'ref_id', 'ref_time', 'org_time', 'rec_time', 'xmt_time', 'num_exts' ]" - code => "event.set('[zeek_cols]', @zeek_ntp_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_ntp" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_ntp" + init => "@zeek_ntp_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'version', 'mode', 'stratum', 'poll', 'precision', 'root_delay', 'root_disp', 'ref_id', 'ref_time', 'org_time', 'rec_time', 'xmt_time', 'num_exts' ]" + code => "event.set('[zeek_cols]', @zeek_ntp_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -2322,24 +2511,26 @@ filter { # ocsp.log # https://docs.zeek.org/en/stable/scripts/policy/files/x509/log-ocsp.zeek.html#type-OCSP::Info - dissect { - id => "dissect_zeek_ocsp" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][fuid]} %{[zeek_cols][hashAlgorithm]} %{[zeek_cols][issuerNameHash]} %{[zeek_cols][issuerKeyHash]} %{[zeek_cols][serialNumber]} %{[zeek_cols][certStatus]} %{[zeek_cols][revoketime]} %{[zeek_cols][revokereason]} %{[zeek_cols][thisUpdate]} %{[zeek_cols][nextUpdate]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_ocsp" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_ocsp" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][fuid]} %{[zeek_cols][hashAlgorithm]} %{[zeek_cols][issuerNameHash]} %{[zeek_cols][issuerKeyHash]} %{[zeek_cols][serialNumber]} %{[zeek_cols][certStatus]} %{[zeek_cols][revoketime]} %{[zeek_cols][revokereason]} %{[zeek_cols][thisUpdate]} %{[zeek_cols][nextUpdate]}" + } } + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_ocsp" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } - ruby { - id => "ruby_zip_zeek_ocsp" - init => "@zeek_ocsp_field_names = [ 'ts', 'fuid', 'hashAlgorithm', 'issuerNameHash', 'issuerKeyHash', 'serialNumber', 'certStatus', 'revoketime', 'revokereason', 'thisUpdate', 'nextUpdate' ]" - code => "event.set('[zeek_cols]', @zeek_ocsp_field_names.zip(event.get('[message]')).to_h)" + ruby { + id => "ruby_zip_zeek_ocsp" + init => "@zeek_ocsp_field_names = [ 'ts', 'fuid', 'hashAlgorithm', 'issuerNameHash', 'issuerKeyHash', 'serialNumber', 'certStatus', 'revoketime', 'revokereason', 'thisUpdate', 'nextUpdate' ]" + code => "event.set('[zeek_cols]', @zeek_ocsp_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -2351,25 +2542,42 @@ filter { # ospf.log # https://github.com/corelight/zeek-spicy-ospf - dissect { - id => "dissect_zeek_ospf" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][orig_h]} %{[zeek_cols][resp_h]} %{[zeek_cols][ospf_type]} %{[zeek_cols][version]} %{[zeek_cols][router_id]} %{[zeek_cols][area_id]} %{[zeek_cols][interface_id]} %{[zeek_cols][netmask]} %{[zeek_cols][desig_router]} %{[zeek_cols][backup_router]} %{[zeek_cols][neighbors]} %{[zeek_cols][lsa_type]} %{[zeek_cols][link_state_id]} %{[zeek_cols][advert_router]} %{[zeek_cols][routers]} %{[zeek_cols][link_id]} %{[zeek_cols][link_data]} %{[zeek_cols][link_type]} %{[zeek_cols][neighbor_router_id]} %{[zeek_cols][metrics]} %{[zeek_cols][fwd_addrs]} %{[zeek_cols][route_tags]} %{[zeek_cols][neighbor_interface_id]} %{[zeek_cols][prefix]} %{[zeek_cols][metric]} %{[zeek_cols][dest_router_id]} %{[zeek_cols][link_prefixes]} %{[zeek_cols][intra_prefixes]}" - } - } - if ("_dissectfailure" in [tags]) { + if ("_jsonparsesuccess" in [tags]) { mutate { - id => "mutate_split_zeek_ospf" + id => "mutate_rename_zeek_json_ospf_fields" + rename => { "[zeek_cols][ip_dst]" => "[zeek_cols][orig_h]" } + rename => { "[zeek_cols][ip_src]" => "[zeek_cols][resp_h]" } + } + + } else { + dissect { + id => "dissect_zeek_ospf" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][orig_h]} %{[zeek_cols][resp_h]} %{[zeek_cols][ospf_type]} %{[zeek_cols][version]} %{[zeek_cols][router_id]} %{[zeek_cols][area_id]} %{[zeek_cols][interface_id]} %{[zeek_cols][netmask]} %{[zeek_cols][desig_router]} %{[zeek_cols][backup_router]} %{[zeek_cols][neighbors]} %{[zeek_cols][lsa_type]} %{[zeek_cols][link_state_id]} %{[zeek_cols][advert_router]} %{[zeek_cols][routers]} %{[zeek_cols][link_id]} %{[zeek_cols][link_data]} %{[zeek_cols][link_type]} %{[zeek_cols][neighbor_router_id]} %{[zeek_cols][metrics]} %{[zeek_cols][fwd_addrs]} %{[zeek_cols][route_tags]} %{[zeek_cols][neighbor_interface_id]} %{[zeek_cols][prefix]} %{[zeek_cols][metric]} %{[zeek_cols][dest_router_id]} %{[zeek_cols][link_prefixes]} %{[zeek_cols][intra_prefixes]}" + } } + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_ospf" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } - ruby { - id => "ruby_zip_zeek_ospf" - init => "@zeek_ospf_field_names = [ 'ts', 'orig_h', 'resp_h', 'ospf_type', 'version', 'router_id', 'area_id', 'interface_id', 'netmask', 'desig_router', 'backup_router', 'neighbors', 'lsa_type', 'link_state_id', 'advert_router', 'routers', 'link_id', 'link_data', 'link_type', 'neighbor_router_id', 'metrics', 'fwd_addrs', 'route_tags', 'neighbor_interface_id', 'prefix', 'metric', 'dest_router_id', 'link_prefixes', 'intra_prefixes' ]" - code => "event.set('[zeek_cols]', @zeek_ospf_field_names.zip(event.get('[message]')).to_h)" + ruby { + id => "ruby_zip_zeek_ospf" + init => "@zeek_ospf_field_names = [ 'ts', 'orig_h', 'resp_h', 'ospf_type', 'version', 'router_id', 'area_id', 'interface_id', 'netmask', 'desig_router', 'backup_router', 'neighbors', 'lsa_type', 'link_state_id', 'advert_router', 'routers', 'link_id', 'link_data', 'link_type', 'neighbor_router_id', 'metrics', 'fwd_addrs', 'route_tags', 'neighbor_interface_id', 'prefix', 'metric', 'dest_router_id', 'link_prefixes', 'intra_prefixes' ]" + code => "event.set('[zeek_cols]', @zeek_ospf_field_names.zip(event.get('[message]')).to_h)" + } } + mutate { id => "mutate_split_zeek_ospf_commas" + split => { "[zeek_cols][neighbors]" => "," + "[zeek_cols][routers]" => "," + "[zeek_cols][metrics]" => "," + "[zeek_cols][fwd_addrs]" => "," + "[zeek_cols][route_tags]" => "," + "[zeek_cols][link_prefixes]" => "," + "[zeek_cols][intra_prefixes]" => "," } } } mutate { @@ -2385,24 +2593,34 @@ filter { # pe.log # https://docs.zeek.org/en/stable/scripts/base/files/pe/main.zeek.html#type-PE::Info - dissect { - id => "dissect_zeek_pe" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][fuid]} %{[zeek_cols][machine]} %{[zeek_cols][compile_ts]} %{[zeek_cols][os]} %{[zeek_cols][subsystem]} %{[zeek_cols][is_exe]} %{[zeek_cols][is_64bit]} %{[zeek_cols][uses_aslr]} %{[zeek_cols][uses_dep]} %{[zeek_cols][uses_code_integrity]} %{[zeek_cols][uses_seh]} %{[zeek_cols][has_import_table]} %{[zeek_cols][has_export_table]} %{[zeek_cols][has_cert_table]} %{[zeek_cols][has_debug_data]} %{[zeek_cols][section_names]}" - } - } - if ("_dissectfailure" in [tags]) { + if ("_jsonparsesuccess" in [tags]) { mutate { - id => "mutate_split_zeek_pe" + id => "mutate_rename_zeek_json_pe_fields" + rename => { "[zeek_cols][id]" => "[zeek_cols][fuid]" } + } + + } else { + dissect { + id => "dissect_zeek_pe" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][fuid]} %{[zeek_cols][machine]} %{[zeek_cols][compile_ts]} %{[zeek_cols][os]} %{[zeek_cols][subsystem]} %{[zeek_cols][is_exe]} %{[zeek_cols][is_64bit]} %{[zeek_cols][uses_aslr]} %{[zeek_cols][uses_dep]} %{[zeek_cols][uses_code_integrity]} %{[zeek_cols][uses_seh]} %{[zeek_cols][has_import_table]} %{[zeek_cols][has_export_table]} %{[zeek_cols][has_cert_table]} %{[zeek_cols][has_debug_data]} %{[zeek_cols][section_names]}" + } } - ruby { - id => "ruby_zip_zeek_pe" - init => "@zeek_pe_field_names = [ 'ts', 'fuid', 'machine', 'compile_ts', 'os', 'subsystem', 'is_exe', 'is_64bit', 'uses_aslr', 'uses_dep', 'uses_code_integrity', 'uses_seh', 'has_import_table', 'has_export_table', 'has_cert_table', 'has_debug_data', 'section_names' ]" - code => "event.set('[zeek_cols]', @zeek_pe_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_pe" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_pe" + init => "@zeek_pe_field_names = [ 'ts', 'fuid', 'machine', 'compile_ts', 'os', 'subsystem', 'is_exe', 'is_64bit', 'uses_aslr', 'uses_dep', 'uses_code_integrity', 'uses_seh', 'has_import_table', 'has_export_table', 'has_cert_table', 'has_debug_data', 'section_names' ]" + code => "event.set('[zeek_cols]', @zeek_pe_field_names.zip(event.get('[message]')).to_h)" + } } + mutate { id => "mutate_split_zeek_pe_section_names" + split => { "[zeek_cols][section_names]" => "," } } } } else if ([log_source] == "profinet") { @@ -2410,23 +2628,25 @@ filter { # profinet.log # https://github.com/amzn/zeek-plugin-profinet/blob/master/scripts/main.zeek - dissect { - id => "dissect_zeek_profinet" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][operation_type]} %{[zeek_cols][block_version]} %{[zeek_cols][slot_number]} %{[zeek_cols][subslot_number]} %{[zeek_cols][index]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_profinet" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_profinet" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][operation_type]} %{[zeek_cols][block_version]} %{[zeek_cols][slot_number]} %{[zeek_cols][subslot_number]} %{[zeek_cols][index]}" + } } - ruby { - id => "ruby_zip_zeek_profinet" - init => "@zeek_profinet_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'operation_type', 'block_version', 'slot_number', 'subslot_number', 'index' ]" - code => "event.set('[zeek_cols]', @zeek_profinet_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_profinet" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_profinet" + init => "@zeek_profinet_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'operation_type', 'block_version', 'slot_number', 'subslot_number', 'index' ]" + code => "event.set('[zeek_cols]', @zeek_profinet_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -2441,23 +2661,25 @@ filter { # profinet_dce_rpc.log # https://github.com/amzn/zeek-plugin-profinet/blob/master/scripts/main.zeek - dissect { - id => "dissect_zeek_profinet_dce_rpc" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][version]} %{[zeek_cols][packet_type]} %{[zeek_cols][object_uuid]} %{[zeek_cols][interface_uuid]} %{[zeek_cols][activity_uuid]} %{[zeek_cols][server_boot_time]} %{[zeek_cols][operation]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_profinet_dce_rpc" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_profinet_dce_rpc" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][version]} %{[zeek_cols][packet_type]} %{[zeek_cols][object_uuid]} %{[zeek_cols][interface_uuid]} %{[zeek_cols][activity_uuid]} %{[zeek_cols][server_boot_time]} %{[zeek_cols][operation]}" + } } - ruby { - id => "ruby_zip_zeek_profinet_dce_rpc" - init => "@zeek_profinet_dce_rpc_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'version', 'packet_type', 'object_uuid', 'interface_uuid', 'activity_uuid', 'server_boot_time', 'operation' ]" - code => "event.set('[zeek_cols]', @zeek_profinet_dce_rpc_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_profinet_dce_rpc" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_profinet_dce_rpc" + init => "@zeek_profinet_dce_rpc_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'version', 'packet_type', 'object_uuid', 'interface_uuid', 'activity_uuid', 'server_boot_time', 'operation' ]" + code => "event.set('[zeek_cols]', @zeek_profinet_dce_rpc_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -2467,96 +2689,116 @@ filter { add_tag => [ "ics" ] } - } else if ([log_source] == "radius") { + } else if ([log_source] == "profinet_io_cm") { ############################################################################################################################# - # radius.log - # https://docs.zeek.org/en/stable/scripts/base/protocols/radius/main.zeek.html#type-RADIUS::Info + # profinet_io_cm.log + # https://github.com/cisagov/icsnpp-profinet-io-cm - dissect { - id => "dissect_zeek_radius" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][user]} %{[zeek_cols][mac]} %{[zeek_cols][framed_addr]} %{[zeek_cols][tunnel_client]} %{[zeek_cols][connect_info]} %{[zeek_cols][reply_msg]} %{[zeek_cols][result]} %{[zeek_cols][ttl]}" - } - } - if ("_dissectfailure" in [tags]) { + if ("_jsonparsesuccess" in [tags]) { mutate { - id => "mutate_split_zeek_radius" + id => "mutate_rename_zeek_json_profinet_io_cm_fields" + rename => { "[zeek_cols][array_of_sel_ack]" => "[zeek_cols][sel_ack]" } + rename => { "[zeek_cols][operation_num]" => "[zeek_cols][operation]" } + } + + } else { + dissect { + id => "dissect_zeek_profinet_io_cm" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][proto]} %{[zeek_cols][rpc_version]} %{[zeek_cols][packet_type]} %{[zeek_cols][reserved_for_impl_1]} %{[zeek_cols][last_fragment]} %{[zeek_cols][fragment]} %{[zeek_cols][no_fragment_requested]} %{[zeek_cols][maybe]} %{[zeek_cols][idempotent]} %{[zeek_cols][broadcast]} %{[zeek_cols][reserved_for_impl_2]} %{[zeek_cols][cancel_was_pending_at_call_end]} %{[zeek_cols][integer_encoding]} %{[zeek_cols][character_encoding]} %{[zeek_cols][floating_point_encoding]} %{[zeek_cols][serial_high]} %{[zeek_cols][object_uuid]} %{[zeek_cols][interface_uuid]} %{[zeek_cols][activity_uuid]} %{[zeek_cols][server_boot_time]} %{[zeek_cols][interface_vers_major]} %{[zeek_cols][interface_vers_minor]} %{[zeek_cols][sequence_num]} %{[zeek_cols][operation]} %{[zeek_cols][interface_hint]} %{[zeek_cols][activity_hint]} %{[zeek_cols][len_of_body]} %{[zeek_cols][fragment_num]} %{[zeek_cols][auth_protocol]} %{[zeek_cols][serial_low]} %{[zeek_cols][vers_fack]} %{[zeek_cols][window_size]} %{[zeek_cols][max_tsdu]} %{[zeek_cols][max_frag_size]} %{[zeek_cols][serial_number]} %{[zeek_cols][sel_ack_len]} %{[zeek_cols][sel_ack]}" + } } - ruby { - id => "ruby_zip_zeek_radius" - init => "@zeek_radius_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'user', 'mac', 'framed_addr', 'tunnel_client', 'connect_info', 'reply_msg', 'result', 'ttl' ]" - code => "event.set('[zeek_cols]', @zeek_radius_field_names.zip(event.get('[message]')).to_h)" + + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_profinet_io_cm" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_profinet_io_cm" + init => "$zeek_profinet_io_cm_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto', 'rpc_version', 'packet_type', 'reserved_for_impl_1', 'last_fragment', 'fragment', 'no_fragment_requested', 'maybe', 'idempotent', 'broadcast', 'reserved_for_impl_2', 'cancel_was_pending_at_call_end', 'integer_encoding', 'character_encoding', 'floating_point_encoding', 'serial_high', 'object_uuid', 'interface_uuid', 'activity_uuid', 'server_boot_time', 'interface_vers_major', 'interface_vers_minor', 'sequence_num', 'operation', 'interface_hint', 'activity_hint', 'len_of_body', 'fragment_num', 'auth_protocol', 'serial_low', 'vers_fack', 'window_size', 'max_tsdu', 'max_frag_size', 'serial_number', 'sel_ack_len', 'sel_ack' ]" + code => "event.set('[zeek_cols]', $zeek_profinet_io_cm_field_names.zip(event.get('[message]')).to_h)" + } } + mutate { id => "mutate_split_zeek_profinet_io_cm_commas" + split => { "[zeek_cols][sel_ack]" => "," } } } mutate { - id => "mutate_add_fields_zeek_radius" - add_field => { "[zeek_cols][service]" => "radius" } + id => "mutate_add_fields_zeek_profinet_io_cm" + add_field => { + "[zeek_cols][proto]" => "udp" + } + add_tag => [ "ics" ] } - } else if ([log_source] == "rdp") { + } else if ([log_source] == "radius") { ############################################################################################################################# - # rdp.log - # https://docs.zeek.org/en/stable/scripts/base/protocols/rdp/main.zeek.html#type-RDP::Info - - if ([@metadata][zeek_fields_bitmap] and [@metadata][zeek_fields_bitmap_version]) { - - # bitmap rdp.log field configuration version 0 - # all fields: 0x03FFFFF / 4194303 - # minus client_channels: 0x03FFDFF / 4193791 + # radius.log + # https://docs.zeek.org/en/stable/scripts/base/protocols/radius/main.zeek.html#type-RADIUS::Info - if ([@metadata][zeek_fields_bitmap_version] == 0) { + if ("_jsonparsesuccess" in [tags]) { + mutate { + id => "mutate_rename_zeek_json_radius_fields" + rename => { "[zeek_cols][username]" => "[zeek_cols][user]" } + } - if ([@metadata][zeek_fields_bitmap] == 4194303) { - dissect { - id => "dissect_zeek_rdp_with_all_fields" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][cookie]} %{[zeek_cols][result]} %{[zeek_cols][security_protocol]} %{[zeek_cols][client_channels]} %{[zeek_cols][keyboard_layout]} %{[zeek_cols][client_build]} %{[zeek_cols][client_name]} %{[zeek_cols][client_dig_product_id]} %{[zeek_cols][desktop_width]} %{[zeek_cols][desktop_height]} %{[zeek_cols][requested_color_depth]} %{[zeek_cols][cert_type]} %{[zeek_cols][cert_count]} %{[zeek_cols][cert_permanent]} %{[zeek_cols][encryption_level]} %{[zeek_cols][encryption_method]}" - } - } - - } else if ([@metadata][zeek_fields_bitmap] == 4193791) { - dissect { - id => "dissect_zeek_rdp_with_all_fields_minus_client_channels" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][cookie]} %{[zeek_cols][result]} %{[zeek_cols][security_protocol]} %{[zeek_cols][keyboard_layout]} %{[zeek_cols][client_build]} %{[zeek_cols][client_name]} %{[zeek_cols][client_dig_product_id]} %{[zeek_cols][desktop_width]} %{[zeek_cols][desktop_height]} %{[zeek_cols][requested_color_depth]} %{[zeek_cols][cert_type]} %{[zeek_cols][cert_count]} %{[zeek_cols][cert_permanent]} %{[zeek_cols][encryption_level]} %{[zeek_cols][encryption_method]}" - } - } - - } else { - # who knows? the rdp.log preprocessed bitmap is not one we're expecting, we've got to guess and cannot use dissect - mutate { id => "mutate_add_tag_dissect_failure_unknown_rdp_bitmap" - add_tag => [ "_dissectfailure" ] } + } else { + dissect { + id => "dissect_zeek_radius" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][user]} %{[zeek_cols][mac]} %{[zeek_cols][framed_addr]} %{[zeek_cols][tunnel_client]} %{[zeek_cols][connect_info]} %{[zeek_cols][reply_msg]} %{[zeek_cols][result]} %{[zeek_cols][ttl]}" } - - } else { - # who knows? the rdp.log preprocessed bitmap field list version is not one we're expecting, we've got to guess and cannot use dissect - mutate { id => "mutate_add_tag_dissect_failure_unknown_bitmap_rdp_version" - add_tag => [ "_dissectfailure" ] } } + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_radius" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_radius" + init => "@zeek_radius_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'user', 'mac', 'framed_addr', 'tunnel_client', 'connect_info', 'reply_msg', 'result', 'ttl' ]" + code => "event.set('[zeek_cols]', @zeek_radius_field_names.zip(event.get('[message]')).to_h)" + } + } + } - } else { - # who knows? the rdp.log was not preprocessed to determine fields, we've got to guess and cannot use dissect - mutate { id => "mutate_add_tag_dissect_failure_missing_rdp_bitmap" - add_tag => [ "_dissectfailure" ] } + mutate { + id => "mutate_add_fields_zeek_radius" + add_field => { "[zeek_cols][service]" => "radius" } } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_rdp" + } else if ([log_source] == "rdp") { + ############################################################################################################################# + # rdp.log + # https://docs.zeek.org/en/stable/scripts/base/protocols/rdp/main.zeek.html#type-RDP::Info + + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_rdp_with_all_fields" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][cookie]} %{[zeek_cols][result]} %{[zeek_cols][security_protocol]} %{[zeek_cols][client_channels]} %{[zeek_cols][keyboard_layout]} %{[zeek_cols][client_build]} %{[zeek_cols][client_name]} %{[zeek_cols][client_dig_product_id]} %{[zeek_cols][desktop_width]} %{[zeek_cols][desktop_height]} %{[zeek_cols][requested_color_depth]} %{[zeek_cols][cert_type]} %{[zeek_cols][cert_count]} %{[zeek_cols][cert_permanent]} %{[zeek_cols][encryption_level]} %{[zeek_cols][encryption_method]}" + } } - ruby { - id => "ruby_zip_zeek_rdp" - init => "@zeek_rdp_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'cookie', 'result', 'security_protocol', 'client_channels', 'keyboard_layout', 'client_build', 'client_name', 'client_dig_product_id', 'desktop_width', 'desktop_height', 'requested_color_depth', 'cert_type', 'cert_count', 'cert_permanent', 'encryption_level', 'encryption_method' ]" - code => "event.set('[zeek_cols]', @zeek_rdp_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_rdp" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_rdp" + init => "@zeek_rdp_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'cookie', 'result', 'security_protocol', 'client_channels', 'keyboard_layout', 'client_build', 'client_name', 'client_dig_product_id', 'desktop_width', 'desktop_height', 'requested_color_depth', 'cert_type', 'cert_count', 'cert_permanent', 'encryption_level', 'encryption_method' ]" + code => "event.set('[zeek_cols]', @zeek_rdp_field_names.zip(event.get('[message]')).to_h)" + } } + mutate { id => "mutate_split_zeek_rdp_client_channels" + split => { "[zeek_cols][client_channels]" => "," } } } mutate { @@ -2573,23 +2815,25 @@ filter { # rfb.log # https://docs.zeek.org/en/stable/scripts/base/protocols/rfb/main.zeek.html#type-RFB::Info - dissect { - id => "dissect_zeek_rfb" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][client_major_version]} %{[zeek_cols][client_minor_version]} %{[zeek_cols][server_major_version]} %{[zeek_cols][server_minor_version]} %{[zeek_cols][authentication_method]} %{[zeek_cols][auth]} %{[zeek_cols][share_flag]} %{[zeek_cols][desktop_name]} %{[zeek_cols][width]} %{[zeek_cols][height]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_rfb" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_rfb" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][client_major_version]} %{[zeek_cols][client_minor_version]} %{[zeek_cols][server_major_version]} %{[zeek_cols][server_minor_version]} %{[zeek_cols][authentication_method]} %{[zeek_cols][auth]} %{[zeek_cols][share_flag]} %{[zeek_cols][desktop_name]} %{[zeek_cols][width]} %{[zeek_cols][height]}" + } } - ruby { - id => "ruby_zip_zeek_rfb" - init => "@zeek_rfb_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'client_major_version', 'client_minor_version', 'server_major_version', 'server_minor_version', 'authentication_method', 'auth', 'share_flag', 'desktop_name', 'width', 'height' ]" - code => "event.set('[zeek_cols]', @zeek_rfb_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_rfb" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_rfb" + init => "@zeek_rfb_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'client_major_version', 'client_minor_version', 'server_major_version', 'server_minor_version', 'authentication_method', 'auth', 'share_flag', 'desktop_name', 'width', 'height' ]" + code => "event.set('[zeek_cols]', @zeek_rfb_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -2603,23 +2847,25 @@ filter { # s7comm.log # https://github.com/cisagov/icsnpp-s7comm - dissect { - id => "dissect_zeek_s7comm" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][rosctr_code]} %{[zeek_cols][rosctr_name]} %{[zeek_cols][pdu_reference]} %{[zeek_cols][function_code]} %{[zeek_cols][function_name]} %{[zeek_cols][subfunction_code]} %{[zeek_cols][subfunction_name]} %{[zeek_cols][error_class]} %{[zeek_cols][error_code]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_s7comm" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_s7comm" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][rosctr_code]} %{[zeek_cols][rosctr_name]} %{[zeek_cols][pdu_reference]} %{[zeek_cols][function_code]} %{[zeek_cols][function_name]} %{[zeek_cols][subfunction_code]} %{[zeek_cols][subfunction_name]} %{[zeek_cols][error_class]} %{[zeek_cols][error_code]}" + } } - ruby { - id => "ruby_zip_zeek_s7comm" - init => "@zeek_s7comm_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'rosctr_code', 'rosctr_name', 'pdu_reference', 'function_code', 'function_name', 'subfunction_code', 'subfunction_name', 'error_class', 'error_code' ]" - code => "event.set('[zeek_cols]', @zeek_s7comm_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_s7comm" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_s7comm" + init => "@zeek_s7comm_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'rosctr_code', 'rosctr_name', 'pdu_reference', 'function_code', 'function_name', 'subfunction_code', 'subfunction_name', 'error_class', 'error_code' ]" + code => "event.set('[zeek_cols]', @zeek_s7comm_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -2637,23 +2883,25 @@ filter { # s7comm_plus.log # https://github.com/cisagov/icsnpp-s7comm - dissect { - id => "dissect_zeek_s7comm_plus" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][version]} %{[zeek_cols][opcode]} %{[zeek_cols][opcode_name]} %{[zeek_cols][function_code]} %{[zeek_cols][function_name]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_s7comm_plus" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_s7comm_plus" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][version]} %{[zeek_cols][opcode]} %{[zeek_cols][opcode_name]} %{[zeek_cols][function_code]} %{[zeek_cols][function_name]}" + } } - ruby { - id => "ruby_zip_zeek_s7comm_plus" - init => "@zeek_s7comm_plus_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'version', 'opcode', 'opcode_name', 'function_code', 'function_name' ]" - code => "event.set('[zeek_cols]', @zeek_s7comm_plus_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_s7comm_plus" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_s7comm_plus" + init => "@zeek_s7comm_plus_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'version', 'opcode', 'opcode_name', 'function_code', 'function_name' ]" + code => "event.set('[zeek_cols]', @zeek_s7comm_plus_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -2671,23 +2919,25 @@ filter { # s7comm_read_szl.log # https://github.com/cisagov/icsnpp-s7comm - dissect { - id => "dissect_zeek_s7comm_read_szl" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][pdu_reference]} %{[zeek_cols][method]} %{[zeek_cols][szl_id]} %{[zeek_cols][szl_id_name]} %{[zeek_cols][szl_index]} %{[zeek_cols][return_code]} %{[zeek_cols][return_code_name]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_s7comm_read_szl" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_s7comm_read_szl" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][pdu_reference]} %{[zeek_cols][method]} %{[zeek_cols][szl_id]} %{[zeek_cols][szl_id_name]} %{[zeek_cols][szl_index]} %{[zeek_cols][return_code]} %{[zeek_cols][return_code_name]}" + } } - ruby { - id => "ruby_zip_zeek_s7comm_read_szl" - init => "@zeek_s7comm_read_szl_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'pdu_reference', 'method', 'szl_id', 'szl_id_name', 'szl_index', 'return_code', 'return_code_name' ]" - code => "event.set('[zeek_cols]', @zeek_s7comm_read_szl_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_s7comm_read_szl" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_s7comm_read_szl" + init => "@zeek_s7comm_read_szl_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'pdu_reference', 'method', 'szl_id', 'szl_id_name', 'szl_index', 'return_code', 'return_code_name' ]" + code => "event.set('[zeek_cols]', @zeek_s7comm_read_szl_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -2705,23 +2955,31 @@ filter { # s7comm_upload_download.log # https://github.com/cisagov/icsnpp-s7comm - dissect { - id => "dissect_zeek_s7comm_upload_download" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][rosctr_name]} %{[zeek_cols][pdu_reference]} %{[zeek_cols][function_name]} %{[zeek_cols][function_status]} %{[zeek_cols][session_id]} %{[zeek_cols][blocklength]} %{[zeek_cols][filename]} %{[zeek_cols][block_type]} %{[zeek_cols][block_number]} %{[zeek_cols][destination_filesystem]}" - } - } - if ("_dissectfailure" in [tags]) { + if ("_jsonparsesuccess" in [tags]) { mutate { - id => "mutate_split_zeek_s7comm_upload_download" + id => "mutate_rename_zeek_json_s7comm_upload_download_fields" + rename => { "[zeek_cols][rosctr]" => "[zeek_cols][rosctr_name]" } + } + + } else { + dissect { + id => "dissect_zeek_s7comm_upload_download" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][rosctr_name]} %{[zeek_cols][pdu_reference]} %{[zeek_cols][function_name]} %{[zeek_cols][function_status]} %{[zeek_cols][session_id]} %{[zeek_cols][blocklength]} %{[zeek_cols][filename]} %{[zeek_cols][block_type]} %{[zeek_cols][block_number]} %{[zeek_cols][destination_filesystem]}" + } } - ruby { - id => "ruby_zip_zeek_s7comm_upload_download" - init => "@zeek_s7comm_upload_download_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'rosctr_name', 'pdu_reference', 'function_name', 'function_status', 'session_id', 'blocklength', 'filename', 'block_type', 'block_number', 'destination_filesystem' ]" - code => "event.set('[zeek_cols]', @zeek_s7comm_upload_download_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_s7comm_upload_download" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_s7comm_upload_download" + init => "@zeek_s7comm_upload_download_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'rosctr_name', 'pdu_reference', 'function_name', 'function_status', 'session_id', 'blocklength', 'filename', 'block_type', 'block_number', 'destination_filesystem' ]" + code => "event.set('[zeek_cols]', @zeek_s7comm_upload_download_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -2738,23 +2996,31 @@ filter { ############################################################################################################################# # signatures.log - dissect { - id => "dissect_zeek_signatures" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][note]} %{[zeek_cols][signature_id]} %{[zeek_cols][event_message]} %{[zeek_cols][sub_message]} %{[zeek_cols][signature_count]} %{[zeek_cols][host_count]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_signatures" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_signatures" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][note]} %{[zeek_cols][signature_id]} %{[zeek_cols][event_message]} %{[zeek_cols][sub_message]} %{[zeek_cols][signature_count]} %{[zeek_cols][host_count]}" + } + } + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_signatures" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_signatures" + init => "@zeek_signatures_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'note', 'signature_id', 'event_message', 'sub_message', 'signature_count', 'host_count' ]" + code => "event.set('[zeek_cols]', @zeek_signatures_field_names.zip(event.get('[message]')).to_h)" + } } - ruby { - id => "ruby_zip_zeek_signatures" - init => "@zeek_signatures_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'note', 'signature_id', 'event_message', 'sub_message', 'signature_count', 'host_count' ]" - code => "event.set('[zeek_cols]', @zeek_signatures_field_names.zip(event.get('[message]')).to_h)" + if ("_carved" in [tags]) { + # Malcolm does some "special" stuff in zeek_carve_logger.py for file carving, sort of hijacking signatures.log for it: + # - _carved signature logs' sub_message contains fuid(s) comma-separated + mutate { id => "mutate_split_zeek_signatures_sub_message" + split => { "[zeek_cols][sub_message]" => "," } } } } @@ -2763,24 +3029,29 @@ filter { # sip.log # https://docs.zeek.org/en/stable/scripts/base/protocols/sip/main.zeek.html#type-SIP::Info - dissect { - id => "dissect_zeek_sip" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][trans_depth]} %{[zeek_cols][method]} %{[zeek_cols][uri]} %{[zeek_cols][date]} %{[zeek_cols][request_from]} %{[zeek_cols][request_to]} %{[zeek_cols][response_from]} %{[zeek_cols][response_to]} %{[zeek_cols][reply_to]} %{[zeek_cols][call_id]} %{[zeek_cols][seq]} %{[zeek_cols][subject]} %{[zeek_cols][request_path]} %{[zeek_cols][response_path]} %{[zeek_cols][user_agent]} %{[zeek_cols][status_code]} %{[zeek_cols][status_msg]} %{[zeek_cols][warning]} %{[zeek_cols][request_body_len]} %{[zeek_cols][response_body_len]} %{[zeek_cols][content_type]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_sip" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_sip" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][trans_depth]} %{[zeek_cols][method]} %{[zeek_cols][uri]} %{[zeek_cols][date]} %{[zeek_cols][request_from]} %{[zeek_cols][request_to]} %{[zeek_cols][response_from]} %{[zeek_cols][response_to]} %{[zeek_cols][reply_to]} %{[zeek_cols][call_id]} %{[zeek_cols][seq]} %{[zeek_cols][subject]} %{[zeek_cols][request_path]} %{[zeek_cols][response_path]} %{[zeek_cols][user_agent]} %{[zeek_cols][status_code]} %{[zeek_cols][status_msg]} %{[zeek_cols][warning]} %{[zeek_cols][request_body_len]} %{[zeek_cols][response_body_len]} %{[zeek_cols][content_type]}" + } } - ruby { - id => "ruby_zip_zeek_sip" - init => "@zeek_sip_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'trans_depth', 'method', 'uri', 'date', 'request_from', 'request_to', 'response_from', 'response_to', 'reply_to', 'call_id', 'seq', 'subject', 'request_path', 'response_path', 'user_agent', 'status_code', 'status_msg', 'warning', 'request_body_len', 'response_body_len', 'content_type' ]" - code => "event.set('[zeek_cols]', @zeek_sip_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_sip" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_sip" + init => "@zeek_sip_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'trans_depth', 'method', 'uri', 'date', 'request_from', 'request_to', 'response_from', 'response_to', 'reply_to', 'call_id', 'seq', 'subject', 'request_path', 'response_path', 'user_agent', 'status_code', 'status_msg', 'warning', 'request_body_len', 'response_body_len', 'content_type' ]" + code => "event.set('[zeek_cols]', @zeek_sip_field_names.zip(event.get('[message]')).to_h)" + } } + mutate { id => "mutate_split_zeek_sip_commas" + split => { "[zeek_cols][request_path]" => "," + "[zeek_cols][response_path]" => "," } } } mutate { @@ -2796,23 +3067,34 @@ filter { # note that smb_cmd.referenced_file is exactly the same structure as the log line for smb_files. later on it will be # merged up as its own top-level entity so I don't have to duplicate the parsing effort below - dissect { - id => "dissect_zeek_smb_cmd" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][command]} %{[zeek_cols][sub_command]} %{[zeek_cols][argument]} %{[zeek_cols][status]} %{[zeek_cols][rtt]} %{[zeek_cols][version]} %{[zeek_cols][user]} %{[zeek_cols][tree]} %{[zeek_cols][tree_service]} %{[zeek_cols][referenced_file][ts]} %{[zeek_cols][referenced_file][uid]} %{[zeek_cols][referenced_file][orig_h]} %{[zeek_cols][referenced_file][orig_p]} %{[zeek_cols][referenced_file][resp_h]} %{[zeek_cols][referenced_file][resp_p]} %{[zeek_cols][referenced_file][fuid]} %{[zeek_cols][referenced_file][action]} %{[zeek_cols][referenced_file][path]} %{[zeek_cols][referenced_file][name]} %{[zeek_cols][referenced_file][size]} %{[zeek_cols][referenced_file][prev_name]} %{[zeek_cols][referenced_file][times_modified]} %{[zeek_cols][referenced_file][times_accessed]} %{[zeek_cols][referenced_file][times_created]} %{[zeek_cols][referenced_file][times_changed]} %{[zeek_cols][referenced_file][data_offset_req]} %{[zeek_cols][referenced_file][data_len_req]} %{[zeek_cols][referenced_file][data_len_rsp]}" - } - } - if ("_dissectfailure" in [tags]) { + if ("_jsonparsesuccess" in [tags]) { mutate { - id => "mutate_split_zeek_smb_cmd" + id => "mutate_rename_zeek_json_smb_cmd_referenced_file_fields" + rename => { "[zeek_cols][referenced_file.id.orig_h]" => "[zeek_cols][referenced_file][orig_h]" } + rename => { "[zeek_cols][referenced_file.id.orig_p]" => "[zeek_cols][referenced_file][orig_p]" } + rename => { "[zeek_cols][referenced_file.id.resp_h]" => "[zeek_cols][referenced_file][resp_h]" } + rename => { "[zeek_cols][referenced_file.id.resp_p]" => "[zeek_cols][referenced_file][resp_p]" } + } + + } else { + dissect { + id => "dissect_zeek_smb_cmd" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][command]} %{[zeek_cols][sub_command]} %{[zeek_cols][argument]} %{[zeek_cols][status]} %{[zeek_cols][rtt]} %{[zeek_cols][version]} %{[zeek_cols][user]} %{[zeek_cols][tree]} %{[zeek_cols][tree_service]} %{[zeek_cols][referenced_file][ts]} %{[zeek_cols][referenced_file][uid]} %{[zeek_cols][referenced_file][orig_h]} %{[zeek_cols][referenced_file][orig_p]} %{[zeek_cols][referenced_file][resp_h]} %{[zeek_cols][referenced_file][resp_p]} %{[zeek_cols][referenced_file][fuid]} %{[zeek_cols][referenced_file][action]} %{[zeek_cols][referenced_file][path]} %{[zeek_cols][referenced_file][name]} %{[zeek_cols][referenced_file][size]} %{[zeek_cols][referenced_file][prev_name]} %{[zeek_cols][referenced_file][times_modified]} %{[zeek_cols][referenced_file][times_accessed]} %{[zeek_cols][referenced_file][times_created]} %{[zeek_cols][referenced_file][times_changed]} %{[zeek_cols][referenced_file][data_offset_req]} %{[zeek_cols][referenced_file][data_len_req]} %{[zeek_cols][referenced_file][data_len_rsp]}" + } } - ruby { - id => "ruby_zip_zeek_smb_cmd" - init => "@zeek_smb_cmd_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'command', 'sub_command', 'argument', 'status', 'rtt', 'version', 'user', 'tree', 'tree_service', 'referenced_file.ts', 'referenced_file.uid', 'referenced_file.orig_h', 'referenced_file.orig_p', 'referenced_file.resp_h', 'referenced_file.resp_p', 'referenced_file.fuid', 'referenced_file.action', 'referenced_file.path', 'referenced_file.name', 'referenced_file.size', 'referenced_file.prev_name', 'referenced_file.times_modified', 'referenced_file.times_accessed', 'referenced_file.times_created', 'referenced_file.times_changed', 'referenced_file.data_offset_req', 'referenced_file.data_len_req', 'referenced_file.data_len_rsp' ]" - code => "event.set('[zeek_cols]', @zeek_smb_cmd_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_smb_cmd" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_smb_cmd" + init => "@zeek_smb_cmd_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'command', 'sub_command', 'argument', 'status', 'rtt', 'version', 'user', 'tree', 'tree_service', 'referenced_file.ts', 'referenced_file.uid', 'referenced_file.orig_h', 'referenced_file.orig_p', 'referenced_file.resp_h', 'referenced_file.resp_p', 'referenced_file.fuid', 'referenced_file.action', 'referenced_file.path', 'referenced_file.name', 'referenced_file.size', 'referenced_file.prev_name', 'referenced_file.times_modified', 'referenced_file.times_accessed', 'referenced_file.times_created', 'referenced_file.times_changed', 'referenced_file.data_offset_req', 'referenced_file.data_len_req', 'referenced_file.data_len_rsp' ]" + code => "event.set('[zeek_cols]', @zeek_smb_cmd_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -2836,65 +3118,25 @@ filter { # smb_files.log # https://docs.zeek.org/en/stable/scripts/base/protocols/smb/main.zeek.html#type-SMB::FileInfo - if ([@metadata][zeek_fields_bitmap] and [@metadata][zeek_fields_bitmap_version]) { - - # bitmap smb_files.log field configuration version 0 - # - # smb_files.log without data_offset_req, data_len_req, data_len_rsp : 0x0000FFFF / 65535 - # smb_files.log with data_offset_req, data_len_req, data_len_rsp : 0x0007FFFF / 524287 - - # smb_files.log with data_offset_req, data_len_req, data_len_rsp - - if ([@metadata][zeek_fields_bitmap_version] == 0) { - - if ([@metadata][zeek_fields_bitmap] == 524287) { - - dissect { - id => "dissect_zeek_smb_files_with_all_fields" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][fuid]} %{[zeek_cols][action]} %{[zeek_cols][path]} %{[zeek_cols][name]} %{[zeek_cols][size]} %{[zeek_cols][prev_name]} %{[zeek_cols][times_modified]} %{[zeek_cols][times_accessed]} %{[zeek_cols][times_created]} %{[zeek_cols][times_changed]} %{[zeek_cols][data_offset_req]} %{[zeek_cols][data_len_req]} %{[zeek_cols][data_len_rsp]}" - } - } - - } else if ([@metadata][zeek_fields_bitmap] == 65535) { - - dissect { - id => "dissect_zeek_smb_files_without_data_lens" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][fuid]} %{[zeek_cols][action]} %{[zeek_cols][path]} %{[zeek_cols][name]} %{[zeek_cols][size]} %{[zeek_cols][prev_name]} %{[zeek_cols][times_modified]} %{[zeek_cols][times_accessed]} %{[zeek_cols][times_created]} %{[zeek_cols][times_changed]}" - } - } - - } else { - # who knows? the smb_files.log preprocessed bitmap is not one we're expecting, we've got to guess and cannot use dissect - mutate { id => "mutate_add_tag_dissect_failure_unknown_smb_files_bitmap" - add_tag => [ "_dissectfailure" ] } - } - - } else { - # who knows? the smb_files.log preprocessed bitmap field list version is not one we're expecting, we've got to guess and cannot use dissect - mutate { id => "mutate_add_tag_dissect_failure_unknown_bitmap_smb_files_version" - add_tag => [ "_dissectfailure" ] } - } - - } else { - # who knows? the smb_files.log was not preprocessed to determine fields, we've got to guess and cannot use dissect - mutate { id => "mutate_add_tag_dissect_failure_missing_smb_files_bitmap" - add_tag => [ "_dissectfailure" ] } - } - - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_smb_files" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_smb_files_with_all_fields" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][fuid]} %{[zeek_cols][action]} %{[zeek_cols][path]} %{[zeek_cols][name]} %{[zeek_cols][size]} %{[zeek_cols][prev_name]} %{[zeek_cols][times_modified]} %{[zeek_cols][times_accessed]} %{[zeek_cols][times_created]} %{[zeek_cols][times_changed]} %{[zeek_cols][data_offset_req]} %{[zeek_cols][data_len_req]} %{[zeek_cols][data_len_rsp]}" + } } - ruby { - id => "ruby_zip_zeek_smb_files" - init => "@zeek_smb_files_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'fuid', 'action', 'path', 'name', 'size', 'prev_name', 'times_modified', 'times_accessed', 'times_created', 'times_changed', 'data_offset_req', 'data_len_req', 'data_len_rsp' ]" - code => "event.set('[zeek_cols]', @zeek_smb_files_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_smb_files" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_smb_files" + init => "@zeek_smb_files_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'fuid', 'action', 'path', 'name', 'size', 'prev_name', 'times_modified', 'times_accessed', 'times_created', 'times_changed', 'data_offset_req', 'data_len_req', 'data_len_rsp' ]" + code => "event.set('[zeek_cols]', @zeek_smb_files_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -2914,23 +3156,25 @@ filter { # smb_mapping.log # https://docs.zeek.org/en/stable/scripts/base/protocols/smb/main.zeek.html#type-SMB::TreeInfo - dissect { - id => "dissect_zeek_smb_mapping" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][path]} %{[zeek_cols][resource_type]} %{[zeek_cols][native_file_system]} %{[zeek_cols][share_type]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_smb_mapping" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_smb_mapping" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][path]} %{[zeek_cols][resource_type]} %{[zeek_cols][native_file_system]} %{[zeek_cols][share_type]}" + } } - ruby { - id => "ruby_zip_zeek_smb_mapping" - init => "@zeek_smb_mapping_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'path', 'resource_type', 'native_file_system', 'share_type' ]" - code => "event.set('[zeek_cols]', @zeek_smb_mapping_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_smb_mapping" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_smb_mapping" + init => "@zeek_smb_mapping_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'path', 'resource_type', 'native_file_system', 'share_type' ]" + code => "event.set('[zeek_cols]', @zeek_smb_mapping_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -2947,24 +3191,31 @@ filter { # smtp.log # https://docs.zeek.org/en/stable/scripts/base/protocols/smtp/main.zeek.html#type-SMTP::Info - dissect { - id => "dissect_zeek_smtp" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][trans_depth]} %{[zeek_cols][helo]} %{[zeek_cols][mailfrom]} %{[zeek_cols][rcptto]} %{[zeek_cols][date]} %{[zeek_cols][from]} %{[zeek_cols][to]} %{[zeek_cols][cc]} %{[zeek_cols][reply_to]} %{[zeek_cols][msg_id]} %{[zeek_cols][in_reply_to]} %{[zeek_cols][subject]} %{[zeek_cols][x_originating_ip]} %{[zeek_cols][first_received]} %{[zeek_cols][second_received]} %{[zeek_cols][last_reply]} %{[zeek_cols][path]} %{[zeek_cols][user_agent]} %{[zeek_cols][tls]} %{[zeek_cols][fuid]} %{[zeek_cols][is_webmail]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_smtp" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_smtp" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][trans_depth]} %{[zeek_cols][helo]} %{[zeek_cols][mailfrom]} %{[zeek_cols][rcptto]} %{[zeek_cols][date]} %{[zeek_cols][from]} %{[zeek_cols][to]} %{[zeek_cols][cc]} %{[zeek_cols][reply_to]} %{[zeek_cols][msg_id]} %{[zeek_cols][in_reply_to]} %{[zeek_cols][subject]} %{[zeek_cols][x_originating_ip]} %{[zeek_cols][first_received]} %{[zeek_cols][second_received]} %{[zeek_cols][last_reply]} %{[zeek_cols][path]} %{[zeek_cols][user_agent]} %{[zeek_cols][tls]} %{[zeek_cols][fuid]} %{[zeek_cols][is_webmail]}" + } } - ruby { - id => "ruby_zip_zeek_smtp" - init => "@zeek_smtp_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'trans_depth', 'helo', 'mailfrom', 'rcptto', 'date', 'from', 'to', 'cc', 'reply_to', 'msg_id', 'in_reply_to', 'subject', 'x_originating_ip', 'first_received', 'second_received', 'last_reply', 'path', 'user_agent', 'tls', 'fuid', 'is_webmail' ]" - code => "event.set('[zeek_cols]', @zeek_smtp_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_smtp" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_smtp" + init => "@zeek_smtp_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'trans_depth', 'helo', 'mailfrom', 'rcptto', 'date', 'from', 'to', 'cc', 'reply_to', 'msg_id', 'in_reply_to', 'subject', 'x_originating_ip', 'first_received', 'second_received', 'last_reply', 'path', 'user_agent', 'tls', 'fuid', 'is_webmail' ]" + code => "event.set('[zeek_cols]', @zeek_smtp_field_names.zip(event.get('[message]')).to_h)" + } } + mutate { id => "mutate_split_zeek_smtp_commas" + split => { "[zeek_cols][rcptto]" => "," + "[zeek_cols][to]" => "," + "[zeek_cols][cc]" => "," + "[zeek_cols][path]" => "," } } } mutate { @@ -2980,23 +3231,25 @@ filter { # snmp.log # https://docs.zeek.org/en/stable/scripts/base/protocols/snmp/main.zeek.html#type-SNMP::Info - dissect { - id => "dissect_zeek_snmp" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][duration]} %{[zeek_cols][version]} %{[zeek_cols][community]} %{[zeek_cols][get_requests]} %{[zeek_cols][get_bulk_requests]} %{[zeek_cols][get_responses]} %{[zeek_cols][set_requests]} %{[zeek_cols][display_string]} %{[zeek_cols][up_since]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_snmp" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_snmp" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][duration]} %{[zeek_cols][version]} %{[zeek_cols][community]} %{[zeek_cols][get_requests]} %{[zeek_cols][get_bulk_requests]} %{[zeek_cols][get_responses]} %{[zeek_cols][set_requests]} %{[zeek_cols][display_string]} %{[zeek_cols][up_since]}" + } } - ruby { - id => "ruby_zip_zeek_snmp" - init => "@zeek_snmp_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'duration', 'version', 'community', 'get_requests', 'get_bulk_requests', 'get_responses', 'set_requests', 'display_string', 'up_since' ]" - code => "event.set('[zeek_cols]', @zeek_snmp_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_snmp" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_snmp" + init => "@zeek_snmp_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'duration', 'version', 'community', 'get_requests', 'get_bulk_requests', 'get_responses', 'set_requests', 'display_string', 'up_since' ]" + code => "event.set('[zeek_cols]', @zeek_snmp_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -3013,23 +3266,25 @@ filter { # socks.log # https://docs.zeek.org/en/stable/scripts/base/protocols/socks/main.zeek.html#type-SOCKS::Info - dissect { - id => "dissect_zeek_socks" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][version]} %{[zeek_cols][user]} %{[zeek_cols][password]} %{[zeek_cols][server_status]} %{[zeek_cols][request_host]} %{[zeek_cols][request_name]} %{[zeek_cols][request_port]} %{[zeek_cols][bound_host]} %{[zeek_cols][bound_name]} %{[zeek_cols][bound_port]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_socks" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_socks" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][version]} %{[zeek_cols][user]} %{[zeek_cols][password]} %{[zeek_cols][server_status]} %{[zeek_cols][request_host]} %{[zeek_cols][request_name]} %{[zeek_cols][request_port]} %{[zeek_cols][bound_host]} %{[zeek_cols][bound_name]} %{[zeek_cols][bound_port]}" + } } - ruby { - id => "ruby_zip_zeek_socks" - init => "@zeek_socks_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'version', 'user', 'password', 'server_status', 'request_host', 'request_name', 'request_port', 'bound_host', 'bound_name', 'bound_port' ]" - code => "event.set('[zeek_cols]', @zeek_socks_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_socks" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_socks" + init => "@zeek_socks_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'version', 'user', 'password', 'server_status', 'request_host', 'request_name', 'request_port', 'bound_host', 'bound_name', 'bound_port' ]" + code => "event.set('[zeek_cols]', @zeek_socks_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -3043,23 +3298,37 @@ filter { # software.log # https://docs.zeek.org/en/stable/scripts/base/frameworks/software/main.zeek.html#type-Software::Info - dissect { - id => "dissect_zeek_software" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][software_type]} %{[zeek_cols][name]} %{[zeek_cols][version_major]} %{[zeek_cols][version_minor]} %{[zeek_cols][version_minor2]} %{[zeek_cols][version_minor3]} %{[zeek_cols][version_addl]} %{[zeek_cols][unparsed_version]} %{[zeek_cols][url]}" - } - } - if ("_dissectfailure" in [tags]) { + if ("_jsonparsesuccess" in [tags]) { mutate { - id => "mutate_split_zeek_software" + id => "mutate_rename_zeek_json_software_fields" + rename => { "[zeek_cols][host]" => "[zeek_cols][orig_h]" } + rename => { "[zeek_cols][host_p]" => "[zeek_cols][orig_p]" } + rename => { "[zeek_cols][version.major]" => "[zeek_cols][version_major]" } + rename => { "[zeek_cols][version.minor]" => "[zeek_cols][version_minor]" } + rename => { "[zeek_cols][version.minor2]" => "[zeek_cols][version_minor2]" } + rename => { "[zeek_cols][version.minor3]" => "[zeek_cols][version_minor3]" } + rename => { "[zeek_cols][version.addl]" => "[zeek_cols][version_addl]" } + } + + } else { + dissect { + id => "dissect_zeek_software" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][software_type]} %{[zeek_cols][name]} %{[zeek_cols][version_major]} %{[zeek_cols][version_minor]} %{[zeek_cols][version_minor2]} %{[zeek_cols][version_minor3]} %{[zeek_cols][version_addl]} %{[zeek_cols][unparsed_version]} %{[zeek_cols][url]}" + } } - ruby { - id => "ruby_zip_zeek_software" - init => "@zeek_software_field_names = [ 'ts', 'orig_h', 'orig_p', 'software_type', 'name', 'version_major', 'version_minor', 'version_minor2', 'version_minor3', 'version_addl', 'unparsed_version', 'url' ]" - code => "event.set('[zeek_cols]', @zeek_software_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_software" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_software" + init => "@zeek_software_field_names = [ 'ts', 'orig_h', 'orig_p', 'software_type', 'name', 'version_major', 'version_minor', 'version_minor2', 'version_minor3', 'version_addl', 'unparsed_version', 'url' ]" + code => "event.set('[zeek_cols]', @zeek_software_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -3068,23 +3337,25 @@ filter { # wireguard.log # https://github.com/corelight/zeek-spicy-wireguard/blob/master/analyzer/main.zeek - dissect { - id => "dissect_zeek_wireguard" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][established]} %{[zeek_cols][initiations]} %{[zeek_cols][responses]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_wireguard" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_wireguard" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][established]} %{[zeek_cols][initiations]} %{[zeek_cols][responses]}" + } } - ruby { - id => "ruby_zip_zeek_wireguard" - init => "@zeek_wireguard_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'established', 'initiations', 'responses' ]" - code => "event.set('[zeek_cols]', @zeek_wireguard_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_wireguard" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_wireguard" + init => "@zeek_wireguard_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'established', 'initiations', 'responses' ]" + code => "event.set('[zeek_cols]', @zeek_wireguard_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -3104,64 +3375,31 @@ filter { # ssh.log # https://docs.zeek.org/en/stable/scripts/base/protocols/ssh/main.zeek.html#type-SSH::Info - if ([@metadata][zeek_fields_bitmap] and [@metadata][zeek_fields_bitmap_version]) { - - # bitmap ssh.log field configuration version 0 - # - # standard ssh.log without HASSH fingerprinting add-on script : 0x007FFFFF / 8388607 - # ssh.log with HASSH fingerprinting add-on script : 0x3FFFFFFF / 1073741823 - - if ([@metadata][zeek_fields_bitmap_version] == 0) { - - if ([@metadata][zeek_fields_bitmap] == 1073741823) { - - dissect { - id => "dissect_zeek_ssh_with_all_fields_with_hassh" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][version]} %{[zeek_cols][auth_success]} %{[zeek_cols][auth_attempts]} %{[zeek_cols][direction]} %{[zeek_cols][client]} %{[zeek_cols][server]} %{[zeek_cols][cipher_alg]} %{[zeek_cols][mac_alg]} %{[zeek_cols][compression_alg]} %{[zeek_cols][kex_alg]} %{[zeek_cols][host_key_alg]} %{[zeek_cols][host_key]} %{[zeek_cols][remote_location_country_code]} %{[zeek_cols][remote_location_region]} %{[zeek_cols][remote_location_city]} %{[zeek_cols][remote_location_latitude]} %{[zeek_cols][remote_location_longitude]} %{[zeek_cols][hasshVersion]} %{[zeek_cols][hassh]} %{[zeek_cols][hasshServer]} %{[zeek_cols][cshka]} %{[zeek_cols][hasshAlgorithms]} %{[zeek_cols][sshka]} %{[zeek_cols][hasshServerAlgorithms]}" - } - } - - } else if ([@metadata][zeek_fields_bitmap] == 8388607) { - - dissect { - id => "dissect_zeek_ssh_with_all_fields_no_hassh" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][version]} %{[zeek_cols][auth_success]} %{[zeek_cols][auth_attempts]} %{[zeek_cols][direction]} %{[zeek_cols][client]} %{[zeek_cols][server]} %{[zeek_cols][cipher_alg]} %{[zeek_cols][mac_alg]} %{[zeek_cols][compression_alg]} %{[zeek_cols][kex_alg]} %{[zeek_cols][host_key_alg]} %{[zeek_cols][host_key]} %{[zeek_cols][remote_location_country_code]} %{[zeek_cols][remote_location_region]} %{[zeek_cols][remote_location_city]} %{[zeek_cols][remote_location_latitude]} %{[zeek_cols][remote_location_longitude]}" - } - } - - } else { - # who knows? the ssh.log preprocessed bitmap is not one we're expecting, we've got to guess and cannot use dissect - mutate { id => "mutate_add_tag_dissect_failure_unknown_ssh_bitmap" - add_tag => [ "_dissectfailure" ] } - } - - } else { - # who knows? the ssh.log preprocessed bitmap field list version is not one we're expecting, we've got to guess and cannot use dissect - mutate { id => "mutate_add_tag_dissect_failure_unknown_bitmap_ssh_version" - add_tag => [ "_dissectfailure" ] } - } - - } else { - # who knows? the ssh.log was not preprocessed to determine fields, we've got to guess and cannot use dissect - mutate { id => "mutate_add_tag_dissect_failure_missing_ssh_bitmap" - add_tag => [ "_dissectfailure" ] } - } - - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_ssh" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_ssh_with_all_fields_with_hassh" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][version]} %{[zeek_cols][auth_success]} %{[zeek_cols][auth_attempts]} %{[zeek_cols][direction]} %{[zeek_cols][client]} %{[zeek_cols][server]} %{[zeek_cols][cipher_alg]} %{[zeek_cols][mac_alg]} %{[zeek_cols][compression_alg]} %{[zeek_cols][kex_alg]} %{[zeek_cols][host_key_alg]} %{[zeek_cols][host_key]} %{[zeek_cols][remote_location_country_code]} %{[zeek_cols][remote_location_region]} %{[zeek_cols][remote_location_city]} %{[zeek_cols][remote_location_latitude]} %{[zeek_cols][remote_location_longitude]} %{[zeek_cols][hasshVersion]} %{[zeek_cols][hassh]} %{[zeek_cols][hasshServer]} %{[zeek_cols][cshka]} %{[zeek_cols][hasshAlgorithms]} %{[zeek_cols][sshka]} %{[zeek_cols][hasshServerAlgorithms]}" + } } - ruby { - id => "ruby_zip_zeek_ssh" - init => "@zeek_ssh_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'version', 'auth_success', 'auth_attempts', 'direction', 'client', 'server', 'cipher_alg', 'mac_alg', 'compression_alg', 'kex_alg', 'host_key_alg', 'host_key', 'remote_location_country_code', 'remote_location_region', 'remote_location_city', 'remote_location_latitude', 'remote_location_longitude', 'hasshVersion', 'hassh', 'hasshServer', 'cshka', 'hasshAlgorithms', 'sshka', 'hasshServerAlgorithms' ]" - code => "event.set('[zeek_cols]', @zeek_ssh_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_ssh" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_ssh" + init => "@zeek_ssh_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'version', 'auth_success', 'auth_attempts', 'direction', 'client', 'server', 'cipher_alg', 'mac_alg', 'compression_alg', 'kex_alg', 'host_key_alg', 'host_key', 'remote_location_country_code', 'remote_location_region', 'remote_location_city', 'remote_location_latitude', 'remote_location_longitude', 'hasshVersion', 'hassh', 'hasshServer', 'cshka', 'hasshAlgorithms', 'sshka', 'hasshServerAlgorithms' ]" + code => "event.set('[zeek_cols]', @zeek_ssh_field_names.zip(event.get('[message]')).to_h)" + } } + mutate { id => "mutate_split_zeek_ssh_commas" + split => { "[zeek_cols][hasshAlgorithms]" => "," + "[zeek_cols][hasshServerAlgorithms]" => "," + "[zeek_cols][cshka]" => "," + "[zeek_cols][sshka]" => "," } } } mutate { @@ -3177,64 +3415,37 @@ filter { # ssl.log # https://docs.zeek.org/en/stable/scripts/base/protocols/ssl/main.zeek.html#type-SSL::Info - if ([@metadata][zeek_fields_bitmap] and [@metadata][zeek_fields_bitmap_version]) { - - # bitmap ssl.log (v4.1.0+) field configuration version 0 - # - # standard ssl.log fields without add-on JA3 fingerprinting script : 0x0007FFFF / 524287 - # ssl.log fields with add-on JA3 fingerprinting script : 0x001FFFFF / 2097151 - - if ([@metadata][zeek_fields_bitmap_version] == 0) { - - if ([@metadata][zeek_fields_bitmap] == 2097151) { - - dissect { - id => "dissect_zeek_ssl_v1_with_ja3" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][ssl_version]} %{[zeek_cols][cipher]} %{[zeek_cols][curve]} %{[zeek_cols][server_name]} %{[zeek_cols][resumed]} %{[zeek_cols][last_alert]} %{[zeek_cols][next_protocol]} %{[zeek_cols][established]} %{[zeek_cols][ssl_history]} %{[zeek_cols][cert_chain_fps]} %{[zeek_cols][client_cert_chain_fps]} %{[zeek_cols][sni_matches_cert]} %{[zeek_cols][validation_status]} %{[zeek_cols][ja3]} %{[zeek_cols][ja3s]}" - } - } - - } else if ([@metadata][zeek_fields_bitmap] == 524287) { - - dissect { - id => "dissect_zeek_ssl_v1_without_ja3" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][ssl_version]} %{[zeek_cols][cipher]} %{[zeek_cols][curve]} %{[zeek_cols][server_name]} %{[zeek_cols][resumed]} %{[zeek_cols][last_alert]} %{[zeek_cols][next_protocol]} %{[zeek_cols][established]} %{[zeek_cols][ssl_history]} %{[zeek_cols][cert_chain_fps]} %{[zeek_cols][client_cert_chain_fps]} %{[zeek_cols][sni_matches_cert]} %{[zeek_cols][validation_status]}" - } - } - - } else { - # who knows? the ssl.log preprocessed bitmap is not one we're expecting, we've got to guess and cannot use dissect - mutate { id => "mutate_add_tag_dissect_failure_unknown_ssl_v1_bitmap" - add_tag => [ "_dissectfailure" ] } - } - - } else { - # who knows? the ssl.log preprocessed bitmap field list version is not one we're expecting, we've got to guess and cannot use dissect - mutate { id => "mutate_add_tag_dissect_failure_unknown_bitmap_ssl_version" - add_tag => [ "_dissectfailure" ] } + if ("_jsonparsesuccess" in [tags]) { + mutate { + id => "mutate_rename_zeek_json_ssl_fields" + rename => { "[zeek_cols][version]" => "[zeek_cols][ssl_version]" } } } else { - # who knows? the ssl.log was not preprocessed to determine fields, we've got to guess and cannot use dissect - mutate { id => "mutate_add_tag_dissect_failure_missing_ssl_bitmap" - add_tag => [ "_dissectfailure" ] } - } - - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_ssl" + dissect { + id => "dissect_zeek_ssl_v1_with_ja3" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][ssl_version]} %{[zeek_cols][cipher]} %{[zeek_cols][curve]} %{[zeek_cols][server_name]} %{[zeek_cols][resumed]} %{[zeek_cols][last_alert]} %{[zeek_cols][next_protocol]} %{[zeek_cols][established]} %{[zeek_cols][ssl_history]} %{[zeek_cols][cert_chain_fps]} %{[zeek_cols][client_cert_chain_fps]} %{[zeek_cols][sni_matches_cert]} %{[zeek_cols][validation_status]} %{[zeek_cols][ja3]} %{[zeek_cols][ja3s]}" + } } - ruby { - id => "ruby_zip_zeek_ssl" - init => "@zeek_ssl_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'ssl_version', 'cipher', 'curve', 'server_name', 'resumed', 'last_alert', 'next_protocol', 'established', 'ssl_history', 'cert_chain_fps', 'client_cert_chain_fps', 'sni_matches_cert', 'validation_status', 'ja3', 'ja3s' ]" - code => "event.set('[zeek_cols]', @zeek_ssl_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_ssl" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_ssl" + init => "@zeek_ssl_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'ssl_version', 'cipher', 'curve', 'server_name', 'resumed', 'last_alert', 'next_protocol', 'established', 'ssl_history', 'cert_chain_fps', 'client_cert_chain_fps', 'sni_matches_cert', 'validation_status', 'ja3', 'ja3s' ]" + code => "event.set('[zeek_cols]', @zeek_ssl_field_names.zip(event.get('[message]')).to_h)" + } } + mutate { id => "mutate_split_zeek_ssl_commas" + split => { "[zeek_cols][cert_chain_fuids]" => "," + "[zeek_cols][client_cert_chain_fuids]" => "," + "[zeek_cols][cert_chain_fps]" => "," + "[zeek_cols][client_cert_chain_fps]" => "," } } } mutate { @@ -3250,24 +3461,36 @@ filter { # stun.log # https://github.com/corelight/zeek-spicy-stun/blob/master/analyzer/main.zeek - dissect { - id => "dissect_zeek_stun" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][proto]} %{[zeek_cols][is_orig]} %{[zeek_cols][trans_id]} %{[zeek_cols][method]} %{[zeek_cols][class]} %{[zeek_cols][attr_type]} %{[zeek_cols][attr_val]}" - } - } - if ("_dissectfailure" in [tags]) { + if ("_jsonparsesuccess" in [tags]) { mutate { - id => "mutate_split_zeek_stun" + id => "mutate_rename_zeek_json_stun_fields" + rename => { "[zeek_cols][attr_types]" => "[zeek_cols][attr_type]" } + rename => { "[zeek_cols][attr_vals]" => "[zeek_cols][attr_val]" } + } + + } else { + dissect { + id => "dissect_zeek_stun" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][proto]} %{[zeek_cols][is_orig]} %{[zeek_cols][trans_id]} %{[zeek_cols][method]} %{[zeek_cols][class]} %{[zeek_cols][attr_type]} %{[zeek_cols][attr_val]}" + } } - ruby { - id => "ruby_zip_zeek_stun" - init => "@zeek_stun_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto', 'is_orig', 'trans_id', 'method', 'class', 'attr_type', 'attr_val' ]" - code => "event.set('[zeek_cols]', @zeek_stun_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_stun" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_stun" + init => "@zeek_stun_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto', 'is_orig', 'trans_id', 'method', 'class', 'attr_type', 'attr_val' ]" + code => "event.set('[zeek_cols]', @zeek_stun_field_names.zip(event.get('[message]')).to_h)" + } } + mutate { id => "mutate_split_field_zeek_stun_commas" + split => { "[zeek_cols][attr_type]" => "," + "[zeek_cols][attr_val]" => "," } } } mutate { @@ -3282,24 +3505,38 @@ filter { # stun.log # https://github.com/corelight/zeek-spicy-stun/blob/master/analyzer/main.zeek - dissect { - id => "dissect_zeek_stun_nat" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][proto]} %{[zeek_cols][is_orig]} %{[zeek_cols][wan_addr]} %{[zeek_cols][wan_port]} %{[zeek_cols][lan_addr]}" - } - } - if ("_dissectfailure" in [tags]) { + if ("_jsonparsesuccess" in [tags]) { mutate { - id => "mutate_split_zeek_stun_nat" + id => "mutate_rename_zeek_json_stun_nat_fields" + rename => { "[zeek_cols][wan_addrs]" => "[zeek_cols][wan_addr]" } + rename => { "[zeek_cols][wan_ports]" => "[zeek_cols][wan_port]" } + rename => { "[zeek_cols][lan_addrs]" => "[zeek_cols][lan_addr]" } + } + + } else { + dissect { + id => "dissect_zeek_stun_nat" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][proto]} %{[zeek_cols][is_orig]} %{[zeek_cols][wan_addr]} %{[zeek_cols][wan_port]} %{[zeek_cols][lan_addr]}" + } } - ruby { - id => "ruby_zip_zeek_stun_nat" - init => "@zeek_stun_nat_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto', 'is_orig', 'wan_addr', 'wan_port', 'lan_addr' ]" - code => "event.set('[zeek_cols]', @zeek_stun_nat_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_stun_nat" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_stun_nat" + init => "@zeek_stun_nat_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto', 'is_orig', 'wan_addr', 'wan_port', 'lan_addr' ]" + code => "event.set('[zeek_cols]', @zeek_stun_nat_field_names.zip(event.get('[message]')).to_h)" + } } + mutate { id => "mutate_split_field_zeek_stun_nat_commas" + split => { "[zeek_cols][wan_addrs]" => "," + "[zeek_cols][wan_ports]" => "," + "[zeek_cols][lan_addrs]" => "," } } } mutate { @@ -3314,24 +3551,25 @@ filter { # synchrophasor.log # main.zeek (https://github.com/cisagov/icsnpp-synchrophasor) - dissect { - id => "dissect_zeek_synchrophasor" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][proto]} %{[zeek_cols][version]} %{[zeek_cols][data_stream_id]} %{[zeek_cols][history]} %{[zeek_cols][frame_size_min]} %{[zeek_cols][frame_size_max]} %{[zeek_cols][frame_size_tot]} %{[zeek_cols][data_frame_count]} %{[zeek_cols][data_rate]}" - } - } - - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_synchrophasor" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_synchrophasor" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][proto]} %{[zeek_cols][version]} %{[zeek_cols][data_stream_id]} %{[zeek_cols][history]} %{[zeek_cols][frame_size_min]} %{[zeek_cols][frame_size_max]} %{[zeek_cols][frame_size_tot]} %{[zeek_cols][data_frame_count]} %{[zeek_cols][data_rate]}" + } } - ruby { - id => "ruby_zip_zeek_synchrophasor" - init => "@zeek_synchrophasor_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto', 'version', 'data_stream_id', 'history', 'frame_size_min', 'frame_size_max', 'frame_size_tot', 'data_frame_count', 'data_rate' ]" - code => "event.set('[zeek_cols]', @zeek_synchrophasor_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_synchrophasor" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_synchrophasor" + init => "@zeek_synchrophasor_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto', 'version', 'data_stream_id', 'history', 'frame_size_min', 'frame_size_max', 'frame_size_tot', 'data_frame_count', 'data_rate' ]" + code => "event.set('[zeek_cols]', @zeek_synchrophasor_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -3349,24 +3587,25 @@ filter { # synchrophasor_cmd.log # main.zeek (https://github.com/cisagov/icsnpp-synchrophasor) - dissect { - id => "dissect_zeek_synchrophasor_cmd" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][proto]} %{[zeek_cols][frame_type]} %{[zeek_cols][frame_size]} %{[zeek_cols][header_time_stamp]} %{[zeek_cols][command]} %{[zeek_cols][extframe]}" - } - } - - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_synchrophasor_cmd" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_synchrophasor_cmd" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][proto]} %{[zeek_cols][frame_type]} %{[zeek_cols][frame_size]} %{[zeek_cols][header_time_stamp]} %{[zeek_cols][command]} %{[zeek_cols][extframe]}" + } } - ruby { - id => "ruby_zip_zeek_synchrophasor_cmd" - init => "@zeek_synchrophasor_cmd_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto', 'frame_type', 'frame_size', 'header_time_stamp', 'command', 'extframe' ]" - code => "event.set('[zeek_cols]', @zeek_synchrophasor_cmd_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_synchrophasor_cmd" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_synchrophasor_cmd" + init => "@zeek_synchrophasor_cmd_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto', 'frame_type', 'frame_size', 'header_time_stamp', 'command', 'extframe' ]" + code => "event.set('[zeek_cols]', @zeek_synchrophasor_cmd_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -3383,24 +3622,25 @@ filter { # synchrophasor_cfg.log # main.zeek (https://github.com/cisagov/icsnpp-synchrophasor) - dissect { - id => "dissect_zeek_synchrophasor_cfg" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][proto]} %{[zeek_cols][frame_type]} %{[zeek_cols][frame_size]} %{[zeek_cols][header_time_stamp]} %{[zeek_cols][cont_idx]} %{[zeek_cols][pmu_count_expected]} %{[zeek_cols][pmu_count_actual]} %{[zeek_cols][data_rate]} %{[zeek_cols][cfg_frame_id]}" - } - } - - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_synchrophasor_cfg" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_synchrophasor_cfg" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][proto]} %{[zeek_cols][frame_type]} %{[zeek_cols][frame_size]} %{[zeek_cols][header_time_stamp]} %{[zeek_cols][cont_idx]} %{[zeek_cols][pmu_count_expected]} %{[zeek_cols][pmu_count_actual]} %{[zeek_cols][data_rate]} %{[zeek_cols][cfg_frame_id]}" + } } - ruby { - id => "ruby_zip_zeek_synchrophasor_cfg" - init => "@zeek_synchrophasor_cfg_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto', 'frame_type', 'frame_size', 'header_time_stamp', 'cont_idx', 'pmu_count_expected', 'pmu_count_actual', 'data_rate', 'cfg_frame_id' ]" - code => "event.set('[zeek_cols]', @zeek_synchrophasor_cfg_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_synchrophasor_cfg" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_synchrophasor_cfg" + init => "@zeek_synchrophasor_cfg_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto', 'frame_type', 'frame_size', 'header_time_stamp', 'cont_idx', 'pmu_count_expected', 'pmu_count_actual', 'data_rate', 'cfg_frame_id' ]" + code => "event.set('[zeek_cols]', @zeek_synchrophasor_cfg_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -3417,24 +3657,25 @@ filter { # synchrophasor_cfg_detail.log # main.zeek (https://github.com/cisagov/icsnpp-synchrophasor) - dissect { - id => "dissect_zeek_synchrophasor_cfg_detail" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][proto]} %{[zeek_cols][frame_type]} %{[zeek_cols][header_time_stamp]} %{[zeek_cols][cfg_frame_id]} %{[zeek_cols][pmu_idx]} %{[zeek_cols][svc_class]} %{[zeek_cols][station_name]} %{[zeek_cols][data_source_id]} %{[zeek_cols][global_pmuid]} %{[zeek_cols][phasor_shape]} %{[zeek_cols][phasor_format]} %{[zeek_cols][analog_format]} %{[zeek_cols][freq_format]} %{[zeek_cols][phnmr]} %{[zeek_cols][annmr]} %{[zeek_cols][dgnmr]} %{[zeek_cols][phnam]} %{[zeek_cols][annam]} %{[zeek_cols][dgnam]} %{[zeek_cols][phasor_conv_phunit]} %{[zeek_cols][phasor_conv_phvalue]} %{[zeek_cols][phasor_conv_upsampled_interpolation]} %{[zeek_cols][phasor_conv_upsampled_extrapolation]} %{[zeek_cols][phasor_conv_downsampled_reselection]} %{[zeek_cols][phasor_conv_downsampled_fir_filter]} %{[zeek_cols][phasor_conv_downsampled_no_fir_filter]} %{[zeek_cols][phasor_conv_filtered_without_changing_sampling]} %{[zeek_cols][phasor_conv_calibration_mag_adj]} %{[zeek_cols][phasor_conv_calibration_phas_adj]} %{[zeek_cols][phasor_conv_rotation_phase_adj]} %{[zeek_cols][phasor_conv_pseudo_phasor_val]} %{[zeek_cols][phasor_conv_mod_appl]} %{[zeek_cols][phasor_conv_phasor_component]} %{[zeek_cols][phasor_conv_phasor_type]} %{[zeek_cols][phasor_conv_user_def]} %{[zeek_cols][phasor_conv_scale_factor]} %{[zeek_cols][phasor_conv_angle_adj]} %{[zeek_cols][analog_conv_analog_flags]} %{[zeek_cols][analog_conv_user_defined_scaling]} %{[zeek_cols][analog_conv_mag_scale]} %{[zeek_cols][analog_conv_offset]} %{[zeek_cols][digital_conv_normal_status_mask]} %{[zeek_cols][digital_conv_valid_inputs_mask]} %{[zeek_cols][pmu_lat]} %{[zeek_cols][pmu_lon]} %{[zeek_cols][pmu_elev]} %{[zeek_cols][window]} %{[zeek_cols][group_delay]} %{[zeek_cols][fnom]} %{[zeek_cols][cfgcnt]}" - } - } - - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_synchrophasor_cfg_detail" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_synchrophasor_cfg_detail" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][proto]} %{[zeek_cols][frame_type]} %{[zeek_cols][header_time_stamp]} %{[zeek_cols][cfg_frame_id]} %{[zeek_cols][pmu_idx]} %{[zeek_cols][svc_class]} %{[zeek_cols][station_name]} %{[zeek_cols][data_source_id]} %{[zeek_cols][global_pmuid]} %{[zeek_cols][phasor_shape]} %{[zeek_cols][phasor_format]} %{[zeek_cols][analog_format]} %{[zeek_cols][freq_format]} %{[zeek_cols][phnmr]} %{[zeek_cols][annmr]} %{[zeek_cols][dgnmr]} %{[zeek_cols][phnam]} %{[zeek_cols][annam]} %{[zeek_cols][dgnam]} %{[zeek_cols][phasor_conv_phunit]} %{[zeek_cols][phasor_conv_phvalue]} %{[zeek_cols][phasor_conv_upsampled_interpolation]} %{[zeek_cols][phasor_conv_upsampled_extrapolation]} %{[zeek_cols][phasor_conv_downsampled_reselection]} %{[zeek_cols][phasor_conv_downsampled_fir_filter]} %{[zeek_cols][phasor_conv_downsampled_no_fir_filter]} %{[zeek_cols][phasor_conv_filtered_without_changing_sampling]} %{[zeek_cols][phasor_conv_calibration_mag_adj]} %{[zeek_cols][phasor_conv_calibration_phas_adj]} %{[zeek_cols][phasor_conv_rotation_phase_adj]} %{[zeek_cols][phasor_conv_pseudo_phasor_val]} %{[zeek_cols][phasor_conv_mod_appl]} %{[zeek_cols][phasor_conv_phasor_component]} %{[zeek_cols][phasor_conv_phasor_type]} %{[zeek_cols][phasor_conv_user_def]} %{[zeek_cols][phasor_conv_scale_factor]} %{[zeek_cols][phasor_conv_angle_adj]} %{[zeek_cols][analog_conv_analog_flags]} %{[zeek_cols][analog_conv_user_defined_scaling]} %{[zeek_cols][analog_conv_mag_scale]} %{[zeek_cols][analog_conv_offset]} %{[zeek_cols][digital_conv_normal_status_mask]} %{[zeek_cols][digital_conv_valid_inputs_mask]} %{[zeek_cols][pmu_lat]} %{[zeek_cols][pmu_lon]} %{[zeek_cols][pmu_elev]} %{[zeek_cols][window]} %{[zeek_cols][group_delay]} %{[zeek_cols][fnom]} %{[zeek_cols][cfgcnt]}" + } } - ruby { - id => "ruby_zip_zeek_synchrophasor_cfg_detail" - init => "@zeek_synchrophasor_cfg_detail_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto', 'frame_type', 'header_time_stamp', 'cfg_frame_id', 'pmu_idx', 'svc_class', 'station_name', 'data_source_id', 'global_pmuid', 'phasor_shape', 'phasor_format', 'analog_format', 'freq_format', 'phnmr', 'annmr', 'dgnmr', 'phnam', 'annam', 'dgnam', 'phasor_conv_phunit', 'phasor_conv_phvalue', 'phasor_conv_upsampled_interpolation', 'phasor_conv_upsampled_extrapolation', 'phasor_conv_downsampled_reselection', 'phasor_conv_downsampled_fir_filter', 'phasor_conv_downsampled_no_fir_filter', 'phasor_conv_filtered_without_changing_sampling', 'phasor_conv_calibration_mag_adj', 'phasor_conv_calibration_phas_adj', 'phasor_conv_rotation_phase_adj', 'phasor_conv_pseudo_phasor_val', 'phasor_conv_mod_appl', 'phasor_conv_phasor_component', 'phasor_conv_phasor_type', 'phasor_conv_user_def', 'phasor_conv_scale_factor', 'phasor_conv_angle_adj', 'analog_conv_analog_flags', 'analog_conv_user_defined_scaling', 'analog_conv_mag_scale', 'analog_conv_offset', 'digital_conv_normal_status_mask', 'digital_conv_valid_inputs_mask', 'pmu_lat', 'pmu_lon', 'pmu_elev', 'window', 'group_delay', 'fnom', 'cfgcnt' ]" - code => "event.set('[zeek_cols]', @zeek_synchrophasor_cfg_detail_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_synchrophasor_cfg_detail" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_synchrophasor_cfg_detail" + init => "@zeek_synchrophasor_cfg_detail_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto', 'frame_type', 'header_time_stamp', 'cfg_frame_id', 'pmu_idx', 'svc_class', 'station_name', 'data_source_id', 'global_pmuid', 'phasor_shape', 'phasor_format', 'analog_format', 'freq_format', 'phnmr', 'annmr', 'dgnmr', 'phnam', 'annam', 'dgnam', 'phasor_conv_phunit', 'phasor_conv_phvalue', 'phasor_conv_upsampled_interpolation', 'phasor_conv_upsampled_extrapolation', 'phasor_conv_downsampled_reselection', 'phasor_conv_downsampled_fir_filter', 'phasor_conv_downsampled_no_fir_filter', 'phasor_conv_filtered_without_changing_sampling', 'phasor_conv_calibration_mag_adj', 'phasor_conv_calibration_phas_adj', 'phasor_conv_rotation_phase_adj', 'phasor_conv_pseudo_phasor_val', 'phasor_conv_mod_appl', 'phasor_conv_phasor_component', 'phasor_conv_phasor_type', 'phasor_conv_user_def', 'phasor_conv_scale_factor', 'phasor_conv_angle_adj', 'analog_conv_analog_flags', 'analog_conv_user_defined_scaling', 'analog_conv_mag_scale', 'analog_conv_offset', 'digital_conv_normal_status_mask', 'digital_conv_valid_inputs_mask', 'pmu_lat', 'pmu_lon', 'pmu_elev', 'window', 'group_delay', 'fnom', 'cfgcnt' ]" + code => "event.set('[zeek_cols]', @zeek_synchrophasor_cfg_detail_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -3451,24 +3692,25 @@ filter { # synchrophasor_data.log # main.zeek (https://github.com/cisagov/icsnpp-synchrophasor) - dissect { - id => "dissect_zeek_synchrophasor_data" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][proto]} %{[zeek_cols][frame_type]} %{[zeek_cols][frame_size]} %{[zeek_cols][header_time_stamp]} %{[zeek_cols][pmu_count_expected]} %{[zeek_cols][pmu_count_actual]} %{[zeek_cols][data_frame_id]}" - } - } - - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_synchrophasor_data" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_synchrophasor_data" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][proto]} %{[zeek_cols][frame_type]} %{[zeek_cols][frame_size]} %{[zeek_cols][header_time_stamp]} %{[zeek_cols][pmu_count_expected]} %{[zeek_cols][pmu_count_actual]} %{[zeek_cols][data_frame_id]}" + } } - ruby { - id => "ruby_zip_zeek_synchrophasor_data" - init => "@zeek_synchrophasor_data_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto', 'frame_type', 'frame_size', 'header_time_stamp', 'pmu_count_expected', 'pmu_count_actual', 'data_frame_id' ]" - code => "event.set('[zeek_cols]', @zeek_synchrophasor_data_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_synchrophasor_data" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_synchrophasor_data" + init => "@zeek_synchrophasor_data_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto', 'frame_type', 'frame_size', 'header_time_stamp', 'pmu_count_expected', 'pmu_count_actual', 'data_frame_id' ]" + code => "event.set('[zeek_cols]', @zeek_synchrophasor_data_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -3485,24 +3727,25 @@ filter { # synchrophasor_data_detail.log # main.zeek (https://github.com/cisagov/icsnpp-synchrophasor) - dissect { - id => "dissect_zeek_synchrophasor_data_detail" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][proto]} %{[zeek_cols][frame_type]} %{[zeek_cols][header_time_stamp]} %{[zeek_cols][data_frame_id]} %{[zeek_cols][pmu_idx]} %{[zeek_cols][trigger_reason]} %{[zeek_cols][unlocked_time]} %{[zeek_cols][pmu_time_quality]} %{[zeek_cols][data_modified]} %{[zeek_cols][config_change]} %{[zeek_cols][pmu_trigger_pickup]} %{[zeek_cols][data_sorting_type]} %{[zeek_cols][pmu_sync_error]} %{[zeek_cols][data_error_indicator]} %{[zeek_cols][est_rectangular_real]} %{[zeek_cols][est_rectangular_imaginary]} %{[zeek_cols][est_polar_magnitude]} %{[zeek_cols][est_polar_angle]} %{[zeek_cols][freq_dev_mhz]} %{[zeek_cols][rocof]} %{[zeek_cols][analog_data]} %{[zeek_cols][digital]}" - } - } - - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_synchrophasor_data_detail" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_synchrophasor_data_detail" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][proto]} %{[zeek_cols][frame_type]} %{[zeek_cols][header_time_stamp]} %{[zeek_cols][data_frame_id]} %{[zeek_cols][pmu_idx]} %{[zeek_cols][trigger_reason]} %{[zeek_cols][unlocked_time]} %{[zeek_cols][pmu_time_quality]} %{[zeek_cols][data_modified]} %{[zeek_cols][config_change]} %{[zeek_cols][pmu_trigger_pickup]} %{[zeek_cols][data_sorting_type]} %{[zeek_cols][pmu_sync_error]} %{[zeek_cols][data_error_indicator]} %{[zeek_cols][est_rectangular_real]} %{[zeek_cols][est_rectangular_imaginary]} %{[zeek_cols][est_polar_magnitude]} %{[zeek_cols][est_polar_angle]} %{[zeek_cols][freq_dev_mhz]} %{[zeek_cols][rocof]} %{[zeek_cols][analog_data]} %{[zeek_cols][digital]}" + } } - ruby { - id => "ruby_zip_zeek_synchrophasor_data_detail" - init => "@zeek_synchrophasor_data_detail_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto', 'frame_type', 'header_time_stamp', 'data_frame_id', 'pmu_idx', 'trigger_reason', 'unlocked_time', 'pmu_time_quality', 'data_modified', 'config_change', 'pmu_trigger_pickup', 'data_sorting_type', 'pmu_sync_error', 'data_error_indicator', 'est_rectangular_real', 'est_rectangular_imaginary', 'est_polar_magnitude', 'est_polar_angle', 'freq_dev_mhz', 'rocof', 'analog_data', 'digital' ]" - code => "event.set('[zeek_cols]', @zeek_synchrophasor_data_detail_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_synchrophasor_data_detail" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_synchrophasor_data_detail" + init => "@zeek_synchrophasor_data_detail_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto', 'frame_type', 'header_time_stamp', 'data_frame_id', 'pmu_idx', 'trigger_reason', 'unlocked_time', 'pmu_time_quality', 'data_modified', 'config_change', 'pmu_trigger_pickup', 'data_sorting_type', 'pmu_sync_error', 'data_error_indicator', 'est_rectangular_real', 'est_rectangular_imaginary', 'est_polar_magnitude', 'est_polar_angle', 'freq_dev_mhz', 'rocof', 'analog_data', 'digital' ]" + code => "event.set('[zeek_cols]', @zeek_synchrophasor_data_detail_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -3519,24 +3762,25 @@ filter { # synchrophasor_hdr.log # main.zeek (https://github.com/cisagov/icsnpp-synchrophasor) - dissect { - id => "dissect_zeek_synchrophasor_hdr" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][proto]} %{[zeek_cols][frame_type]} %{[zeek_cols][frame_size]} %{[zeek_cols][header_time_stamp]} %{[zeek_cols][data]}" - } - } - - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_synchrophasor_hdr" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_synchrophasor_hdr" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][proto]} %{[zeek_cols][frame_type]} %{[zeek_cols][frame_size]} %{[zeek_cols][header_time_stamp]} %{[zeek_cols][data]}" + } } - ruby { - id => "ruby_zip_zeek_synchrophasor_hdr" - init => "@zeek_synchrophasor_hdr_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto', 'frame_type', 'frame_size', 'header_time_stamp', 'data' ]" - code => "event.set('[zeek_cols]', @zeek_synchrophasor_hdr_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_synchrophasor_hdr" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_synchrophasor_hdr" + init => "@zeek_synchrophasor_hdr_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto', 'frame_type', 'frame_size', 'header_time_stamp', 'data' ]" + code => "event.set('[zeek_cols]', @zeek_synchrophasor_hdr_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -3553,23 +3797,25 @@ filter { # syslog.log # https://docs.zeek.org/en/stable/scripts/base/protocols/syslog/main.zeek.html#type-Syslog::Info - dissect { - id => "dissect_zeek_syslog" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][proto]} %{[zeek_cols][facility]} %{[zeek_cols][severity]} %{[zeek_cols][message]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_syslog" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_syslog" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][proto]} %{[zeek_cols][facility]} %{[zeek_cols][severity]} %{[zeek_cols][message]}" + } } - ruby { - id => "ruby_zip_zeek_syslog" - init => "@zeek_syslog_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto', 'facility', 'severity', 'message' ]" - code => "event.set('[zeek_cols]', @zeek_syslog_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_syslog" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_syslog" + init => "@zeek_syslog_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto', 'facility', 'severity', 'message' ]" + code => "event.set('[zeek_cols]', @zeek_syslog_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -3583,23 +3829,25 @@ filter { # tds.log # https://github.com/amzn/zeek-plugin-tds/blob/master/scripts/main.zeek - dissect { - id => "dissect_zeek_tds" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][command]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_tds" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_tds" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][command]}" + } } - ruby { - id => "ruby_zip_zeek_tds" - init => "@zeek_tds_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'command' ]" - code => "event.set('[zeek_cols]', @zeek_tds_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_tds" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_tds" + init => "@zeek_tds_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'command' ]" + code => "event.set('[zeek_cols]', @zeek_tds_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -3616,23 +3864,31 @@ filter { # tds_rpc.log # https://github.com/amzn/zeek-plugin-tds/blob/master/scripts/main.zeek - dissect { - id => "dissect_zeek_tds_rpc" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][procedure_name]} %{[zeek_cols][parameter]}" - } - } - if ("_dissectfailure" in [tags]) { + if ("_jsonparsesuccess" in [tags]) { mutate { - id => "mutate_split_zeek_tds_rpc" + id => "mutate_rename_zeek_json_tds_rpc_fields" + rename => { "[zeek_cols][parameters]" => "[zeek_cols][parameter]" } + } + + } else { + dissect { + id => "dissect_zeek_tds_rpc" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][procedure_name]} %{[zeek_cols][parameter]}" + } } - ruby { - id => "ruby_zip_zeek_tds_rpc" - init => "@zeek_tds_rpc_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'procedure_name', 'parameter' ]" - code => "event.set('[zeek_cols]', @zeek_tds_rpc_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_tds_rpc" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_tds_rpc" + init => "@zeek_tds_rpc_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'procedure_name', 'parameter' ]" + code => "event.set('[zeek_cols]', @zeek_tds_rpc_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -3649,23 +3905,25 @@ filter { # tds_sql_batch.log # https://github.com/amzn/zeek-plugin-tds/blob/master/scripts/main.zeek - dissect { - id => "dissect_zeek_tds_sql_batch" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][header_type]} %{[zeek_cols][query]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_tds_sql_batch" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_tds_sql_batch" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][header_type]} %{[zeek_cols][query]}" + } } - ruby { - id => "ruby_zip_zeek_tds_sql_batch" - init => "@zeek_tds_sql_batch_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'header_type', 'query' ]" - code => "event.set('[zeek_cols]', @zeek_tds_sql_batch_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_tds_sql_batch" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_tds_sql_batch" + init => "@zeek_tds_sql_batch_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'header_type', 'query' ]" + code => "event.set('[zeek_cols]', @zeek_tds_sql_batch_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -3682,23 +3940,25 @@ filter { # tftp.log # https://github.com/zeek/spicy-tftp - dissect { - id => "dissect_zeek_tftp" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][wrq]} %{[zeek_cols][fname]} %{[zeek_cols][mode]} %{[zeek_cols][uid_data]} %{[zeek_cols][size]} %{[zeek_cols][block_sent]} %{[zeek_cols][block_acked]} %{[zeek_cols][error_code]} %{[zeek_cols][error_msg]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_tftp" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_tftp" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][wrq]} %{[zeek_cols][fname]} %{[zeek_cols][mode]} %{[zeek_cols][uid_data]} %{[zeek_cols][size]} %{[zeek_cols][block_sent]} %{[zeek_cols][block_acked]} %{[zeek_cols][error_code]} %{[zeek_cols][error_msg]}" + } } - ruby { - id => "ruby_zip_zeek_tftp" - init => "@zeek_tftp_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'wrq', 'fname', 'mode', 'uid_data', 'size', 'block_sent', 'block_acked', 'error_code', 'error_msg' ]" - code => "event.set('[zeek_cols]', @zeek_tftp_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_tftp" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_tftp" + init => "@zeek_tftp_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'wrq', 'fname', 'mode', 'uid_data', 'size', 'block_sent', 'block_acked', 'error_code', 'error_msg' ]" + code => "event.set('[zeek_cols]', @zeek_tftp_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -3715,23 +3975,25 @@ filter { # tunnel.log # https://docs.zeek.org/en/stable/scripts/base/frameworks/tunnels/main.zeek.html#type-Tunnel::Info - dissect { - id => "dissect_zeek_tunnel" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][tunnel_type]} %{[zeek_cols][action]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_tunnel" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_tunnel" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][tunnel_type]} %{[zeek_cols][action]}" + } } - ruby { - id => "ruby_zip_zeek_tunnel" - init => "@zeek_tunnel_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'tunnel_type', 'action' ]" - code => "event.set('[zeek_cols]', @zeek_tunnel_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_tunnel" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_tunnel" + init => "@zeek_tunnel_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'tunnel_type', 'action' ]" + code => "event.set('[zeek_cols]', @zeek_tunnel_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -3754,23 +4016,25 @@ filter { # weird.log # https://docs.zeek.org/en/stable/scripts/base/frameworks/notice/weird.zeek.html#type-Weird::Info - dissect { - id => "dissect_zeek_weird" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][name]} %{[zeek_cols][addl]} %{[zeek_cols][notice]} %{[zeek_cols][peer]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_weird" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_weird" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][name]} %{[zeek_cols][addl]} %{[zeek_cols][notice]} %{[zeek_cols][peer]} %{[zeek_cols][source]}" + } } - ruby { - id => "ruby_zip_zeek_weird" - init => "@zeek_weird_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'name', 'addl', 'notice', 'peer' ]" - code => "event.set('[zeek_cols]', @zeek_weird_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_weird" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_weird" + init => "@zeek_weird_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'name', 'addl', 'notice', 'peer', 'source' ]" + code => "event.set('[zeek_cols]', @zeek_weird_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -3779,25 +4043,52 @@ filter { # x509.log # https://docs.zeek.org/en/stable/scripts/base/files/x509/main.zeek.html#type-X509::Info - dissect { - id => "dissect_zeek_x509_v1" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][fingerprint]} %{[zeek_cols][certificate_version]} %{[zeek_cols][certificate_serial]} %{[zeek_cols][certificate_subject]} %{[zeek_cols][certificate_issuer]} %{[zeek_cols][certificate_not_valid_before]} %{[zeek_cols][certificate_not_valid_after]} %{[zeek_cols][certificate_key_alg]} %{[zeek_cols][certificate_sig_alg]} %{[zeek_cols][certificate_key_type]} %{[zeek_cols][certificate_key_length]} %{[zeek_cols][certificate_exponent]} %{[zeek_cols][certificate_curve]} %{[zeek_cols][san_dns]} %{[zeek_cols][san_uri]} %{[zeek_cols][san_email]} %{[zeek_cols][san_ip]} %{[zeek_cols][basic_constraints_ca]} %{[zeek_cols][basic_constraints_path_len]} %{[zeek_cols][host_cert]} %{[zeek_cols][client_cert]}" + if ("_jsonparsesuccess" in [tags]) { + mutate { + id => "mutate_rename_zeek_json_x509_fields" + rename => { "[zeek_cols][certificate.version]" => "[zeek_cols][certificate_version]" } + rename => { "[zeek_cols][certificate.serial]" => "[zeek_cols][certificate_serial]" } + rename => { "[zeek_cols][certificate.subject]" => "[zeek_cols][certificate_subject]" } + rename => { "[zeek_cols][certificate.issuer]" => "[zeek_cols][certificate_issuer]" } + rename => { "[zeek_cols][certificate.not_valid_before]" => "[zeek_cols][certificate_not_valid_before]" } + rename => { "[zeek_cols][certificate.not_valid_after]" => "[zeek_cols][certificate_not_valid_after]" } + rename => { "[zeek_cols][certificate.key_alg]" => "[zeek_cols][certificate_key_alg]" } + rename => { "[zeek_cols][certificate.sig_alg]" => "[zeek_cols][certificate_sig_alg]" } + rename => { "[zeek_cols][certificate.key_type]" => "[zeek_cols][certificate_key_type]" } + rename => { "[zeek_cols][certificate.key_length]" => "[zeek_cols][certificate_key_length]" } + rename => { "[zeek_cols][certificate.exponent]" => "[zeek_cols][certificate_exponent]" } + rename => { "[zeek_cols][certificate.curve]" => "[zeek_cols][certificate_curve]" } + rename => { "[zeek_cols][san.dns]" => "[zeek_cols][san_dns]" } + rename => { "[zeek_cols][san.uri]" => "[zeek_cols][san_uri]" } + rename => { "[zeek_cols][san.email]" => "[zeek_cols][san_email]" } + rename => { "[zeek_cols][san.ip]" => "[zeek_cols][san_ip]" } + rename => { "[zeek_cols][basic_constraints.ca]" => "[zeek_cols][basic_constraints_ca]" } + rename => { "[zeek_cols][basic_constraints.path_len]" => "[zeek_cols][basic_constraints_path_len]" } } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_x509" + } else { + dissect { + id => "dissect_zeek_x509_v1" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][fingerprint]} %{[zeek_cols][certificate_version]} %{[zeek_cols][certificate_serial]} %{[zeek_cols][certificate_subject]} %{[zeek_cols][certificate_issuer]} %{[zeek_cols][certificate_not_valid_before]} %{[zeek_cols][certificate_not_valid_after]} %{[zeek_cols][certificate_key_alg]} %{[zeek_cols][certificate_sig_alg]} %{[zeek_cols][certificate_key_type]} %{[zeek_cols][certificate_key_length]} %{[zeek_cols][certificate_exponent]} %{[zeek_cols][certificate_curve]} %{[zeek_cols][san_dns]} %{[zeek_cols][san_uri]} %{[zeek_cols][san_email]} %{[zeek_cols][san_ip]} %{[zeek_cols][basic_constraints_ca]} %{[zeek_cols][basic_constraints_path_len]} %{[zeek_cols][host_cert]} %{[zeek_cols][client_cert]}" + } } - ruby { - id => "ruby_zip_zeek_x509" - init => "@zeek_x509_field_names = [ 'ts', 'fuid', 'certificate_version', 'certificate_serial', 'certificate_subject', 'certificate_issuer', 'certificate_not_valid_before', 'certificate_not_valid_after', 'certificate_key_alg', 'certificate_sig_alg', 'certificate_key_type', 'certificate_key_length', 'certificate_exponent', 'certificate_curve', 'san_dns', 'san_uri', 'san_email', 'san_ip', 'basic_constraints_ca', 'basic_constraints_path_len', 'host_cert', 'client_cert' ]" - code => "event.set('[zeek_cols]', @zeek_x509_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_x509" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_x509" + init => "@zeek_x509_field_names = [ 'ts', 'fuid', 'certificate_version', 'certificate_serial', 'certificate_subject', 'certificate_issuer', 'certificate_not_valid_before', 'certificate_not_valid_after', 'certificate_key_alg', 'certificate_sig_alg', 'certificate_key_type', 'certificate_key_length', 'certificate_exponent', 'certificate_curve', 'san_dns', 'san_uri', 'san_email', 'san_ip', 'basic_constraints_ca', 'basic_constraints_path_len', 'host_cert', 'client_cert' ]" + code => "event.set('[zeek_cols]', @zeek_x509_field_names.zip(event.get('[message]')).to_h)" + } } + mutate { id => "mutate_split_zeek_x509_san_ip" + split => { "[zeek_cols][san_ip]" => "," + "[zeek_cols][fingerprint]" => "," } } } mutate { id => "mutate_add_fields_zeek_x509" @@ -3810,24 +4101,25 @@ filter { # opcua_binary.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - dissect { - id => "dissect_zeek_opcua_binary" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][opcua_link_id]} %{[zeek_cols][msg_type]} %{[zeek_cols][is_final]} %{[zeek_cols][msg_size]} %{[zeek_cols][error]} %{[zeek_cols][reason]} %{[zeek_cols][version]} %{[zeek_cols][rcv_buf_size]} %{[zeek_cols][snd_buf_size]} %{[zeek_cols][max_msg_size]} %{[zeek_cols][max_chunk_cnt]} %{[zeek_cols][endpoint_url]} %{[zeek_cols][sec_channel_id]} %{[zeek_cols][sec_policy_uri_len]} %{[zeek_cols][sec_policy_uri]} %{[zeek_cols][snd_cert_len]} %{[zeek_cols][snd_cert]} %{[zeek_cols][rcv_cert_len]} %{[zeek_cols][rcv_cert]} %{[zeek_cols][seq_number]} %{[zeek_cols][request_id]} %{[zeek_cols][encoding_mask]} %{[zeek_cols][namespace_idx]} %{[zeek_cols][identifier]} %{[zeek_cols][identifier_str]} %{[zeek_cols][req_hdr_node_id_type]} %{[zeek_cols][req_hdr_node_id_namespace_idx]} %{[zeek_cols][req_hdr_node_id_numeric]} %{[zeek_cols][req_hdr_node_id_string]} %{[zeek_cols][req_hdr_node_id_guid]} %{[zeek_cols][req_hdr_node_id_opaque]} %{[zeek_cols][req_hdr_timestamp]} %{[zeek_cols][req_hdr_request_handle]} %{[zeek_cols][req_hdr_return_diag]} %{[zeek_cols][req_hdr_audit_entry_id]} %{[zeek_cols][req_hdr_timeout_hint]} %{[zeek_cols][req_hdr_add_hdr_type_id]} %{[zeek_cols][req_hdr_add_hdr_enc_mask]} %{[zeek_cols][res_hdr_timestamp]} %{[zeek_cols][res_hdr_request_handle]} %{[zeek_cols][status_code_link_id]} %{[zeek_cols][res_hdr_service_diag_encoding]} %{[zeek_cols][res_hdr_add_hdr_type_id]} %{[zeek_cols][res_hdr_add_hdr_enc_mask]}" - } - } - - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_opcua_binary" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_opcua_binary" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][opcua_link_id]} %{[zeek_cols][msg_type]} %{[zeek_cols][is_final]} %{[zeek_cols][msg_size]} %{[zeek_cols][error]} %{[zeek_cols][reason]} %{[zeek_cols][version]} %{[zeek_cols][rcv_buf_size]} %{[zeek_cols][snd_buf_size]} %{[zeek_cols][max_msg_size]} %{[zeek_cols][max_chunk_cnt]} %{[zeek_cols][endpoint_url]} %{[zeek_cols][sec_channel_id]} %{[zeek_cols][sec_policy_uri_len]} %{[zeek_cols][sec_policy_uri]} %{[zeek_cols][snd_cert_len]} %{[zeek_cols][snd_cert]} %{[zeek_cols][rcv_cert_len]} %{[zeek_cols][rcv_cert]} %{[zeek_cols][seq_number]} %{[zeek_cols][request_id]} %{[zeek_cols][encoding_mask]} %{[zeek_cols][namespace_idx]} %{[zeek_cols][identifier]} %{[zeek_cols][identifier_str]} %{[zeek_cols][req_hdr_node_id_type]} %{[zeek_cols][req_hdr_node_id_namespace_idx]} %{[zeek_cols][req_hdr_node_id_numeric]} %{[zeek_cols][req_hdr_node_id_string]} %{[zeek_cols][req_hdr_node_id_guid]} %{[zeek_cols][req_hdr_node_id_opaque]} %{[zeek_cols][req_hdr_timestamp]} %{[zeek_cols][req_hdr_request_handle]} %{[zeek_cols][req_hdr_return_diag]} %{[zeek_cols][req_hdr_audit_entry_id]} %{[zeek_cols][req_hdr_timeout_hint]} %{[zeek_cols][req_hdr_add_hdr_type_id]} %{[zeek_cols][req_hdr_add_hdr_enc_mask]} %{[zeek_cols][res_hdr_timestamp]} %{[zeek_cols][res_hdr_request_handle]} %{[zeek_cols][status_code_link_id]} %{[zeek_cols][res_hdr_service_diag_encoding]} %{[zeek_cols][res_hdr_add_hdr_type_id]} %{[zeek_cols][res_hdr_add_hdr_enc_mask]}" + } } - ruby { - id => "ruby_zip_zeek_opcua_binary" - init => "@zeek_opcua_binary_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'opcua_link_id', 'msg_type', 'is_final', 'msg_size', 'error', 'reason', 'version', 'rcv_buf_size', 'snd_buf_size', 'max_msg_size', 'max_chunk_cnt', 'endpoint_url', 'sec_channel_id', 'sec_policy_uri_len', 'sec_policy_uri', 'snd_cert_len', 'snd_cert', 'rcv_cert_len', 'rcv_cert', 'seq_number', 'request_id', 'encoding_mask', 'namespace_idx', 'identifier', 'identifier_str', 'req_hdr_node_id_type', 'req_hdr_node_id_namespace_idx', 'req_hdr_node_id_numeric', 'req_hdr_node_id_string', 'req_hdr_node_id_guid', 'req_hdr_node_id_opaque', 'req_hdr_timestamp', 'req_hdr_request_handle', 'req_hdr_return_diag', 'req_hdr_audit_entry_id', 'req_hdr_timeout_hint', 'req_hdr_add_hdr_type_id', 'req_hdr_add_hdr_enc_mask', 'res_hdr_timestamp', 'res_hdr_request_handle', 'status_code_link_id', 'res_hdr_service_diag_encoding', 'res_hdr_add_hdr_type_id', 'res_hdr_add_hdr_enc_mask' ]" - code => "event.set('[zeek_cols]', @zeek_opcua_binary_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_opcua_binary" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_opcua_binary" + init => "@zeek_opcua_binary_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'opcua_link_id', 'msg_type', 'is_final', 'msg_size', 'error', 'reason', 'version', 'rcv_buf_size', 'snd_buf_size', 'max_msg_size', 'max_chunk_cnt', 'endpoint_url', 'sec_channel_id', 'sec_policy_uri_len', 'sec_policy_uri', 'snd_cert_len', 'snd_cert', 'rcv_cert_len', 'rcv_cert', 'seq_number', 'request_id', 'encoding_mask', 'namespace_idx', 'identifier', 'identifier_str', 'req_hdr_node_id_type', 'req_hdr_node_id_namespace_idx', 'req_hdr_node_id_numeric', 'req_hdr_node_id_string', 'req_hdr_node_id_guid', 'req_hdr_node_id_opaque', 'req_hdr_timestamp', 'req_hdr_request_handle', 'req_hdr_return_diag', 'req_hdr_audit_entry_id', 'req_hdr_timeout_hint', 'req_hdr_add_hdr_type_id', 'req_hdr_add_hdr_enc_mask', 'res_hdr_timestamp', 'res_hdr_request_handle', 'status_code_link_id', 'res_hdr_service_diag_encoding', 'res_hdr_add_hdr_type_id', 'res_hdr_add_hdr_enc_mask' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -3843,26 +4135,27 @@ filter { } else if ([log_source] == "opcua_binary_activate_session") { ############################################################################################################################# # opcua_binary_activate_session.log - # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - - dissect { - id => "dissect_zeek_opcua_binary_activate_session" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][opcua_link_id]} %{[zeek_cols][client_algorithm]} %{[zeek_cols][client_signature]} %{[zeek_cols][client_software_cert_link_id]} %{[zeek_cols][opcua_locale_link_id]} %{[zeek_cols][ext_obj_type_id_encoding_mask]} %{[zeek_cols][ext_obj_type_id_namespace_idx]} %{[zeek_cols][ext_obj_type_id_numeric]} %{[zeek_cols][ext_obj_type_id_string]} %{[zeek_cols][ext_obj_type_id_guid]} %{[zeek_cols][ext_obj_type_id_opaque]} %{[zeek_cols][ext_obj_type_id_str]} %{[zeek_cols][ext_obj_encoding]} %{[zeek_cols][ext_obj_policy_id]} %{[zeek_cols][ext_obj_user_name]} %{[zeek_cols][ext_obj_password]} %{[zeek_cols][ext_obj_encryption_algorithom]} %{[zeek_cols][ext_obj_certificate_data]} %{[zeek_cols][ext_obj_token_data]} %{[zeek_cols][user_token_algorithm]} %{[zeek_cols][user_token_signature]} %{[zeek_cols][server_nonce]} %{[zeek_cols][status_code_link_id]} %{[zeek_cols][activate_session_diag_info_link_id]}" - } - } + # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_opcua_binary_activate_session" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_opcua_binary_activate_session" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][opcua_link_id]} %{[zeek_cols][client_algorithm]} %{[zeek_cols][client_signature]} %{[zeek_cols][client_software_cert_link_id]} %{[zeek_cols][opcua_locale_link_id]} %{[zeek_cols][ext_obj_type_id_encoding_mask]} %{[zeek_cols][ext_obj_type_id_namespace_idx]} %{[zeek_cols][ext_obj_type_id_numeric]} %{[zeek_cols][ext_obj_type_id_string]} %{[zeek_cols][ext_obj_type_id_guid]} %{[zeek_cols][ext_obj_type_id_opaque]} %{[zeek_cols][ext_obj_type_id_str]} %{[zeek_cols][ext_obj_encoding]} %{[zeek_cols][ext_obj_policy_id]} %{[zeek_cols][ext_obj_user_name]} %{[zeek_cols][ext_obj_password]} %{[zeek_cols][ext_obj_encryption_algorithom]} %{[zeek_cols][ext_obj_certificate_data]} %{[zeek_cols][ext_obj_token_data]} %{[zeek_cols][user_token_algorithm]} %{[zeek_cols][user_token_signature]} %{[zeek_cols][server_nonce]} %{[zeek_cols][status_code_link_id]} %{[zeek_cols][activate_session_diag_info_link_id]}" + } } - ruby { - id => "ruby_zip_zeek_opcua_binary_activate_session" - init => "@zeek_opcua_binary_activate_session_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'opcua_link_id', 'client_algorithm', 'client_signature', 'client_software_cert_link_id', 'opcua_locale_link_id', 'ext_obj_type_id_encoding_mask', 'ext_obj_type_id_namespace_idx', 'ext_obj_type_id_numeric', 'ext_obj_type_id_string', 'ext_obj_type_id_guid', 'ext_obj_type_id_opaque', 'ext_obj_type_id_str', 'ext_obj_encoding', 'ext_obj_policy_id', 'ext_obj_user_name', 'ext_obj_password', 'ext_obj_encryption_algorithom', 'ext_obj_certificate_data', 'ext_obj_token_data', 'user_token_algorithm', 'user_token_signature', 'server_nonce', 'status_code_link_id', 'activate_session_diag_info_link_id' ]" - code => "event.set('[zeek_cols]', @zeek_opcua_binary_activate_session_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_opcua_binary_activate_session" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_opcua_binary_activate_session" + init => "@zeek_opcua_binary_activate_session_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'opcua_link_id', 'client_algorithm', 'client_signature', 'client_software_cert_link_id', 'opcua_locale_link_id', 'ext_obj_type_id_encoding_mask', 'ext_obj_type_id_namespace_idx', 'ext_obj_type_id_numeric', 'ext_obj_type_id_string', 'ext_obj_type_id_guid', 'ext_obj_type_id_opaque', 'ext_obj_type_id_str', 'ext_obj_encoding', 'ext_obj_policy_id', 'ext_obj_user_name', 'ext_obj_password', 'ext_obj_encryption_algorithom', 'ext_obj_certificate_data', 'ext_obj_token_data', 'user_token_algorithm', 'user_token_signature', 'server_nonce', 'status_code_link_id', 'activate_session_diag_info_link_id' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_activate_session_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -3880,24 +4173,25 @@ filter { # opcua_binary_activate_session_client_software_cert.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - dissect { - id => "dissect_zeek_opcua_binary_activate_session_client_software_cert" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][client_software_cert_link_id]} %{[zeek_cols][cert_data]} %{[zeek_cols][cert_signature]}" - } - } - - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_opcua_binary_activate_session_client_software_cert" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_opcua_binary_activate_session_client_software_cert" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][client_software_cert_link_id]} %{[zeek_cols][cert_data]} %{[zeek_cols][cert_signature]}" + } } - ruby { - id => "ruby_zip_zeek_opcua_binary_activate_session_client_software_cert" - init => "@zeek_opcua_binary_activate_session_client_software_cert_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'client_software_cert_link_id', 'cert_data', 'cert_signature' ]" - code => "event.set('[zeek_cols]', @zeek_opcua_binary_activate_session_client_software_cert_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_opcua_binary_activate_session_client_software_cert" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_opcua_binary_activate_session_client_software_cert" + init => "@zeek_opcua_binary_activate_session_client_software_cert_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'client_software_cert_link_id', 'cert_data', 'cert_signature' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_activate_session_client_software_cert_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -3915,24 +4209,25 @@ filter { # opcua_binary_activate_session_locale_id.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - dissect { - id => "dissect_zeek_opcua_binary_activate_session_locale_id" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][opcua_locale_link_id]} %{[zeek_cols][local_id]}" - } - } - - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_opcua_binary_activate_session_locale_id" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_opcua_binary_activate_session_locale_id" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][opcua_locale_link_id]} %{[zeek_cols][local_id]}" + } } - ruby { - id => "ruby_zip_zeek_opcua_binary_activate_session_locale_id" - init => "@zeek_opcua_binary_activate_session_locale_id_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'opcua_locale_link_id', 'local_id' ]" - code => "event.set('[zeek_cols]', @zeek_opcua_binary_activate_session_locale_id_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_opcua_binary_activate_session_locale_id" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_opcua_binary_activate_session_locale_id" + init => "@zeek_opcua_binary_activate_session_locale_id_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'opcua_locale_link_id', 'local_id' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_activate_session_locale_id_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -3950,24 +4245,25 @@ filter { # opcua_binary_aggregate_filter.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - dissect { - id => "dissect_zeek_opcua_binary_aggregate_filter" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][monitored_parameters_link_id]} %{[zeek_cols][start_time]} %{[zeek_cols][start_time_str]} %{[zeek_cols][aggregate_type_encoding_mask]} %{[zeek_cols][aggregate_type_namespace_idx]} %{[zeek_cols][aggregate_type_numeric]} %{[zeek_cols][aggregate_type_string]} %{[zeek_cols][aggregate_type_guid]} %{[zeek_cols][aggregate_type_opaque]} %{[zeek_cols][processing_interval]} %{[zeek_cols][use_server_capabilities_default]} %{[zeek_cols][treat_uncertain_as_bad]} %{[zeek_cols][percent_data_good]} %{[zeek_cols][percent_data_bad]} %{[zeek_cols][use_slopped_extrapolation]} %{[zeek_cols][revised_start_time]} %{[zeek_cols][revised_start_time_str]} %{[zeek_cols][revised_processing_interval]} %{[zeek_cols][revised_use_server_capabilities_default]} %{[zeek_cols][revised_treat_uncertain_as_bad]} %{[zeek_cols][revised_percent_data_good]} %{[zeek_cols][revised_percent_data_bad]} %{[zeek_cols][revised_use_slopped_extrapolation]}" - } - } - - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_opcua_binary_aggregate_filter" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_opcua_binary_aggregate_filter" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][monitored_parameters_link_id]} %{[zeek_cols][start_time]} %{[zeek_cols][start_time_str]} %{[zeek_cols][aggregate_type_encoding_mask]} %{[zeek_cols][aggregate_type_namespace_idx]} %{[zeek_cols][aggregate_type_numeric]} %{[zeek_cols][aggregate_type_string]} %{[zeek_cols][aggregate_type_guid]} %{[zeek_cols][aggregate_type_opaque]} %{[zeek_cols][processing_interval]} %{[zeek_cols][use_server_capabilities_default]} %{[zeek_cols][treat_uncertain_as_bad]} %{[zeek_cols][percent_data_good]} %{[zeek_cols][percent_data_bad]} %{[zeek_cols][use_slopped_extrapolation]} %{[zeek_cols][revised_start_time]} %{[zeek_cols][revised_start_time_str]} %{[zeek_cols][revised_processing_interval]} %{[zeek_cols][revised_use_server_capabilities_default]} %{[zeek_cols][revised_treat_uncertain_as_bad]} %{[zeek_cols][revised_percent_data_good]} %{[zeek_cols][revised_percent_data_bad]} %{[zeek_cols][revised_use_slopped_extrapolation]}" + } } - ruby { - id => "ruby_zip_zeek_opcua_binary_aggregate_filter" - init => "@zeek_opcua_binary_aggregate_filter_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'monitored_parameters_link_id', 'start_time', 'start_time_str', 'aggregate_type_encoding_mask', 'aggregate_type_namespace_idx', 'aggregate_type_numeric', 'aggregate_type_string', 'aggregate_type_guid', 'aggregate_type_opaque', 'processing_interval', 'use_server_capabilities_default', 'treat_uncertain_as_bad', 'percent_data_good', 'percent_data_bad', 'use_slopped_extrapolation', 'revised_start_time', 'revised_start_time_str', 'revised_processing_interval', 'revised_use_server_capabilities_default', 'revised_treat_uncertain_as_bad', 'revised_percent_data_good', 'revised_percent_data_bad', 'revised_use_slopped_extrapolation' ]" - code => "event.set('[zeek_cols]', @zeek_opcua_binary_aggregate_filter_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_opcua_binary_aggregate_filter" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_opcua_binary_aggregate_filter" + init => "@zeek_opcua_binary_aggregate_filter_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'monitored_parameters_link_id', 'start_time', 'start_time_str', 'aggregate_type_encoding_mask', 'aggregate_type_namespace_idx', 'aggregate_type_numeric', 'aggregate_type_string', 'aggregate_type_guid', 'aggregate_type_opaque', 'processing_interval', 'use_server_capabilities_default', 'treat_uncertain_as_bad', 'percent_data_good', 'percent_data_bad', 'use_slopped_extrapolation', 'revised_start_time', 'revised_start_time_str', 'revised_processing_interval', 'revised_use_server_capabilities_default', 'revised_treat_uncertain_as_bad', 'revised_percent_data_good', 'revised_percent_data_bad', 'revised_use_slopped_extrapolation' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_aggregate_filter_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -3985,24 +4281,25 @@ filter { # opcua_binary_event_filter_attribute_operand.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - dissect { - id => "dissect_zeek_opcua_binary_event_filter_attribute_operand" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][content_filter_filter_operand_link_id]} %{[zeek_cols][node_id_encoding_mask]} %{[zeek_cols][node_id_namespace_idx]} %{[zeek_cols][node_id_numeric]} %{[zeek_cols][node_id_string]} %{[zeek_cols][node_id_guid]} %{[zeek_cols][node_id_opaque]} %{[zeek_cols][alias]} %{[zeek_cols][browse_path_element_link_id]} %{[zeek_cols][attribute]} %{[zeek_cols][index_range]}" - } - } - - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_opcua_binary_event_filter_attribute_operand" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_opcua_binary_event_filter_attribute_operand" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][content_filter_filter_operand_link_id]} %{[zeek_cols][node_id_encoding_mask]} %{[zeek_cols][node_id_namespace_idx]} %{[zeek_cols][node_id_numeric]} %{[zeek_cols][node_id_string]} %{[zeek_cols][node_id_guid]} %{[zeek_cols][node_id_opaque]} %{[zeek_cols][alias]} %{[zeek_cols][browse_path_element_link_id]} %{[zeek_cols][attribute]} %{[zeek_cols][index_range]}" + } } - ruby { - id => "ruby_zip_zeek_opcua_binary_event_filter_attribute_operand" - init => "@zeek_opcua_binary_event_filter_attribute_operand_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'content_filter_filter_operand_link_id', 'node_id_encoding_mask', 'node_id_namespace_idx', 'node_id_numeric', 'node_id_string', 'node_id_guid', 'node_id_opaque', 'alias', 'browse_path_element_link_id', 'attribute', 'index_range' ]" - code => "event.set('[zeek_cols]', @zeek_opcua_binary_event_filter_attribute_operand_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_opcua_binary_event_filter_attribute_operand" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_opcua_binary_event_filter_attribute_operand" + init => "@zeek_opcua_binary_event_filter_attribute_operand_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'content_filter_filter_operand_link_id', 'node_id_encoding_mask', 'node_id_namespace_idx', 'node_id_numeric', 'node_id_string', 'node_id_guid', 'node_id_opaque', 'alias', 'browse_path_element_link_id', 'attribute', 'index_range' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_event_filter_attribute_operand_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -4020,24 +4317,25 @@ filter { # opcua_binary_event_filter_attribute_operand_browse_paths.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - dissect { - id => "dissect_zeek_opcua_binary_event_filter_attribute_operand_browse_paths" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][browse_path_element_link_id]} %{[zeek_cols][type_id_encoding_mask]} %{[zeek_cols][type_id_namespace_idx]} %{[zeek_cols][type_id_numeric]} %{[zeek_cols][type_id_string]} %{[zeek_cols][type_id_guid]} %{[zeek_cols][type_id_opaque]} %{[zeek_cols][is_inverse]} %{[zeek_cols][include_subtypes]} %{[zeek_cols][target_name_namespace_idx]} %{[zeek_cols][target_name]}" - } - } - - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_opcua_binary_event_filter_attribute_operand_browse_paths" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_opcua_binary_event_filter_attribute_operand_browse_paths" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][browse_path_element_link_id]} %{[zeek_cols][type_id_encoding_mask]} %{[zeek_cols][type_id_namespace_idx]} %{[zeek_cols][type_id_numeric]} %{[zeek_cols][type_id_string]} %{[zeek_cols][type_id_guid]} %{[zeek_cols][type_id_opaque]} %{[zeek_cols][is_inverse]} %{[zeek_cols][include_subtypes]} %{[zeek_cols][target_name_namespace_idx]} %{[zeek_cols][target_name]}" + } } - ruby { - id => "ruby_zip_zeek_opcua_binary_event_filter_attribute_operand_browse_paths" - init => "@zeek_opcua_binary_event_filter_attribute_operand_browse_paths_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'browse_path_element_link_id', 'type_id_encoding_mask', 'type_id_namespace_idx', 'type_id_numeric', 'type_id_string', 'type_id_guid', 'type_id_opaque', 'is_inverse', 'include_subtypes', 'target_name_namespace_idx', 'target_name' ]" - code => "event.set('[zeek_cols]', @zeek_opcua_binary_event_filter_attribute_operand_browse_paths_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_opcua_binary_event_filter_attribute_operand_browse_paths" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_opcua_binary_event_filter_attribute_operand_browse_paths" + init => "@zeek_opcua_binary_event_filter_attribute_operand_browse_paths_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'browse_path_element_link_id', 'type_id_encoding_mask', 'type_id_namespace_idx', 'type_id_numeric', 'type_id_string', 'type_id_guid', 'type_id_opaque', 'is_inverse', 'include_subtypes', 'target_name_namespace_idx', 'target_name' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_event_filter_attribute_operand_browse_paths_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -4055,24 +4353,25 @@ filter { # opcua_binary_browse.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - dissect { - id => "dissect_zeek_opcua_binary_browse" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][opcua_link_id]} %{[zeek_cols][browse_service_type]} %{[zeek_cols][browse_view_id_encoding_mask]} %{[zeek_cols][browse_view_id_namespace_idx]} %{[zeek_cols][browse_view_id_numeric]} %{[zeek_cols][browse_view_id_string]} %{[zeek_cols][browse_view_id_guid]} %{[zeek_cols][browse_view_id_opaque]} %{[zeek_cols][browse_view_description_timestamp]} %{[zeek_cols][browse_view_description_view_version]} %{[zeek_cols][req_max_ref_nodes]} %{[zeek_cols][browse_description_link_id]} %{[zeek_cols][browse_next_release_continuation_point]} %{[zeek_cols][browse_next_link_id]} %{[zeek_cols][browse_response_link_id]} %{[zeek_cols][browse_diag_info_link_id]}" - } - } - - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_opcua_binary_browse" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_opcua_binary_browse" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][opcua_link_id]} %{[zeek_cols][browse_service_type]} %{[zeek_cols][browse_view_id_encoding_mask]} %{[zeek_cols][browse_view_id_namespace_idx]} %{[zeek_cols][browse_view_id_numeric]} %{[zeek_cols][browse_view_id_string]} %{[zeek_cols][browse_view_id_guid]} %{[zeek_cols][browse_view_id_opaque]} %{[zeek_cols][browse_view_description_timestamp]} %{[zeek_cols][browse_view_description_view_version]} %{[zeek_cols][req_max_ref_nodes]} %{[zeek_cols][browse_description_link_id]} %{[zeek_cols][browse_next_release_continuation_point]} %{[zeek_cols][browse_next_link_id]} %{[zeek_cols][browse_response_link_id]} %{[zeek_cols][browse_diag_info_link_id]}" + } } - ruby { - id => "ruby_zip_zeek_opcua_binary_browse" - init => "@zeek_opcua_binary_browse_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'opcua_link_id', 'browse_service_type', 'browse_view_id_encoding_mask', 'browse_view_id_namespace_idx', 'browse_view_id_numeric', 'browse_view_id_string', 'browse_view_id_guid', 'browse_view_id_opaque', 'browse_view_description_timestamp', 'browse_view_description_view_version', 'req_max_ref_nodes', 'browse_description_link_id', 'browse_next_release_continuation_point', 'browse_next_link_id', 'browse_response_link_id', 'browse_diag_info_link_id' ]" - code => "event.set('[zeek_cols]', @zeek_opcua_binary_browse_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_opcua_binary_browse" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_opcua_binary_browse" + init => "@zeek_opcua_binary_browse_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'opcua_link_id', 'browse_service_type', 'browse_view_id_encoding_mask', 'browse_view_id_namespace_idx', 'browse_view_id_numeric', 'browse_view_id_string', 'browse_view_id_guid', 'browse_view_id_opaque', 'browse_view_description_timestamp', 'browse_view_description_view_version', 'req_max_ref_nodes', 'browse_description_link_id', 'browse_next_release_continuation_point', 'browse_next_link_id', 'browse_response_link_id', 'browse_diag_info_link_id' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_browse_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -4090,24 +4389,25 @@ filter { # opcua_binary_browse_description.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - dissect { - id => "dissect_zeek_opcua_binary_browse_description" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][browse_description_link_id]} %{[zeek_cols][browse_description_encoding_mask]} %{[zeek_cols][browse_description_namespace_idx]} %{[zeek_cols][browse_description_numeric]} %{[zeek_cols][browse_description_string]} %{[zeek_cols][browse_description_guid]} %{[zeek_cols][browse_description_opaque]} %{[zeek_cols][browse_direction]} %{[zeek_cols][browse_description_ref_encoding_mask]} %{[zeek_cols][browse_description_ref_namespace_idx]} %{[zeek_cols][browse_description_ref_numeric]} %{[zeek_cols][browse_description_ref_string]} %{[zeek_cols][browse_description_ref_guid]} %{[zeek_cols][browse_description_ref_opaque]} %{[zeek_cols][browse_description_include_subtypes]} %{[zeek_cols][browse_node_class_mask]} %{[zeek_cols][browse_result_mask]}" - } - } - - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_opcua_binary_browse_description" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_opcua_binary_browse_description" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][browse_description_link_id]} %{[zeek_cols][browse_description_encoding_mask]} %{[zeek_cols][browse_description_namespace_idx]} %{[zeek_cols][browse_description_numeric]} %{[zeek_cols][browse_description_string]} %{[zeek_cols][browse_description_guid]} %{[zeek_cols][browse_description_opaque]} %{[zeek_cols][browse_direction]} %{[zeek_cols][browse_description_ref_encoding_mask]} %{[zeek_cols][browse_description_ref_namespace_idx]} %{[zeek_cols][browse_description_ref_numeric]} %{[zeek_cols][browse_description_ref_string]} %{[zeek_cols][browse_description_ref_guid]} %{[zeek_cols][browse_description_ref_opaque]} %{[zeek_cols][browse_description_include_subtypes]} %{[zeek_cols][browse_node_class_mask]} %{[zeek_cols][browse_result_mask]}" + } } - ruby { - id => "ruby_zip_zeek_opcua_binary_browse_description" - init => "@zeek_opcua_binary_browse_description_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'browse_description_link_id', 'browse_description_encoding_mask', 'browse_description_namespace_idx', 'browse_description_numeric', 'browse_description_string', 'browse_description_guid', 'browse_description_opaque', 'browse_direction', 'browse_description_ref_encoding_mask', 'browse_description_ref_namespace_idx', 'browse_description_ref_numeric', 'browse_description_ref_string', 'browse_description_ref_guid', 'browse_description_ref_opaque', 'browse_description_include_subtypes', 'browse_node_class_mask', 'browse_result_mask' ]" - code => "event.set('[zeek_cols]', @zeek_opcua_binary_browse_description_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_opcua_binary_browse_description" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_opcua_binary_browse_description" + init => "@zeek_opcua_binary_browse_description_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'browse_description_link_id', 'browse_description_encoding_mask', 'browse_description_namespace_idx', 'browse_description_numeric', 'browse_description_string', 'browse_description_guid', 'browse_description_opaque', 'browse_direction', 'browse_description_ref_encoding_mask', 'browse_description_ref_namespace_idx', 'browse_description_ref_numeric', 'browse_description_ref_string', 'browse_description_ref_guid', 'browse_description_ref_opaque', 'browse_description_include_subtypes', 'browse_node_class_mask', 'browse_result_mask' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_browse_description_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -4125,24 +4425,25 @@ filter { # opcua_binary_browse_response_references.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - dissect { - id => "dissect_zeek_opcua_binary_browse_response_references" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][browse_reference_link_id]} %{[zeek_cols][browse_response_ref_encoding_mask]} %{[zeek_cols][browse_response_ref_namespace_idx]} %{[zeek_cols][browse_response_ref_numeric]} %{[zeek_cols][browse_response_ref_string]} %{[zeek_cols][browse_response_ref_guid]} %{[zeek_cols][browse_response_ref_opaque]} %{[zeek_cols][browse_response_is_forward]} %{[zeek_cols][browse_response_ref_type_encoding_mask]} %{[zeek_cols][browse_response_ref_type_namespace_idx]} %{[zeek_cols][browse_response_ref_type_numeric]} %{[zeek_cols][browse_response_ref_type_string]} %{[zeek_cols][browse_response_ref_type_guid]} %{[zeek_cols][browse_response_ref_type_opaque]} %{[zeek_cols][browse_response_ref_type_namespace_uri]} %{[zeek_cols][browse_response_ref_type_server_idx]} %{[zeek_cols][browse_response_ref_name_idx]} %{[zeek_cols][browse_response_ref_name]} %{[zeek_cols][browse_response_display_name_mask]} %{[zeek_cols][browse_response_display_name_locale]} %{[zeek_cols][browse_response_display_name_text]} %{[zeek_cols][browse_response_node_class]} %{[zeek_cols][browse_response_type_def_encoding_mask]} %{[zeek_cols][browse_response_type_def_namespace_idx]} %{[zeek_cols][browse_response_type_def_numeric]} %{[zeek_cols][browse_response_type_def_string]} %{[zeek_cols][browse_response_type_def_guid]} %{[zeek_cols][browse_response_type_def_opaque]} %{[zeek_cols][browse_response_type_def_namespace_uri]} %{[zeek_cols][browse_response_type_def_server_idx]}" - } - } - - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_opcua_binary_browse_response_references" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_opcua_binary_browse_response_references" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][browse_reference_link_id]} %{[zeek_cols][browse_response_ref_encoding_mask]} %{[zeek_cols][browse_response_ref_namespace_idx]} %{[zeek_cols][browse_response_ref_numeric]} %{[zeek_cols][browse_response_ref_string]} %{[zeek_cols][browse_response_ref_guid]} %{[zeek_cols][browse_response_ref_opaque]} %{[zeek_cols][browse_response_is_forward]} %{[zeek_cols][browse_response_ref_type_encoding_mask]} %{[zeek_cols][browse_response_ref_type_namespace_idx]} %{[zeek_cols][browse_response_ref_type_numeric]} %{[zeek_cols][browse_response_ref_type_string]} %{[zeek_cols][browse_response_ref_type_guid]} %{[zeek_cols][browse_response_ref_type_opaque]} %{[zeek_cols][browse_response_ref_type_namespace_uri]} %{[zeek_cols][browse_response_ref_type_server_idx]} %{[zeek_cols][browse_response_ref_name_idx]} %{[zeek_cols][browse_response_ref_name]} %{[zeek_cols][browse_response_display_name_mask]} %{[zeek_cols][browse_response_display_name_locale]} %{[zeek_cols][browse_response_display_name_text]} %{[zeek_cols][browse_response_node_class]} %{[zeek_cols][browse_response_type_def_encoding_mask]} %{[zeek_cols][browse_response_type_def_namespace_idx]} %{[zeek_cols][browse_response_type_def_numeric]} %{[zeek_cols][browse_response_type_def_string]} %{[zeek_cols][browse_response_type_def_guid]} %{[zeek_cols][browse_response_type_def_opaque]} %{[zeek_cols][browse_response_type_def_namespace_uri]} %{[zeek_cols][browse_response_type_def_server_idx]}" + } } - ruby { - id => "ruby_zip_zeek_opcua_binary_browse_response_references" - init => "@zeek_opcua_binary_browse_response_references_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'browse_reference_link_id', 'browse_response_ref_encoding_mask', 'browse_response_ref_namespace_idx', 'browse_response_ref_numeric', 'browse_response_ref_string', 'browse_response_ref_guid', 'browse_response_ref_opaque', 'browse_response_is_forward', 'browse_response_ref_type_encoding_mask', 'browse_response_ref_type_namespace_idx', 'browse_response_ref_type_numeric', 'browse_response_ref_type_string', 'browse_response_ref_type_guid', 'browse_response_ref_type_opaque', 'browse_response_ref_type_namespace_uri', 'browse_response_ref_type_server_idx', 'browse_response_ref_name_idx', 'browse_response_ref_name', 'browse_response_display_name_mask', 'browse_response_display_name_locale', 'browse_response_display_name_text', 'browse_response_node_class', 'browse_response_type_def_encoding_mask', 'browse_response_type_def_namespace_idx', 'browse_response_type_def_numeric', 'browse_response_type_def_string', 'browse_response_type_def_guid', 'browse_response_type_def_opaque', 'browse_response_type_def_namespace_uri', 'browse_response_type_def_server_idx' ]" - code => "event.set('[zeek_cols]', @zeek_opcua_binary_browse_response_references_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_opcua_binary_browse_response_references" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_opcua_binary_browse_response_references" + init => "@zeek_opcua_binary_browse_response_references_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'browse_reference_link_id', 'browse_response_ref_encoding_mask', 'browse_response_ref_namespace_idx', 'browse_response_ref_numeric', 'browse_response_ref_string', 'browse_response_ref_guid', 'browse_response_ref_opaque', 'browse_response_is_forward', 'browse_response_ref_type_encoding_mask', 'browse_response_ref_type_namespace_idx', 'browse_response_ref_type_numeric', 'browse_response_ref_type_string', 'browse_response_ref_type_guid', 'browse_response_ref_type_opaque', 'browse_response_ref_type_namespace_uri', 'browse_response_ref_type_server_idx', 'browse_response_ref_name_idx', 'browse_response_ref_name', 'browse_response_display_name_mask', 'browse_response_display_name_locale', 'browse_response_display_name_text', 'browse_response_node_class', 'browse_response_type_def_encoding_mask', 'browse_response_type_def_namespace_idx', 'browse_response_type_def_numeric', 'browse_response_type_def_string', 'browse_response_type_def_guid', 'browse_response_type_def_opaque', 'browse_response_type_def_namespace_uri', 'browse_response_type_def_server_idx' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_browse_response_references_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -4160,24 +4461,25 @@ filter { # opcua_binary_browse_request_continuation_point.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - dissect { - id => "dissect_zeek_opcua_binary_browse_request_continuation_point" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][browse_next_link_id]} %{[zeek_cols][continuation_point]}" - } - } - - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_opcua_binary_browse_request_continuation_point" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_opcua_binary_browse_request_continuation_point" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][browse_next_link_id]} %{[zeek_cols][continuation_point]}" + } } - ruby { - id => "ruby_zip_zeek_opcua_binary_browse_request_continuation_point" - init => "@zeek_opcua_binary_browse_request_continuation_point_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'browse_next_link_id', 'continuation_point' ]" - code => "event.set('[zeek_cols]', @zeek_opcua_binary_browse_request_continuation_point_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_opcua_binary_browse_request_continuation_point" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_opcua_binary_browse_request_continuation_point" + init => "@zeek_opcua_binary_browse_request_continuation_point_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'browse_next_link_id', 'continuation_point' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_browse_request_continuation_point_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -4195,24 +4497,25 @@ filter { # opcua_binary_browse_result.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - dissect { - id => "dissect_zeek_opcua_binary_browse_result" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][browse_response_link_id]} %{[zeek_cols][status_code_link_id]} %{[zeek_cols][browse_result_continuation_point]} %{[zeek_cols][browse_reference_link_id]}" - } - } - - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_opcua_binary_browse_result" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_opcua_binary_browse_result" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][browse_response_link_id]} %{[zeek_cols][status_code_link_id]} %{[zeek_cols][browse_result_continuation_point]} %{[zeek_cols][browse_reference_link_id]}" + } } - ruby { - id => "ruby_zip_zeek_opcua_binary_browse_result" - init => "@zeek_opcua_binary_browse_result_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'browse_response_link_id', 'status_code_link_id', 'browse_result_continuation_point', 'browse_reference_link_id' ]" - code => "event.set('[zeek_cols]', @zeek_opcua_binary_browse_result_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_opcua_binary_browse_result" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_opcua_binary_browse_result" + init => "@zeek_opcua_binary_browse_result_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'browse_response_link_id', 'status_code_link_id', 'browse_result_continuation_point', 'browse_reference_link_id' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_browse_result_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -4230,24 +4533,25 @@ filter { # opcua_binary_close_session.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - dissect { - id => "dissect_zeek_opcua_binary_close_session" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][opcua_link_id]} %{[zeek_cols][del_subscriptions]}" - } - } - - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_opcua_binary_close_session" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_opcua_binary_close_session" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][opcua_link_id]} %{[zeek_cols][del_subscriptions]}" + } } - ruby { - id => "ruby_zip_zeek_opcua_binary_close_session" - init => "@zeek_opcua_binary_close_session_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'opcua_link_id', 'del_subscriptions' ]" - code => "event.set('[zeek_cols]', @zeek_opcua_binary_close_session_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_opcua_binary_close_session" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_opcua_binary_close_session" + init => "@zeek_opcua_binary_close_session_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'opcua_link_id', 'del_subscriptions' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_close_session_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -4265,24 +4569,25 @@ filter { # opcua_binary_event_filter_where_clause.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - dissect { - id => "dissect_zeek_opcua_binary_event_filter_where_clause" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][where_clause_link_id]} %{[zeek_cols][content_filter_element_link_id]} %{[zeek_cols][content_filter_status_code_link_id]} %{[zeek_cols][content_filter_diag_info_link_id]}" - } - } - - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_opcua_binary_event_filter_where_clause" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_opcua_binary_event_filter_where_clause" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][where_clause_link_id]} %{[zeek_cols][content_filter_element_link_id]} %{[zeek_cols][content_filter_status_code_link_id]} %{[zeek_cols][content_filter_diag_info_link_id]}" + } } - ruby { - id => "ruby_zip_zeek_opcua_binary_event_filter_where_clause" - init => "@zeek_opcua_binary_event_filter_where_clause_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'where_clause_link_id', 'content_filter_element_link_id', 'content_filter_status_code_link_id', 'content_filter_diag_info_link_id' ]" - code => "event.set('[zeek_cols]', @zeek_opcua_binary_event_filter_where_clause_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_opcua_binary_event_filter_where_clause" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_opcua_binary_event_filter_where_clause" + init => "@zeek_opcua_binary_event_filter_where_clause_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'where_clause_link_id', 'content_filter_element_link_id', 'content_filter_status_code_link_id', 'content_filter_diag_info_link_id' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_event_filter_where_clause_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -4300,24 +4605,25 @@ filter { # opcua_binary_event_filter_where_clause_elements.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - dissect { - id => "dissect_zeek_opcua_binary_event_filter_where_clause_elements" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][content_filter_element_link_id]} %{[zeek_cols][filter_operator]} %{[zeek_cols][content_filter_filter_operand_type_id_node_id_encoding_mask]} %{[zeek_cols][content_filter_filter_operand_type_id_node_id_namespace_idx]} %{[zeek_cols][content_filter_filter_operand_type_id_node_id_numeric]} %{[zeek_cols][content_filter_filter_operand_type_id_node_id_string]} %{[zeek_cols][content_filter_filter_operand_type_id_node_id_guid]} %{[zeek_cols][content_filter_filter_operand_type_id_node_id_opaque]} %{[zeek_cols][content_filter_filter_operand_type_id_string]} %{[zeek_cols][content_filter_filter_operand_type_id_encoding]} %{[zeek_cols][content_filter_filter_operand_link_id]} %{[zeek_cols][content_filter_operand_status_code_link_id]} %{[zeek_cols][content_filter_operand_diag_info_link_id]}" - } - } - - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_opcua_binary_event_filter_where_clause_elements" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_opcua_binary_event_filter_where_clause_elements" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][content_filter_element_link_id]} %{[zeek_cols][filter_operator]} %{[zeek_cols][content_filter_filter_operand_type_id_node_id_encoding_mask]} %{[zeek_cols][content_filter_filter_operand_type_id_node_id_namespace_idx]} %{[zeek_cols][content_filter_filter_operand_type_id_node_id_numeric]} %{[zeek_cols][content_filter_filter_operand_type_id_node_id_string]} %{[zeek_cols][content_filter_filter_operand_type_id_node_id_guid]} %{[zeek_cols][content_filter_filter_operand_type_id_node_id_opaque]} %{[zeek_cols][content_filter_filter_operand_type_id_string]} %{[zeek_cols][content_filter_filter_operand_type_id_encoding]} %{[zeek_cols][content_filter_filter_operand_link_id]} %{[zeek_cols][content_filter_operand_status_code_link_id]} %{[zeek_cols][content_filter_operand_diag_info_link_id]}" + } } - ruby { - id => "ruby_zip_zeek_opcua_binary_event_filter_where_clause_elements" - init => "@zeek_opcua_binary_event_filter_where_clause_elements_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'content_filter_element_link_id', 'filter_operator', 'content_filter_filter_operand_type_id_node_id_encoding_mask', 'content_filter_filter_operand_type_id_node_id_namespace_idx', 'content_filter_filter_operand_type_id_node_id_numeric', 'content_filter_filter_operand_type_id_node_id_string', 'content_filter_filter_operand_type_id_node_id_guid', 'content_filter_filter_operand_type_id_node_id_opaque', 'content_filter_filter_operand_type_id_string', 'content_filter_filter_operand_type_id_encoding', 'content_filter_filter_operand_link_id', 'content_filter_operand_status_code_link_id', 'content_filter_operand_diag_info_link_id' ]" - code => "event.set('[zeek_cols]', @zeek_opcua_binary_event_filter_where_clause_elements_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_opcua_binary_event_filter_where_clause_elements" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_opcua_binary_event_filter_where_clause_elements" + init => "@zeek_opcua_binary_event_filter_where_clause_elements_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'content_filter_element_link_id', 'filter_operator', 'content_filter_filter_operand_type_id_node_id_encoding_mask', 'content_filter_filter_operand_type_id_node_id_namespace_idx', 'content_filter_filter_operand_type_id_node_id_numeric', 'content_filter_filter_operand_type_id_node_id_string', 'content_filter_filter_operand_type_id_node_id_guid', 'content_filter_filter_operand_type_id_node_id_opaque', 'content_filter_filter_operand_type_id_string', 'content_filter_filter_operand_type_id_encoding', 'content_filter_filter_operand_link_id', 'content_filter_operand_status_code_link_id', 'content_filter_operand_diag_info_link_id' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_event_filter_where_clause_elements_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -4335,24 +4641,25 @@ filter { # opcua_binary_create_monitored_items.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - dissect { - id => "dissect_zeek_opcua_binary_create_monitored_items" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][opcua_link_id]} %{[zeek_cols][subscription_id]} %{[zeek_cols][timestamps_to_return]} %{[zeek_cols][timestamps_to_return_str]} %{[zeek_cols][create_item_link_id]} %{[zeek_cols][create_monitored_items_diag_info_link_id]}" - } - } - - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_opcua_binary_create_monitored_items" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_opcua_binary_create_monitored_items" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][opcua_link_id]} %{[zeek_cols][subscription_id]} %{[zeek_cols][timestamps_to_return]} %{[zeek_cols][timestamps_to_return_str]} %{[zeek_cols][create_item_link_id]} %{[zeek_cols][create_monitored_items_diag_info_link_id]}" + } } - ruby { - id => "ruby_zip_zeek_opcua_binary_create_monitored_items" - init => "@zeek_opcua_binary_create_monitored_items_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'opcua_link_id', 'subscription_id', 'timestamps_to_return', 'timestamps_to_return_str', 'create_item_link_id', 'create_monitored_items_diag_info_link_id' ]" - code => "event.set('[zeek_cols]', @zeek_opcua_binary_create_monitored_items_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_opcua_binary_create_monitored_items" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_opcua_binary_create_monitored_items" + init => "@zeek_opcua_binary_create_monitored_items_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'opcua_link_id', 'subscription_id', 'timestamps_to_return', 'timestamps_to_return_str', 'create_item_link_id', 'create_monitored_items_diag_info_link_id' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_create_monitored_items_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -4370,24 +4677,25 @@ filter { # opcua_binary_create_monitored_items_create_item.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - dissect { - id => "dissect_zeek_opcua_binary_create_monitored_items_create_item" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][create_item_link_id]} %{[zeek_cols][item_to_monitor_node_id_encoding_mask]} %{[zeek_cols][item_to_monitor_node_id_namespace_idx]} %{[zeek_cols][item_to_monitor_node_id_numeric]} %{[zeek_cols][item_to_monitor_node_id_string]} %{[zeek_cols][item_to_monitor_node_id_guid]} %{[zeek_cols][item_to_monitor_node_id_opaque]} %{[zeek_cols][item_to_monitor_attribute_id]} %{[zeek_cols][item_to_monitor_index_range]} %{[zeek_cols][item_to_monitor_namespace_idx]} %{[zeek_cols][item_to_monitor_name]} %{[zeek_cols][monitoring_mode]} %{[zeek_cols][monitoring_parameters_client_handle]} %{[zeek_cols][monitoring_parameters_sampling_interval]} %{[zeek_cols][monitoring_parameters_queue_size]} %{[zeek_cols][monitoring_parameters_discard_oldest]} %{[zeek_cols][monitoring_parameters_filter_info_type_id_node_id_encoding_mask]} %{[zeek_cols][monitoring_parameters_filter_info_type_id_node_id_namespace_idx]} %{[zeek_cols][monitoring_parameters_filter_info_type_id_node_id_numeric]} %{[zeek_cols][monitoring_parameters_filter_info_type_id_node_id_string]} %{[zeek_cols][monitoring_parameters_filter_info_type_id_node_id_guid]} %{[zeek_cols][monitoring_parameters_filter_info_type_id_node_id_opaque]} %{[zeek_cols][monitoring_parameters_filter_info_type_id_string]} %{[zeek_cols][monitoring_parameters_filter_info_type_id_encoding]} %{[zeek_cols][filter_info_details_link_id]} %{[zeek_cols][monitoring_parameters_status_code_link_id]} %{[zeek_cols][monitored_item_index_id]} %{[zeek_cols][monitoring_parameters_revised_sampling_interval]} %{[zeek_cols][monitoring_parameters_revised_queue_size]}" - } - } - - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_opcua_binary_create_monitored_items_create_item" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_opcua_binary_create_monitored_items_create_item" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][create_item_link_id]} %{[zeek_cols][item_to_monitor_node_id_encoding_mask]} %{[zeek_cols][item_to_monitor_node_id_namespace_idx]} %{[zeek_cols][item_to_monitor_node_id_numeric]} %{[zeek_cols][item_to_monitor_node_id_string]} %{[zeek_cols][item_to_monitor_node_id_guid]} %{[zeek_cols][item_to_monitor_node_id_opaque]} %{[zeek_cols][item_to_monitor_attribute_id]} %{[zeek_cols][item_to_monitor_index_range]} %{[zeek_cols][item_to_monitor_namespace_idx]} %{[zeek_cols][item_to_monitor_name]} %{[zeek_cols][monitoring_mode]} %{[zeek_cols][monitoring_parameters_client_handle]} %{[zeek_cols][monitoring_parameters_sampling_interval]} %{[zeek_cols][monitoring_parameters_queue_size]} %{[zeek_cols][monitoring_parameters_discard_oldest]} %{[zeek_cols][monitoring_parameters_filter_info_type_id_node_id_encoding_mask]} %{[zeek_cols][monitoring_parameters_filter_info_type_id_node_id_namespace_idx]} %{[zeek_cols][monitoring_parameters_filter_info_type_id_node_id_numeric]} %{[zeek_cols][monitoring_parameters_filter_info_type_id_node_id_string]} %{[zeek_cols][monitoring_parameters_filter_info_type_id_node_id_guid]} %{[zeek_cols][monitoring_parameters_filter_info_type_id_node_id_opaque]} %{[zeek_cols][monitoring_parameters_filter_info_type_id_string]} %{[zeek_cols][monitoring_parameters_filter_info_type_id_encoding]} %{[zeek_cols][filter_info_details_link_id]} %{[zeek_cols][monitoring_parameters_status_code_link_id]} %{[zeek_cols][monitored_item_index_id]} %{[zeek_cols][monitoring_parameters_revised_sampling_interval]} %{[zeek_cols][monitoring_parameters_revised_queue_size]}" + } } - ruby { - id => "ruby_zip_zeek_opcua_binary_create_monitored_items_create_item" - init => "@zeek_opcua_binary_create_monitored_items_create_item_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'create_item_link_id', 'item_to_monitor_node_id_encoding_mask', 'item_to_monitor_node_id_namespace_idx', 'item_to_monitor_node_id_numeric', 'item_to_monitor_node_id_string', 'item_to_monitor_node_id_guid', 'item_to_monitor_node_id_opaque', 'item_to_monitor_attribute_id', 'item_to_monitor_index_range', 'item_to_monitor_namespace_idx', 'item_to_monitor_name', 'monitoring_mode', 'monitoring_parameters_client_handle', 'monitoring_parameters_sampling_interval', 'monitoring_parameters_queue_size', 'monitoring_parameters_discard_oldest', 'monitoring_parameters_filter_info_type_id_node_id_encoding_mask', 'monitoring_parameters_filter_info_type_id_node_id_namespace_idx', 'monitoring_parameters_filter_info_type_id_node_id_numeric', 'monitoring_parameters_filter_info_type_id_node_id_string', 'monitoring_parameters_filter_info_type_id_node_id_guid', 'monitoring_parameters_filter_info_type_id_node_id_opaque', 'monitoring_parameters_filter_info_type_id_string', 'monitoring_parameters_filter_info_type_id_encoding', 'filter_info_details_link_id', 'monitoring_parameters_status_code_link_id', 'monitored_item_index_id', 'monitoring_parameters_revised_sampling_interval', 'monitoring_parameters_revised_queue_size' ]" - code => "event.set('[zeek_cols]', @zeek_opcua_binary_create_monitored_items_create_item_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_opcua_binary_create_monitored_items_create_item" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_opcua_binary_create_monitored_items_create_item" + init => "@zeek_opcua_binary_create_monitored_items_create_item_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'create_item_link_id', 'item_to_monitor_node_id_encoding_mask', 'item_to_monitor_node_id_namespace_idx', 'item_to_monitor_node_id_numeric', 'item_to_monitor_node_id_string', 'item_to_monitor_node_id_guid', 'item_to_monitor_node_id_opaque', 'item_to_monitor_attribute_id', 'item_to_monitor_index_range', 'item_to_monitor_namespace_idx', 'item_to_monitor_name', 'monitoring_mode', 'monitoring_parameters_client_handle', 'monitoring_parameters_sampling_interval', 'monitoring_parameters_queue_size', 'monitoring_parameters_discard_oldest', 'monitoring_parameters_filter_info_type_id_node_id_encoding_mask', 'monitoring_parameters_filter_info_type_id_node_id_namespace_idx', 'monitoring_parameters_filter_info_type_id_node_id_numeric', 'monitoring_parameters_filter_info_type_id_node_id_string', 'monitoring_parameters_filter_info_type_id_node_id_guid', 'monitoring_parameters_filter_info_type_id_node_id_opaque', 'monitoring_parameters_filter_info_type_id_string', 'monitoring_parameters_filter_info_type_id_encoding', 'filter_info_details_link_id', 'monitoring_parameters_status_code_link_id', 'monitored_item_index_id', 'monitoring_parameters_revised_sampling_interval', 'monitoring_parameters_revised_queue_size' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_create_monitored_items_create_item_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -4405,24 +4713,25 @@ filter { # opcua_binary_create_session.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - dissect { - id => "dissect_zeek_opcua_binary_create_session" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][opcua_link_id]} %{[zeek_cols][application_uri]} %{[zeek_cols][product_uri]} %{[zeek_cols][encoding_mask]} %{[zeek_cols][locale]} %{[zeek_cols][text]} %{[zeek_cols][application_type]} %{[zeek_cols][gateway_server_uri]} %{[zeek_cols][discovery_profile_uri]} %{[zeek_cols][discovery_profile_link_id]} %{[zeek_cols][server_uri]} %{[zeek_cols][endpoint_url]} %{[zeek_cols][session_name]} %{[zeek_cols][client_nonce]} %{[zeek_cols][client_cert_size]} %{[zeek_cols][client_cert]} %{[zeek_cols][req_session_timeout]} %{[zeek_cols][max_res_msg_size]} %{[zeek_cols][session_id_encoding_mask]} %{[zeek_cols][session_id_namespace_idx]} %{[zeek_cols][session_id_numeric]} %{[zeek_cols][session_id_string]} %{[zeek_cols][session_id_guid]} %{[zeek_cols][session_id_opaque]} %{[zeek_cols][auth_token_encoding_mask]} %{[zeek_cols][auth_token_namespace_idx]} %{[zeek_cols][auth_token_numeric]} %{[zeek_cols][auth_token_string]} %{[zeek_cols][auth_token_guid]} %{[zeek_cols][auth_token_opaque]} %{[zeek_cols][revised_session_timeout]} %{[zeek_cols][server_nonce]} %{[zeek_cols][server_cert_size]} %{[zeek_cols][server_cert]} %{[zeek_cols][endpoint_link_id]} %{[zeek_cols][algorithm]} %{[zeek_cols][signature]} %{[zeek_cols][max_req_msg_size]}" - } - } - - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_opcua_binary_create_session" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_opcua_binary_create_session" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][opcua_link_id]} %{[zeek_cols][application_uri]} %{[zeek_cols][product_uri]} %{[zeek_cols][encoding_mask]} %{[zeek_cols][locale]} %{[zeek_cols][text]} %{[zeek_cols][application_type]} %{[zeek_cols][gateway_server_uri]} %{[zeek_cols][discovery_profile_uri]} %{[zeek_cols][discovery_profile_link_id]} %{[zeek_cols][server_uri]} %{[zeek_cols][endpoint_url]} %{[zeek_cols][session_name]} %{[zeek_cols][client_nonce]} %{[zeek_cols][client_cert_size]} %{[zeek_cols][client_cert]} %{[zeek_cols][req_session_timeout]} %{[zeek_cols][max_res_msg_size]} %{[zeek_cols][session_id_encoding_mask]} %{[zeek_cols][session_id_namespace_idx]} %{[zeek_cols][session_id_numeric]} %{[zeek_cols][session_id_string]} %{[zeek_cols][session_id_guid]} %{[zeek_cols][session_id_opaque]} %{[zeek_cols][auth_token_encoding_mask]} %{[zeek_cols][auth_token_namespace_idx]} %{[zeek_cols][auth_token_numeric]} %{[zeek_cols][auth_token_string]} %{[zeek_cols][auth_token_guid]} %{[zeek_cols][auth_token_opaque]} %{[zeek_cols][revised_session_timeout]} %{[zeek_cols][server_nonce]} %{[zeek_cols][server_cert_size]} %{[zeek_cols][server_cert]} %{[zeek_cols][endpoint_link_id]} %{[zeek_cols][algorithm]} %{[zeek_cols][signature]} %{[zeek_cols][max_req_msg_size]}" + } } - ruby { - id => "ruby_zip_zeek_opcua_binary_create_session" - init => "@zeek_opcua_binary_create_session_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'opcua_link_id', 'application_uri', 'product_uri', 'encoding_mask', 'locale', 'text', 'application_type', 'gateway_server_uri', 'discovery_profile_uri', 'discovery_profile_link_id', 'server_uri', 'endpoint_url', 'session_name', 'client_nonce', 'client_cert_size', 'client_cert', 'req_session_timeout', 'max_res_msg_size', 'session_id_encoding_mask', 'session_id_namespace_idx', 'session_id_numeric', 'session_id_string', 'session_id_guid', 'session_id_opaque', 'auth_token_encoding_mask', 'auth_token_namespace_idx', 'auth_token_numeric', 'auth_token_string', 'auth_token_guid', 'auth_token_opaque', 'revised_session_timeout', 'server_nonce', 'server_cert_size', 'server_cert', 'endpoint_link_id', 'algorithm', 'signature', 'max_req_msg_size' ]" - code => "event.set('[zeek_cols]', @zeek_opcua_binary_create_session_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_opcua_binary_create_session" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_opcua_binary_create_session" + init => "@zeek_opcua_binary_create_session_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'opcua_link_id', 'application_uri', 'product_uri', 'encoding_mask', 'locale', 'text', 'application_type', 'gateway_server_uri', 'discovery_profile_uri', 'discovery_profile_link_id', 'server_uri', 'endpoint_url', 'session_name', 'client_nonce', 'client_cert_size', 'client_cert', 'req_session_timeout', 'max_res_msg_size', 'session_id_encoding_mask', 'session_id_namespace_idx', 'session_id_numeric', 'session_id_string', 'session_id_guid', 'session_id_opaque', 'auth_token_encoding_mask', 'auth_token_namespace_idx', 'auth_token_numeric', 'auth_token_string', 'auth_token_guid', 'auth_token_opaque', 'revised_session_timeout', 'server_nonce', 'server_cert_size', 'server_cert', 'endpoint_link_id', 'algorithm', 'signature', 'max_req_msg_size' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_create_session_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -4440,24 +4749,25 @@ filter { # opcua_binary_create_session_discovery.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - dissect { - id => "dissect_zeek_opcua_binary_create_session_discovery" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][discovery_profile_link_id]} %{[zeek_cols][discovery_profile_uri]} %{[zeek_cols][discovery_profile_url]}" - } - } - - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_opcua_binary_create_session_discovery" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_opcua_binary_create_session_discovery" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][discovery_profile_link_id]} %{[zeek_cols][discovery_profile_uri]} %{[zeek_cols][discovery_profile_url]}" + } } - ruby { - id => "ruby_zip_zeek_opcua_binary_create_session_discovery" - init => "@zeek_opcua_binary_create_session_discovery_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'discovery_profile_link_id', 'discovery_profile_uri', 'discovery_profile_url' ]" - code => "event.set('[zeek_cols]', @zeek_opcua_binary_create_session_discovery_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_opcua_binary_create_session_discovery" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_opcua_binary_create_session_discovery" + init => "@zeek_opcua_binary_create_session_discovery_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'discovery_profile_link_id', 'discovery_profile_uri', 'discovery_profile_url' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_create_session_discovery_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -4475,24 +4785,25 @@ filter { # opcua_binary_create_session_endpoints.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - dissect { - id => "dissect_zeek_opcua_binary_create_session_endpoints" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][endpoint_link_id]} %{[zeek_cols][endpoint_url]} %{[zeek_cols][application_uri]} %{[zeek_cols][product_uri]} %{[zeek_cols][encoding_mask]} %{[zeek_cols][locale]} %{[zeek_cols][text]} %{[zeek_cols][application_type]} %{[zeek_cols][gateway_server_uri]} %{[zeek_cols][discovery_profile_uri]} %{[zeek_cols][discovery_profile_link_id]} %{[zeek_cols][cert_size]} %{[zeek_cols][server_cert]} %{[zeek_cols][message_security_mode]} %{[zeek_cols][security_policy_uri]} %{[zeek_cols][user_token_link_id]} %{[zeek_cols][transport_profile_uri]} %{[zeek_cols][security_level]}" - } - } - - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_opcua_binary_create_session_endpoints" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_opcua_binary_create_session_endpoints" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][endpoint_link_id]} %{[zeek_cols][endpoint_url]} %{[zeek_cols][application_uri]} %{[zeek_cols][product_uri]} %{[zeek_cols][encoding_mask]} %{[zeek_cols][locale]} %{[zeek_cols][text]} %{[zeek_cols][application_type]} %{[zeek_cols][gateway_server_uri]} %{[zeek_cols][discovery_profile_uri]} %{[zeek_cols][discovery_profile_link_id]} %{[zeek_cols][cert_size]} %{[zeek_cols][server_cert]} %{[zeek_cols][message_security_mode]} %{[zeek_cols][security_policy_uri]} %{[zeek_cols][user_token_link_id]} %{[zeek_cols][transport_profile_uri]} %{[zeek_cols][security_level]}" + } } - ruby { - id => "ruby_zip_zeek_opcua_binary_create_session_endpoints" - init => "@zeek_opcua_binary_create_session_endpoints_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'endpoint_link_id', 'endpoint_url', 'application_uri', 'product_uri', 'encoding_mask', 'locale', 'text', 'application_type', 'gateway_server_uri', 'discovery_profile_uri', 'discovery_profile_link_id', 'cert_size', 'server_cert', 'message_security_mode', 'security_policy_uri', 'user_token_link_id', 'transport_profile_uri', 'security_level' ]" - code => "event.set('[zeek_cols]', @zeek_opcua_binary_create_session_endpoints_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_opcua_binary_create_session_endpoints" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_opcua_binary_create_session_endpoints" + init => "@zeek_opcua_binary_create_session_endpoints_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'endpoint_link_id', 'endpoint_url', 'application_uri', 'product_uri', 'encoding_mask', 'locale', 'text', 'application_type', 'gateway_server_uri', 'discovery_profile_uri', 'discovery_profile_link_id', 'cert_size', 'server_cert', 'message_security_mode', 'security_policy_uri', 'user_token_link_id', 'transport_profile_uri', 'security_level' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_create_session_endpoints_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -4510,24 +4821,25 @@ filter { # opcua_binary_create_session_user_token.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - dissect { - id => "dissect_zeek_opcua_binary_create_session_user_token" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][user_token_link_id]} %{[zeek_cols][user_token_policy_id]} %{[zeek_cols][user_token_type]} %{[zeek_cols][user_token_issued_type]} %{[zeek_cols][user_token_endpoint_url]} %{[zeek_cols][user_token_sec_policy_uri]}" - } - } - - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_opcua_binary_create_session_user_token" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_opcua_binary_create_session_user_token" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][user_token_link_id]} %{[zeek_cols][user_token_policy_id]} %{[zeek_cols][user_token_type]} %{[zeek_cols][user_token_issued_type]} %{[zeek_cols][user_token_endpoint_url]} %{[zeek_cols][user_token_sec_policy_uri]}" + } } - ruby { - id => "ruby_zip_zeek_opcua_binary_create_session_user_token" - init => "@zeek_opcua_binary_create_session_user_token_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'user_token_link_id', 'user_token_policy_id', 'user_token_type', 'user_token_issued_type', 'user_token_endpoint_url', 'user_token_sec_policy_uri' ]" - code => "event.set('[zeek_cols]', @zeek_opcua_binary_create_session_user_token_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_opcua_binary_create_session_user_token" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_opcua_binary_create_session_user_token" + init => "@zeek_opcua_binary_create_session_user_token_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'user_token_link_id', 'user_token_policy_id', 'user_token_type', 'user_token_issued_type', 'user_token_endpoint_url', 'user_token_sec_policy_uri' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_create_session_user_token_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -4545,24 +4857,25 @@ filter { # opcua_binary_create_subscription.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - dissect { - id => "dissect_zeek_opcua_binary_create_subscription" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][opcua_link_id]} %{[zeek_cols][requested_publishing_interval]} %{[zeek_cols][requested_lifetime_count]} %{[zeek_cols][requested_max_keep_alive_count]} %{[zeek_cols][max_notifications_per_publish]} %{[zeek_cols][publishing_enabled]} %{[zeek_cols][priority]} %{[zeek_cols][subscription_id]} %{[zeek_cols][revised_publishing_interval]} %{[zeek_cols][revised_lifetime_count]} %{[zeek_cols][revised_max_keep_alive_count]}" - } - } - - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_opcua_binary_create_subscription" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_opcua_binary_create_subscription" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][opcua_link_id]} %{[zeek_cols][requested_publishing_interval]} %{[zeek_cols][requested_lifetime_count]} %{[zeek_cols][requested_max_keep_alive_count]} %{[zeek_cols][max_notifications_per_publish]} %{[zeek_cols][publishing_enabled]} %{[zeek_cols][priority]} %{[zeek_cols][subscription_id]} %{[zeek_cols][revised_publishing_interval]} %{[zeek_cols][revised_lifetime_count]} %{[zeek_cols][revised_max_keep_alive_count]}" + } } - ruby { - id => "ruby_zip_zeek_opcua_binary_create_subscription" - init => "@zeek_opcua_binary_create_subscription_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'opcua_link_id', 'requested_publishing_interval', 'requested_lifetime_count', 'requested_max_keep_alive_count', 'max_notifications_per_publish', 'publishing_enabled', 'priority', 'subscription_id', 'revised_publishing_interval', 'revised_lifetime_count', 'revised_max_keep_alive_count' ]" - code => "event.set('[zeek_cols]', @zeek_opcua_binary_create_subscription_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_opcua_binary_create_subscription" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_opcua_binary_create_subscription" + init => "@zeek_opcua_binary_create_subscription_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'opcua_link_id', 'requested_publishing_interval', 'requested_lifetime_count', 'requested_max_keep_alive_count', 'max_notifications_per_publish', 'publishing_enabled', 'priority', 'subscription_id', 'revised_publishing_interval', 'revised_lifetime_count', 'revised_max_keep_alive_count' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_create_subscription_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -4580,24 +4893,25 @@ filter { # opcua_binary_data_change_filter.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - dissect { - id => "dissect_zeek_opcua_binary_data_change_filter" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][monitored_parameters_link_id]} %{[zeek_cols][trigger]} %{[zeek_cols][deadband_type]} %{[zeek_cols][deadband_value]}" - } - } - - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_opcua_binary_data_change_filter" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_opcua_binary_data_change_filter" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][monitored_parameters_link_id]} %{[zeek_cols][trigger]} %{[zeek_cols][deadband_type]} %{[zeek_cols][deadband_value]}" + } } - ruby { - id => "ruby_zip_zeek_opcua_binary_data_change_filter" - init => "@zeek_opcua_binary_data_change_filter_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'monitored_parameters_link_id', 'trigger', 'deadband_type', 'deadband_value' ]" - code => "event.set('[zeek_cols]', @zeek_opcua_binary_data_change_filter_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_opcua_binary_data_change_filter" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_opcua_binary_data_change_filter" + init => "@zeek_opcua_binary_data_change_filter_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'monitored_parameters_link_id', 'trigger', 'deadband_type', 'deadband_value' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_data_change_filter_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -4615,24 +4929,25 @@ filter { # opcua_binary_diag_info_detail.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - dissect { - id => "dissect_zeek_opcua_binary_diag_info_detail" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][diag_info_link_id]} %{[zeek_cols][root_object_id]} %{[zeek_cols][source]} %{[zeek_cols][source_str]} %{[zeek_cols][inner_diag_level]} %{[zeek_cols][has_symbolic_id]} %{[zeek_cols][symbolic_id]} %{[zeek_cols][symbolic_id_str]} %{[zeek_cols][has_namespace_uri]} %{[zeek_cols][namespace_uri]} %{[zeek_cols][namespace_uri_str]} %{[zeek_cols][has_locale]} %{[zeek_cols][locale]} %{[zeek_cols][locale_str]} %{[zeek_cols][has_locale_txt]} %{[zeek_cols][locale_txt]} %{[zeek_cols][locale_txt_str]} %{[zeek_cols][has_addl_info]} %{[zeek_cols][addl_info]} %{[zeek_cols][has_inner_stat_code]} %{[zeek_cols][inner_stat_code]} %{[zeek_cols][has_inner_diag_info]}" - } - } - - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_opcua_binary_diag_info_detail" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_opcua_binary_diag_info_detail" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][diag_info_link_id]} %{[zeek_cols][root_object_id]} %{[zeek_cols][source]} %{[zeek_cols][source_str]} %{[zeek_cols][inner_diag_level]} %{[zeek_cols][has_symbolic_id]} %{[zeek_cols][symbolic_id]} %{[zeek_cols][symbolic_id_str]} %{[zeek_cols][has_namespace_uri]} %{[zeek_cols][namespace_uri]} %{[zeek_cols][namespace_uri_str]} %{[zeek_cols][has_locale]} %{[zeek_cols][locale]} %{[zeek_cols][locale_str]} %{[zeek_cols][has_locale_txt]} %{[zeek_cols][locale_txt]} %{[zeek_cols][locale_txt_str]} %{[zeek_cols][has_addl_info]} %{[zeek_cols][addl_info]} %{[zeek_cols][has_inner_stat_code]} %{[zeek_cols][inner_stat_code]} %{[zeek_cols][has_inner_diag_info]}" + } } - ruby { - id => "ruby_zip_zeek_opcua_binary_diag_info_detail" - init => "@zeek_opcua_binary_diag_info_detail_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'diag_info_link_id', 'root_object_id', 'source', 'source_str', 'inner_diag_level', 'has_symbolic_id', 'symbolic_id', 'symbolic_id_str', 'has_namespace_uri', 'namespace_uri', 'namespace_uri_str', 'has_locale', 'locale', 'locale_str', 'has_locale_txt', 'locale_txt', 'locale_txt_str', 'has_addl_info', 'addl_info', 'has_inner_stat_code', 'inner_stat_code', 'has_inner_diag_info' ]" - code => "event.set('[zeek_cols]', @zeek_opcua_binary_diag_info_detail_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_opcua_binary_diag_info_detail" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_opcua_binary_diag_info_detail" + init => "@zeek_opcua_binary_diag_info_detail_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'diag_info_link_id', 'root_object_id', 'source', 'source_str', 'inner_diag_level', 'has_symbolic_id', 'symbolic_id', 'symbolic_id_str', 'has_namespace_uri', 'namespace_uri', 'namespace_uri_str', 'has_locale', 'locale', 'locale_str', 'has_locale_txt', 'locale_txt', 'locale_txt_str', 'has_addl_info', 'addl_info', 'has_inner_stat_code', 'inner_stat_code', 'has_inner_diag_info' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_diag_info_detail_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -4650,24 +4965,25 @@ filter { # opcua_binary_event_filter_element_operand.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - dissect { - id => "dissect_zeek_opcua_binary_event_filter_element_operand" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][content_filter_filter_operand_link_id]} %{[zeek_cols][element_index]}" - } - } - - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_opcua_binary_event_filter_element_operand" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_opcua_binary_event_filter_element_operand" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][content_filter_filter_operand_link_id]} %{[zeek_cols][element_index]}" + } } - ruby { - id => "ruby_zip_zeek_opcua_binary_event_filter_element_operand" - init => "@zeek_opcua_binary_event_filter_element_operand_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'content_filter_filter_operand_link_id', 'element_index' ]" - code => "event.set('[zeek_cols]', @zeek_opcua_binary_event_filter_element_operand_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_opcua_binary_event_filter_element_operand" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_opcua_binary_event_filter_element_operand" + init => "@zeek_opcua_binary_event_filter_element_operand_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'content_filter_filter_operand_link_id', 'element_index' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_event_filter_element_operand_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -4685,24 +5001,25 @@ filter { # opcua_binary_event_filter.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - dissect { - id => "dissect_zeek_opcua_binary_event_filter" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][monitored_parameters_link_id]} %{[zeek_cols][select_clause_link_id]} %{[zeek_cols][where_clause_content_filter_link_id]}" - } - } - - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_opcua_binary_event_filter" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_opcua_binary_event_filter" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][monitored_parameters_link_id]} %{[zeek_cols][select_clause_link_id]} %{[zeek_cols][where_clause_content_filter_link_id]}" + } } - ruby { - id => "ruby_zip_zeek_opcua_binary_event_filter" - init => "@zeek_opcua_binary_event_filter_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'monitored_parameters_link_id', 'select_clause_link_id', 'where_clause_content_filter_link_id' ]" - code => "event.set('[zeek_cols]', @zeek_opcua_binary_event_filter_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_opcua_binary_event_filter" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_opcua_binary_event_filter" + init => "@zeek_opcua_binary_event_filter_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'monitored_parameters_link_id', 'select_clause_link_id', 'where_clause_content_filter_link_id' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_event_filter_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -4720,24 +5037,25 @@ filter { # opcua_binary_get_endpoints.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - dissect { - id => "dissect_zeek_opcua_binary_get_endpoints" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][opcua_link_id]} %{[zeek_cols][endpoint_url]} %{[zeek_cols][locale_link_id]} %{[zeek_cols][profile_uri_link_id]} %{[zeek_cols][endpoint_description_link_id]}" - } - } - - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_opcua_binary_get_endpoints" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_opcua_binary_get_endpoints" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][opcua_link_id]} %{[zeek_cols][endpoint_url]} %{[zeek_cols][locale_link_id]} %{[zeek_cols][profile_uri_link_id]} %{[zeek_cols][endpoint_description_link_id]}" + } } - ruby { - id => "ruby_zip_zeek_opcua_binary_get_endpoints" - init => "@zeek_opcua_binary_get_endpoints_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'opcua_link_id', 'endpoint_url', 'locale_link_id', 'profile_uri_link_id', 'endpoint_description_link_id' ]" - code => "event.set('[zeek_cols]', @zeek_opcua_binary_get_endpoints_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_opcua_binary_get_endpoints" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_opcua_binary_get_endpoints" + init => "@zeek_opcua_binary_get_endpoints_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'opcua_link_id', 'endpoint_url', 'locale_link_id', 'profile_uri_link_id', 'endpoint_description_link_id' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_get_endpoints_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -4755,24 +5073,25 @@ filter { # opcua_binary_get_endpoints_description.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - dissect { - id => "dissect_zeek_opcua_binary_get_endpoints_description" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][endpoint_description_link_id]} %{[zeek_cols][endpoint_uri]} %{[zeek_cols][application_uri]} %{[zeek_cols][product_uri]} %{[zeek_cols][encoding_mask]} %{[zeek_cols][locale]} %{[zeek_cols][text]} %{[zeek_cols][application_type]} %{[zeek_cols][gateway_server_uri]} %{[zeek_cols][discovery_profile_uri]} %{[zeek_cols][discovery_profile_link_id]} %{[zeek_cols][cert_size]} %{[zeek_cols][server_cert]} %{[zeek_cols][message_security_mode]} %{[zeek_cols][security_policy_uri]} %{[zeek_cols][user_token_link_id]} %{[zeek_cols][transport_profile_uri]} %{[zeek_cols][security_level]}" - } - } - - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_opcua_binary_get_endpoints_description" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_opcua_binary_get_endpoints_description" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][endpoint_description_link_id]} %{[zeek_cols][endpoint_uri]} %{[zeek_cols][application_uri]} %{[zeek_cols][product_uri]} %{[zeek_cols][encoding_mask]} %{[zeek_cols][locale]} %{[zeek_cols][text]} %{[zeek_cols][application_type]} %{[zeek_cols][gateway_server_uri]} %{[zeek_cols][discovery_profile_uri]} %{[zeek_cols][discovery_profile_link_id]} %{[zeek_cols][cert_size]} %{[zeek_cols][server_cert]} %{[zeek_cols][message_security_mode]} %{[zeek_cols][security_policy_uri]} %{[zeek_cols][user_token_link_id]} %{[zeek_cols][transport_profile_uri]} %{[zeek_cols][security_level]}" + } } - ruby { - id => "ruby_zip_zeek_opcua_binary_get_endpoints_description" - init => "@zeek_opcua_binary_get_endpoints_description_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'endpoint_description_link_id', 'endpoint_uri', 'application_uri', 'product_uri', 'encoding_mask', 'locale', 'text', 'application_type', 'gateway_server_uri', 'discovery_profile_uri', 'discovery_profile_link_id', 'cert_size', 'server_cert', 'message_security_mode', 'security_policy_uri', 'user_token_link_id', 'transport_profile_uri', 'security_level' ]" - code => "event.set('[zeek_cols]', @zeek_opcua_binary_get_endpoints_description_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_opcua_binary_get_endpoints_description" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_opcua_binary_get_endpoints_description" + init => "@zeek_opcua_binary_get_endpoints_description_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'endpoint_description_link_id', 'endpoint_uri', 'application_uri', 'product_uri', 'encoding_mask', 'locale', 'text', 'application_type', 'gateway_server_uri', 'discovery_profile_uri', 'discovery_profile_link_id', 'cert_size', 'server_cert', 'message_security_mode', 'security_policy_uri', 'user_token_link_id', 'transport_profile_uri', 'security_level' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_get_endpoints_description_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -4790,24 +5109,25 @@ filter { # opcua_binary_get_endpoints_discovery.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - dissect { - id => "dissect_zeek_opcua_binary_get_endpoints_discovery" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][discovery_profile_link_id]} %{[zeek_cols][discovery_profile_url]}" - } - } - - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_opcua_binary_get_endpoints_discovery" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_opcua_binary_get_endpoints_discovery" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][discovery_profile_link_id]} %{[zeek_cols][discovery_profile_url]}" + } } - ruby { - id => "ruby_zip_zeek_opcua_binary_get_endpoints_discovery" - init => "@zeek_opcua_binary_get_endpoints_discovery_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'discovery_profile_link_id', 'discovery_profile_url' ]" - code => "event.set('[zeek_cols]', @zeek_opcua_binary_get_endpoints_discovery_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_opcua_binary_get_endpoints_discovery" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_opcua_binary_get_endpoints_discovery" + init => "@zeek_opcua_binary_get_endpoints_discovery_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'discovery_profile_link_id', 'discovery_profile_url' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_get_endpoints_discovery_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -4825,24 +5145,25 @@ filter { # opcua_binary_get_endpoints_locale_id.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - dissect { - id => "dissect_zeek_opcua_binary_get_endpoints_locale_id" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][locale_link_id]} %{[zeek_cols][locale_id]}" - } - } - - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_opcua_binary_get_endpoints_locale_id" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_opcua_binary_get_endpoints_locale_id" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][locale_link_id]} %{[zeek_cols][locale_id]}" + } } - ruby { - id => "ruby_zip_zeek_opcua_binary_get_endpoints_locale_id" - init => "@zeek_opcua_binary_get_endpoints_locale_id_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'locale_link_id', 'locale_id' ]" - code => "event.set('[zeek_cols]', @zeek_opcua_binary_get_endpoints_locale_id_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_opcua_binary_get_endpoints_locale_id" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_opcua_binary_get_endpoints_locale_id" + init => "@zeek_opcua_binary_get_endpoints_locale_id_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'locale_link_id', 'locale_id' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_get_endpoints_locale_id_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -4860,24 +5181,25 @@ filter { # opcua_binary_get_endpoints_profile_uri.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - dissect { - id => "dissect_zeek_opcua_binary_get_endpoints_profile_uri" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][profile_uri_link_id]} %{[zeek_cols][profile_uri]}" - } - } - - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_opcua_binary_get_endpoints_profile_uri" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_opcua_binary_get_endpoints_profile_uri" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][profile_uri_link_id]} %{[zeek_cols][profile_uri]}" + } } - ruby { - id => "ruby_zip_zeek_opcua_binary_get_endpoints_profile_uri" - init => "@zeek_opcua_binary_get_endpoints_profile_uri_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'profile_uri_link_id', 'profile_uri' ]" - code => "event.set('[zeek_cols]', @zeek_opcua_binary_get_endpoints_profile_uri_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_opcua_binary_get_endpoints_profile_uri" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_opcua_binary_get_endpoints_profile_uri" + init => "@zeek_opcua_binary_get_endpoints_profile_uri_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'profile_uri_link_id', 'profile_uri' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_get_endpoints_profile_uri_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -4895,24 +5217,25 @@ filter { # opcua_binary_get_endpoints_user_token.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - dissect { - id => "dissect_zeek_opcua_binary_get_endpoints_user_token" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][user_token_link_id]} %{[zeek_cols][user_token_policy_id]} %{[zeek_cols][user_token_type]} %{[zeek_cols][user_token_issued_type]} %{[zeek_cols][user_token_endpoint_url]} %{[zeek_cols][user_token_sec_policy_uri]}" - } - } - - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_opcua_binary_get_endpoints_user_token" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_opcua_binary_get_endpoints_user_token" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][user_token_link_id]} %{[zeek_cols][user_token_policy_id]} %{[zeek_cols][user_token_type]} %{[zeek_cols][user_token_issued_type]} %{[zeek_cols][user_token_endpoint_url]} %{[zeek_cols][user_token_sec_policy_uri]}" + } } - ruby { - id => "ruby_zip_zeek_opcua_binary_get_endpoints_user_token" - init => "@zeek_opcua_binary_get_endpoints_user_token_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'user_token_link_id', 'user_token_policy_id', 'user_token_type', 'user_token_issued_type', 'user_token_endpoint_url', 'user_token_sec_policy_uri' ]" - code => "event.set('[zeek_cols]', @zeek_opcua_binary_get_endpoints_user_token_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_opcua_binary_get_endpoints_user_token" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_opcua_binary_get_endpoints_user_token" + init => "@zeek_opcua_binary_get_endpoints_user_token_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'user_token_link_id', 'user_token_policy_id', 'user_token_type', 'user_token_issued_type', 'user_token_endpoint_url', 'user_token_sec_policy_uri' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_get_endpoints_user_token_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -4930,24 +5253,25 @@ filter { # opcua_binary_event_filter_literal_operand.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - dissect { - id => "dissect_zeek_opcua_binary_event_filter_literal_operand" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][content_filter_filter_operand_link_id]} %{[zeek_cols][literal_operand_variant_link]}" - } - } - - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_opcua_binary_event_filter_literal_operand" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_opcua_binary_event_filter_literal_operand" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][content_filter_filter_operand_link_id]} %{[zeek_cols][literal_operand_variant_link]}" + } } - ruby { - id => "ruby_zip_zeek_opcua_binary_event_filter_literal_operand" - init => "@zeek_opcua_binary_event_filter_literal_operand_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'content_filter_filter_operand_link_id', 'literal_operand_variant_link' ]" - code => "event.set('[zeek_cols]', @zeek_opcua_binary_event_filter_literal_operand_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_opcua_binary_event_filter_literal_operand" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_opcua_binary_event_filter_literal_operand" + init => "@zeek_opcua_binary_event_filter_literal_operand_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'content_filter_filter_operand_link_id', 'literal_operand_variant_link' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_event_filter_literal_operand_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -4965,24 +5289,25 @@ filter { # opcua_binary_opensecure_channel.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - dissect { - id => "dissect_zeek_opcua_binary_opensecure_channel" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][opcua_link_id]} %{[zeek_cols][client_proto_ver]} %{[zeek_cols][sec_token_request_type]} %{[zeek_cols][message_security_mode]} %{[zeek_cols][client_nonce]} %{[zeek_cols][req_lifetime]} %{[zeek_cols][server_proto_ver]} %{[zeek_cols][sec_token_sec_channel_id]} %{[zeek_cols][sec_token_id]} %{[zeek_cols][sec_token_created_at]} %{[zeek_cols][sec_token_revised_time]} %{[zeek_cols][server_nonce]}" - } - } - - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_opcua_binary_opensecure_channel" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_opcua_binary_opensecure_channel" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][opcua_link_id]} %{[zeek_cols][client_proto_ver]} %{[zeek_cols][sec_token_request_type]} %{[zeek_cols][message_security_mode]} %{[zeek_cols][client_nonce]} %{[zeek_cols][req_lifetime]} %{[zeek_cols][server_proto_ver]} %{[zeek_cols][sec_token_sec_channel_id]} %{[zeek_cols][sec_token_id]} %{[zeek_cols][sec_token_created_at]} %{[zeek_cols][sec_token_revised_time]} %{[zeek_cols][server_nonce]}" + } } - ruby { - id => "ruby_zip_zeek_opcua_binary_opensecure_channel" - init => "@zeek_opcua_binary_opensecure_channel_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'opcua_link_id', 'client_proto_ver', 'sec_token_request_type', 'message_security_mode', 'client_nonce', 'req_lifetime', 'server_proto_ver', 'sec_token_sec_channel_id', 'sec_token_id', 'sec_token_created_at', 'sec_token_revised_time', 'server_nonce' ]" - code => "event.set('[zeek_cols]', @zeek_opcua_binary_opensecure_channel_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_opcua_binary_opensecure_channel" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_opcua_binary_opensecure_channel" + init => "@zeek_opcua_binary_opensecure_channel_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'opcua_link_id', 'client_proto_ver', 'sec_token_request_type', 'message_security_mode', 'client_nonce', 'req_lifetime', 'server_proto_ver', 'sec_token_sec_channel_id', 'sec_token_id', 'sec_token_created_at', 'sec_token_revised_time', 'server_nonce' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_opensecure_channel_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -5000,24 +5325,25 @@ filter { # opcua_binary_read.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - dissect { - id => "dissect_zeek_opcua_binary_read" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][opcua_link_id]} %{[zeek_cols][max_age]} %{[zeek_cols][timestamps_to_return]} %{[zeek_cols][timestamps_to_return_str]} %{[zeek_cols][nodes_to_read_link_id]} %{[zeek_cols][read_results_link_id]} %{[zeek_cols][diag_info_link_id]}" - } - } - - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_opcua_binary_read" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_opcua_binary_read" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][opcua_link_id]} %{[zeek_cols][max_age]} %{[zeek_cols][timestamps_to_return]} %{[zeek_cols][timestamps_to_return_str]} %{[zeek_cols][nodes_to_read_link_id]} %{[zeek_cols][read_results_link_id]} %{[zeek_cols][diag_info_link_id]}" + } } - ruby { - id => "ruby_zip_zeek_opcua_binary_read" - init => "@zeek_opcua_binary_read_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'opcua_link_id', 'max_age', 'timestamps_to_return', 'timestamps_to_return_str', 'nodes_to_read_link_id', 'read_results_link_id', 'diag_info_link_id' ]" - code => "event.set('[zeek_cols]', @zeek_opcua_binary_read_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_opcua_binary_read" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_opcua_binary_read" + init => "@zeek_opcua_binary_read_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'opcua_link_id', 'max_age', 'timestamps_to_return', 'timestamps_to_return_str', 'nodes_to_read_link_id', 'read_results_link_id', 'diag_info_link_id' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_read_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -5035,24 +5361,25 @@ filter { # opcua_binary_read_nodes_to_read.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - dissect { - id => "dissect_zeek_opcua_binary_read_nodes_to_read" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][nodes_to_read_link_id]} %{[zeek_cols][node_id_encoding_mask]} %{[zeek_cols][node_id_namespace_idx]} %{[zeek_cols][node_id_numeric]} %{[zeek_cols][node_id_string]} %{[zeek_cols][node_id_guid]} %{[zeek_cols][node_id_opaque]} %{[zeek_cols][attribute_id]} %{[zeek_cols][attribute_id_str]} %{[zeek_cols][index_range]} %{[zeek_cols][data_encoding_name_idx]} %{[zeek_cols][data_encoding_name]}" - } - } - - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_opcua_binary_read_nodes_to_read" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_opcua_binary_read_nodes_to_read" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][nodes_to_read_link_id]} %{[zeek_cols][node_id_encoding_mask]} %{[zeek_cols][node_id_namespace_idx]} %{[zeek_cols][node_id_numeric]} %{[zeek_cols][node_id_string]} %{[zeek_cols][node_id_guid]} %{[zeek_cols][node_id_opaque]} %{[zeek_cols][attribute_id]} %{[zeek_cols][attribute_id_str]} %{[zeek_cols][index_range]} %{[zeek_cols][data_encoding_name_idx]} %{[zeek_cols][data_encoding_name]}" + } } - ruby { - id => "ruby_zip_zeek_opcua_binary_read_nodes_to_read" - init => "@zeek_opcua_binary_read_nodes_to_read_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'nodes_to_read_link_id', 'node_id_encoding_mask', 'node_id_namespace_idx', 'node_id_numeric', 'node_id_string', 'node_id_guid', 'node_id_opaque', 'attribute_id', 'attribute_id_str', 'index_range', 'data_encoding_name_idx', 'data_encoding_name' ]" - code => "event.set('[zeek_cols]', @zeek_opcua_binary_read_nodes_to_read_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_opcua_binary_read_nodes_to_read" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_opcua_binary_read_nodes_to_read" + init => "@zeek_opcua_binary_read_nodes_to_read_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'nodes_to_read_link_id', 'node_id_encoding_mask', 'node_id_namespace_idx', 'node_id_numeric', 'node_id_string', 'node_id_guid', 'node_id_opaque', 'attribute_id', 'attribute_id_str', 'index_range', 'data_encoding_name_idx', 'data_encoding_name' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_read_nodes_to_read_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -5070,24 +5397,25 @@ filter { # opcua_binary_read_results.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - dissect { - id => "dissect_zeek_opcua_binary_read_results" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][results_link_id]} %{[zeek_cols][level]} %{[zeek_cols][data_value_encoding_mask]} %{[zeek_cols][status_code_link_id]} %{[zeek_cols][source_timestamp]} %{[zeek_cols][source_pico_sec]} %{[zeek_cols][server_timestamp]} %{[zeek_cols][server_pico_sec]} %{[zeek_cols][read_results_variant_metadata_link_id]}" - } - } - - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_opcua_binary_read_results" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_opcua_binary_read_results" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][results_link_id]} %{[zeek_cols][level]} %{[zeek_cols][data_value_encoding_mask]} %{[zeek_cols][status_code_link_id]} %{[zeek_cols][source_timestamp]} %{[zeek_cols][source_pico_sec]} %{[zeek_cols][server_timestamp]} %{[zeek_cols][server_pico_sec]} %{[zeek_cols][read_results_variant_metadata_link_id]}" + } } - ruby { - id => "ruby_zip_zeek_opcua_binary_read_results" - init => "@zeek_opcua_binary_read_results_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'results_link_id', 'level', 'data_value_encoding_mask', 'status_code_link_id', 'source_timestamp', 'source_pico_sec', 'server_timestamp', 'server_pico_sec', 'read_results_variant_metadata_link_id' ]" - code => "event.set('[zeek_cols]', @zeek_opcua_binary_read_results_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_opcua_binary_read_results" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_opcua_binary_read_results" + init => "@zeek_opcua_binary_read_results_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'results_link_id', 'level', 'data_value_encoding_mask', 'status_code_link_id', 'source_timestamp', 'source_pico_sec', 'server_timestamp', 'server_pico_sec', 'read_results_variant_metadata_link_id' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_read_results_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -5105,24 +5433,25 @@ filter { # opcua_binary_event_filter_select_clause.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - dissect { - id => "dissect_zeek_opcua_binary_event_filter_select_clause" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][select_clause_link_id]} %{[zeek_cols][type_id_encoding_mask]} %{[zeek_cols][type_id_namespace_idx]} %{[zeek_cols][type_id_numeric]} %{[zeek_cols][type_id_string]} %{[zeek_cols][type_id_guid]} %{[zeek_cols][type_id_opaque]} %{[zeek_cols][simple_attribute_operand_browse_path_link_id]} %{[zeek_cols][attribute_id]} %{[zeek_cols][index_range]} %{[zeek_cols][select_clause_status_code_link_id]} %{[zeek_cols][select_clause_diagnostic_info_link_id]}" - } - } - - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_opcua_binary_event_filter_select_clause" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_opcua_binary_event_filter_select_clause" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][select_clause_link_id]} %{[zeek_cols][type_id_encoding_mask]} %{[zeek_cols][type_id_namespace_idx]} %{[zeek_cols][type_id_numeric]} %{[zeek_cols][type_id_string]} %{[zeek_cols][type_id_guid]} %{[zeek_cols][type_id_opaque]} %{[zeek_cols][simple_attribute_operand_browse_path_link_id]} %{[zeek_cols][attribute_id]} %{[zeek_cols][index_range]} %{[zeek_cols][select_clause_status_code_link_id]} %{[zeek_cols][select_clause_diagnostic_info_link_id]}" + } } - ruby { - id => "ruby_zip_zeek_opcua_binary_event_filter_select_clause" - init => "@zeek_opcua_binary_event_filter_select_clause_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'select_clause_link_id', 'type_id_encoding_mask', 'type_id_namespace_idx', 'type_id_numeric', 'type_id_string', 'type_id_guid', 'type_id_opaque', 'simple_attribute_operand_browse_path_link_id', 'attribute_id', 'index_range', 'select_clause_status_code_link_id', 'select_clause_diagnostic_info_link_id' ]" - code => "event.set('[zeek_cols]', @zeek_opcua_binary_event_filter_select_clause_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_opcua_binary_event_filter_select_clause" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_opcua_binary_event_filter_select_clause" + init => "@zeek_opcua_binary_event_filter_select_clause_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'select_clause_link_id', 'type_id_encoding_mask', 'type_id_namespace_idx', 'type_id_numeric', 'type_id_string', 'type_id_guid', 'type_id_opaque', 'simple_attribute_operand_browse_path_link_id', 'attribute_id', 'index_range', 'select_clause_status_code_link_id', 'select_clause_diagnostic_info_link_id' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_event_filter_select_clause_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -5140,24 +5469,25 @@ filter { # opcua_binary_event_filter_simple_attribute_operand.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - dissect { - id => "dissect_zeek_opcua_binary_event_filter_simple_attribute_operand" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][content_filter_filter_operand_link_id]} %{[zeek_cols][type_id_encoding_mask]} %{[zeek_cols][type_id_namespace_idx]} %{[zeek_cols][type_id_numeric]} %{[zeek_cols][type_id_string]} %{[zeek_cols][type_id_guid]} %{[zeek_cols][type_id_opaque]} %{[zeek_cols][simple_attribute_operand_browse_path_link_id]} %{[zeek_cols][attribute_id]} %{[zeek_cols][index_range]}" - } - } - - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_opcua_binary_event_filter_simple_attribute_operand" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_opcua_binary_event_filter_simple_attribute_operand" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][content_filter_filter_operand_link_id]} %{[zeek_cols][type_id_encoding_mask]} %{[zeek_cols][type_id_namespace_idx]} %{[zeek_cols][type_id_numeric]} %{[zeek_cols][type_id_string]} %{[zeek_cols][type_id_guid]} %{[zeek_cols][type_id_opaque]} %{[zeek_cols][simple_attribute_operand_browse_path_link_id]} %{[zeek_cols][attribute_id]} %{[zeek_cols][index_range]}" + } } - ruby { - id => "ruby_zip_zeek_opcua_binary_event_filter_simple_attribute_operand" - init => "@zeek_opcua_binary_event_filter_simple_attribute_operand_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'content_filter_filter_operand_link_id', 'type_id_encoding_mask', 'type_id_namespace_idx', 'type_id_numeric', 'type_id_string', 'type_id_guid', 'type_id_opaque', 'simple_attribute_operand_browse_path_link_id', 'attribute_id', 'index_range' ]" - code => "event.set('[zeek_cols]', @zeek_opcua_binary_event_filter_simple_attribute_operand_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_opcua_binary_event_filter_simple_attribute_operand" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_opcua_binary_event_filter_simple_attribute_operand" + init => "@zeek_opcua_binary_event_filter_simple_attribute_operand_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'content_filter_filter_operand_link_id', 'type_id_encoding_mask', 'type_id_namespace_idx', 'type_id_numeric', 'type_id_string', 'type_id_guid', 'type_id_opaque', 'simple_attribute_operand_browse_path_link_id', 'attribute_id', 'index_range' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_event_filter_simple_attribute_operand_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -5175,24 +5505,25 @@ filter { # opcua_binary_event_filter_simple_attribute_operand_browse_paths.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - dissect { - id => "dissect_zeek_opcua_binary_event_filter_simple_attribute_operand_browse_paths" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][simple_attribute_operand_browse_path_link_id]} %{[zeek_cols][browse_path_src]} %{[zeek_cols][namespace_index]} %{[zeek_cols][name]}" - } - } - - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_opcua_binary_event_filter_simple_attribute_operand_browse_paths" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_opcua_binary_event_filter_simple_attribute_operand_browse_paths" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][simple_attribute_operand_browse_path_link_id]} %{[zeek_cols][browse_path_src]} %{[zeek_cols][namespace_index]} %{[zeek_cols][name]}" + } } - ruby { - id => "ruby_zip_zeek_opcua_binary_event_filter_simple_attribute_operand_browse_paths" - init => "@zeek_opcua_binary_event_filter_simple_attribute_operand_browse_paths_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'simple_attribute_operand_browse_path_link_id', 'browse_path_src', 'namespace_index', 'name' ]" - code => "event.set('[zeek_cols]', @zeek_opcua_binary_event_filter_simple_attribute_operand_browse_paths_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_opcua_binary_event_filter_simple_attribute_operand_browse_paths" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_opcua_binary_event_filter_simple_attribute_operand_browse_paths" + init => "@zeek_opcua_binary_event_filter_simple_attribute_operand_browse_paths_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'simple_attribute_operand_browse_path_link_id', 'browse_path_src', 'namespace_index', 'name' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_event_filter_simple_attribute_operand_browse_paths_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -5210,24 +5541,25 @@ filter { # opcua_binary_status_code_detail.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - dissect { - id => "dissect_zeek_opcua_binary_status_code_detail" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][status_code_link_id]} %{[zeek_cols][source]} %{[zeek_cols][source_str]} %{[zeek_cols][source_level]} %{[zeek_cols][status_code]} %{[zeek_cols][severity]} %{[zeek_cols][severity_str]} %{[zeek_cols][sub_code]} %{[zeek_cols][sub_code_str]} %{[zeek_cols][structure_changed]} %{[zeek_cols][semantics_changed]} %{[zeek_cols][info_type]} %{[zeek_cols][info_type_str]} %{[zeek_cols][limit_bits]} %{[zeek_cols][limit_bits_str]} %{[zeek_cols][overflow]} %{[zeek_cols][historian_bits]} %{[zeek_cols][historian_bits_str]} %{[zeek_cols][historianpartial]} %{[zeek_cols][historianextradata]} %{[zeek_cols][historianmultivalue]}" - } - } - - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_opcua_binary_status_code_detail" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_opcua_binary_status_code_detail" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][status_code_link_id]} %{[zeek_cols][source]} %{[zeek_cols][source_str]} %{[zeek_cols][source_level]} %{[zeek_cols][status_code]} %{[zeek_cols][severity]} %{[zeek_cols][severity_str]} %{[zeek_cols][sub_code]} %{[zeek_cols][sub_code_str]} %{[zeek_cols][structure_changed]} %{[zeek_cols][semantics_changed]} %{[zeek_cols][info_type]} %{[zeek_cols][info_type_str]} %{[zeek_cols][limit_bits]} %{[zeek_cols][limit_bits_str]} %{[zeek_cols][overflow]} %{[zeek_cols][historian_bits]} %{[zeek_cols][historian_bits_str]} %{[zeek_cols][historianpartial]} %{[zeek_cols][historianextradata]} %{[zeek_cols][historianmultivalue]}" + } } - ruby { - id => "ruby_zip_zeek_opcua_binary_status_code_detail" - init => "@zeek_opcua_binary_status_code_detail_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'status_code_link_id', 'source', 'source_str', 'source_level', 'status_code', 'severity', 'severity_str', 'sub_code', 'sub_code_str', 'structure_changed', 'semantics_changed', 'info_type', 'info_type_str', 'limit_bits', 'limit_bits_str', 'overflow', 'historian_bits', 'historian_bits_str', 'historianpartial', 'historianextradata', 'historianmultivalue' ]" - code => "event.set('[zeek_cols]', @zeek_opcua_binary_status_code_detail_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_opcua_binary_status_code_detail" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_opcua_binary_status_code_detail" + init => "@zeek_opcua_binary_status_code_detail_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'status_code_link_id', 'source', 'source_str', 'source_level', 'status_code', 'severity', 'severity_str', 'sub_code', 'sub_code_str', 'structure_changed', 'semantics_changed', 'info_type', 'info_type_str', 'limit_bits', 'limit_bits_str', 'overflow', 'historian_bits', 'historian_bits_str', 'historianpartial', 'historianextradata', 'historianmultivalue' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_status_code_detail_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -5245,24 +5577,25 @@ filter { # opcua_binary_variant_array_dims.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - dissect { - id => "dissect_zeek_opcua_binary_variant_array_dims" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][array_dim_link_id]} %{[zeek_cols][dimension]}" - } - } - - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_opcua_binary_variant_array_dims" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_opcua_binary_variant_array_dims" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][array_dim_link_id]} %{[zeek_cols][dimension]}" + } } - ruby { - id => "ruby_zip_zeek_opcua_binary_variant_array_dims" - init => "@zeek_opcua_binary_variant_array_dims_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'array_dim_link_id', 'dimension' ]" - code => "event.set('[zeek_cols]', @zeek_opcua_binary_variant_array_dims_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_opcua_binary_variant_array_dims" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_opcua_binary_variant_array_dims" + init => "@zeek_opcua_binary_variant_array_dims_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'array_dim_link_id', 'dimension' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_variant_array_dims_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -5280,24 +5613,25 @@ filter { # opcua_binary_variant_data.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - dissect { - id => "dissect_zeek_opcua_binary_variant_data" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][variant_data_link_id]} %{[zeek_cols][variant_data_value_signed_numeric]} %{[zeek_cols][variant_data_value_unsigned_numeric]} %{[zeek_cols][variant_data_value_string]} %{[zeek_cols][variant_data_node_id_encoding_mask]} %{[zeek_cols][variant_data_node_id_namespace_idx]} %{[zeek_cols][variant_data_node_id_numeric]} %{[zeek_cols][variant_data_node_id_string]} %{[zeek_cols][variant_data_node_id_guid]} %{[zeek_cols][variant_data_node_id_opaque]} %{[zeek_cols][variant_data_node_id_namespace_uri]} %{[zeek_cols][variant_data_node_id_server_idx]} %{[zeek_cols][variant_data_value_time]} %{[zeek_cols][variant_data_encoding_name_idx]} %{[zeek_cols][variant_data_encoding_name]} %{[zeek_cols][variant_data_mask]} %{[zeek_cols][variant_data_locale]} %{[zeek_cols][variant_data_text]} %{[zeek_cols][variant_data_value_decimal]} %{[zeek_cols][variant_data_status_code_link_id]} %{[zeek_cols][variant_data_diag_info_link_id]} %{[zeek_cols][variant_data_ext_obj_link_id]} %{[zeek_cols][variant_metadata_data_link_id]} %{[zeek_cols][variant_data_value_link_id]}" - } - } - - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_opcua_binary_variant_data" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_opcua_binary_variant_data" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][variant_data_link_id]} %{[zeek_cols][variant_data_value_signed_numeric]} %{[zeek_cols][variant_data_value_unsigned_numeric]} %{[zeek_cols][variant_data_value_string]} %{[zeek_cols][variant_data_node_id_encoding_mask]} %{[zeek_cols][variant_data_node_id_namespace_idx]} %{[zeek_cols][variant_data_node_id_numeric]} %{[zeek_cols][variant_data_node_id_string]} %{[zeek_cols][variant_data_node_id_guid]} %{[zeek_cols][variant_data_node_id_opaque]} %{[zeek_cols][variant_data_node_id_namespace_uri]} %{[zeek_cols][variant_data_node_id_server_idx]} %{[zeek_cols][variant_data_value_time]} %{[zeek_cols][variant_data_encoding_name_idx]} %{[zeek_cols][variant_data_encoding_name]} %{[zeek_cols][variant_data_mask]} %{[zeek_cols][variant_data_locale]} %{[zeek_cols][variant_data_text]} %{[zeek_cols][variant_data_value_decimal]} %{[zeek_cols][variant_data_status_code_link_id]} %{[zeek_cols][variant_data_diag_info_link_id]} %{[zeek_cols][variant_data_ext_obj_link_id]} %{[zeek_cols][variant_metadata_data_link_id]} %{[zeek_cols][variant_data_value_link_id]}" + } } - ruby { - id => "ruby_zip_zeek_opcua_binary_variant_data" - init => "@zeek_opcua_binary_variant_data_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'variant_data_link_id', 'variant_data_value_signed_numeric', 'variant_data_value_unsigned_numeric', 'variant_data_value_string', 'variant_data_node_id_encoding_mask', 'variant_data_node_id_namespace_idx', 'variant_data_node_id_numeric', 'variant_data_node_id_string', 'variant_data_node_id_guid', 'variant_data_node_id_opaque', 'variant_data_node_id_namespace_uri', 'variant_data_node_id_server_idx', 'variant_data_value_time', 'variant_data_encoding_name_idx', 'variant_data_encoding_name', 'variant_data_mask', 'variant_data_locale', 'variant_data_text', 'variant_data_value_decimal', 'variant_data_status_code_link_id', 'variant_data_diag_info_link_id', 'variant_data_ext_obj_link_id', 'variant_metadata_data_link_id', 'variant_data_value_link_id' ]" - code => "event.set('[zeek_cols]', @zeek_opcua_binary_variant_data_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_opcua_binary_variant_data" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_opcua_binary_variant_data" + init => "@zeek_opcua_binary_variant_data_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'variant_data_link_id', 'variant_data_value_signed_numeric', 'variant_data_value_unsigned_numeric', 'variant_data_value_string', 'variant_data_node_id_encoding_mask', 'variant_data_node_id_namespace_idx', 'variant_data_node_id_numeric', 'variant_data_node_id_string', 'variant_data_node_id_guid', 'variant_data_node_id_opaque', 'variant_data_node_id_namespace_uri', 'variant_data_node_id_server_idx', 'variant_data_value_time', 'variant_data_encoding_name_idx', 'variant_data_encoding_name', 'variant_data_mask', 'variant_data_locale', 'variant_data_text', 'variant_data_value_decimal', 'variant_data_status_code_link_id', 'variant_data_diag_info_link_id', 'variant_data_ext_obj_link_id', 'variant_metadata_data_link_id', 'variant_data_value_link_id' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_variant_data_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -5315,24 +5649,25 @@ filter { # opcua_binary_variant_data_value.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - dissect { - id => "dissect_zeek_opcua_binary_variant_data_value" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][variant_data_value_source_link]} %{[zeek_cols][data_value_encoding_mask]} %{[zeek_cols][status_code_link_id]} %{[zeek_cols][source_timestamp]} %{[zeek_cols][source_pico_sec]} %{[zeek_cols][server_timestamp]} %{[zeek_cols][server_pico_sec]} %{[zeek_cols][variant_metadata_link_id]}" - } - } - - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_opcua_binary_variant_data_value" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_opcua_binary_variant_data_value" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][variant_data_value_source_link]} %{[zeek_cols][data_value_encoding_mask]} %{[zeek_cols][status_code_link_id]} %{[zeek_cols][source_timestamp]} %{[zeek_cols][source_pico_sec]} %{[zeek_cols][server_timestamp]} %{[zeek_cols][server_pico_sec]} %{[zeek_cols][variant_metadata_link_id]}" + } } - ruby { - id => "ruby_zip_zeek_opcua_binary_variant_data_value" - init => "@zeek_opcua_binary_variant_data_value_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'variant_data_value_source_link', 'data_value_encoding_mask', 'status_code_link_id', 'source_timestamp', 'source_pico_sec', 'server_timestamp', 'server_pico_sec', 'variant_metadata_link_id' ]" - code => "event.set('[zeek_cols]', @zeek_opcua_binary_variant_data_value_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_opcua_binary_variant_data_value" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_opcua_binary_variant_data_value" + init => "@zeek_opcua_binary_variant_data_value_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'variant_data_value_source_link', 'data_value_encoding_mask', 'status_code_link_id', 'source_timestamp', 'source_pico_sec', 'server_timestamp', 'server_pico_sec', 'variant_metadata_link_id' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_variant_data_value_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -5350,24 +5685,25 @@ filter { # opcua_binary_variant_extension_object.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - dissect { - id => "dissect_zeek_opcua_binary_variant_extension_object" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][ext_obj_link_id]} %{[zeek_cols][ext_obj_node_id_encoding_mask]} %{[zeek_cols][ext_obj_node_id_namespace_idx]} %{[zeek_cols][ext_obj_node_id_numeric]} %{[zeek_cols][ext_obj_node_id_string]} %{[zeek_cols][ext_obj_node_id_guid]} %{[zeek_cols][ext_obj_node_id_opaque]} %{[zeek_cols][ext_obj_type_id_str]} %{[zeek_cols][ext_obj_encoding]}" - } - } - - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_opcua_binary_variant_extension_object" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_opcua_binary_variant_extension_object" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][ext_obj_link_id]} %{[zeek_cols][ext_obj_node_id_encoding_mask]} %{[zeek_cols][ext_obj_node_id_namespace_idx]} %{[zeek_cols][ext_obj_node_id_numeric]} %{[zeek_cols][ext_obj_node_id_string]} %{[zeek_cols][ext_obj_node_id_guid]} %{[zeek_cols][ext_obj_node_id_opaque]} %{[zeek_cols][ext_obj_type_id_str]} %{[zeek_cols][ext_obj_encoding]}" + } } - ruby { - id => "ruby_zip_zeek_opcua_binary_variant_extension_object" - init => "@zeek_opcua_binary_variant_extension_object_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'ext_obj_link_id', 'ext_obj_node_id_encoding_mask', 'ext_obj_node_id_namespace_idx', 'ext_obj_node_id_numeric', 'ext_obj_node_id_string', 'ext_obj_node_id_guid', 'ext_obj_node_id_opaque', 'ext_obj_type_id_str', 'ext_obj_encoding' ]" - code => "event.set('[zeek_cols]', @zeek_opcua_binary_variant_extension_object_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_opcua_binary_variant_extension_object" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_opcua_binary_variant_extension_object" + init => "@zeek_opcua_binary_variant_extension_object_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'ext_obj_link_id', 'ext_obj_node_id_encoding_mask', 'ext_obj_node_id_namespace_idx', 'ext_obj_node_id_numeric', 'ext_obj_node_id_string', 'ext_obj_node_id_guid', 'ext_obj_node_id_opaque', 'ext_obj_type_id_str', 'ext_obj_encoding' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_variant_extension_object_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -5385,24 +5721,25 @@ filter { # opcua_binary_variant_metadata.log # variant-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) - dissect { - id => "dissect_zeek_opcua_binary_variant_metadata" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][variant_source_data_link_id]} %{[zeek_cols][variant_data_source]} %{[zeek_cols][variant_data_source_str]} %{[zeek_cols][dara_variant_encoding_mask]} %{[zeek_cols][data_variant_data_type]} %{[zeek_cols][data_variant_data_type_str]} %{[zeek_cols][built_in_data_type]} %{[zeek_cols][built_in_data_type_str]} %{[zeek_cols][variant_data_link_id]} %{[zeek_cols][variant_data_array_dim]} %{[zeek_cols][variant_data_array_multi_dim_link_id]}" - } - } - - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_opcua_binary_variant_metadata" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_opcua_binary_variant_metadata" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][variant_source_data_link_id]} %{[zeek_cols][variant_data_source]} %{[zeek_cols][variant_data_source_str]} %{[zeek_cols][dara_variant_encoding_mask]} %{[zeek_cols][data_variant_data_type]} %{[zeek_cols][data_variant_data_type_str]} %{[zeek_cols][built_in_data_type]} %{[zeek_cols][built_in_data_type_str]} %{[zeek_cols][variant_data_link_id]} %{[zeek_cols][variant_data_array_dim]} %{[zeek_cols][variant_data_array_multi_dim_link_id]}" + } } - ruby { - id => "ruby_zip_zeek_opcua_binary_variant_metadata" - init => "@zeek_opcua_binary_variant_metadata_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'variant_source_data_link_id', 'variant_data_source', 'variant_data_source_str', 'dara_variant_encoding_mask', 'data_variant_data_type', 'data_variant_data_type_str', 'built_in_data_type', 'built_in_data_type_str', 'variant_data_link_id', 'variant_data_array_dim', 'variant_data_array_multi_dim_link_id' ]" - code => "event.set('[zeek_cols]', @zeek_opcua_binary_variant_metadata_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_opcua_binary_variant_metadata" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_opcua_binary_variant_metadata" + init => "@zeek_opcua_binary_variant_metadata_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'variant_source_data_link_id', 'variant_data_source', 'variant_data_source_str', 'dara_variant_encoding_mask', 'data_variant_data_type', 'data_variant_data_type_str', 'built_in_data_type', 'built_in_data_type_str', 'variant_data_link_id', 'variant_data_array_dim', 'variant_data_array_multi_dim_link_id' ]" + code => "event.set('[zeek_cols]', @zeek_opcua_binary_variant_metadata_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -5417,16 +5754,19 @@ filter { } else { # some other unknown zeek opcua- log file. should start with ts at least! - csv { - id => "csv_zeek_unknown_opcua" - columns => ["ts"] - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - separator => " " - # there's no way to *disable* the csv quote char, so set it to something we'll never see - quote_char => " " + if ("_jsonparsesuccess" not in [tags]) { + csv { + id => "csv_zeek_unknown_opcua" + columns => ["ts"] - target => "[zeek_cols]" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + separator => " " + # there's no way to *disable* the csv quote char, so set it to something we'll never see + quote_char => " " + + target => "[zeek_cols]" + } } mutate { id => "mutate_add_tag_zeek_unknown_opcua" @@ -5445,23 +5785,25 @@ filter { # Zeek Logging analyzer confirmations and violations into analyzer.log # https://docs.zeek.org/en/master/scripts/base/frameworks/analyzer/logging.zeek.html - dissect { - id => "dissect_zeek_diagnostic_analyzer" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][cause]} %{[zeek_cols][analyzer_kind]} %{[zeek_cols][analyzer_name]} %{[zeek_cols][uid]} %{[zeek_cols][fuid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][failure_reason]} %{[zeek_cols][failure_data]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_diagnostic_analyzer" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_diagnostic_analyzer" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][cause]} %{[zeek_cols][analyzer_kind]} %{[zeek_cols][analyzer_name]} %{[zeek_cols][uid]} %{[zeek_cols][fuid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][failure_reason]} %{[zeek_cols][failure_data]}" + } } - ruby { - id => "ruby_zip_zeek_diagnostic_analyzer" - init => "@zeek_diagnostic_analyzer_field_names = [ 'ts', 'cause', 'analyzer_kind', 'analyzer_name', 'uid', 'fuid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'failure_reason', 'failure_data' ]" - code => "event.set('[zeek_cols]', @zeek_diagnostic_analyzer_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_diagnostic_analyzer" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_diagnostic_analyzer" + init => "@zeek_diagnostic_analyzer_field_names = [ 'ts', 'cause', 'analyzer_kind', 'analyzer_name', 'uid', 'fuid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'failure_reason', 'failure_data' ]" + code => "event.set('[zeek_cols]', @zeek_diagnostic_analyzer_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -5473,23 +5815,35 @@ filter { # broker.log # https://docs.zeek.org/en/master/scripts/base/frameworks/broker/log.zeek.html - dissect { - id => "dissect_zeek_diagnostic_broker" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][event_type]} %{[zeek_cols][event_action]} %{[zeek_cols][peer_ip]} %{[zeek_cols][peer_port]} %{[zeek_cols][peer_message]}" - } - } - if ("_dissectfailure" in [tags]) { + if ("_jsonparsesuccess" in [tags]) { mutate { - id => "mutate_split_zeek_diagnostic_broker" + id => "mutate_rename_zeek_json_broker_fields" + rename => { "[zeek_cols][ty]" => "[zeek_cols][event_type]" } + rename => { "[zeek_cols][ev]" => "[zeek_cols][event_action]" } + rename => { "[zeek_cols][peer.address]" => "[zeek_cols][peer_ip]" } + rename => { "[zeek_cols][peer.bound_port]" => "[zeek_cols][peer_port]" } + rename => { "[zeek_cols][message]" => "[zeek_cols][peer_message]" } + } + + } else { + dissect { + id => "dissect_zeek_diagnostic_broker" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][event_type]} %{[zeek_cols][event_action]} %{[zeek_cols][peer_ip]} %{[zeek_cols][peer_port]} %{[zeek_cols][peer_message]}" + } } - ruby { - id => "ruby_zip_zeek_diagnostic_broker" - init => "@zeek_diagnostic_broker_field_names = [ 'ts', 'event_type', 'event_action', 'peer_ip', 'peer_port', 'peer_message' ]" - code => "event.set('[zeek_cols]', @zeek_diagnostic_broker_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_diagnostic_broker" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_diagnostic_broker" + init => "@zeek_diagnostic_broker_field_names = [ 'ts', 'event_type', 'event_action', 'peer_ip', 'peer_port', 'peer_message' ]" + code => "event.set('[zeek_cols]', @zeek_diagnostic_broker_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -5502,23 +5856,25 @@ filter { # Reports analysis of missing traffic. Zeek bases its conclusions on analysis of TCP sequence numbers. # https://docs.zeek.org/en/master/logs/capture-loss-and-reporter.html - dissect { - id => "dissect_zeek_diagnostic_capture_loss" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][ts_delta]} %{[zeek_cols][peer]} %{[zeek_cols][gaps]} %{[zeek_cols][acks]} %{[zeek_cols][percent_lost]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_diagnostic_capture_loss" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_diagnostic_capture_loss" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][ts_delta]} %{[zeek_cols][peer]} %{[zeek_cols][gaps]} %{[zeek_cols][acks]} %{[zeek_cols][percent_lost]}" + } } - ruby { - id => "ruby_zip_zeek_diagnostic_capture_loss" - init => "@zeek_diagnostic_capture_loss_field_names = [ 'ts', 'ts_delta', 'peer', 'gaps', 'acks', 'percent_lost' ]" - code => "event.set('[zeek_cols]', @zeek_diagnostic_capture_loss_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_diagnostic_capture_loss" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_diagnostic_capture_loss" + init => "@zeek_diagnostic_capture_loss_field_names = [ 'ts', 'ts_delta', 'peer', 'gaps', 'acks', 'percent_lost' ]" + code => "event.set('[zeek_cols]', @zeek_diagnostic_capture_loss_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -5531,23 +5887,31 @@ filter { # Logging for establishing and controlling a cluster of Zeek instances # https://docs.zeek.org/en/master/scripts/base/frameworks/cluster/main.zeek.html#type-Cluster::Info - dissect { - id => "dissect_zeek_diagnostic_cluster" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][node]} %{[zeek_cols][node_message]}" - } - } - if ("_dissectfailure" in [tags]) { + if ("_jsonparsesuccess" in [tags]) { mutate { - id => "mutate_split_zeek_diagnostic_cluster" + id => "mutate_rename_zeek_json_cluster_fields" + rename => { "[zeek_cols][message]" => "[zeek_cols][node_message]" } + } + + } else { + dissect { + id => "dissect_zeek_diagnostic_cluster" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][node]} %{[zeek_cols][node_message]}" + } } - ruby { - id => "ruby_zip_zeek_diagnostic_cluster" - init => "@zeek_diagnostic_cluster_field_names = [ 'ts', 'node', 'node_message' ]" - code => "event.set('[zeek_cols]', @zeek_diagnostic_cluster_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_diagnostic_cluster" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_diagnostic_cluster" + init => "@zeek_diagnostic_cluster_field_names = [ 'ts', 'node', 'node_message' ]" + code => "event.set('[zeek_cols]', @zeek_diagnostic_cluster_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -5560,23 +5924,25 @@ filter { # Logging for Zeek configuration changes # https://docs.zeek.org/en/master/scripts/base/frameworks/config/main.zeek.html#type-Config::Info - dissect { - id => "dissect_zeek_diagnostic_config" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][value_name]} %{[zeek_cols][value_old]} %{[zeek_cols][value_new]} %{[zeek_cols][location]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_diagnostic_config" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_diagnostic_config" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][value_name]} %{[zeek_cols][value_old]} %{[zeek_cols][value_new]} %{[zeek_cols][location]}" + } } - ruby { - id => "ruby_zip_zeek_diagnostic_config" - init => "@zeek_diagnostic_config_field_names = [ 'ts', 'value_name', 'value_old', 'value_new', 'location' ]" - code => "event.set('[zeek_cols]', @zeek_diagnostic_config_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_diagnostic_config" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_diagnostic_config" + init => "@zeek_diagnostic_config_field_names = [ 'ts', 'value_name', 'value_old', 'value_new', 'location' ]" + code => "event.set('[zeek_cols]', @zeek_diagnostic_config_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -5588,23 +5954,25 @@ filter { # packet_filter.log # https://docs.zeek.org/en/master/scripts/base/frameworks/packet-filter/main.zeek.html#type-PacketFilter::Info - dissect { - id => "dissect_zeek_diagnostic_packet_filter" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][node]} %{[zeek_cols][filter]} %{[zeek_cols][init]} %{[zeek_cols][success]} %{[zeek_cols][failure_reason]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_diagnostic_packet_filter" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_diagnostic_packet_filter" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][node]} %{[zeek_cols][filter]} %{[zeek_cols][init]} %{[zeek_cols][success]} %{[zeek_cols][failure_reason]}" + } } - ruby { - id => "ruby_zip_zeek_diagnostic_packet_filter" - init => "@zeek_diagnostic_packet_filter_field_names = [ 'ts', 'node', 'filter', 'init', 'success', 'failure_reason' ]" - code => "event.set('[zeek_cols]', @zeek_diagnostic_packet_filter_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_diagnostic_packet_filter" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_diagnostic_packet_filter" + init => "@zeek_diagnostic_packet_filter_field_names = [ 'ts', 'node', 'filter', 'init', 'success', 'failure_reason' ]" + code => "event.set('[zeek_cols]', @zeek_diagnostic_packet_filter_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -5616,29 +5984,30 @@ filter { # print.log # https://docs.zeek.org/en/master/scripts/base/frameworks/logging/main.zeek.html#type-Log::PrintLogInfo - dissect { - id => "dissect_zeek_diagnostic_print" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][vals]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_diagnostic_print" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_diagnostic_print" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][vals]}" + } } - ruby { - id => "ruby_zip_zeek_diagnostic_print" - init => "@zeek_diagnostic_print_field_names = [ 'ts', 'vals' ]" - code => "event.set('[zeek_cols]', @zeek_diagnostic_print_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_diagnostic_print" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_diagnostic_print" + init => "@zeek_diagnostic_print_field_names = [ 'ts', 'vals' ]" + code => "event.set('[zeek_cols]', @zeek_diagnostic_print_field_names.zip(event.get('[message]')).to_h)" + } } + mutate { id => "split_zeek_diagnostic_print_vals" + split => { "[zeek_cols][vals]" => "," } } } - mutate { id => "split_zeek_diagnostic_print_vals" - split => { "[zeek_cols][vals]" => "," } } - mutate { id => "mutate_add_tag_zeek_diagnostic_print" add_tag => [ "_zeekdiagnostic" ] } @@ -5648,23 +6017,31 @@ filter { # reporter.log # https://docs.zeek.org/en/master/scripts/base/frameworks/reporter/main.zeek.html#type-Reporter::Info - dissect { - id => "dissect_zeek_diagnostic_reporter" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][level]} %{[zeek_cols][msg]} %{[zeek_cols][location]}" - } - } - if ("_dissectfailure" in [tags]) { + if ("_jsonparsesuccess" in [tags]) { mutate { - id => "mutate_split_zeek_diagnostic_reporter" + id => "mutate_rename_zeek_json_reporter_fields" + rename => { "[zeek_cols][message]" => "[zeek_cols][msg]" } + } + + } else { + dissect { + id => "dissect_zeek_diagnostic_reporter" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][level]} %{[zeek_cols][msg]} %{[zeek_cols][location]}" + } } - ruby { - id => "ruby_zip_zeek_diagnostic_reporter" - init => "@zeek_diagnostic_reporter_field_names = [ 'ts', 'node', 'filter', 'init', 'success', 'failure_reason' ]" - code => "event.set('[zeek_cols]', @zeek_diagnostic_reporter_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_diagnostic_reporter" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_diagnostic_reporter" + init => "@zeek_diagnostic_reporter_field_names = [ 'ts', 'level', 'msg', 'location' ]" + code => "event.set('[zeek_cols]', @zeek_diagnostic_reporter_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -5676,23 +6053,25 @@ filter { # stats.log # https://docs.zeek.org/en/master/scripts/policy/misc/stats.zeek.html#type-Stats::Info - dissect { - id => "dissect_zeek_diagnostic_stats" - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][peer]} %{[zeek_cols][mem]} %{[zeek_cols][pkts_proc]} %{[zeek_cols][bytes_recv]} %{[zeek_cols][pkts_dropped]} %{[zeek_cols][pkts_link]} %{[zeek_cols][pkt_lag]} %{[zeek_cols][pkts_filtered]} %{[zeek_cols][events_proc]} %{[zeek_cols][events_queued]} %{[zeek_cols][active_tcp_conns]} %{[zeek_cols][active_udp_conns]} %{[zeek_cols][active_icmp_conns]} %{[zeek_cols][tcp_conns]} %{[zeek_cols][udp_conns]} %{[zeek_cols][icmp_conns]} %{[zeek_cols][timers]} %{[zeek_cols][active_timers]} %{[zeek_cols][files]} %{[zeek_cols][active_files]} %{[zeek_cols][dns_requests]} %{[zeek_cols][active_dns_requests]} %{[zeek_cols][reassem_tcp_size]} %{[zeek_cols][reassem_file_size]} %{[zeek_cols][reassem_frag_size]} %{[zeek_cols][reassem_unknown_size]}" - } - } - if ("_dissectfailure" in [tags]) { - mutate { - id => "mutate_split_zeek_diagnostic_stats" + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_diagnostic_stats" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - split => { "[message]" => " " } + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][peer]} %{[zeek_cols][mem]} %{[zeek_cols][pkts_proc]} %{[zeek_cols][bytes_recv]} %{[zeek_cols][pkts_dropped]} %{[zeek_cols][pkts_link]} %{[zeek_cols][pkt_lag]} %{[zeek_cols][pkts_filtered]} %{[zeek_cols][events_proc]} %{[zeek_cols][events_queued]} %{[zeek_cols][active_tcp_conns]} %{[zeek_cols][active_udp_conns]} %{[zeek_cols][active_icmp_conns]} %{[zeek_cols][tcp_conns]} %{[zeek_cols][udp_conns]} %{[zeek_cols][icmp_conns]} %{[zeek_cols][timers]} %{[zeek_cols][active_timers]} %{[zeek_cols][files]} %{[zeek_cols][active_files]} %{[zeek_cols][dns_requests]} %{[zeek_cols][active_dns_requests]} %{[zeek_cols][reassem_tcp_size]} %{[zeek_cols][reassem_file_size]} %{[zeek_cols][reassem_frag_size]} %{[zeek_cols][reassem_unknown_size]}" + } } - ruby { - id => "ruby_zip_zeek_diagnostic_stats" - init => "@zeek_diagnostic_stats_field_names = [ 'ts', 'peer', 'mem', 'pkts_proc', 'bytes_recv', 'pkts_dropped', 'pkts_link', 'pkt_lag', 'pkts_filtered', 'events_proc', 'events_queued', 'active_tcp_conns', 'active_udp_conns', 'active_icmp_conns', 'tcp_conns', 'udp_conns', 'icmp_conns', 'timers', 'active_timers', 'files', 'active_files', 'dns_requests', 'active_dns_requests', 'reassem_tcp_size', 'reassem_file_size', 'reassem_frag_size', 'reassem_unknown_size' ]" - code => "event.set('[zeek_cols]', @zeek_diagnostic_stats_field_names.zip(event.get('[message]')).to_h)" + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_diagnostic_stats" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_diagnostic_stats" + init => "@zeek_diagnostic_stats_field_names = [ 'ts', 'peer', 'mem', 'pkts_proc', 'bytes_recv', 'pkts_dropped', 'pkts_link', 'pkt_lag', 'pkts_filtered', 'events_proc', 'events_queued', 'active_tcp_conns', 'active_udp_conns', 'active_icmp_conns', 'tcp_conns', 'udp_conns', 'icmp_conns', 'timers', 'active_timers', 'files', 'active_files', 'dns_requests', 'active_dns_requests', 'reassem_tcp_size', 'reassem_file_size', 'reassem_frag_size', 'reassem_unknown_size' ]" + code => "event.set('[zeek_cols]', @zeek_diagnostic_stats_field_names.zip(event.get('[message]')).to_h)" + } } } @@ -5700,17 +6079,20 @@ filter { add_tag => [ "_zeekdiagnostic" ] } } else { - # some other unknown zeek log file. should start with ts at least! - csv { - id => "csv_zeek_unknown" - columns => ["ts"] - # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP - separator => " " - # there's no way to *disable* the csv quote char, so set it to something we'll never see - quote_char => " " + if ("_jsonparsesuccess" not in [tags]) { + # some other unknown zeek log file. should start with ts at least! + csv { + id => "csv_zeek_unknown" + columns => ["ts"] + + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + separator => " " + # there's no way to *disable* the csv quote char, so set it to something we'll never see + quote_char => " " - target => "[zeek_cols]" + target => "[zeek_cols]" + } } mutate { id => "mutate_add_tag_zeek_unknown" diff --git a/logstash/pipelines/zeek/12_zeek_mutate.conf b/logstash/pipelines/zeek/12_zeek_mutate.conf index a56866eee..06b74ee15 100644 --- a/logstash/pipelines/zeek/12_zeek_mutate.conf +++ b/logstash/pipelines/zeek/12_zeek_mutate.conf @@ -67,14 +67,13 @@ filter { base64encode => true } - if ([source][mac]) { - mutate { id => "mutate_split_zeek_source_mac" - split => { "[source][mac]" => "," } } - } - - if ([destination][mac]) { - mutate { id => "mutate_split_zeek_destination_mac" - split => { "[destination][mac]" => "," } } + if ("_jsonparsesuccess" not in [tags]) { + mutate { id => "mutate_split_zeek_commas_post_parse" + split => { "[source][mac]" => "," + "[destination][mac]" => "," + "[zeek][fuid]" => "," + "[related][user]" => "," + "[network][protocol]" => "," } } } if ([zeek][uid]) { @@ -87,10 +86,6 @@ filter { merge => { "[event][id]" => "[zeek][uid]" } } } - # if present, FUIDs are always a vector (comma-separated) - if ([zeek][fuid]) { mutate { id => "mutate_split_zeek_fuids" - split => { "[zeek][fuid]" => "," } } } - if ([zeek][ts]) { if ([zeek][ts] == "0.000000") { @@ -140,10 +135,6 @@ filter { } # if ([zeek][ts]) # set user and transport- and application-level protocols if specified - if ([related][user]) { - mutate { id => "mutate_split_related_user" - split => { "[related][user]" => "," } } - } if ([network][transport]) { translate { id => "translate_zeek_proto" @@ -151,17 +142,14 @@ filter { target => "[ipProtocol]" dictionary_path => "/etc/ip_protocol_name_to_number.yaml" } - mutate { id => "mutate_split_zeek_proto" - split => { "[network][transport]" => "," } } + if ("_jsonparsesuccess" not in [tags]) { + mutate { id => "mutate_split_zeek_proto" + split => { "[network][transport]" => "," } } + } mutate { id => "mutate_merge_zeek_proto" merge => { "[protocol]" => "[network][transport]" } } } - if ([network][protocol]) { - mutate { id => "mutate_split_zeek_service" - split => { "[network][protocol]" => "," } } - } - if ([log_source] == "conn") { ############################################################################################################################# # conn.log specific logic @@ -174,9 +162,6 @@ filter { # TODO: can we normalize this with suricata.flow.state somewhere? } - mutate { id => "mutate_split_zeek_conn_tunnel_parents" - split => { "[zeek][conn][tunnel_parents]" => "," } } - if ([zeek][conn][vlan] or [zeek][conn][inner_vlan]) { # ECS - zeek.conn.vlan -> network.vlan.id if ([zeek][conn][vlan]) { @@ -261,8 +246,15 @@ filter { id => "ruby_bsap_ip_rdb_variables" init => "@bsap_ip_rdb_fields = [ :var, :val ]" code => " - vars = event.get('[zeek][bsap_ip_rdb][variables]').to_s.split(',').zip( - event.get('[zeek][bsap_ip_rdb][variable_value]').to_s.split(',')).map{ |x| @bsap_ip_rdb_fields.zip(x).to_h } + vars = event.get('[zeek][bsap_ip_rdb][variables]') + values = event.get('[zeek][bsap_ip_rdb][variable_value]') + if !vars.is_a?(Array) then + vars = vars.to_s.split(',') + end + if !values.is_a?(Array) then + values = values.to_s.split(',') + end + vars = vars.zip(values).map{ |x| @bsap_ip_rdb_fields.zip(x).to_h } event.set('[zeek][bsap_ip_rdb][variables]', vars) event.set('[zeek][bsap_ip_rdb][variable_count]', vars.length) " @@ -280,8 +272,15 @@ filter { id => "ruby_bsap_serial_rdb_variables" init => "@bsap_serial_rdb_fields = [ :var, :val ]" code => " - vars = event.get('[zeek][bsap_serial_rdb][variables]').to_s.split(',').zip( - event.get('[zeek][bsap_serial_rdb][variable_value]').to_s.split(',')).map{ |x| @bsap_serial_rdb_fields.zip(x).to_h } + vars = event.get('[zeek][bsap_serial_rdb][variables]') + values = event.get('[zeek][bsap_serial_rdb][variable_value]') + if !vars.is_a?(Array) then + vars = vars.to_s.split(',') + end + if !values.is_a?(Array) then + values = values.to_s.split(',') + end + vars = vars.zip(values).map{ |x| @bsap_serial_rdb_fields.zip(x).to_h } event.set('[zeek][bsap_serial_rdb][variables]', vars) event.set('[zeek][bsap_serial_rdb][variable_count]', vars.length) " @@ -333,8 +332,6 @@ filter { } if ([zeek][dhcp][msg_types]) { - mutate { id => "mutate_split_zeek_dhcp_msg_types" - split => { "[zeek][dhcp][msg_types]" => "," } } ruby { id => "ruby_zeek_dhcp_msg_types" path => "/usr/share/logstash/malcolm-ruby/make_unique_array.rb" @@ -401,21 +398,20 @@ filter { ############################################################################################################################# # dns.log specific logic - mutate { id => "mutate_split_zeek_dns_TTLs" - split => { "[zeek][dns][TTLs]" => "," } } - # this must be done because [dns][ip] only handles IPv4 or IPv6 addresses # but [answers] can contain hostnames as well if ([zeek][dns][answers]) { ruby { id => "ruby_zeek_dns_answers_ip_extract" # todo: adjust this regex so it at least sort of catches IPv6 as well - code => "event.set('[@metadata][answers_ip]', event.get('[zeek][dns][answers]').scan(/\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b/).join(','))" + code => " + answers = event.get('[zeek][dns][answers]') + if answers.is_a?(Array) then + answers = answers.join(',') + end + event.set('[@metadata][answers_ip]', answers.scan(/\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b/)) + " } - mutate { id => "mutate_split_zeek_dns_answers" - split => { "[zeek][dns][answers]" => "," } } - mutate { id => "mutate_split_zeek_dns_answers_ip" - split => { "[@metadata][answers_ip]" => "," } } # ECS - zeek.dns -> dns.answers ruby { id => "ruby_zeek_dns_answers_to_ecs" @@ -521,12 +517,6 @@ filter { } } - if ([zeek][files][parent_fuid]) { mutate { id => "mutate_split_zeek_files_parent_fuid" - split => { "[zeek][files][parent_fuid]" => "," } } } - - if ([zeek][files][analyzers]) { mutate { id => "mutate_split_zeek_files_analyzers" - split => { "[zeek][files][analyzers]" => "," } } } - if ([zeek][files][source]) { # do some normalization on files source @@ -589,7 +579,11 @@ filter { id => "ruby_genisys_payload" init => "@genisys_fields = [ :address, :data ]" code => " - vars = event.get('[zeek][genisys][payload_raw]').to_s.split(',').map{ |x| x.split('=') }.map{ |x| @genisys_fields.zip(x).to_h } + vars = event.get('[zeek][genisys][payload_raw]') + if !vars.is_a?(Array) then + vars = vars.to_s.split(',') + end + vars = vars.map{ |x| x.split('=') }.map{ |x| @genisys_fields.zip(x).to_h } event.set('[zeek][genisys][payload]', vars) unless vars.nil? or (vars.length == 0) " } @@ -613,7 +607,7 @@ filter { if ([zeek][gquic][version]) { ruby { id => "ruby_zeek_set_quic_version" - code => "event.set('[quic][version]', 'Q' + event.get('[zeek][gquic][version]').rjust(3, '0'))" + code => "event.set('[quic][version]', 'Q' + event.get('[zeek][gquic][version]').to_s.rjust(3, '0'))" } } @@ -622,21 +616,6 @@ filter { ############################################################################################################################# # http.log specific logic - mutate { id => "mutate_split_zeek_http_orig_fuids" - split => { "[zeek][http][orig_fuids]" => "," } } - mutate { id => "mutate_split_zeek_http_orig_filenames" - split => { "[zeek][http][orig_filenames]" => "," } } - mutate { id => "mutate_split_zeek_http_orig_mime_types" - split => { "[zeek][http][orig_mime_types]" => "," } } - mutate { id => "mutate_split_zeek_http_resp_fuids" - split => { "[zeek][http][resp_fuids]" => "," } } - mutate { id => "mutate_split_zeek_http_resp_filenames" - split => { "[zeek][http][resp_filenames]" => "," } } - mutate { id => "mutate_split_zeek_http_resp_mime_types" - split => { "[zeek][http][resp_mime_types]" => "," } } - mutate { id => "mutate_split_zeek_http_proxied" - split => { "[zeek][http][proxied]" => "," } } - # translate Zeek's double-backslashes single backslash in filenames ruby { id => "ruby_zeek_http_orig_filenames_dedoubleslash" @@ -657,8 +636,6 @@ filter { if ([zeek][http][tags]) { # merge zeek.http.tags into tags and then remove original - mutate { id => "mutate_split_zeek_http_tags" - split => { "[zeek][http][tags]" => "," } } mutate { id => "mutate_merge_zeek_http_tags" merge => { "[tags]" => "[zeek][http][tags]" } } mutate { id => "mutate_remove_field_zeek_http_tags" @@ -778,7 +755,7 @@ filter { query = uri.nil? ? nil : uri.partition('?').last event.set('[url][scheme]', scheme) event.set('[url][original]', scheme + '://' + (host.nil? ? '' : host) + (uri.nil? ? '' : uri)) - event.set('[url][full]', scheme + '://' + (user.nil? ? '' : Array(user).first) + (password.nil? ? '' : ':' + password) + ((user.nil? && password.nil?) ? '' : '@') + (host.nil? ? '' : host) + (port.nil? ? '' : ':' + port) + (uri.nil? ? '' : uri)) + event.set('[url][full]', scheme + '://' + (user.nil? ? '' : Array(user).first) + (password.nil? ? '' : ':' + password) + ((user.nil? && password.nil?) ? '' : '@') + (host.nil? ? '' : host) + (port.nil? ? '' : ':' + port.to_s) + (uri.nil? ? '' : uri)) event.set('[url][domain]', host) unless host.nil? event.set('[url][extension]', ext) unless ext.nil? || ext.empty? event.set('[url][fragment]', fragment) unless fragment.nil? || fragment.empty? @@ -797,21 +774,6 @@ filter { ############################################################################################################################# # ipsec.log specific logic - mutate { id => "mutate_split_zeek_ipsec_vendor_ids" - split => { "[zeek][ipsec][vendor_ids]" => "," } } - mutate { id => "mutate_split_zeek_ipsec_notify_messages" - split => { "[zeek][ipsec][notify_messages]" => "," } } - mutate { id => "mutate_split_zeek_ipsec_transforms" - split => { "[zeek][ipsec][transforms]" => "," } } - mutate { id => "mutate_split_zeek_ipsec_ke_dh_groups" - split => { "[zeek][ipsec][ke_dh_groups]" => "," } } - mutate { id => "mutate_split_zeek_ipsec_proposals" - split => { "[zeek][ipsec][proposals]" => "," } } - mutate { id => "mutate_split_zeek_ipsec_certificates" - split => { "[zeek][ipsec][certificates]" => "," } } - mutate { id => "mutate_split_zeek_ipsec_transform_attributes" - split => { "[zeek][ipsec][transform_attributes]" => "," } } - ruby { id => "ruby_zeek_field_ipsec_flags" code => ' @@ -862,15 +824,6 @@ filter { ############################################################################################################################# # intel.log specific logic - if ([zeek][intel][sources]) { mutate { id => "mutate_split_zeek_intel_sources" - split => { "[zeek][intel][sources]" => "," } } } - - if ([zeek][intel][matched]) { mutate { id => "mutate_split_zeek_intel_matched" - split => { "[zeek][intel][matched]" => "," } } } - - if ([zeek][intel][cif_tags]) { mutate { id => "mutate_split_zeek_intel_cif_tags" - split => { "[zeek][intel][cif_tags]" => "," } } } - if ([zeek][intel][sources]) or ([zeek][intel][cif_tags]) { ruby { id => "ruby_zeek_intel_sources_and_tags_comma_fix" @@ -906,23 +859,10 @@ filter { merge => { "[krb5][sname]" => "[zeek][kerberos][sname]" } } } - if ([zeek][kerberos][client_cert_fuid]) { mutate { id => "mutate_split_zeek_kerberos_client_cert_fuid" - split => { "[zeek][kerberos][client_cert_fuid]" => "," } } } - - if ([zeek][kerberos][server_cert_fuid]) { mutate { id => "mutate_split_zeek_kerberos_server_cert_fuid" - split => { "[zeek][kerberos][server_cert_fuid]" => "," } } } - } else if ([log_source] == "ldap") { ############################################################################################################################# # ldap.log specific logic - mutate { id => "mutate_split_zeek_ldap_fields" - split => { "[zeek][ldap][operation]" => "," - "[zeek][ldap][result_code]" => "," - "[zeek][ldap][result_message]" => "," - "[zeek][ldap][object]" => "," - "[zeek][ldap][argument]" => "," } } - if ([zeek][ldap][object]) or ([zeek][ldap][argument]) { ruby { id => "ruby_zeek_ldap_object_and_arguments_adjustment" @@ -972,14 +912,6 @@ filter { ############################################################################################################################# # ldap_search.log specific logic - mutate { id => "mutate_split_zeek_ldap_search_fields" - split => { "[zeek][ldap_search][scope]" => "," - "[zeek][ldap_search][deref]" => "," - "[zeek][ldap_search][attributes]" => "," - "[zeek][ldap_search][base_object]" => "," - "[zeek][ldap_search][result_code]" => "," - "[zeek][ldap_search][result_message]" => "," } } - if ([zeek][ldap_search][base_object]) { ruby { id => "ruby_zeek_ldap_search_base_object_adjustment" @@ -1015,11 +947,6 @@ filter { ############################################################################################################################# # modbus_detailed.log specific logic - if ([zeek][modbus_detailed][values]) { - mutate { id => "mutate_split_zeek_modbus_detailed_values" - split => { "[zeek][modbus_detailed][values]" => "," } } - } - # rename some fields to make correlating modbus easier between logs mutate { id => "mutate_rename_modbus_detailed_fields" @@ -1061,16 +988,6 @@ filter { ############################################################################################################################# # modbus_read_write_multiple_registers.log specific logic - if ([zeek][modbus_read_write_multiple_registers][read_registers]) { - mutate { id => "mutate_split_zeek_modbus_read_write_multiple_registers_read_registers" - split => { "[zeek][modbus_read_write_multiple_registers][read_registers]" => "," } } - } - - if ([zeek][modbus_read_write_multiple_registers][write_registers]) { - mutate { id => "mutate_split_zeek_modbus_read_write_multiple_registers_write_registers" - split => { "[zeek][modbus_read_write_multiple_registers][write_registers]" => "," } } - } - # rename some fields to make correlating modbus easier between logs mutate { id => "mutate_rename_modbus_read_write_multiple_registers_fields" @@ -1095,14 +1012,6 @@ filter { add_field => { "[client][bytes]" => "%{[zeek][mqtt_publish][payload_len]}" } } } - } else if ([log_source] == "mqtt_subscribe") { - ############################################################################################################################# - # mqtt_subscribe.log specific logic - - mutate { id => "mutate_split_zeek_mqtt_subscribe_fields" - split => { "[zeek][mqtt_subscribe][topics]" => "," - "[zeek][mqtt_subscribe][qos_levels]" => "," } } - } else if ([log_source] == "mysql") { ############################################################################################################################# # mysql.log specific logic @@ -1116,9 +1025,6 @@ filter { ############################################################################################################################# # notice.log specific logic - mutate { id => "mutate_split_zeek_notice_actions" - split => { "[zeek][notice][actions]" => "," } } - ruby { id => "ruby_add_field_zeek_notice_event_original" code => " event.set('[event][original]', [event.get('[zeek][notice][msg]'), @@ -1474,31 +1380,31 @@ filter { add_field => { "[zeek][software][software_type]" => "OPCUA" } } } - } else if ([log_source] == "ospf") { + } else if ([log_source] == "profinet_io_cm") { ############################################################################################################################# - # ospf.log specific logic - - mutate { id => "mutate_split_zeek_ospf_neighbors" - split => { "[zeek][ospf][neighbors]" => "," } } - mutate { id => "mutate_split_zeek_ospf_routers" - split => { "[zeek][ospf][routers]" => "," } } - mutate { id => "mutate_split_zeek_ospf_metrics" - split => { "[zeek][ospf][metrics]" => "," } } - mutate { id => "mutate_split_zeek_ospf_fwd_addrs" - split => { "[zeek][ospf][fwd_addrs]" => "," } } - mutate { id => "mutate_split_zeek_ospf_route_tags" - split => { "[zeek][ospf][route_tags]" => "," } } - mutate { id => "mutate_split_zeek_link_prefixes" - split => { "[zeek][ospf][link_prefixes]" => "," } } - mutate { id => "mutate_split_zeek_intra_prefixes" - split => { "[zeek][ospf][intra_prefixes]" => "," } } - - } else if ([log_source] == "pe") { - ############################################################################################################################# - # pe.log specific logic + # profinet_io_cm.log specific logic - mutate { id => "mutate_split_zeek_pe_section_names" - split => { "[zeek][pe][section_names]" => "," } } + if ([zeek][profinet_io_cm][packet_type]) and + ((![zeek][profinet_io_cm][operation]) or ([zeek][profinet_io_cm][operation] == '(empty)') or + ([zeek][profinet_io_cm][operation] == 'unknown') or ([zeek][profinet_io_cm][operation] == '-') or + ([zeek][profinet_io_cm][operation] == '')) { + mutate { id => "mutate_replace_zeek_profinet_io_cm_operation" + replace => { "[zeek][profinet_io_cm][operation]" => "%{[zeek][profinet_io_cm][packet_type]}" } } + } + + if ([zeek][profinet_io_cm][interface_vers_major]) or ([zeek][profinet_io_cm][interface_vers_minor]) { + ruby { + id => "ruby_zeek_profinet_io_cm_interface_vers" + code => ' + event.set("[zeek][profinet_io_cm][uuid_version]", + [event.get("[zeek][profinet_io_cm][interface_vers_major]").to_s, + event.get("[zeek][profinet_io_cm][interface_vers_minor]").to_s].join(".")) + ' + } + mutate { id => "mutate_remove_fields_zeek_profinet_io_cm_interface_vers" + remove_field => [ "[zeek][profinet_io_cm][interface_vers_major]", + "[zeek][profinet_io_cm][interface_vers_minor]" ] } + } } else if ([log_source] == "radius") { ############################################################################################################################# @@ -1526,13 +1432,6 @@ filter { } } - } else if ([log_source] == "rdp") { - ############################################################################################################################# - # rdp.log specific logic - - mutate { id => "mutate_split_zeek_rdp_client_channels" - split => { "[zeek][rdp][client_channels]" => "," } } - } else if ([log_source] == "s7comm_read_szl") { ############################################################################################################################# # s7comm_read_szl.log specific logic @@ -1560,8 +1459,6 @@ filter { # _carved signature logs' sub_message contains fuid(s) comma-separated if ([zeek][signatures][sub_message]) { - mutate { id => "mutate_split_zeek_signatures_sub_message" - split => { "[zeek][signatures][sub_message]" => "," } } mutate { id => "mutate_rename_zeek_signatures_sub_message" rename => { "[zeek][signatures][sub_message]" => "[zeek][fuid]" } } } @@ -1714,12 +1611,6 @@ filter { event.set("[zeek][sip][version]", versions.uniq)' } - mutate { id => "mutate_split_zeek_sip_request_path" - split => { "[zeek][sip][request_path]" => "," } } - - mutate { id => "mutate_split_zeek_sip_response_path" - split => { "[zeek][sip][response_path]" => "," } } - if ([zeek][sip][request_body_len]) { mutate { id => "mutate_add_field_zeek_sip_client_bytes" add_field => { "[client][bytes]" => "%{[zeek][sip][request_body_len]}" } } } @@ -1779,9 +1670,15 @@ filter { } } - # collect referenced file FUID(s) at parent level (here rather than in 12_zeek_normalize.conf because + # collect referenced file UIDs(s)/FUID(s) at parent level (here rather than in 13_zeek_normalize.conf because # this would have already been done as a root-level fuid array in the main "rename" above if we # had not had to move it up a level just now) + if ([zeek][smb_files][uid]) { + mutate { id => "mutate_merge_zeek_smb_files_uid" + merge => { "[zeek][uid]" => "[zeek][smb_files][uid]" } } + mutate { id => "mutate_remove_zeek_smb_files_uid" + remove_field => [ "[zeek][smb_files][uid]" ] } + } if ([zeek][smb_files][fuid]) { mutate { id => "mutate_merge_zeek_smb_files_fuid" merge => { "[zeek][fuid]" => "[zeek][smb_files][fuid]" } } @@ -1886,15 +1783,6 @@ filter { ############################################################################################################################# # smtp.log specific logic - mutate { id => "mutate_split_zeek_smtp_rcptto" - split => { "[zeek][smtp][rcptto]" => "," } } - mutate { id => "mutate_split_zeek_smtp_to" - split => { "[zeek][smtp][to]" => "," } } - mutate { id => "mutate_split_zeek_smtp_cc" - split => { "[zeek][smtp][cc]" => "," } } - mutate { id => "mutate_split_zeek_smtp_path" - split => { "[zeek][smtp][path]" => "," } } - if ([zeek][smtp][user_agent]) { mutate { id => "mutate_merge_zeek_smtp_user_agent" merge => { "[email][useragent]" => "[zeek][smtp][user_agent]" } } @@ -1975,18 +1863,6 @@ filter { merge => { "[ssh][hasshServer]" => "[zeek][ssh][hasshServer]" } } } - if ([zeek][ssh][hasshAlgorithms]) { mutate { id => "mutate_split_zeek_ssh_hasshAlgorithms" - split => { "[zeek][ssh][hasshAlgorithms]" => "," } } } - - if ([zeek][ssh][hasshServerAlgorithms]) { mutate { id => "mutate_split_zeek_ssh_hasshServerAlgorithms" - split => { "[zeek][ssh][hasshServerAlgorithms]" => "," } } } - - if ([zeek][ssh][cshka]) { mutate { id => "mutate_split_zeek_ssh_cshka" - split => { "[zeek][ssh][cshka]" => "," } } } - - if ([zeek][ssh][sshka]) { mutate { id => "mutate_split_zeek_ssh_sshka" - split => { "[zeek][ssh][sshka]" => "," } } } - } else if ([log_source] == "ssl") { ############################################################################################################################# @@ -2098,15 +1974,7 @@ filter { merge => { "[tls][ja3s]" => "[zeek][ssl][ja3s]" } } } - if ([zeek][ssl][cert_chain_fuids]) { mutate { id => "mutate_split_zeek_ssl_cert_chain_fuids" - split => { "[zeek][ssl][cert_chain_fuids]" => "," } } } - - if ([zeek][ssl][client_cert_chain_fuids]) { mutate { id => "mutate_split_zeek_ssl_client_cert_chain_fuids" - split => { "[zeek][ssl][client_cert_chain_fuids]" => "," } } } - if ([zeek][ssl][cert_chain_fps]) { - mutate { id => "mutate_split_zeek_ssl_cert_chain_fps" - split => { "[zeek][ssl][cert_chain_fps]" => "," } } # store certificate chain fingerprints in zeek_x509.fingerprint as well for cross-reference with x509.log mutate { id => "mutate_merge_zeek_ssl_cert_chain_fps" merge => { "[zeek][x509][fingerprint]" => "[zeek][ssl][cert_chain_fps]" } } @@ -2116,8 +1984,6 @@ filter { } if ([zeek][ssl][client_cert_chain_fps]) { - mutate { id => "mutate_split_zeek_ssl_client_cert_chain_fps" - split => { "[zeek][ssl][client_cert_chain_fps]" => "," } } # store certificate chain fingerprints in zeek_x509.fingerprint as well for cross-reference with x509.log mutate { id => "mutate_merge_zeek_ssl_client_cert_chain_fps" merge => { "[zeek][x509][fingerprint]" => "[zeek][ssl][client_cert_chain_fps]" } } @@ -2191,15 +2057,6 @@ filter { if ([zeek][ssl][subject_full]) { mutate { id => "mutate_add_field_ecs_zeek_tls_subject_full" add_field => { "[tls][server][subject]" => "%{[zeek][ssl][subject_full]}" } } } - } else if ([log_source] == "stun") { - ############################################################################################################################# - # stun.log specific logic - - if ([zeek][stun][attr_type]) { - mutate { id => "mutate_split_field_zeek_stun_attr_type" - split => { "[zeek][stun][attr_type]" => "," } } - } - } else if ([log_source] == "stun_nat") { ############################################################################################################################# # stun_nat.log specific logic @@ -2235,42 +2092,44 @@ filter { rename => { "[zeek][synchrophasor_hdr][header_time_stamp]" => "[zeek][synchrophasor][header_time_stamp]" } } - mutate { - id => "mutate_split_synchrophasor_fields" - split => { "[zeek][synchrophasor_cfg_detail][analog_conv_analog_flags]" => "," } - split => { "[zeek][synchrophasor_cfg_detail][analog_conv_mag_scale]" => "," } - split => { "[zeek][synchrophasor_cfg_detail][analog_conv_offset]" => "," } - split => { "[zeek][synchrophasor_cfg_detail][analog_conv_user_defined_scaling]" => "," } - split => { "[zeek][synchrophasor_cfg_detail][annam]" => "," } - split => { "[zeek][synchrophasor_cfg_detail][dgnam]" => "," } - split => { "[zeek][synchrophasor_cfg_detail][digital_conv_normal_status_mask]" => "," } - split => { "[zeek][synchrophasor_cfg_detail][digital_conv_valid_inputs_mask]" => "," } - split => { "[zeek][synchrophasor_cfg_detail][phasor_conv_angle_adj]" => "," } - split => { "[zeek][synchrophasor_cfg_detail][phasor_conv_calibration_mag_adj]" => "," } - split => { "[zeek][synchrophasor_cfg_detail][phasor_conv_calibration_phas_adj]" => "," } - split => { "[zeek][synchrophasor_cfg_detail][phasor_conv_downsampled_fir_filter]" => "," } - split => { "[zeek][synchrophasor_cfg_detail][phasor_conv_downsampled_no_fir_filter]" => "," } - split => { "[zeek][synchrophasor_cfg_detail][phasor_conv_downsampled_reselection]" => "," } - split => { "[zeek][synchrophasor_cfg_detail][phasor_conv_filtered_without_changing_sampling]" => "," } - split => { "[zeek][synchrophasor_cfg_detail][phasor_conv_mod_appl]" => "," } - split => { "[zeek][synchrophasor_cfg_detail][phasor_conv_phasor_component]" => "," } - split => { "[zeek][synchrophasor_cfg_detail][phasor_conv_phasor_type]" => "," } - split => { "[zeek][synchrophasor_cfg_detail][phasor_conv_phunit]" => "," } - split => { "[zeek][synchrophasor_cfg_detail][phasor_conv_phvalue]" => "," } - split => { "[zeek][synchrophasor_cfg_detail][phasor_conv_pseudo_phasor_val]" => "," } - split => { "[zeek][synchrophasor_cfg_detail][phasor_conv_rotation_phase_adj]" => "," } - split => { "[zeek][synchrophasor_cfg_detail][phasor_conv_scale_factor]" => "," } - split => { "[zeek][synchrophasor_cfg_detail][phasor_conv_upsampled_extrapolation]" => "," } - split => { "[zeek][synchrophasor_cfg_detail][phasor_conv_upsampled_interpolation]" => "," } - split => { "[zeek][synchrophasor_cfg_detail][phasor_conv_user_def]" => "," } - split => { "[zeek][synchrophasor_cfg_detail][phnam]" => "," } - split => { "[zeek][synchrophasor_cmd][extframe]" => "," } - split => { "[zeek][synchrophasor_data_detail][analog_data]" => "," } - split => { "[zeek][synchrophasor_data_detail][digital]" => "," } - split => { "[zeek][synchrophasor_data_detail][est_polar_angle]" => "," } - split => { "[zeek][synchrophasor_data_detail][est_polar_magnitude]" => "," } - split => { "[zeek][synchrophasor_data_detail][est_rectangular_imaginary]" => "," } - split => { "[zeek][synchrophasor_data_detail][est_rectangular_real]" => "," } + if ("_jsonparsesuccess" not in [tags]) { + mutate { + id => "mutate_split_synchrophasor_fields" + split => { "[zeek][synchrophasor_cfg_detail][analog_conv_analog_flags]" => "," } + split => { "[zeek][synchrophasor_cfg_detail][analog_conv_mag_scale]" => "," } + split => { "[zeek][synchrophasor_cfg_detail][analog_conv_offset]" => "," } + split => { "[zeek][synchrophasor_cfg_detail][analog_conv_user_defined_scaling]" => "," } + split => { "[zeek][synchrophasor_cfg_detail][annam]" => "," } + split => { "[zeek][synchrophasor_cfg_detail][dgnam]" => "," } + split => { "[zeek][synchrophasor_cfg_detail][digital_conv_normal_status_mask]" => "," } + split => { "[zeek][synchrophasor_cfg_detail][digital_conv_valid_inputs_mask]" => "," } + split => { "[zeek][synchrophasor_cfg_detail][phasor_conv_angle_adj]" => "," } + split => { "[zeek][synchrophasor_cfg_detail][phasor_conv_calibration_mag_adj]" => "," } + split => { "[zeek][synchrophasor_cfg_detail][phasor_conv_calibration_phas_adj]" => "," } + split => { "[zeek][synchrophasor_cfg_detail][phasor_conv_downsampled_fir_filter]" => "," } + split => { "[zeek][synchrophasor_cfg_detail][phasor_conv_downsampled_no_fir_filter]" => "," } + split => { "[zeek][synchrophasor_cfg_detail][phasor_conv_downsampled_reselection]" => "," } + split => { "[zeek][synchrophasor_cfg_detail][phasor_conv_filtered_without_changing_sampling]" => "," } + split => { "[zeek][synchrophasor_cfg_detail][phasor_conv_mod_appl]" => "," } + split => { "[zeek][synchrophasor_cfg_detail][phasor_conv_phasor_component]" => "," } + split => { "[zeek][synchrophasor_cfg_detail][phasor_conv_phasor_type]" => "," } + split => { "[zeek][synchrophasor_cfg_detail][phasor_conv_phunit]" => "," } + split => { "[zeek][synchrophasor_cfg_detail][phasor_conv_phvalue]" => "," } + split => { "[zeek][synchrophasor_cfg_detail][phasor_conv_pseudo_phasor_val]" => "," } + split => { "[zeek][synchrophasor_cfg_detail][phasor_conv_rotation_phase_adj]" => "," } + split => { "[zeek][synchrophasor_cfg_detail][phasor_conv_scale_factor]" => "," } + split => { "[zeek][synchrophasor_cfg_detail][phasor_conv_upsampled_extrapolation]" => "," } + split => { "[zeek][synchrophasor_cfg_detail][phasor_conv_upsampled_interpolation]" => "," } + split => { "[zeek][synchrophasor_cfg_detail][phasor_conv_user_def]" => "," } + split => { "[zeek][synchrophasor_cfg_detail][phnam]" => "," } + split => { "[zeek][synchrophasor_cmd][extframe]" => "," } + split => { "[zeek][synchrophasor_data_detail][analog_data]" => "," } + split => { "[zeek][synchrophasor_data_detail][digital]" => "," } + split => { "[zeek][synchrophasor_data_detail][est_polar_angle]" => "," } + split => { "[zeek][synchrophasor_data_detail][est_polar_magnitude]" => "," } + split => { "[zeek][synchrophasor_data_detail][est_rectangular_imaginary]" => "," } + split => { "[zeek][synchrophasor_data_detail][est_rectangular_real]" => "," } + } } if ([zeek][synchrophasor][frame_size]) { @@ -2290,15 +2149,20 @@ filter { ruby { id => "ruby_zeek_tds_rpc_parse_parameter" code => " - parameterHash = event.get('[zeek][tds_rpc][parameter]').split(',').each_with_object({}) do |a, hash| + params = event.get('[zeek][tds_rpc][parameter]') + if !params.is_a?(Array) then + params = params.to_s.split(',') + end + parameterHash = params.each_with_object({}) do |a, hash| key,value = a.split(/\s*=\s*/) hash[key.to_s.delete_prefix('@')] = value end event.set('[zeek][tds_rpc][parameters]', parameterHash.reject{|k,v| k.nil? || v.nil? || k.empty? || v.empty?})" } - - mutate { id => "mutate_split_field_tds_rpc_parameter" - split => { "[zeek][tds_rpc][parameter]" => "," } } + if ("_jsonparsesuccess" not in [tags]) { + mutate { id => "mutate_split_field_tds_rpc_parameter" + split => { "[zeek][tds_rpc][parameter]" => "," } } + } } } else if ([log_source] == "weird") { @@ -2433,12 +2297,6 @@ filter { } } - if ([zeek][x509][san_ip]) { mutate { id => "mutate_split_zeek_x509_san_ip" - split => { "[zeek][x509][san_ip]" => "," } } } - - if ([zeek][x509][fingerprint]) { mutate { id => "mutate_split_zeek_x509_fingerprint" - split => { "[zeek][x509][fingerprint]" => "," } } } - # TODO: ECS TLS/X509 nestings? # - https://www.elastic.co/guide/en/ecs/current/ecs-tls.html#ecs-tls-nestings # - https://www.elastic.co/guide/en/ecs/current/ecs-x509.html#_field_reuse_27 diff --git a/logstash/pipelines/zeek/13_zeek_normalize.conf b/logstash/pipelines/zeek/13_zeek_normalize.conf index 48d97d88d..917c9198b 100644 --- a/logstash/pipelines/zeek/13_zeek_normalize.conf +++ b/logstash/pipelines/zeek/13_zeek_normalize.conf @@ -81,8 +81,8 @@ filter { id => "ruby_zeek_field_network_protocol_version_ipsec" code => " versions = Array.new - versions << [event.get('[zeek][ipsec][maj_ver]'), - event.get('[zeek][ipsec][min_ver]')].reject{ |e| e.nil? || e&.empty? }.join('.') + versions << [event.get('[zeek][ipsec][maj_ver]').to_s, + event.get('[zeek][ipsec][min_ver]').to_s].reject{ |e| e.nil? || e&.empty? }.join('.') event.set('[network][protocol_version]', versions)" } } @@ -115,6 +115,9 @@ filter { if ([zeek][profinet_dce_rpc][version]) { mutate { id => "mutate_merge_normalize_zeek_profinet_dce_rpc_version" merge => { "[network][protocol_version]" => "[zeek][profinet_dce_rpc][version]" } } } + if ([zeek][profinet_io_cm][rpc_version]) { mutate { id => "mutate_merge_normalize_zeek_profinet_io_cm_rpc_version" + merge => { "[network][protocol_version]" => "[zeek][profinet_io_cm][rpc_version]" } } } + if ([zeek][rfb]) { ruby { id => "ruby_zeek_field_network_protocol_version_rfb" @@ -436,6 +439,12 @@ filter { if ([zeek][profinet_dce_rpc][operation]) { mutate { id => "mutate_merge_normalize_zeek_profinet_dce_rpc_operation" merge => { "[event][action]" => "[zeek][profinet_dce_rpc][operation]" } } } + if ([zeek][profinet_io_cm][packet_type]) { mutate { id => "mutate_merge_normalize_zeek_profinet_io_cm_packet_type" + merge => { "[event][action]" => "[zeek][profinet_io_cm][packet_type]" } } } + + if ([zeek][profinet_io_cm][operation]) { mutate { id => "mutate_merge_normalize_zeek_profinet_io_cm_operation" + merge => { "[event][action]" => "[zeek][profinet_io_cm][operation]" } } } + if ([zeek][rfb][auth]) and ([zeek][rfb][authentication_method]) { # if authentication was attempted, assign an "authenticate" action mutate { id => "mutate_add_field_zeek_rfb_auth_action" @@ -1420,10 +1429,10 @@ filter { merge => { "[related][ip]" => "[zeek][files][rx_hosts]" } } } if ([zeek][files][tx_hosts]) { mutate { id => "mutate_merge_field_related_ip_zeek_files_tx_hosts" merge => { "[related][ip]" => "[zeek][files][tx_hosts]" } } } - if ([zeek][ftp][data_channel_orig_h]) { mutate { id => "mutate_merge_field_related_ip_zeek_ftp_data_channel_orig_h" - merge => { "[related][ip]" => "[zeek][ftp][data_channel_orig_h]" } } } - if ([zeek][ftp][data_channel_resp_h]) { mutate { id => "mutate_merge_field_related_ip_zeek_ftp_data_channel_resp_h" - merge => { "[related][ip]" => "[zeek][ftp][data_channel_resp_h]" } } } + if ([zeek][ftp][data_channel][orig_h]) { mutate { id => "mutate_merge_field_related_ip_zeek_ftp_data_channel_orig_h" + merge => { "[related][ip]" => "[zeek][ftp][data_channel][orig_h]" } } } + if ([zeek][ftp][data_channel][resp_h]) { mutate { id => "mutate_merge_field_related_ip_zeek_ftp_data_channel_resp_h" + merge => { "[related][ip]" => "[zeek][ftp][data_channel][resp_h]" } } } if ([zeek][notice][dst]) { mutate { id => "mutate_merge_field_related_ip_zeek_notice_dst" merge => { "[related][ip]" => "[zeek][notice][dst]" } } } if ([zeek][notice][src]) { mutate { id => "mutate_merge_field_related_ip_zeek_notice_src" @@ -1450,6 +1459,10 @@ filter { merge => { "[related][ip]" => "[zeek][ospf][fwd_addrs]" } } } if ([zeek][ospf][dest_router_id]) { mutate { id => "mutate_merge_field_related_ip_zeek_ospf_dest_router_id" merge => { "[related][ip]" => "[zeek][ospf][dest_router_id]" } } } + if ([zeek][smb_files][orig_h]) { mutate { id => "mutate_merge_field_related_ip_zeek_smb_files_orig_h" + merge => { "[related][ip]" => "[zeek][smb_files][orig_h]" } } } + if ([zeek][smb_files][resp_h]) { mutate { id => "mutate_merge_field_related_ip_zeek_smb_files_resp_h" + merge => { "[related][ip]" => "[zeek][smb_files][resp_h]" } } } if ([zeek][radius][framed_addr]) { mutate { id => "mutate_merge_field_related_ip_zeek_radius_framed_addr" merge => { "[related][ip]" => "[zeek][radius][framed_addr]" } } } if ([zeek][smtp][path]) { mutate { id => "mutate_merge_field_related_ip_zeek_smtp_path" diff --git a/nginx/landingpage/index.html b/nginx/landingpage/index.html index 67b62c54c..308821615 100644 --- a/nginx/landingpage/index.html +++ b/nginx/landingpage/index.html @@ -2,6 +2,7 @@ + diff --git a/nginx/nginx.conf b/nginx/nginx.conf index 87884cd6e..f05786b5c 100644 --- a/nginx/nginx.conf +++ b/nginx/nginx.conf @@ -178,14 +178,6 @@ http { proxy_set_header Authorization ""; } - # Dashboards/Arkime -> extracted file download - location ~* /dl-extracted-files/(.*) { - rewrite ^.*/dl-extracted-files/(.*) /extracted-files/$1 redirect; - proxy_pass http://extracted-file-http-server; - proxy_redirect off; - proxy_set_header Host file-monitor.malcolm.local; - } - # OpenSearch dashboards (or Kibana) location /dashboards { include /etc/nginx/nginx_dashboards_rewrite_rt.conf; @@ -199,6 +191,7 @@ http { proxy_set_header Host dashboards-helper.malcolm.local; } + # extracted file download location ~* ^/extracted-files\b(.*) { include /etc/nginx/nginx_auth_rt.conf; proxy_pass http://extracted-file-http-server$1; diff --git a/nginx/nginx_idark2dash_rewrite_kibana.conf b/nginx/nginx_idark2dash_rewrite_kibana.conf index 47ee989a4..2904b0734 100644 --- a/nginx/nginx_idark2dash_rewrite_kibana.conf +++ b/nginx/nginx_idark2dash_rewrite_kibana.conf @@ -1 +1 @@ -rewrite ^.*/idark2dash/(.*) $dashboards_proxy_url/app/dashboards#/view/0ad3d7c2-3441-485e-9dfe-dbb22e84e576?_g=(refreshInterval:(pause:!t,value:60000),time:(from:'2024-02-01T15:45:45.793Z',to:'2024-02-06T16:00:50.775Z'))&_a=(filters:!((meta:(alias:!n,disabled:!f,index:'$sessions_index',key:$filter_field,negate:!f,params:(query:'$filter_value'),type:phrase),query:(match_phrase:($filter_field:'$filter_value')))))? redirect; \ No newline at end of file +rewrite ^.*/idark2dash/(.*) $dashboards_proxy_url/app/dashboards#/view/0ad3d7c2-3441-485e-9dfe-dbb22e84e576?_g=(refreshInterval:(pause:!t,value:0),time:(from:$filter_start_time,to:$filter_stop_time))&_a=(filters:!((meta:(alias:!n,disabled:!f,index:'$sessions_index',key:$filter_field,negate:!f,params:(query:'$filter_value'),type:phrase),query:(match_phrase:($filter_field:'$filter_value')))))? redirect; \ No newline at end of file diff --git a/nginx/nginx_readonly.conf b/nginx/nginx_readonly.conf index 11c6b001e..1251cf46f 100644 --- a/nginx/nginx_readonly.conf +++ b/nginx/nginx_readonly.conf @@ -110,14 +110,6 @@ http { proxy_set_header Authorization ""; } - # Dashboards/Arkime -> extracted file download - location ~* /dl-extracted-files/(.*) { - rewrite ^.*/dl-extracted-files/(.*) /extracted-files/$1 redirect; - proxy_pass http://extracted-file-http-server; - proxy_redirect off; - proxy_set_header Host file-monitor.malcolm.local; - } - # OpenSearch dashboards (or Kibana) location /dashboards { include /etc/nginx/nginx_dashboards_rewrite_rt.conf; @@ -130,6 +122,7 @@ http { proxy_set_header Host dashboards-helper.malcolm.local; } + # extracted file download location ~* ^/extracted-files\b(.*) { proxy_pass http://extracted-file-http-server$1; proxy_redirect off; diff --git a/scripts/github_image_helper.sh b/scripts/github_image_helper.sh index 931e8faa6..a3f496eee 100755 --- a/scripts/github_image_helper.sh +++ b/scripts/github_image_helper.sh @@ -1,5 +1,21 @@ #!/usr/bin/env bash +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. + +if [ -z "$BASH_VERSION" ]; then + echo "Wrong interpreter, please run \"$0\" with bash" + exit 1 +fi + +[[ "$(uname -s)" = 'Darwin' ]] && REALPATH=grealpath || REALPATH=realpath +[[ "$(uname -s)" = 'Darwin' ]] && DIRNAME=gdirname || DIRNAME=dirname +[[ "$(uname -s)" = 'Darwin' ]] && GREP=ggrep || GREP=grep +if ! (type "$REALPATH" && type "$DIRNAME" && type "$GREP" && type git) > /dev/null; then + echo "$(basename "${BASH_SOURCE[0]}") requires $REALPATH and $DIRNAME and $GREP and git" + exit 1 +fi +SCRIPT_PATH="$($DIRNAME $($REALPATH -e "${BASH_SOURCE[0]}"))" + set -uo pipefail shopt -s nocasematch ENCODING="utf-8" @@ -32,32 +48,42 @@ function _cols() { # get the current git working copy's branch (e.g., main) function _gitbranch() { + pushd "$SCRIPT_PATH/.." >/dev/null 2>&1 git rev-parse --abbrev-ref HEAD + popd >/dev/null 2>&1 } # get the current git working copy's remote name (e.g., origin) function _gitremote() { - git branch -vv | grep "^\*" | cut -d "[" -f2 | cut -d "]" -f1 | cut -d "/" -f1 + pushd "$SCRIPT_PATH/.." >/dev/null 2>&1 + git branch -vv | $GREP "^\*" | cut -d "[" -f2 | cut -d "]" -f1 | cut -d "/" -f1 + popd >/dev/null 2>&1 } # get the current git working copy's top-level directory function _gittoplevel() { + pushd "$SCRIPT_PATH/.." >/dev/null 2>&1 git rev-parse --show-toplevel + popd >/dev/null 2>&1 } # get the current git working copy's remote "owner" (github user or organization, e.g., johndoe) function _gitowner() { + pushd "$SCRIPT_PATH/.." >/dev/null 2>&1 git remote get-url "$(_gitremote)" | sed 's@.*github\.com/@@' | cut -d'/' -f1 + popd >/dev/null 2>&1 } # get the current git working copy's remote repository name (e.g., malcolm) function _gitreponame() { + pushd "$SCRIPT_PATH/.." >/dev/null 2>&1 git remote get-url "$(_gitremote)" | sed 's@.*github\.com/@@' | cut -d'/' -f2 + popd >/dev/null 2>&1 } # get the current git working copy's Malcolm version (grepped from docker-compose.yml, e.g., 5.0.3) function _malcolmversion() { - grep -P "^\s+image:.*/malcolm" "$(_gittoplevel)"/docker-compose.yml | awk '{print $2}' | cut -d':' -f2 | uniq -c | sort -nr | awk '{print $2}' | head -n 1 + $GREP -P "^\s+image:.*/malcolm" "$(_gittoplevel)"/docker-compose.yml | awk '{print $2}' | cut -d':' -f2 | uniq -c | sort -nr | awk '{print $2}' | head -n 1 } ################################################################################ @@ -88,7 +114,7 @@ function PullAndTagGithubWorkflowImages() { VERSION="$(_malcolmversion)" OWNER="$(_gitowner)" echo "Pulling images from ghcr.io/$OWNER ($BRANCH) and tagging as $VERSION ..." - for IMG in $(grep image: "$(_gittoplevel)"/docker-compose.yml | _cols 2 | cut -d: -f1 | sort -u | sed "s/.*\/\(malcolm\)/\1/"); do + for IMG in $($GREP image: "$(_gittoplevel)"/docker-compose.yml | _cols 2 | cut -d: -f1 | sort -u | sed "s/.*\/\(malcolm\)/\1/"); do _PullAndTagGithubWorkflowBuild "$IMG" done echo "done" @@ -180,10 +206,14 @@ function GithubTriggerPackagesBuild () { trap "_cleanup" EXIT +# force-navigate to Malcolm base directory (parent of scripts/ directory) +pushd "$SCRIPT_PATH/.." >/dev/null 2>&1 + # get a list of all the "public" functions (not starting with _) FUNCTIONS=($(declare -F | awk '{print $NF}' | sort -f | egrep -v "^_")) # present the menu to our customer and get their selection +printf "%s\t%s\n" "0" "pull and extract everything" for i in "${!FUNCTIONS[@]}"; do ((IPLUS=i+1)) printf "%s\t%s\n" "$IPLUS" "${FUNCTIONS[$i]}" @@ -191,7 +221,13 @@ done echo -n "Operation:" [[ -n "${1-}" ]] && USER_FUNCTION_IDX="$1" || read USER_FUNCTION_IDX -if (( $USER_FUNCTION_IDX > 0 )) && (( $USER_FUNCTION_IDX <= "${#FUNCTIONS[@]}" )); then +if (( $USER_FUNCTION_IDX == 0 )); then + PullAndTagGithubWorkflowISOImages + ExtractAndLoadImagesFromGithubWorkflowBuildISO + ExtractISOsFromGithubWorkflowBuilds + PullAndTagGithubWorkflowImages + +elif (( $USER_FUNCTION_IDX > 0 )) && (( $USER_FUNCTION_IDX <= "${#FUNCTIONS[@]}" )); then # execute one function, à la carte USER_FUNCTION="${FUNCTIONS[((USER_FUNCTION_IDX-1))]}" echo $USER_FUNCTION diff --git a/scripts/install.py b/scripts/install.py index 5e01049b9..c7e2ddad6 100755 --- a/scripts/install.py +++ b/scripts/install.py @@ -89,8 +89,8 @@ MAC_BREW_DOCKER_COMPOSE_PACKAGE = 'docker-compose' MAC_BREW_DOCKER_SETTINGS = '/Users/{}/Library/Group Containers/group.com.docker/settings.json' -LOGSTASH_JAVA_OPTS_DEFAULT = '-server -Xms2500m -Xmx2500m -Xss1536k -XX:-HeapDumpOnOutOfMemoryError -Djava.security.egd=file:/dev/./urandom -Dlog4j.formatMsgNoLookups=true' -OPENSEARCH_JAVA_OPTS_DEFAULT = '-server -Xms10g -Xmx10g -Xss256k -XX:-HeapDumpOnOutOfMemoryError -Djava.security.egd=file:/dev/./urandom -Dlog4j.formatMsgNoLookups=true' +LOGSTASH_JAVA_OPTS_DEFAULT = '-server -Xmx2500m -Xms2500m -Xss1536k -XX:-HeapDumpOnOutOfMemoryError -Djava.security.egd=file:/dev/./urandom -Dlog4j.formatMsgNoLookups=true' +OPENSEARCH_JAVA_OPTS_DEFAULT = '-server -Xmx10g -Xms10g -Xss256k -XX:-HeapDumpOnOutOfMemoryError -Djava.security.egd=file:/dev/./urandom -Dlog4j.formatMsgNoLookups=true' ################################################################################################### ScriptName = os.path.basename(__file__) @@ -420,10 +420,6 @@ def install_malcolm_files(self, malcolm_install_file, default_config_dir): if os.path.isfile(os.path.join(installPath, "docker-compose.yml")): eprint(f"Malcolm runtime files extracted to {installPath}") result = True - with open(os.path.join(installPath, "install_source.txt"), 'w') as f: - f.write( - f'{os.path.basename(malcolm_install_file)} (installed {str(datetime.datetime.now())})\n' - ) else: eprint(f"Malcolm install file extracted to {installPath}, but missing runtime files?") @@ -711,9 +707,11 @@ def tweak_malcolm_runtime(self, malcolm_install_path): ) while ( not InstallerYesOrNo( - f'Setting {osMemory} for OpenSearch and {lsMemory} for Logstash. Is this OK?' - if opensearchPrimaryMode == DatabaseMode.OpenSearchLocal - else f'Setting {lsMemory} for Logstash. Is this OK?', + ( + f'Setting {osMemory} for OpenSearch and {lsMemory} for Logstash. Is this OK?' + if opensearchPrimaryMode == DatabaseMode.OpenSearchLocal + else f'Setting {lsMemory} for Logstash. Is this OK?' + ), default=True, ) and loopBreaker.increment() @@ -976,6 +974,7 @@ def tweak_malcolm_runtime(self, malcolm_install_path): # make sure paths specified (and their necessary children) exist for pathToCreate in ( + malcolm_install_path, indexDirFull, indexSnapshotDirFull, os.path.join(pcapDirFull, 'arkime-live'), @@ -1075,9 +1074,11 @@ def tweak_malcolm_runtime(self, malcolm_install_path): break if InstallerYesOrNo( - 'Should Malcolm delete the oldest database indices and/or PCAP files based on available storage?' - if ((opensearchPrimaryMode == DatabaseMode.OpenSearchLocal) and (malcolmProfile == PROFILE_MALCOLM)) - else 'Should Arkime delete PCAP files based on available storage (see https://arkime.com/faq#pcap-deletion)?', + ( + 'Should Malcolm delete the oldest database indices and/or PCAP files based on available storage?' + if ((opensearchPrimaryMode == DatabaseMode.OpenSearchLocal) and (malcolmProfile == PROFILE_MALCOLM)) + else 'Should Arkime delete PCAP files based on available storage (see https://arkime.com/faq#pcap-deletion)?' + ), default=args.arkimeManagePCAP or bool(args.indexPruneSizeLimit), ): # delete oldest indexes based on index pattern size @@ -1292,9 +1293,11 @@ def tweak_malcolm_runtime(self, malcolm_install_path): ( x, '', - x == filePreserveModeDefault - if filePreserveModeDefault - else allowedFilePreserveModes[0], + ( + x == filePreserveModeDefault + if filePreserveModeDefault + else allowedFilePreserveModes[0] + ), ) for x in allowedFilePreserveModes ], @@ -1307,9 +1310,11 @@ def tweak_malcolm_runtime(self, malcolm_install_path): 'ZIP downloaded preserved files?', default=args.fileCarveHttpServerZip ) fileCarveHttpServeEncryptKey = InstallerAskForString( - 'Enter ZIP archive password for downloaded preserved files (or leave blank for unprotected)' - if fileCarveHttpServerZip - else 'Enter AES-256-CBC encryption password for downloaded preserved files (or leave blank for unencrypted)', + ( + 'Enter ZIP archive password for downloaded preserved files (or leave blank for unprotected)' + if fileCarveHttpServerZip + else 'Enter AES-256-CBC encryption password for downloaded preserved files (or leave blank for unencrypted)' + ), default=args.fileCarveHttpServeEncryptKey, ) if fileCarveMode is not None: diff --git a/scripts/malcolm_common.py b/scripts/malcolm_common.py index d8e95ded3..dbb31f605 100644 --- a/scripts/malcolm_common.py +++ b/scripts/malcolm_common.py @@ -259,9 +259,11 @@ def AskForString( elif (uiMode & UserInterfaceMode.InteractionDialog) and (MainDialog is not None): code, reply = MainDialog.inputbox( question, - init=default - if (default is not None) and (defaultBehavior & UserInputDefaultsBehavior.DefaultsPrompt) - else "", + init=( + default + if (default is not None) and (defaultBehavior & UserInputDefaultsBehavior.DefaultsPrompt) + else "" + ), ) if (code == Dialog.CANCEL) or (code == Dialog.ESC): raise RuntimeError("Operation cancelled") diff --git a/scripts/third-party-environments/aws/ami/packer_vars.json.example b/scripts/third-party-environments/aws/ami/packer_vars.json.example index 511dfa2b8..6243c5796 100644 --- a/scripts/third-party-environments/aws/ami/packer_vars.json.example +++ b/scripts/third-party-environments/aws/ami/packer_vars.json.example @@ -2,7 +2,7 @@ "aws_access_key": "XXXXXXXXXXXXXXXXXXXX", "aws_secret_key": "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX", "instance_type": "t2.micro", - "malcolm_tag": "v24.02.0", + "malcolm_tag": "v24.03.0", "malcolm_repo": "idaholab/Malcolm", "malcolm_uid": "1000", "ssh_username": "ec2-user", diff --git a/scripts/zeek_script_to_malcolm_boilerplate.py b/scripts/zeek_script_to_malcolm_boilerplate.py index f254099e0..d93323d77 100755 --- a/scripts/zeek_script_to_malcolm_boilerplate.py +++ b/scripts/zeek_script_to_malcolm_boilerplate.py @@ -487,24 +487,26 @@ def main(): f' # {rName}.log', f' # {os.path.basename(val)} ({args.url})', '', - f' dissect {{', - f' id => "dissect_zeek_{rName}"', - f" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP", - f' mapping => {{', - f' "[message]" => "{rFieldsDissect}"', - f' }}', - f' }}', - '', - f' if ("_dissectfailure" in [tags]) {{', - f' mutate {{', - f' id => "mutate_split_zeek_{rName}"', + f' if ("_jsonparsesuccess" not in [tags]) {{', + f' dissect {{', + f' id => "dissect_zeek_{rName}"', f" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP", - f' split => {{ "[message]" => "{ZEEK_DELIMITER_CHAR}" }}', + f' mapping => {{', + f' "[message]" => "{rFieldsDissect}"', + f' }}', f' }}', - f' ruby {{', - f' id => "ruby_zip_zeek_{rName}"', - f' init => "$zeek_{rName}_field_names = [ {rFieldsZip} ]"', - f" code => \"event.set('[zeek_cols]', $zeek_{rName}_field_names.zip(event.get('[message]')).to_h)\"", + '', + f' if ("_dissectfailure" in [tags]) {{', + f' mutate {{', + f' id => "mutate_split_zeek_{rName}"', + f" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP", + f' split => {{ "[message]" => "{ZEEK_DELIMITER_CHAR}" }}', + f' }}', + f' ruby {{', + f' id => "ruby_zip_zeek_{rName}"', + f' init => "$zeek_{rName}_field_names = [ {rFieldsZip} ]"', + f" code => \"event.set('[zeek_cols]', $zeek_{rName}_field_names.zip(event.get('[message]')).to_h)\"", + f' }}', f' }}', f' }}', '', diff --git a/shared/bin/zeek_install_plugins.sh b/shared/bin/zeek_install_plugins.sh index 70ce9b01f..e521a08e5 100755 --- a/shared/bin/zeek_install_plugins.sh +++ b/shared/bin/zeek_install_plugins.sh @@ -76,6 +76,7 @@ ZKG_GITHUB_URLS=( "https://github.com/cisagov/icsnpp-genisys" "https://github.com/cisagov/icsnpp-modbus" "https://github.com/cisagov/icsnpp-opcua-binary" + "https://github.com/cisagov/icsnpp-profinet-io-cm" "https://github.com/cisagov/icsnpp-s7comm" "https://github.com/cisagov/icsnpp-synchrophasor" "https://github.com/corelight/callstranger-detector" diff --git a/shared/bin/zeek_intel_from_threat_feed.py b/shared/bin/zeek_intel_from_threat_feed.py index 456387238..f893a10d7 100755 --- a/shared/bin/zeek_intel_from_threat_feed.py +++ b/shared/bin/zeek_intel_from_threat_feed.py @@ -31,7 +31,7 @@ def main(): [ 'Outputs a Zeek intelligence framework file from various formats used to represent threat information:', ' - "Indicator" objects in STIX™ v2.0/v2.1 JSON files', - ' - MISP core format JSON files', + ' - MISP attributes or core format JSON files', '', 'See:', ' - Malcolm documentation: https://idaholab.github.io/Malcolm/docs/zeek-intel.html#ZeekIntel', @@ -44,6 +44,7 @@ def main(): ' - MISP communities: https://www.misp-project.org/communities/', ' - MISP default feeds: https://www.misp-project.org/feeds/', ' - Managing MISP feeds: https://misp.gitbooks.io/misp-book/content/managing-feeds/', + ' - Expand MISP usage: https://github.com/idaholab/Malcolm/issues/336', '', 'Note: The Zeek intelligence framework only supports simple indicators matched against a single value.', 'The STIX™ standard can express more complex indicators that cannot be expressed with Zeek intelligence items.', @@ -53,30 +54,40 @@ def main(): add_help=False, usage='{} '.format(script_name), ) - parser.add_argument('--verbose', '-v', action='count', default=1, help='Increase verbosity (e.g., -v, -vv, etc.)') parser.add_argument( - '--notice', dest='notice', action='store_true', help='Add fields for policy/frameworks/intel/do_notice.zeek' + '--verbose', + '-v', + action='count', + default=1, + help='Increase verbosity (e.g., -v, -vv, etc.)', ) parser.add_argument( - '--no-notice', + '--notice', dest='notice', - action='store_false', - help='Do not add fields for policy/frameworks/intel/do_notice.zeek', + type=malcolm_utils.str2bool, + nargs='?', + const=True, + default=True, + help='Add fields for policy/frameworks/intel/do_notice.zeek', ) - parser.set_defaults(notice=True) parser.add_argument( '--cif', dest='cif', - action='store_true', + type=malcolm_utils.str2bool, + nargs='?', + const=True, + default=True, help='Add fields for policy/integration/collective-intel/main.zeek', ) parser.add_argument( - '--no-cif', - dest='cif', - action='store_false', - help='Do not add fields for policy/integration/collective-intel/main.zeek', + '--ssl-verify', + dest='sslVerify', + type=malcolm_utils.str2bool, + nargs='?', + const=True, + default=False, + help='Require TLS connections to verify certificates', ) - parser.set_defaults(cif=True) parser.add_argument( '-i', '--input', @@ -161,6 +172,7 @@ def main(): with malcolm_utils.temporary_filename(suffix='.txt') as tmpFileName: dlFileName = zeek_threat_feed_utils.download_to_file( infileArg, + ssl_verify=args.sslVerify, local_filename=tmpFileName, logger=logging, ) @@ -190,6 +202,7 @@ def main(): inputQueue, zeekPrinter, since, + args.sslVerify, defaultNow, workerThreadCount, logging, diff --git a/shared/bin/zeek_intel_setup.sh b/shared/bin/zeek_intel_setup.sh index d4a8ad80b..f2309e352 100755 --- a/shared/bin/zeek_intel_setup.sh +++ b/shared/bin/zeek_intel_setup.sh @@ -15,6 +15,7 @@ SCRIPT_FILESPEC="$(realpath -e "${BASH_SOURCE[0]}")" ZEEK_DIR=${ZEEK_DIR:-"/opt/zeek"} ZEEK_INTEL_ITEM_EXPIRATION=${ZEEK_INTEL_ITEM_EXPIRATION:-"-1min"} ZEEK_INTEL_FEED_SINCE=${ZEEK_INTEL_FEED_SINCE:-""} +ZEEK_INTEL_FEED_SSL_CERTIFICATE_VERIFICATION=${ZEEK_INTEL_FEED_SSL_CERTIFICATE_VERIFICATION:-false} ZEEK_INTEL_REFRESH_THREADS=${ZEEK_INTEL_REFRESH_THREADS:-"2"} INTEL_DIR=${INTEL_DIR:-"${ZEEK_DIR}/share/zeek/site/intel"} INTEL_PRESEED_DIR=${INTEL_PRESEED_DIR:-"${ZEEK_DIR}/share/zeek/site/intel-preseed"} @@ -92,6 +93,7 @@ EOF # process STIX and MISP inputs by converting them to Zeek intel format if ( (( ${#THREAT_JSON_FILES[@]} )) || [[ -r ./STIX/.stix_input.txt ]] || [[ -r ./MISP/.misp_input.txt ]] ) && [[ -x "${THREAT_FEED_TO_ZEEK_SCRIPT}" ]]; then "${THREAT_FEED_TO_ZEEK_SCRIPT}" \ + --ssl-verify ${ZEEK_INTEL_FEED_SSL_CERTIFICATE_VERIFICATION} \ --since "${ZEEK_INTEL_FEED_SINCE}" \ --threads ${ZEEK_INTEL_REFRESH_THREADS} \ --output ./.threat_autogen.zeek.new \ diff --git a/shared/bin/zeek_threat_feed_utils.py b/shared/bin/zeek_threat_feed_utils.py index a7b0c21a7..3716cafe1 100644 --- a/shared/bin/zeek_threat_feed_utils.py +++ b/shared/bin/zeek_threat_feed_utils.py @@ -12,7 +12,7 @@ from contextlib import contextmanager, nullcontext from datetime import datetime from multiprocessing import RawValue -from pymisp import MISPEvent, MISPAttribute +from pymisp import MISPEvent, MISPAttribute, PyMISP from pytz import utc as UTCTimeZone from stix2 import parse as STIXParse from stix2.exceptions import STIXError @@ -30,12 +30,14 @@ from time import sleep, mktime from typing import Tuple, Union from urllib.parse import urljoin, urlparse +from logging import DEBUG as LOGGING_DEBUG import json import os import re import requests +import urllib3 -from malcolm_utils import base64_decode_if_prefixed, LoadStrIfJson, LoadFileIfJson +from malcolm_utils import eprint, base64_decode_if_prefixed, LoadStrIfJson, LoadFileIfJson # keys for dict returned by map_stix_indicator_to_zeek for Zeek intel file fields ZEEK_INTEL_INDICATOR = 'indicator' @@ -53,6 +55,8 @@ TAXII_INDICATOR_FILTER = {'type': 'indicator'} TAXII_PAGE_SIZE = 50 +MISP_PAGE_SIZE_ATTRIBUTES = 500 +MISP_PAGE_SIZE_EVENTS = 10 ZEEK_INTEL_WORKER_THREADS_DEFAULT = 2 @@ -111,6 +115,8 @@ "x509-fingerprint-sha1": "CERT_HASH", } +urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + # get URL directory listing def get_url_paths_from_response(response_text, parent_url='', ext=''): @@ -122,11 +128,11 @@ def get_url_paths_from_response(response_text, parent_url='', ext=''): ] -def get_url_paths(url, session=None, ext='', params={}): +def get_url_paths(url, session=None, ssl_verify=False, ext='', params={}): response = ( - requests.get(url, params=params, allow_redirects=True) + requests.get(url, params=params, allow_redirects=True, verify=ssl_verify) if session is None - else session.get(url, params=params, allow_redirects=True) + else session.get(url, params=params, allow_redirects=True, verify=ssl_verify) ) if response.ok: response_text = response.text @@ -136,12 +142,12 @@ def get_url_paths(url, session=None, ext='', params={}): # download to file -def download_to_file(url, session=None, local_filename=None, chunk_bytes=4096, logger=None): +def download_to_file(url, session=None, local_filename=None, chunk_bytes=4096, ssl_verify=False, logger=None): tmpDownloadedFileSpec = local_filename if local_filename else os.path.basename(urlparse(url).path) r = ( - requests.get(url, stream=True, allow_redirects=True) + requests.get(url, stream=True, allow_redirects=True, verify=ssl_verify) if session is None - else session.get(url, stream=True, allow_redirects=True) + else session.get(url, stream=True, allow_redirects=True, verify=ssl_verify) ) with open(tmpDownloadedFileSpec, "wb") as f: for chunk in r.iter_content(chunk_size=chunk_bytes): @@ -495,78 +501,112 @@ def ProcessMISP( source: Union[Tuple[str], None] = None, url: Union[str, None] = None, ): - try: - event = MISPEvent() - event.from_dict(**toParse) - - if source is None: - source = [] - - if event.Orgc is not None: - source.append(event.Orgc.name) - - description = event.info - - if (event.Tag is not None) and (len(event.Tag) > 0): - tags = [ - x.name - for x in event.Tag - if not x.name.startswith('osint:certainty') - and not x.name.startswith('type:') - and not x.name.startswith('source:') - ] - # TODO: 'slice' object is not subscriptable - source.extend([x.name[7:] for x in event.Tag if x.name.startswith('source:')]) - certaintyTags = [x.name.replace('"', '') for x in event.Tag if x.name.startswith('osint:certainty')] - try: - certainty = float(certaintyTags[0].split('=')[-1]) if len(certaintyTags) > 0 else None - except ValueError: - certainty = None - else: + if isinstance(toParse, dict): + try: + attr = None + event = None + description = '' + if source is None: + source = [] tags = [] certainty = None - for attribute in event.attributes: - # map event attribute to Zeek value(s) - if ( - ((not hasattr(attribute, 'deleted')) or (not attribute.deleted)) - and ((self.since is None) or (event.timestamp >= self.since) or (attribute.timestamp >= self.since)) - and ( - vals := map_misp_attribute_to_zeek( - attribute=attribute, - source=source, - url=url, - description=f"{description}{'. '+attribute.comment if attribute.comment else ''}", - tags=tags, - confidence=certainty, - logger=self.logger, - ) - ) + # determine if we're processing an event or an attribute + if (('Event' in toParse) and isinstance(toParse['Event'], dict) and ('info' in toParse['Event'])) or ( + 'info' in toParse ): - for val in vals: - self.PrintHeader() - with self.lock: - # print the intelligence item fields according to the columns in 'fields' - print('\t'.join([val[key] for key in self.fields]), file=self.outFile) + # this is an event, which may contain an array of attributes + event = MISPEvent() + event.from_dict(**toParse) + + elif ('id' in toParse) and ('type' in toParse): + # processing a single attribute + attr = MISPAttribute() + attr.from_dict(**toParse) + event = MISPEvent() + event.from_dict(**attr.Event) + + if attr or event: + if event: + # format the descriptive info for the Zeek intel item + if hasattr(event, 'Orgc') and event.Orgc: + source.append(event.Orgc.name) + elif hasattr(event, 'orgc') and event.orgc: + source.append(event.orgc.name) + + if hasattr(event, 'info') and event.info: + description = event.info + + if hasattr(event, 'Tag') and (event.Tag is not None) and (len(event.Tag) > 0): + tags = [ + x.name + for x in event.Tag + if not x.name.startswith('osint:certainty') + and not x.name.startswith('type:') + and not x.name.startswith('source:') + ] + source.extend([x.name[7:] for x in event.Tag if x.name.startswith('source:')]) + certaintyTags = [ + x.name.replace('"', '') for x in event.Tag if x.name.startswith('osint:certainty') + ] + try: + certainty = float(certaintyTags[0].split('=')[-1]) if len(certaintyTags) > 0 else None + except ValueError: + certainty = None + + # loop through and process the attribute(s) + for attribute in [attr] if attr else event.attributes: + # map attribute to Zeek value(s) + if ( + ((not hasattr(attribute, 'deleted')) or (not attribute.deleted)) + and ( + (self.since is None) + or (event and hasattr(event, 'timestamp') and (event.timestamp >= self.since)) + or (attribute and hasattr(attribute, 'timestamp') and attribute.timestamp >= self.since) + ) + and ( + vals := map_misp_attribute_to_zeek( + attribute=attribute, + source=source, + url=url, + description=f"{description}{'. '+attribute.comment if (hasattr(attribute, 'comment') and attribute.comment) else ''}", + tags=tags, + confidence=certainty, + logger=self.logger, + ) + ) + ): + for val in vals: + self.PrintHeader() + with self.lock: + # print the intelligence item fields according to the columns in 'fields' + print('\t'.join([val[key] for key in self.fields]), file=self.outFile) - except Exception as e: - if self.logger is not None: - self.logger.warning(e, exc_info=True) + elif self.logger is not None: + self.logger.warning("Unknown MISP object format (could not determine Attribute vs. Event)") + + except Exception as e: + if self.logger is not None: + self.logger.warning(e, exc_info=True) + + elif self.logger is not None: + self.logger.warning(f"Unknown MISP object format ('{type(toParse)}')") def ProcessThreatInputWorker(threatInputWorkerArgs): - inputQueue, zeekPrinter, since, defaultNow, workerThreadCount, logger = ( + inputQueue, zeekPrinter, since, sslVerify, defaultNow, workerThreadCount, logger = ( threatInputWorkerArgs[0], threatInputWorkerArgs[1], threatInputWorkerArgs[2], threatInputWorkerArgs[3], threatInputWorkerArgs[4], threatInputWorkerArgs[5], + threatInputWorkerArgs[6], ) with workerThreadCount as workerId: if logger is not None: - logger.debug(f"[{workerId}]:started") + logger.debug(f"[{workerId}]: started") # the queue was fully populated before we started, so we can run until there are no more elements while len(inputQueue) > 0: @@ -582,21 +622,22 @@ def ProcessThreatInputWorker(threatInputWorkerArgs): # JSON FILE (STIX or MISP) if infileJson := LoadFileIfJson(infile): - if 'type' in infileJson and 'id' in infileJson: - # STIX input file - zeekPrinter.ProcessSTIX( - infileJson, - source=[os.path.splitext(os.path.basename(inarg))[0]], - ) - - elif (len(infileJson.keys()) == 1) and ('Event' in infileJson): - # TODO: is this always the case? anything other than "Event", or multiple objects? - # MISP input file - zeekPrinter.ProcessMISP( - infileJson, - source=[os.path.splitext(os.path.basename(inarg))[0]], - ) + if isinstance(infileJson, dict): + if 'type' in infileJson and 'id' in infileJson: + # STIX input file + zeekPrinter.ProcessSTIX( + infileJson, + source=[os.path.splitext(os.path.basename(inarg))[0]], + ) + elif (len(infileJson.keys()) == 1) and ('Event' in infileJson): + # MISP input file containing "Event" + zeekPrinter.ProcessMISP( + infileJson, + source=[os.path.splitext(os.path.basename(inarg))[0]], + ) + else: + raise Exception(f"Could not identify content in '{inarg}'") else: raise Exception(f"Could not identify content in '{inarg}'") else: @@ -619,6 +660,7 @@ def ProcessThreatInputWorker(threatInputWorkerArgs): mispAuthKey = mispConnInfo[1] with requests.Session() as mispSession: + mispSession.headers.update({'Accept': 'application/json, text/plain, text/html'}) if mispAuthKey is not None: mispSession.headers.update({'Authorization': mispAuthKey}) @@ -626,20 +668,127 @@ def ProcessThreatInputWorker(threatInputWorkerArgs): # - a manifest JSON (https://www.circl.lu/doc/misp/feed-osint/manifest.json) # - a directory listing *containing* a manifest.json (https://www.circl.lu/doc/misp/feed-osint/) # - a directory listing of misc. JSON files without a manifest.json - mispResponse = mispSession.get(mispUrl) + # - an array of Attributes returned for a request via the MISP Automation API to an /attributes endpoint + # - an array of Events returned for a request via the MISP Automation API to an /events endpoint + mispResponse = mispSession.get( + mispUrl, + allow_redirects=True, + verify=sslVerify, + ) mispResponse.raise_for_status() if mispJson := LoadStrIfJson(mispResponse.content): - # the contents are JSON. determine if this is a manifest or a single event - - if (len(mispJson.keys()) == 1) and ('Event' in mispJson): - # TODO: is this always the case? anything other than "Event", or multiple objects? - # this is a MISP event, process it + # the contents are JSON. determine if this is: + # - a single Event + # - an array of Events + # - an array of Attributes + # - a manifest + + if ( + isinstance(mispJson, dict) + and (len(mispJson.keys()) == 1) + and ('Event' in mispJson) + ): + # this is a single MISP Event, process it zeekPrinter.ProcessMISP( mispJson, url=mispUrl, ) - else: + elif isinstance(mispJson, list) and (len(mispJson) > 0): + # are these Attributes or Events? + if ( + isinstance(mispJson[0], dict) + and ('id' in mispJson[0]) + and ('type' in mispJson[0]) + ): + controllerType = 'attributes' + resultKey = 'Attribute' + pageSize = MISP_PAGE_SIZE_ATTRIBUTES + elif isinstance(mispJson[0], dict) and ('info' in mispJson[0]): + controllerType = 'events' + resultKey = 'Event' + pageSize = MISP_PAGE_SIZE_EVENTS + else: + controllerType = None + resultKey = None + pageSize = None + + if controllerType: + # this is an array of either Attributes or Events. + # rather than handling it via additional calls with request, + # let's use the MISP API to do the searching/pulling + # (yeah, we're duplicating the effort of pulling the + # first page, but meh, who cares?) + if mispObject := PyMISP( + mispUrl, + mispAuthKey, + sslVerify, + debug=logger and (LOGGING_DEBUG >= logger.root.level), + ): + # search, looping over the pages pageSize at a time + mispPage = 0 + while True: + mispPage += 1 + resultCount = 0 + mispResults = mispObject.search( + controller=controllerType, + return_format='json', + limit=pageSize, + page=mispPage, + type_attribute=list(MISP_ZEEK_INTEL_TYPE_MAP.keys()), + timestamp=since, + ) + if ( + mispResults + and isinstance(mispResults, dict) + and (resultKey in mispResults) + ): + # Attributes results + resultCount = len(mispResults[resultKey]) + for item in mispResults[resultKey]: + try: + zeekPrinter.ProcessMISP( + item, + url=mispUrl, + ) + except Exception as e: + if logger is not None: + logger.warning( + f"[{workerId}]: {type(e).__name__} for MISP {resultKey}: {e}" + ) + + elif mispResults and isinstance(mispResults, list): + # Events results + resultCount = len(mispResults) + for item in mispResults: + if item and isinstance(item, dict) and (resultKey in item): + try: + zeekPrinter.ProcessMISP( + item[resultKey], + url=mispUrl, + ) + except Exception as e: + if logger is not None: + logger.warning( + f"[{workerId}]: {type(e).__name__} for MISP {resultKey}: {e}" + ) + + else: + # error or unrecognized results, set this to short circuit + resultCount = 0 + + if logger is not None: + logger.debug( + f"[{workerId}]: MISP search page {mispPage} returned {resultCount}" + ) + if not mispResults or (resultCount < pageSize): + break + + else: + # not an Event or an Attribute? what the heck are we even doing? + raise Exception(f"Unknown MISP object '{json.dumps(mispJson)}'") + + elif isinstance(mispJson, dict): # this is a manifest, loop over, retrieve and process the MISP events it references for uri in mispJson: try: @@ -652,7 +801,11 @@ def ProcessThreatInputWorker(threatInputWorkerArgs): else defaultNow ) if (since is None) or (eventTime >= since): - mispObjectReponse = mispSession.get(newUrl) + mispObjectReponse = mispSession.get( + newUrl, + allow_redirects=True, + verify=sslVerify, + ) mispObjectReponse.raise_for_status() zeekPrinter.ProcessMISP( mispObjectReponse.json(), @@ -661,9 +814,12 @@ def ProcessThreatInputWorker(threatInputWorkerArgs): except Exception as e: if logger is not None: logger.warning( - f"{type(e).__name__} for MISP object at '{newUrl}': {e}" + f"[{workerId}]: {type(e).__name__} for MISP object at '{newUrl}': {e}" ) + else: + raise Exception(f"Unknown MISP format '{type(mispJson)}'") + else: # the contents are NOT JSON, it's probably an HTML-formatted directory listing @@ -679,7 +835,11 @@ def ProcessThreatInputWorker(threatInputWorkerArgs): # retrieve it, then loop over it and retrieve and process the MISP events it references for url in manifestPaths: try: - mispManifestResponse = mispSession.get(url) + mispManifestResponse = mispSession.get( + url, + allow_redirects=True, + verify=sslVerify, + ) mispManifestResponse.raise_for_status() mispManifest = mispManifestResponse.json() for uri in mispManifest: @@ -693,7 +853,11 @@ def ProcessThreatInputWorker(threatInputWorkerArgs): ) if (since is None) or (eventTime >= since): newUrl = f'{mispUrl.strip("/")}/{uri}.json' - mispObjectReponse = mispSession.get(newUrl) + mispObjectReponse = mispSession.get( + newUrl, + allow_redirects=True, + verify=sslVerify, + ) mispObjectReponse.raise_for_status() zeekPrinter.ProcessMISP( mispObjectReponse.json(), @@ -702,18 +866,24 @@ def ProcessThreatInputWorker(threatInputWorkerArgs): except Exception as e: if logger is not None: logger.warning( - f"{type(e).__name__} for MISP object at '{mispUrl}/{uri}.json': {e}" + f"[{workerId}]: {type(e).__name__} for MISP object at '{mispUrl}/{uri}.json': {e}" ) except Exception as e: if logger is not None: - logger.warning(f"{type(e).__name__} for manifest at '{url}': {e}") + logger.warning( + f"[{workerId}]: {type(e).__name__} for manifest at '{url}': {e}" + ) else: # the manifest.json does not exist! # just loop over, retrieve and process the .json files in this directory for url in paths: try: - mispObjectReponse = mispSession.get(url) + mispObjectReponse = mispSession.get( + url, + allow_redirects=True, + verify=sslVerify, + ) mispObjectReponse.raise_for_status() zeekPrinter.ProcessMISP( mispObjectReponse.json(), @@ -722,7 +892,7 @@ def ProcessThreatInputWorker(threatInputWorkerArgs): except Exception as e: if logger is not None: logger.warning( - f"{type(e).__name__} for MISP object at '{url}': {e}" + f"[{workerId}]: {type(e).__name__} for MISP object at '{url}': {e}" ) elif inarg.lower().startswith('taxii'): @@ -755,9 +925,13 @@ def ProcessThreatInputWorker(threatInputWorkerArgs): # connect to the server with the appropriate API for the TAXII version if taxiiVersion == '2.0': - server = TaxiiServer_v20(taxiiDisoveryURL, user=taxiiUsername, password=taxiiPassword) + server = TaxiiServer_v20( + taxiiDisoveryURL, user=taxiiUsername, password=taxiiPassword, verify=sslVerify + ) elif taxiiVersion == '2.1': - server = TaxiiServer_v21(taxiiDisoveryURL, user=taxiiUsername, password=taxiiPassword) + server = TaxiiServer_v21( + taxiiDisoveryURL, user=taxiiUsername, password=taxiiPassword, verify=sslVerify + ) else: raise Exception(f"Unsupported TAXII version '{taxiiVersion}'") @@ -776,9 +950,13 @@ def ProcessThreatInputWorker(threatInputWorkerArgs): # connect to and retrieve indicator STIX objects from the collection URL(s) for title, info in collectionUrls.items(): collection = ( - TaxiiCollection_v21(info['url'], user=taxiiUsername, password=taxiiPassword) + TaxiiCollection_v21( + info['url'], user=taxiiUsername, password=taxiiPassword, verify=sslVerify + ) if taxiiVersion == '2.1' - else TaxiiCollection_v20(info['url'], user=taxiiUsername, password=taxiiPassword) + else TaxiiCollection_v20( + info['url'], user=taxiiUsername, password=taxiiPassword, verify=sslVerify + ) ) try: # loop over paginated results @@ -802,11 +980,13 @@ def ProcessThreatInputWorker(threatInputWorkerArgs): except Exception as e: if logger is not None: - logger.warning(f"{type(e).__name__} for object of collection '{title}': {e}") + logger.warning( + f"[{workerId}]: {type(e).__name__} for object of collection '{title}': {e}" + ) except Exception as e: if logger is not None: - logger.warning(f"{type(e).__name__} for '{inarg}': {e}") + logger.warning(f"[{workerId}]: {type(e).__name__} for '{inarg}': {e}") if logger is not None: logger.debug(f"[{workerId}]: finished") diff --git a/suricata/rules-default/OT/malcolm/CVE-2023-6448_Unitronics_VisiLogic.rules b/suricata/rules-default/OT/malcolm/CVE-2023-6448_Unitronics_VisiLogic.rules new file mode 100644 index 000000000..73744defe --- /dev/null +++ b/suricata/rules-default/OT/malcolm/CVE-2023-6448_Unitronics_VisiLogic.rules @@ -0,0 +1,3 @@ +alert tcp any any -> any 20256 (msg:"Suspected Unitronics Default admin Credential Use"; content:"admin"; content:"1111"; sid:1001020;) +alert tcp any any -> any 20256 (msg:"Suspected Unitronics Default user Credential Use"; content:"user"; content:"1111"; sid:1001021;) +alert tcp any any -> any 20256 (msg:"Suspected Unitronics Default operator Credential Use"; content:"operator"; content:"1111"; sid:1001022;) diff --git a/zeek/config/local.zeek b/zeek/config/local.zeek index a610c6c5a..521c2c087 100644 --- a/zeek/config/local.zeek +++ b/zeek/config/local.zeek @@ -4,46 +4,54 @@ ##! https://docs.zeek.org/en/stable/script-reference/scripts.html ##! https://github.com/zeek/zeek/blob/master/scripts/site/local.zeek -global disable_stats = (getenv("ZEEK_DISABLE_STATS") == "") ? F : T; -global disable_hash_all_files = (getenv("ZEEK_DISABLE_HASH_ALL_FILES") == "") ? F : T; -global disable_log_passwords = (getenv("ZEEK_DISABLE_LOG_PASSWORDS") == "") ? F : T; -global disable_ssl_validate_certs = (getenv("ZEEK_DISABLE_SSL_VALIDATE_CERTS") == "") ? F : T; -global disable_track_all_assets = (getenv("ZEEK_DISABLE_TRACK_ALL_ASSETS") == "") ? F : T; -global disable_best_guess_ics = (getenv("ZEEK_DISABLE_BEST_GUESS_ICS") == "") ? F : T; -global synchrophasor_detailed = (getenv("ZEEK_SYNCHROPHASOR_DETAILED") == "") ? F : T; +global true_regex: pattern = /^\s*(?i:t(rue)?|y(es)?|on|1)\s*$/; + +global disable_stats = (getenv("ZEEK_DISABLE_STATS") == true_regex) ? T : F; +global disable_hash_all_files = (getenv("ZEEK_DISABLE_HASH_ALL_FILES") == true_regex) ? T : F; +global disable_log_passwords = (getenv("ZEEK_DISABLE_LOG_PASSWORDS") == true_regex) ? T : F; +global disable_ssl_validate_certs = (getenv("ZEEK_DISABLE_SSL_VALIDATE_CERTS") == true_regex) ? T : F; +global disable_track_all_assets = (getenv("ZEEK_DISABLE_TRACK_ALL_ASSETS") == true_regex) ? T : F; +global disable_best_guess_ics = (getenv("ZEEK_DISABLE_BEST_GUESS_ICS") == true_regex) ? T : F; +global synchrophasor_detailed = (getenv("ZEEK_SYNCHROPHASOR_DETAILED") == true_regex) ? T : F; global synchrophasor_ports_str = getenv("ZEEK_SYNCHROPHASOR_PORTS"); global genisys_ports_str = getenv("ZEEK_GENISYS_PORTS"); global enip_ports_str = getenv("ZEEK_ENIP_PORTS"); global zeek_local_nets_str = getenv("ZEEK_LOCAL_NETS"); -global disable_spicy_dhcp = (getenv("ZEEK_DISABLE_SPICY_DHCP") == "") ? F : T; -global disable_spicy_dns = (getenv("ZEEK_DISABLE_SPICY_DNS") == "") ? F : T; -global disable_spicy_http = (getenv("ZEEK_DISABLE_SPICY_HTTP") == "") ? F : T; -global disable_spicy_ipsec = (getenv("ZEEK_DISABLE_SPICY_IPSEC") == "") ? F : T; -global disable_spicy_ldap = (getenv("ZEEK_DISABLE_SPICY_LDAP") == "") ? F : T; -global disable_spicy_openvpn = (getenv("ZEEK_DISABLE_SPICY_OPENVPN") == "") ? F : T; -global disable_spicy_quic = (getenv("ZEEK_DISABLE_SPICY_QUIC") == "") ? F : T; -global disable_spicy_stun = (getenv("ZEEK_DISABLE_SPICY_STUN") == "") ? F : T; -global disable_spicy_tailscale = (getenv("ZEEK_DISABLE_SPICY_TAILSCALE") == "") ? F : T; -global disable_spicy_tftp = (getenv("ZEEK_DISABLE_SPICY_TFTP") == "") ? F : T; -global disable_spicy_wireguard = (getenv("ZEEK_DISABLE_SPICY_WIREGUARD") == "") ? F : T; +global disable_spicy_dhcp = (getenv("ZEEK_DISABLE_SPICY_DHCP") == true_regex) ? T : F; +global disable_spicy_dns = (getenv("ZEEK_DISABLE_SPICY_DNS") == true_regex) ? T : F; +global disable_spicy_http = (getenv("ZEEK_DISABLE_SPICY_HTTP") == true_regex) ? T : F; +global disable_spicy_ipsec = (getenv("ZEEK_DISABLE_SPICY_IPSEC") == true_regex) ? T : F; +global disable_spicy_ldap = (getenv("ZEEK_DISABLE_SPICY_LDAP") == true_regex) ? T : F; +global disable_spicy_openvpn = (getenv("ZEEK_DISABLE_SPICY_OPENVPN") == true_regex) ? T : F; +global disable_spicy_quic = (getenv("ZEEK_DISABLE_SPICY_QUIC") == true_regex) ? T : F; +global disable_spicy_stun = (getenv("ZEEK_DISABLE_SPICY_STUN") == true_regex) ? T : F; +global disable_spicy_tailscale = (getenv("ZEEK_DISABLE_SPICY_TAILSCALE") == true_regex) ? T : F; +global disable_spicy_tftp = (getenv("ZEEK_DISABLE_SPICY_TFTP") == true_regex) ? T : F; +global disable_spicy_wireguard = (getenv("ZEEK_DISABLE_SPICY_WIREGUARD") == true_regex) ? T : F; -global disable_ics_all = (getenv("ZEEK_DISABLE_ICS_ALL") == "") ? F : T; -global disable_ics_bacnet = (getenv("ZEEK_DISABLE_ICS_BACNET") == "") ? F : T; -global disable_ics_bsap = (getenv("ZEEK_DISABLE_ICS_BSAP") == "") ? F : T; -global disable_ics_dnp3 = (getenv("ZEEK_DISABLE_ICS_DNP3") == "") ? F : T; -global disable_ics_enip = (getenv("ZEEK_DISABLE_ICS_ENIP") == "") ? F : T; -global disable_ics_ethercat = (getenv("ZEEK_DISABLE_ICS_ETHERCAT") == "") ? F : T; -global disable_ics_genisys = (getenv("ZEEK_DISABLE_ICS_GENISYS") == "") ? F : T; -global disable_ics_opcua_binary = (getenv("ZEEK_DISABLE_ICS_OPCUA_BINARY") == "") ? F : T; -global disable_ics_modbus = (getenv("ZEEK_DISABLE_ICS_MODBUS") == "") ? F : T; -global disable_ics_profinet = (getenv("ZEEK_DISABLE_ICS_PROFINET") == "") ? F : T; -global disable_ics_s7comm = (getenv("ZEEK_DISABLE_ICS_S7COMM") == "") ? F : T; -global disable_ics_synchrophasor = (getenv("ZEEK_DISABLE_ICS_SYNCHROPHASOR") == "") ? F : T; +global disable_ics_all = (getenv("ZEEK_DISABLE_ICS_ALL") == true_regex) ? T : F; +global disable_ics_bacnet = (getenv("ZEEK_DISABLE_ICS_BACNET") == true_regex) ? T : F; +global disable_ics_bsap = (getenv("ZEEK_DISABLE_ICS_BSAP") == true_regex) ? T : F; +global disable_ics_dnp3 = (getenv("ZEEK_DISABLE_ICS_DNP3") == true_regex) ? T : F; +global disable_ics_enip = (getenv("ZEEK_DISABLE_ICS_ENIP") == true_regex) ? T : F; +global disable_ics_ethercat = (getenv("ZEEK_DISABLE_ICS_ETHERCAT") == true_regex) ? T : F; +global disable_ics_genisys = (getenv("ZEEK_DISABLE_ICS_GENISYS") == true_regex) ? T : F; +global disable_ics_opcua_binary = (getenv("ZEEK_DISABLE_ICS_OPCUA_BINARY") == true_regex) ? T : F; +global disable_ics_modbus = (getenv("ZEEK_DISABLE_ICS_MODBUS") == true_regex) ? T : F; +global disable_ics_profinet = (getenv("ZEEK_DISABLE_ICS_PROFINET") == true_regex) ? T : F; +global disable_ics_profinet_io_cm = (getenv("ZEEK_DISABLE_ICS_PROFINET_IO_CM") == true_regex) ? T : F; +global disable_ics_s7comm = (getenv("ZEEK_DISABLE_ICS_S7COMM") == true_regex) ? T : F; +global disable_ics_synchrophasor = (getenv("ZEEK_DISABLE_ICS_SYNCHROPHASOR") == true_regex) ? T : F; redef Broker::default_listen_address = "127.0.0.1"; redef ignore_checksums = T; +global json_format = (getenv("ZEEK_JSON") == true_regex) ? T : F; +@if (json_format) + redef LogAscii::use_json = T; +@endif + @load tuning/defaults @load frameworks/software/vulnerable @load frameworks/software/version-changes @@ -139,6 +147,9 @@ event zeek_init() &priority=-5 { if (disable_ics_all || disable_ics_profinet) { Analyzer::disable_analyzer(Analyzer::ANALYZER_PROFINET); } + if (disable_ics_all || disable_ics_profinet_io_cm) { + Spicy::disable_protocol_analyzer(Analyzer::ANALYZER_SPICY_PROFINET_IO_CM); + } if (disable_ics_all || disable_ics_s7comm) { Analyzer::disable_analyzer(Analyzer::ANALYZER_S7COMM_TCP); } @@ -290,3 +301,36 @@ redef CVE_2021_44228::log = F; break; } @endif + +##! Other logs we're just disabling unilaterally + +# amzn/zeek-plugin-profinet's profinet_dce_rpc.log is covered by cisagov/icsnpp-profinet-io-cm +hook Profinet::log_policy_profinet_dce_rpc( + rec: Profinet::Profinet_DCE_RPC, + id: Log::ID, + filter: Log::Filter) { + break; +} + +# we're not tracking the BSAP "unknown" logs +hook Bsap::log_policy_bsap_ip_unknown( + rec: Bsap::BSAP_IP_UNKNOWN, + id: Log::ID, + filter: Log::Filter) { + break; +} + +hook Bsap::log_policy_bsap_serial_unknown( + rec: Bsap::BSAP_SERIAL_UNKNOWN, + id: Log::ID, + filter: Log::Filter) { + break; +} + +# all ARP traffic is logged by the ethercat parser, which is overkill +hook PacketAnalyzer::ECAT::log_policy_ecat_arp( + rec: PacketAnalyzer::ECAT::ECAT_ARP_INFO, + id: Log::ID, + filter: Log::Filter) { + break; +}