diff --git a/.github/workflows/hedgehog-iso-build-docker-wrap-push-ghcr.yml b/.github/workflows/hedgehog-iso-build-docker-wrap-push-ghcr.yml index f88e9a402..6a16d715e 100644 --- a/.github/workflows/hedgehog-iso-build-docker-wrap-push-ghcr.yml +++ b/.github/workflows/hedgehog-iso-build-docker-wrap-push-ghcr.yml @@ -129,6 +129,7 @@ jobs: echo "${{ steps.extract_malcolm_version.outputs.mversion }}" > ./shared/version.txt echo "${{ secrets.MAXMIND_GEOIP_DB_LICENSE_KEY }}" > ./shared/maxmind_license.txt echo "${{ secrets.MAXMIND_GEOIP_DB_ALTERNATE_DOWNLOAD_URL }}" > ./shared/maxmind_url.txt + echo "${{ secrets.ZEEK_DEB_ALTERNATE_DOWNLOAD_URL }}" > ./shared/zeek_url.txt echo "GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }}" > ./shared/environment.chroot echo "VCS_REVSION=${{ steps.extract_commit_sha.outputs.sha }}" > ./shared/environment.chroot echo "BUILD_JOBS=2" > ./shared/environment.chroot diff --git a/.github/workflows/hedgehog-raspi-build-docker-wrap-push-ghcr.yml b/.github/workflows/hedgehog-raspi-build-docker-wrap-push-ghcr.yml index 0d879c261..21c46dbfd 100644 --- a/.github/workflows/hedgehog-raspi-build-docker-wrap-push-ghcr.yml +++ b/.github/workflows/hedgehog-raspi-build-docker-wrap-push-ghcr.yml @@ -86,6 +86,7 @@ jobs: echo "${{ steps.extract_malcolm_version.outputs.mversion }}" > ./shared/version.txt echo "${{ secrets.MAXMIND_GEOIP_DB_LICENSE_KEY }}" > ./shared/maxmind_license.txt echo "${{ secrets.MAXMIND_GEOIP_DB_ALTERNATE_DOWNLOAD_URL }}" > ./shared/maxmind_url.txt + echo "${{ secrets.ZEEK_DEB_ALTERNATE_DOWNLOAD_URL }}" > ./shared/zeek_url.txt echo "GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }}" > ./shared/environment.chroot echo "VCS_REVSION=${{ steps.extract_commit_sha.outputs.sha }}" > ./shared/environment.chroot echo "BUILD_JOBS=2" > ./shared/environment.chroot diff --git a/.github/workflows/zeek-build-and-push-ghcr.yml b/.github/workflows/zeek-build-and-push-ghcr.yml index 4cec0de26..26fac60b3 100644 --- a/.github/workflows/zeek-build-and-push-ghcr.yml +++ b/.github/workflows/zeek-build-and-push-ghcr.yml @@ -105,6 +105,7 @@ jobs: MALCOLM_VERSION=${{ steps.extract_malcolm_version.outputs.mversion }} BUILD_DATE=${{ steps.generate_build_timestamp.outputs.btimestamp }} VCS_REVISION=${{ steps.extract_commit_sha.outputs.sha }} + ZEEK_DEB_ALTERNATE_DOWNLOAD_URL=${{ secrets.ZEEK_DEB_ALTERNATE_DOWNLOAD_URL }} push: true provenance: false platforms: ${{ matrix.platform }} diff --git a/Dockerfiles/arkime.Dockerfile b/Dockerfiles/arkime.Dockerfile index 805af9732..3fc52cab6 100644 --- a/Dockerfiles/arkime.Dockerfile +++ b/Dockerfiles/arkime.Dockerfile @@ -149,7 +149,7 @@ RUN export DEBARCH=$(dpkg --print-architecture) && \ mkdir -p "${ARKIME_DIR}"/plugins && \ curl -fsSL -o "${ARKIME_DIR}/plugins/ja4plus.${DEBARCH}.so" "$(echo "${ARKIME_JA4_SO_URL}" | sed "s/XXX/${DEBARCH}/g")" && \ chmod 755 "${ARKIME_DIR}/plugins/ja4plus.${DEBARCH}.so" && \ - python3 -m pip install --break-system-packages --no-compile --no-cache-dir beautifulsoup4 pyzmq watchdog==5.0.2 && \ + python3 -m pip install --break-system-packages --no-compile --no-cache-dir beautifulsoup4 pyzmq watchdog==5.0.3 && \ ln -sfr $ARKIME_DIR/bin/npm /usr/local/bin/npm && \ ln -sfr $ARKIME_DIR/bin/node /usr/local/bin/node && \ ln -sfr $ARKIME_DIR/bin/npx /usr/local/bin/npx && \ diff --git a/Dockerfiles/dashboards.Dockerfile b/Dockerfiles/dashboards.Dockerfile index 194f8b952..a9cc67d59 100644 --- a/Dockerfiles/dashboards.Dockerfile +++ b/Dockerfiles/dashboards.Dockerfile @@ -1,4 +1,4 @@ -FROM opensearchproject/opensearch-dashboards:2.17.0 +FROM opensearchproject/opensearch-dashboards:2.17.1 LABEL maintainer="malcolm@inl.gov" LABEL org.opencontainers.image.authors='malcolm@inl.gov' @@ -23,7 +23,7 @@ ENV TERM xterm ENV TINI_VERSION v0.19.0 ENV TINI_URL https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini -ENV OSD_TRANSFORM_VIS_VERSION 2.16.0 +ENV OSD_TRANSFORM_VIS_VERSION 2.17.1 ARG NODE_OPTIONS="--max_old_space_size=4096" ENV NODE_OPTIONS $NODE_OPTIONS @@ -42,10 +42,10 @@ RUN export BINARCH=$(uname -m | sed 's/x86_64/amd64/' | sed 's/aarch64/arm64/') # Malcolm manages authentication and encryption via NGINX reverse proxy /usr/share/opensearch-dashboards/bin/opensearch-dashboards-plugin remove securityDashboards --allow-root && \ cd /tmp && \ - unzip transformVis.zip opensearch-dashboards/transformVis/opensearch_dashboards.json opensearch-dashboards/transformVis/package.json && \ - sed -i "s/2\.16\.0/2\.17\.0/g" opensearch-dashboards/transformVis/opensearch_dashboards.json && \ - sed -i "s/2\.16\.0/2\.17\.0/g" opensearch-dashboards/transformVis/package.json && \ - zip transformVis.zip opensearch-dashboards/transformVis/opensearch_dashboards.json opensearch-dashboards/transformVis/package.json && \ + # unzip transformVis.zip opensearch-dashboards/transformVis/opensearch_dashboards.json opensearch-dashboards/transformVis/package.json && \ + # sed -i "s/2\.16\.0/2\.17\.0/g" opensearch-dashboards/transformVis/opensearch_dashboards.json && \ + # sed -i "s/2\.16\.0/2\.17\.0/g" opensearch-dashboards/transformVis/package.json && \ + # zip transformVis.zip opensearch-dashboards/transformVis/opensearch_dashboards.json opensearch-dashboards/transformVis/package.json && \ cd /usr/share/opensearch-dashboards/plugins && \ /usr/share/opensearch-dashboards/bin/opensearch-dashboards-plugin install file:///tmp/transformVis.zip --allow-root && \ rm -rf /tmp/transformVis /tmp/opensearch-dashboards && \ diff --git a/Dockerfiles/file-monitor.Dockerfile b/Dockerfiles/file-monitor.Dockerfile index 9d32be117..e743951a0 100644 --- a/Dockerfiles/file-monitor.Dockerfile +++ b/Dockerfiles/file-monitor.Dockerfile @@ -159,7 +159,7 @@ RUN export BINARCH=$(uname -m | sed 's/x86_64/amd64/' | sed 's/aarch64/arm64/') python-magic \ stream-zip \ supervisor \ - watchdog==5.0.2 \ + watchdog==5.0.3 \ yara-python && \ curl -fsSL -o /usr/local/bin/supercronic "${SUPERCRONIC_URL}${BINARCH}" && \ chmod +x /usr/local/bin/supercronic && \ diff --git a/Dockerfiles/filebeat.Dockerfile b/Dockerfiles/filebeat.Dockerfile index f9756e727..fc2a15e76 100644 --- a/Dockerfiles/filebeat.Dockerfile +++ b/Dockerfiles/filebeat.Dockerfile @@ -1,4 +1,4 @@ -FROM docker.elastic.co/beats/filebeat-oss:8.15.1 +FROM docker.elastic.co/beats/filebeat-oss:8.15.2 # Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. LABEL maintainer="malcolm@inl.gov" @@ -108,7 +108,7 @@ RUN export EVTXARCH=$(uname -m | sed 's/arm64/aarch64/') && \ unzip \ xz-utils && \ ln -s -f -r /usr/bin/python3.9 /usr/bin/python3 && \ - python3.9 -m pip install --no-compile --no-cache-dir patool entrypoint2 pyunpack python-magic ordered-set supervisor watchdog==5.0.2 && \ + python3.9 -m pip install --no-compile --no-cache-dir patool entrypoint2 pyunpack python-magic ordered-set supervisor watchdog==5.0.3 && \ curl -fsSL -o /usr/local/bin/supercronic "${SUPERCRONIC_URL}${BINARCH}" && \ chmod +x /usr/local/bin/supercronic && \ curl -fsSL -o /usr/local/bin/yq "${YQ_URL}${BINARCH}" && \ diff --git a/Dockerfiles/logstash.Dockerfile b/Dockerfiles/logstash.Dockerfile index 75e476f5f..89fde5ef9 100644 --- a/Dockerfiles/logstash.Dockerfile +++ b/Dockerfiles/logstash.Dockerfile @@ -1,4 +1,4 @@ -FROM docker.elastic.co/logstash/logstash-oss:8.15.1 +FROM docker.elastic.co/logstash/logstash-oss:8.15.2 LABEL maintainer="malcolm@inl.gov" LABEL org.opencontainers.image.authors='malcolm@inl.gov' diff --git a/Dockerfiles/opensearch.Dockerfile b/Dockerfiles/opensearch.Dockerfile index c2cefd2f7..78a030676 100644 --- a/Dockerfiles/opensearch.Dockerfile +++ b/Dockerfiles/opensearch.Dockerfile @@ -1,4 +1,4 @@ -FROM opensearchproject/opensearch:2.17.0 +FROM opensearchproject/opensearch:2.17.1 # Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. LABEL maintainer="malcolm@inl.gov" diff --git a/Dockerfiles/pcap-monitor.Dockerfile b/Dockerfiles/pcap-monitor.Dockerfile index f7d46708d..8e3cd1492 100644 --- a/Dockerfiles/pcap-monitor.Dockerfile +++ b/Dockerfiles/pcap-monitor.Dockerfile @@ -67,7 +67,7 @@ RUN apt-get -q update && \ python-magic \ pyzmq \ requests \ - watchdog==5.0.2 && \ + watchdog==5.0.3 && \ groupadd --gid ${DEFAULT_GID} ${PGROUP} && \ useradd -M --uid ${DEFAULT_UID} --gid ${DEFAULT_GID} ${PUSER} diff --git a/Dockerfiles/suricata.Dockerfile b/Dockerfiles/suricata.Dockerfile index 7da6e6deb..cce710791 100644 --- a/Dockerfiles/suricata.Dockerfile +++ b/Dockerfiles/suricata.Dockerfile @@ -108,7 +108,7 @@ RUN export BINARCH=$(uname -m | sed 's/x86_64/amd64/' | sed 's/aarch64/arm64/') apt-get install -q -y --no-install-recommends -t bookworm-backports \ suricata=${SURICATA_VERSION_PATTERN} \ suricata-update && \ - python3 -m pip install --break-system-packages --no-compile --no-cache-dir watchdog==5.0.2 && \ + python3 -m pip install --break-system-packages --no-compile --no-cache-dir watchdog==5.0.3 && \ curl -fsSL -o /usr/local/bin/supercronic "${SUPERCRONIC_URL}${BINARCH}" && \ chmod +x /usr/local/bin/supercronic && \ curl -fsSL -o /usr/bin/yq "${YQ_URL}${BINARCH}" && \ diff --git a/Dockerfiles/zeek.Dockerfile b/Dockerfiles/zeek.Dockerfile index 4b0780c51..02b7cd3b1 100644 --- a/Dockerfiles/zeek.Dockerfile +++ b/Dockerfiles/zeek.Dockerfile @@ -33,8 +33,9 @@ USER root # see PUSER_CHOWN at the bottom of the file (after the other environment variables it references) # for download and install -ARG ZEEK_VERSION=7.0.1-0 +ARG ZEEK_VERSION=7.0.3-0 ENV ZEEK_VERSION $ZEEK_VERSION +ARG ZEEK_DEB_ALTERNATE_DOWNLOAD_URL="" # put Zeek and Spicy in PATH ENV ZEEK_DIR "/opt/zeek" @@ -246,6 +247,7 @@ ARG ZEEK_DISABLE_HASH_ALL_FILES= ARG ZEEK_DISABLE_LOG_PASSWORDS= ARG ZEEK_DISABLE_SSL_VALIDATE_CERTS= ARG ZEEK_DISABLE_TRACK_ALL_ASSETS= +ARG ZEEK_DISABLE_DETECT_ROUTERS=true ARG ZEEK_DISABLE_BEST_GUESS_ICS=true # TODO: assess spicy-analyzer that replace built-in Zeek parsers # for now, disable them by default when a Zeek parser exists @@ -264,6 +266,7 @@ ENV ZEEK_DISABLE_HASH_ALL_FILES $ZEEK_DISABLE_HASH_ALL_FILES ENV ZEEK_DISABLE_LOG_PASSWORDS $ZEEK_DISABLE_LOG_PASSWORDS ENV ZEEK_DISABLE_SSL_VALIDATE_CERTS $ZEEK_DISABLE_SSL_VALIDATE_CERTS ENV ZEEK_DISABLE_TRACK_ALL_ASSETS $ZEEK_DISABLE_TRACK_ALL_ASSETS +ENV ZEEK_DISABLE_DETECT_ROUTERS $ZEEK_DISABLE_DETECT_ROUTERS ENV ZEEK_DISABLE_BEST_GUESS_ICS $ZEEK_DISABLE_BEST_GUESS_ICS ENV ZEEK_DISABLE_SPICY_IPSEC $ZEEK_DISABLE_SPICY_IPSEC diff --git a/api/project/__init__.py b/api/project/__init__.py index 4df34047b..15748ebe9 100644 --- a/api/project/__init__.py +++ b/api/project/__init__.py @@ -743,7 +743,7 @@ def fields(): s = SearchClass( using=databaseClient, index=index_from_args(args), - ).extra(size=5000) + ).extra(size=6000) for hit in [x['_source'] for x in s.execute().to_dict().get('hits', {}).get('hits', [])]: if (fieldname := malcolm_utils.deep_get(hit, ['dbField2'])) and (fieldname not in fields): if debugApi: diff --git a/api/requirements.txt b/api/requirements.txt index 15142558b..aa497c6ab 100644 --- a/api/requirements.txt +++ b/api/requirements.txt @@ -6,5 +6,5 @@ requests==2.32.0 regex==2022.3.2 dateparser==1.1.1 elasticsearch==8.15.1 -elasticsearch-dsl==8.15.3 +elasticsearch-dsl==8.15.4 psutil==5.9.8 \ No newline at end of file diff --git a/arkime/etc/config.ini b/arkime/etc/config.ini index 4e5273e10..4a5ad775c 100644 --- a/arkime/etc/config.ini +++ b/arkime/etc/config.ini @@ -1232,6 +1232,8 @@ zeek.http.resp_mime_types=db:zeek.http.resp_mime_types;group:zeek_http;kind:term zeek.http.post_username=db:zeek.http.post_username;group:zeek_http;kind:termfield;viewerOnly:true;friendly:POST User;help:POST User zeek.http.post_password_plain=db:zeek.http.post_password_plain;group:zeek_http;kind:termfield;viewerOnly:true;friendly:POST Password;help:POST Password zeek.http.ja4h=db:zeek.http.ja4h;group:zeek_http;kind:termfield;viewerOnly:true;friendly:JA4 HTTP Fingerprint;help:JA4 HTTP Fingerprint +zeek.http.client_header_names=db:zeek.http.client_header_names;group:zeek_http;kind:termfield;viewerOnly:true;friendly:Client Header Names;help:Client Header Names +zeek.http.server_header_names=db:zeek.http.server_header_names;group:zeek_http;kind:termfield;viewerOnly:true;friendly:Server Header Names;help:Server Header Names # intel.log # https://docs.zeek.org/en/stable/scripts/base/frameworks/intel/main.zeek.html#type-Intel::Info @@ -1320,6 +1322,10 @@ zeek.known_certs.serial=db:zeek.known_certs.serial;group:zeek_x509;kind:termfiel # https://docs.zeek.org/en/stable/scripts/policy/protocols/modbus/known-masters-slaves.zeek.html#type-Known::ModbusInfo zeek.known_modbus.device_type=db:zeek.known_modbus.device_type;group:zeek_modbus;kind:termfield;viewerOnly:true;friendly:Role;help:Role +# known_routers.log +zeek.known_routers.ttl=db:zeek.known_routers.ttl;group:zeek_conn;kind:integer;viewerOnly:true;friendly:IPv4 Packet Time To Live;help:IPv4 Packet Time To Live +zeek.known_routers.hlim=db:zeek.known_routers.hlim;group:zeek_conn;kind:termfield;viewerOnly:true;friendly:IPv6 Packet Hop Limit;help:IPv6 Packet Hop Limit + # ldap.log # https://github.com/zeek/spicy-ldap/blob/main/analyzer/main.zeek zeek.ldap.message_id=db:zeek.ldap.message_id;group:zeek_ldap;kind:termfield;viewerOnly:true;friendly:Message ID;help:Message ID @@ -3327,7 +3333,7 @@ o_zeek_hart_ip_direct_pdu_command=require:zeek.hart_ip_direct_pdu_command;title: o_zeek_hart_ip=require:zeek.hart_ip;title:Zeek hart_ip.log;fields:zeek.hart_ip.command_number_link_id,zeek.hart_ip.direct_pdu_command_link_id,zeek.hart_ip.session_log_record_link_id,zeek.hart_ip.message_packet_bytes,zeek.hart_ip.header_version,zeek.hart_ip.header_message_type_reserved,zeek.hart_ip.header_message_type_message_type,zeek.hart_ip.header_message_id,zeek.hart_ip.header_status_code,zeek.hart_ip.header_sequence_number,zeek.hart_ip.header_length,zeek.hart_ip.session_initiate_master_type,zeek.hart_ip.session_initiate_inactivity_close_timer,zeek.hart_ip.token_passing_pdu_delimiter_address_type,zeek.hart_ip.token_passing_pdu_delimiter_expansion_bytes,zeek.hart_ip.token_passing_pdu_delimiter_physical_layer_type,zeek.hart_ip.token_passing_pdu_delimiter_frame_type,zeek.hart_ip.token_passing_pdu_address_v4,zeek.hart_ip.token_passing_pdu_address_v6,zeek.hart_ip.token_passing_pdu_command_number,zeek.hart_ip.token_passing_pdu_byte_count,zeek.hart_ip.token_passing_pdu_check_byte,zeek.hart_ip.token_passing_pdu_contents_data_data,zeek.hart_ip.token_passing_pdu_contents_response_response_code,zeek.hart_ip.token_passing_pdu_contents_response_device_status_device_malfunction,zeek.hart_ip.token_passing_pdu_contents_response_device_status_configuration_changed,zeek.hart_ip.token_passing_pdu_contents_response_device_status_cold_start,zeek.hart_ip.token_passing_pdu_contents_response_device_status_more_status_available,zeek.hart_ip.token_passing_pdu_contents_response_device_status_loop_current_fixed,zeek.hart_ip.token_passing_pdu_contents_response_device_status_loop_current_saturated,zeek.hart_ip.token_passing_pdu_contents_response_device_status_non_primary_variable_out_of_limits,zeek.hart_ip.token_passing_pdu_contents_response_device_status_primary_variable_out_of_limits,zeek.hart_ip.direct_pdu_device_status_device_malfunction,zeek.hart_ip.direct_pdu_device_status_configuration_changed,zeek.hart_ip.direct_pdu_device_status_cold_start,zeek.hart_ip.direct_pdu_device_status_more_status_available,zeek.hart_ip.direct_pdu_device_status_loop_current_fixed,zeek.hart_ip.direct_pdu_device_status_loop_current_saturated,zeek.hart_ip.direct_pdu_device_status_non_primary_variable_out_of_limits,zeek.hart_ip.direct_pdu_device_status_primary_variable_out_of_limits,zeek.hart_ip.direct_pdu_extended_status_undefined_bits,zeek.hart_ip.direct_pdu_extended_status_function_check,zeek.hart_ip.direct_pdu_extended_status_out_of_specification,zeek.hart_ip.direct_pdu_extended_status_failure,zeek.hart_ip.direct_pdu_extended_status_critical_power_failure,zeek.hart_ip.direct_pdu_extended_status_device_variable_alert,zeek.hart_ip.direct_pdu_extended_status_maintenance_required,zeek.hart_ip.read_audit_log_start_record,zeek.hart_ip.read_audit_log_number_of_records,zeek.hart_ip.read_audit_log_power_up_time,zeek.hart_ip.read_audit_log_last_security_change,zeek.hart_ip.read_audit_log_server_status_undefined_bits,zeek.hart_ip.read_audit_log_server_status_insecure_syslog_connection,zeek.hart_ip.read_audit_log_server_status_syslog_server_located_but_connection_failed,zeek.hart_ip.read_audit_log_server_status_unable_to_locate_syslog_server,zeek.hart_ip.read_audit_log_session_record_size o_zeek_hart_ip_session_record=require:zeek.hart_ip_session_record;title:Zeek hart_ip_session_record.log;fields:zeek.hart_ip.session_log_record_link_id,zeek.hart_ip_session_record.session_log_record_client_i_pv4_address,zeek.hart_ip_session_record.session_log_record_client_i_pv6_address,zeek.hart_ip_session_record.session_log_record_client_port,zeek.hart_ip_session_record.session_log_record_server_port,zeek.hart_ip_session_record.session_log_record_connect_time,zeek.hart_ip_session_record.session_log_record_disconnect_time,zeek.hart_ip_session_record.session_log_record_session_status_summary_undefined_bits,zeek.hart_ip_session_record.session_log_record_session_status_summary_insecure_session,zeek.hart_ip_session_record.session_log_record_session_status_summary_session_timeout,zeek.hart_ip_session_record.session_log_record_session_status_summary_aborted_session,zeek.hart_ip_session_record.session_log_record_session_status_summary_bad_session_initialization,zeek.hart_ip_session_record.session_log_record_session_status_summary_writes_occured,zeek.hart_ip_session_record.session_log_record_start_configuration_change_count,zeek.hart_ip_session_record.session_log_record_end_configuration_change_count,zeek.hart_ip_session_record.session_log_record_num_publish_pdu,zeek.hart_ip_session_record.session_log_record_num_request_pdu,zeek.hart_ip_session_record.session_log_record_num_response_pdu o_zeek_hart_ip_universal_commands=require:zeek.hart_ip_universal_commands;title:Zeek hart_ip_universal_commands.log;fields:zeek.hart_ip.command_number_link_id,zeek.hart_ip_universal_commands.read_unique_identifier_response_254,zeek.hart_ip_universal_commands.read_unique_identifier_response_expanded_device_type,zeek.hart_ip_universal_commands.read_unique_identifier_response_minimum_preambles_master_slave,zeek.hart_ip_universal_commands.read_unique_identifier_response_hart_protocol_major_revision,zeek.hart_ip_universal_commands.read_unique_identifier_response_device_revision_level,zeek.hart_ip_universal_commands.read_unique_identifier_response_software_revision_level,zeek.hart_ip_universal_commands.read_unique_identifier_response_hardware_revision_level_and_physical_signaling_codes_hardware_revision_level,zeek.hart_ip_universal_commands.read_unique_identifier_response_hardware_revision_level_and_physical_signaling_codes_physical_signaling_code,zeek.hart_ip_universal_commands.read_unique_identifier_response_flags_c8_psk_in_multi_drop_only,zeek.hart_ip_universal_commands.read_unique_identifier_response_flags_c8_psk_capable_field_device,zeek.hart_ip_universal_commands.read_unique_identifier_response_flags_undefined_5,zeek.hart_ip_universal_commands.read_unique_identifier_response_flags_safehart_capable_field_device,zeek.hart_ip_universal_commands.read_unique_identifier_response_flags_ieee_802_15_4_dsss_o_qpsk_modulation,zeek.hart_ip_universal_commands.read_unique_identifier_response_flags_protocol_bridge_device,zeek.hart_ip_universal_commands.read_unique_identifier_response_flags_eeprom_control,zeek.hart_ip_universal_commands.read_unique_identifier_response_flags_mutli_sensor_field_device,zeek.hart_ip_universal_commands.read_unique_identifier_response_device_id,zeek.hart_ip_universal_commands.read_unique_identifier_response_number_preambles_slave_master,zeek.hart_ip_universal_commands.read_unique_identifier_response_last_device_variable_this,zeek.hart_ip_universal_commands.read_unique_identifier_response_configuration_change_counter,zeek.hart_ip_universal_commands.read_unique_identifier_response_extended_field_device_status_undefined_bits,zeek.hart_ip_universal_commands.read_unique_identifier_response_extended_field_device_status_function_check,zeek.hart_ip_universal_commands.read_unique_identifier_response_extended_field_device_status_out_of_specification,zeek.hart_ip_universal_commands.read_unique_identifier_response_extended_field_device_status_failure,zeek.hart_ip_universal_commands.read_unique_identifier_response_extended_field_device_status_critical_power_failure,zeek.hart_ip_universal_commands.read_unique_identifier_response_extended_field_device_status_device_variable_alert,zeek.hart_ip_universal_commands.read_unique_identifier_response_extended_field_device_status_maintenance_required,zeek.hart_ip_universal_commands.read_unique_identifier_response_manufacturer_identification_code,zeek.hart_ip_universal_commands.read_unique_identifier_response_private_label_distributor_code,zeek.hart_ip_universal_commands.read_unique_identifier_response_device_profile,zeek.hart_ip_universal_commands.read_primary_variable_response_primary_variable_units,zeek.hart_ip_universal_commands.read_primary_variable_response_primary_variable,zeek.hart_ip_universal_commands.read_loop_current_response_primary_variable_loop_current,zeek.hart_ip_universal_commands.read_loop_current_response_primary_variable_percent_range,zeek.hart_ip_universal_commands.read_dynamic_variable_response_primary_variable_loop_current,zeek.hart_ip_universal_commands.read_dynamic_variable_response_primary_variable_units,zeek.hart_ip_universal_commands.read_dynamic_variable_response_primary_variable,zeek.hart_ip_universal_commands.read_dynamic_variable_response_secondary_variable_units,zeek.hart_ip_universal_commands.read_dynamic_variable_response_secondary_variable,zeek.hart_ip_universal_commands.read_dynamic_variable_response_tertiary_variable_units,zeek.hart_ip_universal_commands.read_dynamic_variable_response_tertiary_variable,zeek.hart_ip_universal_commands.read_dynamic_variable_response_quaternary_variable_units,zeek.hart_ip_universal_commands.read_dynamic_variable_response_quaternary_variable,zeek.hart_ip_universal_commands.write_polling_address_polling_address_device,zeek.hart_ip_universal_commands.write_polling_address_loop_current_mode,zeek.hart_ip_universal_commands.read_loop_configuration_response_polling_address_device,zeek.hart_ip_universal_commands.read_loop_configuration_response_loop_current_mode,zeek.hart_ip_universal_commands.read_dynamic_variable_classifications_response_primary_variable_classification,zeek.hart_ip_universal_commands.read_dynamic_variable_classifications_response_secondary_variable_classification,zeek.hart_ip_universal_commands.read_dynamic_variable_classifications_response_tertiary_variable_classification,zeek.hart_ip_universal_commands.read_dynamic_variable_classifications_response_quaternary_variable_classification,zeek.hart_ip_universal_commands.read_device_variable_request_slot0_device_variable_code,zeek.hart_ip_universal_commands.read_device_variable_request_slot1_device_variable_code,zeek.hart_ip_universal_commands.read_device_variable_request_slot2_device_variable_code,zeek.hart_ip_universal_commands.read_device_variable_request_slot3_device_variable_code,zeek.hart_ip_universal_commands.read_device_variable_request_slot4_device_variable_code,zeek.hart_ip_universal_commands.read_device_variable_request_slot5_device_variable_code,zeek.hart_ip_universal_commands.read_device_variable_request_slot6_device_variable_code,zeek.hart_ip_universal_commands.read_device_variable_request_slot7_device_variable_code,zeek.hart_ip_universal_commands.read_device_variable_response_extended_field_device_status_undefined_bits,zeek.hart_ip_universal_commands.read_device_variable_response_extended_field_device_status_function_check,zeek.hart_ip_universal_commands.read_device_variable_response_extended_field_device_status_out_of_specification,zeek.hart_ip_universal_commands.read_device_variable_response_extended_field_device_status_failure,zeek.hart_ip_universal_commands.read_device_variable_response_extended_field_device_status_critical_power_failure,zeek.hart_ip_universal_commands.read_device_variable_response_extended_field_device_status_device_variable_alert,zeek.hart_ip_universal_commands.read_device_variable_response_extended_field_device_status_maintenance_required,zeek.hart_ip_universal_commands.read_device_variable_response_slot0_device_variable_code,zeek.hart_ip_universal_commands.read_device_variable_response_slot0_device_variable_class,zeek.hart_ip_universal_commands.read_device_variable_response_slot0_units_code,zeek.hart_ip_universal_commands.read_device_variable_response_slot0_device_variable,zeek.hart_ip_universal_commands.read_device_variable_response_slot0_device_variable_status_process_data_status,zeek.hart_ip_universal_commands.read_device_variable_response_slot0_device_variable_status_limit_status,zeek.hart_ip_universal_commands.read_device_variable_response_slot0_device_variable_status_more_device_variable_status_available,zeek.hart_ip_universal_commands.read_device_variable_response_slot0_device_variable_status_device_family_specific_status,zeek.hart_ip_universal_commands.read_device_variable_response_slot1_device_variable_code,zeek.hart_ip_universal_commands.read_device_variable_response_slot1_device_variable_class,zeek.hart_ip_universal_commands.read_device_variable_response_slot1_units_code,zeek.hart_ip_universal_commands.read_device_variable_response_slot1_device_variable,zeek.hart_ip_universal_commands.read_device_variable_response_slot1_device_variable_status_process_data_status,zeek.hart_ip_universal_commands.read_device_variable_response_slot1_device_variable_status_limit_status,zeek.hart_ip_universal_commands.read_device_variable_response_slot1_device_variable_status_more_device_variable_status_available,zeek.hart_ip_universal_commands.read_device_variable_response_slot1_device_variable_status_device_family_specific_status,zeek.hart_ip_universal_commands.read_device_variable_response_slot2_device_variable_code,zeek.hart_ip_universal_commands.read_device_variable_response_slot2_device_variable_class,zeek.hart_ip_universal_commands.read_device_variable_response_slot2_units_code,zeek.hart_ip_universal_commands.read_device_variable_response_slot2_device_variable,zeek.hart_ip_universal_commands.read_device_variable_response_slot2_device_variable_status_process_data_status,zeek.hart_ip_universal_commands.read_device_variable_response_slot2_device_variable_status_limit_status,zeek.hart_ip_universal_commands.read_device_variable_response_slot2_device_variable_status_more_device_variable_status_available,zeek.hart_ip_universal_commands.read_device_variable_response_slot2_device_variable_status_device_family_specific_status,zeek.hart_ip_universal_commands.read_device_variable_response_slot3_device_variable_code,zeek.hart_ip_universal_commands.read_device_variable_response_slot3_device_variable_class,zeek.hart_ip_universal_commands.read_device_variable_response_slot3_units_code,zeek.hart_ip_universal_commands.read_device_variable_response_slot3_device_variable,zeek.hart_ip_universal_commands.read_device_variable_response_slot3_device_variable_status_process_data_status,zeek.hart_ip_universal_commands.read_device_variable_response_slot3_device_variable_status_limit_status,zeek.hart_ip_universal_commands.read_device_variable_response_slot3_device_variable_status_more_device_variable_status_available,zeek.hart_ip_universal_commands.read_device_variable_response_slot3_device_variable_status_device_family_specific_status,zeek.hart_ip_universal_commands.read_device_variable_response_slot4_device_variable_code,zeek.hart_ip_universal_commands.read_device_variable_response_slot4_device_variable_class,zeek.hart_ip_universal_commands.read_device_variable_response_slot4_units_code,zeek.hart_ip_universal_commands.read_device_variable_response_slot4_device_variable,zeek.hart_ip_universal_commands.read_device_variable_response_slot4_device_variable_status_process_data_status,zeek.hart_ip_universal_commands.read_device_variable_response_slot4_device_variable_status_limit_status,zeek.hart_ip_universal_commands.read_device_variable_response_slot4_device_variable_status_more_device_variable_status_available,zeek.hart_ip_universal_commands.read_device_variable_response_slot4_device_variable_status_device_family_specific_status,zeek.hart_ip_universal_commands.read_device_variable_response_slot5_device_variable_code,zeek.hart_ip_universal_commands.read_device_variable_response_slot5_device_variable_class,zeek.hart_ip_universal_commands.read_device_variable_response_slot5_units_code,zeek.hart_ip_universal_commands.read_device_variable_response_slot5_device_variable,zeek.hart_ip_universal_commands.read_device_variable_response_slot5_device_variable_status_process_data_status,zeek.hart_ip_universal_commands.read_device_variable_response_slot5_device_variable_status_limit_status,zeek.hart_ip_universal_commands.read_device_variable_response_slot5_device_variable_status_more_device_variable_status_available,zeek.hart_ip_universal_commands.read_device_variable_response_slot5_device_variable_status_device_family_specific_status,zeek.hart_ip_universal_commands.read_device_variable_response_slot6_device_variable_code,zeek.hart_ip_universal_commands.read_device_variable_response_slot6_device_variable_class,zeek.hart_ip_universal_commands.read_device_variable_response_slot6_units_code,zeek.hart_ip_universal_commands.read_device_variable_response_slot6_device_variable,zeek.hart_ip_universal_commands.read_device_variable_response_slot6_device_variable_status_process_data_status,zeek.hart_ip_universal_commands.read_device_variable_response_slot6_device_variable_status_limit_status,zeek.hart_ip_universal_commands.read_device_variable_response_slot6_device_variable_status_more_device_variable_status_available,zeek.hart_ip_universal_commands.read_device_variable_response_slot6_device_variable_status_device_family_specific_status,zeek.hart_ip_universal_commands.read_device_variable_response_slot7_device_variable_code,zeek.hart_ip_universal_commands.read_device_variable_response_slot7_device_variable_class,zeek.hart_ip_universal_commands.read_device_variable_response_slot7_units_code,zeek.hart_ip_universal_commands.read_device_variable_response_slot7_device_variable,zeek.hart_ip_universal_commands.read_device_variable_response_slot7_device_variable_status_process_data_status,zeek.hart_ip_universal_commands.read_device_variable_response_slot7_device_variable_status_limit_status,zeek.hart_ip_universal_commands.read_device_variable_response_slot7_device_variable_status_more_device_variable_status_available,zeek.hart_ip_universal_commands.read_device_variable_response_slot7_device_variable_status_device_family_specific_status,zeek.hart_ip_universal_commands.read_device_variable_response_slot0_time,zeek.hart_ip_universal_commands.read_unique_identifier_tag_request_tag,zeek.hart_ip_universal_commands.read_unique_identifier_tag_response_254,zeek.hart_ip_universal_commands.read_unique_identifier_tag_response_expanded_device_type,zeek.hart_ip_universal_commands.read_unique_identifier_tag_response_minimum_preambles_master_slave,zeek.hart_ip_universal_commands.read_unique_identifier_tag_response_hart_protocol_major_revision,zeek.hart_ip_universal_commands.read_unique_identifier_tag_response_device_revision_level,zeek.hart_ip_universal_commands.read_unique_identifier_tag_response_software_revision_level,zeek.hart_ip_universal_commands.read_unique_identifier_tag_response_hardware_revision_level_and_physical_signaling_codes_hardware_revision_level,zeek.hart_ip_universal_commands.read_unique_identifier_tag_response_hardware_revision_level_and_physical_signaling_codes_physical_signaling_code,zeek.hart_ip_universal_commands.read_unique_identifier_tag_response_flags_c8_psk_in_multi_drop_only,zeek.hart_ip_universal_commands.read_unique_identifier_tag_response_flags_c8_psk_capable_field_device,zeek.hart_ip_universal_commands.read_unique_identifier_tag_response_flags_undefined_5,zeek.hart_ip_universal_commands.read_unique_identifier_tag_response_flags_safehart_capable_field_device,zeek.hart_ip_universal_commands.read_unique_identifier_tag_response_flags_ieee_802_15_4_dsss_o_qpsk_modulation,zeek.hart_ip_universal_commands.read_unique_identifier_tag_response_flags_protocol_bridge_device,zeek.hart_ip_universal_commands.read_unique_identifier_tag_response_flags_eeprom_control,zeek.hart_ip_universal_commands.read_unique_identifier_tag_response_flags_mutli_sensor_field_device,zeek.hart_ip_universal_commands.read_unique_identifier_tag_response_device_id,zeek.hart_ip_universal_commands.read_unique_identifier_tag_response_number_preambles_slave_master,zeek.hart_ip_universal_commands.read_unique_identifier_tag_response_last_device_variable_this,zeek.hart_ip_universal_commands.read_unique_identifier_tag_response_configuration_change_counter,zeek.hart_ip_universal_commands.read_unique_identifier_tag_response_extended_field_device_status_undefined_bits,zeek.hart_ip_universal_commands.read_unique_identifier_tag_response_extended_field_device_status_function_check,zeek.hart_ip_universal_commands.read_unique_identifier_tag_response_extended_field_device_status_out_of_specification,zeek.hart_ip_universal_commands.read_unique_identifier_tag_response_extended_field_device_status_failure,zeek.hart_ip_universal_commands.read_unique_identifier_tag_response_extended_field_device_status_critical_power_failure,zeek.hart_ip_universal_commands.read_unique_identifier_tag_response_extended_field_device_status_device_variable_alert,zeek.hart_ip_universal_commands.read_unique_identifier_tag_response_extended_field_device_status_maintenance_required,zeek.hart_ip_universal_commands.read_unique_identifier_tag_response_manufacturer_identification_code,zeek.hart_ip_universal_commands.read_unique_identifier_tag_response_private_label_distributor_code,zeek.hart_ip_universal_commands.read_unique_identifier_tag_response_device_profile,zeek.hart_ip_universal_commands.read_message_response_message,zeek.hart_ip_universal_commands.read_tag_response_tag,zeek.hart_ip_universal_commands.read_tag_response_descriptor,zeek.hart_ip_universal_commands.read_tag_response_date_code,zeek.hart_ip_universal_commands.read_primary_variable_transducer_information_response_p_v_transducer_serial_number,zeek.hart_ip_universal_commands.read_primary_variable_transducer_information_response_p_v_transducer_limits_units,zeek.hart_ip_universal_commands.read_primary_variable_transducer_information_response_p_v_upper_transducer_limit,zeek.hart_ip_universal_commands.read_primary_variable_transducer_information_response_p_v_lower_transducer_limit,zeek.hart_ip_universal_commands.read_primary_variable_transducer_information_response_p_v_minimum_span,zeek.hart_ip_universal_commands.read_device_information_response_p_v_alarm_selection_code,zeek.hart_ip_universal_commands.read_device_information_response_p_v_transfer_function_code,zeek.hart_ip_universal_commands.read_device_information_response_p_v_upper_lower_range,zeek.hart_ip_universal_commands.read_device_information_response_p_v_upper_range_value,zeek.hart_ip_universal_commands.read_device_information_response_p_v_lower_range_value,zeek.hart_ip_universal_commands.read_device_information_response_p_v_damping_value,zeek.hart_ip_universal_commands.read_device_information_response_write_protect_code,zeek.hart_ip_universal_commands.read_device_information_response_250,zeek.hart_ip_universal_commands.read_device_information_response_p_v_analog_channel_flags_undefined_bits,zeek.hart_ip_universal_commands.read_device_information_response_p_v_analog_channel_flags_analog_channel,zeek.hart_ip_universal_commands.read_final_assembly_number_response_final_assembly_number,zeek.hart_ip_universal_commands.write_message_message_string,zeek.hart_ip_universal_commands.write_tag_descriptor_date_tag,zeek.hart_ip_universal_commands.write_tag_descriptor_date_record_keeping_descriptor,zeek.hart_ip_universal_commands.write_tag_descriptor_date_date_code,zeek.hart_ip_universal_commands.write_final_assembly_number_final_assembly_number,zeek.hart_ip_universal_commands.read_long_tag_response_long_tag,zeek.hart_ip_universal_commands.read_unique_identifier_long_tag_request_long_tag,zeek.hart_ip_universal_commands.read_unique_identifier_long_tag_response_254,zeek.hart_ip_universal_commands.read_unique_identifier_long_tag_response_expanded_device_type,zeek.hart_ip_universal_commands.read_unique_identifier_long_tag_response_minimum_preambles_master_slave,zeek.hart_ip_universal_commands.read_unique_identifier_long_tag_response_hart_protocol_major_revision,zeek.hart_ip_universal_commands.read_unique_identifier_long_tag_response_device_revision_level,zeek.hart_ip_universal_commands.read_unique_identifier_long_tag_response_software_revision_level,zeek.hart_ip_universal_commands.read_unique_identifier_long_tag_response_hardware_revision_level_and_physical_signaling_codes_hardware_revision_level,zeek.hart_ip_universal_commands.read_unique_identifier_long_tag_response_hardware_revision_level_and_physical_signaling_codes_physical_signaling_code,zeek.hart_ip_universal_commands.read_unique_identifier_long_tag_response_flags_c8_psk_in_multi_drop_only,zeek.hart_ip_universal_commands.read_unique_identifier_long_tag_response_flags_c8_psk_capable_field_device,zeek.hart_ip_universal_commands.read_unique_identifier_long_tag_response_flags_undefined_5,zeek.hart_ip_universal_commands.read_unique_identifier_long_tag_response_flags_safehart_capable_field_device,zeek.hart_ip_universal_commands.read_unique_identifier_long_tag_response_flags_ieee_802_15_4_dsss_o_qpsk_modulation,zeek.hart_ip_universal_commands.read_unique_identifier_long_tag_response_flags_protocol_bridge_device,zeek.hart_ip_universal_commands.read_unique_identifier_long_tag_response_flags_eeprom_control,zeek.hart_ip_universal_commands.read_unique_identifier_long_tag_response_flags_mutli_sensor_field_device,zeek.hart_ip_universal_commands.read_unique_identifier_long_tag_response_device_id,zeek.hart_ip_universal_commands.read_unique_identifier_long_tag_response_number_preambles_slave_master,zeek.hart_ip_universal_commands.read_unique_identifier_long_tag_response_last_device_variable_this,zeek.hart_ip_universal_commands.read_unique_identifier_long_tag_response_configuration_change_counter,zeek.hart_ip_universal_commands.read_unique_identifier_long_tag_response_extended_field_device_status_undefined_bits,zeek.hart_ip_universal_commands.read_unique_identifier_long_tag_response_extended_field_device_status_function_check,zeek.hart_ip_universal_commands.read_unique_identifier_long_tag_response_extended_field_device_status_out_of_specification,zeek.hart_ip_universal_commands.read_unique_identifier_long_tag_response_extended_field_device_status_failure,zeek.hart_ip_universal_commands.read_unique_identifier_long_tag_response_extended_field_device_status_critical_power_failure,zeek.hart_ip_universal_commands.read_unique_identifier_long_tag_response_extended_field_device_status_device_variable_alert,zeek.hart_ip_universal_commands.read_unique_identifier_long_tag_response_extended_field_device_status_maintenance_required,zeek.hart_ip_universal_commands.read_unique_identifier_long_tag_response_manufacturer_identification_code,zeek.hart_ip_universal_commands.read_unique_identifier_long_tag_response_private_label_distributor_code,zeek.hart_ip_universal_commands.read_unique_identifier_long_tag_response_device_profile,zeek.hart_ip_universal_commands.write_long_tag_long_tag,zeek.hart_ip_universal_commands.reset_configuration_changed_flag_configuration_change_counter,zeek.hart_ip_universal_commands.read_additional_device_status_contents_device_specific_status_0,zeek.hart_ip_universal_commands.read_additional_device_status_contents_extended_field_device_status_undefined_bits,zeek.hart_ip_universal_commands.read_additional_device_status_contents_extended_field_device_status_function_check,zeek.hart_ip_universal_commands.read_additional_device_status_contents_extended_field_device_status_out_of_specification,zeek.hart_ip_universal_commands.read_additional_device_status_contents_extended_field_device_status_failure,zeek.hart_ip_universal_commands.read_additional_device_status_contents_extended_field_device_status_critical_power_failure,zeek.hart_ip_universal_commands.read_additional_device_status_contents_extended_field_device_status_device_variable_alert,zeek.hart_ip_universal_commands.read_additional_device_status_contents_extended_field_device_status_maintenance_required,zeek.hart_ip_universal_commands.read_additional_device_status_contents_device_operating_mode,zeek.hart_ip_universal_commands.read_additional_device_status_contents_standardized_status0_device_configuration_lock,zeek.hart_ip_universal_commands.read_additional_device_status_contents_standardized_status0_electronic_defect,zeek.hart_ip_universal_commands.read_additional_device_status_contents_standardized_status0_environmental_conditions_out_of_range,zeek.hart_ip_universal_commands.read_additional_device_status_contents_standardized_status0_power_supply_conditions_out_of_range,zeek.hart_ip_universal_commands.read_additional_device_status_contents_standardized_status0_watchdog_reset_executed,zeek.hart_ip_universal_commands.read_additional_device_status_contents_standardized_status0_volatile_memory_defect,zeek.hart_ip_universal_commands.read_additional_device_status_contents_standardized_status0_non_volatile_memory_defect,zeek.hart_ip_universal_commands.read_additional_device_status_contents_standardized_status0_device_variable_simulation_active,zeek.hart_ip_universal_commands.read_additional_device_status_contents_standardized_status1_undefined_bits,zeek.hart_ip_universal_commands.read_additional_device_status_contents_standardized_status1_reserved,zeek.hart_ip_universal_commands.read_additional_device_status_contents_standardized_status1_battery_or_power_supply_needs_maintenance,zeek.hart_ip_universal_commands.read_additional_device_status_contents_standardized_status1_event_notification_overflow,zeek.hart_ip_universal_commands.read_additional_device_status_contents_standardized_status1_discrete_variable_simulation_active,zeek.hart_ip_universal_commands.read_additional_device_status_contents_standardized_status1_status_simulation_active,zeek.hart_ip_universal_commands.read_additional_device_status_contents_analog_channel_saturated_undefined_bits,zeek.hart_ip_universal_commands.read_additional_device_status_contents_analog_channel_saturated_quinary_analog,zeek.hart_ip_universal_commands.read_additional_device_status_contents_analog_channel_saturated_quaternary_analog,zeek.hart_ip_universal_commands.read_additional_device_status_contents_analog_channel_saturated_tertiary_analog,zeek.hart_ip_universal_commands.read_additional_device_status_contents_analog_channel_saturated_secondary_analog,zeek.hart_ip_universal_commands.read_additional_device_status_contents_standardized_status2_undefined_bits,zeek.hart_ip_universal_commands.read_additional_device_status_contents_standardized_status2_stale_data_notice,zeek.hart_ip_universal_commands.read_additional_device_status_contents_standardized_status2_sub_device_with_duplicate_id,zeek.hart_ip_universal_commands.read_additional_device_status_contents_standardized_status2_sub_device_mismatch,zeek.hart_ip_universal_commands.read_additional_device_status_contents_standardized_status2_duplicate_master_detected,zeek.hart_ip_universal_commands.read_additional_device_status_contents_standardized_status2_sub_device_list_changed,zeek.hart_ip_universal_commands.read_additional_device_status_contents_standardized_status3_undefined_bits,zeek.hart_ip_universal_commands.read_additional_device_status_contents_standardized_status3_radio_failure,zeek.hart_ip_universal_commands.read_additional_device_status_contents_standardized_status3_block_transfer_pending,zeek.hart_ip_universal_commands.read_additional_device_status_contents_standardized_status3_bandwith_allocation_pending,zeek.hart_ip_universal_commands.read_additional_device_status_contents_standardized_status3_resereved,zeek.hart_ip_universal_commands.read_additional_device_status_contents_standardized_status3_capacity_denied,zeek.hart_ip_universal_commands.read_additional_device_status_contents_analog_channel_undefined_bits,zeek.hart_ip_universal_commands.read_additional_device_status_contents_analog_channel_analog_channel,zeek.hart_ip_universal_commands.read_additional_device_status_contents_device_specific_status_1 -o_zeek_http=require:zeek.http;title:Zeek http.log;fields:zeek.http.trans_depth,zeek.http.method,zeek.http.host,zeek.http.uri,zeek.http.origin,zeek.http.post_password_plain,zeek.http.post_username,zeek.http.referrer,zeek.http.version,zeek.http.user_agent,zeek.http.request_body_len,zeek.http.response_body_len,zeek.http.status_code,zeek.http.status_msg,zeek.http.info_code,zeek.http.info_msg,zeek.http.tags,zeek.http.proxied,zeek.http.orig_fuids,zeek.http.orig_filenames,zeek.http.orig_mime_types,zeek.http.resp_fuids,zeek.http.resp_filenames,zeek.http.resp_mime_types,zeek.http.ja4h +o_zeek_http=require:zeek.http;title:Zeek http.log;fields:zeek.http.trans_depth,zeek.http.method,zeek.http.host,zeek.http.uri,zeek.http.origin,zeek.http.post_password_plain,zeek.http.post_username,zeek.http.referrer,zeek.http.version,zeek.http.user_agent,zeek.http.request_body_len,zeek.http.response_body_len,zeek.http.status_code,zeek.http.status_msg,zeek.http.info_code,zeek.http.info_msg,zeek.http.tags,zeek.http.proxied,zeek.http.orig_fuids,zeek.http.orig_filenames,zeek.http.orig_mime_types,zeek.http.resp_fuids,zeek.http.resp_filenames,zeek.http.resp_mime_types,zeek.http.ja4h,zeek.http.client_header_names,zeek.http.server_header_names o_zeek_intel=require:zeek.intel;title:Zeek intel.log;fields:zeek.intel.cif_tags,zeek.intel.cif_confidence,zeek.intel.cif_source,zeek.intel.cif_description,zeek.intel.cif_firstseen,zeek.intel.cif_lastseen,zeek.intel.seen_indicator,zeek.intel.seen_indicator_type,zeek.intel.seen_where,zeek.intel.seen_node,zeek.intel.matched,zeek.intel.sources,zeek.intel.file_mime_type,zeek.intel.file_description o_zeek_ipsec=require:zeek.ipsec;title:Zeek ipsec.log;fields:zeek.ipsec.initiator_spi,zeek.ipsec.responder_spi,zeek.ipsec.maj_ver,zeek.ipsec.min_ver,zeek.ipsec.exchange_type,zeek.ipsec.doi,zeek.ipsec.flag_e,zeek.ipsec.flag_c,zeek.ipsec.flag_a,zeek.ipsec.flag_i,zeek.ipsec.flag_v,zeek.ipsec.flag_r,zeek.ipsec.flags,zeek.ipsec.message_id,zeek.ipsec.vendor_ids,zeek.ipsec.notify_messages,zeek.ipsec.transforms,zeek.ipsec.ke_dh_groups,zeek.ipsec.proposals,zeek.ipsec.protocol_id,zeek.ipsec.certificates,zeek.ipsec.situation,zeek.ipsec.transform_attributes,zeek.ipsec.length,zeek.ipsec.hash o_zeek_irc=require:zeek.irc;title:Zeek irc.log;fields:zeek.irc.nick,zeek.irc.command,zeek.irc.value,zeek.irc.addl,zeek.irc.dcc_file_name,zeek.irc.dcc_file_size,zeek.irc.dcc_mime_type diff --git a/arkime/wise/source.zeeklogs.js b/arkime/wise/source.zeeklogs.js index 574bce6dd..61d78ac52 100644 --- a/arkime/wise/source.zeeklogs.js +++ b/arkime/wise/source.zeeklogs.js @@ -1638,10 +1638,11 @@ class MalcolmSource extends WISESource { "zeek.hart_ip_universal_commands.write_tag_descriptor_date_date_code", "zeek.hart_ip_universal_commands.write_tag_descriptor_date_record_keeping_descriptor", "zeek.hart_ip_universal_commands.write_tag_descriptor_date_tag", + "zeek.http.client_header_names", "zeek.http.host", - "zeek.http.ja4h", "zeek.http.info_code", "zeek.http.info_msg", + "zeek.http.ja4h", "zeek.http.method", "zeek.http.orig_filenames", "zeek.http.orig_fuids", @@ -1656,6 +1657,7 @@ class MalcolmSource extends WISESource { "zeek.http.resp_fuids", "zeek.http.resp_mime_types", "zeek.http.response_body_len", + "zeek.http.server_header_names", "zeek.http.status_code", "zeek.http.status_msg", "zeek.http.tags", @@ -1727,6 +1729,8 @@ class MalcolmSource extends WISESource { "zeek.known_certs.serial", "zeek.known_certs.subject", "zeek.known_modbus.device_type", + "zeek.known_routers.ttl", + "zeek.known_routers.hlim", "zeek.ldap.argument", "zeek.ldap.message_id", "zeek.ldap.object", diff --git a/config/zeek.env.example b/config/zeek.env.example index ee73ef729..cc87221e6 100644 --- a/config/zeek.env.example +++ b/config/zeek.env.example @@ -68,6 +68,7 @@ ZEEK_DISABLE_HASH_ALL_FILES= ZEEK_DISABLE_LOG_PASSWORDS= ZEEK_DISABLE_SSL_VALIDATE_CERTS= ZEEK_DISABLE_TRACK_ALL_ASSETS= +ZEEK_DISABLE_DETECT_ROUTERS=true ZEEK_DISABLE_SPICY_IPSEC= ZEEK_DISABLE_SPICY_LDAP= ZEEK_DISABLE_SPICY_OPENVPN= diff --git a/dashboards/dashboards/024062a6-48d6-498f-a91a-3bf2da3a3cd3.json b/dashboards/dashboards/024062a6-48d6-498f-a91a-3bf2da3a3cd3.json index 1d7a8e0f5..e434649ee 100644 --- a/dashboards/dashboards/024062a6-48d6-498f-a91a-3bf2da3a3cd3.json +++ b/dashboards/dashboards/024062a6-48d6-498f-a91a-3bf2da3a3cd3.json @@ -18,7 +18,7 @@ "version": 1, "timeRestore": false, "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"highlightAll\":false,\"version\":true,\"query\":{\"language\":\"lucene\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"default_field\":\"*\",\"query\":\"*\",\"filter\":[]}" + "searchSourceJSON": "{\"highlightAll\":false,\"version\":true,\"query\":{\"language\":\"lucene\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"default_field\":\"*\",\"query\":\"*\"}}},\"filter\":[]}" } }, "references": [ diff --git a/dashboards/dashboards/0a490422-0ce9-44bf-9a2d-19329ddde8c3.json b/dashboards/dashboards/0a490422-0ce9-44bf-9a2d-19329ddde8c3.json index 60b26113a..27db8f8a9 100644 --- a/dashboards/dashboards/0a490422-0ce9-44bf-9a2d-19329ddde8c3.json +++ b/dashboards/dashboards/0a490422-0ce9-44bf-9a2d-19329ddde8c3.json @@ -18,7 +18,7 @@ "version": 1, "timeRestore": false, "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"highlightAll\":false,\"version\":true,\"query\":{\"language\":\"lucene\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"default_field\":\"*\",\"query\":\"*\",\"filter\":[]}" + "searchSourceJSON": "{\"highlightAll\":false,\"version\":true,\"query\":{\"language\":\"lucene\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"default_field\":\"*\",\"query\":\"*\"}}},\"filter\":[]}" } }, "references": [ diff --git a/dashboards/dashboards/11be6381-beef-40a7-bdce-88c5398392fc.json b/dashboards/dashboards/11be6381-beef-40a7-bdce-88c5398392fc.json index ce72bd1fd..a6fd63905 100644 --- a/dashboards/dashboards/11be6381-beef-40a7-bdce-88c5398392fc.json +++ b/dashboards/dashboards/11be6381-beef-40a7-bdce-88c5398392fc.json @@ -18,7 +18,7 @@ "version": 1, "timeRestore": false, "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"highlightAll\":false,\"version\":true,\"query\":{\"language\":\"lucene\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"default_field\":\"*\",\"query\":\"*\",\"filter\":[]}" + "searchSourceJSON": "{\"highlightAll\":false,\"version\":true,\"query\":{\"language\":\"lucene\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"default_field\":\"*\",\"query\":\"*\"}}},\"filter\":[]}" } }, "references": [ diff --git a/dashboards/dashboards/11ddd980-e388-11e9-b568-cf17de8e860c.json b/dashboards/dashboards/11ddd980-e388-11e9-b568-cf17de8e860c.json index 64827ff48..2e2f64890 100644 --- a/dashboards/dashboards/11ddd980-e388-11e9-b568-cf17de8e860c.json +++ b/dashboards/dashboards/11ddd980-e388-11e9-b568-cf17de8e860c.json @@ -18,7 +18,7 @@ "version": 1, "timeRestore": false, "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"highlightAll\":false,\"version\":true,\"query\":{\"language\":\"lucene\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"default_field\":\"*\",\"query\":\"*\",\"filter\":[]}" + "searchSourceJSON": "{\"highlightAll\":false,\"version\":true,\"query\":{\"language\":\"lucene\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"default_field\":\"*\",\"query\":\"*\"}}},\"filter\":[]}" } }, "references": [ diff --git a/dashboards/dashboards/1fff49f6-0199-4a0f-820b-721aff9ff1f1.json b/dashboards/dashboards/1fff49f6-0199-4a0f-820b-721aff9ff1f1.json index 21df165ed..6750c90bd 100644 --- a/dashboards/dashboards/1fff49f6-0199-4a0f-820b-721aff9ff1f1.json +++ b/dashboards/dashboards/1fff49f6-0199-4a0f-820b-721aff9ff1f1.json @@ -18,7 +18,7 @@ "version": 1, "timeRestore": false, "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"highlightAll\":false,\"version\":true,\"query\":{\"language\":\"lucene\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"default_field\":\"*\",\"query\":\"*\",\"filter\":[]}" + "searchSourceJSON": "{\"highlightAll\":false,\"version\":true,\"query\":{\"language\":\"lucene\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"default_field\":\"*\",\"query\":\"*\"}}},\"filter\":[]}" } }, "references": [ diff --git a/dashboards/dashboards/2cf94cd0-ecab-40a5-95a7-8419f3a39cd9.json b/dashboards/dashboards/2cf94cd0-ecab-40a5-95a7-8419f3a39cd9.json index 69031e2b0..23bd223d4 100644 --- a/dashboards/dashboards/2cf94cd0-ecab-40a5-95a7-8419f3a39cd9.json +++ b/dashboards/dashboards/2cf94cd0-ecab-40a5-95a7-8419f3a39cd9.json @@ -18,7 +18,7 @@ "version": 1, "timeRestore": false, "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"highlightAll\":false,\"version\":true,\"query\":{\"language\":\"lucene\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"default_field\":\"*\",\"query\":\"*\",\"filter\":[]}" + "searchSourceJSON": "{\"highlightAll\":false,\"version\":true,\"query\":{\"language\":\"lucene\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"default_field\":\"*\",\"query\":\"*\"}}},\"filter\":[]}" } }, "references": [ diff --git a/dashboards/dashboards/432af556-c5c0-4cc3-8166-b274b4e3a406.json b/dashboards/dashboards/432af556-c5c0-4cc3-8166-b274b4e3a406.json index 919af2e88..a48786060 100644 --- a/dashboards/dashboards/432af556-c5c0-4cc3-8166-b274b4e3a406.json +++ b/dashboards/dashboards/432af556-c5c0-4cc3-8166-b274b4e3a406.json @@ -18,7 +18,7 @@ "version": 1, "timeRestore": false, "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"highlightAll\":false,\"version\":true,\"query\":{\"language\":\"lucene\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"default_field\":\"*\",\"query\":\"*\",\"filter\":[]}" + "searchSourceJSON": "{\"highlightAll\":false,\"version\":true,\"query\":{\"language\":\"lucene\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"default_field\":\"*\",\"query\":\"*\"}}},\"filter\":[]}" } }, "references": [ diff --git a/dashboards/dashboards/50ced171-1b10-4c3f-8b67-2db9635661a6.json b/dashboards/dashboards/50ced171-1b10-4c3f-8b67-2db9635661a6.json index 2d3d8d0b0..d5fd80bc8 100644 --- a/dashboards/dashboards/50ced171-1b10-4c3f-8b67-2db9635661a6.json +++ b/dashboards/dashboards/50ced171-1b10-4c3f-8b67-2db9635661a6.json @@ -18,7 +18,7 @@ "version": 1, "timeRestore": false, "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"highlightAll\":false,\"version\":true,\"query\":{\"language\":\"lucene\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"default_field\":\"*\",\"query\":\"*\",\"filter\":[]}" + "searchSourceJSON": "{\"highlightAll\":false,\"version\":true,\"query\":{\"language\":\"lucene\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"default_field\":\"*\",\"query\":\"*\"}}},\"filter\":[]}" } }, "references": [ diff --git a/dashboards/dashboards/543118a9-02d7-43fe-b669-b8652177fc37.json b/dashboards/dashboards/543118a9-02d7-43fe-b669-b8652177fc37.json index d4a74ebf5..5f4c65802 100644 --- a/dashboards/dashboards/543118a9-02d7-43fe-b669-b8652177fc37.json +++ b/dashboards/dashboards/543118a9-02d7-43fe-b669-b8652177fc37.json @@ -18,7 +18,7 @@ "version": 1, "timeRestore": false, "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"highlightAll\":false,\"version\":true,\"query\":{\"language\":\"lucene\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"default_field\":\"*\",\"query\":\"*\",\"filter\":[]}" + "searchSourceJSON": "{\"highlightAll\":false,\"version\":true,\"query\":{\"language\":\"lucene\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"default_field\":\"*\",\"query\":\"*\"}}},\"filter\":[]}" } }, "references": [ diff --git a/dashboards/dashboards/7f41913f-cba8-43f5-82a8-241b7ead03e0.json b/dashboards/dashboards/7f41913f-cba8-43f5-82a8-241b7ead03e0.json index b1b74ea1c..462e488b4 100644 --- a/dashboards/dashboards/7f41913f-cba8-43f5-82a8-241b7ead03e0.json +++ b/dashboards/dashboards/7f41913f-cba8-43f5-82a8-241b7ead03e0.json @@ -18,7 +18,7 @@ "version": 1, "timeRestore": false, "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"highlightAll\":false,\"version\":true,\"query\":{\"language\":\"lucene\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"default_field\":\"*\",\"query\":\"*\",\"filter\":[]}" + "searchSourceJSON": "{\"highlightAll\":false,\"version\":true,\"query\":{\"language\":\"lucene\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"default_field\":\"*\",\"query\":\"*\"}}},\"filter\":[]}" } }, "references": [ diff --git a/dashboards/dashboards/92985909-dc29-4533-9e80-d3182a0ecf1d.json b/dashboards/dashboards/92985909-dc29-4533-9e80-d3182a0ecf1d.json index a9076e452..374bb560d 100644 --- a/dashboards/dashboards/92985909-dc29-4533-9e80-d3182a0ecf1d.json +++ b/dashboards/dashboards/92985909-dc29-4533-9e80-d3182a0ecf1d.json @@ -18,7 +18,7 @@ "version": 1, "timeRestore": false, "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"highlightAll\":false,\"version\":true,\"query\":{\"language\":\"lucene\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"default_field\":\"*\",\"query\":\"*\",\"filter\":[]}" + "searchSourceJSON": "{\"highlightAll\":false,\"version\":true,\"query\":{\"language\":\"lucene\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"default_field\":\"*\",\"query\":\"*\"}}},\"filter\":[]}" } }, "references": [ diff --git a/dashboards/dashboards/a7514350-eba6-11e9-a384-0fcf32210194.json b/dashboards/dashboards/a7514350-eba6-11e9-a384-0fcf32210194.json index c604ca109..2a94b378b 100644 --- a/dashboards/dashboards/a7514350-eba6-11e9-a384-0fcf32210194.json +++ b/dashboards/dashboards/a7514350-eba6-11e9-a384-0fcf32210194.json @@ -18,7 +18,7 @@ "version": 1, "timeRestore": false, "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"highlightAll\":false,\"version\":true,\"query\":{\"language\":\"lucene\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"default_field\":\"*\",\"query\":\"*\",\"filter\":[]}" + "searchSourceJSON": "{\"highlightAll\":false,\"version\":true,\"query\":{\"language\":\"lucene\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"default_field\":\"*\",\"query\":\"*\"}}},\"filter\":[]}" } }, "references": [ diff --git a/dashboards/dashboards/c2549e10-7f2e-11ea-9f8a-1fe1327e2cd2.json b/dashboards/dashboards/c2549e10-7f2e-11ea-9f8a-1fe1327e2cd2.json index bfc03d799..bdafb1f1e 100644 --- a/dashboards/dashboards/c2549e10-7f2e-11ea-9f8a-1fe1327e2cd2.json +++ b/dashboards/dashboards/c2549e10-7f2e-11ea-9f8a-1fe1327e2cd2.json @@ -18,7 +18,7 @@ "version": 1, "timeRestore": false, "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"highlightAll\":false,\"version\":true,\"query\":{\"language\":\"lucene\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"default_field\":\"*\",\"query\":\"*\",\"filter\":[]}" + "searchSourceJSON": "{\"highlightAll\":false,\"version\":true,\"query\":{\"language\":\"lucene\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"default_field\":\"*\",\"query\":\"*\"}}},\"filter\":[]}" } }, "references": [ diff --git a/dashboards/dashboards/caef3ade-d289-4d05-a511-149f3e97f238.json b/dashboards/dashboards/caef3ade-d289-4d05-a511-149f3e97f238.json index b28cef539..84da7d3be 100644 --- a/dashboards/dashboards/caef3ade-d289-4d05-a511-149f3e97f238.json +++ b/dashboards/dashboards/caef3ade-d289-4d05-a511-149f3e97f238.json @@ -18,7 +18,7 @@ "version": 1, "timeRestore": false, "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"highlightAll\":false,\"version\":true,\"query\":{\"language\":\"lucene\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"default_field\":\"*\",\"query\":\"*\",\"filter\":[]}" + "searchSourceJSON": "{\"highlightAll\":false,\"version\":true,\"query\":{\"language\":\"lucene\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"default_field\":\"*\",\"query\":\"*\"}}},\"filter\":[]}" } }, "references": [ diff --git a/dashboards/templates/composable/component/zeek.json b/dashboards/templates/composable/component/zeek.json index 2c94d0618..1c8b30860 100644 --- a/dashboards/templates/composable/component/zeek.json +++ b/dashboards/templates/composable/component/zeek.json @@ -2,7 +2,7 @@ "template": { "settings" : { "index" : { - "mapping.total_fields.limit" : "5000", + "mapping.total_fields.limit" : "6000", "mapping.nested_fields.limit" : "250", "max_docvalue_fields_search" : "200" } @@ -111,6 +111,7 @@ "zeek.gquic.tag_count": { "type": "integer" }, "zeek.gquic.user_agent": { "type": "keyword", "ignore_above": 256, "fields": { "text": { "type": "text" } } }, "zeek.gquic.version": { "type": "keyword" }, + "zeek.http.client_header_names": { "type": "keyword" }, "zeek.http.host": { "type": "keyword" }, "zeek.http.info_code": { "type": "short" }, "zeek.http.info_msg": { "type": "keyword" }, @@ -129,6 +130,7 @@ "zeek.http.resp_fuids": { "type": "keyword" }, "zeek.http.resp_mime_types": { "type": "keyword" }, "zeek.http.response_body_len": { "type": "long" }, + "zeek.http.server_header_names": { "type": "keyword" }, "zeek.http.status_code": { "type": "short" }, "zeek.http.status_msg": { "type": "keyword", "ignore_above": 1024 }, "zeek.http.tags": { "type": "keyword" }, @@ -199,6 +201,8 @@ "zeek.known_certs.issuer_subject": { "type": "keyword" }, "zeek.known_certs.serial": { "type": "keyword" }, "zeek.known_certs.subject": { "type": "keyword" }, + "zeek.known_routers.ttl": { "type": "integer" }, + "zeek.known_routers.hlim": { "type": "integer" }, "zeek.ldap.argument": { "type": "keyword", "ignore_above": 1024, "fields": { "text": { "type": "text", "norms": false } } }, "zeek.ldap.message_id": { "type": "keyword" }, "zeek.ldap.object": { "type": "keyword", "ignore_above": 1024, "fields": { "text": { "type": "text" } } }, diff --git a/dashboards/templates/composable/component/zeek_ot.json b/dashboards/templates/composable/component/zeek_ot.json index 6c49c793a..93877c03b 100644 --- a/dashboards/templates/composable/component/zeek_ot.json +++ b/dashboards/templates/composable/component/zeek_ot.json @@ -2,7 +2,7 @@ "template": { "settings" : { "index" : { - "mapping.total_fields.limit" : "5000", + "mapping.total_fields.limit" : "6000", "mapping.nested_fields.limit" : "250", "max_docvalue_fields_search" : "200" } diff --git a/dashboards/templates/malcolm_beats_template.json b/dashboards/templates/malcolm_beats_template.json index e98ac84e1..31a10a144 100644 --- a/dashboards/templates/malcolm_beats_template.json +++ b/dashboards/templates/malcolm_beats_template.json @@ -29,7 +29,7 @@ "template" :{ "settings" : { "index" : { - "mapping.total_fields.limit" : "5000", + "mapping.total_fields.limit" : "6000", "mapping.nested_fields.limit" : "250", "max_docvalue_fields_search" : "200" } diff --git a/dashboards/templates/malcolm_template.json b/dashboards/templates/malcolm_template.json index dee0b4564..6f4e6ddf8 100644 --- a/dashboards/templates/malcolm_template.json +++ b/dashboards/templates/malcolm_template.json @@ -31,7 +31,7 @@ "template" :{ "settings" : { "index" : { - "mapping.total_fields.limit" : "5000", + "mapping.total_fields.limit" : "6000", "mapping.nested_fields.limit" : "250", "max_docvalue_fields_search" : "200" } diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index c83693703..845c98cf2 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -5,7 +5,7 @@ services: build: context: . dockerfile: Dockerfiles/opensearch.Dockerfile - image: ghcr.io/idaholab/malcolm/opensearch:24.09.0 + image: ghcr.io/idaholab/malcolm/opensearch:24.10.0 # Technically the "hedgehog" profile doesn't have OpenSearch, but in that case # OPENSEARCH_PRIMARY will be set to remote, which means the container will # start but not actually run OpenSearch. It's included in both profiles to @@ -83,7 +83,7 @@ services: build: context: . dockerfile: Dockerfiles/dashboards-helper.Dockerfile - image: ghcr.io/idaholab/malcolm/dashboards-helper:24.09.0 + image: ghcr.io/idaholab/malcolm/dashboards-helper:24.10.0 profiles: ["malcolm"] userns_mode: keep-id logging: @@ -137,7 +137,7 @@ services: build: context: . dockerfile: Dockerfiles/dashboards.Dockerfile - image: ghcr.io/idaholab/malcolm/dashboards:24.09.0 + image: ghcr.io/idaholab/malcolm/dashboards:24.10.0 profiles: ["malcolm"] userns_mode: keep-id logging: @@ -184,7 +184,7 @@ services: build: context: . dockerfile: Dockerfiles/logstash.Dockerfile - image: ghcr.io/idaholab/malcolm/logstash-oss:24.09.0 + image: ghcr.io/idaholab/malcolm/logstash-oss:24.10.0 profiles: ["malcolm"] userns_mode: keep-id logging: @@ -299,7 +299,7 @@ services: build: context: . dockerfile: Dockerfiles/filebeat.Dockerfile - image: ghcr.io/idaholab/malcolm/filebeat-oss:24.09.0 + image: ghcr.io/idaholab/malcolm/filebeat-oss:24.10.0 profiles: ["malcolm", "hedgehog"] userns_mode: keep-id logging: @@ -376,7 +376,7 @@ services: build: context: . dockerfile: Dockerfiles/arkime.Dockerfile - image: ghcr.io/idaholab/malcolm/arkime:24.09.0 + image: ghcr.io/idaholab/malcolm/arkime:24.10.0 profiles: ["malcolm", "hedgehog"] userns_mode: keep-id logging: @@ -462,7 +462,7 @@ services: build: context: . dockerfile: Dockerfiles/arkime.Dockerfile - image: ghcr.io/idaholab/malcolm/arkime:24.09.0 + image: ghcr.io/idaholab/malcolm/arkime:24.10.0 profiles: ["malcolm", "hedgehog"] userns_mode: keep-id logging: @@ -547,7 +547,7 @@ services: build: context: . dockerfile: Dockerfiles/zeek.Dockerfile - image: ghcr.io/idaholab/malcolm/zeek:24.09.0 + image: ghcr.io/idaholab/malcolm/zeek:24.10.0 profiles: ["malcolm", "hedgehog"] userns_mode: keep-id logging: @@ -620,7 +620,7 @@ services: build: context: . dockerfile: Dockerfiles/zeek.Dockerfile - image: ghcr.io/idaholab/malcolm/zeek:24.09.0 + image: ghcr.io/idaholab/malcolm/zeek:24.10.0 profiles: ["malcolm", "hedgehog"] userns_mode: keep-id logging: @@ -685,7 +685,7 @@ services: build: context: . dockerfile: Dockerfiles/suricata.Dockerfile - image: ghcr.io/idaholab/malcolm/suricata:24.09.0 + image: ghcr.io/idaholab/malcolm/suricata:24.10.0 profiles: ["malcolm", "hedgehog"] userns_mode: keep-id logging: @@ -745,7 +745,7 @@ services: build: context: . dockerfile: Dockerfiles/suricata.Dockerfile - image: ghcr.io/idaholab/malcolm/suricata:24.09.0 + image: ghcr.io/idaholab/malcolm/suricata:24.10.0 profiles: ["malcolm", "hedgehog"] userns_mode: keep-id logging: @@ -803,7 +803,7 @@ services: build: context: . dockerfile: Dockerfiles/file-monitor.Dockerfile - image: ghcr.io/idaholab/malcolm/file-monitor:24.09.0 + image: ghcr.io/idaholab/malcolm/file-monitor:24.10.0 profiles: ["malcolm", "hedgehog"] userns_mode: keep-id logging: @@ -859,7 +859,7 @@ services: build: context: . dockerfile: Dockerfiles/pcap-capture.Dockerfile - image: ghcr.io/idaholab/malcolm/pcap-capture:24.09.0 + image: ghcr.io/idaholab/malcolm/pcap-capture:24.10.0 profiles: ["malcolm", "hedgehog"] userns_mode: keep-id logging: @@ -906,7 +906,7 @@ services: build: context: . dockerfile: Dockerfiles/pcap-monitor.Dockerfile - image: ghcr.io/idaholab/malcolm/pcap-monitor:24.09.0 + image: ghcr.io/idaholab/malcolm/pcap-monitor:24.10.0 profiles: ["malcolm", "hedgehog"] userns_mode: keep-id logging: @@ -961,7 +961,7 @@ services: build: context: . dockerfile: Dockerfiles/file-upload.Dockerfile - image: ghcr.io/idaholab/malcolm/file-upload:24.09.0 + image: ghcr.io/idaholab/malcolm/file-upload:24.10.0 profiles: ["malcolm"] userns_mode: keep-id logging: @@ -1003,7 +1003,7 @@ services: retries: 3 start_period: 60s htadmin: - image: ghcr.io/idaholab/malcolm/htadmin:24.09.0 + image: ghcr.io/idaholab/malcolm/htadmin:24.10.0 profiles: ["malcolm"] userns_mode: keep-id logging: @@ -1056,7 +1056,7 @@ services: retries: 3 start_period: 60s freq: - image: ghcr.io/idaholab/malcolm/freq:24.09.0 + image: ghcr.io/idaholab/malcolm/freq:24.10.0 profiles: ["malcolm"] userns_mode: keep-id logging: @@ -1094,7 +1094,7 @@ services: retries: 3 start_period: 60s netbox: - image: ghcr.io/idaholab/malcolm/netbox:24.09.0 + image: ghcr.io/idaholab/malcolm/netbox:24.10.0 profiles: ["malcolm"] userns_mode: keep-id logging: @@ -1161,7 +1161,7 @@ services: retries: 3 start_period: 120s netbox-postgres: - image: ghcr.io/idaholab/malcolm/postgresql:24.09.0 + image: ghcr.io/idaholab/malcolm/postgresql:24.10.0 profiles: ["malcolm"] userns_mode: keep-id logging: @@ -1205,7 +1205,7 @@ services: retries: 3 start_period: 45s netbox-redis: - image: ghcr.io/idaholab/malcolm/redis:24.09.0 + image: ghcr.io/idaholab/malcolm/redis:24.10.0 profiles: ["malcolm"] userns_mode: keep-id logging: @@ -1253,7 +1253,7 @@ services: retries: 3 start_period: 45s netbox-redis-cache: - image: ghcr.io/idaholab/malcolm/redis:24.09.0 + image: ghcr.io/idaholab/malcolm/redis:24.10.0 profiles: ["malcolm"] userns_mode: keep-id logging: @@ -1296,7 +1296,7 @@ services: retries: 3 start_period: 45s api: - image: ghcr.io/idaholab/malcolm/api:24.09.0 + image: ghcr.io/idaholab/malcolm/api:24.10.0 profiles: ["malcolm"] userns_mode: keep-id logging: @@ -1345,7 +1345,7 @@ services: build: context: . dockerfile: Dockerfiles/nginx.Dockerfile - image: ghcr.io/idaholab/malcolm/nginx-proxy:24.09.0 + image: ghcr.io/idaholab/malcolm/nginx-proxy:24.10.0 profiles: ["malcolm"] userns_mode: keep-id logging: diff --git a/docker-compose.yml b/docker-compose.yml index 9fd86ba4f..b2260d162 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,7 +2,7 @@ services: opensearch: - image: ghcr.io/idaholab/malcolm/opensearch:24.09.0 + image: ghcr.io/idaholab/malcolm/opensearch:24.10.0 # Technically the "hedgehog" profile doesn't have OpenSearch, but in that case # OPENSEARCH_PRIMARY will be set to remote, which means the container will # start but not actually run OpenSearch. It's included in both profiles to @@ -77,7 +77,7 @@ services: retries: 3 start_period: 180s dashboards-helper: - image: ghcr.io/idaholab/malcolm/dashboards-helper:24.09.0 + image: ghcr.io/idaholab/malcolm/dashboards-helper:24.10.0 profiles: ["malcolm"] userns_mode: keep-id logging: @@ -128,7 +128,7 @@ services: retries: 3 start_period: 30s dashboards: - image: ghcr.io/idaholab/malcolm/dashboards:24.09.0 + image: ghcr.io/idaholab/malcolm/dashboards:24.10.0 profiles: ["malcolm"] userns_mode: keep-id logging: @@ -172,7 +172,7 @@ services: retries: 3 start_period: 210s logstash: - image: ghcr.io/idaholab/malcolm/logstash-oss:24.09.0 + image: ghcr.io/idaholab/malcolm/logstash-oss:24.10.0 profiles: ["malcolm"] userns_mode: keep-id logging: @@ -260,7 +260,7 @@ services: retries: 3 start_period: 600s filebeat: - image: ghcr.io/idaholab/malcolm/filebeat-oss:24.09.0 + image: ghcr.io/idaholab/malcolm/filebeat-oss:24.10.0 profiles: ["malcolm", "hedgehog"] userns_mode: keep-id logging: @@ -334,7 +334,7 @@ services: retries: 3 start_period: 60s arkime: - image: ghcr.io/idaholab/malcolm/arkime:24.09.0 + image: ghcr.io/idaholab/malcolm/arkime:24.10.0 profiles: ["malcolm", "hedgehog"] userns_mode: keep-id logging: @@ -399,7 +399,7 @@ services: retries: 3 start_period: 210s arkime-live: - image: ghcr.io/idaholab/malcolm/arkime:24.09.0 + image: ghcr.io/idaholab/malcolm/arkime:24.10.0 profiles: ["malcolm", "hedgehog"] userns_mode: keep-id logging: @@ -463,7 +463,7 @@ services: source: ./pcap target: /data/pcap zeek: - image: ghcr.io/idaholab/malcolm/zeek:24.09.0 + image: ghcr.io/idaholab/malcolm/zeek:24.10.0 profiles: ["malcolm", "hedgehog"] userns_mode: keep-id logging: @@ -527,7 +527,7 @@ services: retries: 3 start_period: 60s zeek-live: - image: ghcr.io/idaholab/malcolm/zeek:24.09.0 + image: ghcr.io/idaholab/malcolm/zeek:24.10.0 profiles: ["malcolm", "hedgehog"] userns_mode: keep-id logging: @@ -583,7 +583,7 @@ services: target: /opt/zeek/share/zeek/site/custom read_only: true suricata: - image: ghcr.io/idaholab/malcolm/suricata:24.09.0 + image: ghcr.io/idaholab/malcolm/suricata:24.10.0 profiles: ["malcolm", "hedgehog"] userns_mode: keep-id logging: @@ -640,7 +640,7 @@ services: retries: 3 start_period: 120s suricata-live: - image: ghcr.io/idaholab/malcolm/suricata:24.09.0 + image: ghcr.io/idaholab/malcolm/suricata:24.10.0 profiles: ["malcolm", "hedgehog"] userns_mode: keep-id logging: @@ -695,7 +695,7 @@ services: target: /opt/suricata/include-configs read_only: true file-monitor: - image: ghcr.io/idaholab/malcolm/file-monitor:24.09.0 + image: ghcr.io/idaholab/malcolm/file-monitor:24.10.0 profiles: ["malcolm", "hedgehog"] userns_mode: keep-id logging: @@ -748,7 +748,7 @@ services: retries: 3 start_period: 60s pcap-capture: - image: ghcr.io/idaholab/malcolm/pcap-capture:24.09.0 + image: ghcr.io/idaholab/malcolm/pcap-capture:24.10.0 profiles: ["malcolm", "hedgehog"] userns_mode: keep-id logging: @@ -792,7 +792,7 @@ services: source: ./pcap/upload target: /pcap pcap-monitor: - image: ghcr.io/idaholab/malcolm/pcap-monitor:24.09.0 + image: ghcr.io/idaholab/malcolm/pcap-monitor:24.10.0 profiles: ["malcolm", "hedgehog"] userns_mode: keep-id logging: @@ -844,7 +844,7 @@ services: retries: 3 start_period: 90s upload: - image: ghcr.io/idaholab/malcolm/file-upload:24.09.0 + image: ghcr.io/idaholab/malcolm/file-upload:24.10.0 profiles: ["malcolm"] userns_mode: keep-id logging: @@ -886,7 +886,7 @@ services: retries: 3 start_period: 60s htadmin: - image: ghcr.io/idaholab/malcolm/htadmin:24.09.0 + image: ghcr.io/idaholab/malcolm/htadmin:24.10.0 profiles: ["malcolm"] userns_mode: keep-id logging: @@ -936,7 +936,7 @@ services: retries: 3 start_period: 60s freq: - image: ghcr.io/idaholab/malcolm/freq:24.09.0 + image: ghcr.io/idaholab/malcolm/freq:24.10.0 profiles: ["malcolm"] userns_mode: keep-id logging: @@ -971,7 +971,7 @@ services: retries: 3 start_period: 60s netbox: - image: ghcr.io/idaholab/malcolm/netbox:24.09.0 + image: ghcr.io/idaholab/malcolm/netbox:24.10.0 profiles: ["malcolm"] userns_mode: keep-id logging: @@ -1035,7 +1035,7 @@ services: retries: 3 start_period: 120s netbox-postgres: - image: ghcr.io/idaholab/malcolm/postgresql:24.09.0 + image: ghcr.io/idaholab/malcolm/postgresql:24.10.0 profiles: ["malcolm"] userns_mode: keep-id logging: @@ -1076,7 +1076,7 @@ services: retries: 3 start_period: 45s netbox-redis: - image: ghcr.io/idaholab/malcolm/redis:24.09.0 + image: ghcr.io/idaholab/malcolm/redis:24.10.0 profiles: ["malcolm"] userns_mode: keep-id logging: @@ -1121,7 +1121,7 @@ services: retries: 3 start_period: 45s netbox-redis-cache: - image: ghcr.io/idaholab/malcolm/redis:24.09.0 + image: ghcr.io/idaholab/malcolm/redis:24.10.0 profiles: ["malcolm"] userns_mode: keep-id logging: @@ -1161,7 +1161,7 @@ services: retries: 3 start_period: 45s api: - image: ghcr.io/idaholab/malcolm/api:24.09.0 + image: ghcr.io/idaholab/malcolm/api:24.10.0 profiles: ["malcolm"] userns_mode: keep-id logging: @@ -1204,7 +1204,7 @@ services: retries: 3 start_period: 60s nginx-proxy: - image: ghcr.io/idaholab/malcolm/nginx-proxy:24.09.0 + image: ghcr.io/idaholab/malcolm/nginx-proxy:24.10.0 profiles: ["malcolm"] userns_mode: keep-id logging: diff --git a/docs/api-version.md b/docs/api-version.md index 01cc75a29..0c0fefe76 100644 --- a/docs/api-version.md +++ b/docs/api-version.md @@ -49,6 +49,6 @@ Returns version information about Malcolm and version/[health](https://opensearc } }, "sha": "dad18b1", - "version": "24.09.0" + "version": "24.10.0" } ``` diff --git a/docs/contributing-github-runners.md b/docs/contributing-github-runners.md index 315439767..256ea8ba9 100644 --- a/docs/contributing-github-runners.md +++ b/docs/contributing-github-runners.md @@ -104,7 +104,7 @@ Each container build workflow actually runs two paths in parallel: one for build ## Convenience scripts for development -As mentioned earlier, Malcolm images built using the instructions in this document are are named according to the pattern `ghcr.io/username/malcolm/image:branch`. However, note that the `image:` values found in [`docker-compose.yml`]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/docker-compose.yml) (and in the [Kubernetes](kubernetes.md#Kubernetes) [manifests]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/kubernetes/)) look like `ghcr.io/idaholab/malcolm/opensearch:24.09.0`, using the OpenSearch container as an example. To run a local instance of Malcolm using these images instead of the official `ghcr.io/idaholab` ones, users will need to edit their `docker-compose.yml` file(s) and replace the `image:` tags according to this new pattern, or use the bash helper script [`./scripts/github_image_helper.sh`]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/scripts/github_image_helper.sh) to pull the repository images and re-tag them with `ghcr.io/idaholab` and the current Malcolm version (e.g., `24.09.0`). +As mentioned earlier, Malcolm images built using the instructions in this document are are named according to the pattern `ghcr.io/username/malcolm/image:branch`. However, note that the `image:` values found in [`docker-compose.yml`]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/docker-compose.yml) (and in the [Kubernetes](kubernetes.md#Kubernetes) [manifests]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/kubernetes/)) look like `ghcr.io/idaholab/malcolm/opensearch:24.10.0`, using the OpenSearch container as an example. To run a local instance of Malcolm using these images instead of the official `ghcr.io/idaholab` ones, users will need to edit their `docker-compose.yml` file(s) and replace the `image:` tags according to this new pattern, or use the bash helper script [`./scripts/github_image_helper.sh`]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/scripts/github_image_helper.sh) to pull the repository images and re-tag them with `ghcr.io/idaholab` and the current Malcolm version (e.g., `24.10.0`). Before explaining that script, a discussion of the workflow files for the [Hedgehog Linux](live-analysis.md#Hedgehog) ([hedgehog-iso-build-docker-wrap-push-ghcr.yml ]({{ site.github.repository_url }}/tree/{{ site.github.build_revision }}/.github/workflows/hedgehog-iso-build-docker-wrap-push-ghcr.yml)) and [Malcolm](malcolm-iso.md#ISO) ([malcolm-iso-build-docker-wrap-push-ghcr.yml @@ -141,9 +141,9 @@ These menu options are described below: 3. GithubTriggerPackagesBuild * This option will trigger a [repository dispatch](https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#repository_dispatch) via the [GitHub API](https://docs.github.com/en/rest/actions/workflows?apiVersion=2022-11-28#create-a-workflow-dispatch-event) using `curl`. In order for this operation to work, an environment variable named `GITHUB_OAUTH_TOKEN` must be defined containing a [personal access token](https://docs.github.com/en/rest/authentication/authenticating-to-the-rest-api?apiVersion=2022-11-28#basic-authentication) created for your GitHub user account with the "Actions (write)" repository permission. 4. PullAndTagGithubWorkflowImages - * This option will pull latest Malcolm container images (excluding the installer ISO wrapper container images) from ghcr.io for the user's Malcolm fork, and re-tag them with `ghcr.io/idaholab` and the current Malcolm version (e.g., `24.09.0`) so that they may be run without modifying the local `docker-compose.yml` file. This is probably the option users will select most often. Note that this is different from the action performed in steps 1 and 0 above: this pulls the images directly from the container registry, it does **not** extract them from the Malcolm installer ISO wrapper container image. + * This option will pull latest Malcolm container images (excluding the installer ISO wrapper container images) from ghcr.io for the user's Malcolm fork, and re-tag them with `ghcr.io/idaholab` and the current Malcolm version (e.g., `24.10.0`) so that they may be run without modifying the local `docker-compose.yml` file. This is probably the option users will select most often. Note that this is different from the action performed in steps 1 and 0 above: this pulls the images directly from the container registry, it does **not** extract them from the Malcolm installer ISO wrapper container image. 5. PullAndTagGithubWorkflowISOImages - * This option will pull latest Malcolm installer ISO wrapper container images from ghcr.io for the user's Malcolm fork, and re-tag them with `ghcr.io/idaholab` and the current Malcolm version (e.g., `24.09.0`). + * This option will pull latest Malcolm installer ISO wrapper container images from ghcr.io for the user's Malcolm fork, and re-tag them with `ghcr.io/idaholab` and the current Malcolm version (e.g., `24.10.0`). The script can also be run non-interactively by specifying the option number on the command line (e.g., `./scripts/github_image_helper.sh 4`). @@ -182,7 +182,7 @@ $ ./scripts/github_image_helper.sh 5 PullAndTagGithubWorkflowISOImages Operation:4 PullAndTagGithubWorkflowImages -Pulling images from ghcr.io/romeogdetlevjr (main) and tagging as 24.09.0... +Pulling images from ghcr.io/romeogdetlevjr (main) and tagging as 24.10.0... main: Pulling from romeogdetlevjr/malcolm/api Digest: sha256:xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx Status: Image is up to date for ghcr.io/romeogdetlevjr/malcolm/api:main @@ -196,46 +196,46 @@ ghcr.io/romeogdetlevjr/malcolm/zeek:main xxxxxxxxxxxx: Pull complete ``` -Verify that the images were pulled. Note that users will see two tags for each image: one tagged with the username and branch (e.g., `ghcr.io/romeogdetlevjr/malcolm/api:main`) and another tagged with `ghcr.io/idaholab` and the Malcolm version (e.g., `ghcr.io/idaholab/malcolm/api:24.09.0`). +Verify that the images were pulled. Note that users will see two tags for each image: one tagged with the username and branch (e.g., `ghcr.io/romeogdetlevjr/malcolm/api:main`) and another tagged with `ghcr.io/idaholab` and the Malcolm version (e.g., `ghcr.io/idaholab/malcolm/api:24.10.0`). ```bash $ docker images | grep romeogdetlevjr/malcolm -ghcr.io/idaholab/malcolm/zeek 24.09.0 xxxxxxxxxxxx 10 minutes ago 1.39GB +ghcr.io/idaholab/malcolm/zeek 24.10.0 xxxxxxxxxxxx 10 minutes ago 1.39GB ghcr.io/romeogdetlevjr/malcolm/zeek main xxxxxxxxxxxx 10 minutes ago 1.39GB -ghcr.io/idaholab/malcolm/dashboards 24.09.0 xxxxxxxxxxxx 13 minutes ago 1.55GB +ghcr.io/idaholab/malcolm/dashboards 24.10.0 xxxxxxxxxxxx 13 minutes ago 1.55GB ghcr.io/romeogdetlevjr/malcolm/dashboards main xxxxxxxxxxxx 13 minutes ago 1.55GB -ghcr.io/idaholab/malcolm/suricata 24.09.0 xxxxxxxxxxxx 14 minutes ago 339MB +ghcr.io/idaholab/malcolm/suricata 24.10.0 xxxxxxxxxxxx 14 minutes ago 339MB ghcr.io/romeogdetlevjr/malcolm/suricata main xxxxxxxxxxxx 14 minutes ago 339MB -ghcr.io/idaholab/malcolm/file-monitor 24.09.0 xxxxxxxxxxxx 15 minutes ago 712MB +ghcr.io/idaholab/malcolm/file-monitor 24.10.0 xxxxxxxxxxxx 15 minutes ago 712MB ghcr.io/romeogdetlevjr/malcolm/file-monitor main xxxxxxxxxxxx 15 minutes ago 712MB -ghcr.io/idaholab/malcolm/redis 24.09.0 xxxxxxxxxxxx 15 minutes ago 55.4MB +ghcr.io/idaholab/malcolm/redis 24.10.0 xxxxxxxxxxxx 15 minutes ago 55.4MB ghcr.io/romeogdetlevjr/malcolm/redis main xxxxxxxxxxxx 15 minutes ago 55.4MB -ghcr.io/idaholab/malcolm/nginx-proxy 24.09.0 xxxxxxxxxxxx 16 minutes ago 160MB +ghcr.io/idaholab/malcolm/nginx-proxy 24.10.0 xxxxxxxxxxxx 16 minutes ago 160MB ghcr.io/romeogdetlevjr/malcolm/nginx-proxy main xxxxxxxxxxxx 16 minutes ago 160MB -ghcr.io/idaholab/malcolm/pcap-capture 24.09.0 xxxxxxxxxxxx 16 minutes ago 137MB +ghcr.io/idaholab/malcolm/pcap-capture 24.10.0 xxxxxxxxxxxx 16 minutes ago 137MB ghcr.io/romeogdetlevjr/malcolm/pcap-capture main xxxxxxxxxxxx 16 minutes ago 137MB -ghcr.io/idaholab/malcolm/htadmin 24.09.0 xxxxxxxxxxxx 16 minutes ago 246MB +ghcr.io/idaholab/malcolm/htadmin 24.10.0 xxxxxxxxxxxx 16 minutes ago 246MB ghcr.io/romeogdetlevjr/malcolm/htadmin main xxxxxxxxxxxx 16 minutes ago 246MB ghcr.io/romeogdetlevjr/malcolm/file-upload main xxxxxxxxxxxx 16 minutes ago 250MB -ghcr.io/idaholab/malcolm/file-upload 24.09.0 xxxxxxxxxxxx 16 minutes ago 250MB -ghcr.io/idaholab/malcolm/logstash-oss 24.09.0 xxxxxxxxxxxx 16 minutes ago 1.49GB +ghcr.io/idaholab/malcolm/file-upload 24.10.0 xxxxxxxxxxxx 16 minutes ago 250MB +ghcr.io/idaholab/malcolm/logstash-oss 24.10.0 xxxxxxxxxxxx 16 minutes ago 1.49GB ghcr.io/romeogdetlevjr/malcolm/logstash-oss main xxxxxxxxxxxx 16 minutes ago 1.49GB -ghcr.io/idaholab/malcolm/netbox 24.09.0 xxxxxxxxxxxx 17 minutes ago 1.66GB +ghcr.io/idaholab/malcolm/netbox 24.10.0 xxxxxxxxxxxx 17 minutes ago 1.66GB ghcr.io/romeogdetlevjr/malcolm/netbox main xxxxxxxxxxxx 17 minutes ago 1.66GB ghcr.io/romeogdetlevjr/malcolm/filebeat-oss main xxxxxxxxxxxx 18 minutes ago 405MB -ghcr.io/idaholab/malcolm/filebeat-oss 24.09.0 xxxxxxxxxxxx 18 minutes ago 405MB +ghcr.io/idaholab/malcolm/filebeat-oss 24.10.0 xxxxxxxxxxxx 18 minutes ago 405MB ghcr.io/romeogdetlevjr/malcolm/postgresql main xxxxxxxxxxxx 18 minutes ago 303MB -ghcr.io/idaholab/malcolm/postgresql 24.09.0 xxxxxxxxxxxx 18 minutes ago 303MB -ghcr.io/idaholab/malcolm/arkime 24.09.0 xxxxxxxxxxxx 18 minutes ago 802MB +ghcr.io/idaholab/malcolm/postgresql 24.10.0 xxxxxxxxxxxx 18 minutes ago 303MB +ghcr.io/idaholab/malcolm/arkime 24.10.0 xxxxxxxxxxxx 18 minutes ago 802MB ghcr.io/romeogdetlevjr/malcolm/arkime main xxxxxxxxxxxx 18 minutes ago 802MB -ghcr.io/idaholab/malcolm/opensearch 24.09.0 xxxxxxxxxxxx 18 minutes ago 1.42GB +ghcr.io/idaholab/malcolm/opensearch 24.10.0 xxxxxxxxxxxx 18 minutes ago 1.42GB ghcr.io/romeogdetlevjr/malcolm/opensearch main xxxxxxxxxxxx 18 minutes ago 1.42GB -ghcr.io/idaholab/malcolm/pcap-monitor 24.09.0 xxxxxxxxxxxx 18 minutes ago 176MB +ghcr.io/idaholab/malcolm/pcap-monitor 24.10.0 xxxxxxxxxxxx 18 minutes ago 176MB ghcr.io/romeogdetlevjr/malcolm/pcap-monitor main xxxxxxxxxxxx 18 minutes ago 176MB -ghcr.io/idaholab/malcolm/dashboards-helper 24.09.0 xxxxxxxxxxxx 18 minutes ago 233MB +ghcr.io/idaholab/malcolm/dashboards-helper 24.10.0 xxxxxxxxxxxx 18 minutes ago 233MB ghcr.io/romeogdetlevjr/malcolm/dashboards-helpermain xxxxxxxxxxxx 18 minutes ago 233MB -ghcr.io/idaholab/malcolm/freq 24.09.0 xxxxxxxxxxxx 18 minutes ago 153MB +ghcr.io/idaholab/malcolm/freq 24.10.0 xxxxxxxxxxxx 18 minutes ago 153MB ghcr.io/romeogdetlevjr/malcolm/freq main xxxxxxxxxxxx 18 minutes ago 153MB -ghcr.io/idaholab/malcolm/api 24.09.0 xxxxxxxxxxxx 18 minutes ago 169MB +ghcr.io/idaholab/malcolm/api 24.10.0 xxxxxxxxxxxx 18 minutes ago 169MB ghcr.io/romeogdetlevjr/malcolm/api main xxxxxxxxxxxx 18 minutes ago 169MB ``` diff --git a/docs/contributing-pcap.md b/docs/contributing-pcap.md index 5b56cd07c..c0feeab77 100644 --- a/docs/contributing-pcap.md +++ b/docs/contributing-pcap.md @@ -1,6 +1,6 @@ # PCAP processors -When a PCAP is uploaded (either through Malcolm's [upload web interface](upload.md#Upload) or just copied manually into the `./pcap/upload` directory), the `pcap-monitor` container has a script that picks up those PCAP files and publishes to a [ZeroMQ](https://zeromq.org/) topic that can be subscribed to by any other process that wants to analyze that PCAP. In Malcolm (at the time of the [v24.09.0 release]({{ site.github.repository_url }}/releases/tag/v24.09.0)), there are three such ZeroMQ topics: the `zeek`, `suricata` and `arkime` containers. These actually share the [same script]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/shared/bin/pcap_processor.py) to run the PCAP through Zeek, Suricata, and Arkime, respectively. For an example to follow, the `zeek` container is the less complicated of the two. To integrate a new PCAP processing tool into Malcolm (named `cooltool` for this example) the process would entail: +When a PCAP is uploaded (either through Malcolm's [upload web interface](upload.md#Upload) or just copied manually into the `./pcap/upload` directory), the `pcap-monitor` container has a script that picks up those PCAP files and publishes to a [ZeroMQ](https://zeromq.org/) topic that can be subscribed to by any other process that wants to analyze that PCAP. In Malcolm (at the time of the [v24.10.0 release]({{ site.github.repository_url }}/releases/tag/v24.10.0)), there are three such ZeroMQ topics: the `zeek`, `suricata` and `arkime` containers. These actually share the [same script]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/shared/bin/pcap_processor.py) to run the PCAP through Zeek, Suricata, and Arkime, respectively. For an example to follow, the `zeek` container is the less complicated of the two. To integrate a new PCAP processing tool into Malcolm (named `cooltool` for this example) the process would entail: 1. Define the service as instructed in the [Adding a new service](contributing-new-image.md#NewImage) section * Note how the existing `zeek` and `arkime` services use [bind mounts](contributing-local-modifications.md#Bind) to access the local `./pcap` directory diff --git a/docs/hedgehog-iso-build.md b/docs/hedgehog-iso-build.md index 960bf650a..7713cecb2 100644 --- a/docs/hedgehog-iso-build.md +++ b/docs/hedgehog-iso-build.md @@ -1,19 +1,21 @@ # Appendix A - Generating the ISO -Official downloads of the Hedgehog Linux installer ISO are not provided: however, it can be built easily on an internet-connected Linux host with Vagrant: +Official downloads of the Hedgehog Linux installer ISO [can be downloaded](download.md#DownloadISOs) from the GitHub releases page. It can also be built easily on an Internet-connected system with Vagrant: * [Vagrant](https://www.vagrantup.com/) - - [`vagrant-reload`](https://github.com/aidanns/vagrant-reload) plugin - [`vagrant-sshfs`](https://github.com/dustymabe/vagrant-sshfs) plugin - [`bento/debian-12`](https://app.vagrantup.com/bento/boxes/debian-12) Vagrant box -The build should work with either the [VirtualBox](https://www.virtualbox.org/) provider or the [libvirt](https://libvirt.org/) provider: +The build should work with a variety of [Vagrant providers](https://developer.hashicorp.com/vagrant/docs/providers): -* [VirtualBox](https://www.virtualbox.org/) [provider](https://www.vagrantup.com/docs/providers/virtualbox) - - [`vagrant-vbguest`](https://github.com/dotless-de/vagrant-vbguest) plugin +* [VMware](https://www.vmware.com/) [provider](https://developer.hashicorp.com/vagrant/docs/providers/vmware) + - [`vagrant-vmware-desktop`](https://github.com/hashicorp/vagrant-vmware-desktop) plugin * [libvirt](https://libvirt.org/) - [`vagrant-libvirt`](https://github.com/vagrant-libvirt/vagrant-libvirt) provider plugin - [`vagrant-mutate`](https://github.com/sciurus/vagrant-mutate) plugin to convert [`bento/debian-12`](https://app.vagrantup.com/bento/boxes/debian-12) Vagrant box to `libvirt` format +* [VirtualBox](https://www.virtualbox.org/) [provider](https://developer.hashicorp.com/vagrant/docs/providers/virtualbox) + - [`vagrant-vbguest`](https://github.com/dotless-de/vagrant-vbguest) plugin + To perform a clean build the Hedgehog Linux installer ISO, navigate to your local [Malcolm]({{ site.github.repository_url }}/) working copy and run: @@ -29,8 +31,8 @@ Building the ISO may take 90 minutes or more depending on your system. As the bu ``` … -Finished, created "/sensor-build/hedgehog-24.09.0.iso" +Finished, created "/sensor-build/hedgehog-24.10.0.iso" … ``` -Alternately, if you have forked Malcolm on GitHub, [workflow files]({{ site.github.repository_url }}/tree/{{ site.github.build_revision }}/.github/workflows/) are provided which contain instructions for GitHub to build the images and Hedgehog and [Malcolm]({{ site.github.repository_url }}) installer ISOs, specifically [`hedgehog-iso-build-docker-wrap-push-ghcr.yml`]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/.github/workflows/hedgehog-iso-build-docker-wrap-push-ghcr.yml) for the Hedgehog ISO. The resulting ISO file is wrapped in a image that provides an HTTP server from which the ISO may be downloaded. \ No newline at end of file +Alternately, if you have forked Malcolm on GitHub, [workflow files]({{ site.github.repository_url }}/tree/{{ site.github.build_revision }}/.github/workflows/) are provided which contain instructions for GitHub to build the images and Hedgehog and [Malcolm]({{ site.github.repository_url }}) installer ISOs, specifically [`hedgehog-iso-build-docker-wrap-push-ghcr.yml`]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/.github/workflows/hedgehog-iso-build-docker-wrap-push-ghcr.yml) for the Hedgehog ISO. The resulting ISO file is wrapped in a image that provides an HTTP server from which the ISO may be downloaded. See [Using GitHub runners to build Malcolm images](contributing-github-runners.md#GitHubRunners) for more information. \ No newline at end of file diff --git a/docs/hedgehog-raspi-build.md b/docs/hedgehog-raspi-build.md index 6d1f0cf8d..1a1267e9a 100644 --- a/docs/hedgehog-raspi-build.md +++ b/docs/hedgehog-raspi-build.md @@ -10,19 +10,21 @@ Hedgehog Linux can [also be built]({{ site.github.repository_url }}/tree/{{ site ## Building the Image -Official downloads of the Hedgehog Linux Raspberry Pi image are not provided: however, it can be built easily on an internet-connected Linux host with Vagrant: +Official downloads of the Hedgehog Linux Raspberry Pi image [can be downloaded](download.md#DownloadISOs) from the GitHub releases page. It can also be built easily on an Internet-connected system with Vagrant: * [Vagrant](https://www.vagrantup.com/) - [`vagrant-sshfs`](https://github.com/dustymabe/vagrant-sshfs) plugin - [`bento/debian-12`](https://app.vagrantup.com/bento/boxes/debian-12) Vagrant box -The build should work with either the [VirtualBox](https://www.virtualbox.org/) provider or the [libvirt](https://libvirt.org/) provider: +The build should work with a variety of [Vagrant providers](https://developer.hashicorp.com/vagrant/docs/providers): -* [VirtualBox](https://www.virtualbox.org/) [provider](https://www.vagrantup.com/docs/providers/virtualbox) - - [`vagrant-vbguest`](https://github.com/dotless-de/vagrant-vbguest) plugin +* [VMware](https://www.vmware.com/) [provider](https://developer.hashicorp.com/vagrant/docs/providers/vmware) + - [`vagrant-vmware-desktop`](https://github.com/hashicorp/vagrant-vmware-desktop) plugin * [libvirt](https://libvirt.org/) - [`vagrant-libvirt`](https://github.com/vagrant-libvirt/vagrant-libvirt) provider plugin - [`vagrant-mutate`](https://github.com/sciurus/vagrant-mutate) plugin to convert [`bento/debian-12`](https://app.vagrantup.com/bento/boxes/debian-12) Vagrant box to `libvirt` format +* [VirtualBox](https://www.virtualbox.org/) [provider](https://developer.hashicorp.com/vagrant/docs/providers/virtualbox) + - [`vagrant-vbguest`](https://github.com/dotless-de/vagrant-vbguest) plugin To perform a clean build the Hedgehog Linux Raspberry Pi image, navigate to your local [Malcolm]({{ site.github.repository_url }}/) working copy and run: diff --git a/docs/kubernetes.md b/docs/kubernetes.md index 7887dbcff..8bb51770b 100644 --- a/docs/kubernetes.md +++ b/docs/kubernetes.md @@ -272,28 +272,28 @@ agent2 | agent2 | 192.168.56.12 | agent2 | k3s | 6000m | agent1 | agent1 | 192.168.56.11 | agent1 | k3s | 6000m | 861.34m | 14.36% | 19.55Gi | 9.29Gi | 61.28Gi | 11 | Pod Name | State | Pod IP | Pod Kind | Worker Node | CPU Usage | Memory Usage | Container Name:Restarts | Container Image | -api-deployment-6f4686cf59-bn286 | Running | 10.42.2.14 | ReplicaSet | agent1 | 0.11m | 59.62Mi | api-container:0 | api:24.09.0 | -file-monitor-deployment-855646bd75-vk7st | Running | 10.42.2.16 | ReplicaSet | agent1 | 8.47m | 1.46Gi | file-monitor-container:0 | file-monitor:24.09.0 | -zeek-live-deployment-64b69d4b6f-947vr | Running | 10.42.2.17 | ReplicaSet | agent1 | 0.02m | 12.44Mi | zeek-live-container:0 | zeek:24.09.0 | -dashboards-helper-deployment-69dc54f6b6-ln4sq | Running | 10.42.2.15 | ReplicaSet | agent1 | 10.77m | 38.43Mi | dashboards-helper-container:0 | dashboards-helper:24.09.0 | -upload-deployment-586568844b-4jnk9 | Running | 10.42.2.18 | ReplicaSet | agent1 | 0.15m | 29.78Mi | upload-container:0 | file-upload:24.09.0 | -filebeat-deployment-6ff8bc444f-t7h49 | Running | 10.42.2.20 | ReplicaSet | agent1 | 2.84m | 70.71Mi | filebeat-container:0 | filebeat-oss:24.09.0 | -zeek-offline-deployment-844f4865bd-g2sdm | Running | 10.42.2.21 | ReplicaSet | agent1 | 0.17m | 41.92Mi | zeek-offline-container:0 | zeek:24.09.0 | -logstash-deployment-6fbc9fdcd5-hwx8s | Running | 10.42.2.22 | ReplicaSet | agent1 | 85.55m | 2.91Gi | logstash-container:0 | logstash-oss:24.09.0 | -netbox-deployment-cdcff4977-hbbw5 | Running | 10.42.2.23 | ReplicaSet | agent1 | 807.64m | 702.86Mi | netbox-container:0 | netbox:24.09.0 | -suricata-offline-deployment-6ccdb89478-z5696 | Running | 10.42.2.19 | ReplicaSet | agent1 | 0.22m | 34.88Mi | suricata-offline-container:0 | suricata:24.09.0 | -dashboards-deployment-69b5465db-vz88g | Running | 10.42.1.14 | ReplicaSet | agent2 | 0.94m | 100.12Mi | dashboards-container:0 | dashboards:24.09.0 | -netbox-redis-cache-deployment-5f77d47b8b-z7t2z | Running | 10.42.1.15 | ReplicaSet | agent2 | 3.57m | 7.36Mi | netbox-redis-cache-container:0 | redis:24.09.0 | -suricata-live-deployment-6494c77759-9rlnt | Running | 10.42.1.16 | ReplicaSet | agent2 | 0.02m | 9.69Mi | suricata-live-container:0 | suricata:24.09.0 | -freq-deployment-cfd84fd97-dnngf | Running | 10.42.1.17 | ReplicaSet | agent2 | 0.2m | 26.36Mi | freq-container:0 | freq:24.09.0 | -arkime-deployment-56999cdd66-s98pp | Running | 10.42.1.18 | ReplicaSet | agent2 | 4.15m | 113.07Mi | arkime-container:0 | arkime:24.09.0 | -pcap-monitor-deployment-594ff674c4-fsm7m | Running | 10.42.1.19 | ReplicaSet | agent2 | 1.24m | 48.44Mi | pcap-monitor-container:0 | pcap-monitor:24.09.0 | -pcap-capture-deployment-7c8bf6957-jzpzn | Running | 10.42.1.20 | ReplicaSet | agent2 | 0.02m | 9.64Mi | pcap-capture-container:0 | pcap-capture:24.09.0 | -netbox-postgres-deployment-5879b8dffc-kkt56 | Running | 10.42.1.21 | ReplicaSet | agent2 | 70.91m | 33.02Mi | netbox-postgres-container:0 | postgresql:24.09.0 | -htadmin-deployment-6fc46888b9-sq6ln | Running | 10.42.1.23 | ReplicaSet | agent2 | 0.14m | 30.53Mi | htadmin-container:0 | htadmin:24.09.0 | -netbox-redis-deployment-5bcd8f6c96-j5xpf | Running | 10.42.1.24 | ReplicaSet | agent2 | 1.46m | 7.34Mi | netbox-redis-container:0 | redis:24.09.0 | -nginx-proxy-deployment-69fcc4968d-f68tq | Running | 10.42.1.22 | ReplicaSet | agent2 | 0.31m | 22.63Mi | nginx-proxy-container:0 | nginx-proxy:24.09.0 | -opensearch-deployment-75498799f6-4zmwd | Running | 10.42.1.25 | ReplicaSet | agent2 | 89.8m | 11.03Gi | opensearch-container:0 | opensearch:24.09.0 | +api-deployment-6f4686cf59-bn286 | Running | 10.42.2.14 | ReplicaSet | agent1 | 0.11m | 59.62Mi | api-container:0 | api:24.10.0 | +file-monitor-deployment-855646bd75-vk7st | Running | 10.42.2.16 | ReplicaSet | agent1 | 8.47m | 1.46Gi | file-monitor-container:0 | file-monitor:24.10.0 | +zeek-live-deployment-64b69d4b6f-947vr | Running | 10.42.2.17 | ReplicaSet | agent1 | 0.02m | 12.44Mi | zeek-live-container:0 | zeek:24.10.0 | +dashboards-helper-deployment-69dc54f6b6-ln4sq | Running | 10.42.2.15 | ReplicaSet | agent1 | 10.77m | 38.43Mi | dashboards-helper-container:0 | dashboards-helper:24.10.0 | +upload-deployment-586568844b-4jnk9 | Running | 10.42.2.18 | ReplicaSet | agent1 | 0.15m | 29.78Mi | upload-container:0 | file-upload:24.10.0 | +filebeat-deployment-6ff8bc444f-t7h49 | Running | 10.42.2.20 | ReplicaSet | agent1 | 2.84m | 70.71Mi | filebeat-container:0 | filebeat-oss:24.10.0 | +zeek-offline-deployment-844f4865bd-g2sdm | Running | 10.42.2.21 | ReplicaSet | agent1 | 0.17m | 41.92Mi | zeek-offline-container:0 | zeek:24.10.0 | +logstash-deployment-6fbc9fdcd5-hwx8s | Running | 10.42.2.22 | ReplicaSet | agent1 | 85.55m | 2.91Gi | logstash-container:0 | logstash-oss:24.10.0 | +netbox-deployment-cdcff4977-hbbw5 | Running | 10.42.2.23 | ReplicaSet | agent1 | 807.64m | 702.86Mi | netbox-container:0 | netbox:24.10.0 | +suricata-offline-deployment-6ccdb89478-z5696 | Running | 10.42.2.19 | ReplicaSet | agent1 | 0.22m | 34.88Mi | suricata-offline-container:0 | suricata:24.10.0 | +dashboards-deployment-69b5465db-vz88g | Running | 10.42.1.14 | ReplicaSet | agent2 | 0.94m | 100.12Mi | dashboards-container:0 | dashboards:24.10.0 | +netbox-redis-cache-deployment-5f77d47b8b-z7t2z | Running | 10.42.1.15 | ReplicaSet | agent2 | 3.57m | 7.36Mi | netbox-redis-cache-container:0 | redis:24.10.0 | +suricata-live-deployment-6494c77759-9rlnt | Running | 10.42.1.16 | ReplicaSet | agent2 | 0.02m | 9.69Mi | suricata-live-container:0 | suricata:24.10.0 | +freq-deployment-cfd84fd97-dnngf | Running | 10.42.1.17 | ReplicaSet | agent2 | 0.2m | 26.36Mi | freq-container:0 | freq:24.10.0 | +arkime-deployment-56999cdd66-s98pp | Running | 10.42.1.18 | ReplicaSet | agent2 | 4.15m | 113.07Mi | arkime-container:0 | arkime:24.10.0 | +pcap-monitor-deployment-594ff674c4-fsm7m | Running | 10.42.1.19 | ReplicaSet | agent2 | 1.24m | 48.44Mi | pcap-monitor-container:0 | pcap-monitor:24.10.0 | +pcap-capture-deployment-7c8bf6957-jzpzn | Running | 10.42.1.20 | ReplicaSet | agent2 | 0.02m | 9.64Mi | pcap-capture-container:0 | pcap-capture:24.10.0 | +netbox-postgres-deployment-5879b8dffc-kkt56 | Running | 10.42.1.21 | ReplicaSet | agent2 | 70.91m | 33.02Mi | netbox-postgres-container:0 | postgresql:24.10.0 | +htadmin-deployment-6fc46888b9-sq6ln | Running | 10.42.1.23 | ReplicaSet | agent2 | 0.14m | 30.53Mi | htadmin-container:0 | htadmin:24.10.0 | +netbox-redis-deployment-5bcd8f6c96-j5xpf | Running | 10.42.1.24 | ReplicaSet | agent2 | 1.46m | 7.34Mi | netbox-redis-container:0 | redis:24.10.0 | +nginx-proxy-deployment-69fcc4968d-f68tq | Running | 10.42.1.22 | ReplicaSet | agent2 | 0.31m | 22.63Mi | nginx-proxy-container:0 | nginx-proxy:24.10.0 | +opensearch-deployment-75498799f6-4zmwd | Running | 10.42.1.25 | ReplicaSet | agent2 | 89.8m | 11.03Gi | opensearch-container:0 | opensearch:24.10.0 | ``` The other control scripts (`stop`, `restart`, `logs`, etc.) work in a similar manner as in a Docker-based deployment. One notable difference is the `wipe` script: data on PersistentVolume storage cannot be deleted by `wipe`. It must be deleted manually on the storage media underlying the PersistentVolumes. @@ -554,28 +554,28 @@ agent1 | agent1 | 192.168.56.11 | agent1 | k3s | 6000m | agent2 | agent2 | 192.168.56.12 | agent2 | k3s | 6000m | 552.71m | 9.21% | 19.55Gi | 13.27Gi | 61.28Gi | 12 | Pod Name | State | Pod IP | Pod Kind | Worker Node | CPU Usage | Memory Usage | Container Name:Restarts | Container Image | -netbox-redis-cache-deployment-5f77d47b8b-jr9nt | Running | 10.42.2.6 | ReplicaSet | agent2 | 1.89m | 7.24Mi | netbox-redis-cache-container:0 | redis:24.09.0 | -netbox-redis-deployment-5bcd8f6c96-bkzmh | Running | 10.42.2.5 | ReplicaSet | agent2 | 1.62m | 7.52Mi | netbox-redis-container:0 | redis:24.09.0 | -dashboards-helper-deployment-69dc54f6b6-ks7ps | Running | 10.42.2.4 | ReplicaSet | agent2 | 12.95m | 40.75Mi | dashboards-helper-container:0 | dashboards-helper:24.09.0 | -freq-deployment-cfd84fd97-5bwp6 | Running | 10.42.2.8 | ReplicaSet | agent2 | 0.11m | 26.33Mi | freq-container:0 | freq:24.09.0 | -pcap-capture-deployment-7c8bf6957-hkvkn | Running | 10.42.2.12 | ReplicaSet | agent2 | 0.02m | 9.21Mi | pcap-capture-container:0 | pcap-capture:24.09.0 | -nginx-proxy-deployment-69fcc4968d-m57rz | Running | 10.42.2.10 | ReplicaSet | agent2 | 0.91m | 22.72Mi | nginx-proxy-container:0 | nginx-proxy:24.09.0 | -htadmin-deployment-6fc46888b9-vpt7l | Running | 10.42.2.7 | ReplicaSet | agent2 | 0.16m | 30.21Mi | htadmin-container:0 | htadmin:24.09.0 | -opensearch-deployment-75498799f6-5v92w | Running | 10.42.2.13 | ReplicaSet | agent2 | 139.2m | 10.86Gi | opensearch-container:0 | opensearch:24.09.0 | -zeek-live-deployment-64b69d4b6f-fcb6n | Running | 10.42.2.9 | ReplicaSet | agent2 | 0.02m | 109.55Mi | zeek-live-container:0 | zeek:24.09.0 | -dashboards-deployment-69b5465db-kgsqk | Running | 10.42.2.3 | ReplicaSet | agent2 | 14.98m | 108.85Mi | dashboards-container:0 | dashboards:24.09.0 | -arkime-deployment-56999cdd66-xxpw9 | Running | 10.42.2.11 | ReplicaSet | agent2 | 208.95m | 78.42Mi | arkime-container:0 | arkime:24.09.0 | -api-deployment-6f4686cf59-xt9md | Running | 10.42.1.3 | ReplicaSet | agent1 | 0.14m | 56.88Mi | api-container:0 | api:24.09.0 | -netbox-postgres-deployment-5879b8dffc-lb4qm | Running | 10.42.1.6 | ReplicaSet | agent1 | 141.2m | 48.02Mi | netbox-postgres-container:0 | postgresql:24.09.0 | -pcap-monitor-deployment-594ff674c4-fwq7g | Running | 10.42.1.12 | ReplicaSet | agent1 | 3.93m | 46.44Mi | pcap-monitor-container:0 | pcap-monitor:24.09.0 | -suricata-offline-deployment-6ccdb89478-j5fgj | Running | 10.42.1.10 | ReplicaSet | agent1 | 10.42m | 35.12Mi | suricata-offline-container:0 | suricata:24.09.0 | -suricata-live-deployment-6494c77759-rpt48 | Running | 10.42.1.8 | ReplicaSet | agent1 | 0.01m | 9.62Mi | suricata-live-container:0 | suricata:24.09.0 | -netbox-deployment-cdcff4977-7ns2q | Running | 10.42.1.7 | ReplicaSet | agent1 | 830.47m | 530.7Mi | netbox-container:0 | netbox:24.09.0 | -zeek-offline-deployment-844f4865bd-7x68b | Running | 10.42.1.9 | ReplicaSet | agent1 | 1.44m | 43.66Mi | zeek-offline-container:0 | zeek:24.09.0 | -filebeat-deployment-6ff8bc444f-pdgzj | Running | 10.42.1.11 | ReplicaSet | agent1 | 0.78m | 75.25Mi | filebeat-container:0 | filebeat-oss:24.09.0 | -file-monitor-deployment-855646bd75-nbngq | Running | 10.42.1.4 | ReplicaSet | agent1 | 1.69m | 1.46Gi | file-monitor-container:0 | file-monitor:24.09.0 | -upload-deployment-586568844b-9s7f5 | Running | 10.42.1.13 | ReplicaSet | agent1 | 0.14m | 29.62Mi | upload-container:0 | file-upload:24.09.0 | -logstash-deployment-6fbc9fdcd5-2hhx8 | Running | 10.42.1.5 | ReplicaSet | agent1 | 3236.29m | 357.36Mi | logstash-container:0 | logstash-oss:24.09.0 | +netbox-redis-cache-deployment-5f77d47b8b-jr9nt | Running | 10.42.2.6 | ReplicaSet | agent2 | 1.89m | 7.24Mi | netbox-redis-cache-container:0 | redis:24.10.0 | +netbox-redis-deployment-5bcd8f6c96-bkzmh | Running | 10.42.2.5 | ReplicaSet | agent2 | 1.62m | 7.52Mi | netbox-redis-container:0 | redis:24.10.0 | +dashboards-helper-deployment-69dc54f6b6-ks7ps | Running | 10.42.2.4 | ReplicaSet | agent2 | 12.95m | 40.75Mi | dashboards-helper-container:0 | dashboards-helper:24.10.0 | +freq-deployment-cfd84fd97-5bwp6 | Running | 10.42.2.8 | ReplicaSet | agent2 | 0.11m | 26.33Mi | freq-container:0 | freq:24.10.0 | +pcap-capture-deployment-7c8bf6957-hkvkn | Running | 10.42.2.12 | ReplicaSet | agent2 | 0.02m | 9.21Mi | pcap-capture-container:0 | pcap-capture:24.10.0 | +nginx-proxy-deployment-69fcc4968d-m57rz | Running | 10.42.2.10 | ReplicaSet | agent2 | 0.91m | 22.72Mi | nginx-proxy-container:0 | nginx-proxy:24.10.0 | +htadmin-deployment-6fc46888b9-vpt7l | Running | 10.42.2.7 | ReplicaSet | agent2 | 0.16m | 30.21Mi | htadmin-container:0 | htadmin:24.10.0 | +opensearch-deployment-75498799f6-5v92w | Running | 10.42.2.13 | ReplicaSet | agent2 | 139.2m | 10.86Gi | opensearch-container:0 | opensearch:24.10.0 | +zeek-live-deployment-64b69d4b6f-fcb6n | Running | 10.42.2.9 | ReplicaSet | agent2 | 0.02m | 109.55Mi | zeek-live-container:0 | zeek:24.10.0 | +dashboards-deployment-69b5465db-kgsqk | Running | 10.42.2.3 | ReplicaSet | agent2 | 14.98m | 108.85Mi | dashboards-container:0 | dashboards:24.10.0 | +arkime-deployment-56999cdd66-xxpw9 | Running | 10.42.2.11 | ReplicaSet | agent2 | 208.95m | 78.42Mi | arkime-container:0 | arkime:24.10.0 | +api-deployment-6f4686cf59-xt9md | Running | 10.42.1.3 | ReplicaSet | agent1 | 0.14m | 56.88Mi | api-container:0 | api:24.10.0 | +netbox-postgres-deployment-5879b8dffc-lb4qm | Running | 10.42.1.6 | ReplicaSet | agent1 | 141.2m | 48.02Mi | netbox-postgres-container:0 | postgresql:24.10.0 | +pcap-monitor-deployment-594ff674c4-fwq7g | Running | 10.42.1.12 | ReplicaSet | agent1 | 3.93m | 46.44Mi | pcap-monitor-container:0 | pcap-monitor:24.10.0 | +suricata-offline-deployment-6ccdb89478-j5fgj | Running | 10.42.1.10 | ReplicaSet | agent1 | 10.42m | 35.12Mi | suricata-offline-container:0 | suricata:24.10.0 | +suricata-live-deployment-6494c77759-rpt48 | Running | 10.42.1.8 | ReplicaSet | agent1 | 0.01m | 9.62Mi | suricata-live-container:0 | suricata:24.10.0 | +netbox-deployment-cdcff4977-7ns2q | Running | 10.42.1.7 | ReplicaSet | agent1 | 830.47m | 530.7Mi | netbox-container:0 | netbox:24.10.0 | +zeek-offline-deployment-844f4865bd-7x68b | Running | 10.42.1.9 | ReplicaSet | agent1 | 1.44m | 43.66Mi | zeek-offline-container:0 | zeek:24.10.0 | +filebeat-deployment-6ff8bc444f-pdgzj | Running | 10.42.1.11 | ReplicaSet | agent1 | 0.78m | 75.25Mi | filebeat-container:0 | filebeat-oss:24.10.0 | +file-monitor-deployment-855646bd75-nbngq | Running | 10.42.1.4 | ReplicaSet | agent1 | 1.69m | 1.46Gi | file-monitor-container:0 | file-monitor:24.10.0 | +upload-deployment-586568844b-9s7f5 | Running | 10.42.1.13 | ReplicaSet | agent1 | 0.14m | 29.62Mi | upload-container:0 | file-upload:24.10.0 | +logstash-deployment-6fbc9fdcd5-2hhx8 | Running | 10.42.1.5 | ReplicaSet | agent1 | 3236.29m | 357.36Mi | logstash-container:0 | logstash-oss:24.10.0 | ``` View container logs for the Malcolm deployment with `./scripts/logs` (if **[stern](https://github.com/stern/stern)** present in `$PATH`): diff --git a/docs/malcolm-iso.md b/docs/malcolm-iso.md index 55efa14d4..7406ac341 100644 --- a/docs/malcolm-iso.md +++ b/docs/malcolm-iso.md @@ -12,20 +12,21 @@ Malcolm can be packaged into an installer ISO based on the current [stable relea ## Generating the ISO -Official downloads of the Malcolm installer ISO are not provided: however, it can be built easily on an Internet-connected Linux host with Vagrant: +Official downloads of the Malcolm installer ISO [can be downloaded](download.md#DownloadISOs) from the GitHub releases page. It can also be built easily on an Internet-connected system with Vagrant: * [Vagrant](https://www.vagrantup.com/) - - [`vagrant-reload`](https://github.com/aidanns/vagrant-reload) plugin - [`vagrant-sshfs`](https://github.com/dustymabe/vagrant-sshfs) plugin - [`bento/debian-12`](https://app.vagrantup.com/bento/boxes/debian-12) Vagrant box -The build should work with either the [VirtualBox](https://www.virtualbox.org/) provider or the [libvirt](https://libvirt.org/) provider: +The build should work with a variety of [Vagrant providers](https://developer.hashicorp.com/vagrant/docs/providers): -* [VirtualBox](https://www.virtualbox.org/) [provider](https://www.vagrantup.com/docs/providers/virtualbox) - - [`vagrant-vbguest`](https://github.com/dotless-de/vagrant-vbguest) plugin +* [VMware](https://www.vmware.com/) [provider](https://developer.hashicorp.com/vagrant/docs/providers/vmware) + - [`vagrant-vmware-desktop`](https://github.com/hashicorp/vagrant-vmware-desktop) plugin * [libvirt](https://libvirt.org/) - [`vagrant-libvirt`](https://github.com/vagrant-libvirt/vagrant-libvirt) provider plugin - [`vagrant-mutate`](https://github.com/sciurus/vagrant-mutate) plugin to convert [`bento/debian-12`](https://app.vagrantup.com/bento/boxes/debian-12) Vagrant box to `libvirt` format +* [VirtualBox](https://www.virtualbox.org/) [provider](https://developer.hashicorp.com/vagrant/docs/providers/virtualbox) + - [`vagrant-vbguest`](https://github.com/dotless-de/vagrant-vbguest) plugin To perform a clean build of the Malcolm installer ISO, navigate to the local Malcolm working copy and run: @@ -41,7 +42,7 @@ Building the ISO may take 30 minutes or more depending on the system. As the bui ``` … -Finished, created "/malcolm-build/malcolm-iso/malcolm-24.09.0.iso" +Finished, created "/malcolm-build/malcolm-iso/malcolm-24.10.0.iso" … ``` @@ -54,7 +55,7 @@ $ ./malcolm-iso/build_via_vagrant.sh -f -d malcolm_YYYYMMDD_HHNNSS_xxxxxxx_image A system installed from the resulting ISO will load the Malcolm images upon first boot. This method is desirable when the ISO is to be installed in an "air gapped" environment or for distribution to non-networked machines. -Alternately, if users have forked Malcolm on GitHub, [workflow files]({{ site.github.repository_url }}/tree/{{ site.github.build_revision }}/.github/workflows/) are provided that contain instructions for GitHub to build the images and [sensor](live-analysis.md#Hedgehog) and [Malcolm](#ISO) installer ISOs - specifically [`malcolm-iso-build-docker-wrap-push-ghcr.yml`]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/.github/workflows/malcolm-iso-build-docker-wrap-push-ghcr.yml) for the Malcolm ISO. Users must run the workflows to build and push the fork's Malcolm images before building the ISO. The resulting ISO file is wrapped in a image that provides an HTTP server from which the ISO may be downloaded. +Alternately, if users have forked Malcolm on GitHub, [workflow files]({{ site.github.repository_url }}/tree/{{ site.github.build_revision }}/.github/workflows/) are provided that contain instructions for GitHub to build the images and [sensor](live-analysis.md#Hedgehog) and [Malcolm](#ISO) installer ISOs - specifically [`malcolm-iso-build-docker-wrap-push-ghcr.yml`]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/.github/workflows/malcolm-iso-build-docker-wrap-push-ghcr.yml) for the Malcolm ISO. Users must run the workflows to build and push the fork's Malcolm images before building the ISO. The resulting ISO file is wrapped in a image that provides an HTTP server from which the ISO may be downloaded. See [Using GitHub runners to build Malcolm images](contributing-github-runners.md#GitHubRunners) for more information. ## Installation diff --git a/docs/quickstart.md b/docs/quickstart.md index 5e2b96410..33f0a1f46 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -60,25 +60,25 @@ You can then observe the images have been retrieved by running `docker images`: ``` $ docker images REPOSITORY TAG IMAGE ID CREATED SIZE -ghcr.io/idaholab/malcolm/api 24.09.0 xxxxxxxxxxxx 3 days ago 158MB -ghcr.io/idaholab/malcolm/arkime 24.09.0 xxxxxxxxxxxx 3 days ago 816MB -ghcr.io/idaholab/malcolm/dashboards 24.09.0 xxxxxxxxxxxx 3 days ago 1.02GB -ghcr.io/idaholab/malcolm/dashboards-helper 24.09.0 xxxxxxxxxxxx 3 days ago 184MB -ghcr.io/idaholab/malcolm/file-monitor 24.09.0 xxxxxxxxxxxx 3 days ago 588MB -ghcr.io/idaholab/malcolm/file-upload 24.09.0 xxxxxxxxxxxx 3 days ago 259MB -ghcr.io/idaholab/malcolm/filebeat-oss 24.09.0 xxxxxxxxxxxx 3 days ago 624MB -ghcr.io/idaholab/malcolm/freq 24.09.0 xxxxxxxxxxxx 3 days ago 132MB -ghcr.io/idaholab/malcolm/htadmin 24.09.0 xxxxxxxxxxxx 3 days ago 242MB -ghcr.io/idaholab/malcolm/logstash-oss 24.09.0 xxxxxxxxxxxx 3 days ago 1.35GB -ghcr.io/idaholab/malcolm/netbox 24.09.0 xxxxxxxxxxxx 3 days ago 1.01GB -ghcr.io/idaholab/malcolm/nginx-proxy 24.09.0 xxxxxxxxxxxx 3 days ago 121MB -ghcr.io/idaholab/malcolm/opensearch 24.09.0 xxxxxxxxxxxx 3 days ago 1.17GB -ghcr.io/idaholab/malcolm/pcap-capture 24.09.0 xxxxxxxxxxxx 3 days ago 121MB -ghcr.io/idaholab/malcolm/pcap-monitor 24.09.0 xxxxxxxxxxxx 3 days ago 213MB -ghcr.io/idaholab/malcolm/postgresql 24.09.0 xxxxxxxxxxxx 3 days ago 268MB -ghcr.io/idaholab/malcolm/redis 24.09.0 xxxxxxxxxxxx 3 days ago 34.2MB -ghcr.io/idaholab/malcolm/suricata 24.09.0 xxxxxxxxxxxx 3 days ago 278MB -ghcr.io/idaholab/malcolm/zeek 24.09.0 xxxxxxxxxxxx 3 days ago 1GB +ghcr.io/idaholab/malcolm/api 24.10.0 xxxxxxxxxxxx 3 days ago 158MB +ghcr.io/idaholab/malcolm/arkime 24.10.0 xxxxxxxxxxxx 3 days ago 816MB +ghcr.io/idaholab/malcolm/dashboards 24.10.0 xxxxxxxxxxxx 3 days ago 1.02GB +ghcr.io/idaholab/malcolm/dashboards-helper 24.10.0 xxxxxxxxxxxx 3 days ago 184MB +ghcr.io/idaholab/malcolm/file-monitor 24.10.0 xxxxxxxxxxxx 3 days ago 588MB +ghcr.io/idaholab/malcolm/file-upload 24.10.0 xxxxxxxxxxxx 3 days ago 259MB +ghcr.io/idaholab/malcolm/filebeat-oss 24.10.0 xxxxxxxxxxxx 3 days ago 624MB +ghcr.io/idaholab/malcolm/freq 24.10.0 xxxxxxxxxxxx 3 days ago 132MB +ghcr.io/idaholab/malcolm/htadmin 24.10.0 xxxxxxxxxxxx 3 days ago 242MB +ghcr.io/idaholab/malcolm/logstash-oss 24.10.0 xxxxxxxxxxxx 3 days ago 1.35GB +ghcr.io/idaholab/malcolm/netbox 24.10.0 xxxxxxxxxxxx 3 days ago 1.01GB +ghcr.io/idaholab/malcolm/nginx-proxy 24.10.0 xxxxxxxxxxxx 3 days ago 121MB +ghcr.io/idaholab/malcolm/opensearch 24.10.0 xxxxxxxxxxxx 3 days ago 1.17GB +ghcr.io/idaholab/malcolm/pcap-capture 24.10.0 xxxxxxxxxxxx 3 days ago 121MB +ghcr.io/idaholab/malcolm/pcap-monitor 24.10.0 xxxxxxxxxxxx 3 days ago 213MB +ghcr.io/idaholab/malcolm/postgresql 24.10.0 xxxxxxxxxxxx 3 days ago 268MB +ghcr.io/idaholab/malcolm/redis 24.10.0 xxxxxxxxxxxx 3 days ago 34.2MB +ghcr.io/idaholab/malcolm/suricata 24.10.0 xxxxxxxxxxxx 3 days ago 278MB +ghcr.io/idaholab/malcolm/zeek 24.10.0 xxxxxxxxxxxx 3 days ago 1GB ``` ### Import from pre-packaged tarballs diff --git a/docs/third-party-envs.md b/docs/third-party-envs.md index f62b2ccbc..134b9b1ad 100644 --- a/docs/third-party-envs.md +++ b/docs/third-party-envs.md @@ -43,7 +43,7 @@ The files referenced in this section can be found in [scripts/third-party-enviro amazon-ebs: output will be in this color. ==> amazon-ebs: Prevalidating any provided VPC information - ==> amazon-ebs: Prevalidating AMI Name: malcolm-v24.09.0-arm64-2024-05-30T13-57-31Z + ==> amazon-ebs: Prevalidating AMI Name: malcolm-v24.10.0-arm64-2024-05-30T13-57-31Z amazon-ebs: Found Image ID: ami-xxxxxxxxxxxxxxxxx ... @@ -53,7 +53,7 @@ The files referenced in this section can be found in [scripts/third-party-enviro ==> amazon-ebs: Adding tags to AMI (ami-xxxxxxxxxxxxxxxxx)... ==> amazon-ebs: Tagging snapshot: snap-xxxxxxxxxxxxxxxxx ==> amazon-ebs: Creating AMI tags - amazon-ebs: Adding tag: "Malcolm": "idaholab/Malcolm/v24.09.0" + amazon-ebs: Adding tag: "Malcolm": "idaholab/Malcolm/v24.10.0" amazon-ebs: Adding tag: "source_ami_name": "amzn2-ami-kernel-5.10-hvm-2.0.20240521.0-arm64-gp2" ==> amazon-ebs: Creating snapshot tags ==> amazon-ebs: Terminating the source AWS instance... @@ -79,7 +79,7 @@ The files referenced in this section can be found in [scripts/third-party-enviro "Architecture": "arm64", "CreationDate": "2024-05-30T14:02:21.000Z", "ImageId": "ami-xxxxxxxxxxxxxxxxx", - "ImageLocation": "xxxxxxxxxxxx/malcolm-v24.09.0-arm64-2024-05-30T13-57-31Z", + "ImageLocation": "xxxxxxxxxxxx/malcolm-v24.10.0-arm64-2024-05-30T13-57-31Z", "ImageType": "machine", "Public": false, "OwnerId": "xxxxxxxxxxxx", @@ -100,14 +100,14 @@ The files referenced in this section can be found in [scripts/third-party-enviro ], "EnaSupport": true, "Hypervisor": "xen", - "Name": "malcolm-v24.09.0-arm64-2024-05-30T13-57-31Z", + "Name": "malcolm-v24.10.0-arm64-2024-05-30T13-57-31Z", "RootDeviceName": "/dev/xvda", "RootDeviceType": "ebs", "SriovNetSupport": "simple", "Tags": [ { "Key": "Malcolm", - "Value": "idaholab/Malcolm/v24.09.0" + "Value": "idaholab/Malcolm/v24.10.0" }, { "Key": "source_ami_name", diff --git a/docs/ubuntu-install-example.md b/docs/ubuntu-install-example.md index c867cf9f4..9e77c9c8d 100644 --- a/docs/ubuntu-install-example.md +++ b/docs/ubuntu-install-example.md @@ -261,25 +261,25 @@ Pulling zeek ... done user@host:~/Malcolm$ docker images REPOSITORY TAG IMAGE ID CREATED SIZE -ghcr.io/idaholab/malcolm/api 24.09.0 xxxxxxxxxxxx 3 days ago 158MB -ghcr.io/idaholab/malcolm/arkime 24.09.0 xxxxxxxxxxxx 3 days ago 816MB -ghcr.io/idaholab/malcolm/dashboards 24.09.0 xxxxxxxxxxxx 3 days ago 1.02GB -ghcr.io/idaholab/malcolm/dashboards-helper 24.09.0 xxxxxxxxxxxx 3 days ago 184MB -ghcr.io/idaholab/malcolm/file-monitor 24.09.0 xxxxxxxxxxxx 3 days ago 588MB -ghcr.io/idaholab/malcolm/file-upload 24.09.0 xxxxxxxxxxxx 3 days ago 259MB -ghcr.io/idaholab/malcolm/filebeat-oss 24.09.0 xxxxxxxxxxxx 3 days ago 624MB -ghcr.io/idaholab/malcolm/freq 24.09.0 xxxxxxxxxxxx 3 days ago 132MB -ghcr.io/idaholab/malcolm/htadmin 24.09.0 xxxxxxxxxxxx 3 days ago 242MB -ghcr.io/idaholab/malcolm/logstash-oss 24.09.0 xxxxxxxxxxxx 3 days ago 1.35GB -ghcr.io/idaholab/malcolm/netbox 24.09.0 xxxxxxxxxxxx 3 days ago 1.01GB -ghcr.io/idaholab/malcolm/nginx-proxy 24.09.0 xxxxxxxxxxxx 3 days ago 121MB -ghcr.io/idaholab/malcolm/opensearch 24.09.0 xxxxxxxxxxxx 3 days ago 1.17GB -ghcr.io/idaholab/malcolm/pcap-capture 24.09.0 xxxxxxxxxxxx 3 days ago 121MB -ghcr.io/idaholab/malcolm/pcap-monitor 24.09.0 xxxxxxxxxxxx 3 days ago 213MB -ghcr.io/idaholab/malcolm/postgresql 24.09.0 xxxxxxxxxxxx 3 days ago 268MB -ghcr.io/idaholab/malcolm/redis 24.09.0 xxxxxxxxxxxx 3 days ago 34.2MB -ghcr.io/idaholab/malcolm/suricata 24.09.0 xxxxxxxxxxxx 3 days ago 278MB -ghcr.io/idaholab/malcolm/zeek 24.09.0 xxxxxxxxxxxx 3 days ago 1GB +ghcr.io/idaholab/malcolm/api 24.10.0 xxxxxxxxxxxx 3 days ago 158MB +ghcr.io/idaholab/malcolm/arkime 24.10.0 xxxxxxxxxxxx 3 days ago 816MB +ghcr.io/idaholab/malcolm/dashboards 24.10.0 xxxxxxxxxxxx 3 days ago 1.02GB +ghcr.io/idaholab/malcolm/dashboards-helper 24.10.0 xxxxxxxxxxxx 3 days ago 184MB +ghcr.io/idaholab/malcolm/file-monitor 24.10.0 xxxxxxxxxxxx 3 days ago 588MB +ghcr.io/idaholab/malcolm/file-upload 24.10.0 xxxxxxxxxxxx 3 days ago 259MB +ghcr.io/idaholab/malcolm/filebeat-oss 24.10.0 xxxxxxxxxxxx 3 days ago 624MB +ghcr.io/idaholab/malcolm/freq 24.10.0 xxxxxxxxxxxx 3 days ago 132MB +ghcr.io/idaholab/malcolm/htadmin 24.10.0 xxxxxxxxxxxx 3 days ago 242MB +ghcr.io/idaholab/malcolm/logstash-oss 24.10.0 xxxxxxxxxxxx 3 days ago 1.35GB +ghcr.io/idaholab/malcolm/netbox 24.10.0 xxxxxxxxxxxx 3 days ago 1.01GB +ghcr.io/idaholab/malcolm/nginx-proxy 24.10.0 xxxxxxxxxxxx 3 days ago 121MB +ghcr.io/idaholab/malcolm/opensearch 24.10.0 xxxxxxxxxxxx 3 days ago 1.17GB +ghcr.io/idaholab/malcolm/pcap-capture 24.10.0 xxxxxxxxxxxx 3 days ago 121MB +ghcr.io/idaholab/malcolm/pcap-monitor 24.10.0 xxxxxxxxxxxx 3 days ago 213MB +ghcr.io/idaholab/malcolm/postgresql 24.10.0 xxxxxxxxxxxx 3 days ago 268MB +ghcr.io/idaholab/malcolm/redis 24.10.0 xxxxxxxxxxxx 3 days ago 34.2MB +ghcr.io/idaholab/malcolm/suricata 24.10.0 xxxxxxxxxxxx 3 days ago 278MB +ghcr.io/idaholab/malcolm/zeek 24.10.0 xxxxxxxxxxxx 3 days ago 1GB ``` Finally, start Malcolm. When Malcolm starts it will stream informational and debug messages to the console until it has completed initializing. diff --git a/hedgehog-iso/build.sh b/hedgehog-iso/build.sh index 0e6b10a13..8390e6c63 100755 --- a/hedgehog-iso/build.sh +++ b/hedgehog-iso/build.sh @@ -5,7 +5,7 @@ IMAGE_PUBLISHER=cisagov IMAGE_VERSION=1.0.0 IMAGE_DISTRIBUTION=bookworm -BEATS_VER="8.15.1" +BEATS_VER="8.15.2" BEATS_OSS="-oss" ARKIME_VER="5.4.0" @@ -190,7 +190,7 @@ if [ -d "$WORKDIR" ]; then "https://github.com/arkime/arkime/releases/download/v${ARKIME_VER}/arkime_${ARKIME_VER}-1.debian12_amd64.deb" # download Zeek .deb packages - bash "$SCRIPT_PATH/shared/bin/zeek-deb-download.sh" -o ./config/packages.chroot/ + bash "$SCRIPT_PATH/shared/bin/zeek-deb-download.sh" -o ./config/packages.chroot/ -f "$SCRIPT_PATH/shared/zeek_url.txt" # reclaim some space docker system prune --volumes --force diff --git a/hedgehog-iso/build_via_vagrant.sh b/hedgehog-iso/build_via_vagrant.sh index bd66f5ea5..c15457f83 100755 --- a/hedgehog-iso/build_via_vagrant.sh +++ b/hedgehog-iso/build_via_vagrant.sh @@ -100,6 +100,7 @@ YML_IMAGE_VERSION="$(grep -P "^\s+image:.*/malcolm/" "$SCRIPT_PATH"/../docker-co [[ -n $YML_IMAGE_VERSION ]] && echo "$YML_IMAGE_VERSION" > "$SCRIPT_PATH"/shared/version.txt [[ ${#MAXMIND_GEOIP_DB_LICENSE_KEY} -gt 1 ]] && echo "$MAXMIND_GEOIP_DB_LICENSE_KEY" > "$SCRIPT_PATH"/shared/maxmind_license.txt [[ ${#MAXMIND_GEOIP_DB_ALTERNATE_DOWNLOAD_URL} -gt 1 ]] && echo "$MAXMIND_GEOIP_DB_ALTERNATE_DOWNLOAD_URL" > "$SCRIPT_PATH"/shared/maxmind_url.txt +[[ ${#ZEEK_DEB_ALTERNATE_DOWNLOAD_URL} -gt 1 ]] && echo "$ZEEK_DEB_ALTERNATE_DOWNLOAD_URL" > "$SCRIPT_PATH"/shared/zeek_url.txt [[ ${#GITHUB_TOKEN} -gt 1 ]] && echo "GITHUB_TOKEN=$GITHUB_TOKEN" >> "$SCRIPT_PATH"/shared/environment.chroot echo "VCS_REVSION=$( git rev-parse --short HEAD 2>/dev/null || echo main )" >> "$SCRIPT_PATH"/shared/environment.chroot trap cleanup_shared_and_docs EXIT diff --git a/hedgehog-iso/config/hooks/normal/0169-pip-installs.hook.chroot b/hedgehog-iso/config/hooks/normal/0169-pip-installs.hook.chroot index 68617deee..2a992298c 100755 --- a/hedgehog-iso/config/hooks/normal/0169-pip-installs.hook.chroot +++ b/hedgehog-iso/config/hooks/normal/0169-pip-installs.hook.chroot @@ -20,4 +20,4 @@ pip3 install --break-system-packages --no-compile --no-cache-dir --force-reinsta stix2 \ stream-zip \ taxii2-client \ - watchdog==5.0.2 + watchdog==5.0.3 diff --git a/hedgehog-iso/config/includes.chroot/usr/local/etc/zeek/local.zeek b/hedgehog-iso/config/includes.chroot/usr/local/etc/zeek/local.zeek index 65b3acf43..9b66dccfc 100644 --- a/hedgehog-iso/config/includes.chroot/usr/local/etc/zeek/local.zeek +++ b/hedgehog-iso/config/includes.chroot/usr/local/etc/zeek/local.zeek @@ -56,7 +56,6 @@ global json_format = (getenv("ZEEK_JSON") == true_regex) ? T : F; redef LogAscii::use_json = T; @endif -@load tuning/defaults @load frameworks/software/vulnerable @load frameworks/software/version-changes @load frameworks/software/windows-version-detection @@ -70,6 +69,7 @@ global json_format = (getenv("ZEEK_JSON") == true_regex) ? T : F; @load protocols/ftp/software @load protocols/http/detect-sqli @load protocols/http/detect-webapps +@load protocols/http/header-names @load protocols/http/software @load protocols/http/software-browser-plugins @load protocols/mysql/software @@ -282,6 +282,8 @@ event zeek_init() &priority=-5 { redef LDAP::default_capture_password = T; @endif +redef HTTP::log_client_header_names = T; +redef HTTP::log_server_header_names = T; redef LDAP::default_log_search_attributes = F; redef SNIFFPASS::notice_log_enable = F; redef CVE_2021_44228::log = F; diff --git a/hedgehog-iso/interface/sensor_ctl/control_vars.conf b/hedgehog-iso/interface/sensor_ctl/control_vars.conf index 1a4e030e0..4dd4eac96 100644 --- a/hedgehog-iso/interface/sensor_ctl/control_vars.conf +++ b/hedgehog-iso/interface/sensor_ctl/control_vars.conf @@ -121,6 +121,7 @@ export ZEEK_DISABLE_HASH_ALL_FILES= export ZEEK_DISABLE_LOG_PASSWORDS= export ZEEK_DISABLE_SSL_VALIDATE_CERTS= export ZEEK_DISABLE_TRACK_ALL_ASSETS= +export ZEEK_DISABLE_DETECT_ROUTERS=true export ZEEK_DISABLE_SPICY_IPSEC= export ZEEK_DISABLE_SPICY_LDAP= export ZEEK_DISABLE_SPICY_OPENVPN= diff --git a/hedgehog-iso/vagrant/Vagrantfile b/hedgehog-iso/vagrant/Vagrantfile index 04cfb86f6..bc1c3cf90 100644 --- a/hedgehog-iso/vagrant/Vagrantfile +++ b/hedgehog-iso/vagrant/Vagrantfile @@ -36,6 +36,16 @@ Vagrant.configure("2") do |config| lv.cpus = 4 end + config.vm.provider "vmware_desktop" do |vd| + vd.memory = "8192" + vd.cpus = 4 + end + + config.vm.provider "vmware_fusion" do |vf| + vf.memory = "8192" + vf.cpus = 4 + end + config.vm.provision "shell", inline: <<-STEP1 dpkg-reconfigure debconf -f noninteractive -p critical export DEBIAN_FRONTEND=noninteractive diff --git a/hedgehog-raspi/build_via_vagrant.sh b/hedgehog-raspi/build_via_vagrant.sh index fac209dea..643f4378b 100755 --- a/hedgehog-raspi/build_via_vagrant.sh +++ b/hedgehog-raspi/build_via_vagrant.sh @@ -99,6 +99,7 @@ YML_IMAGE_VERSION="$(grep -P "^\s+image:.*/malcolm/" "$SCRIPT_PATH"/../docker-co [[ -n $YML_IMAGE_VERSION ]] && echo "$YML_IMAGE_VERSION" > "$SCRIPT_PATH"/shared/version.txt [[ ${#MAXMIND_GEOIP_DB_LICENSE_KEY} -gt 1 ]] && echo "$MAXMIND_GEOIP_DB_LICENSE_KEY" > "$SCRIPT_PATH"/shared/maxmind_license.txt [[ ${#MAXMIND_GEOIP_DB_ALTERNATE_DOWNLOAD_URL} -gt 1 ]] && echo "$MAXMIND_GEOIP_DB_ALTERNATE_DOWNLOAD_URL" > "$SCRIPT_PATH"/shared/maxmind_url.txt +[[ ${#ZEEK_DEB_ALTERNATE_DOWNLOAD_URL} -gt 1 ]] && echo "$ZEEK_DEB_ALTERNATE_DOWNLOAD_URL" > "$SCRIPT_PATH"/shared/zeek_url.txt [[ ${#GITHUB_TOKEN} -gt 1 ]] && echo "GITHUB_TOKEN=$GITHUB_TOKEN" >> "$SCRIPT_PATH"/shared/environment.chroot echo "VCS_REVSION=$( git rev-parse --short HEAD 2>/dev/null || echo main )" >> "$SCRIPT_PATH"/shared/environment.chroot trap cleanup_shared_and_docs EXIT diff --git a/hedgehog-raspi/sensor_install.sh b/hedgehog-raspi/sensor_install.sh index 9c930a869..0310104e6 100644 --- a/hedgehog-raspi/sensor_install.sh +++ b/hedgehog-raspi/sensor_install.sh @@ -34,7 +34,7 @@ SENSOR_DIR='/opt/sensor' ARKIME_VERSION="5.4.0" -BEATS_VER="8.15.1" +BEATS_VER="8.15.2" BEATS_OSS="-oss" # Option to build from sources if desired @@ -197,7 +197,7 @@ build_yara_src() { build_zeek() { # install zeek from debs from OpenSUSE mkdir -p /tmp/zeek-debs - /bin/bash /usr/local/bin/zeek-deb-download.sh -o /tmp/zeek-debs + /bin/bash /usr/local/bin/zeek-deb-download.sh -o /tmp/zeek-debs -f "$SHARED_DIR/zeek_url.txt" dpkg -i /tmp/zeek-debs/*.deb } @@ -212,7 +212,7 @@ build_zeek_src() { export PYTHONUNBUFFERED=1 zeek_url=https://github.com/zeek/zeek.git - zeek_version=7.0.1 + zeek_version=7.0.3 zeek_release=1 zeek_dir=/opt/zeek # Zeek's build eats a ton of resources; prevent OOM from the killing build process @@ -369,8 +369,8 @@ install_files() { fi echo "BUILD_ID=\"$(date +\'%Y-%m-%d\')-${IMAGE_VERSION}\"" > "$sensor_ver_file" - echo "VARIANT=\"Hedgehog Linux (Sensor) v${IMAGE_VERSION}\"" >> "$sensor_ver_file" - echo "VARIANT_ID=\"hedgehog-sensor\"" >> "$sensor_ver_file" + echo "VARIANT=\"Hedgehog Linux (Minihog Sensor) v${IMAGE_VERSION}\"" >> "$sensor_ver_file" + echo "VARIANT_ID=\"hedgehog-minihog-sensor\"" >> "$sensor_ver_file" echo "ID_LIKE=\"debian\"" >> "$sensor_ver_file" echo "HOME_URL=\"https://${IMAGE_PUBLISHER}.github.io/Malcolm\"" >> "$sensor_ver_file" echo "DOCUMENTATION_URL=\"https://${IMAGE_PUBLISHER}.github.io/Malcolm/docs/hedgehog.html\"" >> "$sensor_ver_file" diff --git a/hedgehog-raspi/vagrant/Vagrantfile b/hedgehog-raspi/vagrant/Vagrantfile index cb46f2255..897e6a371 100644 --- a/hedgehog-raspi/vagrant/Vagrantfile +++ b/hedgehog-raspi/vagrant/Vagrantfile @@ -26,14 +26,24 @@ Vagrant.configure("2") do |config| config.vbguest.auto_update = false end - config.vm.provider "virtualbox" do |vb| - vb.memory = "8192" - vb.cpus = 4 + config.vm.provider "virtualbox" do |vm| + vm.memory = "8192" + vm.cpus = 4 end - config.vm.provider "libvirt" do |lv| - lv.memory = "8192" - lv.cpus = 4 + config.vm.provider "libvirt" do |vm| + vm.memory = "8192" + vm.cpus = 4 + end + + config.vm.provider "vmware_desktop" do |vm| + vm.memory = "8192" + vm.cpus = 4 + end + + config.vm.provider "vmware_fusion" do |vm| + vm.memory = "8192" + vm.cpus = 4 end config.vm.provision "shell", inline: <<-STEP1 diff --git a/kubernetes/03-opensearch.yml b/kubernetes/03-opensearch.yml index 73757072e..d216b4aa0 100644 --- a/kubernetes/03-opensearch.yml +++ b/kubernetes/03-opensearch.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: opensearch-container - image: ghcr.io/idaholab/malcolm/opensearch:24.09.0 + image: ghcr.io/idaholab/malcolm/opensearch:24.10.0 imagePullPolicy: Always stdin: false tty: true @@ -71,7 +71,7 @@ spec: subPath: "opensearch" initContainers: - name: opensearch-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.09.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.10.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/04-dashboards.yml b/kubernetes/04-dashboards.yml index 6db70b00f..37f58106a 100644 --- a/kubernetes/04-dashboards.yml +++ b/kubernetes/04-dashboards.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: dashboards-container - image: ghcr.io/idaholab/malcolm/dashboards:24.09.0 + image: ghcr.io/idaholab/malcolm/dashboards:24.10.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/05-upload.yml b/kubernetes/05-upload.yml index c32afadc6..e094081c0 100644 --- a/kubernetes/05-upload.yml +++ b/kubernetes/05-upload.yml @@ -34,7 +34,7 @@ spec: spec: containers: - name: upload-container - image: ghcr.io/idaholab/malcolm/file-upload:24.09.0 + image: ghcr.io/idaholab/malcolm/file-upload:24.10.0 imagePullPolicy: Always stdin: false tty: true @@ -73,7 +73,7 @@ spec: subPath: "upload" initContainers: - name: upload-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.09.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.10.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/06-pcap-monitor.yml b/kubernetes/06-pcap-monitor.yml index 7450ac064..c947523c8 100644 --- a/kubernetes/06-pcap-monitor.yml +++ b/kubernetes/06-pcap-monitor.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: pcap-monitor-container - image: ghcr.io/idaholab/malcolm/pcap-monitor:24.09.0 + image: ghcr.io/idaholab/malcolm/pcap-monitor:24.10.0 imagePullPolicy: Always stdin: false tty: true @@ -70,7 +70,7 @@ spec: name: pcap-monitor-zeek-volume initContainers: - name: pcap-monitor-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.09.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.10.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/07-arkime.yml b/kubernetes/07-arkime.yml index d619b4483..bf363d3a4 100644 --- a/kubernetes/07-arkime.yml +++ b/kubernetes/07-arkime.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: arkime-container - image: ghcr.io/idaholab/malcolm/arkime:24.09.0 + image: ghcr.io/idaholab/malcolm/arkime:24.10.0 imagePullPolicy: Always stdin: false tty: true @@ -81,7 +81,7 @@ spec: name: arkime-pcap-volume initContainers: - name: arkime-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.09.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.10.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/08-api.yml b/kubernetes/08-api.yml index 3d57ac4ad..860ed225a 100644 --- a/kubernetes/08-api.yml +++ b/kubernetes/08-api.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: api-container - image: ghcr.io/idaholab/malcolm/api:24.09.0 + image: ghcr.io/idaholab/malcolm/api:24.10.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/09-dashboards-helper.yml b/kubernetes/09-dashboards-helper.yml index 7ad4015c5..98769d4b3 100644 --- a/kubernetes/09-dashboards-helper.yml +++ b/kubernetes/09-dashboards-helper.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: dashboards-helper-container - image: ghcr.io/idaholab/malcolm/dashboards-helper:24.09.0 + image: ghcr.io/idaholab/malcolm/dashboards-helper:24.10.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/10-zeek.yml b/kubernetes/10-zeek.yml index 9c244f867..7900c7a29 100644 --- a/kubernetes/10-zeek.yml +++ b/kubernetes/10-zeek.yml @@ -16,7 +16,7 @@ spec: spec: containers: - name: zeek-offline-container - image: ghcr.io/idaholab/malcolm/zeek:24.09.0 + image: ghcr.io/idaholab/malcolm/zeek:24.10.0 imagePullPolicy: Always stdin: false tty: true @@ -62,7 +62,7 @@ spec: subPath: "zeek/intel" initContainers: - name: zeek-offline-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.09.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.10.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/11-suricata.yml b/kubernetes/11-suricata.yml index ca41760c9..4af3177ae 100644 --- a/kubernetes/11-suricata.yml +++ b/kubernetes/11-suricata.yml @@ -16,7 +16,7 @@ spec: spec: containers: - name: suricata-offline-container - image: ghcr.io/idaholab/malcolm/suricata:24.09.0 + image: ghcr.io/idaholab/malcolm/suricata:24.10.0 imagePullPolicy: Always stdin: false tty: true @@ -55,7 +55,7 @@ spec: name: suricata-offline-custom-configs-volume initContainers: - name: suricata-offline-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.09.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.10.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/12-file-monitor.yml b/kubernetes/12-file-monitor.yml index 0b17bc114..dc2f4b4b5 100644 --- a/kubernetes/12-file-monitor.yml +++ b/kubernetes/12-file-monitor.yml @@ -33,7 +33,7 @@ spec: spec: containers: - name: file-monitor-container - image: ghcr.io/idaholab/malcolm/file-monitor:24.09.0 + image: ghcr.io/idaholab/malcolm/file-monitor:24.10.0 imagePullPolicy: Always stdin: false tty: true @@ -83,7 +83,7 @@ spec: name: file-monitor-yara-rules-custom-volume initContainers: - name: file-monitor-live-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.09.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.10.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/13-filebeat.yml b/kubernetes/13-filebeat.yml index 02b8e9d2a..a9c45be20 100644 --- a/kubernetes/13-filebeat.yml +++ b/kubernetes/13-filebeat.yml @@ -33,7 +33,7 @@ spec: spec: containers: - name: filebeat-container - image: ghcr.io/idaholab/malcolm/filebeat-oss:24.09.0 + image: ghcr.io/idaholab/malcolm/filebeat-oss:24.10.0 imagePullPolicy: Always stdin: false tty: true @@ -83,7 +83,7 @@ spec: subPath: "nginx" initContainers: - name: filebeat-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.09.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.10.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/14-logstash.yml b/kubernetes/14-logstash.yml index 955626096..e1c8823e8 100644 --- a/kubernetes/14-logstash.yml +++ b/kubernetes/14-logstash.yml @@ -49,7 +49,7 @@ spec: # topologyKey: "kubernetes.io/hostname" containers: - name: logstash-container - image: ghcr.io/idaholab/malcolm/logstash-oss:24.09.0 + image: ghcr.io/idaholab/malcolm/logstash-oss:24.10.0 imagePullPolicy: Always stdin: false tty: true @@ -115,7 +115,7 @@ spec: subPath: "logstash" initContainers: - name: logstash-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.09.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.10.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/15-netbox-redis.yml b/kubernetes/15-netbox-redis.yml index 450ec2fad..4267666bd 100644 --- a/kubernetes/15-netbox-redis.yml +++ b/kubernetes/15-netbox-redis.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: netbox-redis-container - image: ghcr.io/idaholab/malcolm/redis:24.09.0 + image: ghcr.io/idaholab/malcolm/redis:24.10.0 imagePullPolicy: Always stdin: false tty: true @@ -83,7 +83,7 @@ spec: subPath: netbox/redis initContainers: - name: netbox-redis-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.09.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.10.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/16-netbox-redis-cache.yml b/kubernetes/16-netbox-redis-cache.yml index 2f419e85c..8bf968227 100644 --- a/kubernetes/16-netbox-redis-cache.yml +++ b/kubernetes/16-netbox-redis-cache.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: netbox-redis-cache-container - image: ghcr.io/idaholab/malcolm/redis:24.09.0 + image: ghcr.io/idaholab/malcolm/redis:24.10.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/17-netbox-postgres.yml b/kubernetes/17-netbox-postgres.yml index 6167c1c92..c96d1bb4c 100644 --- a/kubernetes/17-netbox-postgres.yml +++ b/kubernetes/17-netbox-postgres.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: netbox-postgres-container - image: ghcr.io/idaholab/malcolm/postgresql:24.09.0 + image: ghcr.io/idaholab/malcolm/postgresql:24.10.0 imagePullPolicy: Always stdin: false tty: true @@ -74,7 +74,7 @@ spec: subPath: netbox/postgres initContainers: - name: netbox-postgres-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.09.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.10.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/18-netbox.yml b/kubernetes/18-netbox.yml index 11d1d8d93..5d3d59a15 100644 --- a/kubernetes/18-netbox.yml +++ b/kubernetes/18-netbox.yml @@ -36,7 +36,7 @@ spec: spec: containers: - name: netbox-container - image: ghcr.io/idaholab/malcolm/netbox:24.09.0 + image: ghcr.io/idaholab/malcolm/netbox:24.10.0 imagePullPolicy: Always stdin: false tty: true @@ -88,7 +88,7 @@ spec: subPath: netbox/media initContainers: - name: netbox-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.09.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.10.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/19-htadmin.yml b/kubernetes/19-htadmin.yml index d592c32bc..9438eae0c 100644 --- a/kubernetes/19-htadmin.yml +++ b/kubernetes/19-htadmin.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: htadmin-container - image: ghcr.io/idaholab/malcolm/htadmin:24.09.0 + image: ghcr.io/idaholab/malcolm/htadmin:24.10.0 imagePullPolicy: Always stdin: false tty: true @@ -63,7 +63,7 @@ spec: subPath: "htadmin" initContainers: - name: htadmin-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.09.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.10.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/20-pcap-capture.yml b/kubernetes/20-pcap-capture.yml index 130570b12..44a13f65d 100644 --- a/kubernetes/20-pcap-capture.yml +++ b/kubernetes/20-pcap-capture.yml @@ -16,7 +16,7 @@ spec: spec: containers: - name: pcap-capture-container - image: ghcr.io/idaholab/malcolm/pcap-capture:24.09.0 + image: ghcr.io/idaholab/malcolm/pcap-capture:24.10.0 imagePullPolicy: Always stdin: false tty: true @@ -50,7 +50,7 @@ spec: subPath: "upload" initContainers: - name: pcap-capture-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.09.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.10.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/21-zeek-live.yml b/kubernetes/21-zeek-live.yml index 59cab5575..4c2d1fd69 100644 --- a/kubernetes/21-zeek-live.yml +++ b/kubernetes/21-zeek-live.yml @@ -16,7 +16,7 @@ spec: spec: containers: - name: zeek-live-container - image: ghcr.io/idaholab/malcolm/zeek:24.09.0 + image: ghcr.io/idaholab/malcolm/zeek:24.10.0 imagePullPolicy: Always stdin: false tty: true @@ -61,7 +61,7 @@ spec: subPath: "zeek/intel" initContainers: - name: zeek-live-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.09.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.10.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/22-suricata-live.yml b/kubernetes/22-suricata-live.yml index aefa5b1de..80d085b12 100644 --- a/kubernetes/22-suricata-live.yml +++ b/kubernetes/22-suricata-live.yml @@ -16,7 +16,7 @@ spec: spec: containers: - name: suricata-live-container - image: ghcr.io/idaholab/malcolm/suricata:24.09.0 + image: ghcr.io/idaholab/malcolm/suricata:24.10.0 imagePullPolicy: Always stdin: false tty: true @@ -56,7 +56,7 @@ spec: name: suricata-live-custom-configs-volume initContainers: - name: suricata-live-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.09.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.10.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/23-arkime-live.yml b/kubernetes/23-arkime-live.yml index f00b31ba7..39f5deb30 100644 --- a/kubernetes/23-arkime-live.yml +++ b/kubernetes/23-arkime-live.yml @@ -16,7 +16,7 @@ spec: spec: containers: - name: arkime-live-container - image: ghcr.io/idaholab/malcolm/arkime:24.09.0 + image: ghcr.io/idaholab/malcolm/arkime:24.10.0 imagePullPolicy: Always stdin: false tty: true @@ -64,7 +64,7 @@ spec: name: arkime-live-pcap-volume initContainers: - name: arkime-live-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.09.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.10.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/24-freq.yml b/kubernetes/24-freq.yml index 8792acf61..87bd1ff95 100644 --- a/kubernetes/24-freq.yml +++ b/kubernetes/24-freq.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: freq-container - image: ghcr.io/idaholab/malcolm/freq:24.09.0 + image: ghcr.io/idaholab/malcolm/freq:24.10.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/98-nginx-proxy.yml b/kubernetes/98-nginx-proxy.yml index 31e1ca3d0..9608d5163 100644 --- a/kubernetes/98-nginx-proxy.yml +++ b/kubernetes/98-nginx-proxy.yml @@ -39,7 +39,7 @@ spec: spec: containers: - name: nginx-proxy-container - image: ghcr.io/idaholab/malcolm/nginx-proxy:24.09.0 + image: ghcr.io/idaholab/malcolm/nginx-proxy:24.10.0 imagePullPolicy: Always stdin: false tty: true @@ -99,7 +99,7 @@ spec: subPath: "nginx" initContainers: - name: nginx-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.09.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.10.0 imagePullPolicy: Always stdin: false tty: true diff --git a/logstash/maps/zeek_log_ecs_categories.yaml b/logstash/maps/zeek_log_ecs_categories.yaml index a357eb26e..0c3235d98 100644 --- a/logstash/maps/zeek_log_ecs_categories.yaml +++ b/logstash/maps/zeek_log_ecs_categories.yaml @@ -49,6 +49,7 @@ "known_certs": ["file"] "known_hosts": ["network"] "known_modbus": ["ot", "network"] +"known_routers": ["network"] "known_services": ["network"] "ldap": ["authentication", "iam", "network"] "login": ["authentication", "network"] diff --git a/logstash/pipelines/enrichment/21_netbox.conf b/logstash/pipelines/enrichment/21_netbox.conf index ae2113eac..a42ec3009 100644 --- a/logstash/pipelines/enrichment/21_netbox.conf +++ b/logstash/pipelines/enrichment/21_netbox.conf @@ -12,8 +12,7 @@ filter { ruby { id => "ruby_determine_netbox_suitability" - # @logtypes = {"suricata"=>["alert"], "zeek"=>["conn", "known_hosts", "known_services", "notice", "signatures", "software", "weird"]} - init => "logtypesStr = ENV['LOGSTASH_NETBOX_ENRICHMENT_DATASETS'] || 'suricata.alert,zeek.conn,zeek.dhcp,zeek.dns,zeek.known_hosts,zeek.known_services,zeek.ntlm,zeek.notice,zeek.signatures,zeek.software,zeek.weird' ; logtypesArr = logtypesStr.gsub(/\s+/, '').split(','); @logtypes = logtypesArr.group_by { |logtype| logtype.split('.').first }.transform_values { |values| values.map { |v| v.split('.')[1] } }" + init => "logtypesStr = ENV['LOGSTASH_NETBOX_ENRICHMENT_DATASETS'] || 'suricata.alert,zeek.conn,zeek.dhcp,zeek.dns,zeek.known_hosts,zeek.known_routers,zeek.known_services,zeek.ntlm,zeek.notice,zeek.signatures,zeek.software,zeek.weird' ; logtypesArr = logtypesStr.gsub(/\s+/, '').split(','); @logtypes = logtypesArr.group_by { |logtype| logtype.split('.').first }.transform_values { |values| values.map { |v| v.split('.')[1] } }" code => " provider = event.get('[event][provider]').to_s dataset = event.get('[event][dataset]').to_s diff --git a/logstash/pipelines/zeek/11_zeek_parse.conf b/logstash/pipelines/zeek/11_zeek_parse.conf index 4707bd0f1..83daff13b 100644 --- a/logstash/pipelines/zeek/11_zeek_parse.conf +++ b/logstash/pipelines/zeek/11_zeek_parse.conf @@ -1735,7 +1735,7 @@ filter { id => "dissect_zeek_http_with_all_fields" # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP mapping => { - "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][trans_depth]} %{[zeek_cols][method]} %{[zeek_cols][host]} %{[zeek_cols][uri]} %{[zeek_cols][referrer]} %{[zeek_cols][version]} %{[zeek_cols][user_agent]} %{[zeek_cols][origin]} %{[zeek_cols][request_body_len]} %{[zeek_cols][response_body_len]} %{[zeek_cols][status_code]} %{[zeek_cols][status_msg]} %{[zeek_cols][info_code]} %{[zeek_cols][info_msg]} %{[zeek_cols][tags]} %{[zeek_cols][user]} %{[zeek_cols][password]} %{[zeek_cols][proxied]} %{[zeek_cols][orig_fuids]} %{[zeek_cols][orig_filenames]} %{[zeek_cols][orig_mime_types]} %{[zeek_cols][resp_fuids]} %{[zeek_cols][resp_filenames]} %{[zeek_cols][resp_mime_types]} %{[zeek_cols][ja4h]} %{[zeek_cols][post_username]} %{[zeek_cols][post_password_plain]} %{[zeek_cols][post_password_md5]} %{[zeek_cols][post_password_sha1]} %{[zeek_cols][post_password_sha256]}" + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][trans_depth]} %{[zeek_cols][method]} %{[zeek_cols][host]} %{[zeek_cols][uri]} %{[zeek_cols][referrer]} %{[zeek_cols][version]} %{[zeek_cols][user_agent]} %{[zeek_cols][origin]} %{[zeek_cols][request_body_len]} %{[zeek_cols][response_body_len]} %{[zeek_cols][status_code]} %{[zeek_cols][status_msg]} %{[zeek_cols][info_code]} %{[zeek_cols][info_msg]} %{[zeek_cols][tags]} %{[zeek_cols][user]} %{[zeek_cols][password]} %{[zeek_cols][proxied]} %{[zeek_cols][orig_fuids]} %{[zeek_cols][orig_filenames]} %{[zeek_cols][orig_mime_types]} %{[zeek_cols][resp_fuids]} %{[zeek_cols][resp_filenames]} %{[zeek_cols][resp_mime_types]} %{[zeek_cols][client_header_names]} %{[zeek_cols][server_header_names]} %{[zeek_cols][ja4h]} %{[zeek_cols][post_username]} %{[zeek_cols][post_password_plain]} %{[zeek_cols][post_password_md5]} %{[zeek_cols][post_password_sha1]} %{[zeek_cols][post_password_sha256]}" } } if ("_dissectfailure" in [tags]) { @@ -1746,18 +1746,20 @@ filter { } ruby { id => "ruby_zip_zeek_http" - init => "@zeek_http_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'trans_depth', 'method', 'host', 'uri', 'referrer', 'version', 'user_agent', 'origin', 'request_body_len', 'response_body_len', 'status_code', 'status_msg', 'info_code', 'info_msg', 'tags', 'user', 'password', 'proxied', 'orig_fuids', 'orig_filenames', 'orig_mime_types', 'resp_fuids', 'resp_filenames', 'resp_mime_types', 'ja4h', 'post_username', 'post_password_plain', 'post_password_md5', 'post_password_sha1', 'post_password_sha256' ]" + init => "@zeek_http_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'trans_depth', 'method', 'host', 'uri', 'referrer', 'version', 'user_agent', 'origin', 'request_body_len', 'response_body_len', 'status_code', 'status_msg', 'info_code', 'info_msg', 'tags', 'user', 'password', 'proxied', 'orig_fuids', 'orig_filenames', 'orig_mime_types', 'resp_fuids', 'resp_filenames', 'resp_mime_types', 'client_header_names', 'server_header_names', 'ja4h', 'post_username', 'post_password_plain', 'post_password_md5', 'post_password_sha1', 'post_password_sha256' ]" code => "event.set('[zeek_cols]', @zeek_http_field_names.zip(event.get('[message]')).to_h)" } } mutate { id => "mutate_split_zeek_http_commas" - split => { "[zeek_cols][orig_fuids]" => "," + split => { "[zeek_cols][client_header_names]" => "," "[zeek_cols][orig_filenames]" => "," + "[zeek_cols][orig_fuids]" => "," "[zeek_cols][orig_mime_types]" => "," - "[zeek_cols][resp_fuids]" => "," + "[zeek_cols][proxied]" => "," "[zeek_cols][resp_filenames]" => "," + "[zeek_cols][resp_fuids]" => "," "[zeek_cols][resp_mime_types]" => "," - "[zeek_cols][proxied]" => "," + "[zeek_cols][server_header_names]" => "," "[zeek_cols][tags]" => "," } } } @@ -2127,6 +2129,32 @@ filter { mutate { id => "mutate_add_tag_ics_known_modbus_log" add_tag => [ "ics" ] } + } else if ([log_source] == "known_routers") { + ############################################################################################################################# + # known_routers.log + + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_known_routers" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_l2_addr]} %{[zeek_cols][ttl]} %{[zeek_cols][hlim]}" + } + } + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_known_routers" + # zeek's default delimiter is a literal tab, MAKE SURE YOUR EDITOR DOESN'T SCREW IT UP + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_known_routers" + init => "@zeek_known_routers_field_names = [ 'ts', 'orig_h', 'orig_l2_addr', 'ttl', 'hlim' ]" + code => "event.set('[zeek_cols]', @zeek_known_routers_field_names.zip(event.get('[message]')).to_h)" + } + } + } + } else if ([log_source] == "known_services") { ############################################################################################################################# # known_services.log diff --git a/malcolm-iso/vagrant/Vagrantfile b/malcolm-iso/vagrant/Vagrantfile index ecc7852a2..8057a54f5 100644 --- a/malcolm-iso/vagrant/Vagrantfile +++ b/malcolm-iso/vagrant/Vagrantfile @@ -36,6 +36,16 @@ Vagrant.configure("2") do |config| lv.cpus = 4 end + config.vm.provider "vmware_desktop" do |vd| + vd.memory = "8192" + vd.cpus = 4 + end + + config.vm.provider "vmware_fusion" do |vf| + vf.memory = "8192" + vf.cpus = 4 + end + config.vm.provision "shell", inline: <<-STEP1 dpkg-reconfigure debconf -f noninteractive -p critical export DEBIAN_FRONTEND=noninteractive diff --git a/scripts/build.sh b/scripts/build.sh index ff21bebd8..050128dbc 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -122,9 +122,9 @@ fi # build the image(s) DOCKER_COMPOSE_COMMAND="${DOCKER_COMPOSE_BIN[@]} --profile malcolm -f "$CONFIG_FILE"" if [[ $CONFIRMATION =~ ^[Yy] ]]; then - $DOCKER_COMPOSE_COMMAND --progress=plain build --force-rm --no-cache --build-arg TARGETPLATFORM="$TARGET_PLATFORM" --build-arg GITHUB_TOKEN="$GITHUB_API_TOKEN" --build-arg MAXMIND_GEOIP_DB_LICENSE_KEY="$MAXMIND_API_KEY" --build-arg MAXMIND_GEOIP_DB_ALTERNATE_DOWNLOAD_URL="${MAXMIND_GEOIP_DB_ALTERNATE_DOWNLOAD_URL:-}" --build-arg BUILD_DATE="$BUILD_DATE" --build-arg MALCOLM_VERSION="$MALCOLM_VERSION" --build-arg VCS_REVISION="$VCS_REVISION" "$@" + $DOCKER_COMPOSE_COMMAND --progress=plain build --force-rm --no-cache --build-arg TARGETPLATFORM="$TARGET_PLATFORM" --build-arg GITHUB_TOKEN="$GITHUB_API_TOKEN" --build-arg MAXMIND_GEOIP_DB_LICENSE_KEY="$MAXMIND_API_KEY" --build-arg MAXMIND_GEOIP_DB_ALTERNATE_DOWNLOAD_URL="${MAXMIND_GEOIP_DB_ALTERNATE_DOWNLOAD_URL:-}" --build-arg ZEEK_DEB_ALTERNATE_DOWNLOAD_URL="${ZEEK_DEB_ALTERNATE_DOWNLOAD_URL:-}" --build-arg BUILD_DATE="$BUILD_DATE" --build-arg MALCOLM_VERSION="$MALCOLM_VERSION" --build-arg VCS_REVISION="$VCS_REVISION" "$@" else - $DOCKER_COMPOSE_COMMAND --progress=plain build --build-arg TARGETPLATFORM="$TARGET_PLATFORM" --build-arg GITHUB_TOKEN="$GITHUB_API_TOKEN" --build-arg MAXMIND_GEOIP_DB_LICENSE_KEY="$MAXMIND_API_KEY" --build-arg MAXMIND_GEOIP_DB_ALTERNATE_DOWNLOAD_URL="${MAXMIND_GEOIP_DB_ALTERNATE_DOWNLOAD_URL:-}" --build-arg BUILD_DATE="$BUILD_DATE" --build-arg MALCOLM_VERSION="$MALCOLM_VERSION" --build-arg VCS_REVISION="$VCS_REVISION" "$@" + $DOCKER_COMPOSE_COMMAND --progress=plain build --build-arg TARGETPLATFORM="$TARGET_PLATFORM" --build-arg GITHUB_TOKEN="$GITHUB_API_TOKEN" --build-arg MAXMIND_GEOIP_DB_LICENSE_KEY="$MAXMIND_API_KEY" --build-arg MAXMIND_GEOIP_DB_ALTERNATE_DOWNLOAD_URL="${MAXMIND_GEOIP_DB_ALTERNATE_DOWNLOAD_URL:-}" --build-arg ZEEK_DEB_ALTERNATE_DOWNLOAD_URL="${ZEEK_DEB_ALTERNATE_DOWNLOAD_URL:-}" --build-arg BUILD_DATE="$BUILD_DATE" --build-arg MALCOLM_VERSION="$MALCOLM_VERSION" --build-arg VCS_REVISION="$VCS_REVISION" "$@" fi # we're going to do some validation that some things got pulled/built correctly diff --git a/scripts/demo/Vagrantfile b/scripts/demo/Vagrantfile index e80b63314..d90236f7d 100644 --- a/scripts/demo/Vagrantfile +++ b/scripts/demo/Vagrantfile @@ -43,6 +43,19 @@ Vagrant.configure("2") do |config| libvirt.storage :file, :size => '150G' end + + config.vm.provider "vmware_desktop" do |vd| + vd.cpus = 8 + vd.memory = 24576 + config.vm.disk :disk, size: "150G" + end + + config.vm.provider "vmware_fusion" do |vf| + vf.cpus = 8 + vf.memory = 24576 + config.vm.disk :disk, size: "150G" + end + config.vm.provision "shell", privileged: true, inline: <<-STEP0 yum install -y btrfs-progs DISK=$(lsblk -o NAME,FSTYPE -dsn | awk '$2 == "" {print $1}' | grep -P "^[a-z]+$" | tail -n 1) diff --git a/scripts/third-party-environments/aws/ami/packer_vars.json.example b/scripts/third-party-environments/aws/ami/packer_vars.json.example index 89f04d1a5..ea5f709a6 100644 --- a/scripts/third-party-environments/aws/ami/packer_vars.json.example +++ b/scripts/third-party-environments/aws/ami/packer_vars.json.example @@ -3,7 +3,7 @@ "aws_secret_key": "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX", "instance_type": "t2.micro", "instance_arch": "x86_64", - "malcolm_tag": "v24.09.0", + "malcolm_tag": "v24.10.0", "malcolm_repo": "cisagov/Malcolm", "malcolm_uid": "1000", "ssh_username": "ec2-user", diff --git a/scripts/third-party-environments/aws/ami/scripts/Malcolm_AMI_Setup.sh b/scripts/third-party-environments/aws/ami/scripts/Malcolm_AMI_Setup.sh index 3108ec095..082eb7c20 100755 --- a/scripts/third-party-environments/aws/ami/scripts/Malcolm_AMI_Setup.sh +++ b/scripts/third-party-environments/aws/ami/scripts/Malcolm_AMI_Setup.sh @@ -32,7 +32,7 @@ fi # -u UID (user UID, e.g., 1000) VERBOSE_FLAG= MALCOLM_REPO=${MALCOLM_REPO:-cisagov/Malcolm} -MALCOLM_TAG=${MALCOLM_TAG:-v24.09.0} +MALCOLM_TAG=${MALCOLM_TAG:-v24.10.0} [[ -z "$MALCOLM_UID" ]] && ( [[ $EUID -eq 0 ]] && MALCOLM_UID=1000 || MALCOLM_UID="$(id -u)" ) while getopts 'vr:t:u:' OPTION; do case "$OPTION" in diff --git a/scripts/third-party-logs/fluent-bit-setup.ps1 b/scripts/third-party-logs/fluent-bit-setup.ps1 index c1a2bf5fe..f56b74132 100644 --- a/scripts/third-party-logs/fluent-bit-setup.ps1 +++ b/scripts/third-party-logs/fluent-bit-setup.ps1 @@ -9,7 +9,7 @@ ############################################################################### $fluent_bit_version = '3.1' -$fluent_bit_full_version = '3.1.8' +$fluent_bit_full_version = '3.1.9' ############################################################################### # select an item from a menu provided in an array diff --git a/shared/bin/capa-build.sh b/shared/bin/capa-build.sh index c77970a3f..60ab5d207 100644 --- a/shared/bin/capa-build.sh +++ b/shared/bin/capa-build.sh @@ -1,6 +1,6 @@ #!/bin/bash -export CAPA_VERSION="7.2.0" +export CAPA_VERSION="7.4.0" export CAPA_SRC_URL="https://github.com/mandiant/capa/archive/refs/tags/v${CAPA_VERSION}.zip" export CAPA_RULES_URL="https://github.com/mandiant/capa-rules/archive/refs/tags/v${CAPA_VERSION}.zip" diff --git a/shared/bin/zeek-deb-download.sh b/shared/bin/zeek-deb-download.sh index cbe8e6fcf..c695ccad2 100755 --- a/shared/bin/zeek-deb-download.sh +++ b/shared/bin/zeek-deb-download.sh @@ -6,13 +6,19 @@ unset VERBOSE command -v dpkg >/dev/null 2>&1 && ARCH="$(dpkg --print-architecture)" || ARCH=amd64 DISTRO=Debian_12 OUTPUT_DIR=/tmp -ZEEK_VERSION=7.0.1-0 +ZEEK_VERSION=7.0.3-0 +PRESERVE_HIERARCHY=false +ZEEK_DEB_ALTERNATE_DOWNLOAD_URL=${ZEEK_DEB_ALTERNATE_DOWNLOAD_URL:-} +ZEEK_DEB_ALTERNATE_DOWNLOAD_URL_FILE=${ZEEK_DEB_ALTERNATE_DOWNLOAD_URL_FILE:-} -while getopts a:d:o:vz: opts; do +while getopts a:d:f:ho:u:vz: opts; do case ${opts} in a) ARCH=${OPTARG} ;; d) DISTRO=${OPTARG} ;; + f) ZEEK_DEB_ALTERNATE_DOWNLOAD_URL_FILE=${OPTARG} ;; + h) PRESERVE_HIERARCHY=true ;; o) OUTPUT_DIR=${OPTARG} ;; + u) ZEEK_DEB_ALTERNATE_DOWNLOAD_URL=${OPTARG} ;; v) VERBOSE=1 ;; z) ZEEK_VERSION=${OPTARG} ;; esac @@ -23,23 +29,37 @@ if [[ -n $VERBOSE ]]; then set -x fi -URL_PREFIX="https://downloadcontentcdn.opensuse.org/repositories/security:/zeek/${DISTRO}" -URLS=( - "${URL_PREFIX}/${ARCH}/libbroker-dev_${ZEEK_VERSION}_${ARCH}.deb" - "${URL_PREFIX}/${ARCH}/zeek-core-dev_${ZEEK_VERSION}_${ARCH}.deb" - "${URL_PREFIX}/${ARCH}/zeek-core_${ZEEK_VERSION}_${ARCH}.deb" - "${URL_PREFIX}/${ARCH}/zeek-spicy-dev_${ZEEK_VERSION}_${ARCH}.deb" - "${URL_PREFIX}/${ARCH}/zeek_${ZEEK_VERSION}_${ARCH}.deb" - "${URL_PREFIX}/${ARCH}/zeekctl_${ZEEK_VERSION}_${ARCH}.deb" - "${URL_PREFIX}/all/zeek-client_${ZEEK_VERSION}_all.deb" - "${URL_PREFIX}/all/zeek-zkg_${ZEEK_VERSION}_all.deb" - "${URL_PREFIX}/all/zeek-btest_${ZEEK_VERSION}_all.deb" - "${URL_PREFIX}/all/zeek-btest-data_${ZEEK_VERSION}_all.deb" +if [[ -z "${ZEEK_DEB_ALTERNATE_DOWNLOAD_URL}" ]] && [[ -f "${ZEEK_DEB_ALTERNATE_DOWNLOAD_URL_FILE}" ]]; then + ZEEK_DEB_ALTERNATE_DOWNLOAD_URL="$(head -n 1 "${ZEEK_DEB_ALTERNATE_DOWNLOAD_URL_FILE}")" +fi + +URL_PREFIXES=( + "https://downloadcontentcdn.opensuse.org/repositories/security:/zeek" +) +[[ -n "$ZEEK_DEB_ALTERNATE_DOWNLOAD_URL" ]] && URL_PREFIXES+=( "$ZEEK_DEB_ALTERNATE_DOWNLOAD_URL" ) + +URL_SUFFIXES=( + "${DISTRO}/${ARCH}/libbroker-dev_${ZEEK_VERSION}_${ARCH}.deb" + "${DISTRO}/${ARCH}/zeek-core-dev_${ZEEK_VERSION}_${ARCH}.deb" + "${DISTRO}/${ARCH}/zeek-core_${ZEEK_VERSION}_${ARCH}.deb" + "${DISTRO}/${ARCH}/zeek-spicy-dev_${ZEEK_VERSION}_${ARCH}.deb" + "${DISTRO}/${ARCH}/zeek_${ZEEK_VERSION}_${ARCH}.deb" + "${DISTRO}/${ARCH}/zeekctl_${ZEEK_VERSION}_${ARCH}.deb" + "${DISTRO}/all/zeek-client_${ZEEK_VERSION}_all.deb" + "${DISTRO}/all/zeek-zkg_${ZEEK_VERSION}_all.deb" + "${DISTRO}/all/zeek-btest_${ZEEK_VERSION}_all.deb" + "${DISTRO}/all/zeek-btest-data_${ZEEK_VERSION}_all.deb" ) pushd "$OUTPUT_DIR" >/dev/null 2>&1 -for URL in ${URLS[@]}; do - curl -fsSL -O -J "${URL}" +for URL_SUFFIX in ${URL_SUFFIXES[@]}; do + [[ "$PRESERVE_HIERARCHY" == "true" ]] && OUTPUT_DIR_REL="$(dirname "$URL_SUFFIX")" || OUTPUT_DIR_REL=. + mkdir -p "$OUTPUT_DIR_REL" + pushd "$OUTPUT_DIR_REL" >/dev/null 2>&1 + for URL_PREFIX in ${URL_PREFIXES[@]}; do + curl -fsSL -O -J "${URL_PREFIX%/}/${URL_SUFFIX}" && break + done + popd >/dev/null 2>&1 done popd >/dev/null 2>&1 diff --git a/shared/bin/zeek_install_plugins.sh b/shared/bin/zeek_install_plugins.sh index f1ea57e27..9907331fc 100755 --- a/shared/bin/zeek_install_plugins.sh +++ b/shared/bin/zeek_install_plugins.sh @@ -30,18 +30,26 @@ function get_latest_github_tagged_release() { # zkg_install_github_repo # # zkg install the latest GitHub release tag if available (else, master/HEAD) -# release tag/branch can be overriden by specifying the branch name with after the URL delimited by a | -# +# some optional overrides can be specified using | as a delimiter in the URL parameter: +# URL|branch|environment variables +# URL - the full GitHub URL to install +# branch - the branch to checkout and install (otherwise, the latest release tag will be used, or the default branch if no releases exist) +# environment variables - semicolon-separated list of environment variables to set before calling zkg (e.g., CMAKE_BUILD_TYPE=Debug;HILTI_CXX_FLAGS=-fno-var-tracking-assignments) function zkg_install_github_repo() { URL_PARAM="$1" - URL_BRANCH_DELIM='|' - URL_BRANCH_DELIM_COUNT="$(awk -F"${URL_BRANCH_DELIM}" '{print NF-1}' <<< "${URL_PARAM}")" - if (( $URL_BRANCH_DELIM_COUNT > 0 )); then + URL_DELIM='|' + URL_DELIM_COUNT="$(awk -F"${URL_DELIM}" '{print NF-1}' <<< "${URL_PARAM}")" + REPO_URL="" + BRANCH_OVERRIDE="" + ENV_LIST="" + if (( $URL_DELIM_COUNT >= 0 )); then REPO_URL="$(echo "$URL_PARAM" | cut -d'|' -f1)" + fi + if (( $URL_DELIM_COUNT >= 1 )); then BRANCH_OVERRIDE="$(echo "$URL_PARAM" | cut -d'|' -f2)" - else - REPO_URL="$URL_PARAM" - BRANCH_OVERRIDE="" + fi + if (( $URL_DELIM_COUNT >= 2 )); then + ENV_LIST="$(echo "$URL_PARAM" | cut -d'|' -f3)" fi if [[ -n $REPO_URL ]]; then if [[ -n $BRANCH_OVERRIDE ]]; then @@ -49,6 +57,12 @@ function zkg_install_github_repo() { else REPO_LATEST_RELEASE="$(get_latest_github_tagged_release "$REPO_URL")" fi + if [[ -n $ENV_LIST ]]; then + IFS=';' read -ra ENVS <<< "${ENV_LIST}" + for ENV in "${ENVS[@]}"; do + export "$ENV" + done + fi if [[ -n $REPO_LATEST_RELEASE ]]; then zkg install --nodeps --force --skiptests --version "$REPO_LATEST_RELEASE" "$REPO_URL" else @@ -57,9 +71,12 @@ function zkg_install_github_repo() { fi } -# don't consume as many resources when building spicy-analyzers, even if it's slower. -# https://github.com/zeek/spicy-analyzers/pull/60 -export SPICY_ZKG_PROCESSES=1 +# don't consume as many resources when building spicy analyzers, even if it's slower. +# https://docs.zeek.org/projects/spicy/en/latest/toolchain.html +TOTAL_CPUS="$(nproc --all 2>/dev/null || echo '1')" +TOTAL_CPUS=$(( TOTAL_CPUS / 2 )) +(( $TOTAL_CPUS <= 0 )) && TOTAL_CPUS=1 +export HILTI_JIT_PARALLELISM=${BUILD_JOBS:-$TOTAL_CPUS} # install Zeek packages that install nicely using zkg ZKG_GITHUB_URLS=( @@ -113,6 +130,7 @@ ZKG_GITHUB_URLS=( "https://github.com/zeek/spicy-tftp" "https://github.com/zeek/spicy-zip" ) + for i in ${ZKG_GITHUB_URLS[@]}; do zkg_install_github_repo "$i" done diff --git a/zeek/config/local.zeek b/zeek/config/local.zeek index f8ff4b04b..e3aea8183 100644 --- a/zeek/config/local.zeek +++ b/zeek/config/local.zeek @@ -56,7 +56,6 @@ global json_format = (getenv("ZEEK_JSON") == true_regex) ? T : F; redef LogAscii::use_json = T; @endif -@load tuning/defaults @load frameworks/software/vulnerable @load frameworks/software/version-changes @load frameworks/software/windows-version-detection @@ -70,6 +69,7 @@ global json_format = (getenv("ZEEK_JSON") == true_regex) ? T : F; @load protocols/ftp/software @load protocols/http/detect-sqli @load protocols/http/detect-webapps +@load protocols/http/header-names @load protocols/http/software @load protocols/http/software-browser-plugins @load protocols/mysql/software @@ -282,6 +282,8 @@ event zeek_init() &priority=-5 { redef LDAP::default_capture_password = T; @endif +redef HTTP::log_client_header_names = T; +redef HTTP::log_server_header_names = T; redef LDAP::default_log_search_attributes = F; redef SNIFFPASS::notice_log_enable = F; redef CVE_2021_44228::log = F;