From b0c84a0d3700cf903152dc8531258ee6bd58c494 Mon Sep 17 00:00:00 2001 From: weliasz <77732905+weliasz@users.noreply.github.com> Date: Tue, 9 Nov 2021 09:33:56 +0100 Subject: [PATCH] Feat/suffix for events (#192) * feat: suffix for events * feat: suffix for events * fix: build fix * fix: build fix --- .../manager/hec_sender.py | 19 ++++--- .../manager/static/mib_enricher.py | 35 ++++--------- tests/test_additional_data_extraction.py | 2 - tests/test_hec_sender.py | 52 +++++++++++-------- tests/test_mib_enricher.py | 26 +++++----- 5 files changed, 62 insertions(+), 72 deletions(-) diff --git a/splunk_connect_for_snmp_poller/manager/hec_sender.py b/splunk_connect_for_snmp_poller/manager/hec_sender.py index e6de371..258b665 100644 --- a/splunk_connect_for_snmp_poller/manager/hec_sender.py +++ b/splunk_connect_for_snmp_poller/manager/hec_sender.py @@ -159,8 +159,11 @@ def _enrich_event_data(mib_enricher: MibEnricher, variables_binds: dict) -> str: interface_index="1" interface_desc="lo" ' """ metric_result = json.loads(variables_binds["metric"]) + parsed_index = metric_result.get("parsed_index") non_metric_result = variables_binds["non_metric"] - additional_dimensions = mib_enricher.append_additional_dimensions(metric_result) + additional_dimensions = mib_enricher.append_additional_dimensions( + metric_result, parsed_index + ) logger.debug(additional_dimensions) for field_name in additional_dimensions: if field_name in metric_result: @@ -187,7 +190,7 @@ def build_metric_data( EventField.FREQUENCY.value: ir.frequency_str, } if mib_enricher: - _enrich_metric_data(mib_enricher, json_val, fields) + _enrich_metric_data(mib_enricher, json_val, fields, parsed_index) if additional_metric_fields: fields = ir.extend_dict_with_provided_data(fields, additional_metric_fields) @@ -199,17 +202,13 @@ def build_metric_data( else: builder.add(EventField.SOURCETYPE, "sc4snmp:metric") - extract_additional_properties(fields, metric_name, metric_value, parsed_index) + extract_additional_properties(fields, parsed_index) builder.add_fields(fields) return builder.build() -def extract_additional_properties(fields, metric_name, metric_value, parsed_index): - stripped = metric_name[: metric_name.index("_")] - del fields["metric_name:" + metric_name] - fields["metric_name:" + stripped] = metric_value - +def extract_additional_properties(fields, parsed_index): if parsed_index: for key, value in parsed_index.items(): fields[key] = value @@ -227,10 +226,10 @@ def build_error_data( def _enrich_metric_data( - mib_enricher: MibEnricher, variables_binds: dict, fields: dict + mib_enricher: MibEnricher, variables_binds: dict, fields: dict, parsed_index ) -> None: additional_if_mib_dimensions = mib_enricher.append_additional_dimensions( - variables_binds + variables_binds, parsed_index ) for field_name in additional_if_mib_dimensions: if field_name in variables_binds: diff --git a/splunk_connect_for_snmp_poller/manager/static/mib_enricher.py b/splunk_connect_for_snmp_poller/manager/static/mib_enricher.py index 4f8c114..af97033 100644 --- a/splunk_connect_for_snmp_poller/manager/static/mib_enricher.py +++ b/splunk_connect_for_snmp_poller/manager/static/mib_enricher.py @@ -22,28 +22,15 @@ logger = logging.getLogger(__name__) -def extract_current_index_from_metric(current_translated_oid): - try: - if current_translated_oid: - return ( - int( - current_translated_oid[ - current_translated_oid.rindex("_") + 1 : # noqa: E203 - ] - ) - - 1 - ) - except ValueError: - logger.warning( - f"Could not find any index for {current_translated_oid}. This will not be enriched" - ) - +def extract_current_index_from_metric(parsed_index): + if parsed_index and "ifIndex" in parsed_index: + return int(parsed_index["ifIndex"]) - 1 return None def extract_dimension_name_and_value(dimension, index): all_keys = dimension.keys() - if len(all_keys) == 1: + if len(all_keys) == 1 and index is not None: dimension_name = [key for key in all_keys][0] dimension_values = dimension[dimension_name] # We need to enrich only table data. Static values like IF-MIB::ifNumber.0 won't be enriched (it doesn't @@ -68,7 +55,7 @@ def get_by_oid_and_type(self, oid_family, type): oid_record = self.get_by_oid(oid_family.split(".")[1]) return oid_record.get(type, {}) - def __enrich_if_mib_existing(self, metric_name): + def __enrich_if_mib_existing(self, metric_name, parsed_index): result = [] if metric_name and metric_name.startswith(InterfaceMib.IF_MIB_METRIC_PREFIX): if self._mib_static_data_collection: @@ -76,7 +63,7 @@ def __enrich_if_mib_existing(self, metric_name): InterfaceMib.IF_MIB_METRIC_PREFIX, enricher_existing_varbinds ) for dimension in if_mib_record: - index = extract_current_index_from_metric(metric_name) + index = extract_current_index_from_metric(parsed_index) ( dimension_name, dimension_value, @@ -85,11 +72,11 @@ def __enrich_if_mib_existing(self, metric_name): result.append({dimension_name: dimension_value}) return result - def __enrich_if_mib_additional(self, metric_name): + def __enrich_if_mib_additional(self, metric_name, parsed_index): for oid_family in self._mib_static_data_collection.keys(): if oid_family in metric_name: try: - index = extract_current_index_from_metric(metric_name) + 1 + index = extract_current_index_from_metric(parsed_index) + 1 index_field = self.get_by_oid_and_type( oid_family, enricher_additional_varbinds )["indexNum"] @@ -100,17 +87,17 @@ def __enrich_if_mib_additional(self, metric_name): logger.debug(f"Can't get the index from metric name: {metric_name}") return [] - def append_additional_dimensions(self, translated_var_bind): + def append_additional_dimensions(self, translated_var_bind, parsed_index): if translated_var_bind: metric_name = translated_var_bind[InterfaceMib.METRIC_NAME_KEY] additional_if_mib_dimensions = [] fields_list = [] if self._mib_static_data_collection: additional_if_mib_dimensions += self.__enrich_if_mib_existing( - metric_name + metric_name, parsed_index ) additional_if_mib_dimensions += self.__enrich_if_mib_additional( - metric_name + metric_name, parsed_index ) for more_data in additional_if_mib_dimensions: translated_var_bind.update(more_data) diff --git a/tests/test_additional_data_extraction.py b/tests/test_additional_data_extraction.py index dbbf56c..119e0a8 100644 --- a/tests/test_additional_data_extraction.py +++ b/tests/test_additional_data_extraction.py @@ -34,8 +34,6 @@ def test_data_extraction(self): extract_additional_properties( fields, - "sc4snmp.TCP-MIB.tcpConnLocalPort_192_168_0_1_161_127_0_0_1_5", - "1111", parsed_index, ) diff --git a/tests/test_hec_sender.py b/tests/test_hec_sender.py index e23f3e7..400d323 100644 --- a/tests/test_hec_sender.py +++ b/tests/test_hec_sender.py @@ -46,103 +46,109 @@ def test__enrich_metric_data_index_0(self): fields = {"metric_name:sc4snmp.IF-MIB.ifNumber_0": "2"} fields_initial = fields.copy() variables_binds = { - "metric_name": "sc4snmp.IF-MIB.ifNumber_0", + "metric_name": "sc4snmp.IF-MIB.ifNumber", "_value": "2", "metric_type": "Integer", } variables_binds_initial = variables_binds.copy() - _enrich_metric_data(_MibEnricher, variables_binds, fields) + _enrich_metric_data(_MibEnricher, variables_binds, fields, {"ifIndex": 0}) self.assertEqual(variables_binds, variables_binds_initial) self.assertEqual(fields, fields_initial) def test__enrich_metric_data_index_1(self): fields = { - "metric_name:sc4snmp.IF-MIB.ifIndex_1": "1", + "metric_name:sc4snmp.IF-MIB.ifIndex": "1", "interface_desc": "lo", "interface_index": "1", } - fields_initial = {"metric_name:sc4snmp.IF-MIB.ifIndex_1": "1"} + fields_initial = {"metric_name:sc4snmp.IF-MIB.ifIndex": "1"} variables_binds = { - "metric_name": "sc4snmp.IF-MIB.ifIndex_1", + "metric_name": "sc4snmp.IF-MIB.ifIndex", "_value": "1", "metric_type": "Integer", "interface_desc": "lo", "interface_index": "1", } variables_binds_initial = { - "metric_name": "sc4snmp.IF-MIB.ifIndex_1", + "metric_name": "sc4snmp.IF-MIB.ifIndex", "_value": "1", "metric_type": "Integer", } - _enrich_metric_data(_MibEnricher, variables_binds_initial, fields_initial) + _enrich_metric_data( + _MibEnricher, variables_binds_initial, fields_initial, {"ifIndex": 1} + ) self.assertEqual(variables_binds, variables_binds_initial) self.assertEqual(fields, fields_initial) def test__enrich_metric_data_index_2(self): fields = { - "metric_name:sc4snmp.IF-MIB.ifIndex_2": "2", + "metric_name:sc4snmp.IF-MIB.ifIndex": "2", "interface_desc": "eth0", "interface_index": "2", } - fields_initial = {"metric_name:sc4snmp.IF-MIB.ifIndex_2": "2"} + fields_initial = {"metric_name:sc4snmp.IF-MIB.ifIndex": "2"} variables_binds = { - "metric_name": "sc4snmp.IF-MIB.ifIndex_2", + "metric_name": "sc4snmp.IF-MIB.ifIndex", "_value": "2", "metric_type": "Integer", "interface_desc": "eth0", "interface_index": "2", } variables_binds_initial = { - "metric_name": "sc4snmp.IF-MIB.ifIndex_2", + "metric_name": "sc4snmp.IF-MIB.ifIndex", "_value": "2", "metric_type": "Integer", } - _enrich_metric_data(_MibEnricher, variables_binds_initial, fields_initial) + _enrich_metric_data( + _MibEnricher, variables_binds_initial, fields_initial, {"ifIndex": 2} + ) self.assertEqual(variables_binds, variables_binds_initial) self.assertEqual(fields, fields_initial) def test__enrich_metric_data_index_3(self): fields = { - "metric_name:sc4snmp.IF-MIB.ifIndex_3": "3", + "metric_name:sc4snmp.IF-MIB.ifIndex": "3", "interface_desc": "eth1", "interface_index": "3", } - fields_initial = {"metric_name:sc4snmp.IF-MIB.ifIndex_3": "3"} + fields_initial = {"metric_name:sc4snmp.IF-MIB.ifIndex": "3"} variables_binds = { - "metric_name": "sc4snmp.IF-MIB.ifIndex_3", + "metric_name": "sc4snmp.IF-MIB.ifIndex", "_value": "3", "metric_type": "Integer", "interface_desc": "eth1", "interface_index": "3", } variables_binds_initial = { - "metric_name": "sc4snmp.IF-MIB.ifIndex_3", + "metric_name": "sc4snmp.IF-MIB.ifIndex", "_value": "3", "metric_type": "Integer", } - _enrich_metric_data(_MibEnricher, variables_binds_initial, fields_initial) + _enrich_metric_data( + _MibEnricher, variables_binds_initial, fields_initial, {"ifIndex": 3} + ) self.assertEqual(variables_binds, variables_binds_initial) self.assertEqual(fields, fields_initial) def test__enrich_event_data_index_1(self): variables_binds = { "metric": '{"metric_name": "sc4snmp.IF-MIB.ifDescr_1", "_value": "lo", ' - '"metric_type": "OctetString"}', + '"metric_type": "OctetString", "parsed_index": {"ifIndex": 1}}', "metric_name": "sc4snmp.IF-MIB.ifDescr_1", "non_metric": 'oid-type1="ObjectIdentity" value1-type="OctetString" ' '1.3.6.1.2.1.2.2.1.2.1="lo" value1="lo" IF-MIB::ifDescr.1="lo" ', } - variables_binds_result = ( + variables_binds_expected = ( 'oid-type1="ObjectIdentity" value1-type="OctetString" 1.3.6.1.2.1.2.2.1.2.1="lo" ' 'value1="lo" IF-MIB::ifDescr.1="lo" interface_index="1" interface_desc="lo" ' ) - variables_binds_processed = _enrich_event_data(_MibEnricher, variables_binds) - self.assertEqual(variables_binds_processed, variables_binds_result) + variables_binds_actual = _enrich_event_data(_MibEnricher, variables_binds) + self.assertEqual(variables_binds_expected, variables_binds_actual) def test__enrich_event_data_index_2(self): variables_binds = { "metric": '{"metric_name": "sc4snmp.IF-MIB.ifDescr_2", "_value": "eth0", ' - '"metric_type": "OctetString"}', + '"metric_type": "OctetString", "parsed_index": {"ifIndex": 2}}', "metric_name": "sc4snmp.IF-MIB.ifDescr_2", "non_metric": 'oid-type1="ObjectIdentity" value1-type="OctetString" ' '1.3.6.1.2.1.2.2.1.2.1="lo" value1="eth0" IF-MIB::ifDescr.2="eth0" ', @@ -157,7 +163,7 @@ def test__enrich_event_data_index_2(self): def test__enrich_event_data_index_3(self): variables_binds = { "metric": '{"metric_name": "sc4snmp.IF-MIB.ifDescr_3", "_value": "eth1", ' - '"metric_type": "OctetString"}', + '"metric_type": "OctetString", "parsed_index": {"ifIndex": 3}}', "metric_name": "sc4snmp.IF-MIB.ifDescr_3", "non_metric": 'oid-type1="ObjectIdentity" value1-type="OctetString" ' '1.3.6.1.2.1.2.2.1.2.1="lo" value1="eth1" IF-MIB::ifDescr.3="eth1" ', diff --git a/tests/test_mib_enricher.py b/tests/test_mib_enricher.py index 6b17078..fee1d93 100644 --- a/tests/test_mib_enricher.py +++ b/tests/test_mib_enricher.py @@ -41,16 +41,16 @@ class TestMibEnricher(TestCase): def test_process_one_none_input_parameter(self): - MibEnricher(mib_static_data_coll).append_additional_dimensions(None) + MibEnricher(mib_static_data_coll).append_additional_dimensions(None, None) def test_process_one_valid_no_if_mib_entry(self): translated_metric = { - "metric_name": "sc4snmp.TCP-MIB::tcpInErrs_0", + "metric_name": "sc4snmp.TCP-MIB::tcpInErrs", "_value": "3", "metric_type": "Counter32", } enricher = MibEnricher(mib_static_data_coll) - enricher.append_additional_dimensions(translated_metric) + enricher.append_additional_dimensions(translated_metric, {"ifIndex": 0}) self.assertTrue(len(translated_metric) == 3) self.assertEqual( {"metric_name", "_value", "metric_type"}, translated_metric.keys() @@ -58,12 +58,12 @@ def test_process_one_valid_no_if_mib_entry(self): def test_process_one_valid_if_mib_entry_iwith_zero_index(self): translated_metric = { - "metric_name": "sc4snmp.IF-MIB::ifNumber.0", + "metric_name": "sc4snmp.IF-MIB::ifNumber", "_value": "2", "metric_type": "Integer", } enricher = MibEnricher(mib_static_data_coll) - enricher.append_additional_dimensions(translated_metric) + enricher.append_additional_dimensions(translated_metric, {"ifIndex": 0}) self.assertTrue(len(translated_metric) == 3) self.assertEqual( {"metric_name", "_value", "metric_type"}, translated_metric.keys() @@ -71,12 +71,12 @@ def test_process_one_valid_if_mib_entry_iwith_zero_index(self): def test_process_one_valid_if_mib_entry_without_proper_mongo_static_data(self): translated_metric = { - "metric_name": "sc4snmp.IF-MIB.ifIndex_2", + "metric_name": "sc4snmp.IF-MIB.ifIndex", "_value": "2", "metric_type": "Integer", } enricher = MibEnricher(None) - enricher.append_additional_dimensions(translated_metric) + enricher.append_additional_dimensions(translated_metric, {"ifIndex": 2}) self.assertTrue(len(translated_metric) == 3) self.assertEqual( {"metric_name", "_value", "metric_type"}, translated_metric.keys() @@ -84,12 +84,12 @@ def test_process_one_valid_if_mib_entry_without_proper_mongo_static_data(self): def test_process_one_valid_if_mib_entry(self): translated_metric = { - "metric_name": "sc4snmp.IF-MIB.ifIndex_2", + "metric_name": "sc4snmp.IF-MIB.ifIndex", "_value": "2", "metric_type": "Integer", } enricher = MibEnricher(mib_static_data_coll) - enricher.append_additional_dimensions(translated_metric) + enricher.append_additional_dimensions(translated_metric, {"ifIndex": 2}) self.assertTrue("interface_index" in translated_metric) self.assertTrue("interface_desc" in translated_metric) self.assertFalse("index_num" in translated_metric) @@ -97,23 +97,23 @@ def test_process_one_valid_if_mib_entry(self): def test_process_one_valid_snmpv2_mib_entry(self): translated_metric = { "_value": "2", - "metric_name": "sc4snmp.SNMPv2-MIB.sysORUpTime_2", + "metric_name": "sc4snmp.SNMPv2-MIB.sysORUpTime", "metric_type": "TimeStamp", } enricher = MibEnricher(mib_static_data_coll) - enricher.append_additional_dimensions(translated_metric) + enricher.append_additional_dimensions(translated_metric, {"ifIndex": 2}) self.assertFalse("interface_index" in translated_metric) self.assertFalse("interface_desc" in translated_metric) self.assertTrue("index_num" in translated_metric) def test_additional_variable(self): translated_metric = { - "metric_name": "sc4snmp.IF-MIB.ifIndex_2", + "metric_name": "sc4snmp.IF-MIB.ifIndex", "_value": "2", "metric_type": "Integer", } enricher = MibEnricher(mib_static_data_coll_additional) - enricher.append_additional_dimensions(translated_metric) + enricher.append_additional_dimensions(translated_metric, {"ifIndex": 2}) self.assertTrue("interface_index" in translated_metric) self.assertTrue("interface_desc" in translated_metric) self.assertTrue("index_num" in translated_metric)