From ef8cddcf0a68228c5ad44b174f6a72b853f02271 Mon Sep 17 00:00:00 2001 From: sgoral-splunk <138458044+sgoral-splunk@users.noreply.github.com> Date: Tue, 30 Jan 2024 11:42:28 +0100 Subject: [PATCH 1/5] chore: flake8 pre commit (#345) Add flake8 to pre-commit checks --- .pre-commit-config.yaml | 4 ++ tests/integration/_search.py | 5 +- .../bin/solnlib_demo_collector.py | 2 +- tests/integration/test__kvstore.py | 5 +- tests/integration/test_acl.py | 5 +- tests/integration/test_conf_manager.py | 7 ++- tests/integration/test_credentials.py | 6 +-- tests/integration/test_hec_config.py | 6 +-- tests/integration/test_hec_event_writer.py | 5 +- tests/integration/test_logger.py | 4 +- tests/integration/test_splunkenv.py | 2 +- tests/integration/test_time_parser.py | 6 +-- tests/integration/test_user_access.py | 6 +-- tests/unit/test_acl.py | 38 ++++++++++++--- tests/unit/test_modular_input.py | 41 +++++++++++----- tests/unit/test_modular_input_event.py | 48 ++++++++++++++----- tests/unit/test_modular_input_event_writer.py | 12 +++-- tests/unit/test_server_info.py | 8 +++- tests/unit/test_user_access.py | 8 ++-- 19 files changed, 149 insertions(+), 69 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c9acf0b1..3224e995 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -13,3 +13,7 @@ repos: hooks: - id: docformatter args: [--in-place] +- repo: https://github.com/PyCQA/flake8 + rev: 5.0.4 + hooks: + - id: flake8 diff --git a/tests/integration/_search.py b/tests/integration/_search.py index ac6c88f4..9e9abe63 100644 --- a/tests/integration/_search.py +++ b/tests/integration/_search.py @@ -13,15 +13,16 @@ # See the License for the specific language governing permissions and # limitations under the License. # +import context import os.path as op import sys import time -sys.path.insert(0, op.dirname(op.dirname(op.abspath(__file__)))) -import context from splunklib import client from splunklib import results as splunklib_results +sys.path.insert(0, op.dirname(op.dirname(op.abspath(__file__)))) + def search(session_key, query): service = client.connect(host=context.host, token=session_key) diff --git a/tests/integration/data/solnlib_demo/bin/solnlib_demo_collector.py b/tests/integration/data/solnlib_demo/bin/solnlib_demo_collector.py index fbe00127..ff61ca72 100644 --- a/tests/integration/data/solnlib_demo/bin/solnlib_demo_collector.py +++ b/tests/integration/data/solnlib_demo/bin/solnlib_demo_collector.py @@ -2,7 +2,7 @@ import uuid from solnlib import log -from solnlib.modular_input import * +from solnlib.modular_input import ModularInput, Argument # Set log context log.Logs.set_context(namespace="solnlib_demo", root_logger_log_file="collector") diff --git a/tests/integration/test__kvstore.py b/tests/integration/test__kvstore.py index 540fc343..da93a34d 100644 --- a/tests/integration/test__kvstore.py +++ b/tests/integration/test__kvstore.py @@ -14,6 +14,7 @@ # limitations under the License. # +import context import json import os.path as op import sys @@ -22,11 +23,11 @@ import pytest -sys.path.insert(0, op.dirname(op.dirname(op.abspath(__file__)))) -import context from splunklib import binding, client from splunklib.binding import HTTPError +sys.path.insert(0, op.dirname(op.dirname(op.abspath(__file__)))) + def test_kvstore(): session_key = context.get_session_key() diff --git a/tests/integration/test_acl.py b/tests/integration/test_acl.py index 3ba29ea8..5acfc43f 100644 --- a/tests/integration/test_acl.py +++ b/tests/integration/test_acl.py @@ -14,13 +14,12 @@ # limitations under the License. # +import context import os.path as op import sys +from solnlib import acl sys.path.insert(0, op.dirname(op.dirname(op.abspath(__file__)))) -import context - -from solnlib import acl def test_acl_manager(): diff --git a/tests/integration/test_conf_manager.py b/tests/integration/test_conf_manager.py index ad116e45..ae28fe7e 100644 --- a/tests/integration/test_conf_manager.py +++ b/tests/integration/test_conf_manager.py @@ -14,16 +14,15 @@ # limitations under the License. # +import context import os.path as op import sys +import pytest +from solnlib import conf_manager from unittest import mock -import pytest sys.path.insert(0, op.dirname(op.dirname(op.abspath(__file__)))) -import context - -from solnlib import conf_manager def _build_conf_manager(session_key: str) -> conf_manager.ConfManager: diff --git a/tests/integration/test_credentials.py b/tests/integration/test_credentials.py index 09de4a02..e6999ec0 100644 --- a/tests/integration/test_credentials.py +++ b/tests/integration/test_credentials.py @@ -14,16 +14,14 @@ # limitations under the License. # +import context import os.path as op import sys from typing import Optional - import pytest +from solnlib import credentials sys.path.insert(0, op.dirname(op.dirname(op.abspath(__file__)))) -import context - -from solnlib import credentials def _build_credential_manager( diff --git a/tests/integration/test_hec_config.py b/tests/integration/test_hec_config.py index 001fc796..38159e60 100644 --- a/tests/integration/test_hec_config.py +++ b/tests/integration/test_hec_config.py @@ -14,14 +14,14 @@ # limitations under the License. # +import context import os.path as op import sys -sys.path.insert(0, op.dirname(op.dirname(op.abspath(__file__)))) -import context - from solnlib import hec_config +sys.path.insert(0, op.dirname(op.dirname(op.abspath(__file__)))) + def test_hec_config(): session_key = context.get_session_key() diff --git a/tests/integration/test_hec_event_writer.py b/tests/integration/test_hec_event_writer.py index 6530095b..3a19ef74 100644 --- a/tests/integration/test_hec_event_writer.py +++ b/tests/integration/test_hec_event_writer.py @@ -13,16 +13,17 @@ # See the License for the specific language governing permissions and # limitations under the License. # +import context import os.path as op import sys import time -sys.path.insert(0, op.dirname(op.dirname(op.abspath(__file__)))) -import context from _search import search from solnlib.modular_input import event_writer as hew +sys.path.insert(0, op.dirname(op.dirname(op.abspath(__file__)))) + def test_hec_event_writer(): session_key = context.get_session_key() diff --git a/tests/integration/test_logger.py b/tests/integration/test_logger.py index 4829dd00..cd834a72 100644 --- a/tests/integration/test_logger.py +++ b/tests/integration/test_logger.py @@ -13,13 +13,13 @@ # See the License for the specific language governing permissions and # limitations under the License. # +import context import os.path as op import sys import time +from _search import search sys.path.insert(0, op.dirname(op.dirname(op.abspath(__file__)))) -import context -from _search import search def test_CVE_2023_32712(): diff --git a/tests/integration/test_splunkenv.py b/tests/integration/test_splunkenv.py index a53b1afb..d9a0c47f 100644 --- a/tests/integration/test_splunkenv.py +++ b/tests/integration/test_splunkenv.py @@ -17,9 +17,9 @@ import os import os.path as op import sys +from solnlib import splunkenv sys.path.insert(0, op.dirname(op.dirname(op.abspath(__file__)))) -from solnlib import splunkenv def test_splunkenv(): diff --git a/tests/integration/test_time_parser.py b/tests/integration/test_time_parser.py index c696337c..e21b0a67 100644 --- a/tests/integration/test_time_parser.py +++ b/tests/integration/test_time_parser.py @@ -14,16 +14,14 @@ # limitations under the License. # +import context import datetime import os.path as op import sys - import pytest +from solnlib import time_parser sys.path.insert(0, op.dirname(op.dirname(op.abspath(__file__)))) -import context - -from solnlib import time_parser def test_time_parser(): diff --git a/tests/integration/test_user_access.py b/tests/integration/test_user_access.py index 2ba6272b..62a8cfe0 100644 --- a/tests/integration/test_user_access.py +++ b/tests/integration/test_user_access.py @@ -14,16 +14,16 @@ # limitations under the License. # +import context import os.path as op import sys import pytest -sys.path.insert(0, op.dirname(op.dirname(op.abspath(__file__)))) -import context - from solnlib import user_access +sys.path.insert(0, op.dirname(op.dirname(op.abspath(__file__)))) + def test_object_acl_manager(): session_key = context.get_session_key() diff --git a/tests/unit/test_acl.py b/tests/unit/test_acl.py index 1aac9527..95490533 100644 --- a/tests/unit/test_acl.py +++ b/tests/unit/test_acl.py @@ -22,13 +22,37 @@ from solnlib import acl -_old_acl = '{"entry": [{"author": "nobody", "name": "transforms", "acl": {"sharing": "global", "perms": {"read": ["*"], "write": ["*"]}, "app": "unittest", "modifiable": true, "owner": "nobody", "can_change_perms": true, "can_share_global": true, "can_list": true, "can_share_user": false, "can_share_app": true, "removable": false, "can_write": true}}]}' - -_new_acl1 = '{"entry": [{"author": "nobody", "name": "transforms", "acl": {"sharing": "global", "perms": {"read": ["admin"], "write": ["admin"]}, "app": "unittest", "modifiable": true, "owner": "nobody", "can_change_perms": true, "can_share_global": true, "can_list": true, "can_share_user": false, "can_share_app": true, "removable": false, "can_write": true}}]}' - -_new_acl2 = '{"entry": [{"author": "nobody", "name": "transforms", "acl": {"sharing": "global", "perms": {"read": ["admin"], "write": ["*"]}, "app": "unittest", "modifiable": true, "owner": "nobody", "can_change_perms": true, "can_share_global": true, "can_list": true, "can_share_user": false, "can_share_app": true, "removable": false, "can_write": true}}]}' - -_new_acl3 = '{"entry": [{"author": "nobody", "name": "transforms", "acl": {"sharing": "global", "perms": {"read": ["*"], "write": ["admin"]}, "app": "unittest", "modifiable": true, "owner": "nobody", "can_change_perms": true, "can_share_global": true, "can_list": true, "can_share_user": false, "can_share_app": true, "removable": false, "can_write": true}}]}' +_old_acl = ( + '{"entry": [{"author": "nobody", "name": "transforms", ' + '"acl": {"sharing": "global", "perms": {"read": ["*"], "write": ["*"]}, ' + '"app": "unittest", "modifiable": true, "owner": "nobody", "can_change_perms": true, ' + '"can_share_global": true, "can_list": true, "can_share_user": false, "can_share_app": true, ' + '"removable": false, "can_write": true}}]}' +) + +_new_acl1 = ( + '{"entry": [{"author": "nobody", "name": "transforms", ' + '"acl": {"sharing": "global", "perms": {"read": ["admin"], "write": ["admin"]}, ' + '"app": "unittest", "modifiable": true, "owner": "nobody", "can_change_perms": true, ' + '"can_share_global": true, "can_list": true, "can_share_user": false, "can_share_app": true, ' + '"removable": false, "can_write": true}}]}' +) + +_new_acl2 = ( + '{"entry": [{"author": "nobody", "name": "transforms", ' + '"acl": {"sharing": "global", "perms": {"read": ["admin"], "write": ["*"]}, ' + '"app": "unittest", "modifiable": true, "owner": "nobody", "can_change_perms": true, ' + '"can_share_global": true, "can_list": true, "can_share_user": false, "can_share_app": true, ' + '"removable": false, "can_write": true}}]}' +) + +_new_acl3 = ( + '{"entry": [{"author": "nobody", "name": "transforms", ' + '"acl": {"sharing": "global", "perms": {"read": ["*"], "write": ["admin"]}, ' + '"app": "unittest", "modifiable": true, "owner": "nobody", "can_change_perms": true, ' + '"can_share_global": true, "can_list": true, "can_share_user": false, "can_share_app": true, ' + '"removable": false, "can_write": true}}]}' +) def _mock_get(self, path_segment, owner=None, app=None, sharing=None, **query): diff --git a/tests/unit/test_modular_input.py b/tests/unit/test_modular_input.py index 3eb29736..1af95a6d 100644 --- a/tests/unit/test_modular_input.py +++ b/tests/unit/test_modular_input.py @@ -149,13 +149,22 @@ def flush(self): md.execute() assert ( sys.stdout.read() - == 'unittest app collectorUnittest app collectortruefalsexmlUnittest app collector statestringfalsetrue' + == "unittest app collectorUnittest app collector" + "truefalse" + 'xml' + "Unittest app collector statestring" + "falsetrue" + "" ) sys.argv = [None, "--validate-arguments"] - validate_arugments_input = 'lli-mbpr.localhttps://127.0.0.1:8089{session_key}{checkpoint_dir}success'.format( - session_key=common.SESSION_KEY, checkpoint_dir=checkpoint_dir - ) + validate_arugments_input = ( + "lli-mbpr.local" + "https://127.0.0.1:8089" + "{session_key}" + '{checkpoint_dir}' + 'success' + ).format(session_key=common.SESSION_KEY, checkpoint_dir=checkpoint_dir) with open(".validate-arguments.xml", "w") as fp: fp.write(validate_arugments_input) mock_stdin = open(".validate-arguments.xml", "rb") @@ -165,9 +174,13 @@ def flush(self): os.remove(".validate-arguments.xml") sys.argv = [None, "--validate-arguments"] - validate_arugments_input = 'lli-mbpr.localhttps://127.0.0.1:8089{session_key}{checkpoint_dir}fail'.format( - session_key=common.SESSION_KEY, checkpoint_dir=checkpoint_dir - ) + validate_arugments_input = ( + "lli-mbpr.local" + "https://127.0.0.1:8089" + "{session_key}" + "{checkpoint_dir}" + 'fail' + ).format(session_key=common.SESSION_KEY, checkpoint_dir=checkpoint_dir) with open(".validate-arguments.xml", "w") as fp: fp.write(validate_arugments_input) mock_stdin = open(".validate-arguments.xml", "rb") @@ -178,9 +191,12 @@ def flush(self): os.remove(".validate-arguments.xml") sys.argv = [None] - run_input = 'lli-mbpr.localhttps://127.0.0.1:8089{session_key}{checkpoint_dir}success'.format( - session_key=common.SESSION_KEY, checkpoint_dir=checkpoint_dir - ) + run_input = ( + "lli-mbpr.localhttps://127.0.0.1:8089" + "{session_key}{checkpoint_dir}" + 'success' + "" + ).format(session_key=common.SESSION_KEY, checkpoint_dir=checkpoint_dir) with open(".run.xml", "w") as fp: fp.write(run_input) mock_stdin = open(".run.xml", "rb") @@ -188,7 +204,10 @@ def flush(self): md.execute() assert ( sys.stdout.read() - == 'unittestappunittestapp{"id": 12345, "time": 1461394857.301}unittestappunittestapp12345' + == "unittestapp" + 'unittestapp{"id": 12345, "time": 1461394857.301}' + "unittestappunittestapp" + "12345" ) mock_stdin.close() os.remove(".run.xml") diff --git a/tests/unit/test_modular_input_event.py b/tests/unit/test_modular_input_event.py index d4c22c90..b08f1d41 100644 --- a/tests/unit/test_modular_input_event.py +++ b/tests/unit/test_modular_input_event.py @@ -73,26 +73,41 @@ def setup_class(cls): def test_str(self, monkeypatch): assert ( to_sorted_json_string(self.xe1) - == '{"data": "This is a test data1.", "done": false, "host": "localhost", "index": "main", "source": "Splunk", "sourcetype": "misc", "stanza": "test_scheme://test", "time": 1372274622.493, "unbroken": true}' + == '{"data": "This is a test data1.", "done": false, "host": "localhost", "index": "main", ' + '"source": "Splunk", "sourcetype": "misc", "stanza": "test_scheme://test", ' + '"time": 1372274622.493, "unbroken": true}' ) assert ( to_sorted_json_string(self.xe2) - == '{"data": "This is a test data2.", "done": true, "host": "localhost", "index": "main", "source": "Splunk", "sourcetype": "misc", "stanza": "test_scheme://test", "time": 1372274622.493, "unbroken": true}' + == '{"data": "This is a test data2.", "done": true, "host": "localhost", "index": "main", ' + '"source": "Splunk", "sourcetype": "misc", "stanza": "test_scheme://test", ' + '"time": 1372274622.493, "unbroken": true}' ) assert ( to_sorted_json_string(self.xe3) - == '{"data": "This is a test data3.", "done": false, "host": "localhost", "index": "main", "source": "Splunk", "sourcetype": "misc", "stanza": "test_scheme://test", "time": 1372274622.493, "unbroken": false}' + == '{"data": "This is a test data3.", "done": false, "host": "localhost", "index": "main", ' + '"source": "Splunk", "sourcetype": "misc", "stanza": "test_scheme://test", ' + '"time": 1372274622.493, "unbroken": false}' ) def test_format_events(self, monkeypatch): assert XMLEvent.format_events([self.xe1, self.xe2]) == [ - 'mainlocalhostSplunkmiscThis is a test data1.mainlocalhostSplunkmiscThis is a test data2.' + '' + "mainlocalhostSplunk" + "miscThis is a test data1." + '' + "mainlocalhostSplunkmisc" + "This is a test data2." ] assert XMLEvent.format_events([self.xe3]) == [ - 'mainlocalhostSplunkmiscThis is a test data3.' + '' + "mainlocalhostSplunkmisc" + "This is a test data3." ] assert XMLEvent.format_events([self.xe4]) == [ - 'mainlocalhostSplunkmiscThis is utf-8 \u2603 data4.' + '' + "mainlocalhostSplunkmisc" + "This is utf-8 \u2603 data4." ] @@ -136,15 +151,21 @@ def setup_class(cls): def test_str(self, monkeypatch): assert ( to_sorted_json_string(self.he1) - == '{"data": "This is a test data1.", "done": false, "host": "localhost", "index": "main", "source": "Splunk", "sourcetype": "misc", "stanza": "test_scheme://test", "time": 1372274622.493, "unbroken": true}' + == '{"data": "This is a test data1.", "done": false, "host": "localhost", "index": "main", ' + '"source": "Splunk", "sourcetype": "misc", "stanza": "test_scheme://test", ' + '"time": 1372274622.493, "unbroken": true}' ) assert ( to_sorted_json_string(self.he2) - == '{"data": "This is a test data2.", "done": true, "host": "localhost", "index": "main", "source": "Splunk", "sourcetype": "misc", "stanza": "test_scheme://test", "time": 1372274622.493, "unbroken": true}' + == '{"data": "This is a test data2.", "done": true, "host": "localhost", "index": "main", ' + '"source": "Splunk", "sourcetype": "misc", "stanza": "test_scheme://test", ' + '"time": 1372274622.493, "unbroken": true}' ) assert ( to_sorted_json_string(self.he3) - == '{"data": "This is a test data3.", "done": false, "host": "localhost", "index": "main", "source": "Splunk", "sourcetype": "misc", "stanza": "test_scheme://test", "time": 1372274622.493, "unbroken": false}' + == '{"data": "This is a test data3.", "done": false, "host": "localhost", "index": "main", ' + '"source": "Splunk", "sourcetype": "misc", "stanza": "test_scheme://test", ' + '"time": 1372274622.493, "unbroken": false}' ) def test_format_events(self, monkeypatch): @@ -157,11 +178,13 @@ def test_format_events(self, monkeypatch): assert len(event_strings) == 2 assert ( event_strings[0] - == '{"event": "This is a test data1.", "host": "localhost", "index": "main", "source": "Splunk", "sourcetype": "misc", "time": 1372274622.493}' + == '{"event": "This is a test data1.", "host": "localhost", "index": "main", ' + '"source": "Splunk", "sourcetype": "misc", "time": 1372274622.493}' ) assert ( event_strings[1] - == '{"event": "This is a test data2.", "host": "localhost", "index": "main", "source": "Splunk", "sourcetype": "misc", "time": 1372274622.493}' + == '{"event": "This is a test data2.", "host": "localhost", "index": "main", ' + '"source": "Splunk", "sourcetype": "misc", "time": 1372274622.493}' ) formatted_events = HECEvent.format_events([self.he3]) @@ -173,5 +196,6 @@ def test_format_events(self, monkeypatch): assert len(event_strings) == 1 assert ( event_strings[0] - == '{"event": "This is a test data3.", "host": "localhost", "index": "main", "source": "Splunk", "sourcetype": "misc", "time": 1372274622.493}' + == '{"event": "This is a test data3.", "host": "localhost", "index": "main", ' + '"source": "Splunk", "sourcetype": "misc", "time": 1372274622.493}' ) diff --git a/tests/unit/test_modular_input_event_writer.py b/tests/unit/test_modular_input_event_writer.py index e0e8cf87..1bfcdabf 100644 --- a/tests/unit/test_modular_input_event_writer.py +++ b/tests/unit/test_modular_input_event_writer.py @@ -76,7 +76,11 @@ def flush(self): assert ( mock_stdout.read() - == 'mainlocalhostSplunkmiscThis is a test data1.mainlocalhostSplunkmiscThis is a test data2.' + == 'main' + "localhostSplunkmisc" + 'This is a test data1.mainlocalhostSplunkmisc" + "This is a test data2." ) assert mock_stdout.write_count == 1 @@ -102,11 +106,13 @@ def mock_post( assert ( event_strings[0] - == '{"event": "This is a test data1.", "host": "localhost", "index": "main", "source": "Splunk", "sourcetype": "misc", "time": 1372274622.493}' + == '{"event": "This is a test data1.", "host": "localhost", "index": "main", ' + '"source": "Splunk", "sourcetype": "misc", "time": 1372274622.493}' ) assert ( event_strings[1] - == '{"event": "This is a test data2.", "host": "localhost", "index": "main", "source": "Splunk", "sourcetype": "misc", "time": 1372274622.493}' + == '{"event": "This is a test data2.", "host": "localhost", "index": "main", ' + '"source": "Splunk", "sourcetype": "misc", "time": 1372274622.493}' ) def mock_get_hec_config( diff --git a/tests/unit/test_server_info.py b/tests/unit/test_server_info.py index 0b0d13c3..f312baad 100644 --- a/tests/unit/test_server_info.py +++ b/tests/unit/test_server_info.py @@ -35,7 +35,13 @@ class TestServerInfo: def test_get_shc_members(self, monkeypatch): def _mock_get(self, path_segment, owner=None, app=None, sharing=None, **query): return common.make_response_record( - '{"entry": [{"name": "5B4A53C7-B824-4103-B8CC-C22E1EC6480F", "content": {"peer_scheme_host_port": "https://192.168.1.85:8089", "label": "SHC01_SearchHead02_1_85"}}, {"name": "D7E3BA03-85CE-449A-9736-38F2DA58236B", "content": {"peer_scheme_host_port": "https://192.168.1.86:8089", "label": "SHC01_SearchHead03_1_86"}}, {"name": "DA72938A-72C4-46F3-86BE-2E200EC56C76", "content": {"peer_scheme_host_port": "https://192.168.1.84:8089", "label": "SHC01_SearchHead01_1_84"}}]}' + '{"entry": [{"name": "5B4A53C7-B824-4103-B8CC-C22E1EC6480F", ' + '"content": ' + '{"peer_scheme_host_port": "https://192.168.1.85:8089", "label": "SHC01_SearchHead02_1_85"}}, ' + '{"name": "D7E3BA03-85CE-449A-9736-38F2DA58236B", "content": ' + '{"peer_scheme_host_port": "https://192.168.1.86:8089", "label": "SHC01_SearchHead03_1_86"}}, ' + '{"name": "DA72938A-72C4-46F3-86BE-2E200EC56C76", "content": {"peer_scheme_host_port": ' + '"https://192.168.1.84:8089", "label": "SHC01_SearchHead01_1_84"}}]}' ) common.mock_splunkhome(monkeypatch) diff --git a/tests/unit/test_user_access.py b/tests/unit/test_user_access.py index b5cdbefa..6124421d 100644 --- a/tests/unit/test_user_access.py +++ b/tests/unit/test_user_access.py @@ -52,7 +52,7 @@ def mock_kvstore_collection_data_batch_save(self, *documents): def mock_kvstore_collection_data_query_by_id(self, id): try: return object_acls[id] - except: + except Exception: raise binding.HTTPError(common.make_response_record("", status=404)) def mock_kvstore_collection_data_query(self, **query): @@ -68,7 +68,7 @@ def mock_kvstore_collection_data_query(self, **query): def mock_kvstore_collection_data_delete_by_id(self, id): try: del object_acls[id] - except: + except Exception: raise binding.HTTPError(common.make_response_record("", status=404)) def mock_kvstore_collection_data_delete(self, query=None): @@ -210,13 +210,13 @@ def mock_kvstore_collection_data_batch_save(self, *documents): def mock_kvstore_collection_data_query_by_id(self, id): try: return app_capabilities[id] - except: + except Exception: raise binding.HTTPError(common.make_response_record("", status=404)) def mock_kvstore_collection_data_delete_by_id(self, id): try: del app_capabilities[id] - except: + except Exception: raise binding.HTTPError(None, status=404) common.mock_splunkhome(monkeypatch) From 44e35c4a4bf75f41edbd22bc43c547b740ede3f9 Mon Sep 17 00:00:00 2001 From: sgoral-splunk <138458044+sgoral-splunk@users.noreply.github.com> Date: Tue, 30 Jan 2024 12:21:17 +0100 Subject: [PATCH 2/5] feat: add exception logging function, add tests (#344) --- solnlib/log.py | 23 +++++++++++++++++++++++ tests/unit/test_log.py | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+) diff --git a/solnlib/log.py b/solnlib/log.py index ecb23a5a..0d624c9b 100644 --- a/solnlib/log.py +++ b/solnlib/log.py @@ -19,6 +19,7 @@ import logging import logging.handlers import os.path as op +import traceback from threading import Lock from typing import Dict, Any @@ -263,3 +264,25 @@ def events_ingested( "n_events": n_events, }, ) + + +def log_exception( + logger: logging.Logger, + e: Exception, + full_msg: bool = True, + msg_before: str = None, + msg_after: str = None, + log_level: int = logging.ERROR, +): + """General function to log exceptions.""" + exc_type, exc_value, exc_traceback = type(e), e, e.__traceback__ + if full_msg: + error = traceback.format_exception(exc_type, exc_value, exc_traceback) + else: + error = traceback.format_exception_only(exc_type, exc_value) + + msg_start = msg_before if msg_before is not None else "" + msg_mid = "".join(error) + msg_end = msg_after if msg_after is not None else "" + msg = f"{msg_start}\n{msg_mid}\n{msg_end}" + logger.log(log_level, msg) diff --git a/tests/unit/test_log.py b/tests/unit/test_log.py index 93fa5de2..d19a14f0 100644 --- a/tests/unit/test_log.py +++ b/tests/unit/test_log.py @@ -15,10 +15,12 @@ # import logging +import json import multiprocessing import os import shutil import threading +import traceback import time from unittest import mock @@ -197,3 +199,34 @@ def test_events_ingested(): logging.INFO, "action=events_ingested modular_input_name=modular_input_name sourcetype_ingested=sourcetype n_events=5", ) + + +def test_log_exceptions_full_msg(): + start_msg = "some msg before exception" + with mock.patch("logging.Logger") as mock_logger: + try: + test_jsons = "{'a': 'aa'" + json.loads(test_jsons) + except Exception as e: + log.log_exception(mock_logger, e, msg_before=start_msg) + mock_logger.log.assert_called_with( + logging.ERROR, f"{start_msg}\n{traceback.format_exc()}\n" + ) + + +def test_log_exceptions_partial_msg(): + start_msg = "some msg before exception" + end_msg = "some msg after exception" + with mock.patch("logging.Logger") as mock_logger: + try: + test_jsons = "{'a': 'aa'" + json.loads(test_jsons) + except Exception as e: + log.log_exception( + mock_logger, e, full_msg=False, msg_before=start_msg, msg_after=end_msg + ) + mock_logger.log.assert_called_with( + logging.ERROR, + "some msg before exception\njson.decoder.JSONDecodeError: Expecting property name enclosed in double " + "quotes: line 1 column 2 (char 1)\n\nsome msg after exception", + ) From 5a71f28a74af365f57d4ce9daf0db20cf4bbc763 Mon Sep 17 00:00:00 2001 From: srv-rr-github-token <94607705+srv-rr-github-token@users.noreply.github.com> Date: Thu, 7 Mar 2024 14:39:01 +0000 Subject: [PATCH 3/5] chore(release): 4.13.0-beta.1 # [4.13.0-beta.1](https://github.com/splunk/addonfactory-solutions-library-python/compare/v4.12.0...v4.13.0-beta.1) (2024-03-07) ### Features * add exception logging function, add tests ([#344](https://github.com/splunk/addonfactory-solutions-library-python/issues/344)) ([44e35c4](https://github.com/splunk/addonfactory-solutions-library-python/commit/44e35c4a4bf75f41edbd22bc43c547b740ede3f9)) [ci skip] --- pyproject.toml | 2 +- solnlib/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 18c0c0ed..846524aa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,7 +16,7 @@ [tool.poetry] name = "solnlib" -version = "4.12.0" +version = "4.13.0-beta.1" description = "The Splunk Software Development Kit for Splunk Solutions" authors = ["Splunk "] license = "Apache-2.0" diff --git a/solnlib/__init__.py b/solnlib/__init__.py index 686c127b..02933834 100644 --- a/solnlib/__init__.py +++ b/solnlib/__init__.py @@ -54,4 +54,4 @@ "utils", ] -__version__ = "4.12.0" +__version__ = "4.13.0-beta.1" From fef1d80821bd362610d8c4ebc8b80e198397cfeb Mon Sep 17 00:00:00 2001 From: sgoral-splunk <138458044+sgoral-splunk@users.noreply.github.com> Date: Thu, 7 Mar 2024 15:42:09 +0100 Subject: [PATCH 4/5] feat: HECEventWriter can get scheme from HEC global settings (#351) Jira: https://splunk.atlassian.net/browse/ADDON-68827 Add schema calculations based on global HEC settings for HECEventWriter --------- Co-authored-by: Artem Rys --- solnlib/modular_input/event_writer.py | 32 ++++++++++++++++--- solnlib/modular_input/modular_input.py | 2 ++ solnlib/splunkenv.py | 25 +++++++++++++++ tests/unit/common.py | 7 ++++ .../apps/splunk_httpinput/local/inputs.conf | 3 ++ tests/unit/test_modular_input_event_writer.py | 23 ++++++++++--- 6 files changed, 84 insertions(+), 8 deletions(-) create mode 100644 tests/unit/data/mock_splunk/etc/apps/splunk_httpinput/local/inputs.conf diff --git a/solnlib/modular_input/event_writer.py b/solnlib/modular_input/event_writer.py index 788de75c..3d14a4c3 100644 --- a/solnlib/modular_input/event_writer.py +++ b/solnlib/modular_input/event_writer.py @@ -32,7 +32,7 @@ from .. import splunk_rest_client as rest_client from .. import utils from ..hec_config import HECConfig -from ..splunkenv import get_splunkd_access_info +from ..splunkenv import get_splunkd_access_info, get_scheme_from_hec_settings from ..utils import retry from .event import HECEvent, XMLEvent @@ -203,6 +203,7 @@ def __init__( port: int = None, hec_uri: str = None, hec_token: str = None, + global_settings_schema: bool = True, logger: logging.Logger = None, **context: dict ): @@ -217,6 +218,7 @@ def __init__( hec_uri: (optional) If hec_uri and hec_token are provided, they will higher precedence than hec_input_name. hec_token: (optional) HEC token. + global_settings_schema: (optional) if True, scheme will be set based on HEC global settings, default False. logger: Logger object. context: Other configurations for Splunk rest client. """ @@ -237,6 +239,9 @@ def __init__( hec_input_name, session_key, scheme, host, port, **context ) + if global_settings_schema: + scheme = get_scheme_from_hec_settings() + if not context.get("pool_connections"): context["pool_connections"] = 10 @@ -249,7 +254,10 @@ def __init__( @staticmethod def create_from_token( - hec_uri: str, hec_token: str, **context: dict + hec_uri: str, + hec_token: str, + global_settings_schema: bool = False, + **context: dict ) -> "HECEventWriter": """Given HEC URI and HEC token, create HECEventWriter object. This function simplifies the standalone mode HECEventWriter usage (not in a @@ -258,6 +266,7 @@ def create_from_token( Arguments: hec_uri: HTTP Event Collector URI, like https://localhost:8088. hec_token: HTTP Event Collector token. + global_settings_schema: (optional) if True, scheme will be set based on HEC global settings, default False. context: Other configurations. Returns: @@ -272,12 +281,17 @@ def create_from_token( None, hec_uri=hec_uri, hec_token=hec_token, + global_settings_schema=global_settings_schema, **context ) @staticmethod def create_from_input( - hec_input_name: str, splunkd_uri: str, session_key: str, **context: dict + hec_input_name: str, + splunkd_uri: str, + session_key: str, + global_settings_schema: bool = False, + **context: dict ) -> "HECEventWriter": """Given HEC input stanza name, splunkd URI and splunkd session key, create HECEventWriter object. HEC URI and token etc will be discovered @@ -289,6 +303,7 @@ def create_from_input( hec_input_name: Splunk HEC input name. splunkd_uri: Splunkd URI, like https://localhost:8089 session_key: Splunkd access token. + global_settings_schema: (optional) if True, scheme will be set based on HEC global settings, default False. context: Other configurations. Returns: @@ -297,7 +312,13 @@ def create_from_input( scheme, host, port = utils.extract_http_scheme_host_port(splunkd_uri) return HECEventWriter( - hec_input_name, session_key, scheme, host, port, **context + hec_input_name, + session_key, + scheme, + host, + port, + global_settings_schema=global_settings_schema, + **context ) @staticmethod @@ -306,6 +327,7 @@ def create_from_token_with_session_key( session_key: str, hec_uri: str, hec_token: str, + global_settings_schema: bool = False, **context: dict ) -> "HECEventWriter": """Given Splunkd URI, Splunkd session key, HEC URI and HEC token, @@ -318,6 +340,7 @@ def create_from_token_with_session_key( session_key: Splunkd access token. hec_uri: Http Event Collector URI, like https://localhost:8088. hec_token: Http Event Collector token. + global_settings_schema: (optional) if True, scheme will be set based on HEC global settings, default False. context: Other configurations. Returns: @@ -333,6 +356,7 @@ def create_from_token_with_session_key( port, hec_uri=hec_uri, hec_token=hec_token, + global_settings_schema=global_settings_schema, **context ) diff --git a/solnlib/modular_input/modular_input.py b/solnlib/modular_input/modular_input.py index c93b0365..959fe772 100644 --- a/solnlib/modular_input/modular_input.py +++ b/solnlib/modular_input/modular_input.py @@ -107,6 +107,7 @@ class ModularInput(metaclass=ABCMeta): # Input name of Splunk HEC, must be overridden if use_hec_event_writer # is True hec_input_name = None + hec_global_settings_schema = False def __init__(self): # Validate properties @@ -230,6 +231,7 @@ def _create_event_writer(self): scheme=self.server_scheme, host=self.server_host, port=self.server_port, + global_settings_schema=self.hec_global_settings_schema, ) except binding.HTTPError: logging.error( diff --git a/solnlib/splunkenv.py b/solnlib/splunkenv.py index da98a473..20484072 100644 --- a/solnlib/splunkenv.py +++ b/solnlib/splunkenv.py @@ -31,6 +31,7 @@ "get_splunk_host_info", "get_splunk_bin", "get_splunkd_access_info", + "get_scheme_from_hec_settings", "get_splunkd_uri", "get_conf_key_value", "get_conf_stanza", @@ -198,6 +199,30 @@ def get_splunkd_access_info() -> Tuple[str, str, int]: return scheme, host, port +def get_scheme_from_hec_settings() -> str: + """Get scheme from HEC global settings. + + Returns: + scheme (str) + """ + try: + ssl_enabled = get_conf_key_value("inputs", "http", "enableSSL") + except KeyError: + raise KeyError( + "Cannot get enableSSL setting form conf: 'inputs' and stanza: '[http]'. " + "Verify that your Splunk instance has the inputs.conf file with the correct [http] stanza. " + "For more information see: " + "https://docs.splunk.com/Documentation/Splunk/9.2.0/Data/UseHECusingconffiles" + ) + + if is_true(ssl_enabled): + scheme = "https" + else: + scheme = "http" + + return scheme + + def get_splunkd_uri() -> str: """Get splunkd uri. diff --git a/tests/unit/common.py b/tests/unit/common.py index 30f5e56b..fa79619f 100644 --- a/tests/unit/common.py +++ b/tests/unit/common.py @@ -57,6 +57,13 @@ def communicate(self, input=None): file_path = op.sep.join( [cur_dir, "data/mock_splunk/etc/system/default/server.conf"] ) + elif self._conf == "inputs": + file_path = op.sep.join( + [ + cur_dir, + "data/mock_splunk/etc/apps/splunk_httpinput/local/inputs.conf", + ] + ) else: file_path = op.sep.join( [cur_dir, "data/mock_splunk/etc/system/default/web.conf"] diff --git a/tests/unit/data/mock_splunk/etc/apps/splunk_httpinput/local/inputs.conf b/tests/unit/data/mock_splunk/etc/apps/splunk_httpinput/local/inputs.conf new file mode 100644 index 00000000..6e74c6dc --- /dev/null +++ b/tests/unit/data/mock_splunk/etc/apps/splunk_httpinput/local/inputs.conf @@ -0,0 +1,3 @@ +[http] +disabled = 0 +enableSSL = 0 diff --git a/tests/unit/test_modular_input_event_writer.py b/tests/unit/test_modular_input_event_writer.py index 1bfcdabf..5eddbae8 100644 --- a/tests/unit/test_modular_input_event_writer.py +++ b/tests/unit/test_modular_input_event_writer.py @@ -207,11 +207,21 @@ def mock_post_2( # test that post is called 2 times assert len(times_post_called) == 2 + for i in range(4): + ev = create_hec_event_writer(i) + assert ev._rest_client.scheme == "https" + for i in range(4): + ev = create_hec_event_writer(i, hec=True) + assert ev._rest_client.scheme == "http" -def create_hec_event_writer(i): + +def create_hec_event_writer(i, hec=False): if i == 1: return HECEventWriter.create_from_input( - "HECTestInput", "https://localhost:8089", common.SESSION_KEY + "HECTestInput", + "https://localhost:8089", + common.SESSION_KEY, + global_settings_schema=hec, ) elif i == 2: return HECEventWriter.create_from_token_with_session_key( @@ -219,8 +229,13 @@ def create_hec_event_writer(i): common.SESSION_KEY, "https://localhost:8090", "test_token", + global_settings_schema=hec, ) elif i == 3: - return HECEventWriter.create_from_token("https://localhost:8090", "test_token") + return HECEventWriter.create_from_token( + "https://localhost:8090", "test_token", global_settings_schema=hec + ) else: - return HECEventWriter("HECTestInput", common.SESSION_KEY) + return HECEventWriter( + "HECTestInput", common.SESSION_KEY, global_settings_schema=hec + ) From 79fa69323339f4f3748b258789e158e77fae8260 Mon Sep 17 00:00:00 2001 From: srv-rr-github-token <94607705+srv-rr-github-token@users.noreply.github.com> Date: Thu, 7 Mar 2024 14:46:35 +0000 Subject: [PATCH 5/5] chore(release): 4.13.0-beta.2 # [4.13.0-beta.2](https://github.com/splunk/addonfactory-solutions-library-python/compare/v4.13.0-beta.1...v4.13.0-beta.2) (2024-03-07) ### Features * HECEventWriter can get scheme from HEC global settings ([#351](https://github.com/splunk/addonfactory-solutions-library-python/issues/351)) ([fef1d80](https://github.com/splunk/addonfactory-solutions-library-python/commit/fef1d80821bd362610d8c4ebc8b80e198397cfeb)) [ci skip] --- pyproject.toml | 2 +- solnlib/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 846524aa..6252affc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,7 +16,7 @@ [tool.poetry] name = "solnlib" -version = "4.13.0-beta.1" +version = "4.13.0-beta.2" description = "The Splunk Software Development Kit for Splunk Solutions" authors = ["Splunk "] license = "Apache-2.0" diff --git a/solnlib/__init__.py b/solnlib/__init__.py index 02933834..620cfeda 100644 --- a/solnlib/__init__.py +++ b/solnlib/__init__.py @@ -54,4 +54,4 @@ "utils", ] -__version__ = "4.13.0-beta.1" +__version__ = "4.13.0-beta.2"