refactor: SplunkRestClient initialization

Removed check_css_params function from net_utils and replaced it with validate_scheme_host_port.

.github/workflows/build-test-release.yml

@@ -127,14 +127,6 @@ jobs:
           source "$HOME/.poetry/env"
           poetry install
           poetry build
-      - uses: actions/[email protected]
-        with:
-          name: dist
-          path: dist
-      - uses: actions/[email protected]
-        with:
-          name: output
-          path: output
 
   run-unit-tests:
     name: test-unit
@@ -153,12 +145,12 @@ jobs:
           # shellcheck disable=SC1090
           source "$HOME/.poetry/env"
           poetry install
-          poetry run pytest --junitxml=test-results/results.xml --cov=solnlib --cov-report=html tests
-      - uses: actions/upload-artifact@v2.2.4
+          poetry run pytest --junitxml=test-results/results.xml --cov=solnlib --cov-report=html tests/unit
+      - uses: actions/upload-artifact@v2
         with:
           name: unit tests test-results
           path: test-results
-      - uses: actions/upload-artifact@v2.2.4
+      - uses: actions/upload-artifact@v2
         with:
           name: unit tests htmlcov report
           path: htmlcov
@@ -195,7 +187,7 @@ jobs:
           export SPLUNK_HOME=/opt/splunk
           wget -qO /tmp/splunk.tgz "${SPLUNK_BUILD_URL}"
           sudo tar -C /opt -zxf /tmp/splunk.tgz
-          sudo cp -r examples/data/solnlib_demo $SPLUNK_HOME/etc/apps
+          sudo cp -r tests/integration/data/solnlib_demo $SPLUNK_HOME/etc/apps
           sudo cp -r solnlib $SPLUNK_HOME/etc/apps/solnlib_demo/bin/
           sudo mkdir -p $SPLUNK_HOME/etc/apps/Splunk_TA_test/default/
           sudo chown -R "$USER":"$USER" /opt/splunk
@@ -211,7 +203,7 @@ jobs:
           # shellcheck disable=SC1090
           source "$HOME/.poetry/env"
           poetry install
-          SPLUNK_HOME=/opt/splunk/ poetry run pytest --junitxml=test-results/results.xml -v examples
+          SPLUNK_HOME=/opt/splunk/ poetry run pytest --junitxml=test-results/results.xml -v tests/integration
       - uses: actions/[email protected]
         with:
           name: test-splunk test-results

.licenserc.yaml

@@ -25,7 +25,6 @@ header:
     - "*.lock"
     - "tests/**"
     - ".*"
-    - "examples/data"
     - "mkdocs.yml"
     - "docs/"
 

.semgrepignore

@@ -5,5 +5,4 @@
 :include .gitignore
 
 tests/
-examples/
 .github/

solnlib/credentials.py

@@ -22,12 +22,7 @@
 from splunklib import binding
 
 from . import splunk_rest_client as rest_client
-from .net_utils import (
-    check_css_params,
-    is_valid_hostname,
-    is_valid_port,
-    is_valid_scheme,
-)
+from .net_utils import validate_scheme_host_port
 from .splunkenv import get_splunkd_access_info
 from .utils import retry
 
@@ -294,7 +289,6 @@ def _get_all_passwords(self):
 
 
 @retry(exceptions=[binding.HTTPError])
-@check_css_params(scheme=is_valid_scheme, host=is_valid_hostname, port=is_valid_port)
 def get_session_key(
     username: str,
     password: str,
@@ -318,10 +312,12 @@ def get_session_key(
 
     Raises:
         CredentialException: If username/password are invalid.
+        ValueError: if scheme, host or port are invalid.
 
     Examples:
        >>> credentials.get_session_key('user', 'password')
     """
+    validate_scheme_host_port(scheme, host, port)
 
     if any([scheme is None, host is None, port is None]):
         scheme, host, port = get_splunkd_access_info()

solnlib/net_utils.py

@@ -15,12 +15,10 @@
 #
 
 """Net utilities."""
-import inspect
 import re
 import socket
-from functools import wraps
 
-__all__ = ["resolve_hostname"]
+__all__ = ["resolve_hostname", "validate_scheme_host_port"]
 
 from typing import Optional, Union
 
@@ -134,35 +132,20 @@ def is_valid_scheme(scheme: str) -> bool:
     return scheme.lower() in ("http", "https")
 
 
-def check_css_params(**validators):
-    """A decorator for validating arguments for function with specified
-    validating function which returns True or False.
+def validate_scheme_host_port(scheme: str, host: str, port: Union[str, int]):
+    """Validates scheme, host and port.
 
     Arguments:
-        validators: argument and it's validation function.
+        scheme: scheme to validate.
+        host: hostname to validate.
+        port: port to validate.
 
     Raises:
-        ValueError: If validation fails.
+        ValueError: if scheme, host or port are invalid.
     """
-
-    def decorator(f):
-        @wraps(f)
-        def wrapper(*args, **kwargs):
-            arg_spec = inspect.getfullargspec(f)
-            actual_args = dict(list(zip(arg_spec.args, args)) + list(kwargs.items()))
-            dfs = arg_spec.defaults
-            optional = dict(list(zip(arg_spec.args[-len(dfs) :], dfs))) if dfs else {}
-
-            for arg, func in list(validators.items()):
-                if arg not in actual_args:
-                    continue
-                value = actual_args[arg]
-                if arg in optional and optional[arg] == value:
-                    continue
-                if not func(value):
-                    raise ValueError(f"Illegal argument: {arg}={value}")
-            return f(*args, **kwargs)
-
-        return wrapper
-
-    return decorator
+    if scheme is not None and not is_valid_scheme(scheme):
+        raise ValueError("Invalid scheme")
+    if host is not None and not is_valid_hostname(host):
+        raise ValueError("Invalid host")
+    if port is not None and not is_valid_port(port):
+        raise ValueError("Invalid port")

solnlib/splunk_rest_client.py

@@ -29,12 +29,7 @@
 
 from splunklib import binding, client
 
-from .net_utils import (
-    check_css_params,
-    is_valid_hostname,
-    is_valid_port,
-    is_valid_scheme,
-)
+from .net_utils import validate_scheme_host_port
 from .splunkenv import get_splunkd_access_info
 
 __all__ = ["SplunkRestClient"]
@@ -151,7 +146,7 @@ def request(url, message, **kwargs):
                 cert=cert,
                 **kwargs,
             )
-        except Exception as e:
+        except Exception:
             logging.error(
                 "Failed to issue http request=%s to url=%s, error=%s",
                 method,
@@ -171,15 +166,8 @@ def request(url, message, **kwargs):
 
 
 class SplunkRestClient(client.Service):
-    """Splunk REST client.
+    """Splunk REST client."""
 
-    If any of scheme, host and port is None, will discover local splunkd
-    access info automatically.
-    """
-
-    @check_css_params(
-        scheme=is_valid_scheme, host=is_valid_hostname, port=is_valid_port
-    )
     def __init__(
         self,
         session_key: str,
@@ -192,24 +180,40 @@ def __init__(
     ):
         """Initializes SplunkRestClient.
 
+        Arguments `scheme`, `host` and `port` are optional in the Splunk
+        environment (when environment variable SPLUNK_HOME is set). In this
+        situation `get_splunkd_access_info` will be used to set `scheme`,
+        `host` and `port`. In case of using `SplunkRestClient` outside of
+        Splunk environment - `scheme`, `host` and `port` should be provided.
+
         Arguments:
             session_key: Splunk access token.
             app: App name of namespace.
             owner: Owner of namespace, default is `nobody`.
             scheme: The access scheme, default is None.
             host: The host name, default is None.
             port: The port number, default is None.
-            context: Other configurations, it can contains `proxy_hostname`,
+            context: Other configurations, it can contain `proxy_hostname`,
                 `proxy_port`, `proxy_username`, `proxy_password`, then proxy will
-                be accounted and setup, all REST APIs to Splunkd will be through
+                be accounted and setup, all REST APIs to splunkd will be through
                 the proxy. If `context` contains `key_file`, `cert_file`, then
-                certification will be accounted and setup, all REST APIs to Splunkd
+                certification will be accounted and setup, all REST APIs to splunkd
                 will use certification. If `context` contains `pool_connections`,
-                `pool_maxsize`, then HTTP Connection will be pooled.
+                `pool_maxsize`, then HTTP connection will be pooled.
+
+        Raises:
+            ValueError: if scheme, host or port are invalid.
         """
-        # Only do splunkd URI discovery in SPLUNK env (SPLUNK_HOME is set)
+        # Only do splunkd URI discovery in SPLUNK env (SPLUNK_HOME is set).
         if not all([scheme, host, port]) and os.environ.get("SPLUNK_HOME"):
             scheme, host, port = get_splunkd_access_info()
+        if os.environ.get("SPLUNK_HOME") is None:
+            if not all([scheme, host, port]):
+                raise ValueError(
+                    "scheme, host, port should be provided outside of Splunk environment"
+                )
+
+        validate_scheme_host_port(scheme, host, port)
 
         handler = _request_handler(context)
         super().__init__(

solnlib/splunkenv.py

@@ -172,14 +172,14 @@ def get_splunk_bin() -> str:
     return make_splunkhome_path(("bin", splunk_bin))
 
 
-def get_splunkd_access_info() -> Tuple:
+def get_splunkd_access_info() -> Tuple[str, str, int]:
     """Get splunkd server access info.
 
     Returns:
         Tuple of (scheme, host, port).
     """
 
-    if utils.is_true(get_conf_key_value("server", "sslConfig", "enableSplunkdSSL")):
+    if get_conf_key_value("server", "sslConfig", "enableSplunkdSSL") == "true":
         scheme = "https"
     else:
         scheme = "http"
@@ -194,7 +194,7 @@ def get_splunkd_access_info() -> Tuple:
         port_idx = bindip.rfind(":")
         host = bindip[:port_idx] if port_idx > 0 else bindip
 
-    return (scheme, host, port)
+    return scheme, host, port
 
 
 def get_splunkd_uri() -> str:
@@ -211,7 +211,7 @@ def get_splunkd_uri() -> str:
     return f"{scheme}://{host}:{port}"
 
 
-def get_conf_key_value(conf_name: str, stanza: str, key: str) -> Tuple[str, List, dict]:
+def get_conf_key_value(conf_name: str, stanza: str, key: str) -> Union[str, List, dict]:
     """Get value of `key` of `stanza` in `conf_name`.
 
     Arguments:

solnlib/timer_queue.py

@@ -199,11 +199,12 @@ class TimerQueue:
     that the timers are just a simple functions which inject themselvies to
     a task queue and then they are picked up by a threading/process pool to
     execute, as shows below:
-    Timers --enqueue---> TimerQueue --------expiration-----------
-                                                                |
-                                                                |
-                                                               \|/
-    Threading/Process Pool <---- TaskQueue <--enqueue-- Timers' callback (nonblocking)
+
+        Timers --enqueue---> TimerQueue --------expiration-----------
+                                                                    |
+                                                                    |
+                                                                   \|/
+        Threading/Process Pool <---- TaskQueue <--enqueue-- Timers' callback (nonblocking)
 
     Examples:
            >>> from solnlib import timer_queue

examples/test__kvstore.py → tests/integration/test__kvstore.py

@@ -27,17 +27,9 @@
 from splunklib import binding, client
 from splunklib.binding import HTTPError
 
-from solnlib.credentials import get_session_key
-
 
 def test_kvstore():
-    session_key = get_session_key(
-        context.username,
-        context.password,
-        scheme=context.scheme,
-        host=context.host,
-        port=context.port,
-    )
+    session_key = context.get_session_key()
     kvstore = client.Service(
         scheme=context.scheme,
         host=context.host,

examples/test_acl.py → tests/integration/test_acl.py

@@ -21,17 +21,10 @@
 import context
 
 from solnlib import acl
-from solnlib.credentials import get_session_key
 
 
 def test_acl_manager():
-    session_key = get_session_key(
-        context.username,
-        context.password,
-        scheme=context.scheme,
-        host=context.host,
-        port=context.port,
-    )
+    session_key = context.get_session_key()
 
     aclm = acl.ACLManager(
         session_key,

examples/test_conf_manager.py → tests/integration/test_conf_manager.py

@@ -22,18 +22,11 @@
 sys.path.insert(0, op.dirname(op.dirname(op.abspath(__file__))))
 import context
 
-from solnlib import conf_manager, credentials
+from solnlib import conf_manager
 
 
 def test_conf_manager():
-    session_key = credentials.get_session_key(
-        context.username,
-        context.password,
-        scheme=context.scheme,
-        host=context.host,
-        port=context.port,
-    )
-
+    session_key = context.get_session_key()
     cfm = conf_manager.ConfManager(
         session_key,
         context.app,

examples/test_hec_config.py → tests/integration/test_hec_config.py

@@ -20,17 +20,11 @@
 sys.path.insert(0, op.dirname(op.dirname(op.abspath(__file__))))
 import context
 
-from solnlib import credentials, hec_config
+from solnlib import hec_config
 
 
 def test_hec_config():
-    session_key = credentials.get_session_key(
-        context.username,
-        context.password,
-        scheme=context.scheme,
-        host=context.host,
-        port=context.port,
-    )
+    session_key = context.get_session_key()
     config = hec_config.HECConfig(session_key)
     stanza = {
         "index": "main",

examples/test_hec_event_writer.py → tests/integration/test_hec_event_writer.py

@@ -20,18 +20,11 @@
 sys.path.insert(0, op.dirname(op.dirname(op.abspath(__file__))))
 import context
 
-from solnlib import credentials
 from solnlib.modular_input import event_writer as hew
 
 
 def test_hec_event_writer():
-    session_key = credentials.get_session_key(
-        context.username,
-        context.password,
-        scheme=context.scheme,
-        host=context.host,
-        port=context.port,
-    )
+    session_key = context.get_session_key()
 
     ew = hew.HECEventWriter("test", session_key)
     m1 = {}