Merge pull request #256 from phenobarbital/vault-reader-loader

fix over logstash configuration

examples/test_logstash.py

@@ -0,0 +1,41 @@
+from time import sleep
+from elasticsearch import Elasticsearch
+from navconfig.logging import logger
+
+## set a new logger name
+logger.setName('config.example')
+# verbose debugging:
+logger.verbose('This is a verbose message')
+
+# Give Elasticsearch some time to index the new log entry
+sleep(5)
+
+# Initialize the Elasticsearch client
+es = Elasticsearch(
+    ['http://localhost:9200'],
+    basic_auth=('elastic', '12345678')
+)
+
+# Define the search query
+search_query = {
+    "match": {
+        "message": "This is a verbose message"
+    }
+}
+
+# Perform the search
+res = es.search(index="logstash-*", query=search_query)
+
+
+# Extract the total number of hits
+total_hits = res['hits']['total']['value']
+
+# If the log entry was found in Elasticsearch
+if total_hits > 0:
+    logger.verbose("Found the log entry in Elasticsearch!")
+else:
+    logger.verbose("Could not find the log entry in Elasticsearch.")
+
+# Print out the first log entry (if any were found)
+if total_hits > 0:
+    print(res['hits']['hits'][0]['_source'])

navconfig/config.py

@@ -242,7 +242,7 @@ def _get_external(self, key: str) -> Any:
         """
         for _, reader in self._readers.items():
             try:
-                if reader.exists(key) is True:
+                if reader.enabled is True and reader.exists(key) is True:
                     return reader.get(key)
             except RuntimeError:
                 continue

navconfig/logging/__init__.py

@@ -21,7 +21,9 @@
 
 APP_NAME = config.get('APP_NAME', fallback='navigator')
 APP_TITLE = config.get('APP_TITLE', section='info', fallback='navigator')
-LOG_DIR = config.get('logdir', section='logging', fallback=str(BASE_DIR.joinpath('logs')))
+LOG_DIR = config.get(
+    'logdir', section='logging', fallback=str(BASE_DIR.joinpath('logs'))
+)
 LOG_NAME = config.get('logname', section='logging', fallback=APP_TITLE)
 TMP_DIR = config.get('temp_path', section='temp', fallback='/tmp')
 
@@ -46,9 +48,15 @@
     'logging_enable_filehandler', section='logging', fallback=False
 )
 
-logging_host = config.get('logging_host', section='logging', fallback="localhost")
-logging_admin = config.get('logging_admin', section='logging', fallback="[email protected]")
-logging_email = config.get('logging_email', section='logging', fallback='[email protected]')
+logging_host = config.get(
+    'logging_host', section='logging', fallback="localhost"
+)
+logging_admin = config.get(
+    'logging_admin', section='logging', fallback="[email protected]"
+)
+logging_email = config.get(
+    'logging_email', section='logging', fallback='[email protected]'
+)
 
 # Path version of the log directory
 logdir = Path(LOG_DIR).resolve()
@@ -92,20 +100,20 @@
             'level': loglevel
         },
         'StreamHandler': {
-                'class': 'logging.StreamHandler',
-                'formatter': 'default',
-                "stream": "ext://sys.stdout",
-                'level': loglevel
+            'class': 'logging.StreamHandler',
+            'formatter': 'default',
+            "stream": "ext://sys.stdout",
+            'level': loglevel
         },
         'ErrorFileHandler': {
-                'class': 'logging.handlers.RotatingFileHandler',
-                'level': logging.ERROR,
-                'filename': f'{LOG_DIR}/{LOG_NAME}.error.log',
-                'formatter': 'error',
-                'maxBytes': (1048576*5),
-                'backupCount': 2,
-                'mode': 'a',
-         }
+            'class': 'logging.handlers.RotatingFileHandler',
+            'level': logging.ERROR,
+            'filename': f'{LOG_DIR}/{LOG_NAME}.error.log',
+            'formatter': 'error',
+            'maxBytes': (1048576 * 5),
+            'backupCount': 2,
+            'mode': 'a',
+        }
     },
     loggers={
         APP_NAME: {
@@ -134,7 +142,7 @@
     logging_config['handlers']['RotatingFileHandler'] = {
         'class': 'logging.handlers.RotatingFileHandler',
         'filename': f'{LOG_DIR}/{LOG_NAME}.log',
-        'maxBytes': (1048576*5),
+        'maxBytes': (1048576 * 5),
         'backupCount': 2,
         'encoding': 'utf-8',
         'formatter': 'file',
@@ -147,7 +155,7 @@
         'level': logging.CRITICAL,
         'formatter': 'error',
         'class': 'logging.handlers.SMTPHandler',
-        'mailhost' : logging_host,
+        'mailhost': logging_host,
         'fromaddr': logging_email,
         'toaddrs': [logging_admin],
         'subject': f'Critical Error on {APP_NAME}'
@@ -158,42 +166,50 @@
     logging_config['root']['handlers'].append('console')
 
 if logstash_logging:
+    logging.debug(
+        "Logstash configuration Enabled."
+    )
     environment = config.ENV if config.ENV is not None else 'production'
     # basic configuration of Logstash
     try:
-        import logstash_async # pylint: disable=W0611
+        import logstash_async  # pylint: disable=W0611
     except ImportError as ex:
         raise RuntimeError(
-            "NavConfig: Logstash Logging is enabled but Logstash async dependency is not installed.\
+            "NavConfig: Logstash Logging is enabled but Logstash async \
+                dependency is not installed.\
             Hint: run 'pip install logstash_async'."
         ) from ex
     LOGSTASH_HOST = config.get('LOGSTASH_HOST', fallback='localhost')
-    LOGSTASH_PORT = config.get('LOGSTASH_PORT', fallback=6000)
+    LOGSTASH_PORT = config.getint('LOGSTASH_PORT', fallback=6000)
     LOG_TAG = config.get('FLUENT_TAG', fallback='app.log')
     logging_config['formatters']['logstash'] = {
-      '()': 'logstash_async.formatter.LogstashFormatter',
-      'message_type': 'python-logstash',
-      'fqdn': False,  # Fully qualified domain name. Default value: false.
-      'extra_prefix': 'dev',
-      'extra': {
-          'application': f'{APP_NAME}',
-          'project_path': f'{BASE_DIR}',
-          'environment': environment
-      }
+        '()': 'logstash_async.formatter.LogstashFormatter',
+        'message_type': 'python-logstash',
+        'fqdn': False,  # Fully qualified domain name. Default value: false.
+        'extra_prefix': 'dev',
+        'extra': {
+            'application': f'{APP_NAME}',
+            'project_path': f'{BASE_DIR}',
+            'environment': environment
+        }
     }
     logging_config['handlers']['LogstashHandler'] = {
-            'class': 'logstash_async.handler.AsynchronousLogstashHandler',
-            'formatter': 'logstash',
-            'transport': 'logstash_async.transport.TcpTransport',
-            'host': LOGSTASH_HOST,
-            'port': LOGSTASH_PORT,
-            'level': loglevel,
-            'database_path': f'{LOG_DIR}/logstash.db',
+        'class': 'logstash_async.handler.AsynchronousLogstashHandler',
+        'formatter': 'logstash',
+        'transport': 'logstash_async.transport.TcpTransport',
+        'host': LOGSTASH_HOST,
+        'port': int(LOGSTASH_PORT),
+        'level': loglevel,
+        'database_path': f'{LOG_DIR}/logstash.db',
     }
-    if APP_NAME in logging_config:
-        logging_config[APP_NAME]['handlers'] = [
-            'LogstashHandler', 'StreamHandler'
-        ]
+    if 'root' not in logging_config:
+        logging_config['root'] = {}
+    if 'handlers' not in logging_config['root']:
+        logging_config['root']['handlers'] = []
+    logging_config['root']['handlers'].extend(
+        ['LogstashHandler', 'StreamHandler']
+    )
+
 
 ### Load Logging Configuration:
 dictConfig(logging_config)

navconfig/readers/abstract.py

@@ -7,6 +7,8 @@ class AbstractReader(ABC):
 
     Description: Abstract class for External Readers.
     """
+    enabled: bool = True
+
     @abstractmethod
     def get(self, key: str, default: Any = None) -> Any:
         pass

navconfig/readers/memcache.py

@@ -20,7 +20,8 @@ def __init__(self) -> None:
                 binary=True,
                 behaviors=self._args
             )
-        except Exception as err: # pylint: disable=W0703
+        except Exception as err:  # pylint: disable=W0703
+            self.enabled = False
             logging.exception(err, stack_info=True)
 
     def get(self, key, default=None):

navconfig/readers/redis.py

@@ -30,15 +30,18 @@ def __init__(self):
                 url=self.redis_url, **self.params
             )
         except (TimeoutError) as err:
+            self.enabled = False
             raise Exception(
                 f"Redis Config: Redis Timeout: {err}"
             ) from err
         except (RedisError, ConnectionError) as err:
+            self.enabled = False
             raise Exception(
                 f"Redis Config: Unable to connect to Redis: {err}"
             ) from err
         except Exception as err:
             logging.exception(err)
+            self.enabled = False
             raise
 
     def set(self, key, value):

navconfig/readers/vault.py

@@ -22,10 +22,11 @@ def __init__(self):
         try:
             self.client = hvac.Client(url=url, token=token)
             self.open()
-        except Exception as err: # pylint: disable=W0703
-            logging.exception(
-                f"Vault Error: {err}", stack_info=True
+        except Exception as err:  # pylint: disable=W0703
+            logging.error(
+                f"Vault Error: {err}", stack_info=False
             )
+            self.enabled = False
 
     def open(self) -> bool:
         if self.client.is_authenticated():

navconfig/version.py

@@ -3,7 +3,7 @@
 __title__ = 'navconfig'
 __description__ = ('Configuration tool for all Navigator Services '
                    'Tool for accessing Config info from different sources.')
-__version__ = '1.2.0'
+__version__ = '1.2.1'
 __author__ = 'Jesus Lara'
 __author_email__ = '[email protected]'
 __license__ = 'MIT'

setup.py

@@ -136,7 +136,7 @@ def readme():
         'aiofiles==23.1.0',
         'aiofile==3.8.1',
     ],
-    extras_require = {
+    extras_require={
         "memcache": [
             "pylibmc==1.6.3",
             "aiomcache==0.8.1",
@@ -147,6 +147,7 @@ def readme():
         "logstash": [
             'python-logstash-async==2.5.0',
             'aiologstash==2.0.0',
+            'elasticsearch==8.8.0'
         ],
         "redis": [
             'redis==4.5.1',
@@ -156,10 +157,10 @@ def readme():
             'pytomlpp==1.0.11'
         ],
         "yaml": [
-          'PyYAML>=6.0',
+            'PyYAML>=6.0',
         ],
         "hvac": [
-          "hvac==1.1.0"
+            "hvac==1.1.0"
         ],
         "default": [
             'pytomlpp==1.0.11',