diff --git a/examples/test_logstash.py b/examples/test_logstash.py new file mode 100644 index 0000000..38cd0cf --- /dev/null +++ b/examples/test_logstash.py @@ -0,0 +1,41 @@ +from time import sleep +from elasticsearch import Elasticsearch +from navconfig.logging import logger + +## set a new logger name +logger.setName('config.example') +# verbose debugging: +logger.verbose('This is a verbose message') + +# Give Elasticsearch some time to index the new log entry +sleep(5) + +# Initialize the Elasticsearch client +es = Elasticsearch( + ['http://localhost:9200'], + basic_auth=('elastic', '12345678') +) + +# Define the search query +search_query = { + "match": { + "message": "This is a verbose message" + } +} + +# Perform the search +res = es.search(index="logstash-*", query=search_query) + + +# Extract the total number of hits +total_hits = res['hits']['total']['value'] + +# If the log entry was found in Elasticsearch +if total_hits > 0: + logger.verbose("Found the log entry in Elasticsearch!") +else: + logger.verbose("Could not find the log entry in Elasticsearch.") + +# Print out the first log entry (if any were found) +if total_hits > 0: + print(res['hits']['hits'][0]['_source']) diff --git a/navconfig/config.py b/navconfig/config.py index edb84f5..bb3c3b2 100644 --- a/navconfig/config.py +++ b/navconfig/config.py @@ -242,7 +242,7 @@ def _get_external(self, key: str) -> Any: """ for _, reader in self._readers.items(): try: - if reader.exists(key) is True: + if reader.enabled is True and reader.exists(key) is True: return reader.get(key) except RuntimeError: continue diff --git a/navconfig/logging/__init__.py b/navconfig/logging/__init__.py index 9dcbe84..f8a1db7 100644 --- a/navconfig/logging/__init__.py +++ b/navconfig/logging/__init__.py @@ -21,7 +21,9 @@ APP_NAME = config.get('APP_NAME', fallback='navigator') APP_TITLE = config.get('APP_TITLE', section='info', fallback='navigator') -LOG_DIR = config.get('logdir', section='logging', fallback=str(BASE_DIR.joinpath('logs'))) +LOG_DIR = config.get( + 'logdir', section='logging', fallback=str(BASE_DIR.joinpath('logs')) +) LOG_NAME = config.get('logname', section='logging', fallback=APP_TITLE) TMP_DIR = config.get('temp_path', section='temp', fallback='/tmp') @@ -46,9 +48,15 @@ 'logging_enable_filehandler', section='logging', fallback=False ) -logging_host = config.get('logging_host', section='logging', fallback="localhost") -logging_admin = config.get('logging_admin', section='logging', fallback="dev@domain.com") -logging_email = config.get('logging_email', section='logging', fallback='no-reply@domain.com') +logging_host = config.get( + 'logging_host', section='logging', fallback="localhost" +) +logging_admin = config.get( + 'logging_admin', section='logging', fallback="dev@domain.com" +) +logging_email = config.get( + 'logging_email', section='logging', fallback='no-reply@domain.com' +) # Path version of the log directory logdir = Path(LOG_DIR).resolve() @@ -92,20 +100,20 @@ 'level': loglevel }, 'StreamHandler': { - 'class': 'logging.StreamHandler', - 'formatter': 'default', - "stream": "ext://sys.stdout", - 'level': loglevel + 'class': 'logging.StreamHandler', + 'formatter': 'default', + "stream": "ext://sys.stdout", + 'level': loglevel }, 'ErrorFileHandler': { - 'class': 'logging.handlers.RotatingFileHandler', - 'level': logging.ERROR, - 'filename': f'{LOG_DIR}/{LOG_NAME}.error.log', - 'formatter': 'error', - 'maxBytes': (1048576*5), - 'backupCount': 2, - 'mode': 'a', - } + 'class': 'logging.handlers.RotatingFileHandler', + 'level': logging.ERROR, + 'filename': f'{LOG_DIR}/{LOG_NAME}.error.log', + 'formatter': 'error', + 'maxBytes': (1048576 * 5), + 'backupCount': 2, + 'mode': 'a', + } }, loggers={ APP_NAME: { @@ -134,7 +142,7 @@ logging_config['handlers']['RotatingFileHandler'] = { 'class': 'logging.handlers.RotatingFileHandler', 'filename': f'{LOG_DIR}/{LOG_NAME}.log', - 'maxBytes': (1048576*5), + 'maxBytes': (1048576 * 5), 'backupCount': 2, 'encoding': 'utf-8', 'formatter': 'file', @@ -147,7 +155,7 @@ 'level': logging.CRITICAL, 'formatter': 'error', 'class': 'logging.handlers.SMTPHandler', - 'mailhost' : logging_host, + 'mailhost': logging_host, 'fromaddr': logging_email, 'toaddrs': [logging_admin], 'subject': f'Critical Error on {APP_NAME}' @@ -158,42 +166,50 @@ logging_config['root']['handlers'].append('console') if logstash_logging: + logging.debug( + "Logstash configuration Enabled." + ) environment = config.ENV if config.ENV is not None else 'production' # basic configuration of Logstash try: - import logstash_async # pylint: disable=W0611 + import logstash_async # pylint: disable=W0611 except ImportError as ex: raise RuntimeError( - "NavConfig: Logstash Logging is enabled but Logstash async dependency is not installed.\ + "NavConfig: Logstash Logging is enabled but Logstash async \ + dependency is not installed.\ Hint: run 'pip install logstash_async'." ) from ex LOGSTASH_HOST = config.get('LOGSTASH_HOST', fallback='localhost') - LOGSTASH_PORT = config.get('LOGSTASH_PORT', fallback=6000) + LOGSTASH_PORT = config.getint('LOGSTASH_PORT', fallback=6000) LOG_TAG = config.get('FLUENT_TAG', fallback='app.log') logging_config['formatters']['logstash'] = { - '()': 'logstash_async.formatter.LogstashFormatter', - 'message_type': 'python-logstash', - 'fqdn': False, # Fully qualified domain name. Default value: false. - 'extra_prefix': 'dev', - 'extra': { - 'application': f'{APP_NAME}', - 'project_path': f'{BASE_DIR}', - 'environment': environment - } + '()': 'logstash_async.formatter.LogstashFormatter', + 'message_type': 'python-logstash', + 'fqdn': False, # Fully qualified domain name. Default value: false. + 'extra_prefix': 'dev', + 'extra': { + 'application': f'{APP_NAME}', + 'project_path': f'{BASE_DIR}', + 'environment': environment + } } logging_config['handlers']['LogstashHandler'] = { - 'class': 'logstash_async.handler.AsynchronousLogstashHandler', - 'formatter': 'logstash', - 'transport': 'logstash_async.transport.TcpTransport', - 'host': LOGSTASH_HOST, - 'port': LOGSTASH_PORT, - 'level': loglevel, - 'database_path': f'{LOG_DIR}/logstash.db', + 'class': 'logstash_async.handler.AsynchronousLogstashHandler', + 'formatter': 'logstash', + 'transport': 'logstash_async.transport.TcpTransport', + 'host': LOGSTASH_HOST, + 'port': int(LOGSTASH_PORT), + 'level': loglevel, + 'database_path': f'{LOG_DIR}/logstash.db', } - if APP_NAME in logging_config: - logging_config[APP_NAME]['handlers'] = [ - 'LogstashHandler', 'StreamHandler' - ] + if 'root' not in logging_config: + logging_config['root'] = {} + if 'handlers' not in logging_config['root']: + logging_config['root']['handlers'] = [] + logging_config['root']['handlers'].extend( + ['LogstashHandler', 'StreamHandler'] + ) + ### Load Logging Configuration: dictConfig(logging_config) diff --git a/navconfig/readers/abstract.py b/navconfig/readers/abstract.py index 81fc88e..66d03b5 100644 --- a/navconfig/readers/abstract.py +++ b/navconfig/readers/abstract.py @@ -7,6 +7,8 @@ class AbstractReader(ABC): Description: Abstract class for External Readers. """ + enabled: bool = True + @abstractmethod def get(self, key: str, default: Any = None) -> Any: pass diff --git a/navconfig/readers/memcache.py b/navconfig/readers/memcache.py index b3d7e58..9350abb 100644 --- a/navconfig/readers/memcache.py +++ b/navconfig/readers/memcache.py @@ -20,7 +20,8 @@ def __init__(self) -> None: binary=True, behaviors=self._args ) - except Exception as err: # pylint: disable=W0703 + except Exception as err: # pylint: disable=W0703 + self.enabled = False logging.exception(err, stack_info=True) def get(self, key, default=None): diff --git a/navconfig/readers/redis.py b/navconfig/readers/redis.py index 504f4d8..501776f 100644 --- a/navconfig/readers/redis.py +++ b/navconfig/readers/redis.py @@ -30,15 +30,18 @@ def __init__(self): url=self.redis_url, **self.params ) except (TimeoutError) as err: + self.enabled = False raise Exception( f"Redis Config: Redis Timeout: {err}" ) from err except (RedisError, ConnectionError) as err: + self.enabled = False raise Exception( f"Redis Config: Unable to connect to Redis: {err}" ) from err except Exception as err: logging.exception(err) + self.enabled = False raise def set(self, key, value): diff --git a/navconfig/readers/vault.py b/navconfig/readers/vault.py index d1eae59..f183405 100644 --- a/navconfig/readers/vault.py +++ b/navconfig/readers/vault.py @@ -22,10 +22,11 @@ def __init__(self): try: self.client = hvac.Client(url=url, token=token) self.open() - except Exception as err: # pylint: disable=W0703 - logging.exception( - f"Vault Error: {err}", stack_info=True + except Exception as err: # pylint: disable=W0703 + logging.error( + f"Vault Error: {err}", stack_info=False ) + self.enabled = False def open(self) -> bool: if self.client.is_authenticated(): diff --git a/navconfig/version.py b/navconfig/version.py index 4c09ce6..9e9149a 100644 --- a/navconfig/version.py +++ b/navconfig/version.py @@ -3,7 +3,7 @@ __title__ = 'navconfig' __description__ = ('Configuration tool for all Navigator Services ' 'Tool for accessing Config info from different sources.') -__version__ = '1.2.0' +__version__ = '1.2.1' __author__ = 'Jesus Lara' __author_email__ = 'jesuslarag@gmail.com' __license__ = 'MIT' diff --git a/setup.py b/setup.py index 0b566f5..fe78641 100644 --- a/setup.py +++ b/setup.py @@ -136,7 +136,7 @@ def readme(): 'aiofiles==23.1.0', 'aiofile==3.8.1', ], - extras_require = { + extras_require={ "memcache": [ "pylibmc==1.6.3", "aiomcache==0.8.1", @@ -147,6 +147,7 @@ def readme(): "logstash": [ 'python-logstash-async==2.5.0', 'aiologstash==2.0.0', + 'elasticsearch==8.8.0' ], "redis": [ 'redis==4.5.1', @@ -156,10 +157,10 @@ def readme(): 'pytomlpp==1.0.11' ], "yaml": [ - 'PyYAML>=6.0', + 'PyYAML>=6.0', ], "hvac": [ - "hvac==1.1.0" + "hvac==1.1.0" ], "default": [ 'pytomlpp==1.0.11',