House keeping improvements (#49)

* Updated dependencies versions

* fixed side efects of sanic version update

Now we cannot assign values directly in sanic application instances. This should be done throug app.ctx attribute

* Improved contract endpoint and fixed bapi instance

* Contracts are now obtained in a concurrent way without i/o blocks
* Also fix BeedataApi instantiaton

* // WIP: custom responses

* Unified error responses

* Unified error responses

* improve sentries identification

* refactored f1 endpoints

* fixed f1 endpoints to not block main process

* added PageNotFoudError in cch endpoints

* validate report body

* Updated requirements and ignored files

* Fixed report tests

Fixed report code and all side effects involved.

* Fixed sanic version update

* cleaned unused imports

* cleaned unused imports

* fixed beedata_api_client tests

* improved cch_endpoint

* added retry_expired decorator

* added `retry_expired` decorator to manage expired conexions

* //WIP: improve reports

* fixed retry_expired decorator

* reordered code in BeedataApiClient

* updated requirements

* installed pudb

* fixed report types in beedata_api

* Added black code style and pre-commit conf

* added .editorconfig

* blacked code o_O

* refactor(BeedataApiClient): improved interface

* http session in client
* _request is now a clojure
* clean connections when client is deleted
* download_report returns only one report

* Test: fixed beedaapi client tests

* feat(ReportRequest): created a model to track each report request that api recieve

* refactor(reports): refactored reports flow

- added ReportRequest to track reports requests
- improved report view
- improved report flow processing

* test(reports): added invalid report case

* test(reports): fixed report api call

* build: updated dependencies

* test: record mode as environ variable

* test(reports): fix beedata instantiation

* fix(cch): fixed contract_id parameter in pagination

* refactor: adapted app creation to sanic new version

* fix(cch): fixed import

* fix(pagination): fixed url generation for result pagination

* refactor: rename api view files to genereic view name

* fix(contractId -> contract_id): renamed to contract_id in order to use paginationlinksmixin

- fixed bugs getting parameters in modcontracts

* refactor: splited build_app factory function from app instantiation

* refactor: addapted run.py to new app instantiation

* fix(tests): fixed base tase class andk test_contract

* fix: fixed db initialitation in build_app and app import

* doc: updated README

* fix: removed harcoded dummy passwd

.editorconfig

@@ -0,0 +1,18 @@
+root = true
+
+[*]
+indent_style = spaces
+end_of_line = lf
+trim_trailing_whitespace = true
+insert_final_newline = true
+charset = utf-8
+
+[*.py]
+max_line_length = 90
+indent_size = 4
+
+[*.json]
+indent_size = 4
+
+[*.yaml]
+indent_size = 2

.env.example

@@ -41,4 +41,4 @@ PASSWORD = "test1234"
 N_WORKERS = 2
 
 # Sentry SENTRY_DSN
-SENTRY_DSN='https://[email protected]/0'
+SENTRY_DSN='https://[email protected]/0'

.gitignore

@@ -116,4 +116,4 @@ json4test.yaml
 .vscode
 cert.crt
 cert.key
-fixtures
+vcr_cassetes

.pre-commit-config.yaml

@@ -0,0 +1,11 @@
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v2.3.0
+    hooks:
+    -   id: end-of-file-fixer
+    -   id: trailing-whitespace
+  - repo: https://github.com/psf/black
+    rev: 22.6.0
+    hooks:
+      - id: black
+        language_version: python3.9

Pipfile

@@ -11,31 +11,33 @@ pylint = "*"
 coverage = "*"
 pytest = "*"
 requests = "*"
-pytest-aiohttp = "*"
 fakeredis = "*"
 vcrpy = "*"
+sanic-testing = "*"
+pytest-env = "*"
+pytest-asyncio = "*"
+nest-asyncio = "*"
+pudb = "*"
+pre-commit = "*"
+black = "*"
 
 [packages]
 connexion = "*"
 python-jose = "*"
-sanic = "==20.12.2"
+sanic = ">=22.3"
 erppeek = "*"
 sanic-jwt = "*"
 pytz = "*"
 passlib = "*"
 pony = "*"
 environs = "*"
-uvicorn = "==0.11.5"
-aioredis = "==1.3.0"
-pymongo = "*"
-motor = "*"
+uvicorn = "*"
+aioredis = "*"
+pymongo = ">=3.12,<4.0"
+motor = ">=2.5,<3.0"
 pool-transport = {git = "https://github.com/Som-Energia/pool_transport.git"}
-aiohttp = {extras = ["speedups"],version = "*"}
 pyyaml = "*"
 sentry-sdk = "*"
-aiocontextvars = "*"
-fakeredis = "*"
-vcrpy = "*"
-
-[requires]
-python_version = "3.8"
+aiohttp = "*"
+hiredis = "*"
+ipdb = "*"

README.md

@@ -40,6 +40,13 @@ Now our api is ready to run. You can simply execute `pipenv run python run.py` a
 
 #### Changes
 
+### 2.0.0-rc1
+- updated dependencies versions
+- adapted code to that dependencies
+- reimplemented report processing
+- a lot of bugs fixed
+- improved perfomance
+
 ### 1.5.0
 - Adapted endpoints to new tolls
 
@@ -73,7 +80,7 @@ Now our api is ready to run. You can simply execute `pipenv run python run.py` a
 
 ### 1.3.0
 - Added f5d endpoint
-- Support for parallel requests to one ERP 
+- Support for parallel requests to one ERP
 
 ### 1.2.1
 - Fixed minor bugs

config/__init__.py

@@ -1,12 +1,12 @@
 import os
 from importlib import import_module
 
-ENVIRONMENT_VARIABLE = 'INFOENERGIA_MODULE_SETTINGS'
+ENVIRONMENT_VARIABLE = "INFOENERGIA_MODULE_SETTINGS"
 
-os.environ.setdefault('INFOENERGIA_MODULE_SETTINGS', 'config.settings.devel')
+os.environ.setdefault("INFOENERGIA_MODULE_SETTINGS", "config.settings.devel")
 
-class ImproperlyConfigured(Exception):
 
+class ImproperlyConfigured(Exception):
     def __init__(self, msg):
         super(ImproperlyConfigured, self).__init__()
         self.msg = msg
@@ -17,12 +17,12 @@ def __repr__(self):
     def __str__(self):
         return self.__repr__()
 
-class Settings(object):
 
+class Settings(object):
     def __init__(self, settings_module):
         self.SETTINGS_MODULE = settings_module
         if not self.SETTINGS_MODULE:
-            msg = 'Environment variable \"{}\" is not set'
+            msg = 'Environment variable "{}" is not set'
             raise ImproperlyConfigured(msg.format(ENVIRONMENT_VARIABLE))
 
         mod = import_module(self.SETTINGS_MODULE)

config/samples/supervisor/infoenergia_api.ini.sample

@@ -13,4 +13,4 @@ stdout_logfile_backups=15
 stdout_events_enabled=false
 stderr_logfile=/var/log/infoenergia_api/api.err.log
 stderr_logfile_maxbytes=1MB
-stderr_logfile_backups=15
+stderr_logfile_backups=15

config/settings/base.py

@@ -1,6 +1,6 @@
-'''
+"""
 Base and common settings for api configuration
-'''
+"""
 import os
 
 from environs import Env
@@ -11,20 +11,20 @@
 BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 # Supermega secret key
-SECRET_KEY = '1234'
+SECRET_KEY = "1234"
 
 # Max threads in pool
-MAX_THREADS = 5
+MAX_THREADS = 200
 
 # Max concurrent tasks
 MAX_TASKS = 1
 
 # I will wait until this timeout seconds
-TASK_TIMEOUT = 30
+TASKS_TIMEOUT = 0.5
 
 # Response timeout
 RESPONSE_TIMEOUT = 36000
 
-SANIC_JWT_USER_ID = 'id'
+SANIC_JWT_USER_ID = "id"
 
-SENTRY_DSN = env.str('SENTRY_DSN')
+SENTRY_DSN = env.str("SENTRY_DSN")

config/settings/devel.py

@@ -1,62 +1,58 @@
 from .base import *
 
 # ERP configuration
-ERP_CONF = env.json('ERP_CONF')
+ERP_CONF = env.json("ERP_CONF")
 
 # Transport Pool configuration
-TRANSPORT_POOL_CONF = env.json('TRANSPORT_POOL_CONF')
+TRANSPORT_POOL_CONF = env.json("TRANSPORT_POOL_CONF")
 
 # Mongo configuration
-MONGO_CONF = env.str('MONGO_CONF')
+MONGO_CONF = env.str("MONGO_CONF")
 
 # DATABASE configuration
-DB_CONF = env.json('DATABASE_CONF')
+DB_CONF = env.json("DATABASE_CONF")
 
 # Redis configuration
-REDIS_CONF = env.str('REDIS_CONF')
+REDIS_CONF = env.str("REDIS_CONF")
 
 MAX_THREADS = 10
 
 INVITATION_EXP_DAYS = 1
 
 SANIC_JWT_SECRET = SECRET_KEY
 
-RESULTS_TTL = env.int('RESULTS_TTL')
+RESULTS_TTL = env.int("RESULTS_TTL")
 
 DATA_DIR = BASE_DIR
 LOGGING = {
-    'version': 1,
-    'disable_existing_loggers': True,
-    'formatters': {
-        'verbose': {
-            'format': '[%(asctime)s] [%(process)d] [%(levelname)s]'
-                      '[%(module)s.%(funcName)s:%(lineno)s] %(message)s'
+    "version": 1,
+    "disable_existing_loggers": True,
+    "formatters": {
+        "verbose": {
+            "format": "[%(asctime)s] [%(process)d] [%(levelname)s]"
+            "[%(module)s.%(funcName)s:%(lineno)s] %(message)s"
         },
     },
-    'handlers': {
-        'console': {
-            'level': 'DEBUG',
-            'class': 'logging.StreamHandler',
-            'formatter': 'verbose'
+    "handlers": {
+        "console": {
+            "level": "DEBUG",
+            "class": "logging.StreamHandler",
+            "formatter": "verbose",
         }
     },
-    'loggers': {
-        'api': {
-            'handlers': ['console'],
-            'level': 'DEBUG',
-            'propagate': True
-        },
-    }
+    "loggers": {
+        "api": {"handlers": ["console"], "level": "DEBUG", "propagate": True},
+    },
 }
 
-#Access BeeData API:
-CERT_FILE = env.str('CERT_FILE')
-KEY_FILE = env.str('KEY_FILE')
-COMPANY_ID = env.int('COMPANY_ID')
-BASE_URL = env.str('BASE_URL')
-APIVERSION = env.str('APIVERSION')
-USERNAME = env.str('USERNAME')
-PASSWORD = env.str('PASSWORD')
-
-#Number of workers for process Reports
-N_WORKERS = env.int('N_WORKERS')
+# Access BeeData API:
+CERT_FILE = env.str("CERT_FILE")
+KEY_FILE = env.str("KEY_FILE")
+COMPANY_ID = env.int("COMPANY_ID")
+BASE_URL = env.str("BASE_URL")
+APIVERSION = env.str("APIVERSION")
+USERNAME = env.str("USERNAME")
+PASSWORD = env.str("PASSWORD")
+
+# Number of workers for process Reports
+N_WORKERS = env.int("N_WORKERS")

config/settings/prod.py

@@ -1,73 +1,72 @@
 from .base import *
 
 # ERP configuration
-ERP_CONF = env.json('ERP_CONF')
+ERP_CONF = env.json("ERP_CONF")
 
 # Transport Pool configuration
-TRANSPORT_POOL_CONF = env.json('TRANSPORT_POOL_CONF')
+TRANSPORT_POOL_CONF = env.json("TRANSPORT_POOL_CONF")
 
 # Mongo configuration
-MONGO_CONF = env.str('MONGO_CONF')
+MONGO_CONF = env.str("MONGO_CONF")
 
 # DATABASE configuration
-DB_CONF = env.json('DATABASE_CONF')
+DB_CONF = env.json("DATABASE_CONF")
 
 # Redis configuration
-REDIS_CONF = env.str('REDIS_CONF')
+REDIS_CONF = env.str("REDIS_CONF")
 
 ACCESS_LOG = False
 
 PROXIES_COUNT = 1
 
-MAX_THREADS = env.int('MAX_THREADS')
+MAX_THREADS = env.int("MAX_THREADS")
 
-INVITATION_EXP_DAYS = env.int('EXP_DAYS')
+# I will wait until this timeout seconds
+TASKS_TIMEOUT = env.float("TASKS_TIMEOUT")
 
-SECRET_KEY = env.str('SECRET_KEY')
+INVITATION_EXP_DAYS = env.int("EXP_DAYS")
+
+SECRET_KEY = env.str("SECRET_KEY")
 
 SANIC_JWT_SECRET = SECRET_KEY
-SANIC_JWT_EXPIRATION_DELTA = env.int('JWT_EXPIRATION_DELTA') * 48
+SANIC_JWT_EXPIRATION_DELTA = env.int("JWT_EXPIRATION_DELTA") * 48
 
-RESULTS_TTL = env.int('RESULTS_TTL') * 18
+RESULTS_TTL = env.int("RESULTS_TTL") * 18
 
-DATA_DIR = env.str('DATA_DIR')
+DATA_DIR = env.str("DATA_DIR")
 
 LOGGING = {
-    'version': 1,
-    'disable_existing_loggers': True,
-    'formatters': {
-        'verbose': {
-            'format': '[%(asctime)s] [%(process)d] [%(levelname)s]'
-                      '[%(module)s.%(funcName)s:%(lineno)s] %(message)s'
+    "version": 1,
+    "disable_existing_loggers": True,
+    "formatters": {
+        "verbose": {
+            "format": "[%(asctime)s] [%(process)d] [%(levelname)s]"
+            "[%(module)s.%(funcName)s:%(lineno)s] %(message)s"
         },
     },
-    'handlers': {
-        'file': {
-            'level': 'INFO',
-            'class': 'logging.handlers.TimedRotatingFileHandler',
-            'filename': 'logs/infoenergia-api.log',
-            'when': 'midnight',
-            'backupCount': 7,
-            'formatter': 'verbose'
+    "handlers": {
+        "file": {
+            "level": "INFO",
+            "class": "logging.handlers.TimedRotatingFileHandler",
+            "filename": "logs/infoenergia-api.log",
+            "when": "midnight",
+            "backupCount": 7,
+            "formatter": "verbose",
         }
     },
-    'loggers': {
-        'api': {
-            'handlers': ['file'],
-            'level': 'INFO',
-            'propagate': True
-        },
-    }
+    "loggers": {
+        "api": {"handlers": ["file"], "level": "INFO", "propagate": True},
+    },
 }
 
-#Access BeeData API:
-CERT_FILE = env.str('CERT_FILE')
-KEY_FILE = env.str('KEY_FILE')
-COMPANY_ID = env.int('COMPANY_ID')
-BASE_URL = env.str('BASE_URL')
-APIVERSION = env.str('APIVERSION')
-USERNAME = env.str('USERNAME')
-PASSWORD = env.str('PASSWORD')
-
-#Number of workers for process Reports
-N_WORKERS = env.int('N_WORKERS')
+# Access BeeData API:
+CERT_FILE = env.str("CERT_FILE")
+KEY_FILE = env.str("KEY_FILE")
+COMPANY_ID = env.int("COMPANY_ID")
+BASE_URL = env.str("BASE_URL")
+APIVERSION = env.str("APIVERSION")
+USERNAME = env.str("USERNAME")
+PASSWORD = env.str("PASSWORD")
+
+# Number of workers for process Reports
+N_WORKERS = env.int("N_WORKERS")