Add logging configuration for both dev and prod environments

FNNDSC · Jun 28, 2020 · 20a0f38 · 20a0f38
1 parent 70717f5
commit 20a0f38
Show file tree

Hide file tree

Showing 17 changed files with 139 additions and 71 deletions.
diff --git a/README.md b/README.md
@@ -81,7 +81,7 @@ pip freeze --local
 Start ChRIS Store services by running the make bash script from the repository source directory
 
 ```bash
-./docker-make.sh up
+./make.sh up
 ```
 All the steps performed by the above script are properly documented in the script itself. 
 
@@ -133,7 +133,7 @@ docker run --rm -v /tmp/json:/json fnndsc/pl-simplefsapp simplefsapp.py --savejs
 ```
 Then upload the plugin representation json file to the ChRIS Store as part of the `POST` request:
 ```bash
-http -a cubeadmin:cubeadmin1234 -f POST http://localhost:8010/api/v1/plugins/ dock_image=fnndsc/pl-simplefsapp descriptor_file@/tmp/json/SimpleFSApp.json public_repo=https://github.com/FNNDSC/pl-simplefsapp name=simplefsapp
+http -a cubeadmin:cubeadmin1234 -f POST http://localhost:8010/api/v1/plugins/ dock_image=fnndsc/pl-simplefsapp descriptor_file@/tmp/json/SimpleFSApp.json public_repo=https://github.com/FNNDSC/pl-simplefsapp name=pl-simplefsapp
 ```
 
 #### An unauthenticated POST request to create a new ChRIS store user account:
@@ -151,7 +151,7 @@ swift -A http://127.0.0.1:8080/auth/v1.0 -U chris:chris1234 -K testing list stor
 Stop and remove ChRIS Store services by running the make bash script from the repository source directory
 
 ```bash
-./docker-make.sh down
+./make.sh down
 ```
 
 ### REST API Documentation

diff --git a/docker-make.sh → make.sh b/docker-make.sh → make.sh
@@ -41,6 +41,7 @@ if [[ "$1" == 'up' ]]; then
     docker pull mysql:5
     docker pull fnndsc/docker-swift-onlyone
     docker pull fnndsc/chris_store:dev
+    echo ""
     echo "docker-compose -f docker-compose_dev.yml up -d"
     docker-compose -f docker-compose_dev.yml up -d
     windowBottom
@@ -76,23 +77,24 @@ if [[ "$1" == 'up' ]]; then
     title -d 1 "Automatically uploading some plugins to the ChRIS STORE..."
     # Declare an array variable for the list of plugin names to be automatically registered
     # Add a new plugin name to the list if you want it to be automatically registered
-    declare -a plugins=( "simplefsapp"
-                         "simpledsapp"
-                         "pacsquery"
-                         "pacsretrieve"
-                         "s3retrieve"
-                         "s3push"
-                         "dircopy"
+    declare -a plugins=( "pl-simplefsapp"
+                         "pl-simpledsapp"
+                         "pl-pacsquery"
+                         "pl-pacsretrieve"
+                         "pl-s3retrieve"
+                         "pl-s3push"
+                         "pl-dircopy"
     )
     declare -i i=1
     declare -i STEP=7
     for plugin in "${plugins[@]}"; do
         echo ""
         echo "${STEP}.$i: Uploading $plugin representation to the ChRIS store..."
-        PLUGIN_DOCK="fnndsc/pl-${plugin}"
+        PLUGIN_DOCK="fnndsc/${plugin}"
+        PLUGIN_MODULE="${plugin:3}"
         docker pull "$PLUGIN_DOCK"
-        PLUGIN_REP=$(docker run --rm "$PLUGIN_DOCK" "${plugin}.py" --json 2> /dev/null;)
-        docker-compose -f docker-compose_dev.yml exec chris_store_dev python plugins/services/manager.py add "${plugin}" cubeadmin https://github.com/FNNDSC "$PLUGIN_DOCK" --descriptorstring "$PLUGIN_REP"
+        PLUGIN_REP=$(docker run --rm "$PLUGIN_DOCK" "${PLUGIN_MODULE}.py" --json 2> /dev/null;)
+        docker-compose -f docker-compose_dev.yml exec chris_store_dev python plugins/services/manager.py add "${plugin}" cubeadmin https://github.com/FNNDSC "$PLUGIN_DOCK" --descriptorstring "$PLUGIN_REP" >/dev/null
         ((i++))
     done
     windowBottom
@@ -103,9 +105,9 @@ if [[ "$1" == 'up' ]]; then
     PIPELINE_NAME="s3retrieve_v${S3_PLUGIN_VER}-simpledsapp_v${SIMPLEDS_PLUGIN_VER}"
     echo "Creating pipeline named '$PIPELINE_NAME'"
 
-    STR1='[{"plugin_name": "s3retrieve", "plugin_version": "'
+    STR1='[{"plugin_name": "pl-s3retrieve", "plugin_version": "'
     STR2='", "plugin_parameter_defaults": [{"name": "awssecretkey", "default": "somekey"},{"name": "awskeyid", "default": "somekeyid"}], "previous_index": null},
-    {"plugin_name": "simpledsapp", "plugin_version": "'
+    {"plugin_name": "pl-simpledsapp", "plugin_version": "'
     STR3='", "previous_index": 0}]'
     PLUGIN_TREE=${STR1}${S3_PLUGIN_VER}${STR2}${SIMPLEDS_PLUGIN_VER}${STR3}
 
@@ -116,9 +118,9 @@ if [[ "$1" == 'up' ]]; then
     PIPELINE_NAME="simpledsapp_v${SIMPLEDS_PLUGIN_VER}-simpledsapp_v${SIMPLEDS_PLUGIN_VER}-simpledsapp_v${SIMPLEDS_PLUGIN_VER}"
     echo "Creating pipeline named '$PIPELINE_NAME'"
 
-    STR4='[{"plugin_name": "simpledsapp", "plugin_version": "'
-    STR5='", "previous_index": null},{"plugin_name": "simpledsapp", "plugin_version": "'
-    STR6='", "previous_index": 0},{"plugin_name": "simpledsapp", "plugin_version": "'
+    STR4='[{"plugin_name": "pl-simpledsapp", "plugin_version": "'
+    STR5='", "previous_index": null},{"plugin_name": "pl-simpledsapp", "plugin_version": "'
+    STR6='", "previous_index": 0},{"plugin_name": "pl-simpledsapp", "plugin_version": "'
     STR7='", "previous_index": 0}]'
     PLUGIN_TREE=${STR4}${SIMPLEDS_PLUGIN_VER}${STR5}${SIMPLEDS_PLUGIN_VER}${STR6}${SIMPLEDS_PLUGIN_VER}${STR7}
 

diff --git a/store_backend/collectionjson/tests/test_parsers.py b/store_backend/collectionjson/tests/test_parsers.py
@@ -1,5 +1,6 @@
 
 import json
+import logging
 
 from django.conf.urls import url, include
 from django.test.utils import override_settings
@@ -16,8 +17,14 @@ class SimplePOSTTest(TestCase):
     endpoint = ''
 
     def setUp(self):
+        # avoid cluttered console output (for instance logging all the http requests)
+        logging.disable(logging.WARNING)
         self.content_type='application/vnd.collection+json'
 
+    def tearDown(self):
+        # re-enable logging
+        logging.disable(logging.NOTSET)
+
 
 class TestCollectionJsonParser(SimplePOSTTest):
     endpoint = '/rest-api/moron/'

diff --git a/store_backend/collectionjson/tests/test_renderers.py b/store_backend/collectionjson/tests/test_renderers.py
@@ -27,15 +27,15 @@ class SimpleGetTest(TestCase):
 
     def setUp(self):
         # avoid cluttered console output (for instance logging all the http requests)
-        logging.disable(logging.CRITICAL)
+        logging.disable(logging.WARNING)
         self.response = self.client.get(self.endpoint)
         content = json.loads(self.response.content.decode('utf8'))
         self.total = content['collection'].pop('total', None)  # remove the non-standard 'total' property
         self.collection = Collection.from_json(json.dumps(content))
 
     def tearDown(self):
         # re-enable logging
-        logging.disable(logging.DEBUG)
+        logging.disable(logging.NOTSET)
 
 
 class TestCollectionJsonRenderer(SimpleGetTest):

diff --git a/store_backend/collectionjson/tests/test_services.py b/store_backend/collectionjson/tests/test_services.py
@@ -18,12 +18,12 @@ class SimpleGetTest(TestCase):
 
     def setUp(self):
         # avoid cluttered console output (for instance logging all the http requests)
-        logging.disable(logging.CRITICAL)
+        logging.disable(logging.WARNING)
         self.response = self.client.get(self.endpoint)
 
     def tearDown(self):
         # re-enable logging
-        logging.disable(logging.DEBUG)
+        logging.disable(logging.NOTSET)
 
 
 class FunctionTests(SimpleGetTest):

diff --git a/store_backend/config/settings/local.py b/store_backend/config/settings/local.py
@@ -23,6 +23,38 @@
 # SECURITY WARNING: don't run with debug turned on in production!
 DEBUG = True
 
+# LOGGING CONFIGURATION
+# See http://docs.djangoproject.com/en/2.2/topics/logging for
+# more details on how to customize your logging configuration.
+LOGGING = {
+    'version': 1,
+    'disable_existing_loggers': False,
+    'formatters': {
+        'console': {
+            'format': '[%(asctime)s] [%(levelname)s][%(name)s][%(filename)s:%(lineno)d %(funcName)s] %(message)s'
+        },
+    },
+    'handlers': {
+        'console': {
+            'level': 'DEBUG',
+            'class': 'logging.StreamHandler',
+            'formatter': 'console',
+        },
+    },
+    'loggers': {
+        '': {  # root logger
+            'level': 'INFO',
+            'handlers': ['console'],
+        }
+    }
+}
+for app in ['collectionjson', 'plugins', 'pipelines', 'users']:
+    LOGGING['loggers'][app] = {
+            'level': 'DEBUG',
+            'handlers': ['console'],
+            'propagate': False  # required to avoid double logging with root logger
+        }
+
 # swift service settings
 DEFAULT_FILE_STORAGE = 'swift.storage.SwiftStorage'
 SWIFT_AUTH_URL = 'http://swift_service_dev:8080/auth/v1.0'

diff --git a/store_backend/config/settings/production.py b/store_backend/config/settings/production.py
@@ -61,53 +61,33 @@ def get_secret(setting, secret_type=env):
 
 
 # LOGGING CONFIGURATION
-# ------------------------------------------------------------------------------
-# See: https://docs.djangoproject.com/en/dev/ref/settings/#logging
-# A sample logging configuration. The only tangible logging
-# performed by this configuration is to send an email to
-# the site admins on every HTTP 500 error when DEBUG=False.
-# See http://docs.djangoproject.com/en/dev/topics/logging for
+# See http://docs.djangoproject.com/en/2.2/topics/logging for
 # more details on how to customize your logging configuration.
+ADMINS = [('FNNDSC Developers', '[email protected]')]
 LOGGING = {
     'version': 1,
     'disable_existing_loggers': False,
-    'filters': {
-        'require_debug_false': {
-            '()': 'django.utils.log.RequireDebugFalse'
-        }
-    },
     'formatters': {
-        'verbose': {
-            'format': '%(levelname)s %(asctime)s %(module)s '
-                      '%(process)d %(thread)d %(message)s'
+        'console': {
+            'format': '[%(levelname)s][%(name)s][%(filename)s:%(lineno)d %(funcName)s] %(message)s'
         },
     },
     'handlers': {
-        'mail_admins': {
-            'level': 'ERROR',
-            'filters': ['require_debug_false'],
-            'class': 'django.utils.log.AdminEmailHandler'
-        },
         'console': {
-            'level': 'DEBUG',
+            'level': 'INFO',
             'class': 'logging.StreamHandler',
-            'formatter': 'verbose',
+            'formatter': 'console',
         },
     },
     'loggers': {
-        'django.request': {
-            'handlers': ['mail_admins'],
-            'level': 'ERROR',
-            'propagate': True
-        },
-        'django.security.DisallowedHost': {
-            'level': 'ERROR',
-            'handlers': ['console', 'mail_admins'],
-            'propagate': True
+        '': {  # root logger
+            'level': 'INFO',
+            'handlers': ['console'],
         }
     }
 }
 
+
 # Your production stuff: Below this line define 3rd party library settings
 
 # CORSHEADERS

diff --git a/store_backend/pipelines/tests/test_manager.py b/store_backend/pipelines/tests/test_manager.py
@@ -14,7 +14,7 @@ class PipelineManagerTests(TestCase):
 
     def setUp(self):
         # avoid cluttered console output (for instance logging all the http requests)
-        logging.disable(logging.CRITICAL)
+        logging.disable(logging.WARNING)
 
         self.plugin_ds_name = "simpledsapp"
         self.plugin_ds_version = "0.1"
@@ -53,7 +53,7 @@ def setUp(self):
 
     def tearDown(self):
         # re-enable logging
-        logging.disable(logging.DEBUG)
+        logging.disable(logging.NOTSET)
 
     def test_mananger_can_add_pipeline(self):
         """

diff --git a/store_backend/pipelines/tests/test_models.py b/store_backend/pipelines/tests/test_models.py
@@ -15,7 +15,7 @@ class ModelTests(TestCase):
 
     def setUp(self):
         # avoid cluttered console output (for instance logging all the http requests)
-        logging.disable(logging.CRITICAL)
+        logging.disable(logging.WARNING)
 
         self.plugin_ds_name = "simpledsapp"
         self.plugin_ds_parameters = {'prefix': {'type': 'string', 'optional': False}}
@@ -60,7 +60,7 @@ def setUp(self):
 
     def tearDown(self):
         # re-enable logging
-        logging.disable(logging.DEBUG)
+        logging.disable(logging.NOTSET)
 
 
 class PipelineModelTests(ModelTests):

diff --git a/store_backend/pipelines/tests/test_serializers.py b/store_backend/pipelines/tests/test_serializers.py
@@ -17,7 +17,7 @@ class SerializerTests(TestCase):
 
     def setUp(self):
         # avoid cluttered console output (for instance logging all the http requests)
-        logging.disable(logging.CRITICAL)
+        logging.disable(logging.WARNING)
 
         self.plugin_fs_name = "simplefsapp"
         self.plugin_fs_parameters = {'dir': {'type': 'string', 'optional': True,
@@ -64,7 +64,7 @@ def setUp(self):
 
     def tearDown(self):
         # re-enable logging
-        logging.disable(logging.DEBUG)
+        logging.disable(logging.NOTSET)
 
 
 class PipelineSerializerTests(SerializerTests):

diff --git a/store_backend/pipelines/tests/test_views.py b/store_backend/pipelines/tests/test_views.py
@@ -19,7 +19,7 @@ class ViewTests(TestCase):
 
     def setUp(self):
         # avoid cluttered console output (for instance logging all the http requests)
-        logging.disable(logging.CRITICAL)
+        logging.disable(logging.WARNING)
 
         self.content_type = 'application/vnd.collection+json'
 
@@ -68,7 +68,7 @@ def setUp(self):
 
     def tearDown(self):
         # re-enable logging
-        logging.disable(logging.DEBUG)
+        logging.disable(logging.NOTSET)
 
 
 class PipelineViewTests(ViewTests):

diff --git a/store_backend/plugins/tests/test_manager.py b/store_backend/plugins/tests/test_manager.py
@@ -15,7 +15,7 @@ class PluginManagerTests(TestCase):
 
     def setUp(self):
         # avoid cluttered console output (for instance logging all the http requests)
-        logging.disable(logging.CRITICAL)
+        logging.disable(logging.WARNING)
 
         self.username = 'foo'
         self.password = 'foopassword'
@@ -73,7 +73,7 @@ def setUp(self):
 
     def tearDown(self):
         # re-enable logging
-        logging.disable(logging.DEBUG)
+        logging.disable(logging.NOTSET)
 
     def test_mananger_can_add_plugin(self):
         """

diff --git a/store_backend/plugins/tests/test_models.py b/store_backend/plugins/tests/test_models.py
@@ -1,4 +1,6 @@
 
+import logging
+
 from django.test import TestCase
 from django.contrib.auth.models import User
 
@@ -9,6 +11,9 @@
 class ModelTests(TestCase):
 
     def setUp(self):
+        # avoid cluttered console output (for instance logging all the http requests)
+        logging.disable(logging.WARNING)
+
         self.username = 'foo'
         self.password = 'foopassword'
         self.email = '[email protected]'
@@ -44,6 +49,10 @@ def setUp(self):
         plugin.descriptor_file.name = self.plugin_name + '.json'
         plugin.save()
 
+    def tearDown(self):
+        # re-enable logging
+        logging.disable(logging.NOTSET)
+
 
 class PluginMetaModelTests(ModelTests):