Issue 429 Add list of 10 most recent FAILED builds to status page (#67)

* TxManager - added job counts per module and total. DashboardTest - unit test for TxManager.generate_dashboard()

* bumped version number

* test_manager.py - fixed test_generate_dashboard to account for job counts.  Removed dashboard_tests.

* test_manager.py - improved test_generate_dashboard to verify job counts.  manager.py - tweaked ids of totals for easy validation.

* TxManager.generate_dashboard - get list of failed jobs.

* TxManager.generate_dashboard - First pass with failure table entries. ManagerTest.test_generate_dashboard() - added error table validation

* TxManager.generate_dashboard - Added more failure table entries.

* TxManager.generate_dashboard - code cleanup.

* TxManager.generate_dashboard - code cleanup.

* TxManager.generate_dashboard - code cleanup.

* TxManager.generate_dashboard - code cleanup.

* TxManager.generate_dashboard - put in guesstimated links to source and destination.

* TxManager.generate_dashboard - tweaks to table format.

* TxManager.generate_dashboard - tweaks to table format.

* ManagerTest.test_generate_dashboard() - init gogs_url. TxManager.generate_dashboard - tweaks for testing.

* TxManager.generate_dashboard - default value for self.gogs_url.

* ManagerTest.test_generate_dashboard() - changed to use output field for destination. TxManager.generate_dashboard - tweaks for testing.

* DashboardHandler - added support for max failures count.  ManagerTest.test_generate_dashboard() - added max_failures parameter. test_dashboardHandler.py - expanded thoroughness of test stealing from manager_tests.

* test_dashboardHandler.py - fixing imports to get to run on travis.

* test_dashboardHandler.py - fix path to query string.

* test_dashboardHandler.py - fix path to query string.

* test_dashboardHandler.py - narrowed down tests to just test that max_failures is getting set properly.

* generate_dashboard() - Now has links to source repo, preconverted, converted, and destination build log. ManagerTest - added test_generate_dashboard_max_two to test max failures parameter.

* DashboardHandlerTest - fix for unit testing on Travis.

* DynamoDBHandler.query_items() - fix for when return data exceeds 1MB.  added exclusive_start_key parameter and detection when query returns partial data.
TxManager.query_jobs() - added fix to for when only partial data is returned.  Continues to query until all data fetched.
TxManager.generate_dashboard() - added check for unregistered modules in jobs.

* DynamoDBHandler.query_items() - return last_key as tuple.
mock_db_handler.query_items() - return last_key (None) as tuple.

* DynamoDBHandlerTests - fix for query_items() to handle last_key tuple.

* TxManager - tweak missing modules.

* TxManager - tweaks to improve code coverage.  Added more jobs to test_manager.

* Tweaked test_manager to reflect higher job count.

* Tweaked test_manager to reflect higher job count.

* Tweaked test_manager to reflect higher job count.

* DynamoDBHandler.query_items() - removed last_key in return value and modified grab all chunks.  Also added a query limit.

* DynamoDBHandler.query_items() - fixed problem that when only one query item, it gets ignored.  TxManager.generate_dashboard() - no loads only supported modules.

* DynamoDBHandler - added get_item_count().  TxManager - added get_job_count(), generate_dashboard() - now only loads jobs for supported modules, added unregistered job count.

* TxManager - generate_dashboard() - fix for issue that AWS table size is only updated every 6 hours.

aws_tools/dynamodb_handler.py

@@ -73,9 +73,24 @@ def delete_item(self, keys):
             Key=keys
         )
 
-    def query_items(self, query=None, only_fields_with_values=True):
+    def get_item_count(self):
+        """
+        get number of items in table - one caveat is that this value may be off since AWS only updates it every 6 hours
+        :return: 
+        """
+        return self.table.item_count
+
+    def query_items(self, query=None, only_fields_with_values=True, queryChunkLimit=-1):
+        """
+        gets items from database
+        :param query: 
+        :param only_fields_with_values: 
+        :param queryChunkLimit: not an absolute count, but a threshold where we stop fetching more chunks
+                        (if negative then no limit, but will read all chunks)
+        :return: 
+        """
         filter_expression = None
-        if query and len(query) > 1:
+        if query and len(query) >= 1:
             for field, value in iteritems(query):
                 value2 = None
                 if isinstance(value, dict) and 'condition' in value and 'value' in value:
@@ -124,11 +139,33 @@ def query_items(self, query=None, only_fields_with_values=True):
         else:
             response = self.table.scan()
 
-        if response and 'Items' in response:
-            return response['Items']
-        else:
+        if not response or not('Items' in response):
             return None
 
+        # finished if there is no more data to read
+        if not('LastEvaluatedKey' in response):
+            return response['Items']
+
+        items = response['Items']
+
+        # read chunks until end or threshold is reached
+        while 'LastEvaluatedKey' in response:
+            if filter_expression is not None:
+                response = self.table.scan(
+                    FilterExpression=filter_expression,
+                    ExclusiveStartKey = response['LastEvaluatedKey']
+                )
+            else:
+                response = self.table.scan(ExclusiveStartKey = response['LastEvaluatedKey'])
+
+            if response and ('Items' in response):
+                items += response['Items']
+
+            itemCount = len(items)
+            if (queryChunkLimit >= 0) and (itemCount >= queryChunkLimit):
+                break
+
+        return items
 
 RESERVED_WORDS = [
     'ABORT',

lambda_handlers/dashboard_handler.py

@@ -24,4 +24,13 @@ def _handle(self, event, context):
             'job_table_name': self.retrieve(event['vars'], 'job_table_name', 'Environment Vars'),
             'module_table_name': self.retrieve(event['vars'], 'module_table_name', 'Environment Vars')
         }
-        return TxManager(**env_vars).generate_dashboard()
+
+        max_failures = TxManager.MAX_FAILURES
+        try:
+            querystring = event['api-gateway']['params']['querystring']
+            max_failures = int(querystring['failures'])
+
+        except:
+            pass
+
+        return TxManager(**env_vars).generate_dashboard(max_failures)

manager/manager.py

@@ -11,10 +11,10 @@
 from job import TxJob
 from module import TxModule
 
-
 class TxManager(object):
     JOB_TABLE_NAME = 'tx-job'
     MODULE_TABLE_NAME = 'tx-module'
+    MAX_FAILURES = 10
 
     def __init__(self, api_url=None, gogs_url=None, cdn_url=None, cdn_bucket=None, quiet=False,
                  aws_access_key_id=None, aws_secret_access_key=None,
@@ -151,6 +151,13 @@ def setup_job(self, data):
             ],
         }
 
+    def get_job_count(self):
+        """
+        get number of jobs in database - one caveat is that this value may be off since AWS only updates it every 6 hours
+        :return: 
+        """
+        return self.job_db_handler.get_item_count()
+
     def list_jobs(self, data, must_be_authenticated=True):
         if must_be_authenticated:
             if 'gogs_user_token' not in data:
@@ -452,7 +459,7 @@ def update_module(self, module):
     def delete_module(self, module):
         return self.module_db_handler.delete_item({'name': module.name})
 
-    def generate_dashboard(self):
+    def generate_dashboard(self, max_failures = MAX_FAILURES):
         """
         Generate page with metrics indicating configuration of tx-manager.
 
@@ -469,12 +476,21 @@ def generate_dashboard(self):
         }
 
         items = sorted(self.module_db_handler.query_items(), key=lambda k: k['name'])
-        totalJobs = self.list_jobs({},False)
-
         if items and len(items):
+            moduleNames = []
+            for item in items:
+                moduleNames.append(item["name"])
+
+            registeredJobs = self.list_jobs({ "convert_module" : { "condition" : "is_in", "value" : moduleNames}
+                                    }, False)
+            totalJobCount = self.get_job_count()
+            registeredJobCount = len(registeredJobs)
+            if registeredJobCount > totalJobCount: # sanity check since AWS can be slow to update job count reported in table (every 6 hours)
+                totalJobCount = registeredJobCount
+
             self.logger.info("  Found: " + str(len(items)) + " item[s] in tx-module")
 
-            body = BeautifulSoup('<h1>TX-Manager Dashboard</h1><h2>Module Attributes</h2><br><table></table>',
+            body = BeautifulSoup('<h1>TX-Manager Dashboard</h1><h2>Module Attributes</h2><br><table id="status"></table>',
                                  'html.parser')
             for item in items:
                 # self.logger.info(json.dumps(item))
@@ -484,7 +500,7 @@ def generate_dashboard(self):
                     '<tr id="' + moduleName + '"><td class="hdr" colspan="2">' + str(moduleName) + '</td></tr>',
                     'html.parser'))
 
-                jobs = self.get_jobs_for_module(totalJobs, moduleName)
+                jobs = self.get_jobs_for_module(registeredJobs, moduleName)
                 self.get_jobs_counts(jobs)
 
                 # TBD the following code almosts walks the db record replacing next 11 lines
@@ -549,7 +565,7 @@ def generate_dashboard(self):
                     str(self.jobs_total) + '</td></tr>',
                     'html.parser'))
 
-            self.get_jobs_counts(totalJobs)
+            self.get_jobs_counts(registeredJobs)
             body.table.append(BeautifulSoup(
                 '<tr id="totals"><td class="hdr" colspan="2">Total Jobs</td></tr>',
                 'html.parser'))
@@ -565,11 +581,65 @@ def generate_dashboard(self):
                 '<tr id="totals-job-failure" class="module-public-links"><td class="lbl">Failures:</td><td>' +
                 str(self.jobs_failures) + '</td></tr>',
                 'html.parser'))
+            body.table.append(BeautifulSoup(
+                '<tr id="totals-job-unregistered" class="module-public-links"><td class="lbl">Unregistered:</td><td>' +
+                str(totalJobCount - self.jobs_total) + '</td></tr>',
+                'html.parser'))
             body.table.append(BeautifulSoup(
                 '<tr id="totals-job-total" class="module-public-links"><td class="lbl">Total:</td><td>' +
-                str(self.jobs_total) + '</td></tr>',
+                str(totalJobCount) + '</td></tr>',
+                'html.parser'))
+
+            # build job failures table
+
+            jobFailures = self.get_job_failures(registeredJobs)
+            body.append(BeautifulSoup('<h2>Failed Jobs</h2>', 'html.parser'))
+            failureTable = BeautifulSoup('<table id="failed" cellpadding="4" border="1" style="border-collapse:collapse"></table>','html.parser')
+            failureTable.table.append(BeautifulSoup(
+                '<tr id="header">'
+                '<th class="hdr">Time</th>'
+                '<th class="hdr">Errors</th>'
+                '<th class="hdr">Repo</th>'
+                '<th class="hdr">PreConvert</th>'
+                '<th class="hdr">Converted</th>'
+                '<th class="hdr">Destination</th>'
+                '<th class="hdr">Job ID</th></tr>',
                 'html.parser'))
 
+            gogs_url = self.gogs_url
+            if gogs_url == None :
+                gogs_url = 'https://git.door43.org'
+
+            for i in range(0, max_failures):
+                if i >= len(jobFailures):
+                    break
+
+                item = jobFailures[i]
+
+                try :
+                    identifier = item['identifier']
+                    owner_name, repo_name, commit_id = identifier.split('/')
+                    sourceSubPath = 'u/{0}/{1}'.format(owner_name, repo_name)
+                    cdn_bucket = item['cdn_bucket']
+                    destinationUrl = 'https://{0}/u/{1}/{2}/{3}/build_log.json'.format(cdn_bucket, owner_name, repo_name, commit_id)
+                    repoUrl = gogs_url + "/" + sourceSubPath
+                    preconvertedUrl = item['source']
+                    convertedUrl = item['output']
+                    failureTable.table.append(BeautifulSoup(
+                        '<tr id="failure-' + str(i) + '" class="module-job-id">'
+                        + '<td>' + item['created_at'] + '</td>'
+                        + '<td>' + str(item['errors']) + '</td>'
+                        + '<td><a href="' + repoUrl + '">' + repoUrl + '</a></td>'
+                        + '<td><a href="' + preconvertedUrl + '">' + preconvertedUrl + '</a></td>'
+                        + '<td><a href="' + convertedUrl + '">' + convertedUrl + '</a></td>'
+                        + '<td><a href="' + destinationUrl + '">' + destinationUrl + '</a></td>'
+                        + '<td>' + item['job_id'] + '</td>'
+                        + '</tr>',
+                        'html.parser'))
+                except:
+                    pass
+
+            body.append(failureTable)
             dashboard['body'] = body.prettify('UTF-8')
         else:
             self.logger.info("No modules found.")
@@ -592,20 +662,26 @@ def get_jobs_counts(self, jobs):
         self.jobs_failures = 0
         self.jobs_success = 0
         for job in jobs:
-            try:
-                errors = job['errors']
-                if len(errors) > 0:
-                    self.jobs_failures+=1
-                    continue
+            errors = job['errors']
+            if len(errors) > 0:
+                self.jobs_failures+=1
+                continue
 
-                warnings = job['warnings']
-                if len(warnings) > 0:
-                    self.jobs_warnings+=1
-                    continue
+            warnings = job['warnings']
+            if len(warnings) > 0:
+                self.jobs_warnings+=1
+                continue
 
-                self.jobs_success+=1
+            self.jobs_success+=1
 
-            except:
-                self.jobs_failures+=1
 
+    def get_job_failures(self, jobs):
+        failedJobs = []
+        for job in jobs:
+            errors = job['errors']
+            if len(errors) > 0:
+                failedJobs.append(job)
+
+        failedJobs = sorted(failedJobs, key=lambda k: k['created_at'], reverse=True)
+        return failedJobs
 

setup.py

@@ -15,7 +15,7 @@ def read(f_name):
 
 setup(
     name='tx-manager',
-    version='0.2.62',
+    version='0.2.63',
     packages=[
         'client',
         'manager',

test-setup.py

@@ -2,7 +2,7 @@
 
 setup(
     name='tx-manager',
-    version='0.2.62',
+    version='0.2.63',
     packages=[
         'client',
         'manager',

tests/lambda_handlers_tests/test_dashboardHandler.py

@@ -2,14 +2,18 @@
 import mock
 from unittest import TestCase
 from lambda_handlers.dashboard_handler import DashboardHandler
+from manager.manager import TxManager
 
+def new_generate_dashboard( max_failures):
+    return max_failures; # return the parameter for testing
 
-class TestListJobsHandler(TestCase):
+class DashboardHandlerTest(TestCase):
 
     @mock.patch('manager.manager.TxManager.setup_resources')
     @mock.patch('manager.manager.TxManager.generate_dashboard')
     def test_handle(self, mock_generate_dashboard, mock_setup_resources):
-        mock_generate_dashboard.return_value = None
+        mock_generate_dashboard.side_effect=new_generate_dashboard
+        expectedMaxFailures = TxManager.MAX_FAILURES
         event = {
             'data': {},
             'body-json': {},
@@ -23,5 +27,33 @@ def test_handle(self, mock_generate_dashboard, mock_setup_resources):
             }
         }
         handler = DashboardHandler()
-        self.assertIsNone(handler.handle(event, None))
+        maxFailures = handler.handle(event, expectedMaxFailures)
+        self.assertEqual(maxFailures, expectedMaxFailures)
+
+    @mock.patch('manager.manager.TxManager.setup_resources')
+    @mock.patch('manager.manager.TxManager.generate_dashboard')
+    def test_dashboard_handler_max_two(self, mock_generate_dashboard, mock_setup_resources):
+        mock_generate_dashboard.side_effect=new_generate_dashboard
+        expectedMaxFailures = 2
+        event = {
+            "vars" : {
+                'data': {},
+                'body-json': {},
+                'api_url': 'https://test-api.door43.org',
+                'gogs_url': 'https://git.door43.org',
+                'cdn_url': 'https://test-cdn.door43.org',
+                'job_table_name': 'test-tx-job',
+                'module_table_name': 'test-tx-module'
+            },
+            "api-gateway" : {
+                "params" : {
+                    'querystring': {
+                        'failures': str(expectedMaxFailures)
+                    }
+                }
+            }
+        }
+        handler = DashboardHandler()
+        maxFailures = handler.handle(event, expectedMaxFailures)
+        self.assertEqual(maxFailures, expectedMaxFailures)
 

tests/manager_tests/mock_utils.py

@@ -19,12 +19,16 @@ def get_item(keys):
             return data[key]
         return None
 
-    def query_items(*ignored):
+    def get_item_count():
+        return len(data.values())
+
+    def query_items(query=None, only_fields_with_values=True, queryChunkLimit=-1):
         return data.values()
 
     handler = MagicMock()
     handler.get_item = MagicMock(side_effect=get_item)
     handler.query_items = MagicMock(side_effect=query_items)
+    handler.get_item_count = MagicMock(side_effect=get_item_count)
     handler.setup_resources = MagicMock(side_effects=setup_resources)
     handler.mock_data = data
     return handler