Merge pull request #70 from unfoldingWord-dev/develop

Pull latest tx-manager develop branch into master

aws_tools/dynamodb_handler.py

@@ -73,9 +73,24 @@ def delete_item(self, keys):
             Key=keys
         )
 
-    def query_items(self, query=None, only_fields_with_values=True):
+    def get_item_count(self):
+        """
+        get number of items in table - one caveat is that this value may be off since AWS only updates it every 6 hours
+        :return: 
+        """
+        return self.table.item_count
+
+    def query_items(self, query=None, only_fields_with_values=True, queryChunkLimit=-1):
+        """
+        gets items from database
+        :param query: 
+        :param only_fields_with_values: 
+        :param queryChunkLimit: not an absolute count, but a threshold where we stop fetching more chunks
+                        (if negative then no limit, but will read all chunks)
+        :return: 
+        """
         filter_expression = None
-        if query and len(query) > 1:
+        if query and len(query) >= 1:
             for field, value in iteritems(query):
                 value2 = None
                 if isinstance(value, dict) and 'condition' in value and 'value' in value:
@@ -124,11 +139,33 @@ def query_items(self, query=None, only_fields_with_values=True):
         else:
             response = self.table.scan()
 
-        if response and 'Items' in response:
-            return response['Items']
-        else:
+        if not response or not('Items' in response):
             return None
 
+        # finished if there is no more data to read
+        if not('LastEvaluatedKey' in response):
+            return response['Items']
+
+        items = response['Items']
+
+        # read chunks until end or threshold is reached
+        while 'LastEvaluatedKey' in response:
+            if filter_expression is not None:
+                response = self.table.scan(
+                    FilterExpression=filter_expression,
+                    ExclusiveStartKey = response['LastEvaluatedKey']
+                )
+            else:
+                response = self.table.scan(ExclusiveStartKey = response['LastEvaluatedKey'])
+
+            if response and ('Items' in response):
+                items += response['Items']
+
+            itemCount = len(items)
+            if (queryChunkLimit >= 0) and (itemCount >= queryChunkLimit):
+                break
+
+        return items
 
 RESERVED_WORDS = [
     'ABORT',

door43_tools/language_handler.py

@@ -4,6 +4,9 @@
 
 
 class Language(object):
+
+    language_list = None
+
     def __init__(self, json_obj=None):
         """
         Optionally accepts an object for initialization.
@@ -27,11 +30,18 @@ def __init__(self, json_obj=None):
 
     @staticmethod
     def load_languages():
-        return_val = []
+        """
+        Gets the list of Languages. Retrieves the list from tD if needed.
+        :return: list<Language>
+        """
+
+        if Language.language_list is None:
+
+            Language.language_list = []
 
-        lang_file = 'http://td.unfoldingword.org/exports/langnames.json'
-        langs = json.loads(get_url(lang_file))
-        for lang in langs:
-            return_val.append(Language(lang))
+            lang_file = 'http://td.unfoldingword.org/exports/langnames.json'
+            langs = json.loads(get_url(lang_file))
+            for lang in langs:
+                Language.language_list.append(Language(lang))
 
-        return return_val
+        return Language.language_list

door43_tools/templaters.py

@@ -4,6 +4,7 @@
 import codecs
 from glob import glob
 from bs4 import BeautifulSoup
+from door43_tools.language_handler import Language
 from general_tools.file_utils import write_file
 from door43_tools.manifest_handler import Manifest
 
@@ -70,7 +71,7 @@ def build_page_nav(self, filename=None):
             if title == "Conversion requested..." or title == "Conversion successful" or title == "Index":
                 continue
             if filename != fname:
-                html += '<li><a href="{0}">{1}</a></li>'.format(os.path.basename(fname),title)
+                html += '<li><a href="{0}">{1}</a></li>'.format(os.path.basename(fname), title)
             else:
                 html += '<li>{0}</li>'.format(title)
         html += """
@@ -136,6 +137,12 @@ def apply_template(self):
             outer_content_div.clear()
             outer_content_div.append(body)
             soup.html['lang'] = language_code
+
+            languages = Language.load_languages()
+            language = [lang for lang in languages if lang.lc == language_code]
+            if language and language[0].ld:
+                soup.html['dir'] = language[0].ld
+
             soup.head.title.clear()
             soup.head.title.append(heading+' - '+title)
 

lambda_handlers/dashboard_handler.py

@@ -24,4 +24,13 @@ def _handle(self, event, context):
             'job_table_name': self.retrieve(event['vars'], 'job_table_name', 'Environment Vars'),
             'module_table_name': self.retrieve(event['vars'], 'module_table_name', 'Environment Vars')
         }
-        return TxManager(**env_vars).generate_dashboard()
+
+        max_failures = TxManager.MAX_FAILURES
+        try:
+            querystring = event['api-gateway']['params']['querystring']
+            max_failures = int(querystring['failures'])
+
+        except:
+            pass
+
+        return TxManager(**env_vars).generate_dashboard(max_failures)

manager/manager.py

@@ -11,10 +11,10 @@
 from job import TxJob
 from module import TxModule
 
-
 class TxManager(object):
     JOB_TABLE_NAME = 'tx-job'
     MODULE_TABLE_NAME = 'tx-module'
+    MAX_FAILURES = 10
 
     def __init__(self, api_url=None, gogs_url=None, cdn_url=None, cdn_bucket=None, quiet=False,
                  aws_access_key_id=None, aws_secret_access_key=None,
@@ -151,6 +151,13 @@ def setup_job(self, data):
             ],
         }
 
+    def get_job_count(self):
+        """
+        get number of jobs in database - one caveat is that this value may be off since AWS only updates it every 6 hours
+        :return: 
+        """
+        return self.job_db_handler.get_item_count()
+
     def list_jobs(self, data, must_be_authenticated=True):
         if must_be_authenticated:
             if 'gogs_user_token' not in data:
@@ -452,7 +459,7 @@ def update_module(self, module):
     def delete_module(self, module):
         return self.module_db_handler.delete_item({'name': module.name})
 
-    def generate_dashboard(self):
+    def generate_dashboard(self, max_failures = MAX_FAILURES):
         """
         Generate page with metrics indicating configuration of tx-manager.
 
@@ -469,12 +476,21 @@ def generate_dashboard(self):
         }
 
         items = sorted(self.module_db_handler.query_items(), key=lambda k: k['name'])
-        totalJobs = self.list_jobs({},False)
-
         if items and len(items):
+            moduleNames = []
+            for item in items:
+                moduleNames.append(item["name"])
+
+            registeredJobs = self.list_jobs({ "convert_module" : { "condition" : "is_in", "value" : moduleNames}
+                                    }, False)
+            totalJobCount = self.get_job_count()
+            registeredJobCount = len(registeredJobs)
+            if registeredJobCount > totalJobCount: # sanity check since AWS can be slow to update job count reported in table (every 6 hours)
+                totalJobCount = registeredJobCount
+
             self.logger.info("  Found: " + str(len(items)) + " item[s] in tx-module")
 
-            body = BeautifulSoup('<h1>TX-Manager Dashboard</h1><h2>Module Attributes</h2><br><table></table>',
+            body = BeautifulSoup('<h1>TX-Manager Dashboard</h1><h2>Module Attributes</h2><br><table id="status"></table>',
                                  'html.parser')
             for item in items:
                 # self.logger.info(json.dumps(item))
@@ -484,7 +500,7 @@ def generate_dashboard(self):
                     '<tr id="' + moduleName + '"><td class="hdr" colspan="2">' + str(moduleName) + '</td></tr>',
                     'html.parser'))
 
-                jobs = self.get_jobs_for_module(totalJobs, moduleName)
+                jobs = self.get_jobs_for_module(registeredJobs, moduleName)
                 self.get_jobs_counts(jobs)
 
                 # TBD the following code almosts walks the db record replacing next 11 lines
@@ -549,7 +565,7 @@ def generate_dashboard(self):
                     str(self.jobs_total) + '</td></tr>',
                     'html.parser'))
 
-            self.get_jobs_counts(totalJobs)
+            self.get_jobs_counts(registeredJobs)
             body.table.append(BeautifulSoup(
                 '<tr id="totals"><td class="hdr" colspan="2">Total Jobs</td></tr>',
                 'html.parser'))
@@ -565,11 +581,65 @@ def generate_dashboard(self):
                 '<tr id="totals-job-failure" class="module-public-links"><td class="lbl">Failures:</td><td>' +
                 str(self.jobs_failures) + '</td></tr>',
                 'html.parser'))
+            body.table.append(BeautifulSoup(
+                '<tr id="totals-job-unregistered" class="module-public-links"><td class="lbl">Unregistered:</td><td>' +
+                str(totalJobCount - self.jobs_total) + '</td></tr>',
+                'html.parser'))
             body.table.append(BeautifulSoup(
                 '<tr id="totals-job-total" class="module-public-links"><td class="lbl">Total:</td><td>' +
-                str(self.jobs_total) + '</td></tr>',
+                str(totalJobCount) + '</td></tr>',
+                'html.parser'))
+
+            # build job failures table
+
+            jobFailures = self.get_job_failures(registeredJobs)
+            body.append(BeautifulSoup('<h2>Failed Jobs</h2>', 'html.parser'))
+            failureTable = BeautifulSoup('<table id="failed" cellpadding="4" border="1" style="border-collapse:collapse"></table>','html.parser')
+            failureTable.table.append(BeautifulSoup(
+                '<tr id="header">'
+                '<th class="hdr">Time</th>'
+                '<th class="hdr">Errors</th>'
+                '<th class="hdr">Repo</th>'
+                '<th class="hdr">PreConvert</th>'
+                '<th class="hdr">Converted</th>'
+                '<th class="hdr">Destination</th>'
+                '<th class="hdr">Job ID</th></tr>',
                 'html.parser'))
 
+            gogs_url = self.gogs_url
+            if gogs_url == None :
+                gogs_url = 'https://git.door43.org'
+
+            for i in range(0, max_failures):
+                if i >= len(jobFailures):
+                    break
+
+                item = jobFailures[i]
+
+                try :
+                    identifier = item['identifier']
+                    owner_name, repo_name, commit_id = identifier.split('/')
+                    sourceSubPath = 'u/{0}/{1}'.format(owner_name, repo_name)
+                    cdn_bucket = item['cdn_bucket']
+                    destinationUrl = 'https://{0}/u/{1}/{2}/{3}/build_log.json'.format(cdn_bucket, owner_name, repo_name, commit_id)
+                    repoUrl = gogs_url + "/" + sourceSubPath
+                    preconvertedUrl = item['source']
+                    convertedUrl = item['output']
+                    failureTable.table.append(BeautifulSoup(
+                        '<tr id="failure-' + str(i) + '" class="module-job-id">'
+                        + '<td>' + item['created_at'] + '</td>'
+                        + '<td>' + str(item['errors']) + '</td>'
+                        + '<td><a href="' + repoUrl + '">' + repoUrl + '</a></td>'
+                        + '<td><a href="' + preconvertedUrl + '">' + preconvertedUrl + '</a></td>'
+                        + '<td><a href="' + convertedUrl + '">' + convertedUrl + '</a></td>'
+                        + '<td><a href="' + destinationUrl + '">' + destinationUrl + '</a></td>'
+                        + '<td>' + item['job_id'] + '</td>'
+                        + '</tr>',
+                        'html.parser'))
+                except:
+                    pass
+
+            body.append(failureTable)
             dashboard['body'] = body.prettify('UTF-8')
         else:
             self.logger.info("No modules found.")
@@ -592,20 +662,26 @@ def get_jobs_counts(self, jobs):
         self.jobs_failures = 0
         self.jobs_success = 0
         for job in jobs:
-            try:
-                errors = job['errors']
-                if len(errors) > 0:
-                    self.jobs_failures+=1
-                    continue
+            errors = job['errors']
+            if len(errors) > 0:
+                self.jobs_failures+=1
+                continue
 
-                warnings = job['warnings']
-                if len(warnings) > 0:
-                    self.jobs_warnings+=1
-                    continue
+            warnings = job['warnings']
+            if len(warnings) > 0:
+                self.jobs_warnings+=1
+                continue
 
-                self.jobs_success+=1
+            self.jobs_success+=1
 
-            except:
-                self.jobs_failures+=1
 
+    def get_job_failures(self, jobs):
+        failedJobs = []
+        for job in jobs:
+            errors = job['errors']
+            if len(errors) > 0:
+                failedJobs.append(job)
+
+        failedJobs = sorted(failedJobs, key=lambda k: k['created_at'], reverse=True)
+        return failedJobs
 

setup.py

@@ -15,7 +15,7 @@ def read(f_name):
 
 setup(
     name='tx-manager',
-    version='0.2.62',
+    version='0.2.63',
     packages=[
         'client',
         'manager',

test-setup.py

@@ -2,7 +2,7 @@
 
 setup(
     name='tx-manager',
-    version='0.2.62',
+    version='0.2.63',
     packages=[
         'client',
         'manager',

tests/door43_tools_tests/test_templaters.py

@@ -1,8 +1,10 @@
 from __future__ import absolute_import, unicode_literals, print_function
+import codecs
 import os
 import tempfile
 import unittest
 import shutil
+from bs4 import BeautifulSoup
 from door43_tools.templaters import Templater
 from general_tools.file_utils import unzip
 
@@ -62,6 +64,18 @@ def testCommitToDoor43MissingChapter50(self):
         deployer, success = self.doTemplater(test_file_path)
         self.verifyObsTemplater(success, expect_success, deployer.output_dir, missing_chapters)
 
+    def testTemplaterRightToLeft(self):
+        test_folder_name = os.path.join('converted_projects', 'glk_obs_text_obs-complete.zip')
+        test_file_path = self.extractZipFiles(test_folder_name)
+        deployer, success = self.doTemplater(test_file_path)
+
+        # check for dir attribute in html tag
+        with codecs.open(os.path.join(deployer.output_dir, '01.html'), 'r', 'utf-8-sig') as f:
+            soup = BeautifulSoup(f, 'html.parser')
+
+        self.assertIn('dir', soup.html.attrs)
+        self.assertEqual('rtl', soup.html.attrs['dir'])
+
     def extractZipFiles(self, test_folder_name):
         file_path = os.path.join(self.resources_dir, test_folder_name)
         self.temp_dir = tempfile.mkdtemp(prefix='repo_')