Skip to content
This repository has been archived by the owner on May 22, 2024. It is now read-only.

Commit

Permalink
Issue 431: add job counts to dashboard (#63)
Browse files Browse the repository at this point in the history 
* TxManager - added job counts per module and total. DashboardTest - unit test for TxManager.generate_dashboard()

* bumped version number

* test_manager.py - fixed test_generate_dashboard to account for job counts.  Removed dashboard_tests.

* test_manager.py - improved test_generate_dashboard to verify job counts.  manager.py - tweaked ids of totals for easy validation.

* TxManager.generate_dashboard - code cleanup.

* TxManager.generate_dashboard - code cleanup.

* ManagerTest - fixed error count for test_start_job1 to reflect changes I made to mock test data.
  • Loading branch information
@PhotoNomad0
PhotoNomad0 authored and Phil Hopper committed May 11, 2017
1 parent 4faeef2 commit 410ead3
Show file tree
Hide file tree
Showing 4 changed files with 167 additions and 25 deletions.
98 changes: 88 additions & 10 deletions manager/manager.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -469,6 +469,7 @@ def generate_dashboard(self):
}

items = sorted(self.module_db_handler.query_items(), key=lambda k: k['name'])
totalJobs = self.list_jobs({},False)

if items and len(items):
self.logger.info(" Found: " + str(len(items)) + " item[s] in tx-module")
Expand All @@ -477,57 +478,134 @@ def generate_dashboard(self):
'html.parser')
for item in items:
# self.logger.info(json.dumps(item))
self.logger.info(item["name"])
moduleName = item["name"]
self.logger.info(moduleName)
body.table.append(BeautifulSoup(
'<tr id="' + item['name'] + '"><td class="hdr" colspan="2">' + str(item["name"]) + '</td></tr>',
'<tr id="' + moduleName + '"><td class="hdr" colspan="2">' + str(moduleName) + '</td></tr>',
'html.parser'))

jobs = self.get_jobs_for_module(totalJobs, moduleName)
self.get_jobs_counts(jobs)

# TBD the following code almosts walks the db record replacing next 11 lines
# for attr, val in item:
# if (attr != 'name') and (len(attr) > 0):
# rec += ' <tr><td class="lbl">' + attr.replace("_", " ").title() + ':</td><td>' + "lst(val)" + "</td></tr>\n"
# rec += '<tr><td colspan="2"></td></tr>'

body.table.append(BeautifulSoup(
'<tr id="' + item['name'] + '-type" class="module-type"><td class="lbl">Type:</td><td>' +
'<tr id="' + moduleName + '-type" class="module-type"><td class="lbl">Type:</td><td>' +
str(item["type"]) + '</td></tr>',
'html.parser'))
body.table.append(BeautifulSoup(
'<tr id="' + item['name'] + '-input" class="module-input"><td class="lbl">Input Format:</td><td>' +
'<tr id="' + moduleName + '-input" class="module-input"><td class="lbl">Input Format:</td><td>' +
json.dumps(item["input_format"]) + '</td></tr>',
'html.parser'))
body.table.append(BeautifulSoup(
'<tr id="' + item['name'] + '-output" class="module-output"><td class="lbl">Output Format:</td><td>' +
'<tr id="' + moduleName + '-output" class="module-output"><td class="lbl">Output Format:</td><td>' +
json.dumps(item["output_format"]) + '</td></tr>',
'html.parser'))
body.table.append(BeautifulSoup(
'<tr id="' + item['name'] + '-resource" class="module-resource"><td class="lbl">Resource Types:</td><td>' +
'<tr id="' + moduleName + '-resource" class="module-resource"><td class="lbl">Resource Types:</td><td>' +
json.dumps(item["resource_types"]) + '</td></tr>',
'html.parser'))
body.table.append(BeautifulSoup(
'<tr id="' + item['name'] + '-version" class="module-version"><td class="lbl">Version:</td><td>' +
'<tr id="' + moduleName + '-version" class="module-version"><td class="lbl">Version:</td><td>' +
str(item["version"]) + '</td></tr>',
'html.parser'))

if len(item["options"]) > 0:
body.table.append(BeautifulSoup(
'<tr id="' + item['name'] + '-options" class="module-options"><td class="lbl">Options:</td><td>' +
'<tr id="' + moduleName + '-options" class="module-options"><td class="lbl">Options:</td><td>' +
json.dumps(item["options"]) + '</td></tr>',
'html.parser'))

if len(item["private_links"]) > 0:
body.table.append(BeautifulSoup(
'<tr id="' + item['name'] + '-private-links" class="module-private-links"><td class="lbl">Private Links:</td><td>' +
'<tr id="' + moduleName + '-private-links" class="module-private-links"><td class="lbl">Private Links:</td><td>' +
json.dumps(item["private_links"]) + '</td></tr>',
'html.parser'))

if len(item["public_links"]) > 0:
body.table.append(BeautifulSoup(
'<tr id="' + item['name'] + '-public-links" class="module-public-links"><td class="lbl">Public Links:</td><td>' +
'<tr id="' + moduleName + '-public-links" class="module-public-links"><td class="lbl">Public Links:</td><td>' +
json.dumps(item["public_links"]) + '</td></tr>',
'html.parser'))

body.table.append(BeautifulSoup(
'<tr id="' + moduleName + '-job-success" class="module-public-links"><td class="lbl">Job Successes:</td><td>' +
str(self.jobs_success) + '</td></tr>',
'html.parser'))
body.table.append(BeautifulSoup(
'<tr id="' + moduleName + '-job-warning" class="module-public-links"><td class="lbl">Job Warnings:</td><td>' +
str(self.jobs_warnings) + '</td></tr>',
'html.parser'))
body.table.append(BeautifulSoup(
'<tr id="' + moduleName + '-job-failure" class="module-public-links"><td class="lbl">Job Failures:</td><td>' +
str(self.jobs_failures) + '</td></tr>',
'html.parser'))
body.table.append(BeautifulSoup(
'<tr id="' + moduleName + '-job-total" class="module-public-links"><td class="lbl">Jobs Total:</td><td>' +
str(self.jobs_total) + '</td></tr>',
'html.parser'))

self.get_jobs_counts(totalJobs)
body.table.append(BeautifulSoup(
'<tr id="totals"><td class="hdr" colspan="2">Total Jobs</td></tr>',
'html.parser'))
body.table.append(BeautifulSoup(
'<tr id="totals-job-success" class="module-public-links"><td class="lbl">Success:</td><td>' +
str(self.jobs_success) + '</td></tr>',
'html.parser'))
body.table.append(BeautifulSoup(
'<tr id="totals-job-warning" class="module-public-links"><td class="lbl">Warnings:</td><td>' +
str(self.jobs_warnings) + '</td></tr>',
'html.parser'))
body.table.append(BeautifulSoup(
'<tr id="totals-job-failure" class="module-public-links"><td class="lbl">Failures:</td><td>' +
str(self.jobs_failures) + '</td></tr>',
'html.parser'))
body.table.append(BeautifulSoup(
'<tr id="totals-job-total" class="module-public-links"><td class="lbl">Total:</td><td>' +
str(self.jobs_total) + '</td></tr>',
'html.parser'))

dashboard['body'] = body.prettify('UTF-8')
else:
self.logger.info("No modules found.")

return dashboard

def get_jobs_for_module(self, jobs, moduleName):
jobsInModule = []
for job in jobs:
if "convert_module" in job:
name = job["convert_module"]
if name == moduleName:
jobsInModule.append(job)

return jobsInModule

def get_jobs_counts(self, jobs):
self.jobs_total = len(jobs)
self.jobs_warnings = 0
self.jobs_failures = 0
self.jobs_success = 0
for job in jobs:
try:
errors = job['errors']
if len(errors) > 0:
self.jobs_failures+=1
continue

warnings = job['warnings']
if len(warnings) > 0:
self.jobs_warnings+=1
continue

self.jobs_success+=1

except:
self.jobs_failures+=1


2 changes: 1 addition & 1 deletion setup.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ def read(f_name):

setup(
name='tx-manager',
version='0.2.61',
version='0.2.62',
packages=[
'client',
'manager',
Expand Down
2 changes: 1 addition & 1 deletion test-setup.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

setup(
name='tx-manager',
version='0.2.61',
version='0.2.62',
packages=[
'client',
'manager',
Expand Down
90 changes: 77 additions & 13 deletions tests/manager_tests/test_manager.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,14 +46,18 @@ def setUpClass(cls):
"status": "started",
"resource_type": "obs",
"input_format": "md",
"output_format": "html"
"output_format": "html",
"convert_module": "module1",
"errors" : []
},
"1": {
"job_id": "1",
"status": "requested",
"resource_type": "obs",
"input_format": "md",
"output_format": "html"
"output_format": "html",
"convert_module": "module1",
"errors" : [ "error" ]
},
"2": {
"job_id": "2",
Expand All @@ -62,34 +66,42 @@ def setUpClass(cls):
"input_format": "usfm",
"output_format": "html",
"callback": ManagerTest.MOCK_CALLBACK_URL,
"convert_module": "module1",
"warnings" : []
},
"3": {
"job_id": "3",
"status": "requested",
"resource_type": "other",
"input_format": "md",
"output_format": "html"
"output_format": "html",
"convert_module": "module1",
"warnings" : [ "warning" ]
},
"4": {
"job_id": "4",
"status": "requested",
"resource_type": "unsupported",
"input_format": "md",
"output_format": "html"
"output_format": "html",
"convert_module": "module1",
"warnings" : [ "warning1", "warning2" ]
},
"6": {
"job_id": "6",
"status": "requested",
"resource_type": "obs",
"input_format": "md",
"output_format": "html"
"output_format": "html",
"convert_module": "module2"
},
"7": {
"job_id": "7",
"status": "requested",
"resource_type": "obs",
"input_format": "md",
"output_format": "html"
"output_format": "html",
"convert_module": "module2"
}
}, keyname="job_id")
cls.mock_module_db = mock_utils.mock_db_handler(data={
Expand Down Expand Up @@ -312,7 +324,7 @@ def test_start_job1(self, mock_requests_post):
data = args[1]
self.assertIsInstance(data, dict)
self.assertIn("errors", data)
self.assertEqual(len(data["errors"]), 0)
self.assertEqual(len(data["errors"]), 1)
self.assertIn("warnings", data)
self.assertTrue(len(data["warnings"]) > 0)

Expand Down Expand Up @@ -581,15 +593,67 @@ def test_generate_dashboard(self):
soup = BeautifulSoup(dashboard['body'], 'html.parser')
# there should be a table tag
self.assertIsNotNone(soup.find('table'))
# module1 should have 8 rows of info
self.assertEquals(len(soup.table.findAll('tr', id=lambda x: x and x.startswith('module1-'))), 8)
# module2 should have 7 rows of info
self.assertEquals(len(soup.table.findAll('tr', id=lambda x: x and x.startswith('module2-'))), 7)
# module3 should have 5 rows of info
self.assertEquals(len(soup.table.findAll('tr', id=lambda x: x and x.startswith('module3-'))), 5)

moduleName = 'module1'
expectedRowCount = 12
expectedSuccessCount = 2
expectedWarningCount = 2
expectedFailureCount = 1
self.validateModule(soup, moduleName, expectedRowCount, expectedSuccessCount, expectedFailureCount,
expectedWarningCount)

moduleName = 'module2'
expectedRowCount = 11
expectedSuccessCount = 2
expectedWarningCount = 0
expectedFailureCount = 0
self.validateModule(soup, moduleName, expectedRowCount, expectedSuccessCount, expectedFailureCount,
expectedWarningCount)

moduleName = 'module3'
expectedRowCount = 9
expectedSuccessCount = 0
expectedWarningCount = 0
expectedFailureCount = 0
self.validateModule(soup, moduleName, expectedRowCount, expectedSuccessCount, expectedFailureCount,
expectedWarningCount)

moduleName = 'totals'
expectedRowCount = 4
expectedSuccessCount = 4
expectedWarningCount = 2
expectedFailureCount = 1
self.validateModule(soup, moduleName, expectedRowCount, expectedSuccessCount, expectedFailureCount,
expectedWarningCount)

# helper methods #

def validateModule(self, soup, moduleName, expectedRowCount, expectedSuccessCount, expectedFailureCount,
expectedWarningCount):
module = soup.table.findAll('tr', id=lambda x: x and x.startswith(moduleName + '-'))
rowCount = len(module)
self.assertEquals(rowCount, expectedRowCount)
successCount = self.getCountFromRow(soup, moduleName + '-job-success')
self.assertEquals(successCount, expectedSuccessCount)
warningCount = self.getCountFromRow(soup, moduleName + '-job-warning')
self.assertEquals(warningCount, expectedWarningCount)
failureCount = self.getCountFromRow(soup, moduleName + '-job-failure')
self.assertEquals(failureCount, expectedFailureCount)
expectedTotalCount = expectedFailureCount + expectedSuccessCount + expectedWarningCount
totalCount = self.getCountFromRow(soup, moduleName + '-job-total')
self.assertEquals(totalCount, expectedTotalCount)

def getCountFromRow(self, soup, rowID):
success = soup.table.findAll('tr', id=lambda x: x == rowID)
dataFields = success[0].findAll("td")
strings = dataFields[1].stripped_strings # get data from second column
count = -1
for string in strings:
count = int(string)
break

return count

def call_args(self, mock_object, num_args, num_kwargs=0):
"""
:param mock_object: mock object that is expected to have been called
Expand Down

0 comments on commit 410ead3

Please sign in to comment.