Skip to content

Commit

Permalink
Added timeout and limit to jira_controller.py
Browse files Browse the repository at this point in the history
  • Loading branch information
@blupants
blupants committed Jun 20, 2024
1 parent c9ab687 commit 3c98271
Show file tree
Hide file tree
Showing 2 changed files with 65 additions and 32 deletions.
35 changes: 15 additions & 20 deletions src/n0s1/controllers/confluence_controller.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -124,42 +124,38 @@ def get_data(self, include_coments=False, limit=None):
if not self._client:
return None, None, None, None, None

start = 0
space_start = 0
if not limit or limit < 0:
limit = 50
space_limit = limit
finished = False
while not finished:
try:
res = self._client.get_all_spaces(start=start, limit=space_limit)
res = self._client.get_all_spaces(start=space_start, limit=limit)
spaces = res.get("results", [])
except Exception as e:
self.log_message(str(e), logging.WARNING)
message = str(e) + f" get_all_spaces(start={space_start}, limit={limit})"
self.log_message(message, logging.WARNING)
spaces = [{}]
time.sleep(1)
continue

start = space_limit
space_limit += start
space_start += limit

for s in spaces:
key = s.get("key", "")
self.log_message(f"Scanning Confluence space: [{key}]...")
if len(key) > 0:
pages_start = 0
pages_limit = limit
pages_finished = False
while not pages_finished:
try:
pages = self._client.get_all_pages_from_space(key, start=pages_start, limit=pages_limit)
pages = self._client.get_all_pages_from_space(key, start=pages_start, limit=limit)
except Exception as e:
self.log_message(str(e), logging.WARNING)
message = str(e) + f" get_all_pages_from_space({key}, start={pages_start}, limit={limit})"
self.log_message(message, logging.WARNING)
pages = [{}]
time.sleep(1)
continue

pages_start = pages_limit
pages_limit += pages_start
pages_start += limit

for p in pages:
comments = []
Expand All @@ -168,7 +164,8 @@ def get_data(self, include_coments=False, limit=None):
try:
body = self._client.get_page_by_id(page_id, expand="body.storage")
except Exception as e:
self.log_message(str(e), logging.WARNING)
message = str(e) + f" get_page_by_id({page_id})"
self.log_message(message, logging.WARNING)
body = {}
time.sleep(1)
continue
Expand All @@ -177,20 +174,18 @@ def get_data(self, include_coments=False, limit=None):
url = body.get("_links", {}).get("base", "") + p.get("_links", {}).get("webui", "")
if len(page_id) > 0 and include_coments:
comments_start = 0
comments_limit = limit
comments_finished = False
while not comments_finished:
try:
comments_response = self._client.get_page_comments(page_id, expand="body.storage", start=comments_start, limit=comments_limit)
comments_response = self._client.get_page_comments(page_id, expand="body.storage", start=comments_start, limit=limit)
comments_result = comments_response.get("results", [])
except Exception as e:
self.log_message(str(e), logging.WARNING)
message = str(e) + f" get_page_comments({page_id}, expand=\"body.storage\", start={comments_start}, limit={limit})"
self.log_message(message, logging.WARNING)
comments_result = [{}]
time.sleep(1)
continue

comments_start = comments_limit
comments_limit += comments_start
comments_start += limit

for c in comments_result:
comment = c.get("body", {}).get("storage", {}).get("value", "")
Expand Down
62 changes: 50 additions & 12 deletions src/n0s1/controllers/jira_controller.py
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import logging
import time

try:
from . import hollow_controller as hollow_controller
Expand All @@ -16,10 +17,17 @@ def set_config(self, config):
SERVER = config.get("server", "")
EMAIL = config.get("email", "")
TOKEN = config.get("token", "")
TIMEOUT = config.get("timeout", -1)
if EMAIL and len(EMAIL) > 0:
self._client = JIRA(SERVER, basic_auth=(EMAIL, TOKEN))
if TIMEOUT and TIMEOUT > 0:
self._client = JIRA(SERVER, basic_auth=(EMAIL, TOKEN), timeout=TIMEOUT)
else:
self._client = JIRA(SERVER, basic_auth=(EMAIL, TOKEN))
else:
self._client = JIRA(SERVER, token_auth=TOKEN)
if TIMEOUT and TIMEOUT > 0:
self._client = JIRA(SERVER, token_auth=TOKEN, timeout=TIMEOUT)
else:
self._client = JIRA(SERVER, token_auth=TOKEN)
return self.is_connected()

def get_name(self):
Expand Down Expand Up @@ -57,18 +65,48 @@ def is_connected(self):
def get_data(self, include_coments=False, limit=None):
if not self._client:
return None, None, None, None, None
for key in self._client.projects():
start = 0
if not limit or limit < 0:
limit = 50
try:
projects = self._client.projects()
except Exception as e:
message = str(e) + f" client.projects()"
self.log_message(message, logging.WARNING)
projects = []

for key in projects:
ql = f"project = '{key}'"
self.log_message(f"Scanning Jira project: [{key}]...")
for issue in self._client.search_issues(ql):
url = issue.self.split('/rest/api')[0] + "/browse/" + issue.key;
title = issue.fields.summary
description = issue.fields.description
comments = []
if include_coments:
issue_comments = self._client.comments(issue.id)
comments.extend(c.body for c in issue_comments)
yield title, description, comments, url, issue.key
issues_finished = False
issue_start = start
while not issues_finished:
try:
issues = self._client.search_issues(ql, startAt=issue_start, maxResults=limit)
except Exception as e:
message = str(e) + f" client.search_issues({ql}, startAt={issue_start}, maxResults={limit})"
self.log_message(message, logging.WARNING)
issues = [{}]
time.sleep(1)
continue
issue_start += limit
issues_finished = len(issues) <= 0
for issue in issues:
url = issue.self.split('/rest/api')[0] + "/browse/" + issue.key;
title = issue.fields.summary
description = issue.fields.description
comments = []
if include_coments:
try:
issue_comments = self._client.comments(issue.id)
comments.extend(c.body for c in issue_comments)
except Exception as e:
message = str(e) + f" client.comments({issue.id})"
self.log_message(message, logging.WARNING)
comments = []
time.sleep(1)

yield title, description, comments, url, issue.key

def post_comment(self, issue, comment):
if not self._client:
Expand Down

0 comments on commit 3c98271

Please sign in to comment.