diff --git a/src/n0s1/controllers/confluence_controller.py b/src/n0s1/controllers/confluence_controller.py index 8232784..2eeda8c 100644 --- a/src/n0s1/controllers/confluence_controller.py +++ b/src/n0s1/controllers/confluence_controller.py @@ -124,42 +124,38 @@ def get_data(self, include_coments=False, limit=None): if not self._client: return None, None, None, None, None - start = 0 + space_start = 0 if not limit or limit < 0: limit = 50 - space_limit = limit finished = False while not finished: try: - res = self._client.get_all_spaces(start=start, limit=space_limit) + res = self._client.get_all_spaces(start=space_start, limit=limit) spaces = res.get("results", []) except Exception as e: - self.log_message(str(e), logging.WARNING) + message = str(e) + f" get_all_spaces(start={space_start}, limit={limit})" + self.log_message(message, logging.WARNING) spaces = [{}] time.sleep(1) continue - - start = space_limit - space_limit += start + space_start += limit for s in spaces: key = s.get("key", "") self.log_message(f"Scanning Confluence space: [{key}]...") if len(key) > 0: pages_start = 0 - pages_limit = limit pages_finished = False while not pages_finished: try: - pages = self._client.get_all_pages_from_space(key, start=pages_start, limit=pages_limit) + pages = self._client.get_all_pages_from_space(key, start=pages_start, limit=limit) except Exception as e: - self.log_message(str(e), logging.WARNING) + message = str(e) + f" get_all_pages_from_space({key}, start={pages_start}, limit={limit})" + self.log_message(message, logging.WARNING) pages = [{}] time.sleep(1) continue - - pages_start = pages_limit - pages_limit += pages_start + pages_start += limit for p in pages: comments = [] @@ -168,7 +164,8 @@ def get_data(self, include_coments=False, limit=None): try: body = self._client.get_page_by_id(page_id, expand="body.storage") except Exception as e: - self.log_message(str(e), logging.WARNING) + message = str(e) + f" get_page_by_id({page_id})" + self.log_message(message, logging.WARNING) body = {} time.sleep(1) continue @@ -177,20 +174,18 @@ def get_data(self, include_coments=False, limit=None): url = body.get("_links", {}).get("base", "") + p.get("_links", {}).get("webui", "") if len(page_id) > 0 and include_coments: comments_start = 0 - comments_limit = limit comments_finished = False while not comments_finished: try: - comments_response = self._client.get_page_comments(page_id, expand="body.storage", start=comments_start, limit=comments_limit) + comments_response = self._client.get_page_comments(page_id, expand="body.storage", start=comments_start, limit=limit) comments_result = comments_response.get("results", []) except Exception as e: - self.log_message(str(e), logging.WARNING) + message = str(e) + f" get_page_comments({page_id}, expand=\"body.storage\", start={comments_start}, limit={limit})" + self.log_message(message, logging.WARNING) comments_result = [{}] time.sleep(1) continue - - comments_start = comments_limit - comments_limit += comments_start + comments_start += limit for c in comments_result: comment = c.get("body", {}).get("storage", {}).get("value", "") diff --git a/src/n0s1/controllers/jira_controller.py b/src/n0s1/controllers/jira_controller.py index 6bb2beb..fe21e36 100644 --- a/src/n0s1/controllers/jira_controller.py +++ b/src/n0s1/controllers/jira_controller.py @@ -1,4 +1,5 @@ import logging +import time try: from . import hollow_controller as hollow_controller @@ -16,10 +17,17 @@ def set_config(self, config): SERVER = config.get("server", "") EMAIL = config.get("email", "") TOKEN = config.get("token", "") + TIMEOUT = config.get("timeout", -1) if EMAIL and len(EMAIL) > 0: - self._client = JIRA(SERVER, basic_auth=(EMAIL, TOKEN)) + if TIMEOUT and TIMEOUT > 0: + self._client = JIRA(SERVER, basic_auth=(EMAIL, TOKEN), timeout=TIMEOUT) + else: + self._client = JIRA(SERVER, basic_auth=(EMAIL, TOKEN)) else: - self._client = JIRA(SERVER, token_auth=TOKEN) + if TIMEOUT and TIMEOUT > 0: + self._client = JIRA(SERVER, token_auth=TOKEN, timeout=TIMEOUT) + else: + self._client = JIRA(SERVER, token_auth=TOKEN) return self.is_connected() def get_name(self): @@ -57,18 +65,48 @@ def is_connected(self): def get_data(self, include_coments=False, limit=None): if not self._client: return None, None, None, None, None - for key in self._client.projects(): + start = 0 + if not limit or limit < 0: + limit = 50 + try: + projects = self._client.projects() + except Exception as e: + message = str(e) + f" client.projects()" + self.log_message(message, logging.WARNING) + projects = [] + + for key in projects: ql = f"project = '{key}'" self.log_message(f"Scanning Jira project: [{key}]...") - for issue in self._client.search_issues(ql): - url = issue.self.split('/rest/api')[0] + "/browse/" + issue.key; - title = issue.fields.summary - description = issue.fields.description - comments = [] - if include_coments: - issue_comments = self._client.comments(issue.id) - comments.extend(c.body for c in issue_comments) - yield title, description, comments, url, issue.key + issues_finished = False + issue_start = start + while not issues_finished: + try: + issues = self._client.search_issues(ql, startAt=issue_start, maxResults=limit) + except Exception as e: + message = str(e) + f" client.search_issues({ql}, startAt={issue_start}, maxResults={limit})" + self.log_message(message, logging.WARNING) + issues = [{}] + time.sleep(1) + continue + issue_start += limit + issues_finished = len(issues) <= 0 + for issue in issues: + url = issue.self.split('/rest/api')[0] + "/browse/" + issue.key; + title = issue.fields.summary + description = issue.fields.description + comments = [] + if include_coments: + try: + issue_comments = self._client.comments(issue.id) + comments.extend(c.body for c in issue_comments) + except Exception as e: + message = str(e) + f" client.comments({issue.id})" + self.log_message(message, logging.WARNING) + comments = [] + time.sleep(1) + + yield title, description, comments, url, issue.key def post_comment(self, issue, comment): if not self._client: