Merge pull request #73 from bcgov/feature/SRIS-20

SRIS-20

.github/workflows/ci-openshift-prod.yaml

@@ -68,6 +68,8 @@ jobs:
             AGOL_UPDATE_FLAG=${{ secrets.AGOL_UPDATE_FLAG }} \
             EMAIL_NOTIFY_FLAG=${{ secrets.EMAIL_NOTIFY_FLAG }} \
             TIMEOUT_SECONDS=${{ secrets.TIMEOUT_SECONDS }} \
+            CHEFS_API_TIMEOUT=${{ secrets.CHEFS_API_TIMEOUT }} \
+            CHES_API_TIMEOUT=${{ secrets.CHES_API_TIMEOUT }} \
   release:
     name: Release
     runs-on: ubuntu-22.04

.github/workflows/ci-openshift-test.yaml

@@ -64,3 +64,5 @@ jobs:
             AGOL_UPDATE_FLAG=${{ secrets.AGOL_UPDATE_FLAG }} \
             EMAIL_NOTIFY_FLAG=${{ secrets.EMAIL_NOTIFY_FLAG }} \
             TIMEOUT_SECONDS=${{ secrets.TIMEOUT_SECONDS }} \
+            CHEFS_API_TIMEOUT=${{ secrets.CHEFS_API_TIMEOUT }} \
+            CHES_API_TIMEOUT=${{ secrets.CHES_API_TIMEOUT }} \

helper.py

@@ -33,6 +33,16 @@
 AUTH_URL = os.getenv('AUTH_URL')
 CHES_URL = os.getenv('CHES_URL')
 LOGLEVEL = os.getenv('LOGLEVEL')
+CHEFS_API_TIMEOUT = os.getenv('CHEFS_API_TIMEOUT')
+CHES_API_TIMEOUT = os.getenv('CHES_API_TIMEOUT')
+
+# If there is no CHEFS_API_TIMEOUT, set it to 60 seconds (1 minutes)
+if not CHEFS_API_TIMEOUT:
+    CHEFS_API_TIMEOUT = '60'
+
+# If there is no CHES_API_TIMEOUT, set it to 60 seconds (1 minutes)
+if not CHES_API_TIMEOUT:
+    CHES_API_TIMEOUT = '60'
 
 def load_env():
     """Loading environment variables from .env files - for local testing"""
@@ -119,7 +129,8 @@ def get_ches_token():
             'Content-Type': 'application/x-www-form-urlencoded',
             'Authorization': 'Basic ' + CHES_API_OAUTH_SECRET
         }
-        _auth_response = requests.request("POST", AUTH_URL, headers=_auth_headers, data=_auth_pay_load, timeout=5) # timeout in seconds
+        ches_api_timeout = int(CHES_API_TIMEOUT)
+        _auth_response = requests.request("POST", AUTH_URL, headers=_auth_headers, data=_auth_pay_load, timeout=ches_api_timeout) # timeout in seconds
         _auth_response_json = json.loads(_auth_response.content)
         if _auth_response_json.get('access_token'):
             return _auth_response_json['access_token']
@@ -131,6 +142,8 @@ def get_ches_token():
         logging.exception("Email could not be sent due to an authorization issue:%s", _ke)
     except Timeout:
         logging.error('The request timed out to get CHES token! - %s', AUTH_URL)
+    except ValueError:
+        logging.error('Invalid timeout value: %s. Must be an int or float.', CHES_API_TIMEOUT)
     return _auth_response
 
 def check_ches_health():
@@ -142,7 +155,8 @@ def check_ches_health():
     }
     _ches_api_health_endpoint = CHES_URL + '/api/v1/health'
     try:
-        _ches_response = requests.request("GET", _ches_api_health_endpoint, headers=ches_headers, timeout=5) # timeout in seconds
+        ches_api_timeout = int(CHES_API_TIMEOUT)
+        _ches_response = requests.request("GET", _ches_api_health_endpoint, headers=ches_headers, timeout=ches_api_timeout) # timeout in seconds
         if _ches_response.status_code == 200:
             logging.info(constant.CHES_HEALTH_200_STATUS)
         elif _ches_response.status_code == 401:
@@ -153,6 +167,8 @@ def check_ches_health():
             logging.error("CHES Health returned staus code:%s, text:%s", str(_ches_response.status_code), _ches_response.text)
     except Timeout:
         logging.error('The request timed out to check CHES Health! - %s', _ches_api_health_endpoint)
+    except ValueError:
+        logging.error('Invalid timeout value: %s. Must be an int or float.', CHES_API_TIMEOUT)
 
 def send_single_email(to_email, subject, message):
     """Send email via CHES API"""
@@ -167,34 +183,42 @@ def send_single_email(to_email, subject, message):
         }
         _ches_api_single_email_endpoint = CHES_URL + '/api/v1/email'
         try:
-            _ches_response = requests.request("POST", _ches_api_single_email_endpoint, headers=ches_headers, data=ches_pay_load, timeout=5) # timeout in seconds
+            ches_api_timeout = int(CHES_API_TIMEOUT)
+            _ches_response = requests.request("POST", _ches_api_single_email_endpoint, headers=ches_headers, data=ches_pay_load, timeout=ches_api_timeout) # timeout in seconds
         except Timeout:
             logging.error('The request timed out to send email! - %s', _ches_api_single_email_endpoint)
+        except ValueError:
+            logging.error('Invalid timeout value: %s. Must be an int or float.', CHES_API_TIMEOUT)
     return _ches_response
 
-def site_list(form_id, form_key, form_version):
+def get_chefs_form_data(form_id, form_key, form_version):
     """Retrieve CHEFS form data via CHEFS API"""
     content = None
     chefs_api_request_url = CHEFS_API_URL + '/forms/' + form_id + '/export?format=json&type=submissions&version=' + form_version
     try:
-        request = requests.get(chefs_api_request_url, auth=HTTPBasicAuth(form_id, form_key), headers={'Content-type': 'application/json'}, timeout=5) # timeout in seconds
+        chefs_api_timeout = int(CHEFS_API_TIMEOUT)
+        request = requests.get(chefs_api_request_url, auth=HTTPBasicAuth(form_id, form_key), headers={'Content-type': 'application/json'}, timeout=chefs_api_timeout) # timeout in seconds
         content = json.loads(request.content)
     except Timeout:
         logging.error('The request timed out! %s', chefs_api_request_url)
+        os._exit(1)
+    except ValueError:
+        logging.error('Invalid timeout value: %s. Must be an int or float.', CHEFS_API_TIMEOUT)
+        os._exit(1)
     return content
 
-def fetch_columns(form_id, form_key):
-    """Retrieve CHEFS form columns"""
-    attributes = None
-    try:
-        request = requests.get(CHEFS_API_URL + '/forms/' + form_id + '/versions', auth=HTTPBasicAuth(form_id, form_key), headers={'Content-type': 'application/json'}, timeout=5) # timeout in seconds
-        request_json = json.loads(request.content)
-        version = request_json[0]['id']
-        attribute_request = requests.get(CHEFS_API_URL + '/forms/' + form_id + '/versions/' + version + '/fields', auth=HTTPBasicAuth(form_id, form_key), headers={'Content-type': 'application/json'}, timeout=5) # timeout in seconds
-        attributes = json.loads(attribute_request.content)
-    except Timeout:
-        logging.error('The request timed out to retrieve CHEFS form columns!')
-    return attributes
+# def fetch_columns(form_id, form_key):
+#     """Retrieve CHEFS form columns"""
+#     attributes = None
+#     try:
+#         request = requests.get(CHEFS_API_URL + '/forms/' + form_id + '/versions', auth=HTTPBasicAuth(form_id, form_key), headers={'Content-type': 'application/json'}, timeout=5) # timeout in seconds
+#         request_json = json.loads(request.content)
+#         version = request_json[0]['id']
+#         attribute_request = requests.get(CHEFS_API_URL + '/forms/' + form_id + '/versions/' + version + '/fields', auth=HTTPBasicAuth(form_id, form_key), headers={'Content-type': 'application/json'}, timeout=5) # timeout in seconds
+#         attributes = json.loads(attribute_request.content)
+#     except Timeout:
+#         logging.error('The request timed out to retrieve CHEFS form columns!')
+#     return attributes
 
 def get_difference_datetimes_in_hour(datetime1, datetime2):
     """Calculates and returns the difference between two given dates/times in hours"""
@@ -298,6 +322,8 @@ def chefs_rcv_param(key, rcv_clz):
     Returns receiver parameter name that matching in receiver dictionary
     according to receiver classfication(1st, 2nd, 3rd receiver)
     """
+    name = None
+
     if rcv_clz == 1: # 1st receiver
         name = constant.CHEFS_RCV1_PARAM_DIC.get(key)
     elif rcv_clz == 2: # 2nd receiver

openshift/sris-schedule-job.yml

@@ -97,6 +97,12 @@ parameters:
   - name: TIMEOUT_SECONDS
     description: Timeout to AGOL API call
     required: true
+  - name: CHEFS_API_TIMEOUT
+    description: Timeout to CHEFS API call
+    required: true
+  - name: CHES_API_TIMEOUT
+    description: Timeout to CHES API call
+    required: true
 objects:
   - kind: ConfigMap
     apiVersion: v1
@@ -132,6 +138,8 @@ objects:
       AGOL_UPDATE_FLAG: ${AGOL_UPDATE_FLAG}
       EMAIL_NOTIFY_FLAG: ${EMAIL_NOTIFY_FLAG}
       TIMEOUT_SECONDS: ${TIMEOUT_SECONDS}
+      CHEFS_API_TIMEOUT: ${CHEFS_API_TIMEOUT}
+      CHES_API_TIMEOUT: ${CHES_API_TIMEOUT}
 
   - apiVersion: batch/v1
     kind: CronJob

submission_loader.py

@@ -56,7 +56,7 @@ def load_submissions():
     if soils_form_versions:
         for version in soils_form_versions:
             submission_loaded = False
-            response = helper.site_list(CHEFS_SOILS_FORM_ID, CHEFS_SOILS_API_KEY, version)
+            response = helper.get_chefs_form_data(CHEFS_SOILS_FORM_ID, CHEFS_SOILS_API_KEY, version)
             if isinstance(response, dict) and int(response.get('status')) > 201:
                 logging.error("Loading Submissions List failed: %s %s, %s", response['status'], response['title'], response['detail'])
             elif response is None:
@@ -95,7 +95,7 @@ def load_submissions():
     if hv_form_versions:
         for version in hv_form_versions:
             hvs_loaded = False
-            response = helper.site_list(CHEFS_HV_FORM_ID, CHEFS_HV_API_KEY, version)
+            response = helper.get_chefs_form_data(CHEFS_HV_FORM_ID, CHEFS_HV_API_KEY, version)
             if isinstance(response, dict) and int(response.get('status')) > 201:
                 logging.error("Loading High Volume Sites list failed: %s %s, %s", response['status'], response['title'], response['detail'])
             elif response is None:
@@ -134,7 +134,7 @@ def load_submissions():
     if mail_form_versions:
         for version in mail_form_versions:
             subscribers_loaded = False
-            response = helper.site_list(CHEFS_MAIL_FORM_ID, CHEFS_MAIL_API_KEY, version)
+            response = helper.get_chefs_form_data(CHEFS_MAIL_FORM_ID, CHEFS_MAIL_API_KEY, version)
             if isinstance(response, dict) and int(response.get('status')) > 201:
                 logging.error("Loading submission subscribers list failed: %s %s, %s", response['status'], response['title'], response['detail'])
             elif response is None: