Merge pull request #368 from ianco/master

Tweak for active and inactive relationship dates from LEAR

data-pipeline/bcreg/bcreg_core.py

@@ -643,6 +643,7 @@ def get_adhoc_query(self, sql, use_sec=False):
     lear_other_other_tables = ['parties',
                           'parties_version',]
     lear_other_other_other_tables = ['transaction',]
+    lear_other_other_other_other_tables = ['filings',]
 
     # in colin if the corp_num is numeric we pre-pend 'BC' to it
     def bc_ifiy_one(self, corp_num):
@@ -726,6 +727,10 @@ def cache_lear_bcreg_corp_tables(self, specific_corps, generate_individual_sql=F
                 for other_table in self.lear_other_other_other_tables:
                     _rows = self.get_bcreg_table(other_table, txn_id_where, '', True, generate_individual_sql, use_sec=use_sec)
 
+                txn_id_where = 'transaction_id in (' + self.id_where_in(txn_ids_list, True) + ')' if 0 < len(txn_ids_list) else "transaction_id = 0"
+                for other_table in self.lear_other_other_other_other_tables:
+                    _rows = self.get_bcreg_table(other_table, txn_id_where, '', True, generate_individual_sql, use_sec=use_sec)
+
     # load all bc registries data for the specified corps into our in-mem cache
     def cache_lear_bcreg_code_tables(self, generate_individual_sql=False, use_sec=False):
         if self.use_local_cache():

data-pipeline/bcreg/bcreg_lear.py

@@ -25,12 +25,14 @@
 
 MIN_START_DATE = datetime.datetime(datetime.MINYEAR+1, 1, 1)
 MAX_END_DATE   = datetime.datetime(datetime.MAXYEAR-1, 12, 31)
+LEAR_CONVERSION_DATE = datetime.datetime(2022, 10, 22)
 
 # for now, we are in PST time
 timezone = pytz.timezone("PST8PDT")
 
 MIN_START_DATE_TZ = timezone.localize(MIN_START_DATE)
 MAX_END_DATE_TZ   = timezone.localize(MAX_END_DATE)
+LEAR_CONVERSION_DATE_TZ = timezone.localize(LEAR_CONVERSION_DATE)
 
 LOGGER = logging.getLogger(__name__)
 
@@ -371,7 +373,7 @@ def get_unprocessed_corps(self, last_event_id, last_event_dt):
                        party_roles_version roles
                   WHERE businesses.id = roles.business_id
                     AND roles.party_id = parties.id 
-                    AND parties.party_type = 'organization' and roles.role = 'proprietor'
+                    AND parties.party_type = 'organization' and roles.role in ('proprietor','partner')
                     AND (parties.identifier = %s
                          OR roles.business_id = (select id from businesses where identifier = %s))"""
         new_corps = []
@@ -799,6 +801,7 @@ def get_lear_relationship_info(self, corp_info):
                                parties.transaction_id as transaction_id,
                                parties.end_transaction_id as end_transaction_id,
                                roles.cessation_date as cessation_dt,
+                               roles.appointment_date as appointment_dt,
                                parties.last_name as last_nme, 
                                parties.middle_initial as middle_nme, 
                                parties.first_name as first_nme, 
@@ -814,7 +817,7 @@ def get_lear_relationship_info(self, corp_info):
                            """ + self.get_table_prefix() + """party_roles_version roles
                       WHERE businesses.id = roles.business_id
                         AND roles.party_id = parties.id 
-                        AND parties.party_type = 'organization' and roles.role = 'proprietor'
+                        AND parties.party_type = 'organization' and roles.role in ('proprietor','partner')
                         AND (parties.identifier = """ + self.get_db_sql_param() + """ 
                              OR roles.business_id = (select id from """ + self.get_table_prefix() + """businesses where identifier = """ + self.get_db_sql_param() + """))
                         """
@@ -851,27 +854,28 @@ def get_lear_relationship_info(self, corp_info):
                     corp_party['effective_end_date'] = MAX_END_DATE
                 corp_party['cessation_dt'] = row[7]
                 # use "cessation_date" as the end date (if available), regardless of the transaction status
-                corp_party['last_nme'] = row[8]
-                corp_party['middle_nme'] = row[9]
-                corp_party['first_nme'] = row[10]
-                corp_party['business_nme'] = row[11]
-                corp_party['bus_company_num'] = row[12]
-                corp_party['email_address'] = row[13]
+                corp_party['appointment_dt'] = row[8]
+                corp_party['last_nme'] = row[9]
+                corp_party['middle_nme'] = row[10]
+                corp_party['first_nme'] = row[11]
+                corp_party['business_nme'] = row[12]
+                corp_party['bus_company_num'] = row[13]
+                corp_party['email_address'] = row[14]
                 corp_party['corp_party_seq_num'] = None
                 corp_party['office_notification_dt'] = None
                 corp_party['phone'] = None
                 corp_party['reason_typ_cd'] = None
-                corp_party['role_id'] = row[14]
-                corp_party['role'] = row[15]
-                role_transaction_id = row[16]
+                corp_party['role_id'] = row[15]
+                corp_party['role'] = row[16]
+                role_transaction_id = row[17]
                 corp_party['role_transaction_id'] = role_transaction_id
                 if role_transaction_id and 0 < role_transaction_id:
                     role_transaction = self.get_event(corp_party['corp_num'], role_transaction_id)
                     corp_party['role_transaction'] = role_transaction
                 else:
                     corp_party['role_transaction'] = {}
                 corp_party['role_effective_start_date'] = self.get_corp_version_effective_date(corp_party, txn_field='role_transaction', filing_field=None)
-                role_end_transaction_id = row[17]
+                role_end_transaction_id = row[18]
                 corp_party['role_end_transaction_id'] = role_end_transaction_id
                 if role_end_transaction_id and 0 < role_end_transaction_id:
                     role_end_transaction = self.get_event(corp_party['corp_num'], role_end_transaction_id)

data-pipeline/bcreg/eventprocessor.py

@@ -15,7 +15,7 @@
 from bcreg.config import config
 from bcreg.bcregistries import BCRegistries, CustomJsonEncoder, event_dict, is_data_conversion_event, system_type, CORP_TYPES_IN_SCOPE
 from bcreg.bcreg_core import CORP_WITHDRAWN_STATE
-from bcreg.bcreg_lear import BCReg_Lear, lear_system_type, LEAR_CORP_TYPES_IN_SCOPE
+from bcreg.bcreg_lear import BCReg_Lear, lear_system_type, LEAR_CORP_TYPES_IN_SCOPE, LEAR_CONVERSION_DATE_TZ
 from bcreg.rocketchat_hooks import log_error, log_warning, log_info
 
 
@@ -1312,15 +1312,15 @@ def generate_relationship_creds(self, corp_info, prev_event, last_event, corp_nu
                     dba_cred['relationship_description'] = 'TBD' # party['']
                     dba_cred['associated_registration_name'] = ''
                 dba_cred['relationship_status'] = 'ACT'
-                dba_cred['effective_date'] = party['effective_start_date']
+                dba_cred['effective_date'] = party['appointment_dt']
 
                 # if the start event is 'ADMIN' type and there is only one party record, use the firm effective date
                 # if party['start_event']['event_typ_cd'] == 'ADMIN' and party_count[party['corp_info']['corp_num']] == 1 and party['corp_info']['recognition_dts']:
                 #     dba_cred['effective_date'] = party['corp_info']['recognition_dts']
 
                 dba_cred['relationship_status_effective'] = self.filter_min_date(dba_cred['effective_date'])
-                if party['end_transaction_id'] is not None and self.compare_dates(party['end_transaction']['effective_date'], "<=", corp_info['current_date'], "Relationships"):
-                    dba_cred['expiry_date'] = party['effective_end_date']
+                if party['end_transaction_id'] is not None and self.compare_dates(party['end_transaction']['effective_date'], "<=", LEAR_CONVERSION_DATE_TZ, "Relationships"):
+                    dba_cred['expiry_date'] = party['end_transaction']['effective_date']
                 elif party['cessation_dt'] is not None:
                     # scenario where the "cessation_dt" is the expiry date, with no transaction
                     dba_cred['expiry_date'] = party['cessation_dt']