diff --git a/data-pipeline/bcreg/bcreg_core.py b/data-pipeline/bcreg/bcreg_core.py index 535fa0b..1ccc982 100644 --- a/data-pipeline/bcreg/bcreg_core.py +++ b/data-pipeline/bcreg/bcreg_core.py @@ -643,6 +643,7 @@ def get_adhoc_query(self, sql, use_sec=False): lear_other_other_tables = ['parties', 'parties_version',] lear_other_other_other_tables = ['transaction',] + lear_other_other_other_other_tables = ['filings',] # in colin if the corp_num is numeric we pre-pend 'BC' to it def bc_ifiy_one(self, corp_num): @@ -726,6 +727,10 @@ def cache_lear_bcreg_corp_tables(self, specific_corps, generate_individual_sql=F for other_table in self.lear_other_other_other_tables: _rows = self.get_bcreg_table(other_table, txn_id_where, '', True, generate_individual_sql, use_sec=use_sec) + txn_id_where = 'transaction_id in (' + self.id_where_in(txn_ids_list, True) + ')' if 0 < len(txn_ids_list) else "transaction_id = 0" + for other_table in self.lear_other_other_other_other_tables: + _rows = self.get_bcreg_table(other_table, txn_id_where, '', True, generate_individual_sql, use_sec=use_sec) + # load all bc registries data for the specified corps into our in-mem cache def cache_lear_bcreg_code_tables(self, generate_individual_sql=False, use_sec=False): if self.use_local_cache(): diff --git a/data-pipeline/bcreg/bcreg_lear.py b/data-pipeline/bcreg/bcreg_lear.py index 8d306f6..ec9ec06 100644 --- a/data-pipeline/bcreg/bcreg_lear.py +++ b/data-pipeline/bcreg/bcreg_lear.py @@ -25,12 +25,14 @@ MIN_START_DATE = datetime.datetime(datetime.MINYEAR+1, 1, 1) MAX_END_DATE = datetime.datetime(datetime.MAXYEAR-1, 12, 31) +LEAR_CONVERSION_DATE = datetime.datetime(2022, 10, 22) # for now, we are in PST time timezone = pytz.timezone("PST8PDT") MIN_START_DATE_TZ = timezone.localize(MIN_START_DATE) MAX_END_DATE_TZ = timezone.localize(MAX_END_DATE) +LEAR_CONVERSION_DATE_TZ = timezone.localize(LEAR_CONVERSION_DATE) LOGGER = logging.getLogger(__name__) @@ -371,7 +373,7 @@ def get_unprocessed_corps(self, last_event_id, last_event_dt): party_roles_version roles WHERE businesses.id = roles.business_id AND roles.party_id = parties.id - AND parties.party_type = 'organization' and roles.role = 'proprietor' + AND parties.party_type = 'organization' and roles.role in ('proprietor','partner') AND (parties.identifier = %s OR roles.business_id = (select id from businesses where identifier = %s))""" new_corps = [] @@ -799,6 +801,7 @@ def get_lear_relationship_info(self, corp_info): parties.transaction_id as transaction_id, parties.end_transaction_id as end_transaction_id, roles.cessation_date as cessation_dt, + roles.appointment_date as appointment_dt, parties.last_name as last_nme, parties.middle_initial as middle_nme, parties.first_name as first_nme, @@ -814,7 +817,7 @@ def get_lear_relationship_info(self, corp_info): """ + self.get_table_prefix() + """party_roles_version roles WHERE businesses.id = roles.business_id AND roles.party_id = parties.id - AND parties.party_type = 'organization' and roles.role = 'proprietor' + AND parties.party_type = 'organization' and roles.role in ('proprietor','partner') AND (parties.identifier = """ + self.get_db_sql_param() + """ OR roles.business_id = (select id from """ + self.get_table_prefix() + """businesses where identifier = """ + self.get_db_sql_param() + """)) """ @@ -851,19 +854,20 @@ def get_lear_relationship_info(self, corp_info): corp_party['effective_end_date'] = MAX_END_DATE corp_party['cessation_dt'] = row[7] # use "cessation_date" as the end date (if available), regardless of the transaction status - corp_party['last_nme'] = row[8] - corp_party['middle_nme'] = row[9] - corp_party['first_nme'] = row[10] - corp_party['business_nme'] = row[11] - corp_party['bus_company_num'] = row[12] - corp_party['email_address'] = row[13] + corp_party['appointment_dt'] = row[8] + corp_party['last_nme'] = row[9] + corp_party['middle_nme'] = row[10] + corp_party['first_nme'] = row[11] + corp_party['business_nme'] = row[12] + corp_party['bus_company_num'] = row[13] + corp_party['email_address'] = row[14] corp_party['corp_party_seq_num'] = None corp_party['office_notification_dt'] = None corp_party['phone'] = None corp_party['reason_typ_cd'] = None - corp_party['role_id'] = row[14] - corp_party['role'] = row[15] - role_transaction_id = row[16] + corp_party['role_id'] = row[15] + corp_party['role'] = row[16] + role_transaction_id = row[17] corp_party['role_transaction_id'] = role_transaction_id if role_transaction_id and 0 < role_transaction_id: role_transaction = self.get_event(corp_party['corp_num'], role_transaction_id) @@ -871,7 +875,7 @@ def get_lear_relationship_info(self, corp_info): else: corp_party['role_transaction'] = {} corp_party['role_effective_start_date'] = self.get_corp_version_effective_date(corp_party, txn_field='role_transaction', filing_field=None) - role_end_transaction_id = row[17] + role_end_transaction_id = row[18] corp_party['role_end_transaction_id'] = role_end_transaction_id if role_end_transaction_id and 0 < role_end_transaction_id: role_end_transaction = self.get_event(corp_party['corp_num'], role_end_transaction_id) diff --git a/data-pipeline/bcreg/eventprocessor.py b/data-pipeline/bcreg/eventprocessor.py index 879dd88..e7bab40 100644 --- a/data-pipeline/bcreg/eventprocessor.py +++ b/data-pipeline/bcreg/eventprocessor.py @@ -15,7 +15,7 @@ from bcreg.config import config from bcreg.bcregistries import BCRegistries, CustomJsonEncoder, event_dict, is_data_conversion_event, system_type, CORP_TYPES_IN_SCOPE from bcreg.bcreg_core import CORP_WITHDRAWN_STATE -from bcreg.bcreg_lear import BCReg_Lear, lear_system_type, LEAR_CORP_TYPES_IN_SCOPE +from bcreg.bcreg_lear import BCReg_Lear, lear_system_type, LEAR_CORP_TYPES_IN_SCOPE, LEAR_CONVERSION_DATE_TZ from bcreg.rocketchat_hooks import log_error, log_warning, log_info @@ -1312,15 +1312,15 @@ def generate_relationship_creds(self, corp_info, prev_event, last_event, corp_nu dba_cred['relationship_description'] = 'TBD' # party[''] dba_cred['associated_registration_name'] = '' dba_cred['relationship_status'] = 'ACT' - dba_cred['effective_date'] = party['effective_start_date'] + dba_cred['effective_date'] = party['appointment_dt'] # if the start event is 'ADMIN' type and there is only one party record, use the firm effective date # if party['start_event']['event_typ_cd'] == 'ADMIN' and party_count[party['corp_info']['corp_num']] == 1 and party['corp_info']['recognition_dts']: # dba_cred['effective_date'] = party['corp_info']['recognition_dts'] dba_cred['relationship_status_effective'] = self.filter_min_date(dba_cred['effective_date']) - if party['end_transaction_id'] is not None and self.compare_dates(party['end_transaction']['effective_date'], "<=", corp_info['current_date'], "Relationships"): - dba_cred['expiry_date'] = party['effective_end_date'] + if party['end_transaction_id'] is not None and self.compare_dates(party['end_transaction']['effective_date'], "<=", LEAR_CONVERSION_DATE_TZ, "Relationships"): + dba_cred['expiry_date'] = party['end_transaction']['effective_date'] elif party['cessation_dt'] is not None: # scenario where the "cessation_dt" is the expiry date, with no transaction dba_cred['expiry_date'] = party['cessation_dt']