From c936f3dc93b4a93a87f187684f02a9f6afc5f35c Mon Sep 17 00:00:00 2001 From: travis Date: Mon, 16 Oct 2023 09:20:46 -0700 Subject: [PATCH 1/5] Update ABEL workflows --- tools/abel/abel.py | 9 +-- tools/abel/model/export_helper.py | 83 ++++++--------------- tools/abel/model/from_spreadsheet.py | 11 +-- tools/abel/model/workflow.py | 104 +++++++++++++++------------ 4 files changed, 95 insertions(+), 112 deletions(-) diff --git a/tools/abel/abel.py b/tools/abel/abel.py index b88eddfd43..b991bd6540 100644 --- a/tools/abel/abel.py +++ b/tools/abel/abel.py @@ -21,19 +21,20 @@ def main(parsed_args: ParseArgs) -> None: + print( '\nHow would you like to use ABEL?\n' - + '1: Modify a spreadsheet/building config for an existing building\n' - + '2: Create a spreadsheet for a new building\n' + + '1: Create a building config yaml file from a spreadsheet.\n' + + '2: Create a spreadsheet from a building config.\n' + '3: Split a building config\n' + 'q: quit\n' ) function_choice = input('Please select an option: ') new_workflow = Workflow(parsed_args) if function_choice == '1': - new_workflow.UpdateWorkflow() + new_workflow.SpreadsheetWorkflow() elif function_choice == '2': - new_workflow.InitWorkflow() + new_workflow.ConfigWorkflow() elif function_choice == '3': new_workflow.SplitWorkflow() elif function_choice == 'q': diff --git a/tools/abel/model/export_helper.py b/tools/abel/model/export_helper.py index ce7053abfc..e9797aebb0 100644 --- a/tools/abel/model/export_helper.py +++ b/tools/abel/model/export_helper.py @@ -13,6 +13,7 @@ # limitations under the License. """Helper module for exporting a valid Building Configuration or spreadsheet.""" +import enum from typing import Any, Dict, List, Optional # pylint: disable=g-importing-member @@ -51,6 +52,13 @@ from model.model_error import SpreadsheetAuthorizationError from validate.field_translation import FieldTranslation +class ConfigMode(enum.Enum): + + # config mode for a building config being updated. + UPDATE = 'UPDATE' + + # config mode for a building being initialized. + INITIALIZE = 'INITIALIZE' class GoogleSheetExport(object): """Class to help write ABEL data types to a Google Sheets spreadsheet. @@ -112,7 +120,7 @@ def __init__(self, model: Model): """ self.model = model - def ExportUpdateBuildingConfiguration( + def ExportBuildingConfiguration( self, filepath: str, operations: List[EntityOperation] ) -> Dict[str, Any]: """Exports a building Config under the UPDATE operation. @@ -126,14 +134,17 @@ def ExportUpdateBuildingConfiguration( Dictionary mapping of a building config under the update operation. """ site = self.model.site - entity_yaml_dict = {CONFIG_METADATA: {CONFIG_OPERATION: 'UPDATE'}} + config_mode = ConfigMode.UPDATE + if not site.etag: + config_mode = ConfigMode.INITIALIZE + entity_yaml_dict = {CONFIG_METADATA: {CONFIG_OPERATION: config_mode.value}} for operation in operations: entity = operation.entity if isinstance(entity, ReportingEntity): entity_yaml_dict.update( { entity.bc_guid: self._GetReportingEntityBuildingConfigBlock( - entity, operation + entity, config_mode, operation ) } ) @@ -141,7 +152,7 @@ def ExportUpdateBuildingConfiguration( entity_yaml_dict.update( { entity.bc_guid: self._GetVirtualEntityBuildingConfigBlock( - entity, operation + entity, config_mode, operation ) } ) @@ -151,10 +162,11 @@ def ExportUpdateBuildingConfiguration( site.guid: { CONFIG_CODE: site.code, CONFIG_TYPE: site.namespace + '/' + site.type_name, - CONFIG_ETAG: site.etag, } } ) + if site.etag: + entity_yaml_dict.get(site.guid).update({CONFIG_ETAG: site.etag}) try: with open(filepath, WRITE, encoding=UTF_8) as file: for key, value in entity_yaml_dict.items(): @@ -165,56 +177,6 @@ def ExportUpdateBuildingConfiguration( return entity_yaml_dict # TODO(b/233756557) Allow user to set config_metadata operation. - def ExportInitBuildingConfiguration(self, filepath: str) -> Dict[str, Any]: - """Exports an ABEL concrete model graph to a Building Config file. - - Args: - filepath: Absolute export path for a Building Config. - - Returns: - A dictionary model of the exported building config. - - Raises: - PermissionError: When ABEL is denied access to filepath. - """ - site = self.model.site - entity_yaml_dict = {CONFIG_METADATA: {CONFIG_OPERATION: CONFIG_INITIALIZE}} - for entity_guid in site.entities: - entity = self.model.guid_to_entity_map.GetEntityByGuid(entity_guid) - if isinstance(entity, ReportingEntity): - entity_yaml_dict.update( - { - entity.bc_guid: self._GetReportingEntityBuildingConfigBlock( - entity=entity, - operation=None, - ) - } - ) - elif isinstance(entity, VirtualEntity): - entity_yaml_dict.update( - { - entity.bc_guid: self._GetVirtualEntityBuildingConfigBlock( - entity=entity, operation=None - ) - } - ) - - entity_yaml_dict.update( - { - site.guid: { - CONFIG_CODE: site.code, - CONFIG_TYPE: site.namespace + '/' + site.type_name, - } - } - ) - try: - with open(filepath, WRITE, encoding=UTF_8) as file: - for key, value in entity_yaml_dict.items(): - file.write(as_document({key: value}).as_yaml()) - file.write('\n') - except PermissionError: - print(f'Permission denied when writing to {filepath}') - return entity_yaml_dict def _AddOperationToBlock( self, operation: EntityOperation @@ -229,13 +191,14 @@ def _AddOperationToBlock( return update_dict def _GetReportingEntityBuildingConfigBlock( - self, entity: ReportingEntity, operation: Optional[EntityOperation] + self, entity: ReportingEntity, config_mode: ConfigMode, operation: Optional[EntityOperation] ) -> Dict[str, object]: """Returns a Building Config formatted reporting entity block dictionary. Args: entity: A ReportingEntity instance. - operation: + operation: The operation acting on the entity... + config_mode: ConfigMode instance representing a building config's metadata config mode. Returns: A dictionary in Building Config format ready to be parsed into yaml. @@ -273,12 +236,12 @@ def _GetReportingEntityBuildingConfigBlock( reporting_entity_yaml.update( {CONFIG_TYPE: entity.namespace.value + '/' + str(entity.type_name)} ) - if operation: + if operation and not operation.operation == EntityOperationType.EXPORT and config_mode == ConfigMode.UPDATE: reporting_entity_yaml.update(self._AddOperationToBlock(operation)) return reporting_entity_yaml def _GetVirtualEntityBuildingConfigBlock( - self, entity: VirtualEntity, operation: Optional[EntityOperation] + self, entity: VirtualEntity, config_mode: ConfigMode, operation: Optional[EntityOperation] ) -> Dict[str, object]: """Returns a Building Config formatted virtual entity block dictionary. @@ -298,7 +261,7 @@ def _GetVirtualEntityBuildingConfigBlock( virtual_entity_yaml.update( {CONFIG_TYPE: entity.namespace.value + '/' + str(entity.type_name)} ) - if operation: + if operation and not operation.operation == EntityOperationType.EXPORT and config_mode == ConfigMode.UPDATE: virtual_entity_yaml.update(self._AddOperationToBlock(operation)) return virtual_entity_yaml diff --git a/tools/abel/model/from_spreadsheet.py b/tools/abel/model/from_spreadsheet.py index 738c4cd1b6..4a5ff90319 100644 --- a/tools/abel/model/from_spreadsheet.py +++ b/tools/abel/model/from_spreadsheet.py @@ -128,10 +128,11 @@ def LoadStatesFromSpreadsheet( states = [] for state_entry in state_entries: - state_entry[BC_GUID] = guid_to_entity_map.GetEntityGuidByCode( + state_entry[REPORTING_ENTITY_GUID] = guid_to_entity_map.GetEntityGuidByCode( state_entry[REPORTING_ENTITY_CODE] ) - states.append(State.FromDict(states_dict=state_entry)) + new_state = State.FromDict(states_dict=state_entry) + states.append(new_state) return states @@ -172,7 +173,7 @@ def LoadConnectionsFromSpreadsheet( def LoadOperationsFromSpreadsheet( entity_entries: Dict[str, str], guid_to_entity_map: GuidToEntityMap ) -> List[EntityOperation]: - """loads a list of entity dicitionary mappings into EntityOperation instances. + """loads a list of entity dictionary mappings into EntityOperation instances. Args: entity_entries: A list of Python Dictionaries mapping entity attributes @@ -192,8 +193,10 @@ def LoadOperationsFromSpreadsheet( new_entity.bc_guid = str(uuid.uuid4()) guid_to_entity_map.AddEntity(new_entity) operation = entity_entry.get(OPERATION) - if not operation: + if not operation and new_entity.etag: operation = EntityOperationType.EXPORT + elif not operation and not new_entity.etag: + operation = EntityOperationType.ADD operation_instance = EntityOperation( entity=new_entity, operation=EntityOperationType(operation) ) diff --git a/tools/abel/model/workflow.py b/tools/abel/model/workflow.py index faa39417d1..cc19b49c63 100644 --- a/tools/abel/model/workflow.py +++ b/tools/abel/model/workflow.py @@ -14,6 +14,7 @@ """Module for various ABEL workflows..""" import os +import sys from typing import List, Optional import webbrowser @@ -119,7 +120,7 @@ def _ImportBCAndBuildModel(self) -> Model: def _ExportAndWriteToSpreadsheet( self, model: Model, operations: Optional[List[EntityOperation]] = None - ) -> None: + ) -> tuple[str, str]: """Exports an ABEL spreadsheet for a user to interact with. Args: @@ -154,17 +155,12 @@ def _ValidateAndExportBuildingConfig( operations: [Optional] A list of EntityOperation instances for the model. """ bc_path = os.path.join(self.output_dir, EXPORT_BUILDING_CONFIG_NAME) - if operations: - export_helper.BuildingConfigExport( - model - ).ExportUpdateBuildingConfiguration( - filepath=bc_path, - operations=operations, - ) - else: - export_helper.BuildingConfigExport(model).ExportInitBuildingConfiguration( - filepath=bc_path - ) + export_helper.BuildingConfigExport( + model + ).ExportBuildingConfiguration( + filepath=bc_path, + operations=operations, + ) # Run instance validator print('Validating Export.') report_name = handler.RunValidation( @@ -178,7 +174,6 @@ def _ValidateAndExportBuildingConfig( print(f'Instance validator log: {report_name}') print(f'Exported Building Configuration: {bc_path}') - # TODO: b/296067948 - Ingest and export spreadsheet with update operations. def UpdateWorkflow(self) -> None: """Workflow for generating a building config with update operations. @@ -205,6 +200,9 @@ def UpdateWorkflow(self) -> None: spreadsheet_url, spreadsheet_id = self._ExportAndWriteToSpreadsheet( self.bc_model, bc_operations ) + self.ss_model, ss_operations = self._ImportSpreadsheetAndBuildModel( + spreadsheet_id + ) # Write to spreadsheet webbrowser.open(spreadsheet_url) @@ -212,43 +210,23 @@ def UpdateWorkflow(self) -> None: # Wait for user to edit spreadsheet input('Edit spreadsheet and press any key when done.') - # Build model from spreadsheet - self.ss_model, ss_operations = self._ImportSpreadsheetAndBuildModel( - spreadsheet_id=spreadsheet_id - ) - - # Determine entity operations on an updated model - generated_operations = model_helper.DetermineEntityOperations( - current_model=self.bc_model, updated_model=self.ss_model - ) - operations_list = model_helper.ReconcileOperations( - generated_operations=generated_operations, - model_operations=ss_operations, - ) - - # Finally export to building config - export_engine = export_helper.BuildingConfigExport(model=self.ss_model) - export_engine.ExportUpdateBuildingConfiguration( - filepath=os.path.join(self.output_dir, EXPORT_BUILDING_CONFIG_NAME), - operations=operations_list, - ) elif function_choice == '2': assert self.spreadsheet_id is not None and self.bc_filepath is not None self.bc_model = self._ImportBCAndBuildModel()[0] self.ss_model, ss_operations = self._ImportSpreadsheetAndBuildModel( self.spreadsheet_id ) - generated_operations = model_helper.DetermineEntityOperations( - current_model=self.bc_model, updated_model=self.ss_model - ) - operations_list = model_helper.ReconcileOperations( - generated_operations=generated_operations, - model_operations=ss_operations, - ) - self._ValidateAndExportBuildingConfig( - model=self.ss_model, - operations=operations_list, - ) + generated_operations = model_helper.DetermineEntityOperations( + current_model=self.bc_model, updated_model=self.ss_model + ) + operations_list = model_helper.ReconcileOperations( + generated_operations=generated_operations, + model_operations=ss_operations, + ) + self._ValidateAndExportBuildingConfig( + model=self.ss_model, + operations=operations_list, + ) def InitWorkflow(self) -> None: """Workflow for generating a building config under the INITIALIZE operation. @@ -284,3 +262,41 @@ def SplitWorkflow(self) -> None: self._ValidateAndExportBuildingConfig( model=split_model, operations=split_operations ) + + def SpreadsheetWorkflow(self) -> None: + """Workflow to write a Building Config from a spreadsheet + + Can either take just a spreadsheet or a spreadsheet and a building config.""" + pass + + def ConfigWorkflow(self) -> None: + """Workflow to create a Google sheets spreadsheet from a building config""" + + self.bc_model, bc_operations = self._ImportBCAndBuildModel() + spreadsheet_url, spreadsheet_id = self._ExportAndWriteToSpreadsheet( + self.bc_model, bc_operations + ) + + # Write to spreadsheet + webbrowser.open(spreadsheet_url) + + # Wait for user to edit spreadsheet + decision = input('press q to exit or edit spreadsheet and press return to continue.') + if decision == 'q': + sys.exit() + + # If user doesn't exit, write spreadsheet to a building config. + self.ss_model, ss_operations = self._ImportSpreadsheetAndBuildModel( + spreadsheet_id + ) + generated_operations = model_helper.DetermineEntityOperations( + current_model=self.bc_model, updated_model=self.ss_model + ) + operations_list = model_helper.ReconcileOperations( + generated_operations=generated_operations, + model_operations=ss_operations, + ) + self._ValidateAndExportBuildingConfig( + model=self.ss_model, + operations=operations_list, + ) From 41f68ca4654fa6322a4b18876769b5cdae5fb8fc Mon Sep 17 00:00:00 2001 From: travis Date: Wed, 17 Jan 2024 14:15:20 -0800 Subject: [PATCH 2/5] Update ABEL workflows --- tools/abel/abel.py | 3 - tools/abel/model/model_helper.py | 8 ++ tools/abel/model/workflow.py | 127 +++++-------------------------- 3 files changed, 29 insertions(+), 109 deletions(-) diff --git a/tools/abel/abel.py b/tools/abel/abel.py index b991bd6540..e2b0c1b035 100644 --- a/tools/abel/abel.py +++ b/tools/abel/abel.py @@ -26,7 +26,6 @@ def main(parsed_args: ParseArgs) -> None: '\nHow would you like to use ABEL?\n' + '1: Create a building config yaml file from a spreadsheet.\n' + '2: Create a spreadsheet from a building config.\n' - + '3: Split a building config\n' + 'q: quit\n' ) function_choice = input('Please select an option: ') @@ -35,8 +34,6 @@ def main(parsed_args: ParseArgs) -> None: new_workflow.SpreadsheetWorkflow() elif function_choice == '2': new_workflow.ConfigWorkflow() - elif function_choice == '3': - new_workflow.SplitWorkflow() elif function_choice == 'q': print('Bye bye') sys.exit() diff --git a/tools/abel/model/model_helper.py b/tools/abel/model/model_helper.py index edcdba07fc..025d784d76 100644 --- a/tools/abel/model/model_helper.py +++ b/tools/abel/model/model_helper.py @@ -179,6 +179,14 @@ def DetermineEntityOperations( operation which is being performed on it. """ operations = [] + if not current_model: + for entity in updated_model.entities: + if not entity.etag: + operations.append(entity_operation.EntityOperation( + entity, + operation=entity_enumerations.EntityOperationType.ADD + )) + return operations for import_entity in updated_model.entities: if import_entity.bc_guid not in set( entity.bc_guid for entity in current_model.entities diff --git a/tools/abel/model/workflow.py b/tools/abel/model/workflow.py index cc19b49c63..38f7bcfac3 100644 --- a/tools/abel/model/workflow.py +++ b/tools/abel/model/workflow.py @@ -174,101 +174,37 @@ def _ValidateAndExportBuildingConfig( print(f'Instance validator log: {report_name}') print(f'Exported Building Configuration: {bc_path}') - def UpdateWorkflow(self) -> None: - """Workflow for generating a building config with update operations. + def SpreadsheetWorkflow(self) -> None: + """Workflow to write a Building Config from a spreadsheet - STEPS: - 1. Ingest a building config - 2. write to a spreadsheet - 3. Either let the user quit or build an update bc from updated spreadsheet - 4. If build update BC, ingest modified spreadsheet(same id) and bc(already - persisting) - 5. Create operations with two models. - 6. Write to building config. - """ + Can either take just a spreadsheet or a spreadsheet and a building config.""" + # If user doesn't exit, write spreadsheet to a building config. + if not self.bc_model: + # This case statement will go away once ABEL can call DB API. + # If a person just wants to create a bc, they shouldn't need to provide + # a spreadsheet. + print("No building config input") + #sys.exit(0) + pass + elif not self.ss_model: + print("No spreadsheet id input") + sys.exit(0) - print( - '\nHow would you like to use ABEL?\n' - + '1: Edit or update an existing building config\n' - + '2: Create building config from an updated spreadsheet\n' - + 'q: quit\n' + self.ss_model, ss_operations = self._ImportSpreadsheetAndBuildModel( + self.spreadsheet_id ) - function_choice = input('Please select an option: ') - if function_choice == '1': - # Export BC and build model - self.bc_model, bc_operations = self._ImportBCAndBuildModel() - spreadsheet_url, spreadsheet_id = self._ExportAndWriteToSpreadsheet( - self.bc_model, bc_operations - ) - self.ss_model, ss_operations = self._ImportSpreadsheetAndBuildModel( - spreadsheet_id - ) - - # Write to spreadsheet - webbrowser.open(spreadsheet_url) - - # Wait for user to edit spreadsheet - input('Edit spreadsheet and press any key when done.') - - elif function_choice == '2': - assert self.spreadsheet_id is not None and self.bc_filepath is not None - self.bc_model = self._ImportBCAndBuildModel()[0] - self.ss_model, ss_operations = self._ImportSpreadsheetAndBuildModel( - self.spreadsheet_id - ) generated_operations = model_helper.DetermineEntityOperations( - current_model=self.bc_model, updated_model=self.ss_model + current_model=self.bc_model, updated_model=self.ss_model ) operations_list = model_helper.ReconcileOperations( - generated_operations=generated_operations, - model_operations=ss_operations, + generated_operations=generated_operations, + model_operations=ss_operations, ) self._ValidateAndExportBuildingConfig( - model=self.ss_model, - operations=operations_list, + model=self.ss_model, + operations=operations_list, ) - def InitWorkflow(self) -> None: - """Workflow for generating a building config under the INITIALIZE operation. - - STEPS: - 1. Ingest and parse an ABEL spreadsheet. - 2. Export and validate a building config. - """ - if not self.spreadsheet_id: - self.spreadsheet_id = input( - 'Please provide a Google Sheets spreadsheet id.' - ) - self.ss_model = self._ImportSpreadsheetAndBuildModel(self.spreadsheet_id)[0] - self._ValidateAndExportBuildingConfig(model=self.ss_model) - - def SplitWorkflow(self) -> None: - """Workflow to take in a building config and split it. - - STEPS: - 1. ingest building config. - 2. Split building config. - 3. Export the split building config. - """ - if not self.bc_filepath: - self.bc_filepath = input('Please provide a building config file path.') - else: - namespace_string = input('Desired namespace: ') - namespace = entity_enumerations.EntityNamespace(namespace_string.upper()) - self.bc_model, bc_operations = self._ImportBCAndBuildModel() - split_model, split_operations = model_helper.Split( - self.bc_model, bc_operations, namespace - ) - self._ValidateAndExportBuildingConfig( - model=split_model, operations=split_operations - ) - - def SpreadsheetWorkflow(self) -> None: - """Workflow to write a Building Config from a spreadsheet - - Can either take just a spreadsheet or a spreadsheet and a building config.""" - pass - def ConfigWorkflow(self) -> None: """Workflow to create a Google sheets spreadsheet from a building config""" @@ -279,24 +215,3 @@ def ConfigWorkflow(self) -> None: # Write to spreadsheet webbrowser.open(spreadsheet_url) - - # Wait for user to edit spreadsheet - decision = input('press q to exit or edit spreadsheet and press return to continue.') - if decision == 'q': - sys.exit() - - # If user doesn't exit, write spreadsheet to a building config. - self.ss_model, ss_operations = self._ImportSpreadsheetAndBuildModel( - spreadsheet_id - ) - generated_operations = model_helper.DetermineEntityOperations( - current_model=self.bc_model, updated_model=self.ss_model - ) - operations_list = model_helper.ReconcileOperations( - generated_operations=generated_operations, - model_operations=ss_operations, - ) - self._ValidateAndExportBuildingConfig( - model=self.ss_model, - operations=operations_list, - ) From 3f3e79abfd5f705bf1efb8a79e59d44061a172e0 Mon Sep 17 00:00:00 2001 From: travis Date: Wed, 17 Jan 2024 14:47:40 -0800 Subject: [PATCH 3/5] Pylint fixes --- tools/abel/model/export_helper.py | 20 ++++++++++++++------ tools/abel/model/workflow.py | 13 ++++++------- 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/tools/abel/model/export_helper.py b/tools/abel/model/export_helper.py index e9797aebb0..d40582e8df 100644 --- a/tools/abel/model/export_helper.py +++ b/tools/abel/model/export_helper.py @@ -26,7 +26,6 @@ from model.constants import CONFIG_CODE from model.constants import CONFIG_CONNECTIONS from model.constants import CONFIG_ETAG -from model.constants import CONFIG_INITIALIZE from model.constants import CONFIG_LINKS from model.constants import CONFIG_METADATA from model.constants import CONFIG_OPERATION @@ -191,14 +190,18 @@ def _AddOperationToBlock( return update_dict def _GetReportingEntityBuildingConfigBlock( - self, entity: ReportingEntity, config_mode: ConfigMode, operation: Optional[EntityOperation] + self, + entity: ReportingEntity, + config_mode: ConfigMode, + operation: Optional[EntityOperation] ) -> Dict[str, object]: """Returns a Building Config formatted reporting entity block dictionary. Args: entity: A ReportingEntity instance. operation: The operation acting on the entity... - config_mode: ConfigMode instance representing a building config's metadata config mode. + config_mode: ConfigMode instance representing a building config's metadata + config mode. Returns: A dictionary in Building Config format ready to be parsed into yaml. @@ -236,12 +239,16 @@ def _GetReportingEntityBuildingConfigBlock( reporting_entity_yaml.update( {CONFIG_TYPE: entity.namespace.value + '/' + str(entity.type_name)} ) - if operation and not operation.operation == EntityOperationType.EXPORT and config_mode == ConfigMode.UPDATE: + if operation and not operation.operation == EntityOperationType.EXPORT and \ + config_mode == ConfigMode.UPDATE: reporting_entity_yaml.update(self._AddOperationToBlock(operation)) return reporting_entity_yaml def _GetVirtualEntityBuildingConfigBlock( - self, entity: VirtualEntity, config_mode: ConfigMode, operation: Optional[EntityOperation] + self, + entity: VirtualEntity, + config_mode: ConfigMode, + operation: Optional[EntityOperation] ) -> Dict[str, object]: """Returns a Building Config formatted virtual entity block dictionary. @@ -261,7 +268,8 @@ def _GetVirtualEntityBuildingConfigBlock( virtual_entity_yaml.update( {CONFIG_TYPE: entity.namespace.value + '/' + str(entity.type_name)} ) - if operation and not operation.operation == EntityOperationType.EXPORT and config_mode == ConfigMode.UPDATE: + if operation and not operation.operation == EntityOperationType.EXPORT and \ + config_mode == ConfigMode.UPDATE: virtual_entity_yaml.update(self._AddOperationToBlock(operation)) return virtual_entity_yaml diff --git a/tools/abel/model/workflow.py b/tools/abel/model/workflow.py index 38f7bcfac3..d302157bea 100644 --- a/tools/abel/model/workflow.py +++ b/tools/abel/model/workflow.py @@ -20,7 +20,6 @@ # pylint: disable=g-importing-member from model import authenticator -from model import entity_enumerations from model import export_helper from model import import_helper from model import model_helper @@ -177,17 +176,17 @@ def _ValidateAndExportBuildingConfig( def SpreadsheetWorkflow(self) -> None: """Workflow to write a Building Config from a spreadsheet - Can either take just a spreadsheet or a spreadsheet and a building config.""" + Can either take just a spreadsheet or a spreadsheet and a building config. + """ # If user doesn't exit, write spreadsheet to a building config. if not self.bc_model: # This case statement will go away once ABEL can call DB API. # If a person just wants to create a bc, they shouldn't need to provide # a spreadsheet. - print("No building config input") - #sys.exit(0) + print('No building config input') pass elif not self.ss_model: - print("No spreadsheet id input") + print('No spreadsheet id input.\nExiting ABEL...') sys.exit(0) self.ss_model, ss_operations = self._ImportSpreadsheetAndBuildModel( @@ -209,9 +208,9 @@ def ConfigWorkflow(self) -> None: """Workflow to create a Google sheets spreadsheet from a building config""" self.bc_model, bc_operations = self._ImportBCAndBuildModel() - spreadsheet_url, spreadsheet_id = self._ExportAndWriteToSpreadsheet( + spreadsheet_details = self._ExportAndWriteToSpreadsheet( self.bc_model, bc_operations ) # Write to spreadsheet - webbrowser.open(spreadsheet_url) + webbrowser.open(spreadsheet_details[0]) From 8c0f625bfb7e6d9c26d7abf1f40cf7026ce2febb Mon Sep 17 00:00:00 2001 From: travis Date: Wed, 17 Jan 2024 15:46:55 -0800 Subject: [PATCH 4/5] update export_helper.py and tests --- tools/abel/model/export_helper.py | 14 +++++++++++--- tools/abel/tests/export_helper_test.py | 19 ++++++++++++------- 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/tools/abel/model/export_helper.py b/tools/abel/model/export_helper.py index a8cb3b0efe..8d09ee0ffd 100644 --- a/tools/abel/model/export_helper.py +++ b/tools/abel/model/export_helper.py @@ -120,7 +120,7 @@ def __init__(self, model: Model): self.model = model def ExportBuildingConfiguration( - self, filepath: str, operations: List[EntityOperation] + self, filepath: str, operations: List[EntityOperation] = None ) -> Dict[str, Any]: """Exports a building Config under the UPDATE operation. @@ -134,11 +134,19 @@ def ExportBuildingConfiguration( """ site = self.model.site config_mode = ConfigMode.UPDATE + operations_map = {} if not site.etag: config_mode = ConfigMode.INITIALIZE entity_yaml_dict = {CONFIG_METADATA: {CONFIG_OPERATION: config_mode.value}} - for operation in operations: - entity = operation.entity + if not operations: + iterator = [self.model.guid_to_entity_map.GetEntityByGuid(entity_guid) + for entity_guid in site.entities] + else: + iterator = [operation.entity for operation in operations] + operations_map = {operation.entity.bc_guid: operation for operation in + operations} + for entity in iterator: + operation = operations_map.get(entity.bc_guid) if isinstance(entity, ReportingEntity): entity_yaml_dict.update( { diff --git a/tools/abel/tests/export_helper_test.py b/tools/abel/tests/export_helper_test.py index 930de310b5..88d14aa9e4 100644 --- a/tools/abel/tests/export_helper_test.py +++ b/tools/abel/tests/export_helper_test.py @@ -150,7 +150,8 @@ def testWriteOneSheetRaisesSpreadsheetAuthorizationError(self): def testExportBuildingConfigExportsVirtualEntityKeysCorrectly(self): exported_building_config = ( - self.export_helper.ExportInitBuildingConfiguration(self.export_filepath) + self.export_helper.ExportBuildingConfiguration( + self.export_filepath) ) expected_keys = ['code', 'etag', 'connections', 'links', 'type'] exported_keys = list( @@ -163,7 +164,8 @@ def testExportBuildingConfigExportsVirtualEntityKeysCorrectly(self): def testExportBuildingConfigExportsReportingEntityKeysCorrectly(self): exported_building_config = ( - self.export_helper.ExportInitBuildingConfiguration(self.export_filepath) + self.export_helper.ExportBuildingConfiguration( + self.export_filepath) ) expected_keys = ['cloud_device_id', 'code', 'etag', 'translation', 'type'] exported_keys = list( @@ -174,7 +176,8 @@ def testExportBuildingConfigExportsReportingEntityKeysCorrectly(self): def testExportBuildingConfigExportsCloudDeviceIDAsString(self): exported_building_config = ( - self.export_helper.ExportInitBuildingConfiguration(self.export_filepath) + self.export_helper.ExportBuildingConfiguration( + self.export_filepath) ) expected_cdid = '2541901344105616' exported_cdid = exported_building_config.get('test_reporting_guid').get( @@ -186,7 +189,8 @@ def testExportBuildingConfigExportsCloudDeviceIDAsString(self): def testExportBuildingConfigExportsMultiStateValueFieldStates(self): exported_building_config = ( - self.export_helper.ExportInitBuildingConfiguration(self.export_filepath) + self.export_helper.ExportBuildingConfiguration( + self.export_filepath) ) multi_state_value_field_states = ( exported_building_config.get('test_reporting_guid') @@ -200,7 +204,8 @@ def testExportBuildingConfigExportsMultiStateValueFieldStates(self): def testExportBuildingConfigExportsUnitsCorrectly(self): exported_building_config = ( - self.export_helper.ExportInitBuildingConfiguration(self.export_filepath) + self.export_helper.ExportBuildingConfiguration( + self.export_filepath) ) exported_units = ( exported_building_config.get('test_reporting_guid') @@ -215,7 +220,7 @@ def testExportBuildingConfigExportsUnitsCorrectly(self): def testExportBuildingConfigExportsLinksCorrectly(self): exported_building_config = ( - self.export_helper.ExportInitBuildingConfiguration(self.export_filepath) + self.export_helper.ExportBuildingConfiguration(self.export_filepath) ) exported_links = exported_building_config.get('test_virtual_guid').get( 'links' @@ -245,7 +250,7 @@ def testExportBuildingConfigRaisesValueErrorForBadMultistate(self): export_helper = BuildingConfigExport(model) with self.assertRaises(ValueError): - export_helper.ExportInitBuildingConfiguration(self.export_filepath) + export_helper.ExportBuildingConfiguration(self.export_filepath, []) if __name__ == '__main__': From f650d9d40bd593722e25063cb92191b300aeda70 Mon Sep 17 00:00:00 2001 From: travis Date: Tue, 23 Jan 2024 15:08:49 -0800 Subject: [PATCH 5/5] Update ABEL workflows --- tools/abel/abel.py | 15 ++++-- tools/abel/model/model_builder.py | 6 ++- tools/abel/model/workflow.py | 85 ++++++++++++++++++------------- 3 files changed, 66 insertions(+), 40 deletions(-) diff --git a/tools/abel/abel.py b/tools/abel/abel.py index e2b0c1b035..eeebb60824 100644 --- a/tools/abel/abel.py +++ b/tools/abel/abel.py @@ -29,11 +29,20 @@ def main(parsed_args: ParseArgs) -> None: + 'q: quit\n' ) function_choice = input('Please select an option: ') - new_workflow = Workflow(parsed_args) + new_workflow = Workflow( + parsed_args.credential, + parsed_args.modified_types_filepath, + parsed_args.output_dir + ) if function_choice == '1': - new_workflow.SpreadsheetWorkflow() + new_workflow.SpreadsheetWorkflow( + parsed_args.spreadsheet_id, + parsed_args.building_config, + parsed_args.subscription, + parsed_args.timeout + ) elif function_choice == '2': - new_workflow.ConfigWorkflow() + new_workflow.ConfigWorkflow(parsed_args.building_config) elif function_choice == 'q': print('Bye bye') sys.exit() diff --git a/tools/abel/model/model_builder.py b/tools/abel/model/model_builder.py index f229253339..767f90a60c 100644 --- a/tools/abel/model/model_builder.py +++ b/tools/abel/model/model_builder.py @@ -14,6 +14,7 @@ """Helper module for concrete model construction.""" import datetime +import uuid from typing import Dict, List, Optional # pylint: disable=g-importing-member @@ -203,8 +204,9 @@ def Build(self) -> ...: # For each entity, Add connections where entity is the source for guid in self.site.entities: entity = self.guid_to_entity_map.GetEntityByGuid(guid) + guid = uuid.UUID(guid) for connection in self.connections: - if connection.target_entity_guid == guid: + if uuid.UUID(connection.target_entity_guid) == guid: entity.AddConnection(connection) # For each field in the model for field in self.fields: @@ -212,7 +214,7 @@ def Build(self) -> ...: for state in self.states: # Create edges between states and their corresponding Multi-state # value field in stances. - if state.reporting_entity_guid == guid: + if uuid.UUID(state.reporting_entity_guid) == guid: if state.std_field_name in ( field.reporting_entity_field_name, field.std_field_name, diff --git a/tools/abel/model/workflow.py b/tools/abel/model/workflow.py index d302157bea..15d6fcaba1 100644 --- a/tools/abel/model/workflow.py +++ b/tools/abel/model/workflow.py @@ -16,6 +16,7 @@ import os import sys from typing import List, Optional +import warnings import webbrowser # pylint: disable=g-importing-member @@ -45,21 +46,13 @@ class Workflow(object): 3: Ingest an exported BC and write to spreadsheet that user can then modify. """ - def __init__(self, argset: ParseArgs): + def __init__(self, oauth_credential: str, modified_types_filepath: str = + None, output_dir: str = None): self.google_sheets_service = ( - authenticator.GetGoogleSheetsServiceByCredential(argset.credential) + authenticator.GetGoogleSheetsServiceByCredential(oauth_credential) ) - self.subscription = argset.subscription - self.spreadsheet_id = argset.spreadsheet_id - if not argset.building_config: - self.bc_filepath = None - else: - self.bc_filepath = os.path.expanduser(argset.building_config) - self.timeout = argset.timeout - self.modified_types_filepath = argset.modified_types_filepath - self.output_dir = argset.output_dir - self.bc_model = None - self.ss_model = None + self.modified_types_filepath = modified_types_filepath + self.output_dir = output_dir def _ImportSpreadsheetAndBuildModel(self, spreadsheet_id) -> Model: """Reads a Google Sheets spreadsheet and builds an ABEL model. @@ -99,14 +92,15 @@ def _ImportSpreadsheetAndBuildModel(self, spreadsheet_id) -> Model: ) return unbuilt_model.Build(), operations - def _ImportBCAndBuildModel(self) -> Model: + def _ImportBCAndBuildModel(self, bc_filepath: str) -> Model: """Reads a local Building Config YAML file and returns and ABEL Model. Returns: An ABEL model instance and a list of EntityOperation instances. """ + warnings.simplefilter("ignore", UserWarning) imported_building_config = import_helper.DeserializeBuildingConfiguration( - filepath=self.bc_filepath + filepath=bc_filepath ) # STEP 2 @@ -146,6 +140,8 @@ def _ValidateAndExportBuildingConfig( self, model: Model, operations: list[EntityOperation] = None, + subscription: str = None, + timeout: str = None ) -> None: """Helper function for validating and exporting a building config. @@ -161,55 +157,74 @@ def _ValidateAndExportBuildingConfig( operations=operations, ) # Run instance validator - print('Validating Export.') + print('Validating building config yaml file export.') report_name = handler.RunValidation( filenames=[bc_path], report_directory=self.output_dir, modified_types_filepath=self.modified_types_filepath, default_types_filepath=ONTOLOGY_ROOT, - subscription=self.subscription, - timeout=self.timeout, + subscription=subscription, + timeout=timeout, ) print(f'Instance validator log: {report_name}') print(f'Exported Building Configuration: {bc_path}') - def SpreadsheetWorkflow(self) -> None: - """Workflow to write a Building Config from a spreadsheet + def SpreadsheetWorkflow( + self, + spreadsheet_id: str, + bc_filepath: str = None, + gcp_subscription: str = None, + timeout: int = None + ) -> None: + """Workflow that Ingests a a spreadsheet and outputs a valid building + config. + + If a building config path is provided to this workflow, then ABEL can + compare entities in a spreadsheet to entities in a building config, + and determine update operations. If no building config path is provided + then the spreadsheet will be treating as an init config. - Can either take just a spreadsheet or a spreadsheet and a building config. + Args: + spreadsheet_id: Google Sheets spreadsheet ID. + bc_filepath: Relative or Absolute filepath to a building config yaml + file. + gcp_subscription: Path to a GCP PubSub subscription. + timeout: Timeout duration in seconds for listening to PubSub. """ - # If user doesn't exit, write spreadsheet to a building config. - if not self.bc_model: - # This case statement will go away once ABEL can call DB API. - # If a person just wants to create a bc, they shouldn't need to provide - # a spreadsheet. + # TODO: b/ + if bc_filepath: + bc_model, bc_operations = self._ImportBCAndBuildModel(bc_filepath) + else: print('No building config input') - pass - elif not self.ss_model: + bc_model = None + + if not spreadsheet_id: print('No spreadsheet id input.\nExiting ABEL...') sys.exit(0) - self.ss_model, ss_operations = self._ImportSpreadsheetAndBuildModel( - self.spreadsheet_id + ss_model, ss_operations = self._ImportSpreadsheetAndBuildModel( + spreadsheet_id ) generated_operations = model_helper.DetermineEntityOperations( - current_model=self.bc_model, updated_model=self.ss_model + current_model=bc_model, updated_model=ss_model ) operations_list = model_helper.ReconcileOperations( generated_operations=generated_operations, model_operations=ss_operations, ) self._ValidateAndExportBuildingConfig( - model=self.ss_model, + model=ss_model, operations=operations_list, + subscription=gcp_subscription, + timeout=timeout ) - def ConfigWorkflow(self) -> None: + def ConfigWorkflow(self, bc_filepath: str) -> None: """Workflow to create a Google sheets spreadsheet from a building config""" - self.bc_model, bc_operations = self._ImportBCAndBuildModel() + bc_model, bc_operations = self._ImportBCAndBuildModel(bc_filepath) spreadsheet_details = self._ExportAndWriteToSpreadsheet( - self.bc_model, bc_operations + bc_model, bc_operations ) # Write to spreadsheet