diff --git a/ontology/docs/building_config.md b/ontology/docs/building_config.md index b2e0b97a51..ad4fce29df 100644 --- a/ontology/docs/building_config.md +++ b/ontology/docs/building_config.md @@ -204,8 +204,8 @@ ENTITY-CODE: represents this entity. * **Code:** The human readable identifier for the entity. This should be unique in document scope. -* **cloud_device_id:** the cloud device id from the cloud iot registry. - This field is mandatory when a translation exists. +* **cloud_device_id:** the numeric device id from cloud iot registry. + This field is mandatory when a translation exists. Must be a numeric string. * **Connections:** Used to specify connections from other entities (sources) pointing to this entity, with connection types. Entities are keys and cannot be repeated. Values are one or more connections, specified as a single diff --git a/ontology/yaml/resources/HVAC/entity_types/ABSTRACT.yaml b/ontology/yaml/resources/HVAC/entity_types/ABSTRACT.yaml index 71f6f468f6..047e7031b0 100644 --- a/ontology/yaml/resources/HVAC/entity_types/ABSTRACT.yaml +++ b/ontology/yaml/resources/HVAC/entity_types/ABSTRACT.yaml @@ -2700,6 +2700,33 @@ HP2ZC: implements: - CONTROL +HP3ZC: + guid: "bffa568b-5f69-4b08-8d8c-1377cd74d36e" + description: "Zone temp heat pump control with three compressors." + is_abstract: true + opt_uses: + - compressor_speed_percentage_command + - cooling_thermal_power_capacity + - discharge_air_temperature_sensor + - heating_thermal_power_capacity + - failed_discharge_air_temperature_alarm + - failed_zone_air_temperature_alarm + - high_zone_air_temperature_alarm + - low_zone_air_temperature_alarm + uses: + - compressor_run_command_1 + - compressor_run_command_2 + - compressor_run_command_3 + - compressor_run_status_1 + - compressor_run_status_2 + - compressor_run_status_3 + - reversing_valve_command + - zone_air_cooling_temperature_setpoint + - zone_air_heating_temperature_setpoint + - zone_air_temperature_sensor + implements: + - CONTROL + HWDC: guid: "353bd503-92eb-4f72-8a82-ec1a91413f89" diff --git a/ontology/yaml/resources/HVAC/entity_types/AHU.yaml b/ontology/yaml/resources/HVAC/entity_types/AHU.yaml index 5e6da03b06..a52c2b22e6 100644 --- a/ontology/yaml/resources/HVAC/entity_types/AHU.yaml +++ b/ontology/yaml/resources/HVAC/entity_types/AHU.yaml @@ -390,6 +390,17 @@ AHU_DFSS_DSP_ECONMD_HP2ZC: - ECONMD - HP2ZC +AHU_DFSS_DSP_ECONMD_HP3ZC: + guid: "f80e10d8-7889-489c-a195-48ee2be843fd" + description: "Single zone heat pump AHU with economizer mode control." + is_canonical: true + implements: + - AHU + - DFSS + - DSP + - ECONMD + - HP3ZC + AHU_DFSS_DSP_DX2ZC_ECONZ_EFSS: guid: "ceb87a73-0847-4203-bf37-d6f57619f4fd" description: "Single zone AHU." @@ -2004,7 +2015,7 @@ AHU_BSPC_DX3SC_ECONM_EFSS_EFVSC_SARC_SFSS_SFVSC_SSPC: implements: - AHU - BSPC - - DX2SC + - DX3SC - ECONM - EFSS - EFVSC diff --git a/ontology/yaml/resources/HVAC/entity_types/CHWS.yaml b/ontology/yaml/resources/HVAC/entity_types/CHWS.yaml index 25dfd10bf8..d23dcfd09e 100644 --- a/ontology/yaml/resources/HVAC/entity_types/CHWS.yaml +++ b/ontology/yaml/resources/HVAC/entity_types/CHWS.yaml @@ -16,6 +16,15 @@ ### Canonical Types ### ######################## +CHWS_RWTC_WDT: + guid: "c6b1443d-86c3-4012-8db3-698d133432da" + description: "Chilled water system with temperature control and monitoring." + is_canonical: true + implements: + - CHWS + - RWTC + - WDT + CHWS_WDT: guid: "b8073a03-10fd-410e-a4d9-c8002181775b" description: "Chilled water system with only basic delta-T monitoring." diff --git a/ontology/yaml/resources/HVAC/entity_types/FAN.yaml b/ontology/yaml/resources/HVAC/entity_types/FAN.yaml index 27a9e204ab..7ea542e684 100644 --- a/ontology/yaml/resources/HVAC/entity_types/FAN.yaml +++ b/ontology/yaml/resources/HVAC/entity_types/FAN.yaml @@ -37,6 +37,16 @@ FAN_SS_VSC: - SS - VSC +FAN_SS_VSC_ZSPC: + guid: "b9f968cc-211b-4b25-915d-525216aebc4e" + description: "Fan with start/stop status, open-loop speed control & zone static pressure control." + is_canonical: true + implements: + - FAN + - SS + - VSC + - ZSPC + FAN_SS_WDPM: guid: "d12cbe88-243a-40a5-b624-3a2f355e7cfc" description: "Fan with start/stop and differential pressure monitoring." diff --git a/ontology/yaml/resources/METERS/entity_types/ABSTRACT.yaml b/ontology/yaml/resources/METERS/entity_types/ABSTRACT.yaml index f75ff88832..e7b7482630 100644 --- a/ontology/yaml/resources/METERS/entity_types/ABSTRACT.yaml +++ b/ontology/yaml/resources/METERS/entity_types/ABSTRACT.yaml @@ -62,6 +62,8 @@ PVCM: guid: "97b8b9c1-21dd-4201-9424-c776e28de758" description: "Phase-level current and voltage monitoring." is_abstract: true + opt_uses: + - neutral_line_current_sensor uses: - phase1_line_current_sensor - phase2_line_current_sensor @@ -83,6 +85,14 @@ PWMRDM: - generator_run_time_accumulator - electricalgrid_run_time_accumulator +TPWM: + guid: "141dc3bf-85dd-4fce-9342-9a5462b3c274" + description: "Thermal power and thermal energy monitoring." + is_abstract: true + uses: + - thermal_energy_accumulator + - thermal_power_sensor + VCM: guid: "dc4d2839-c00d-4935-bc67-013f7721535f" description: "Current and voltage monitoring" @@ -90,4 +100,3 @@ VCM: uses: - current_sensor - voltage_sensor - diff --git a/ontology/yaml/resources/METERS/entity_types/GM.yaml b/ontology/yaml/resources/METERS/entity_types/GM.yaml index af15b83b94..4d9aff6171 100644 --- a/ontology/yaml/resources/METERS/entity_types/GM.yaml +++ b/ontology/yaml/resources/METERS/entity_types/GM.yaml @@ -32,3 +32,13 @@ GM_PULSE: uses: - gas_flowrate_sensor - gas_volume_accumulator + +GM_TPWM: + guid: "223b81fb-9fcf-475b-a2f5-ad5d1364454c" + description: "Gas meter with thermal power and thermal energy monitoring." + is_canonical: true + implements: + - GM + - TPWM + opt_uses: + - gas_temperature_sensor diff --git a/tools/README.md b/tools/README.md index 1ee7d50758..7e266861ae 100644 --- a/tools/README.md +++ b/tools/README.md @@ -75,6 +75,14 @@ $ ./tools/docker_run.sh abel 2. Follow setup instructions for the [GUID Generator](./guid_generator). 3. Run `sudo python setup.py` for this directory. +##### Authentication + +To Authenticate against GCP PubSub for telemetry validation you must install +and initialize the [gcloud CLI](https://cloud.google.com/sdk/docs/install). +Then, use the +[gcloud auth application-default-login](https://cloud.google.com/sdk/gcloud/reference/auth/application-default/login) +command to create credentials for default logins. + ### Toolkit CLI Workflow Run `python toolkit.py` and provide the following arguments: @@ -87,17 +95,18 @@ Run `python toolkit.py` and provide the following arguments: 4. `--validate` or `-v`: Runs instance validator to validate the building configuration file. -5. [Optional] Telemetry Validation: After a building configuration's entity types are validated, validation must also be run on the telemetry payload using the following parameters. **NOTE:** The OAuth credential (`-c`) and subscription (`-s`) are provided by the Google team. Please reach out to your IoT TPM for guidance. +5. [Optional] Telemetry Validation: After a building configuration's entity + types are validated, validation must also be run on the telemetry payload + using the following parameters. **NOTE:** The subscription (`-s`) is + provided by the Google team. Please reach out to your IoT TPM for guidance. * `--subscription` or `-s`: The fully-qualified path to a Google Cloud Pubsub subscription (e.g., `projects/google.com:your-project/subscriptions/your-subscription`). - * `--credential` or `-c`: An absolute or relative path to an OAuth client credential JSON file. - * `--timeout` or `-t` **[Optional]**: The timeout duration in seconds for the telemetry validation test. The default value is 600 seconds, or 10 minutes. If this time limit is exceeded before the validator receives a test pubsub message for each of the entities configured in the given instance config file, the test will fail with an error and report the entities that were not heard from. -7. `--report-directory` or `-d` **[Optional]**: Writes instance validation (instance_validation_report.txt) and telemetry validation (telemetry_validation_report.json) reports to the specified `report-directory`. By default, writes instance validation output to the console and telemetry validation output to the current working directory. +6`--report-directory` or `-d` **[Optional]**: Writes instance validation(instance_validation_report.txt) and telemetry validation (telemetry_validation_report.json) reports to the specified `report-directory`. By default, writes instance validation output to the console and telemetry validation output to the current working directory. -8. `--udmi` **[Optional]**: Validates entity metadata as [UDMI](https://github.com/faucetsdn/udmi/). Flag is set to `True` by default; change this parameter to `--udmi=False` when not validating against UDMI. +7`--udmi` **[Optional]**: Validates entity metadata as [UDMI](https://github.com/faucetsdn/udmi/). Flag is set to `True` by default; change this parameter to `--udmi=False` when not validating against UDMI. For example, the following input @@ -107,10 +116,13 @@ python toolkit.py -i //path/to/building/configuration/file.yaml -g -v -s subscri results in these actions: 1. Ingests a building configuration file. -2. Generates guids for every entity instance in the buiding configuration file and re-writes building configuration file in the new format. -4. Validates the building configuration file with the instance validator. -5. Validates the telemetry payloads for each reporting entity in the building configuration file. -6. Writes instance and telemetry validation results to the report directory as `//path/to/report-directory/instance_validation_report.txt` and `//path/to/report-directory/telemetry_validation_report.json`. +2. Generates guids for every entity instance in the building configuration + file and re-writes building configuration file in the new format. +3. Validates the building configuration file with the instance validator. +4. Validates the telemetry payloads for each reporting entity in the building + configuration file. +5. Writes instance and telemetry validation results to the report directory as + `//path/to/report-directory/instance_validation_report.txt` and `//path/to/report-directory/telemetry_validation_report.json`. **NOTE:** The new building configuration format requires that entities are keyed by Version 4 UUIDs (referred to as guids) instead of the code. To convert from old format to the new format, run your building configuration file(.yaml) through the [guid generator](https://github.com/google/digitalbuildings/tree/master/tools/guid_generator). diff --git a/tools/abel/model/entity_field.py b/tools/abel/model/entity_field.py index 52a4f5cecd..720b19fdc1 100644 --- a/tools/abel/model/entity_field.py +++ b/tools/abel/model/entity_field.py @@ -86,7 +86,7 @@ def __init__( def __eq__(self, other): if not isinstance(other, MissingField): - raise TypeError(f'{str(other)} must be a MissingField instance') + return False standard_field_name_eq = self.std_field_name == other.std_field_name entity_guid_eq = self.entity_guid == other.entity_guid reporting_field_eq = ( @@ -396,7 +396,7 @@ def __init__( def __eq__(self, other: ...) -> bool: if not isinstance(other, DimensionalValueField): - raise TypeError(f'{str(other)} must be an DimensionalValueField instance') + return False standard_field_name_eq = self.std_field_name == other.std_field_name raw_field_name_eq = self.raw_field_name == other.raw_field_name entity_guid_eq = self.entity_guid == other.entity_guid diff --git a/tools/abel/model/from_spreadsheet.py b/tools/abel/model/from_spreadsheet.py index 4a5ff90319..e8d52809a9 100644 --- a/tools/abel/model/from_spreadsheet.py +++ b/tools/abel/model/from_spreadsheet.py @@ -75,11 +75,11 @@ def LoadFieldsFromSpreadsheet( entity_field_entries: List[Dict[str, str]], guid_to_entity_map: GuidToEntityMap, ) -> List[FieldTranslation]: - """Loads list of entity field maps into FieldTranslation instances. + """Loads list of entity fields from a spreadsheet into FieldTranslation + instances. Once the entity field mapping is loaded into an FieldTranslation instance, - it - is then added to the ABEL internal model. + it is then added to the ABEL internal model. Args: entity_field_entries: A list of python dictionaries mapping entity field diff --git a/tools/abel/model/model_builder.py b/tools/abel/model/model_builder.py index 767f90a60c..66e298de8c 100644 --- a/tools/abel/model/model_builder.py +++ b/tools/abel/model/model_builder.py @@ -200,6 +200,17 @@ def Build(self) -> ...: Returns: built Model instance """ + # First add states to fields + for field in self.fields: + # For each state in the model + if isinstance(field, MultistateValueField): + for state in self.states: + # Create edges between states and their corresponding Multi-state + # value field in stances. + if state.reporting_entity_guid == field.reporting_entity_guid: + if state.std_field_name in (field.reporting_entity_field_name, + field.std_field_name): + field.AddState(state) self.site.entities = self.entities # For each entity, Add connections where entity is the source for guid in self.site.entities: @@ -210,17 +221,6 @@ def Build(self) -> ...: entity.AddConnection(connection) # For each field in the model for field in self.fields: - # For each state in the model - for state in self.states: - # Create edges between states and their corresponding Multi-state - # value field in stances. - if uuid.UUID(state.reporting_entity_guid) == guid: - if state.std_field_name in ( - field.reporting_entity_field_name, - field.std_field_name, - ): - if isinstance(field, MultistateValueField): - field.AddState(state) # Link field to entity if entity is virtual if isinstance(entity, VirtualEntity): if field.entity_guid == guid: diff --git a/tools/abel/tests/entity_field_test.py b/tools/abel/tests/entity_field_test.py index 62234581e8..f1bab4c379 100644 --- a/tools/abel/tests/entity_field_test.py +++ b/tools/abel/tests/entity_field_test.py @@ -94,8 +94,7 @@ def testMissingFieldEqualityRaisesTypeError(self): test_missing_field = MissingField.FromDict(TEST_MISSING_FIELD_DICT) # pylint: disable=unnecessary-dunder-call - with self.assertRaises(TypeError): - test_missing_field.__eq__('not a field') + self.assertFalse(test_missing_field.__eq__('not a field')) @mock.patch.object(GuidToEntityMap, 'GetEntityCodeByGuid') def testMissingFieldGetSpreadsheetRowMapping(self, test_get_code): diff --git a/tools/abel/validators/spreadsheet_validator.py b/tools/abel/validators/spreadsheet_validator.py index e4b9fa27b9..8ebcf908fe 100644 --- a/tools/abel/validators/spreadsheet_validator.py +++ b/tools/abel/validators/spreadsheet_validator.py @@ -376,16 +376,16 @@ def _ValidateConnections( validation_errors.append( ConnectionDependencyError( row=row_number, - missing_code=connection.get(TARGET_ENTITY_CODE), - present_code=connection.get(SOURCE_ENTITY_CODE), + missing_code=connection.get(SOURCE_ENTITY_CODE), + present_code=connection.get(TARGET_ENTITY_CODE), ) ) if connection[TARGET_ENTITY_CODE] not in codes: validation_errors.append( ConnectionDependencyError( row=row_number, - missing_code=connection[SOURCE_ENTITY_CODE], - present_code=connection[TARGET_ENTITY_CODE], + missing_code=connection.get(TARGET_ENTITY_CODE), + present_code=connection.get(SOURCE_ENTITY_CODE), ) ) return validation_errors diff --git a/tools/validators/instance_validator/README.md b/tools/validators/instance_validator/README.md index 46b366aca8..490a8379fe 100644 --- a/tools/validators/instance_validator/README.md +++ b/tools/validators/instance_validator/README.md @@ -74,11 +74,22 @@ Navigate to `digitalbuildings/tools/validators/instance_validator` and run `pyth ### Telemetry Validation -The validator supports a telemetry validation mode. When this mode is enabled, the validator will listen on a provided pubsub subscription for telemetry messages, and validate the message contents against the instance configuration. **It is recommended that you first use the instance validator with telemetry validation mode disabled, and then enable it after that passes.** +#### Authentication + +To Authenticate against GCP PubSub for telemetry validation you must install +and initialize the [gcloud CLI](https://cloud.google.com/sdk/docs/install). +Then, use the +[gcloud auth application-default-login](https://cloud.google.com/sdk/gcloud/reference/auth/application-default/login) +command to create credentials for default logins. -If you would like to use the telemetry validation mode, you must provide the `--subscription` parameter and the `--credential` parameter. **NOTE:** The OAuth credential and subscription are provided by the Google team. Please reach out to your IoT TPM for guidance. If a GCP Oauth client credential is not provided, then application default credentials will be used to authenticate against Google APIs. Running telemetry validation will also output a machine-readable log of the validation performed on a set of devices. This log will be output as `telemetry_validation_log.json` in the current working directory, unless otherwise specefied using the `--report_directory` parameter. +#### Validation + +The validator supports a telemetry validation mode. When this mode is enabled, the validator will listen on a provided pubsub subscription for telemetry messages, and validate the message contents against the instance configuration. **It is recommended that you first use the instance validator with telemetry validation mode disabled, and then enable it after that passes.** -1. `--credential` or `-c`: An absolute or relative path to an OAuth client credential JSON file. +If you would like to use the telemetry validation mode, you must provide the +`--subscription` parameter. **NOTE:** The subscription is provided by the +Google team. Please reach out to your IoT TPM for guidance. Running +telemetry validation will also output a machine-readable log of the validation performed on a set of devices. This log will be output as `telemetry_validation_log.json` in the current working directory, unless otherwise specefied using the `--report_directory` parameter. 2. `--subscription` or `-s`: The fully-qualified path to a Google Cloud Pubsub subscription (e.g., `projects/google.com:your-project/subscriptions/your-subscription`). diff --git a/tools/validators/instance_validator/instance_validator.py b/tools/validators/instance_validator/instance_validator.py index 38b8a261d4..18688d040d 100644 --- a/tools/validators/instance_validator/instance_validator.py +++ b/tools/validators/instance_validator/instance_validator.py @@ -69,14 +69,6 @@ def _ParseArgs() -> argparse.ArgumentParser: help='Pubsub subscription for telemetry to validate', metavar='subscription') - parser.add_argument( - '-c', - '--credential', - dest='gcp_credential', - required=False, - help='gcp credential used to authenticate against pubsub api', - metavar='gcp credential') - parser.add_argument( '-t', '--timeout', @@ -114,7 +106,6 @@ def _ParseArgs() -> argparse.ArgumentParser: filenames=args.filenames, modified_types_filepath=args.modified_types_filepath, subscription=args.subscription, - gcp_credential_path=args.gcp_credential, report_directory=args.report_directory, timeout=int(args.timeout), is_udmi=is_udmi, diff --git a/tools/validators/instance_validator/tests/fake_instances/GOOD/bc_entity_with_id.yaml b/tools/validators/instance_validator/tests/fake_instances/GOOD/bc_entity_with_id.yaml index d67bf5befb..4f3a8c5e0e 100644 --- a/tools/validators/instance_validator/tests/fake_instances/GOOD/bc_entity_with_id.yaml +++ b/tools/validators/instance_validator/tests/fake_instances/GOOD/bc_entity_with_id.yaml @@ -17,7 +17,7 @@ DMP_EDM-17-GUID: type: HVAC/DMP_EDM code: DMP_EDM-17 - cloud_device_id: "1234567890123456" + cloud_device_id: "123456" translation: exhaust_air_damper_command: present_value: "points.exhaust_air_damper_command.present_value" diff --git a/tools/validators/instance_validator/validate/entity_instance.py b/tools/validators/instance_validator/validate/entity_instance.py index 67abcf1e82..3b5693a79c 100644 --- a/tools/validators/instance_validator/validate/entity_instance.py +++ b/tools/validators/instance_validator/validate/entity_instance.py @@ -47,7 +47,7 @@ ) _UDMI_UNIT_FIELD_PATTERN = re.compile(_UDMI_UNIT_FIELD_REGEX) -_DEVICE_NUMERIC_ID_REGEX = r'[0-9]{16}' +_DEVICE_NUMERIC_ID_REGEX = r'[0-9]+' _DEVICE_NUMERIC_ID_PATTERN = re.compile(_DEVICE_NUMERIC_ID_REGEX) # Faciltities naming patterns @@ -487,16 +487,16 @@ def _IsAllMissingFields(self, entity: EntityInstance) -> bool: return True return False - def _ValidateEnumerations(self, entity_translation: Dict[str, Any]) -> bool: + def _ValidateEnumerations(self, entity: EntityInstance) -> bool: """Validate that a translation is properly enumerated. Args: - entity_translation: Dictionary of written field names to - FieldTranslation instances. + entity: Instance of EntityInstance. Returns: """ + entity_translation = entity.translation enumeration_map = {} is_valid = True for written_field_name in entity_translation: @@ -520,8 +520,9 @@ def _ValidateEnumerations(self, entity_translation: Dict[str, Any]) -> bool: enumeration_map[base_field_name] = [1, 0] for base_field_mapping, enum_list in enumeration_map.items(): if enum_list[0] > enum_list[1] and enum_list != [1, 0]: - print(f'[ERROR]\t field name {base_field_name} is enumerated and ' - 'un-enumerated in the same translation block.') + print(f'[ERROR]\t {entity.guid}: {entity.code} has field name' + f' {base_field_name} which is enumerated and ' + 'not enumerated in the same translation block.') is_valid = False return is_valid @@ -554,7 +555,7 @@ def _ValidateTranslation( if self._IsAllMissingFields(entity): return False - is_valid = self._ValidateEnumerations(entity.translation) + is_valid = self._ValidateEnumerations(entity) # Check that defined fields are in the type for as_written_field_name, ft in entity.translation.items(): qualified_field_name = _GetAllowedField( diff --git a/tools/validators/instance_validator/validate/subscriber.py b/tools/validators/instance_validator/validate/subscriber.py index 73bb53dbb1..f470150ad3 100644 --- a/tools/validators/instance_validator/validate/subscriber.py +++ b/tools/validators/instance_validator/validate/subscriber.py @@ -49,7 +49,7 @@ def __init__( assert subscription_name self.subscription_name = subscription_name - def Listen(self, callback, gcp_credential_path: str): + def Listen(self, callback, gcp_credential_path: str = None): """Listens to a pubsub subscription. Args: @@ -71,9 +71,7 @@ def Listen(self, callback, gcp_credential_path: str): ) from err except MutualTLSChannelError as err: raise MutualTLSChannelError( - 'ABEL cannot authenticate against Google Sheets API with the' - ' provided client credential.' - ) from err + 'Instance Validator cannot authenticate against GCP.') from err else: print( '[INFO]\tNo GCP client credential. Using application default'