From 209babf06b10ad92f08e2b929e494c27abff0ac0 Mon Sep 17 00:00:00 2001 From: Carson Eng Date: Thu, 12 Oct 2023 16:23:32 +0000 Subject: [PATCH 1/3] Project import generated by Copybara. PiperOrigin-RevId: 572927519 --- CONTRIBUTING.md | 15 ++-- README.md | 52 ++++++------- tools/README.md | 35 ++------- tools/abel/README.md | 84 ++++++--------------- tools/abel/model/guid_to_entity_map.py | 7 +- tools/abel/model/model_helper.py | 5 +- tools/abel/model/site.py | 13 +++- tools/abel/tests/entity_field_test.py | 1 - tools/abel/tests/guid_to_entity_map_test.py | 5 +- tools/abel/validators/README.md | 6 -- 10 files changed, 90 insertions(+), 133 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d54511702c..bc66d03329 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -6,7 +6,7 @@ just a few small guidelines you need to follow. ## Contributor License Agreement Contributions to this project must be accompanied by a Contributor License -Agreement (CLA). You (or your employer) retain the copyright to your contribution; +Agreement. You (or your employer) retain the copyright to your contribution; this simply gives us permission to use and redistribute your contributions as part of the project. Head over to to see your current agreements on file or to sign a new one. @@ -29,12 +29,13 @@ This project follows ## Pull requests -If you are Google partner, please make sure not to add sensitive data on GitHub when performing the following actions: -* Extending the ontology types through a pull request +If you are Google partner, please make sure not to add sensitive data on GitHub when: +* Extending the ontology types through a pull request * Opening an issue * Asking a question -Sensitive data is considered anything in any of the following categories: -* **Buildings:** name, location, street address, information about floors/rooms/zones within the building, etc. -* **Devices/Equipment:** name, id, location, etc. -* **Other:** project id, subscription names, etc. +Sensitive data is one of the following: +* Building names or their locations, same applied for floors, rooms and zones. +* Identifiers such as project id, subscription names +* Equipment names and their locations. +* Others diff --git a/README.md b/README.md index 548c277bf3..3be98ea652 100644 --- a/README.md +++ b/README.md @@ -12,57 +12,55 @@ The Digital Buildings project originated from the need to manage a very large, h In creating the Digital Buildings project, we have considered the following: -* Human readability +* Human Readability * Machine readability and interpretation * Composable functionality -* Dimensional analysis +* Dimensional Analysis * Correctness validation -* Cross-compatibility +* Cross compatibility ## Project Structure This project is structured as follows: -* An [ontology](./ontology/README.md) that defines the parameters of the semantic data model ("Terminology box") and tools for building, validating, and associating real equipment with a specific model. It contains the following formats: +* An [**ontology**](./ontology/README.md) that defines the parameters of the semantic data model (Terminology box) and tools for building, validating, and associating real equipment with a specific model. It contains the following formats: * [Yaml format](./ontology/yaml/README.md) * [RDF/OWL format](./ontology/rdf/README.md) - -* A [model instance configuration](./ontology/docs/building_config.md) (a.k.a building configuration file) that contains a mapping between the ontology and the "raw" data. Building configuration files are the "Assertion box." -* Tools that enable the following: - * [**ABEL**](./tools/abel/README.md)**:** facilitates easier building configration construction by converting from a templatized Google Sheet to a [building configuration file](./ontology/docs/building_config.md) (and from a building configuration file back to a Google Sheet). - * [**Explorer**](./tools/explorer/README.md)**:** allows users to explore ontology type fields and compare ontology types to each other. - * [**Instance Validator**](./tools/validators/ontology_validator/README.md) which allows to validate the yaml ontology upon a change or an extension. - * [**Ontology Validator**](./tools/validators/ontology_validator/README.md)**:** validates the ontology upon a change or an extension (currently only for the Yaml format). - * [**RDF/OWL Generator**](./tools/rdf_generator/README.md)**:** generates an RDF version from the yaml ontology files. -* An [Internal Building Representation (IBR)](./ibr/README.md) file format to represent data from different verticals such as spatial or assets. +* [Building Configuration](./ontology/docs/building_config.md) which contains a mapping between the ontology and the "raw" data. Building Config is the Assertion Box. +* Tools that allow the following: + * [ABEL](./tools/abel/README.md) generates from/to Google spreadsheet/[Building Configuration](./ontology/docs/building_config.md). + * [Explorer](./tools/explorer/README.md) allows users to explorer the ontology types and their associated fields. + * [Yaml Validator](./tools/validators/ontology_validator/README.md) which allows to validate the yaml ontology upon a change or an extension. + * [RDF/OWL Generator](./tools/rdf_generator/README.md) which allows to generate an RDF version from the yaml ontology files. +* [Internal Building Representation](./ibr/README.md) (IBR) File Format to represent data from different verticals such as spatial, assets. ## Learning Modules -The learning modules provide an overview of the following key concepts: -* The main concepts and components of the Digital Buildings Ontology -* How to model entities and extend types in the ontology -* The components of building configuration files -* How to use the validation tools for ontology extensions and building configuration files +The learning modules provide an overview of: +* The main concepts of the Digital Buildings Ontology. +* How to model and extend types. +* The Building configuration file concepts. +* The validation tools for the Building Configuration file. ### Module 1: Digital Buildings Ontology (DBO) In this module, you’ll fully explore the core modeling and organizational concepts of the DBO. These are essential concepts for data modeling and creating building configuration files. * [Lesson 1: Introduction to the DBO](./ontology/docs/learning/Module_1_Lesson_1_Introduction_to_the_DBO.pdf) -* [Lesson 2: Conceptual model](./ontology/docs/learning/Module_1_Lesson_2_Conceptual_model.pdf) +* [Lesson 2: Conceptual Model](./ontology/docs/learning/Module_1_Lesson_2_Conceptual_model.pdf) * [Lesson 3: Subfields](./ontology/docs/learning/Module_1_Lesson_3_Subfields.pdf) * [Lesson 4: Fields](./ontology/docs/learning/Module_1_Lesson_4_Fields.pdf) -* [Lesson 5: States and multi-states](./ontology/docs/learning/Module_1_Lesson_5_States_and_multistates.pdf) -* [Lesson 6: Entity types](./ontology/docs/learning/Module_1_Lesson_6_Entity_types.pdf) +* [Lesson 5: States and Multi-states](./ontology/docs/learning/Module_1_Lesson_5_States_and_multistates.pdf) +* [Lesson 6: Entity Types](./ontology/docs/learning/Module_1_Lesson_6_Entity_types.pdf) * [Lesson 7: Mappings](./ontology/docs/learning/Module_1_Lesson_7_Mappings.pdf) * [Lesson 8: Connections](./ontology/docs/learning/Module_1_Lesson_8_Connections.pdf) * [Lesson 9: Namespaces](./ontology/docs/learning/Module_1_Lesson_9_Namespaces.pdf) -### Module 2: Data Modeling with the DBO +### Module 2: Module 2: Data Modeling with the DBO In this module, you’ll deepen your understanding of the DBO and practice applying it. Through several hands-on activities, you'll walk through the recommended workflow for creating a building configuration file -* [Lesson 1: Get ready to data model](./ontology/docs/learning/Module_2_Lesson_1_Get_ready_to_data_model.pdf) +* [Lesson 1: Get Ready to data model](./ontology/docs/learning/Module_2_Lesson_1_Get_ready_to_data_model.pdf) * [Lesson 2: Determine which devices need to be modeled](./ontology/docs/learning/Module_2_Lesson_2_Determine_which_devices_need_to_be_modeled.pdf) * [Lesson 3: Determine which data points are required](./ontology/docs/learning/Module_2_Lesson_3_Determine_which_data_points_are_required.pdf) * [Lesson 4: Name each data point using the DBO](./ontology/docs/learning/Module_2_Lesson_4_Name_each_data_point_using_the_DBO.pdf) @@ -72,16 +70,16 @@ In this module, you’ll deepen your understanding of the DBO and practice apply ## Issues -Please post issues in the [Issues](https://github.com/google/digitalbuildings/issues) section. +Please post issues in [Issues](https://github.com/google/digitalbuildings/issues) section. ## Discussion -Our team has created an open mailing list to discuss Google's Digital Building effort. The discussion could include general questions, standards, APIs, and more. Join the discussion here: [google-digital-building-discuss@googlegroups.com](mailto:google-digital-building-discuss@googlegroups.com). +Open mailing list to discuss Google's Digital Building effort. The discussion could include general questions, standards, APIs, and more. [google-digital-building-discuss@googlegroups.com](mailto:google-digital-building-discuss@googlegroups.com) -Members are expected to adhere to this code of conduct: [https://opensource.google.com/conduct](https://opensource.google.com/conduct). +Members are expected to adhere to this code of conduct: [https://opensource.google.com/conduct](https://opensource.google.com/conduct) ## How to Contribute -Please see the [contribution](CONTRIBUTING.md) section. +Please see the [contribution section](CONTRIBUTING.md) ## License ``` diff --git a/tools/README.md b/tools/README.md index 9a68a0c667..8d524159b9 100644 --- a/tools/README.md +++ b/tools/README.md @@ -1,33 +1,16 @@ -# Digital Buildings Tools - -Various tools have been developed to support the use of the Digital Buildings Ontology and Building Configuration files. - -The tools are: - * [ABEL](./abel/README.md) generates from/to Google spreadsheet/[Building Configuration](../ontology/docs/building_config.md). - * [Explorer](./explorer/README.md) allows users to explorer the ontology types and their associated fields. - * [Instance Validator](./validators/instance_validator/README.md) which allows to validate the yaml ontology upon a change or an extension. - * A sub function of the Instance Validator is to also [validate telemetry messages](./validators/instance_validator/README.md#telemetry-validation) - corresponding to entity blocks in a building configuration file. - * [Ontology Validator](./validators/ontology_validator/README.md) which allows to validate the yaml ontology upon a change or an extension. - -## Digital Buildings Toolkit +# Digital Buildings Toolkit The Digital Buildings Toolkit provides a centralized method for interfacing with all of the tools contained within the Digital Buildings Repository. -### Toolkit Web Application - -The [web-based toolkit application](dbo-toolkit-app.azurewebsites.net) also exists to provide a user-friendly interface to all of the Digital Buildings tools. -Only the Instance Validator is currently supported, but other tools are planned to be added in the near future. - -### Install +## Install To install please follow the instructions below. -#### First create a virtual env +### First create a virtual env -Create the virtual environment with `venv` followed by the environment name, in this example: `tooling`, in the digitalbuildings repository. +Create the virtual environment with `venv` followed by the environment name, in this example: `tooling` ``` python -m venv tooling @@ -49,12 +32,12 @@ tooling\Scripts\activate Then you can either use pip or setuptools. -#### Pip +### Pip 1. Run `python3 -m pip install --upgrade pip` to ensure that your Python package management tools are up-to-date. 2. Run `bash pip_install.sh` or `pip_install.bat` (windows) from the following directory digitalbuildings/tools. -#### Docker +### Docker 1. Install [Docker Desktop](https://docs.docker.com/desktop/) 2. Run `./tools/docker_run.sh` to build the docker image. @@ -63,13 +46,13 @@ Then you can either use pip or setuptools. $ ./tools/docker_run.sh abel ``` -#### Setup (to be deprecated) +### Setup (to be deprecated) 1. Follow setup instructions for the [Instance Validator](./validators/instance_validator). 2. Follow setup instructions for the [GUID Generator](./guid_generator). 3. Run `sudo python setup.py` for this directory. -### Toolkit Workflow +## Toolkit Workflow Run `python toolkit.py` and provide the following arguments: @@ -109,5 +92,3 @@ For example: **NOTE:** The new building configuration format switches entities being keyed by codes to being keyed by guids, and Ids are removed. To convert from old format to the new format, run your building configuration file(.yaml) through the [guid generator](https://github.com/google/digitalbuildings/tree/master/tools/guid_generator). - -Any questions or concerns can be emailed to **BOS-GPS@google.com** diff --git a/tools/abel/README.md b/tools/abel/README.md index 2d3937e905..8b8c960ba5 100644 --- a/tools/abel/README.md +++ b/tools/abel/README.md @@ -18,12 +18,11 @@ Total setup process should only take about 15 minutes. Before starting the setup and installation process, please ensure that the dependencies are met: 1. You are running **Python 3.9** or higher +3. You have installed [virtualenv](https://pypi.org/project/virtualenv/) 2. you have installed the [Google Cloud CLI](https://cloud.google.com/sdk/docs/install) ### Set up a tooling environment -If environment is already set up then the following steps can be ignored. - 1. Clone the Digital Buildings repository ``` @@ -60,7 +59,7 @@ If environment is already set up then the following steps can be ignored. * Linux/MacOs: ``` - ./pip_install.sh + bash pip_install.sh ``` * Windows: @@ -83,15 +82,11 @@ cd abel ## Using ABEL ABEL has a few pieces of core functionality, they are: -1. Modify a spreadsheet or a building config for an existing building to produce an [UPDATE building config](../../ontology/docs/building_config.md#update) - * [Update Workflow](#update-workflow) -2. Create a [Building Config](../../ontology/docs/building_config.md) from an [ABEL spreadsheet](../../tools/abel/validators/README.md) for a new building - * [Initialization workflow](#initialization-workflow) -3. Split a building configuration file on a namespace - * [Split Workflow](#split-functionality) +* Ingest an [ABEL spreadsheet](../../tools/abel/validators/README.md) and export a valid [Building Config](../../ontology/docs/building_config.md) file +* Ingest a [Building Config](../../ontology/docs/building_config.md) and export an [ABEL spreadsheet](../../tools/abel/validators/README.md) ### Command-line arguments for ABEL: -`-c` or `--credential` **required** absolute or relative path to a gcp OAuth client +`-c` or `--credential` absolute or relative path to a gcp OAuth client credential file. An OAuth client credential is required for authentication against Google Sheets service. @@ -100,7 +95,7 @@ against Google Sheets service. * A Google Sheets ID is found embedded into the spreadsheet's url. Required when ABEL is creating a building config from a Google spreadsheet. e.g. `https://docs/google/com/spreadsheets/d//edit#gid=123467` -`-b` or `--building_config` absolute o relative path to a local building configuration +`-b` or `--building_config` absolute path to a local building configuration file. Only required for the `Building Config -> Spreadsheet` workflow. * [Building Configuration Docs](../../ontology/docs/building_config.md) @@ -114,71 +109,42 @@ Validator](../validators/ontology_validator) will surface validation results from the modified ontology, and Building Configuration files will be validated against the modified ontology. -`-d` or `--output-dir` fully qualified or relative file path to a directory which ABEL can write files to. ABEL must have write access to this directory or else an error will be raised. +`-d` or `--output-dir` fully qualified or relative file path to a directory which ABEL can write validation logs to. ABEL must have write access to this directory or else an error will be raised. ### The ABEL Spreadsheet The ABEL spreadsheet serves as a user-friendly interface for ABEL and is what -allows a user to make changes to machine-readable documents like [Building +allows a user to make changes to machine readable documents like [Building Config](../../ontology/docs/building_config.md). Please see the [ABEL spreadsheet docs](../../tools/abel/validators/README.md) for detailed instructions on how to create your own spreadsheet. -### Update Workflow - -If you would like to create a building config to update an already onboarded building, then there are two options: - -Generate an ABEL spreadsheet from an exported building config. - - 1. In `digitalbuildings/tools/abel` run ABEL with the command: - ``` - python3 abel.py -c /path/to/credential.json -b absolute/path/to/building/config - ``` -To generate a building config from an already updated spreadsheet. - - 1. In `digitalbuildings/tools/abel` run ABEL with the command: - ``` - python3 abel.py -s -c -d - ``` - 2. If your spreadsheet does not pass the validation criteria found in the - [spreadsheet docs](../../tools/abel/validators/README.md) then ABEL will fast - fail and a validation - report will be created in your current directory with the name, - `spreadsheet_validation_.log` - 3. The resulting Building Config and instance validation report will be written - to the current directory with names: - * `bc_export_.yaml` - * `instance_validation_.log` - -### Initialization workflow -If you would like create a building configuration file under the initialization operation +### Spreadsheet -> Building Config -The process for using an ABEL spreadsheet to generate a new Building Config is as +The process for using an ABEL spreadsheet to generate a Building Config is as follows: -1. Create a spreadsheet for ABEL from [ABEL Spreadsheet template](https://docs.google.com/spreadsheets/d/1nlFVwVvmumBSIAAAv7xq1-xuqGy0k1ONyrG2rMqrgdE/copy#gid=980240783) -2. Populate your spreadsheet. A well-defined guide on how to populate your +1. Create a spreadsheet for ABEL from [ABEL Spreadsheet template](https://docs.google.com/spreadsheets/d/1tcLjFnHiXUT-xh5C1hRKiUVaUH_CzgSI8zFQ_B8q7vs/copy#gid=980240783) +2. Populate your spreadsheet. A well defined guide on how to populate your spreadsheet can be found in the [spreadsheet docs](../../tools/abel/validators/README.md) 3. In `digitalbuildings/tools/abel` run ABEL with the command: - ``` - python3 abel.py -s -c -d - ``` -4. Choose option 2: Create a spreadsheet for a new building -5. If your spreadsheet does not pass the validation criteria found in the +``` +python3 abel.py -c -s +``` +4. If your spreadsheet does not pass the validation criteria found in the [spreadsheet docs](../../tools/abel/validators/README.md) then ABEL will fast fail and a validation report will be created in the directory specified with the `-d` arguments with the name, `spreadsheet_validation_.log` -6. The resulting Building Config and instance validation report will be written +5. The resulting Building Config and instance validation report will be written to the same directory with names: * `bc_export_.yaml` * `instance_validation_.log` -### Split Functionality -To split a building config file on a certain namespace: +### Building Config -> Spreadsheet + +The process for generating an ABEL spreadsheet from a building config is as +follows: -1. Run abel with a json credential and a building config: - ``` - python3 abel.py -c -b - ``` -2. Select option 3: `Split a building config` -3. ABEL will generate a new building config split on the desired namespace - along with any dependencies. Provide the `-d` argument if you would like ABEL to write to a specific directory. +1. In `digitalbuildings/tools/abel` run ABEL with the command: +``` +python3 abel.py -c /path/to/credential.json -b absolute/path/to/building/config +``` diff --git a/tools/abel/model/guid_to_entity_map.py b/tools/abel/model/guid_to_entity_map.py index c8010b40c2..545127d141 100644 --- a/tools/abel/model/guid_to_entity_map.py +++ b/tools/abel/model/guid_to_entity_map.py @@ -65,10 +65,13 @@ def AddEntity(self, entity: ...) -> None: """ if entity is None: raise ValueError('Cannot add None values to the guid to entity map.') - if not entity.bc_guid: + elif not entity.bc_guid: raise AttributeError(f'{entity.code}: guid missing') - if entity.bc_guid not in self._guid_to_entity_map: + elif entity.bc_guid not in list(self._guid_to_entity_map.keys()): self._guid_to_entity_map[entity.bc_guid] = entity + elif self._guid_to_entity_map.get(entity.bc_guid) == entity: + # Do nothing, this mapping already exists + pass else: raise KeyError( f'{entity.bc_guid} maps to {self._guid_to_entity_map[entity.bc_guid]}' diff --git a/tools/abel/model/model_helper.py b/tools/abel/model/model_helper.py index edcdba07fc..5b9ba89a67 100644 --- a/tools/abel/model/model_helper.py +++ b/tools/abel/model/model_helper.py @@ -20,6 +20,7 @@ from model import entity_field as ef from model import entity_operation from model import model_builder as mb +from model import site as site_lib from validate import field_translation as ft @@ -304,7 +305,9 @@ def GetConnectionDependencies(entity, dependencies, split_entities): A list of entities connected to entity. """ split_entities.append(entity) - if not entity.connections and entity not in split_entities: + if isinstance(entity, site_lib.Site): + return dependencies + elif not entity.connections and entity not in split_entities: dependencies.append(entity) return dependencies else: diff --git a/tools/abel/model/site.py b/tools/abel/model/site.py index 213e138925..4f1c43f50f 100644 --- a/tools/abel/model/site.py +++ b/tools/abel/model/site.py @@ -78,6 +78,11 @@ def FromDict(cls, site_dict: Dict[str, object]) -> ...: ) return site_instance + @property + def bc_guid(self) -> str: + """Returns the BC GUID for a site.""" + return self.guid + @property def entities(self) -> List[str]: """Returns a list of entity guids contained in a site.""" @@ -104,10 +109,14 @@ def AddEntity(self, entity: Entity) -> None: # pylint: disable=unused-argument def GetSpreadsheetRowMapping(self, *args) -> Dict[str, str]: """Returns a dictionary of Site attributes by spreadsheet headers.""" - return { + site_row_mapping = { VALUES: [ {USER_ENTERED_VALUE: {STRING_VALUE: self.code}}, {USER_ENTERED_VALUE: {STRING_VALUE: self.guid}}, - {USER_ENTERED_VALUE: {STRING_VALUE: self.etag}}, ] } + if self.etag: + site_row_mapping.get(VALUES).append( + {USER_ENTERED_VALUE: {STRING_VALUE: self.etag}} + ) + return site_row_mapping diff --git a/tools/abel/tests/entity_field_test.py b/tools/abel/tests/entity_field_test.py index 0c0819a93b..9728eedb42 100644 --- a/tools/abel/tests/entity_field_test.py +++ b/tools/abel/tests/entity_field_test.py @@ -93,7 +93,6 @@ def testMissingFieldInequality(self): def testMissingFieldEqualityRaisesTypeError(self): test_missing_field = MissingField.FromDict(TEST_MISSING_FIELD_DICT) - # pylint: disable=unnecessary-dunder-call with self.assertRaises(TypeError): test_missing_field.__eq__('not a field') diff --git a/tools/abel/tests/guid_to_entity_map_test.py b/tools/abel/tests/guid_to_entity_map_test.py index 5784e5cf85..634ba1c99e 100644 --- a/tools/abel/tests/guid_to_entity_map_test.py +++ b/tools/abel/tests/guid_to_entity_map_test.py @@ -83,10 +83,13 @@ def testAddEntityRaisesAttributeError(self): _TEST_REPORTING_ENTITY.bc_guid = TEST_REPORTING_GUID def testAddEntityRaisesKeyError(self): + """Test adding a different entity with a already existing guid.""" self.guid_to_entity_map.AddEntity(_TEST_REPORTING_ENTITY) + test_virtual_entity = VirtualEntity.FromDict(TEST_VIRTUAL_ENTITY_DICT) + test_virtual_entity.bc_guid = TEST_REPORTING_GUID with self.assertRaises(KeyError): - self.guid_to_entity_map.AddEntity(_TEST_REPORTING_ENTITY) + self.guid_to_entity_map.AddEntity(test_virtual_entity) def testGetEntityByGuidRaisesKeyError(self): with self.assertRaises(KeyError): diff --git a/tools/abel/validators/README.md b/tools/abel/validators/README.md index d1ff16712f..8d93abae49 100644 --- a/tools/abel/validators/README.md +++ b/tools/abel/validators/README.md @@ -58,10 +58,6 @@ A [human-readable code](../../../ontology/docs/building_config.md#identifiers) f [UUID4](https://en.wikipedia.org/wiki/Universally_unique_identifier#Version_4_(random)) [identifier](../../../ontology/docs/building_config.md#identifiers) for a building. Unless you are onboarding a Google building, this value is generated by ABEL and does not need to be provided for initial onboarding. -`Etag` *string* - -Version control tag for entities in an entity type database. Used only when updating an existing building config. - ***Note:*** A Site should not be considered an entity and should not be listed in the `Entities` tab. @@ -112,8 +108,6 @@ An [update operation](../../../ontology/docs/building_config.md#update) for an e * `DELETE` - Must be indicated by a user that an entity is being removed from a building. * `EXPORT` - Will be generated automatically and indicates that an entity exists in a building. -Operation specifies if and how an entity is being updated from an existing configuration. - ### Entity Fields The `Entity Fields` tab defines fields translated by [reporting entities](../../../ontology/docs/building_config.md#reporting-physical-devices) and linked to [virtual entities](../../../onotology/docs/building_config.md#virtual-devices) defined in the `Entities` tab. From 99c8e205e2df87df3b1893303ae32c0c5a5d5b91 Mon Sep 17 00:00:00 2001 From: travis Date: Thu, 12 Oct 2023 09:42:51 -0700 Subject: [PATCH 2/3] revert copybara doc changes --- CONTRIBUTING.md | 15 +++--- README.md | 52 ++++++++++---------- tools/README.md | 35 ++++++++++---- tools/abel/README.md | 84 +++++++++++++++++++++++---------- tools/abel/validators/README.md | 6 +++ 5 files changed, 126 insertions(+), 66 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index bc66d03329..d54511702c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -6,7 +6,7 @@ just a few small guidelines you need to follow. ## Contributor License Agreement Contributions to this project must be accompanied by a Contributor License -Agreement. You (or your employer) retain the copyright to your contribution; +Agreement (CLA). You (or your employer) retain the copyright to your contribution; this simply gives us permission to use and redistribute your contributions as part of the project. Head over to to see your current agreements on file or to sign a new one. @@ -29,13 +29,12 @@ This project follows ## Pull requests -If you are Google partner, please make sure not to add sensitive data on GitHub when: -* Extending the ontology types through a pull request +If you are Google partner, please make sure not to add sensitive data on GitHub when performing the following actions: +* Extending the ontology types through a pull request * Opening an issue * Asking a question -Sensitive data is one of the following: -* Building names or their locations, same applied for floors, rooms and zones. -* Identifiers such as project id, subscription names -* Equipment names and their locations. -* Others +Sensitive data is considered anything in any of the following categories: +* **Buildings:** name, location, street address, information about floors/rooms/zones within the building, etc. +* **Devices/Equipment:** name, id, location, etc. +* **Other:** project id, subscription names, etc. diff --git a/README.md b/README.md index 3be98ea652..548c277bf3 100644 --- a/README.md +++ b/README.md @@ -12,55 +12,57 @@ The Digital Buildings project originated from the need to manage a very large, h In creating the Digital Buildings project, we have considered the following: -* Human Readability +* Human readability * Machine readability and interpretation * Composable functionality -* Dimensional Analysis +* Dimensional analysis * Correctness validation -* Cross compatibility +* Cross-compatibility ## Project Structure This project is structured as follows: -* An [**ontology**](./ontology/README.md) that defines the parameters of the semantic data model (Terminology box) and tools for building, validating, and associating real equipment with a specific model. It contains the following formats: +* An [ontology](./ontology/README.md) that defines the parameters of the semantic data model ("Terminology box") and tools for building, validating, and associating real equipment with a specific model. It contains the following formats: * [Yaml format](./ontology/yaml/README.md) * [RDF/OWL format](./ontology/rdf/README.md) -* [Building Configuration](./ontology/docs/building_config.md) which contains a mapping between the ontology and the "raw" data. Building Config is the Assertion Box. -* Tools that allow the following: - * [ABEL](./tools/abel/README.md) generates from/to Google spreadsheet/[Building Configuration](./ontology/docs/building_config.md). - * [Explorer](./tools/explorer/README.md) allows users to explorer the ontology types and their associated fields. - * [Yaml Validator](./tools/validators/ontology_validator/README.md) which allows to validate the yaml ontology upon a change or an extension. - * [RDF/OWL Generator](./tools/rdf_generator/README.md) which allows to generate an RDF version from the yaml ontology files. -* [Internal Building Representation](./ibr/README.md) (IBR) File Format to represent data from different verticals such as spatial, assets. + +* A [model instance configuration](./ontology/docs/building_config.md) (a.k.a building configuration file) that contains a mapping between the ontology and the "raw" data. Building configuration files are the "Assertion box." +* Tools that enable the following: + * [**ABEL**](./tools/abel/README.md)**:** facilitates easier building configration construction by converting from a templatized Google Sheet to a [building configuration file](./ontology/docs/building_config.md) (and from a building configuration file back to a Google Sheet). + * [**Explorer**](./tools/explorer/README.md)**:** allows users to explore ontology type fields and compare ontology types to each other. + * [**Instance Validator**](./tools/validators/ontology_validator/README.md) which allows to validate the yaml ontology upon a change or an extension. + * [**Ontology Validator**](./tools/validators/ontology_validator/README.md)**:** validates the ontology upon a change or an extension (currently only for the Yaml format). + * [**RDF/OWL Generator**](./tools/rdf_generator/README.md)**:** generates an RDF version from the yaml ontology files. +* An [Internal Building Representation (IBR)](./ibr/README.md) file format to represent data from different verticals such as spatial or assets. ## Learning Modules -The learning modules provide an overview of: -* The main concepts of the Digital Buildings Ontology. -* How to model and extend types. -* The Building configuration file concepts. -* The validation tools for the Building Configuration file. +The learning modules provide an overview of the following key concepts: +* The main concepts and components of the Digital Buildings Ontology +* How to model entities and extend types in the ontology +* The components of building configuration files +* How to use the validation tools for ontology extensions and building configuration files ### Module 1: Digital Buildings Ontology (DBO) In this module, you’ll fully explore the core modeling and organizational concepts of the DBO. These are essential concepts for data modeling and creating building configuration files. * [Lesson 1: Introduction to the DBO](./ontology/docs/learning/Module_1_Lesson_1_Introduction_to_the_DBO.pdf) -* [Lesson 2: Conceptual Model](./ontology/docs/learning/Module_1_Lesson_2_Conceptual_model.pdf) +* [Lesson 2: Conceptual model](./ontology/docs/learning/Module_1_Lesson_2_Conceptual_model.pdf) * [Lesson 3: Subfields](./ontology/docs/learning/Module_1_Lesson_3_Subfields.pdf) * [Lesson 4: Fields](./ontology/docs/learning/Module_1_Lesson_4_Fields.pdf) -* [Lesson 5: States and Multi-states](./ontology/docs/learning/Module_1_Lesson_5_States_and_multistates.pdf) -* [Lesson 6: Entity Types](./ontology/docs/learning/Module_1_Lesson_6_Entity_types.pdf) +* [Lesson 5: States and multi-states](./ontology/docs/learning/Module_1_Lesson_5_States_and_multistates.pdf) +* [Lesson 6: Entity types](./ontology/docs/learning/Module_1_Lesson_6_Entity_types.pdf) * [Lesson 7: Mappings](./ontology/docs/learning/Module_1_Lesson_7_Mappings.pdf) * [Lesson 8: Connections](./ontology/docs/learning/Module_1_Lesson_8_Connections.pdf) * [Lesson 9: Namespaces](./ontology/docs/learning/Module_1_Lesson_9_Namespaces.pdf) -### Module 2: Module 2: Data Modeling with the DBO +### Module 2: Data Modeling with the DBO In this module, you’ll deepen your understanding of the DBO and practice applying it. Through several hands-on activities, you'll walk through the recommended workflow for creating a building configuration file -* [Lesson 1: Get Ready to data model](./ontology/docs/learning/Module_2_Lesson_1_Get_ready_to_data_model.pdf) +* [Lesson 1: Get ready to data model](./ontology/docs/learning/Module_2_Lesson_1_Get_ready_to_data_model.pdf) * [Lesson 2: Determine which devices need to be modeled](./ontology/docs/learning/Module_2_Lesson_2_Determine_which_devices_need_to_be_modeled.pdf) * [Lesson 3: Determine which data points are required](./ontology/docs/learning/Module_2_Lesson_3_Determine_which_data_points_are_required.pdf) * [Lesson 4: Name each data point using the DBO](./ontology/docs/learning/Module_2_Lesson_4_Name_each_data_point_using_the_DBO.pdf) @@ -70,16 +72,16 @@ In this module, you’ll deepen your understanding of the DBO and practice apply ## Issues -Please post issues in [Issues](https://github.com/google/digitalbuildings/issues) section. +Please post issues in the [Issues](https://github.com/google/digitalbuildings/issues) section. ## Discussion -Open mailing list to discuss Google's Digital Building effort. The discussion could include general questions, standards, APIs, and more. [google-digital-building-discuss@googlegroups.com](mailto:google-digital-building-discuss@googlegroups.com) +Our team has created an open mailing list to discuss Google's Digital Building effort. The discussion could include general questions, standards, APIs, and more. Join the discussion here: [google-digital-building-discuss@googlegroups.com](mailto:google-digital-building-discuss@googlegroups.com). -Members are expected to adhere to this code of conduct: [https://opensource.google.com/conduct](https://opensource.google.com/conduct) +Members are expected to adhere to this code of conduct: [https://opensource.google.com/conduct](https://opensource.google.com/conduct). ## How to Contribute -Please see the [contribution section](CONTRIBUTING.md) +Please see the [contribution](CONTRIBUTING.md) section. ## License ``` diff --git a/tools/README.md b/tools/README.md index 8d524159b9..9a68a0c667 100644 --- a/tools/README.md +++ b/tools/README.md @@ -1,16 +1,33 @@ -# Digital Buildings Toolkit +# Digital Buildings Tools + +Various tools have been developed to support the use of the Digital Buildings Ontology and Building Configuration files. + +The tools are: + * [ABEL](./abel/README.md) generates from/to Google spreadsheet/[Building Configuration](../ontology/docs/building_config.md). + * [Explorer](./explorer/README.md) allows users to explorer the ontology types and their associated fields. + * [Instance Validator](./validators/instance_validator/README.md) which allows to validate the yaml ontology upon a change or an extension. + * A sub function of the Instance Validator is to also [validate telemetry messages](./validators/instance_validator/README.md#telemetry-validation) + corresponding to entity blocks in a building configuration file. + * [Ontology Validator](./validators/ontology_validator/README.md) which allows to validate the yaml ontology upon a change or an extension. + +## Digital Buildings Toolkit The Digital Buildings Toolkit provides a centralized method for interfacing with all of the tools contained within the Digital Buildings Repository. -## Install +### Toolkit Web Application + +The [web-based toolkit application](dbo-toolkit-app.azurewebsites.net) also exists to provide a user-friendly interface to all of the Digital Buildings tools. +Only the Instance Validator is currently supported, but other tools are planned to be added in the near future. + +### Install To install please follow the instructions below. -### First create a virtual env +#### First create a virtual env -Create the virtual environment with `venv` followed by the environment name, in this example: `tooling` +Create the virtual environment with `venv` followed by the environment name, in this example: `tooling`, in the digitalbuildings repository. ``` python -m venv tooling @@ -32,12 +49,12 @@ tooling\Scripts\activate Then you can either use pip or setuptools. -### Pip +#### Pip 1. Run `python3 -m pip install --upgrade pip` to ensure that your Python package management tools are up-to-date. 2. Run `bash pip_install.sh` or `pip_install.bat` (windows) from the following directory digitalbuildings/tools. -### Docker +#### Docker 1. Install [Docker Desktop](https://docs.docker.com/desktop/) 2. Run `./tools/docker_run.sh` to build the docker image. @@ -46,13 +63,13 @@ Then you can either use pip or setuptools. $ ./tools/docker_run.sh abel ``` -### Setup (to be deprecated) +#### Setup (to be deprecated) 1. Follow setup instructions for the [Instance Validator](./validators/instance_validator). 2. Follow setup instructions for the [GUID Generator](./guid_generator). 3. Run `sudo python setup.py` for this directory. -## Toolkit Workflow +### Toolkit Workflow Run `python toolkit.py` and provide the following arguments: @@ -92,3 +109,5 @@ For example: **NOTE:** The new building configuration format switches entities being keyed by codes to being keyed by guids, and Ids are removed. To convert from old format to the new format, run your building configuration file(.yaml) through the [guid generator](https://github.com/google/digitalbuildings/tree/master/tools/guid_generator). + +Any questions or concerns can be emailed to **BOS-GPS@google.com** diff --git a/tools/abel/README.md b/tools/abel/README.md index 8b8c960ba5..2d3937e905 100644 --- a/tools/abel/README.md +++ b/tools/abel/README.md @@ -18,11 +18,12 @@ Total setup process should only take about 15 minutes. Before starting the setup and installation process, please ensure that the dependencies are met: 1. You are running **Python 3.9** or higher -3. You have installed [virtualenv](https://pypi.org/project/virtualenv/) 2. you have installed the [Google Cloud CLI](https://cloud.google.com/sdk/docs/install) ### Set up a tooling environment +If environment is already set up then the following steps can be ignored. + 1. Clone the Digital Buildings repository ``` @@ -59,7 +60,7 @@ dependencies are met: * Linux/MacOs: ``` - bash pip_install.sh + ./pip_install.sh ``` * Windows: @@ -82,11 +83,15 @@ cd abel ## Using ABEL ABEL has a few pieces of core functionality, they are: -* Ingest an [ABEL spreadsheet](../../tools/abel/validators/README.md) and export a valid [Building Config](../../ontology/docs/building_config.md) file -* Ingest a [Building Config](../../ontology/docs/building_config.md) and export an [ABEL spreadsheet](../../tools/abel/validators/README.md) +1. Modify a spreadsheet or a building config for an existing building to produce an [UPDATE building config](../../ontology/docs/building_config.md#update) + * [Update Workflow](#update-workflow) +2. Create a [Building Config](../../ontology/docs/building_config.md) from an [ABEL spreadsheet](../../tools/abel/validators/README.md) for a new building + * [Initialization workflow](#initialization-workflow) +3. Split a building configuration file on a namespace + * [Split Workflow](#split-functionality) ### Command-line arguments for ABEL: -`-c` or `--credential` absolute or relative path to a gcp OAuth client +`-c` or `--credential` **required** absolute or relative path to a gcp OAuth client credential file. An OAuth client credential is required for authentication against Google Sheets service. @@ -95,7 +100,7 @@ against Google Sheets service. * A Google Sheets ID is found embedded into the spreadsheet's url. Required when ABEL is creating a building config from a Google spreadsheet. e.g. `https://docs/google/com/spreadsheets/d//edit#gid=123467` -`-b` or `--building_config` absolute path to a local building configuration +`-b` or `--building_config` absolute o relative path to a local building configuration file. Only required for the `Building Config -> Spreadsheet` workflow. * [Building Configuration Docs](../../ontology/docs/building_config.md) @@ -109,42 +114,71 @@ Validator](../validators/ontology_validator) will surface validation results from the modified ontology, and Building Configuration files will be validated against the modified ontology. -`-d` or `--output-dir` fully qualified or relative file path to a directory which ABEL can write validation logs to. ABEL must have write access to this directory or else an error will be raised. +`-d` or `--output-dir` fully qualified or relative file path to a directory which ABEL can write files to. ABEL must have write access to this directory or else an error will be raised. ### The ABEL Spreadsheet The ABEL spreadsheet serves as a user-friendly interface for ABEL and is what -allows a user to make changes to machine readable documents like [Building +allows a user to make changes to machine-readable documents like [Building Config](../../ontology/docs/building_config.md). Please see the [ABEL spreadsheet docs](../../tools/abel/validators/README.md) for detailed instructions on how to create your own spreadsheet. -### Spreadsheet -> Building Config +### Update Workflow + +If you would like to create a building config to update an already onboarded building, then there are two options: + +Generate an ABEL spreadsheet from an exported building config. + + 1. In `digitalbuildings/tools/abel` run ABEL with the command: + ``` + python3 abel.py -c /path/to/credential.json -b absolute/path/to/building/config + ``` +To generate a building config from an already updated spreadsheet. + + 1. In `digitalbuildings/tools/abel` run ABEL with the command: + ``` + python3 abel.py -s -c -d + ``` + 2. If your spreadsheet does not pass the validation criteria found in the + [spreadsheet docs](../../tools/abel/validators/README.md) then ABEL will fast + fail and a validation + report will be created in your current directory with the name, + `spreadsheet_validation_.log` + 3. The resulting Building Config and instance validation report will be written + to the current directory with names: + * `bc_export_.yaml` + * `instance_validation_.log` + +### Initialization workflow +If you would like create a building configuration file under the initialization operation -The process for using an ABEL spreadsheet to generate a Building Config is as +The process for using an ABEL spreadsheet to generate a new Building Config is as follows: -1. Create a spreadsheet for ABEL from [ABEL Spreadsheet template](https://docs.google.com/spreadsheets/d/1tcLjFnHiXUT-xh5C1hRKiUVaUH_CzgSI8zFQ_B8q7vs/copy#gid=980240783) -2. Populate your spreadsheet. A well defined guide on how to populate your +1. Create a spreadsheet for ABEL from [ABEL Spreadsheet template](https://docs.google.com/spreadsheets/d/1nlFVwVvmumBSIAAAv7xq1-xuqGy0k1ONyrG2rMqrgdE/copy#gid=980240783) +2. Populate your spreadsheet. A well-defined guide on how to populate your spreadsheet can be found in the [spreadsheet docs](../../tools/abel/validators/README.md) 3. In `digitalbuildings/tools/abel` run ABEL with the command: -``` -python3 abel.py -c -s -``` -4. If your spreadsheet does not pass the validation criteria found in the + ``` + python3 abel.py -s -c -d + ``` +4. Choose option 2: Create a spreadsheet for a new building +5. If your spreadsheet does not pass the validation criteria found in the [spreadsheet docs](../../tools/abel/validators/README.md) then ABEL will fast fail and a validation report will be created in the directory specified with the `-d` arguments with the name, `spreadsheet_validation_.log` -5. The resulting Building Config and instance validation report will be written +6. The resulting Building Config and instance validation report will be written to the same directory with names: * `bc_export_.yaml` * `instance_validation_.log` -### Building Config -> Spreadsheet - -The process for generating an ABEL spreadsheet from a building config is as -follows: +### Split Functionality +To split a building config file on a certain namespace: -1. In `digitalbuildings/tools/abel` run ABEL with the command: -``` -python3 abel.py -c /path/to/credential.json -b absolute/path/to/building/config -``` +1. Run abel with a json credential and a building config: + ``` + python3 abel.py -c -b + ``` +2. Select option 3: `Split a building config` +3. ABEL will generate a new building config split on the desired namespace + along with any dependencies. Provide the `-d` argument if you would like ABEL to write to a specific directory. diff --git a/tools/abel/validators/README.md b/tools/abel/validators/README.md index 8d93abae49..d1ff16712f 100644 --- a/tools/abel/validators/README.md +++ b/tools/abel/validators/README.md @@ -58,6 +58,10 @@ A [human-readable code](../../../ontology/docs/building_config.md#identifiers) f [UUID4](https://en.wikipedia.org/wiki/Universally_unique_identifier#Version_4_(random)) [identifier](../../../ontology/docs/building_config.md#identifiers) for a building. Unless you are onboarding a Google building, this value is generated by ABEL and does not need to be provided for initial onboarding. +`Etag` *string* + +Version control tag for entities in an entity type database. Used only when updating an existing building config. + ***Note:*** A Site should not be considered an entity and should not be listed in the `Entities` tab. @@ -108,6 +112,8 @@ An [update operation](../../../ontology/docs/building_config.md#update) for an e * `DELETE` - Must be indicated by a user that an entity is being removed from a building. * `EXPORT` - Will be generated automatically and indicates that an entity exists in a building. +Operation specifies if and how an entity is being updated from an existing configuration. + ### Entity Fields The `Entity Fields` tab defines fields translated by [reporting entities](../../../ontology/docs/building_config.md#reporting-physical-devices) and linked to [virtual entities](../../../onotology/docs/building_config.md#virtual-devices) defined in the `Entities` tab. From c87a477522b278eadb0e59bac22b143d3ea34853 Mon Sep 17 00:00:00 2001 From: travis Date: Thu, 12 Oct 2023 09:50:11 -0700 Subject: [PATCH 3/3] pylint fixes --- tools/abel/model/guid_to_entity_map.py | 2 +- tools/abel/tests/entity_field_test.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/abel/model/guid_to_entity_map.py b/tools/abel/model/guid_to_entity_map.py index 545127d141..5576d3becd 100644 --- a/tools/abel/model/guid_to_entity_map.py +++ b/tools/abel/model/guid_to_entity_map.py @@ -67,7 +67,7 @@ def AddEntity(self, entity: ...) -> None: raise ValueError('Cannot add None values to the guid to entity map.') elif not entity.bc_guid: raise AttributeError(f'{entity.code}: guid missing') - elif entity.bc_guid not in list(self._guid_to_entity_map.keys()): + elif entity.bc_guid not in list(self._guid_to_entity_map): self._guid_to_entity_map[entity.bc_guid] = entity elif self._guid_to_entity_map.get(entity.bc_guid) == entity: # Do nothing, this mapping already exists diff --git a/tools/abel/tests/entity_field_test.py b/tools/abel/tests/entity_field_test.py index 9728eedb42..0c0819a93b 100644 --- a/tools/abel/tests/entity_field_test.py +++ b/tools/abel/tests/entity_field_test.py @@ -93,6 +93,7 @@ def testMissingFieldInequality(self): def testMissingFieldEqualityRaisesTypeError(self): test_missing_field = MissingField.FromDict(TEST_MISSING_FIELD_DICT) + # pylint: disable=unnecessary-dunder-call with self.assertRaises(TypeError): test_missing_field.__eq__('not a field')