Merge branch 'main' into dependabot/pip/types-requests-2.31.0.0

.pre-commit-config.yaml

@@ -3,21 +3,25 @@ repos:
     hooks:
       - id: isort
         name: isort
+        exclude: ^(contxt/generated)
         entry: poetry run isort
         language: system
         types: [python]
       - id: black
         name: black
+        exclude: ^(contxt/generated)
         entry: poetry run black
         language: system
         types: [python]
       - id: flake8
         name: flake8
+        exclude: ^(graphql/|contxt/generated)
         entry: poetry run flake8
         language: system
         types: [python]
       - id: mypy
         name: mypy
+        exclude: ^(contxt/generated)
         entry: poetry run mypy --ignore-missing-imports
         language: system
         require_serial: true

CONTRIBUTING.md

@@ -41,6 +41,19 @@ Our [CI pipeline](.github/workflows/ci.yaml) will run these tools on each commit
 poetry run pre-commit install
 ```
 
+### Updating the Nionic Graphql artifacts
+
+
+```sh
+poetry shell
+
+python3 -m sgqlc.introspection --exclude-deprecated --exclude-description http://localhost:3000/graphql graphql/nionic_schema.json
+
+sgqlc-codegen schema graphql/nionic_schema.json contxt/generated/nionic_schema.py
+
+sgqlc-codegen operation --schema graphql/nionic_schema.json .nionic_schema contxt/generated/nionic_queries.py graphql/nionic_queries.graphql
+```
+
 ### Create Release
 
 On a commit to main, our [CI pipeline](.github/workflows/ci.yaml) will bump the version (determined by [conventional commits](https://www.conventionalcommits.org/)) and publish a new release to PyPI.

README.md

@@ -31,3 +31,4 @@ Please refer to <https://ndustrialio.github.io/contxt-sdk-python>.
 ## Contributing
 
 Please refer to [CONTRIBUTING.md](CONTRIBUTING.md).
+

contxt/cli/commands/ems.py

@@ -35,36 +35,46 @@ def mains(
     clients: Clients, facility_id: int, resource_type: ResourceType, fields: List[str], sort: str
 ) -> None:
     """Get main services"""
-    items = clients.ems.get_main_services(facility_id=facility_id, resource_type=resource_type)
+    results = clients.nionic.get_main_services(facility_id=facility_id, resource_type=resource_type)
+    items = [
+        MainService(
+            x.id, x.facility_id, x.name, x.type, x.demand.id, x.usage.id, x.created_at, x.updated_at
+        )
+        for x in results
+    ]
     print_table(items=items, keys=fields, sort_by=sort)
 
 
 @ems.command()
 @click.argument("facility_id")
 @click.option("--resource-type", type=ResourceType, default="electric", help="Resource type")
-@click.option("--start", type=click.DateTime(), help="Start time")
-@click.option("--end", type=click.DateTime(), help="End time")
+@click.option("--start", type=click.DateTime(), help="Start time", required=True)
+@click.option("--end", type=click.DateTime(), help="End time", required=True)
 @click.pass_obj
 def main_data(
     clients: Clients, facility_id: int, resource_type: ResourceType, start: datetime, end: datetime
 ) -> None:
     """Get main service data"""
     data: Dict[datetime, Dict[str, Any]] = defaultdict(dict)
-    services = clients.ems.get_main_services(facility_id=facility_id, resource_type=resource_type)
+    services = clients.nionic.get_main_services(facility_id=facility_id, resource_type=resource_type)
     with click.progressbar(
         services,
         label="Downloading main service data",
         item_show_func=lambda s: f"Service {s.name}" if s else "",
     ) as services_:
         for service in services_:
-            for t, v in clients.iot.get_time_series_for_field(
-                field=service.usage_field,
-                start_time=start,
-                end_time=end,
+            for t, v in clients.nionic.get_data_point_data(
+                data_source_name=service.usage.data_source_name,
+                name=service.usage.name,
+                start=start.astimezone().isoformat(),
+                end=end.astimezone().isoformat(),
                 window=Window.MINUTELY,
                 per_page=5000,
             ):
-                data[t][service.usage_field.field_human_name] = v
+                if t in data and service.usage.name in data[t]:
+                    data[t][service.usage.name] = v + data[t][service.usage.name]
+                else:
+                    data[t][service.usage.name] = v
 
     # Dump
     print_table(items=data)
@@ -119,11 +129,11 @@ def usage(
 
 
 @ems.command()
-@click.argument("facility_ids", nargs=-1)
+@click.argument("facility_ids", nargs=-1, type=click.INT)
 @click.option(
     "--include",
     default="all",
-    callback=csv_callback(options=["spend", "usage"]),
+    callback=csv_callback(options=["mains", "usage"]),
     help="Data to export",
 )
 @click.option("--start", type=click.DateTime(), default=LAST_WEEK.isoformat(), help="Start time")
@@ -141,9 +151,9 @@ def export(
     """Export data for facilities"""
     with click.progressbar(
         facility_ids, label="Downloading data", item_show_func=lambda f: f"Facility {f}" if f else ""
-    ) as facility_ids_:
+    ):
         for facility in clients.nionic.get_facilities():
-            if facility.id not in facility_ids_:
+            if facility.id not in facility_ids:
                 continue
             fpath = output / facility.slug
 
@@ -157,16 +167,20 @@ def export(
             # Main service data
             if "mains" in include:
                 data: Dict[datetime, Dict[str, Any]] = defaultdict(dict)
-                services = clients.ems.get_main_services(facility_id=facility.id)
+                services = clients.nionic.get_main_services(facility_id=facility.id)
                 for service in services:
-                    for t, v in clients.iot.get_time_series_for_field(
-                        field=service.usage_field,
-                        start_time=start,
-                        end_time=end,
+                    for t, v in clients.nionic.get_data_point_data(
+                        data_source_name=service.usage.data_source_name,
+                        name=service.usage.name,
+                        start=start.astimezone().isoformat(),
+                        end=end.astimezone().isoformat(),
                         window=Window.MINUTELY,
                         per_page=5000,
                     ):
-                        data[t][service.usage_field.field_human_name] = v
+                        if t in data and service.usage.name in data[t]:
+                            data[t][service.usage.name] = v + data[t][service.usage.name]
+                        else:
+                            data[t][service.usage.name] = v
                 Serializer.to_csv(data, fpath / "ems" / "main_service_usage.csv")
 
     print(f"Wrote data to {output}")

contxt/cli/commands/iot.py

@@ -10,11 +10,27 @@
 from slugify import slugify
 
 from contxt.cli.clients import Clients
-from contxt.cli.utils import LAST_WEEK, NOW, ClickPath, fields_option, print_table, sort_option
+from contxt.cli.utils import (
+    LAST_WEEK,
+    NOW,
+    ClickPath,
+    fields_option,
+    print_table,
+    sort_option,
+    str_to_bool,
+)
 from contxt.models.iot import Feed, Field, FieldCategory, FieldGrouping, FieldValueType, Window
 from contxt.utils.serializer import Serializer
 
-NEW_FIELD_ATTRS = ["field_descriptor", "label", "value_type", "units", "grouping", "category"]
+NEW_FIELD_ATTRS = [
+    "field_descriptor",
+    "label",
+    "value_type",
+    "units",
+    "is_totalizer",
+    "grouping",
+    "category",
+]
 
 
 @click.group()
@@ -182,6 +198,7 @@ def create(clients: Clients, feed_key: str, input: IO[str]) -> None:
                     units=r["units"],
                     value_type=FieldValueType(r["value_type"].lower()),
                     is_hidden=False,
+                    is_totalizer=str_to_bool(r["is_totalizer"]),
                 ),
                 r["grouping"],
                 r["category"],

contxt/cli/commands/metrics.py

@@ -1,3 +1,5 @@
+from datetime import datetime
+
 import click
 
 from contxt.cli.clients import Clients
@@ -19,15 +21,14 @@ def labels(clients: Clients) -> None:
 
 @metrics.command()
 @click.option(
-    "--source-id",
+    "--facility-id",
     required=True,
-    help="Organization ID, defaults to first value in token if not specified",
-)
-@click.option(
-    "--label", required=True, help="Organization ID, defaults to first value in token if not specified"
+    help="Facility ID",
 )
+@click.option("--label", required=True, help="Facility ID")
+@click.option("--start", required=True, type=click.DateTime())
 @click.pass_obj
-def data(clients: Clients, source_id: str, label: str) -> None:
-    """Get facilities"""
-    facilities = clients.nionic.get_metric_data(label, source_id)
-    print(Serializer.to_table(facilities))
+def data(clients: Clients, facility_id: int, label: str, start: datetime) -> None:
+    """Get metric data"""
+    results = clients.nionic.get_metric_data(label, facility_id, start.astimezone().isoformat())
+    print(Serializer.to_table(results))

contxt/cli/commands/service_instances.py

@@ -3,7 +3,7 @@
 import click
 
 from contxt.cli.clients import Clients
-from contxt.cli.utils import OPTIONAL_PROMPT_KWARGS, fields_option, print_item, print_table, sort_option
+from contxt.cli.utils import fields_option, print_item, print_table, sort_option
 from contxt.models.contxt import ServiceInstance
 
 
@@ -29,7 +29,7 @@ def get(clients: Clients, service_id: Optional[str], fields: List[str], sort: st
 
 @service_instances.command()
 @click.option("--service-id", prompt=True)
-@click.option("--project-environment-id", **OPTIONAL_PROMPT_KWARGS)
+@click.option("--project-environment-id", required=False)
 @click.option("--name", prompt=True)
 @click.option("--slug", prompt=True)
 @click.option("--descriptor", prompt=True)

contxt/cli/utils.py

@@ -104,3 +104,12 @@ def __init__(self) -> None:
 
     def convert(self, *args, **kwargs) -> Any:
         return super().convert(*args, **kwargs).date()
+
+
+def str_to_bool(s):
+    if s.lower() in ("true", "t", "1"):
+        return True
+    elif s.lower() in ("false", "f", "0"):
+        return False
+    else:
+        raise ValueError("Cannot covert {} to a bool".format(s))

contxt/generated/nionic_queries.py

@@ -0,0 +1,79 @@
+import sgqlc.types
+import sgqlc.operation
+from . import nionic_schema
+
+_schema = nionic_schema
+_schema_root = _schema.nionic_schema
+
+__all__ = ('Operations',)
+
+
+def mutation_create_facility():
+    _op = sgqlc.operation.Operation(_schema_root.mutation_type, name='createFacility', variables=dict(name=sgqlc.types.Arg(sgqlc.types.non_null(_schema.String)), slug=sgqlc.types.Arg(sgqlc.types.non_null(_schema.String)), timezoneName=sgqlc.types.Arg(sgqlc.types.non_null(_schema.String))))
+    _op_create_facility = _op.create_facility(input={'facility': {'name': sgqlc.types.Variable('name'), 'slug': sgqlc.types.Variable('slug'), 'timezoneName': sgqlc.types.Variable('timezoneName')}})
+    _op_create_facility_facility = _op_create_facility.facility()
+    _op_create_facility_facility.id()
+    return _op
+
+
+class Mutation:
+    create_facility = mutation_create_facility()
+
+
+def query_facilities():
+    _op = sgqlc.operation.Operation(_schema_root.query_type, name='facilities')
+    _op_facilities = _op.facilities()
+    _op_facilities_nodes = _op_facilities.nodes()
+    _op_facilities_nodes.id()
+    _op_facilities_nodes.name()
+    _op_facilities_nodes.slug()
+    _op_facilities_nodes.address()
+    _op_facilities_nodes.city()
+    _op_facilities_nodes.state()
+    _op_facilities_nodes.zip()
+    _op_facilities_nodes.timezone_name()
+    return _op
+
+
+def query_metric_labels():
+    _op = sgqlc.operation.Operation(_schema_root.query_type, name='metricLabels')
+    _op_source_labels = _op.source_labels()
+    _op_source_labels.source_id()
+    _op_source_labels.label()
+    return _op
+
+
+def query_main_services():
+    _op = sgqlc.operation.Operation(_schema_root.query_type, name='mainServices', variables=dict(facilityId=sgqlc.types.Arg(sgqlc.types.non_null(_schema.Int))))
+    _op_main_services = _op.main_services(filter={'facilityId': {'equalTo': sgqlc.types.Variable('facilityId')}})
+    _op_main_services_nodes = _op_main_services.nodes()
+    _op_main_services_nodes.id()
+    _op_main_services_nodes.facility_id()
+    _op_main_services_nodes.name()
+    _op_main_services_nodes.type()
+    _op_main_services_nodes_usage = _op_main_services_nodes.usage()
+    _op_main_services_nodes_usage.id()
+    _op_main_services_nodes_usage.data_source_name()
+    _op_main_services_nodes_usage.name()
+    _op_main_services_nodes_usage.alias()
+    _op_main_services_nodes_usage.data_type()
+    _op_main_services_nodes_demand = _op_main_services_nodes.demand()
+    _op_main_services_nodes_demand.id()
+    _op_main_services_nodes_demand.data_source_name()
+    _op_main_services_nodes_demand.name()
+    _op_main_services_nodes_demand.alias()
+    _op_main_services_nodes_demand.data_type()
+    _op_main_services_nodes.created_at()
+    _op_main_services_nodes.updated_at()
+    return _op
+
+
+class Query:
+    facilities = query_facilities()
+    main_services = query_main_services()
+    metric_labels = query_metric_labels()
+
+
+class Operations:
+    mutation = Mutation
+    query = Query

contxt/models/ems.py

@@ -5,7 +5,6 @@
 
 from . import ApiField, ApiObject, Formatters, Parsers
 from .events import Event
-from .iot import Field
 
 
 class ResourceType(Enum):
@@ -22,10 +21,8 @@ class MainService(ApiObject):
         ApiField("facility_id", data_type=int),
         ApiField("name"),
         ApiField("type", attr_key="resource_type", data_type=ResourceType),
-        ApiField("demand_field_id", data_type=int),
-        ApiField("usage_field_id", data_type=int),
-        ApiField("demand_field", data_type=Field),
-        ApiField("usage_field", data_type=Field),
+        ApiField("demand_datapoint_id", data_type=int),
+        ApiField("usage_datapointid", data_type=int),
         ApiField("created_at", data_type=Parsers.datetime),
         ApiField("updated_at", data_type=Parsers.datetime),
     )
@@ -34,10 +31,8 @@ class MainService(ApiObject):
     facility_id: int
     name: str
     resource_type: ResourceType
-    demand_field_id: int
-    usage_field_id: int
-    demand_field: Field
-    usage_field: Field
+    demand_datapoint_id: int
+    usage_datapoint_id: int
     created_at: datetime
     updated_at: datetime
 
@@ -94,25 +89,6 @@ class MetricValue(ApiObject):
     is_estimated: Optional[bool] = None
 
 
-@dataclass
-class Facility(ApiObject):
-    _api_fields: ClassVar = (
-        ApiField("id", data_type=int),
-        ApiField("name"),
-        ApiField("organization_id"),
-        ApiField("main_services", data_type=MainService),
-        ApiField("created_at", data_type=Parsers.datetime),
-        ApiField("updated_at", data_type=Parsers.datetime),
-    )
-
-    id: int
-    name: str
-    organization_id: str
-    main_services: List[MainService]
-    created_at: datetime
-    updated_at: datetime
-
-
 @dataclass
 class UtilityPeriod(ApiObject):
     _api_fields: ClassVar = (