From 5deebc8b85f7d5ca68fe9306ada14897a9c26ebb Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 14 Aug 2023 20:32:00 +0000 Subject: [PATCH 01/20] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.0.282 → v0.0.284](https://github.com/astral-sh/ruff-pre-commit/compare/v0.0.282...v0.0.284) - [github.com/pre-commit/mirrors-mypy: v1.4.1 → v1.5.0](https://github.com/pre-commit/mirrors-mypy/compare/v1.4.1...v1.5.0) --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 71f520d..18ab770 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,12 +9,12 @@ repos: hooks: - id: black - repo: https://github.com/astral-sh/ruff-pre-commit - rev: 'v0.0.282' + rev: 'v0.0.284' hooks: - id: ruff args: [--fix, --exit-non-zero-on-fix] - repo: https://github.com/pre-commit/mirrors-mypy - rev: "v1.4.1" + rev: "v1.5.0" hooks: - id: mypy - repo: https://github.com/igorshubovych/markdownlint-cli From e12f5be58038cee30aae8089b0fc38ce6362a9cb Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 21 Aug 2023 20:37:37 +0000 Subject: [PATCH 02/20] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.0.284 → v0.0.285](https://github.com/astral-sh/ruff-pre-commit/compare/v0.0.284...v0.0.285) - [github.com/pre-commit/mirrors-mypy: v1.5.0 → v1.5.1](https://github.com/pre-commit/mirrors-mypy/compare/v1.5.0...v1.5.1) --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 18ab770..1ad0631 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,12 +9,12 @@ repos: hooks: - id: black - repo: https://github.com/astral-sh/ruff-pre-commit - rev: 'v0.0.284' + rev: 'v0.0.285' hooks: - id: ruff args: [--fix, --exit-non-zero-on-fix] - repo: https://github.com/pre-commit/mirrors-mypy - rev: "v1.5.0" + rev: "v1.5.1" hooks: - id: mypy - repo: https://github.com/igorshubovych/markdownlint-cli From 0fcf665c7d1c0754178de1adad2d52b83cae9699 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 28 Aug 2023 20:23:25 +0000 Subject: [PATCH 03/20] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.0.285 → v0.0.286](https://github.com/astral-sh/ruff-pre-commit/compare/v0.0.285...v0.0.286) --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1ad0631..a8dec92 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,7 +9,7 @@ repos: hooks: - id: black - repo: https://github.com/astral-sh/ruff-pre-commit - rev: 'v0.0.285' + rev: 'v0.0.286' hooks: - id: ruff args: [--fix, --exit-non-zero-on-fix] From 048e75fe34fc2e0d74650b80a6b2a6cab54e9171 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 1 Sep 2023 21:45:57 +0000 Subject: [PATCH 04/20] Bump ruff from 0.0.282 to 0.0.287 Bumps [ruff](https://github.com/astral-sh/ruff) from 0.0.282 to 0.0.287. - [Release notes](https://github.com/astral-sh/ruff/releases) - [Changelog](https://github.com/astral-sh/ruff/blob/main/BREAKING_CHANGES.md) - [Commits](https://github.com/astral-sh/ruff/compare/v0.0.282...v0.0.287) --- updated-dependencies: - dependency-name: ruff dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index cb94635..7864e3f 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -129,7 +129,7 @@ pyxlsb==1.0.10 # via pandas pyyaml==6.0 # via pre-commit -ruff==0.0.282 +ruff==0.0.287 # via datahub (pyproject.toml) six==1.16.0 # via python-dateutil From cde14f4bca7f699a790e6737d081765fb6d46a68 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 1 Sep 2023 21:48:59 +0000 Subject: [PATCH 05/20] Bump anyio from 3.7.0 to 4.0.0 Bumps [anyio](https://github.com/agronholm/anyio) from 3.7.0 to 4.0.0. - [Release notes](https://github.com/agronholm/anyio/releases) - [Changelog](https://github.com/agronholm/anyio/blob/master/docs/versionhistory.rst) - [Commits](https://github.com/agronholm/anyio/compare/3.7.0...4.0.0) --- updated-dependencies: - dependency-name: anyio dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 7864e3f..edd1522 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -4,7 +4,7 @@ # # pip-compile --extra=dev --output-file=requirements-dev.txt # -anyio==3.7.0 +anyio==4.0.0 # via # httpcore # starlette diff --git a/requirements.txt b/requirements.txt index 00e139b..897adba 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ # # pip-compile # -anyio==3.7.0 +anyio==4.0.0 # via starlette click==8.1.6 # via uvicorn From f7d88413dba035a2d14a0dd62c87f9ee2dd1702e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 1 Sep 2023 21:52:19 +0000 Subject: [PATCH 06/20] Bump build from 0.10.0 to 1.0.0 Bumps [build](https://github.com/pypa/build) from 0.10.0 to 1.0.0. - [Release notes](https://github.com/pypa/build/releases) - [Changelog](https://github.com/pypa/build/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pypa/build/compare/0.10.0...1.0.0) --- updated-dependencies: - dependency-name: build dependency-type: direct:development update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index edd1522..6a09546 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -12,7 +12,7 @@ attrs==23.1.0 # via pytest-mypy black==23.3.0 # via datahub (pyproject.toml) -build==0.10.0 +build==1.0.0 # via pip-tools certifi==2023.7.22 # via From c6fe86eb0379d9fbc96ce271ab05a455d686f086 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 1 Sep 2023 21:59:27 +0000 Subject: [PATCH 07/20] Bump httpcore from 0.17.2 to 0.17.3 Bumps [httpcore](https://github.com/encode/httpcore) from 0.17.2 to 0.17.3. - [Release notes](https://github.com/encode/httpcore/releases) - [Changelog](https://github.com/encode/httpcore/blob/master/CHANGELOG.md) - [Commits](https://github.com/encode/httpcore/compare/0.17.2...0.17.3) --- updated-dependencies: - dependency-name: httpcore dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 6a09546..05be631 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -49,7 +49,7 @@ h11==0.14.0 # uvicorn h5py==3.9.0 # via datahub (pyproject.toml) -httpcore==0.17.2 +httpcore==0.17.3 # via httpx httpx==0.24.1 # via datahub (pyproject.toml) From d3859f9dd2b44e4a1cac399c79aa0ffc597c1200 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 4 Sep 2023 20:43:08 +0000 Subject: [PATCH 08/20] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.0.286 → v0.0.287](https://github.com/astral-sh/ruff-pre-commit/compare/v0.0.286...v0.0.287) - [github.com/igorshubovych/markdownlint-cli: v0.35.0 → v0.36.0](https://github.com/igorshubovych/markdownlint-cli/compare/v0.35.0...v0.36.0) --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a8dec92..7f7fc59 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,7 +9,7 @@ repos: hooks: - id: black - repo: https://github.com/astral-sh/ruff-pre-commit - rev: 'v0.0.286' + rev: 'v0.0.287' hooks: - id: ruff args: [--fix, --exit-non-zero-on-fix] @@ -18,7 +18,7 @@ repos: hooks: - id: mypy - repo: https://github.com/igorshubovych/markdownlint-cli - rev: v0.35.0 + rev: v0.36.0 hooks: - id: markdownlint args: ["--disable", "MD013", "--"] From cede5007c8e8036a36039ee31e607e797f407ccd Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 11 Sep 2023 21:03:22 +0000 Subject: [PATCH 09/20] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/psf/black: 23.7.0 → 23.9.1](https://github.com/psf/black/compare/23.7.0...23.9.1) - [github.com/astral-sh/ruff-pre-commit: v0.0.287 → v0.0.288](https://github.com/astral-sh/ruff-pre-commit/compare/v0.0.287...v0.0.288) --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7f7fc59..406c5f5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,11 +5,11 @@ repos: - id: check-merge-conflict - id: debug-statements - repo: https://github.com/psf/black - rev: "23.7.0" + rev: "23.9.1" hooks: - id: black - repo: https://github.com/astral-sh/ruff-pre-commit - rev: 'v0.0.287' + rev: 'v0.0.288' hooks: - id: ruff args: [--fix, --exit-non-zero-on-fix] From 2ed76353bf3825d38fa43a55e05e522b84819c25 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 18 Sep 2023 20:48:33 +0000 Subject: [PATCH 10/20] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.0.288 → v0.0.290](https://github.com/astral-sh/ruff-pre-commit/compare/v0.0.288...v0.0.290) --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 406c5f5..c7fc71a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,7 +9,7 @@ repos: hooks: - id: black - repo: https://github.com/astral-sh/ruff-pre-commit - rev: 'v0.0.288' + rev: 'v0.0.290' hooks: - id: ruff args: [--fix, --exit-non-zero-on-fix] From 3b15920d001653dea7fc7f65d741d3ffaadbad73 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 25 Sep 2023 21:08:30 +0000 Subject: [PATCH 11/20] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.0.290 → v0.0.291](https://github.com/astral-sh/ruff-pre-commit/compare/v0.0.290...v0.0.291) - [github.com/igorshubovych/markdownlint-cli: v0.36.0 → v0.37.0](https://github.com/igorshubovych/markdownlint-cli/compare/v0.36.0...v0.37.0) --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c7fc71a..661beea 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,7 +9,7 @@ repos: hooks: - id: black - repo: https://github.com/astral-sh/ruff-pre-commit - rev: 'v0.0.290' + rev: 'v0.0.291' hooks: - id: ruff args: [--fix, --exit-non-zero-on-fix] @@ -18,7 +18,7 @@ repos: hooks: - id: mypy - repo: https://github.com/igorshubovych/markdownlint-cli - rev: v0.36.0 + rev: v0.37.0 hooks: - id: markdownlint args: ["--disable", "MD013", "--"] From 9eb3cf3007114c8b806b264ae00bb3ae780d3e4e Mon Sep 17 00:00:00 2001 From: CWestICL Date: Thu, 21 Sep 2023 16:25:58 +0100 Subject: [PATCH 12/20] DSR start parameter --- datahub/main.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/datahub/main.py b/datahub/main.py index 9318def..845d712 100644 --- a/datahub/main.py +++ b/datahub/main.py @@ -155,7 +155,9 @@ def upload_dsr(file: UploadFile) -> dict[str, str | None]: @app.get("/dsr", response_class=ORJSONResponse) -def get_dsr_data(start: int = 0, end: int | None = None) -> ORJSONResponse: +def get_dsr_data( + start: int = len(dt.dsr_data) - 1, end: int | None = None +) -> ORJSONResponse: """GET method function for getting DSR data as JSON. It takes optional query parameters of: From df46580dcc25c7a5d5efdb26d54f772b3b114eac Mon Sep 17 00:00:00 2001 From: CWestICL Date: Fri, 22 Sep 2023 08:46:36 +0100 Subject: [PATCH 13/20] DSR column filtering --- datahub/dsr.py | 7 +++++++ datahub/main.py | 31 +++++++++++++++++++++++++++---- 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/datahub/dsr.py b/datahub/dsr.py index eebc691..3a15cfb 100644 --- a/datahub/dsr.py +++ b/datahub/dsr.py @@ -38,6 +38,13 @@ class Config: allow_population_by_field_name = True +dsr_headers = { + field["title"]: name + for name, field in DSRModel.schema(by_alias=False)["properties"].items() + if name != "frame" +} + + def validate_dsr_data(data: dict[str, NDArray | str]) -> None: """Validate the shapes of the arrays in the DSR data. diff --git a/datahub/main.py b/datahub/main.py index 845d712..75450aa 100644 --- a/datahub/main.py +++ b/datahub/main.py @@ -4,7 +4,7 @@ from . import data as dt from . import log -from .dsr import read_dsr_file, validate_dsr_data +from .dsr import dsr_headers, read_dsr_file, validate_dsr_data from .opal import OpalArrayData, OpalModel from .wesim import get_wesim @@ -156,7 +156,7 @@ def upload_dsr(file: UploadFile) -> dict[str, str | None]: @app.get("/dsr", response_class=ORJSONResponse) def get_dsr_data( - start: int = len(dt.dsr_data) - 1, end: int | None = None + start: int = len(dt.dsr_data) - 1, end: int | None = None, col: str | None = None ) -> ORJSONResponse: """GET method function for getting DSR data as JSON. @@ -176,22 +176,45 @@ def get_dsr_data( Args: start: Starting index for exported list end: Last index that will be included in exported list + col: Column names to filter by Returns: A Dict containing the DSR list """ # noqa: D301 log.info("Sending DSR data...") - log.debug(f"Query parameters:\n\nstart={start}\nend={end}\n") + log.debug(f"Query parameters:\n\nstart={start}\nend={end}\ncol={col}\n") if isinstance(end, int) and end < start: message = "End parameter cannot be less than Start parameter." log.error(message) raise HTTPException(status_code=400, detail=message) - log.info("Filtering data...") + log.info("Filtering data by index...") log.debug(f"Current DSR data length:\n\n{len(dt.dsr_data)}") filtered_data = dt.dsr_data[start : end + 1 if end else end] log.debug(f"Filtered DSR data length:\n\n{len(dt.dsr_data)}") + if isinstance(col, str): + log.debug(f"Columns:\n\n{col.split(',')}\n") + columns = col.lower().split(",") + + for col_name in columns: + dsr_columns = [x.lower() for x in dsr_headers.keys()] + + if col_name not in dsr_columns: + message = "One or more of the specified columns are invalid." + log.error(message) + raise HTTPException(status_code=400, detail=message) + + log.info("Filtering data by column...") + for frame in filtered_data: + delete_keys = [] + for key in frame.keys(): + if key.lower() not in columns: + delete_keys.append(key) + + for key in delete_keys: + del frame[key] + return ORJSONResponse({"data": filtered_data}) From 2eed8efc0e0802133b5725351e28d65d36e5534c Mon Sep 17 00:00:00 2001 From: CWestICL Date: Fri, 22 Sep 2023 14:48:33 +0100 Subject: [PATCH 14/20] DSR GET Column test --- datahub/main.py | 21 +++++++++++++-------- tests/test_dsr_api.py | 9 +++++++++ 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/datahub/main.py b/datahub/main.py index 75450aa..bf44feb 100644 --- a/datahub/main.py +++ b/datahub/main.py @@ -190,7 +190,8 @@ def get_dsr_data( log.info("Filtering data by index...") log.debug(f"Current DSR data length:\n\n{len(dt.dsr_data)}") - filtered_data = dt.dsr_data[start : end + 1 if end else end] + data = dt.dsr_data.copy() + filtered_index_data = data[start : end + 1 if end else end] log.debug(f"Filtered DSR data length:\n\n{len(dt.dsr_data)}") if isinstance(col, str): @@ -206,16 +207,20 @@ def get_dsr_data( raise HTTPException(status_code=400, detail=message) log.info("Filtering data by column...") - for frame in filtered_data: - delete_keys = [] + filtered_data = [] + for frame in filtered_index_data: + log.debug(f"Frame keys: {frame.keys()}") + filtered_keys = {} for key in frame.keys(): - if key.lower() not in columns: - delete_keys.append(key) + log.debug(f"Key: {key}") + if key.lower() in columns: + filtered_keys[key] = frame[key] + log.debug(f"Filtered keys: {filtered_keys.keys()}") + filtered_data.append(filtered_keys) - for key in delete_keys: - del frame[key] + return ORJSONResponse({"data": filtered_data}) - return ORJSONResponse({"data": filtered_data}) + return ORJSONResponse({"data": filtered_index_data}) @app.get("/wesim") diff --git a/tests/test_dsr_api.py b/tests/test_dsr_api.py index 2874532..06d5783 100644 --- a/tests/test_dsr_api.py +++ b/tests/test_dsr_api.py @@ -76,3 +76,12 @@ def test_get_dsr_api(dsr_data): response = client.get("/dsr?start=1") assert response.json()["data"][0]["Name"] == dt.dsr_data[1]["Name"] + + response = client.get("/dsr?col=activities") + assert len(response.json()["data"][0].keys()) == 1 + assert "Activities" in response.json()["data"][0].keys() + + response = client.get("/dsr?col=activity types,kwh cost") + assert len(response.json()["data"][0].keys()) == 2 + assert "Activity Types" in response.json()["data"][0].keys() + assert "kWh Cost" in response.json()["data"][0].keys() From f4b0221c71610b12d9e531ad9eeef4a711a599ba Mon Sep 17 00:00:00 2001 From: CWestICL Date: Fri, 22 Sep 2023 16:01:56 +0100 Subject: [PATCH 15/20] Removed some debug logs --- datahub/main.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/datahub/main.py b/datahub/main.py index bf44feb..fa20d59 100644 --- a/datahub/main.py +++ b/datahub/main.py @@ -209,13 +209,10 @@ def get_dsr_data( log.info("Filtering data by column...") filtered_data = [] for frame in filtered_index_data: - log.debug(f"Frame keys: {frame.keys()}") filtered_keys = {} for key in frame.keys(): - log.debug(f"Key: {key}") if key.lower() in columns: filtered_keys[key] = frame[key] - log.debug(f"Filtered keys: {filtered_keys.keys()}") filtered_data.append(filtered_keys) return ORJSONResponse({"data": filtered_data}) From e92f96e3c36284ffa31f2f4b93a06bfd76f3dd3e Mon Sep 17 00:00:00 2001 From: CWestICL Date: Fri, 22 Sep 2023 16:45:15 +0100 Subject: [PATCH 16/20] Index filtering tests --- tests/test_dsr_api.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/test_dsr_api.py b/tests/test_dsr_api.py index 06d5783..ef44935 100644 --- a/tests/test_dsr_api.py +++ b/tests/test_dsr_api.py @@ -73,10 +73,25 @@ def test_get_dsr_api(dsr_data): new_data = dsr_data.copy() new_data["Name"] = "A new entry" dt.dsr_data.append(new_data) + new_data = dsr_data.copy() + new_data["Name"] = "Another new entry" + dt.dsr_data.append(new_data) + response = client.get("/dsr") + assert response.json()["data"][0]["Name"] == dt.dsr_data[2]["Name"] + + # Checks index filtering response = client.get("/dsr?start=1") + assert len(response.json()["data"]) == 2 assert response.json()["data"][0]["Name"] == dt.dsr_data[1]["Name"] + assert response.json()["data"][1]["Name"] == dt.dsr_data[2]["Name"] + + response = client.get("/dsr?start=0&end=1") + assert len(response.json()["data"]) == 2 + assert response.json()["data"][0]["Name"] == dt.dsr_data[0]["Name"] + assert response.json()["data"][1]["Name"] == dt.dsr_data[1]["Name"] + # Checks column filtering response = client.get("/dsr?col=activities") assert len(response.json()["data"][0].keys()) == 1 assert "Activities" in response.json()["data"][0].keys() From 230dccda4d96fc61d5c7c03e646e5d50f2bd0bb1 Mon Sep 17 00:00:00 2001 From: CWestICL Date: Mon, 25 Sep 2023 11:02:50 +0100 Subject: [PATCH 17/20] Col and index filtering test --- tests/test_dsr_api.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/test_dsr_api.py b/tests/test_dsr_api.py index ef44935..7dda35b 100644 --- a/tests/test_dsr_api.py +++ b/tests/test_dsr_api.py @@ -100,3 +100,10 @@ def test_get_dsr_api(dsr_data): assert len(response.json()["data"][0].keys()) == 2 assert "Activity Types" in response.json()["data"][0].keys() assert "kWh Cost" in response.json()["data"][0].keys() + + response = client.get("/dsr?start=1&col=activities") + assert len(response.json()["data"]) == 2 + assert len(response.json()["data"][0].keys()) == 1 + assert "Activities" in response.json()["data"][0].keys() + assert len(response.json()["data"][1].keys()) == 1 + assert "Activities" in response.json()["data"][1].keys() From 0552da44eac0658f33b227752afd28847a11b2eb Mon Sep 17 00:00:00 2001 From: CWestICL Date: Mon, 25 Sep 2023 11:11:01 +0100 Subject: [PATCH 18/20] Updated docstring --- datahub/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datahub/main.py b/datahub/main.py index fa20d59..9896a11 100644 --- a/datahub/main.py +++ b/datahub/main.py @@ -176,7 +176,7 @@ def get_dsr_data( Args: start: Starting index for exported list end: Last index that will be included in exported list - col: Column names to filter by + col: Column names to filter by, multiple values seperated by comma Returns: A Dict containing the DSR list From 500f581b9b0857912d095db8d110739cb046fc30 Mon Sep 17 00:00:00 2001 From: CWestICL Date: Mon, 25 Sep 2023 11:23:46 +0100 Subject: [PATCH 19/20] Updated dsr_headers --- datahub/dsr.py | 1 - 1 file changed, 1 deletion(-) diff --git a/datahub/dsr.py b/datahub/dsr.py index 3a15cfb..2091da0 100644 --- a/datahub/dsr.py +++ b/datahub/dsr.py @@ -41,7 +41,6 @@ class Config: dsr_headers = { field["title"]: name for name, field in DSRModel.schema(by_alias=False)["properties"].items() - if name != "frame" } From f6dafe27199f61ff54500fd74494626835181b7e Mon Sep 17 00:00:00 2001 From: Adrian D'Alessandro Date: Mon, 25 Sep 2023 17:22:37 +0100 Subject: [PATCH 20/20] Update dsr get method to take non-whitespace column names --- datahub/main.py | 17 ++++++++--------- tests/test_dsr_api.py | 2 +- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/datahub/main.py b/datahub/main.py index 9896a11..c41eb95 100644 --- a/datahub/main.py +++ b/datahub/main.py @@ -156,21 +156,22 @@ def upload_dsr(file: UploadFile) -> dict[str, str | None]: @app.get("/dsr", response_class=ORJSONResponse) def get_dsr_data( - start: int = len(dt.dsr_data) - 1, end: int | None = None, col: str | None = None + start: int = -1, end: int | None = None, col: str | None = None ) -> ORJSONResponse: """GET method function for getting DSR data as JSON. It takes optional query parameters of: - - `start`: Starting index for exported list - - `end`: Last index that will be included in exported list + - `start`: Starting index for exported list. Defaults to -1 for the most recent + entry only. + - `end`: Last index that will be included in exported list. + - `col`: A comma-separated list of which columns/keys within the data to get. + These values are all lower-case and spaces are replaced by underscores. And returns a dictionary containing the DSR data in JSON format. This can be converted back to a DataFrame using the following: `pd.DataFrame(**data)` - TODO: Ensure data is json serializable or returned in binary format - \f Args: @@ -199,9 +200,7 @@ def get_dsr_data( columns = col.lower().split(",") for col_name in columns: - dsr_columns = [x.lower() for x in dsr_headers.keys()] - - if col_name not in dsr_columns: + if col_name not in dsr_headers.values(): message = "One or more of the specified columns are invalid." log.error(message) raise HTTPException(status_code=400, detail=message) @@ -211,7 +210,7 @@ def get_dsr_data( for frame in filtered_index_data: filtered_keys = {} for key in frame.keys(): - if key.lower() in columns: + if dsr_headers[key.title()] in columns: filtered_keys[key] = frame[key] filtered_data.append(filtered_keys) diff --git a/tests/test_dsr_api.py b/tests/test_dsr_api.py index 7dda35b..baa8c39 100644 --- a/tests/test_dsr_api.py +++ b/tests/test_dsr_api.py @@ -96,7 +96,7 @@ def test_get_dsr_api(dsr_data): assert len(response.json()["data"][0].keys()) == 1 assert "Activities" in response.json()["data"][0].keys() - response = client.get("/dsr?col=activity types,kwh cost") + response = client.get("/dsr?col=activity_types,kwh_cost") assert len(response.json()["data"][0].keys()) == 2 assert "Activity Types" in response.json()["data"][0].keys() assert "kWh Cost" in response.json()["data"][0].keys()