[issue 2279] Update how we generate ERD diagrams for the database sch…

…ema (#2478) ## Summary Fixes #{[2279](#2279 (comment))} ### Time to review: __5 mins__ ## Changes proposed Replace sadisplay library with eralchemy to generate ERDs ## Context for reviewers Updated existing script `create_erds` to use the new library for generating ERDs. Newly generated diagrams(included) contain all the table and columns in the database. ## Additional information Screenshots, GIF demos, code examples or output to help show the changes working as expected. --------- Co-authored-by: nava-platform-bot <[email protected]>
HHS · Oct 17, 2024 · 9a7cf92 · 9a7cf92
1 parent 40ea8c7
commit 9a7cf92
Show file tree

Hide file tree

Showing 8 changed files with 708 additions and 674 deletions.
diff --git a/api/bin/create_erds.py b/api/bin/create_erds.py
@@ -1,19 +1,13 @@
 # Generate database schema diagrams from our SQLAlchemy models
-import codecs
 import logging
-import os
 import pathlib
-from typing import Any
 
-import pydot
-import sadisplay
+from eralchemy import render_er
+from sqlalchemy import MetaData
 
-import src.db.models.staging.forecast as staging_forecast_models
-import src.db.models.staging.opportunity as staging_opportunity_models
-import src.db.models.staging.synopsis as staging_synopsis_models
 import src.logging
-from src.db.models import agency_models, opportunity_models
-from src.db.models.transfer import topportunity_models
+from src.db.models.base import ApiSchemaTable
+from src.db.models.staging.staging_base import StagingBase
 
 logger = logging.getLogger(__name__)
 
@@ -23,56 +17,20 @@
 ERD_FOLDER = pathlib.Path(__file__).parent.resolve()
 
 # If we want to generate separate files for more specific groups, we can set that up here
-API_MODULES = (
-    opportunity_models,
-    agency_models,
-)
-STAGING_TABLE_MODULES = (
-    staging_opportunity_models,
-    staging_forecast_models,
-    staging_synopsis_models,
-)
-TRANSFER_TABLE_MODULES = (topportunity_models,)
+STAGING_BASE_METADATA = StagingBase.metadata
+API_BASE_METADATA = ApiSchemaTable.metadata
 
-# Any modules you add above, merge together for the full schema to be generated
-ALL_MODULES = API_MODULES + STAGING_TABLE_MODULES + TRANSFER_TABLE_MODULES
 
-
-def create_erds(modules: Any, file_name: str) -> None:
+def create_erds(metadata: MetaData, file_name: str) -> None:
     logger.info("Generating ERD diagrams for %s", file_name)
 
-    items = []
-    for module in modules:
-        items.extend([getattr(module, attr) for attr in dir(module)])
-
-    description = sadisplay.describe(
-        items,
-        show_methods=True,
-        show_properties=True,
-        show_indexes=True,
-    )
-
-    dot_file_name = ERD_FOLDER / f"{file_name}.dot"
-
-    # We create a temporary .dot file which we then convert to a png
-    with codecs.open(str(dot_file_name), "w", encoding="utf8") as f:
-        f.write(sadisplay.dot(description))
-
-    (graph,) = pydot.graph_from_dot_file(dot_file_name)
-
-    png_file_path = ERD_FOLDER / f"{file_name}.png"
-    logger.info("Creating ERD diagram at %s", png_file_path)
-    graph.write_png(png_file_path)
-
-    # remove the temporary .dot file
-    os.remove(dot_file_name)
+    png_file_path = str(ERD_FOLDER / f"{file_name}.png")
+    render_er(metadata, png_file_path)
 
 
 def main() -> None:
     with src.logging.init(__package__):
         logger.info("Generating ERD diagrams")
 
-        create_erds(ALL_MODULES, "full-schema")
-        create_erds(API_MODULES, "api-schema")
-        create_erds(STAGING_TABLE_MODULES, "staging-schema")
-        create_erds(TRANSFER_TABLE_MODULES, "transfer-schema")
+        create_erds(STAGING_BASE_METADATA, "staging-schema")
+        create_erds(API_BASE_METADATA, "api-schema")
diff --git a/api/poetry.lock b/api/poetry.lock
diff --git a/api/pyproject.toml b/api/pyproject.toml
@@ -46,6 +46,9 @@ ruff = "^0.6.0"
 debugpy = "^1.8.1"
 freezegun = "^1.5.0"
 types-requests = "^2.31"
+graphviz = "^0.20.3"
+eralchemy = "^1.5.0"
+
 
 [build-system]
 requires = ["poetry-core>=1.0.0"]

diff --git a/documentation/api/database/erds/README.md b/documentation/api/database/erds/README.md
@@ -1,16 +1,14 @@
 # Overview
-This folder contains ERD diagrams representing our database schema for both our postgres DB
+This folder contains ERD diagrams representing the `api` and `staging` schema in the `grants-db` database. 
 
 Diagrams can be manually generated by running `make create-erds` from the api folder.
 
 # Dependencies
 If running outside of Docker, you must install `graphviz` (`brew install graphviz`) for this to work, this should be automatically installed as part of the Dockerfile inside Docker.
 
 # Caveats
-The diagrams generated are based on our SQLAlchemy models, and not the database itself, so there are a few differences.
+The diagrams generated are based on our SQLAlchemy models metadata, and not the database itself, so there are a few differences.
 
-* Fields that we name different in-code will have a different name
-* The table names use the class name
 * Property fields are SQLAlchemy only and generally represent relationships (ie. values fetched via a foreign key `join`)
 
 # Files
@@ -20,9 +18,3 @@ The diagrams generated are based on our SQLAlchemy models, and not the database
 
 ## Staging Schema
 ![Staging Table ERD](staging-schema.png)
-
-## Transfer Table Schema
-![Transfer Table ERD](transfer-schema.png)
-
-## Full Schema
-![Postgres ERD](full-schema.png)
diff --git a/documentation/api/database/erds/api-schema.png b/documentation/api/database/erds/api-schema.png
diff --git a/documentation/api/database/erds/full-schema.png b/documentation/api/database/erds/full-schema.png
diff --git a/documentation/api/database/erds/staging-schema.png b/documentation/api/database/erds/staging-schema.png
diff --git a/documentation/api/database/erds/transfer-schema.png b/documentation/api/database/erds/transfer-schema.png