WIP refactoring

.github/workflows/update-archives.yml

@@ -19,10 +19,10 @@ jobs:
           git config --global user.name "modm update bot"
 
 
-      - name: Setup Python 3.11
+      - name: Setup Python 3.12
         uses: actions/setup-python@v4
         with:
-          python-version: "3.11"
+          python-version: "3.12"
 
       - name: Install Python Dependencies
         run: |
@@ -98,10 +98,10 @@ jobs:
           git config --global user.name "modm update bot"
 
 
-      - name: Setup Python 3.11
+      - name: Setup Python 3.12
         uses: actions/setup-python@v4
         with:
-          python-version: "3.11"
+          python-version: "3.12"
 
       - name: Install Python Dependencies
         run: |

src/modm_data/html/document.py

@@ -1,7 +1,7 @@
 # Copyright 2022, Niklas Hauser
 # SPDX-License-Identifier: MPL-2.0
 
-import re
+import re, os
 import logging
 from pathlib import Path
 from functools import cached_property
@@ -13,7 +13,7 @@
 class Document:
     def __init__(self, path: str):
         self.path = Path(path)
-        self.relpath = self.path.relative_to(Path().cwd())
+        self.relpath = os.path.relpath(self.path, Path().cwd())
         self.fullname = self.path.stem
         self.name = self.fullname.split("-")[0]
         self.version = self.fullname.split("-")[1]
@@ -41,6 +41,8 @@ def chapter(self, pattern: str) -> Chapter:
             LOGGER.error(f"Cannot find chapter with pattern '{pattern}'!")
         if len(chapters) > 1:
             LOGGER.error(f"Found multiple chapters with pattern '{pattern}'!")
+            for chapter in chapters:
+                LOGGER.error(f"  - {chapter.name}")
         assert len(chapters) == 1
         return chapters[0]
 

src/modm_data/html/stmicro/__init__.py

@@ -1,7 +1,8 @@
 # Copyright 2022, Niklas Hauser
 # SPDX-License-Identifier: MPL-2.0
 
-from .datasheet import DatasheetMicro, DatasheetSensor
+from .datasheet_sensor import DatasheetSensor
+from .datasheet_stm32 import DatasheetStm32
 from .reference import ReferenceManual
 from .document import load_documents, load_document_devices
 from .document import datasheet_for_device, reference_manual_for_device

src/modm_data/html/stmicro/datasheet_sensor.py

@@ -0,0 +1,25 @@
+# Copyright 2022, Niklas Hauser
+# SPDX-License-Identifier: MPL-2.0
+
+import re
+import itertools
+from pathlib import Path
+from functools import cached_property, cache
+from collections import defaultdict
+
+from .helper import split_device_filter, split_package
+from ...html.text import ReDict
+
+import modm_data.html as html
+
+
+class DatasheetSensor(html.Document):
+    def __init__(self, path: str):
+        super().__init__(path)
+
+    def __repr__(self) -> str:
+        return f"DSsensor({self.fullname})"
+
+    @cache
+    def register_map(self, assert_table=True):
+        pass

src/modm_data/html/stmicro/datasheet.py → src/modm_data/html/stmicro/datasheet_stm32.py

@@ -14,14 +14,14 @@
 import modm_data.html as html
 
 
-class DatasheetMicro(html.Document):
+class DatasheetStm32(html.Document):
     def __init__(self, path: str):
         super().__init__(path)
         self._id = {}
         self._devices = {}
 
     def __repr__(self) -> str:
-        return f"DSµC({self.fullname})"
+        return f"DSstm32({self.fullname})"
 
     @cached_property
     def device_family(self) -> str:
@@ -247,11 +247,3 @@ def packages_pins(self):
                         data_pin["alternate"][af].extend(signals)
 
         return data_packages, data_pins
-
-
-class DatasheetSensor(html.Document):
-    def __init__(self, path: str):
-        super().__init__(path)
-
-    def __repr__(self) -> str:
-        return f"DSsens({self.fullname})"

src/modm_data/html/stmicro/document.py

@@ -5,7 +5,8 @@
 from collections import defaultdict
 from ...html import Document
 from ...utils import cache_path, ext_path
-from .datasheet import DatasheetMicro, DatasheetSensor
+from .datasheet_stm32 import DatasheetStm32
+from .datasheet_sensor import DatasheetSensor
 from .reference import ReferenceManual
 from ...owl import DeviceIdentifier
 from ...owl.stmicro import did_from_string
@@ -27,15 +28,15 @@ def load_documents() -> list:
             # FIXME: Better detection that DS13252 is a STM32WB55 module, not a chip!
             if any("STM32" in h.html for h in chap[0].headings()) and \
                 "DS13252" not in doc.name and "DS14096" not in doc.name:
-                documents[doc.name][doc.version] = DatasheetMicro(path)
+                documents[doc.name][doc.version] = DatasheetStm32(path)
             else:
                 documents[doc.name][doc.version] = DatasheetSensor(path)
         elif "RM" in doc.name:
             documents[doc.name][doc.version] = ReferenceManual(path)
     return documents
 
 
-def load_document_devices(use_cached=True) -> tuple[dict[DeviceIdentifier, DatasheetMicro],
+def load_document_devices(use_cached=True) -> tuple[dict[DeviceIdentifier, DatasheetStm32],
                                                     dict[DeviceIdentifier, ReferenceManual]]:
     global DOCUMENT_CACHE
     if DOCUMENT_CACHE is not None:
@@ -48,7 +49,7 @@ def load_document_devices(use_cached=True) -> tuple[dict[DeviceIdentifier, Datas
 
         docs = {}
         for path in set(json_data["ds"].values()):
-            docs[path] = DatasheetMicro(path)
+            docs[path] = DatasheetStm32(path)
         for path in set(json_data["rm"].values()):
             docs[path] = ReferenceManual(path)
         datasheets = {did_from_string(did): docs[path]
@@ -63,7 +64,7 @@ def load_document_devices(use_cached=True) -> tuple[dict[DeviceIdentifier, Datas
             doc = list(versions.values())[-1]
             # print(doc.path_pdf.relative_to(Path().cwd()), doc.path.relative_to(Path().cwd()))
             # print(doc.devices)
-            if isinstance(doc, DatasheetMicro):
+            if isinstance(doc, DatasheetStm32):
                 if not doc.devices:
                     raise ValueError(f"{doc} has no associated devices!")
                 for dev in doc.devices:
@@ -120,7 +121,7 @@ def _document_for_device(did: DeviceIdentifier, documents):
     return None
 
 
-def datasheet_for_device(did: DeviceIdentifier) -> DatasheetMicro:
+def datasheet_for_device(did: DeviceIdentifier) -> DatasheetStm32:
     datasheets, _ = load_document_devices()
     return _document_for_device(did, datasheets)
 

src/modm_data/html2owl/stmicro/__main__.py

@@ -9,7 +9,7 @@
 from collections import defaultdict
 from multiprocessing.pool import ThreadPool
 
-from modm_data.html.stmicro import DatasheetMicro, ReferenceManual, load_documents
+from modm_data.html.stmicro import DatasheetStm32, ReferenceManual, load_documents
 from modm_data.owl import Store
 from modm_data.py2owl.stmicro import owl_from_doc
 
@@ -25,7 +25,7 @@ def main():
         for name, versions in load_documents().items():
             # always use latest version for now
             doc = list(versions.values())[-1]
-            if isinstance(doc, DatasheetMicro):
+            if isinstance(doc, DatasheetStm32):
                 docs.append(doc)
             elif isinstance(doc, ReferenceManual):
                 docs.append(doc)
@@ -40,7 +40,7 @@ def main():
 
     path = Path(args.document).absolute()
     if path.stem.startswith("DS"):
-        doc = DatasheetMicro(path)
+        doc = DatasheetStm32(path)
     elif path.stem.startswith("RM"):
         doc = ReferenceManual(path)
 

src/modm_data/html2svd/stmicro/__init__.py

@@ -2,3 +2,4 @@
 # SPDX-License-Identifier: MPL-2.0
 
 from .reference import memory_map_from_reference_manual
+from .datasheet import memory_map_from_datasheet

src/modm_data/html2svd/stmicro/__main__.py

@@ -8,16 +8,17 @@
 from pathlib import Path
 from multiprocessing.pool import ThreadPool
 
-from modm_data.html.stmicro import ReferenceManual, load_documents
-from modm_data.html2svd.stmicro import memory_map_from_reference_manual
+from modm_data.html.stmicro import ReferenceManual, DatasheetSensor, load_documents
+from modm_data.html2svd.stmicro import memory_map_from_reference_manual, memory_map_from_datasheet
 from modm_data.svd import format_svd, write_svd
 from modm_data.utils import ext_path
 from anytree import RenderTree
 
 
 def main():
     parser = argparse.ArgumentParser()
-    parser.add_argument("--document", type=str, default="")
+    parser.add_argument("--stm32", type=Path)
+    parser.add_argument("--sensor", type=Path)
     parser.add_argument("--all", action="store_true", default=False)
     args = parser.parse_args()
 
@@ -30,20 +31,25 @@ def main():
                 docs.append(doc)
 
         Path("log/stmicro/svd").mkdir(exist_ok=True, parents=True)
-        calls = [f"python3 -m modm_data.html2svd.stmicro --document {doc.path} "
+        calls = [f"python3 -m modm_data.html2svd.stmicro --stm32 {doc.path} "
                  f"> log/stmicro/svd/html_{doc.name}.txt 2>&1" for doc in docs]
         with ThreadPool() as pool:
             retvals = list(tqdm.tqdm(pool.imap(lambda c: subprocess.run(c, shell=True), calls), total=len(calls)))
         for retval, call in zip(retvals, calls):
             if retval.returncode != 0: print(call)
         return all(r.returncode == 0 for r in retvals)
 
-    path = Path(args.document).absolute()
-    doc = ReferenceManual(path)
+    if args.stm32:
+        doc = ReferenceManual(args.stm32.absolute())
+    elif args.sensor:
+        doc = DatasheetSensor(args.sensor.absolute())
     print(doc.path_pdf.relative_to(Path().cwd()),
           doc.path.relative_to(Path().cwd()))
 
-    mmaptrees = memory_map_from_reference_manual(doc)
+    if args.stm32:
+        mmaptrees = memory_map_from_reference_manual(doc)
+    elif args.sensor:
+        mmaptrees = memory_map_from_datasheet(doc)
     for mmaptree in mmaptrees:
         print(RenderTree(mmaptree, maxlevel=2))
         svd = format_svd(mmaptree)