diff --git a/src/modm_data/__init__.py b/src/modm_data/__init__.py index 9f7e6ec..62d6876 100644 --- a/src/modm_data/__init__.py +++ b/src/modm_data/__init__.py @@ -6,28 +6,49 @@ """ from importlib.metadata import version, PackageNotFoundError + try: __version__ = version("modm_data") except PackageNotFoundError: __version__ = "0.0.1" -from . import cubehal -from . import cubemx -from . import cube2owl -from . import dl -from . import header2svd -from . import html -from . import html2owl -from . import html2svd -from . import owl -from . import pdf -from . import pdf2html -from . import svd -from . import utils +from . import ( + cubehal, + cubemx, + cube2owl, + dl, + header2svd, + html, + html2owl, + html2svd, + owl, + pdf, + pdf2html, + svd, + utils, +) + +__all__ = [ + "cube2owl", + "cubehal", + "cubemx", + "dl", + "header2svd", + "html", + "html2owl", + "html2svd", + "owl", + "pdf", + "pdf2html", + "svd", + "utils", +] # Silence warnings about path import order when calling modules directly import sys + if not sys.warnoptions: import warnings + warnings.filterwarnings("ignore", category=RuntimeWarning, module="runpy") diff --git a/src/modm_data/cube2owl/__main__.py b/src/modm_data/cube2owl/__main__.py index 75e4da5..9177916 100644 --- a/src/modm_data/cube2owl/__main__.py +++ b/src/modm_data/cube2owl/__main__.py @@ -1,19 +1,17 @@ # Copyright 2022, Niklas Hauser # SPDX-License-Identifier: MPL-2.0 -import re import tqdm import argparse import subprocess from pathlib import Path -from collections import defaultdict from multiprocessing.pool import ThreadPool from modm_data.cubemx import devices_from_prefix, devices_from_partname from modm_data.header2svd.stmicro import Header from modm_data.html.stmicro import datasheet_for_device, reference_manual_for_device from modm_data.owl.stmicro import create_ontology -from modm_data.py2owl.stmicro import * +from modm_data.py2owl.stmicro import owl_from_did, owl_from_cubemx, owl_from_header, owl_from_doc def main(): @@ -29,23 +27,26 @@ def main(): partnames = sorted(list(set(p[:9] for p in partnames))) Path("log/stmicro/owl").mkdir(exist_ok=True, parents=True) - calls = [f"python3 -m modm_data.cubemx2owl --prefix {partname} " - f"> log/stmicro/owl/cubemx_{partname}.txt 2>&1" - for partname in partnames] + calls = [ + f"python3 -m modm_data.cube2owl --prefix {partname} > log/stmicro/owl/cubemx_{partname}.txt 2>&1" + for partname in partnames + ] with ThreadPool() as pool: retvals = list(tqdm.tqdm(pool.imap(lambda c: subprocess.run(c, shell=True), calls), total=len(calls))) for retval, call in zip(retvals, calls): - if retval.returncode != 0: print(call) + if retval.returncode != 0: + print(call) return all(r.returncode == 0 for r in retvals) - for partname in devices_from_prefix(args.prefix.lower()): ds, rm = None, None for device in devices_from_partname(partname): did = device["id"] # Only change the documentation object if necessary to preserve caching - if ds != (nds := datasheet_for_device(did)): ds = nds - if rm != (nrm := reference_manual_for_device(did)): rm = nrm + if ds != (nds := datasheet_for_device(did)): + ds = nds + if rm != (nrm := reference_manual_for_device(did)): + rm = nrm print(did, ds, rm) if ds is None or rm is None: print(f"Ignoring {did} due to lack of documents") diff --git a/src/modm_data/cubehal/__init__.py b/src/modm_data/cubehal/__init__.py index a21f43a..456886f 100644 --- a/src/modm_data/cubehal/__init__.py +++ b/src/modm_data/cubehal/__init__.py @@ -11,3 +11,5 @@ """ from .dmamux_requests import read_request_map + +__all__ = ["read_request_map"] diff --git a/src/modm_data/cubehal/dmamux_requests.py b/src/modm_data/cubehal/dmamux_requests.py index 98b6985..ad85821 100644 --- a/src/modm_data/cubehal/dmamux_requests.py +++ b/src/modm_data/cubehal/dmamux_requests.py @@ -4,17 +4,18 @@ import re from pathlib import Path from ..utils import ext_path +from ..owl import DeviceIdentifier _CUBE_PATH = ext_path("stmicro/cubehal") _DMAMUX_PATTERN = re.compile(r"^\s*#define\s+(?P(LL_DMAMUX_REQ_\w+))\s+(?P(0x[0-9A-Fa-f]+))U") _REQUEST_PATTERN = re.compile(r"^\s*#define\s+(?P(DMA_REQUEST_\w+))\s+(?P([0-9]+))U") -def read_request_map(did: "modm_data.owl.DeviceIdentifier") -> dict[str, int]: + +def read_request_map(did: DeviceIdentifier) -> dict[str, int]: """ Reads the DMA requests mapping from the Low-Level (LL) CubeHAL header files. :param did: Device to query for. - :return: A dictionary of DMA trigger name to trigger position. """ dma_header = _get_hal_dma_header_path(did.family) @@ -27,7 +28,7 @@ def read_request_map(did: "modm_data.owl.DeviceIdentifier") -> dict[str, int]: elif did.family == "l4" and did.name[0] in ["p", "q", "r", "s"]: request_map = _read_requests_l4(did.name in ["p5", "q5"]) else: - raise RuntimeError("No DMAMUX request data available for {}".format(did)) + raise RuntimeError(f"No DMAMUX request data available for {did}") _fix_request_data(request_map) return request_map @@ -63,20 +64,21 @@ def _fix_request_data(request_map): else: m = dac_pattern.match(name) if m: - fix_requests["{}_CH{}".format(m.group("dac"), m.group("ch"))] = number + fix_requests[f'{m.group("dac")}_CH{m.group("ch")}'] = number request_map.update(fix_requests) + def _get_include_path(family): - return _CUBE_PATH / Path("stm32{}xx/Inc".format(family)) + return _CUBE_PATH / Path(f"stm32{family}xx/Inc") def _get_hal_dma_header_path(family): - return _get_include_path(family) / Path("stm32{}xx_hal_dma.h".format(family)) + return _get_include_path(family) / Path(f"stm32{family}xx_hal_dma.h") def _get_ll_dmamux_header_path(family): - return _get_include_path(family) / Path("stm32{}xx_ll_dmamux.h".format(family)) + return _get_include_path(family) / Path(f"stm32{family}xx_ll_dmamux.h") # For G4, H7 and L5 @@ -91,7 +93,7 @@ def _read_requests(hal_dma_file): # For G0, WB and WL def _read_requests_from_ll_dmamux(hal_dma_file, ll_dmamux_file): dmamux_map = _read_map(ll_dmamux_file, _DMAMUX_PATTERN) - request_pattern = re.compile("^\s*#define\s+(?P(DMA_REQUEST_\w+))\s+(?P(LL_DMAMUX?_REQ_\w+))\s*") + request_pattern = re.compile(r"^\s*#define\s+(?P(DMA_REQUEST_\w+))\s+(?P(LL_DMAMUX?_REQ_\w+))\s*") requests_map = _read_map(hal_dma_file, request_pattern) out_map = {} for r in requests_map.keys(): @@ -130,7 +132,7 @@ def _read_requests_l4(read_for_p5_q5): if m: name = m.group("name").replace("DMA_REQUEST_", "", 1) if name in out_map: - raise RuntimeError("Duplicate entry {}".format(name)) + raise RuntimeError(f"Duplicate entry {name}") out_map[name] = int(m.group("id")) return out_map @@ -143,6 +145,6 @@ def _read_map(filename, pattern): if m: name = m.group("name") if name in out_map: - raise RuntimeError("Duplicate entry {}".format(name)) + raise RuntimeError(f"Duplicate entry {name}") out_map[name] = m.group("id") return out_map diff --git a/src/modm_data/cubemx/__init__.py b/src/modm_data/cubemx/__init__.py index 67a40c8..3ce9f76 100644 --- a/src/modm_data/cubemx/__init__.py +++ b/src/modm_data/cubemx/__init__.py @@ -3,8 +3,13 @@ """ # STMicro STM32CubeMX Database - - """ -from .device_data import devices_from_prefix, devices_from_partname, cubemx_device_list +from .device_data import devices_from_family, devices_from_prefix, devices_from_partname, cubemx_device_list + +__all__ = [ + "devices_from_family", + "devices_from_prefix", + "devices_from_partname", + "cubemx_device_list", +] diff --git a/src/modm_data/cubemx/device_data.py b/src/modm_data/cubemx/device_data.py index 2791b3b..9ffd7b0 100644 --- a/src/modm_data/cubemx/device_data.py +++ b/src/modm_data/cubemx/device_data.py @@ -8,6 +8,7 @@ from collections import defaultdict from ..owl.stmicro import did_from_string +from ..owl import DeviceIdentifier from ..utils import ext_path, XmlReader from . import stm32_data from ..cubehal import read_request_map as dmamux_request_map @@ -19,6 +20,7 @@ _MCU_PATH = ext_path("stmicro/cubemx/mcu") _FAMILY_FILE = None + def _family_file() -> XmlReader: global _FAMILY_FILE if _FAMILY_FILE is None: @@ -63,7 +65,8 @@ def devices_from_prefix(prefix: str) -> list[str]: LOGGER.info("Found devices for prefix '{}': {}".format(prefix, ", ".join(devices))) return list(sorted(devices)) -def cubemx_device_list() -> list["modm_data.owl.DeviceIdentifier"]: + +def cubemx_device_list() -> list[DeviceIdentifier]: """ :return: A list of all STM32 device identifiers. """ @@ -83,16 +86,18 @@ def devices_from_partname(partname: str) -> list[dict[str]]: :param partname: A full STM32 device name. :return: a list of dictionaries containing a device specific data structure. """ - deviceNames = _family_file().query('//Family/SubFamily/Mcu[starts-with(@RefName,"{}")]' - .format(partname[:12] + "x" + partname[13:])) + deviceNames = _family_file().query( + f'//Family/SubFamily/Mcu[starts-with(@RefName,"{partname[:12]}x{partname[13:]}")]' + ) comboDeviceName = sorted([d.get("Name") for d in deviceNames])[0] device_file = XmlReader(os.path.join(_MCU_PATH, comboDeviceName + ".xml")) did = did_from_string(partname.lower()) - LOGGER.info("Parsing '{}'".format(did.string)) + LOGGER.info(f"Parsing '{did.string}'") # information about the core and architecture - cores = [c.text.lower().replace("arm ", "") for c in device_file.query('//Core')] - if len(cores) > 1: did.naming_schema += "@{core}" + cores = [c.text.lower().replace("arm ", "") for c in device_file.query("//Core")] + if len(cores) > 1: + did.naming_schema += "@{core}" devices = [_properties_from_id(comboDeviceName, device_file, did.copy(), c) for c in cores] return [d for d in devices if d is not None] @@ -107,9 +112,10 @@ def _properties_from_id(comboDeviceName, device_file, did, core): p = {"id": did, "core": core} # Maximum operating frequency - max_frequency = float(device_file.query('//Frequency')[0].text) + max_frequency = float(device_file.query("//Frequency")[0].text) # H7 dual-core devices run the M4 core at half the frequency as the M7 core - if did.get("core", "") == "m4": max_frequency /= 2.0; + if did.get("core", "") == "m4": + max_frequency /= 2.0 p["max_frequency"] = int(max_frequency * 1e6) # Information from the CMSIS headers @@ -131,30 +137,30 @@ def _properties_from_id(comboDeviceName, device_file, did, core): sizeIndexFlash = sizeArray.index(did.size) sizeIndexRam = sizeIndexFlash - rams = sorted([int(r.text) for r in device_file.query('//Ram')]) + rams = sorted([int(r.text) for r in device_file.query("//Ram")]) if sizeIndexRam >= len(rams): sizeIndexRam = len(rams) - 1 - flashs = sorted([int(f.text) for f in device_file.query('//Flash')]) + flashs = sorted([int(f.text) for f in device_file.query("//Flash")]) if sizeIndexFlash >= len(flashs): sizeIndexFlash = len(flashs) - 1 - p["ram"] = rams[sizeIndexRam] * 1024 p["flash"] = flashs[sizeIndexFlash] * 1024 memories = [] - for (mem_name, mem_start, mem_size) in stm32_data.getMemoryForDevice(did, p["flash"], p["ram"]): + for mem_name, mem_start, mem_size in stm32_data.getMemoryForDevice(did, p["flash"], p["ram"]): access = "rwx" - if did.family == "f4" and mem_name == "ccm": access = "rw"; - if "flash" in mem_name: access = "rx"; - memories.append({"name": mem_name, "access": access, "size": str(mem_size), - "start": "0x{:02X}".format(mem_start)}) + if did.family == "f4" and mem_name == "ccm": + access = "rw" + if "flash" in mem_name: + access = "rx" + memories.append({"name": mem_name, "access": access, "size": str(mem_size), "start": f"0x{mem_start:02X}"}) p["memories"] = memories # packaging - package = device_file.query('//@Package')[0] + package = device_file.query("//@Package")[0] p["pin-count"] = re.findall(r"[0-9]+", package)[0] p["package"] = re.findall(r"[A-Za-z\.]+", package)[0] @@ -169,12 +175,31 @@ def clean_up_version(version): modules = [] dmaFile = None - for ip in device_file.query('//IP'): + for ip in device_file.query("//IP"): # These IPs are all software modules, NOT hardware modules. Their version string is weird too. - software_ips = {"GFXSIMULATOR", "GRAPHICS", "FATFS", "TOUCHSENSING", "PDM2PCM", - "MBEDTLS", "FREERTOS", "CORTEX_M", "NVIC", "USB_DEVICE", - "USB_HOST", "LWIP", "LIBJPEG", "GUI_INTERFACE", "TRACER", - "FILEX", "LEVELX", "THREADX", "USBX", "LINKEDLIST", "NETXDUO"} + software_ips = { + "GFXSIMULATOR", + "GRAPHICS", + "FATFS", + "TOUCHSENSING", + "PDM2PCM", + "MBEDTLS", + "FREERTOS", + "CORTEX_M", + "NVIC", + "USB_DEVICE", + "USB_HOST", + "LWIP", + "LIBJPEG", + "GUI_INTERFACE", + "TRACER", + "FILEX", + "LEVELX", + "THREADX", + "USBX", + "LINKEDLIST", + "NETXDUO", + } if any(ip.get("Name").upper().startswith(p) for p in software_ips): continue @@ -193,12 +218,11 @@ def clean_up_version(version): modules.append(tuple([m.lower() for m in module])) - modules.append( ("flash", "flash", "v1.0")) + modules.append(("flash", "flash", "v1.0")) modules = [m + peripherals.getPeripheralData(did, m) for m in modules] p["modules"] = modules LOGGER.debug("Available Modules are:\n" + _modulesToString(modules)) - instances = [m[1] for m in modules] # print("\n".join(str(m) for m in modules)) # p["stm_header"] = stm_header @@ -212,12 +236,14 @@ def clean_up_version(version): gpioFile = XmlReader(ip_file) pins = device_file.query('//Pin[@Type="I/O"][starts-with(@Name,"P")]') + def raw_pin_sort(p): port = p.get("Name")[1:2] pin = p.get("Name")[:4] if len(pin) > 3 and not pin[3].isdigit(): pin = pin[:3] return (port, int(pin[2:])) + pins = sorted(pins, key=raw_pin_sort) # Remove package remaps from GPIO data (but not from package) pins.sort(key=lambda p: "PINREMAP" not in p.get("Variant", "")) @@ -232,11 +258,13 @@ def pin_name(name): # Find the remap pin pairs, if they exist double_pinouts = defaultdict(list) - for pin in device_file.query('//Pin'): + for pin in device_file.query("//Pin"): double_pinouts[pin.get("Position")].append((pin.get("Name"), pin.get("Variant", "DEFAULT"))) - double_pinouts = {pos: {pin:variant for (pin, variant) in pins} - for pos, pins in double_pinouts.items() - if len(pins) > 1 and any("PINREMAP" in pin[1] for pin in pins)} + double_pinouts = { + pos: {pin: variant for (pin, variant) in pins} + for pos, pins in double_pinouts.items() + if len(pins) > 1 and any("PINREMAP" in pin[1] for pin in pins) + } # Get the pinout for this package with correct remap variants pinout = [] @@ -249,9 +277,9 @@ def pin_name(name): "type": pin.get("Type"), } variant = double_pinouts.get(pos, {}).get(name) - if (variant is not None and (pin.get("Type") != "I/O" or ( - pin_name(name)[0] in ['a'] and - pin_name(name)[1] in ['9', '10', '11', '12']))): + if variant is not None and ( + pin.get("Type") != "I/O" or (pin_name(name)[0] in ["a"] and pin_name(name)[1] in ["9", "10", "11", "12"]) + ): pinv["variant"] = "remap" if "PINREMAP" in variant else "remap-default" pinout.append(pinv) @@ -265,7 +293,7 @@ def split_af(af): minst = [m for m in modules if af.startswith(m[1] + "_")] # print(af, mdriv, minst) if len(minst) > 1: - LOGGER.warning("Ambiguos driver: {} {}".format(af, minst)) + LOGGER.warning(f"Ambiguos driver: {af} {minst}") exit(1) minst = minst[0] if len(minst) else None @@ -273,15 +301,16 @@ def split_af(af): driver = minst[0] if minst else (mdriv[0] if mdriv else None) if not driver: - LOGGER.debug("Unknown driver: {}".format(af)) + LOGGER.debug(f"Unknown driver: {af}") instance = None if minst and driver: pinst = minst[1].replace(driver, "") - if len(pinst): instance = pinst; + if len(pinst): + instance = pinst if minst or mdriv: name = af.replace((minst[1] if minst else mdriv[0]) + "_", "") if not len(name): - LOGGER.error("Unknown name: {} {}".format(af, minst, mdriv)) + LOGGER.error(f"Unknown name: {af} {minst}") exit(1) else: name = af @@ -289,15 +318,14 @@ def split_af(af): return (driver, instance, name) def split_multi_af(af): - af = af.replace("ir_", "irtim_") \ - .replace("crs_", "rcc_crs_") \ - .replace("timx_", "tim_") - if af == "cec": af = "hdmi_cec_cec"; + af = af.replace("ir_", "irtim_").replace("crs_", "rcc_crs_").replace("timx_", "tim_") + if af == "cec": + af = "hdmi_cec_cec" driver, instance, names = split_af(af) rafs = [] for name in names.split("-"): - rafs.append( (driver, instance, name) ) + rafs.append((driver, instance, name)) return rafs if dmaFile is not None: @@ -313,10 +341,12 @@ def split_multi_af(af): instance = parent.split("_")[0][3:] parent = parent.split("_")[1] - request = dmaFile.query('//RefMode[@Name="{}"]'.format(name))[0] + request = dmaFile.query(f'//RefMode[@Name="{name}"]')[0] + def rv(param, default=[]): - vls = request.xpath('./Parameter[@Name="{}"]/PossibleValue/text()'.format(param)) - if not len(vls): vls = default; + vls = request.xpath(f'./Parameter[@Name="{param}"]/PossibleValue/text()') + if not len(vls): + vls = default return vls name = name.lower().split(":")[0] @@ -325,9 +355,12 @@ def rv(param, default=[]): # Several corrections name = name.replace("spdif_rx", "spdifrx") - if name.startswith("dac") and "_" not in name: name = "dac_{}".format(name); - if any(name == n for n in ["sdio", "sdmmc2", "sdmmc1"]): continue - if len(name.split("_")) < 2: name = "{}_default".format(name); + if name.startswith("dac") and "_" not in name: + name = f"dac_{name}" + if any(name == n for n in ["sdio", "sdmmc2", "sdmmc1"]): + continue + if len(name.split("_")) < 2: + name = f"{name}_default" driver, inst, name = split_af(name) if "[" in parent: @@ -349,58 +382,63 @@ def rv(param, default=[]): stream = channel p["dma_naming"] = (None, "channel", "signal") - if driver is None: # peripheral is not part of this device - dma_dumped.append( (instance, stream, name) ) + if driver is None: # peripheral is not part of this device + dma_dumped.append((instance, stream, name)) continue mode = [v[4:].lower() for v in rv("Mode")] - for sname in ([None] if name == "default" else name.split("/")): + for sname in [None] if name == "default" else name.split("/"): signal = { "driver": driver, "name": sname, - "direction": [v[4:].replace("PERIPH", "p").replace("MEMORY", "m").replace("_TO_", "2") for v in rv("Direction")], + "direction": [ + v[4:].replace("PERIPH", "p").replace("MEMORY", "m").replace("_TO_", "2") + for v in rv("Direction") + ], "mode": mode, "increase": "ENABLE" in rv("PeriphInc", ["DMA_PINC_ENABLE"])[0], } - if inst: signal["instance"] = inst; + if inst: + signal["instance"] = inst remaps = stm32_data.getDmaRemap(did, instance, channel, driver, inst, sname) - if remaps: signal["remap"] = remaps; + if remaps: + signal["remap"] = remaps dma_streams[instance][stream][channel].append(signal) # print(instance, stream, channel) # print(signal) # Manually handle condition expressions from XML for # (STM32F030CCTx|STM32F030RCTx) and (STM32F070CBTx|STM32F070RBTx) - if did.family in ['f0']: - if (did.name == '30' and did.size == 'c'): - dma_streams['1'].pop('6') - dma_streams['1'].pop('7') - dma_streams.pop('2') - if (did.name == '70' and did.size == 'b'): - dma_streams['1'].pop('6') - dma_streams['1'].pop('7') + if did.family in ["f0"]: + if did.name == "30" and did.size == "c": + dma_streams["1"].pop("6") + dma_streams["1"].pop("7") + dma_streams.pop("2") + if did.name == "70" and did.size == "b": + dma_streams["1"].pop("6") + dma_streams["1"].pop("7") # De-duplicate DMA signal entries - def deduplicate_list(l): - return [i for n, i in enumerate(l) if i not in l[n + 1:]] + def deduplicate_list(dl): + return [i for n, i in enumerate(dl) if i not in dl[n + 1 :]] + for stream in dma_streams: for channel in dma_streams[stream]: for signal in dma_streams[stream][channel]: - dma_streams[stream][channel][signal] = deduplicate_list( - dma_streams[stream][channel][signal]) + dma_streams[stream][channel][signal] = deduplicate_list(dma_streams[stream][channel][signal]) # if p["dma_naming"][1] == "request": # print(did, dmaFile.filename) p["dma"] = dma_streams if len(dma_dumped): for instance, stream, name in sorted(dma_dumped): - LOGGER.debug("DMA{}#{}: dumping {}".format(instance, stream, name)) + LOGGER.debug(f"DMA{instance}#{stream}: dumping {name}") # If DMAMUX is used, add DMAMUX to DMA peripheral channel mappings if p["dma_naming"] == (None, "request", "signal"): # There can be multiple "//RefParameter[@Name="Instance"]" nodes constrained by # a child node filtering by the STM32 die id # Try to match a node with condition first, if nothing matches choose the default one - die_id = device_file.query('//Die')[0].text + die_id = device_file.query("//Die")[0].text q = '//RefParameter[@Name="Instance"]/Condition[@Expression="%s"]/../PossibleValue/@Value' % die_id channels = dmaFile.query(q) if len(channels) == 0: @@ -413,13 +451,17 @@ def deduplicate_list(l): for mux_ch_position, channel in enumerate(channels): m = mux_channel_regex.match(channel) assert m is not None - mux_channels.append({'position' : mux_ch_position, - 'dma-instance' : int(m.group("instance")), - 'dma-channel' : int(m.group("channel"))}) + mux_channels.append( + { + "position": mux_ch_position, + "dma-instance": int(m.group("instance")), + "dma-channel": int(m.group("channel")), + } + ) p["dma_mux_channels"] = mux_channels if did.family == "f1": - grouped_f1_signals = gpioFile.compactQuery('//GPIO_Pin/PinSignal/@Name') + grouped_f1_signals = gpioFile.compactQuery("//GPIO_Pin/PinSignal/@Name") _seen_gpio = set() for pin in pins: @@ -427,16 +469,20 @@ def deduplicate_list(l): name = pin_name(rname) # the analog channels are only available in the Mcu file, not the GPIO file - localSignals = device_file.compactQuery('//Pin[@Name="{}"]/Signal[not(@Name="GPIO")]/@Name'.format(rname)) + localSignals = device_file.compactQuery(f'//Pin[@Name="{rname}"]/Signal[not(@Name="GPIO")]/@Name') # print(name, localSignals) altFunctions = [] if did.family == "f1": - altFunctions = [ (s.lower(), "-1") for s in localSignals if s not in grouped_f1_signals] + altFunctions = [(s.lower(), "-1") for s in localSignals if s not in grouped_f1_signals] else: - allSignals = gpioFile.compactQuery('//GPIO_Pin[@Name="{}"]/PinSignal/SpecificParameter[@Name="GPIO_AF"]/..'.format(rname)) - signalMap = { a.get("Name"): a[0][0].text.lower().replace("gpio_af", "")[:2].replace("_", "") for a in allSignals } - altFunctions = [ (s.lower(), (signalMap[s] if s in signalMap else "-1")) for s in localSignals ] + allSignals = gpioFile.compactQuery( + f'//GPIO_Pin[@Name="{rname}"]/PinSignal/SpecificParameter[@Name="GPIO_AF"]/..' + ) + signalMap = { + a.get("Name"): a[0][0].text.lower().replace("gpio_af", "")[:2].replace("_", "") for a in allSignals + } + altFunctions = [(s.lower(), (signalMap[s] if s in signalMap else "-1")) for s in localSignals] afs = [] for af in altFunctions: @@ -444,38 +490,40 @@ def deduplicate_list(l): naf = {} naf["driver"], naf["instance"], naf["name"] = raf naf["af"] = af[1] if int(af[1]) >= 0 else None - if "exti" in naf["name"]: continue; + if "exti" in naf["name"]: + continue afs.append(naf) gpio = (name[0], name[1], afs) if name not in _seen_gpio: gpios.append(gpio) _seen_gpio.add(name) - # print(gpio[0].upper(), gpio[1], afs) - # LOGGER.debug("{}{}: {} ->".format(gpio[0].upper(), gpio[1])) + # LOGGER.debug(f"{gpio[0].upper()}{gpio[1]}: {afs}") remaps = {} if did.family == "f1": - for remap in gpioFile.compactQuery('//GPIO_Pin/PinSignal/RemapBlock/@Name'): + for remap in gpioFile.compactQuery("//GPIO_Pin/PinSignal/RemapBlock/@Name"): module = remap.split("_")[0].lower() config = remap.split("_")[1].replace("REMAP", "").replace("IREMAP", "") mapping = stm32_data.getGpioRemapForModuleConfig(module, config) mpins = [] - for pin in gpioFile.compactQuery('//GPIO_Pin/PinSignal/RemapBlock[@Name="{}"]/..'.format(remap)): + for pin in gpioFile.compactQuery(f'//GPIO_Pin/PinSignal/RemapBlock[@Name="{remap}"]/..'): name = pin.getparent().get("Name")[:4].split("-")[0].split("/")[0].strip().lower() pport, ppin = name[1:2], name[2:] if not any([pp[0] == pport and pp[1] == ppin for pp in gpios]): continue mmm = {"port": pport, "pin": ppin} driver, _, name = split_af(pin.get("Name").lower()) - if driver is None: continue; - mmm["name"] = name; + if driver is None: + continue + mmm["name"] = name mpins.append(mmm) if module not in remaps: driver, instance, _ = split_af(module + "_lol") - if not driver: continue; + if not driver: + continue remaps[module] = { "mask": mapping["mask"], "position": mapping["position"], @@ -485,7 +533,9 @@ def deduplicate_list(l): } if len(mpins) > 0: remaps[module]["groups"][mapping["mapping"]] = mpins - LOGGER.debug("{:<20}{}".format(module + "_" + config, ["{}{}:{}".format(b["port"], b["pin"], b["name"]) for b in mpins])) + LOGGER.debug( + "{:<20}{}".format(module + "_" + config, [f'{b["port"]}{b["pin"]}:{b["name"]}' for b in mpins]) + ) # import json # print(json.dumps(remaps, indent=4)) diff --git a/src/modm_data/cubemx/peripherals.py b/src/modm_data/cubemx/peripherals.py index fdfc1b2..a964507 100644 --- a/src/modm_data/cubemx/peripherals.py +++ b/src/modm_data/cubemx/peripherals.py @@ -1,507 +1,439 @@ # Copyright 2017, Niklas Hauser # SPDX-License-Identifier: MPL-2.0 -stm_peripherals = \ -{ - 'adc': [{ - 'instances': '*', - 'groups': [ - { - 'hardware': 'stm32-f0', - 'features': [], - 'protocols': ['analog-in'], - 'devices': [{'family': ['f0']}] - },{ - 'hardware': 'stm32-l0', - 'features': ['oversampler', 'calfact', 'prescaler'], - 'protocols': ['analog-in'], - 'devices': [{'family': ['l0']}] - },{ - 'hardware': 'stm32-g0', - 'features': ['oversampler', 'calfact', 'prescaler'], - 'protocols': ['analog-in'], - 'devices': [{'family': ['g0']}] - },{ - # F373 & F378 has a non-special ADC - 'hardware': 'stm32', - 'features': [], - 'protocols': ['analog-in'], - 'devices': [{'family': ['f3'], 'name': ['73', '78']}] - },{ - 'hardware': 'stm32-f3', - 'features': [], - 'protocols': ['analog-in'], - 'devices': [{'family': ['f3', 'l4', 'l5', 'g4', 'wb']}] - },{ - 'hardware': 'stm32-h7', - 'features': [], - 'protocols': ['analog-in'], - 'devices': [{'family': ['h7']}] - },{ - 'hardware': 'stm32', - 'features': [], - 'protocols': ['analog-in'], - 'devices': '*' - } - ] - }], - 'sdadc': [{ - 'instances': '*', - 'groups': [ - { - 'hardware': 'stm32-f3', - 'features': [], - 'protocols': ['analog-in'], - 'devices': [{'family': ['f3']}] - } - ] - }], - 'can': [{ - 'instances': '*', - 'groups': [ - { - # 14 shared filters - 'hardware': 'stm32', - 'features': ['filter-14'], - 'protocols': ['can-v2.0a', 'can-v2.0b'], - 'devices': [{'family': ['f0', 'g0', 'f1']}] - },{ - # 28 shared filters - 'hardware': 'stm32', - 'features': ['filter-28'], - 'protocols': ['can-v2.0a', 'can-v2.0b'], - 'devices': '*' - } - ] - }], - 'fdcan': [{ - 'instances': '*', - 'groups': [ - { - 'hardware': 'stm32-h7', - 'features': [], - 'protocols': [], - 'devices': [{'family': ['h7']}] - },{ - 'hardware': 'stm32', - 'features': [], - 'protocols': [], - 'devices': '*' - } - ] - }], - 'crc': [{ - 'instances': '*', - 'groups': [ - { - # Custom polynomial and reverse data - 'hardware': 'stm32', - 'features': ['polynomial', 'reverse'], - 'protocols': ['crc32'], - 'devices': [{'family': ['f0', 'f3', 'f7', 'h7', 'l5', 'u5']}] - },{ - # Custom polynomial and reverse data - 'hardware': 'stm32', - 'features': ['reverse'], - 'protocols': ['crc32'], - 'devices': [{'family': ['g0', 'g4']}] - },{ - # no poly size - 'hardware': 'stm32', - 'features': [], - 'protocols': ['crc32'], - 'devices': '*' - } - ] - }], - 'dma': [{ - 'instances': '*', - 'groups': [ - { - 'hardware': 'stm32-mux', - 'features': [], - 'protocols': ['mem2mem', 'mem2per', 'per2per'], - 'devices': [{'family': ['g0', 'g4', 'l5', 'wb', 'wl']}, {'family': ['l4'], 'name': ['p5', 'p7', 'p9', 'q5', 'q7', 'q9', 'r5', 'r7', 'r9', 's5', 's7', 's9']}] - }, - { - 'hardware': 'stm32-mux-stream', - 'features': [], - 'protocols': ['mem2mem', 'mem2per', 'per2per'], - 'devices': [{'family': ['h7']}] - }, - { - 'hardware': 'stm32-stream-channel', - 'features': [], - 'protocols': ['mem2mem', 'mem2per', 'per2per'], - 'devices': [{'family': ['f2', 'f4', 'f7']}] - }, - { - 'hardware': 'stm32-channel-request', - 'features': [], - 'protocols': ['mem2mem', 'mem2per', 'per2per'], - 'devices': [{'family': ['l0', 'l4']}, {'family': ['f0'], 'name': ['91', '98']}, {'family': ['f0'], 'name': ['30'], 'size': ['c']}] - }, - { - 'hardware': 'stm32-channel', - 'features': [], - 'protocols': ['mem2mem', 'mem2per', 'per2per'], - 'devices': '*' - } - ] - }], - 'iwdg': [{ - 'instances': '*', - 'groups': [ - { - 'hardware': 'stm32', - 'features': ['window'], - 'protocols': [], - 'devices': [{'family': ['f0', 'f3', 'f7', 'g0', 'g4', 'l5', 'u5', 'wb']}] - },{ - 'hardware': 'stm32', - 'features': [], - 'protocols': [], - 'devices': '*' - } - ] - }], - 'spi': [{ - 'instances': '*', - 'groups': [ - { - 'hardware': 'stm32', - 'features': ['data-size', 'nss-pulse', 'fifo'], - 'protocols': [], - 'devices': [{'family': ['f0', 'g0', 'f3', 'f7', 'l4', 'l5', 'g4', 'wb']}] - },{ - 'hardware': 'stm32-extended', - 'features': [], - 'protocols': [], - 'devices': [{'family': ['h7', 'u5']}] - },{ - 'hardware': 'stm32', - 'features': [], - 'protocols': [], - 'devices': '*' - } - ] - }], - 'dac': [{ - 'instances': '*', - 'groups': [ - { - 'hardware': 'stm32', - 'features': [], - 'protocols': [], - 'devices': [{'family': ['f1']}] - },{ - 'hardware': 'stm32', - 'features': ['status'], - 'protocols': [], - 'devices': '*' - } - ] - }], - 'dcmi': [{ - 'instances': '*', - 'groups': [ - { - 'hardware': 'stm32', - 'features': [], - 'protocols': [], - 'devices': '*' - } - ] - }], - 'dsi': [{ - 'instances': '*', - 'groups': [ - { - 'hardware': 'stm32', - 'features': [], - 'protocols': [], - 'devices': '*' - } - ] - }], - 'sdio': [{ - 'instances': '*', - 'groups': [ - { - 'hardware': 'stm32', - 'features': [], - 'protocols': [], - 'devices': '*' - } - ] - }], - 'tim': [ +stm_peripherals = { + "adc": [ { - 'instances': ['1', '8', '20'], - 'groups': [ + "instances": "*", + "groups": [ + {"hardware": "stm32-f0", "features": [], "protocols": ["analog-in"], "devices": [{"family": ["f0"]}]}, { - 'hardware': 'stm32-advanced', - 'features': [], - 'protocols': [], - 'devices': '*' - } - ] - },{ - 'instances': ['2', '3', '4', '5'], - 'groups': [ - { - 'hardware': 'stm32-general-purpose', - 'features': [], - 'protocols': [], - 'devices': '*' - } - ] - },{ - 'instances': ['9', '10', '11', '12', '13', '14', '15', '16', '17'], - 'groups': [ - { - 'hardware': 'stm32-general-purpose', - 'features': [], - 'protocols': [], - 'devices': '*' - } - ] - },{ - 'instances': ['6', '7'], - 'groups': [ - { - 'hardware': 'stm32-basic', - 'features': [], - 'protocols': [], - 'devices': '*' - } - ] + "hardware": "stm32-l0", + "features": ["oversampler", "calfact", "prescaler"], + "protocols": ["analog-in"], + "devices": [{"family": ["l0"]}], + }, + { + "hardware": "stm32-g0", + "features": ["oversampler", "calfact", "prescaler"], + "protocols": ["analog-in"], + "devices": [{"family": ["g0"]}], + }, + { + # F373 & F378 has a non-special ADC + "hardware": "stm32", + "features": [], + "protocols": ["analog-in"], + "devices": [{"family": ["f3"], "name": ["73", "78"]}], + }, + { + "hardware": "stm32-f3", + "features": [], + "protocols": ["analog-in"], + "devices": [{"family": ["f3", "l4", "l5", "g4", "wb"]}], + }, + {"hardware": "stm32-h7", "features": [], "protocols": ["analog-in"], "devices": [{"family": ["h7"]}]}, + {"hardware": "stm32", "features": [], "protocols": ["analog-in"], "devices": "*"}, + ], + } + ], + "sdadc": [ + { + "instances": "*", + "groups": [ + {"hardware": "stm32-f3", "features": [], "protocols": ["analog-in"], "devices": [{"family": ["f3"]}]} + ], + } + ], + "can": [ + { + "instances": "*", + "groups": [ + { + # 14 shared filters + "hardware": "stm32", + "features": ["filter-14"], + "protocols": ["can-v2.0a", "can-v2.0b"], + "devices": [{"family": ["f0", "g0", "f1"]}], + }, + { + # 28 shared filters + "hardware": "stm32", + "features": ["filter-28"], + "protocols": ["can-v2.0a", "can-v2.0b"], + "devices": "*", + }, + ], + } + ], + "fdcan": [ + { + "instances": "*", + "groups": [ + {"hardware": "stm32-h7", "features": [], "protocols": [], "devices": [{"family": ["h7"]}]}, + {"hardware": "stm32", "features": [], "protocols": [], "devices": "*"}, + ], + } + ], + "crc": [ + { + "instances": "*", + "groups": [ + { + # Custom polynomial and reverse data + "hardware": "stm32", + "features": ["polynomial", "reverse"], + "protocols": ["crc32"], + "devices": [{"family": ["f0", "f3", "f7", "h7", "l5", "u5"]}], + }, + { + # Custom polynomial and reverse data + "hardware": "stm32", + "features": ["reverse"], + "protocols": ["crc32"], + "devices": [{"family": ["g0", "g4"]}], + }, + { + # no poly size + "hardware": "stm32", + "features": [], + "protocols": ["crc32"], + "devices": "*", + }, + ], + } + ], + "dma": [ + { + "instances": "*", + "groups": [ + { + "hardware": "stm32-mux", + "features": [], + "protocols": ["mem2mem", "mem2per", "per2per"], + "devices": [ + {"family": ["g0", "g4", "l5", "wb", "wl"]}, + { + "family": ["l4"], + "name": ["p5", "p7", "p9", "q5", "q7", "q9", "r5", "r7", "r9", "s5", "s7", "s9"], + }, + ], + }, + { + "hardware": "stm32-mux-stream", + "features": [], + "protocols": ["mem2mem", "mem2per", "per2per"], + "devices": [{"family": ["h7"]}], + }, + { + "hardware": "stm32-stream-channel", + "features": [], + "protocols": ["mem2mem", "mem2per", "per2per"], + "devices": [{"family": ["f2", "f4", "f7"]}], + }, + { + "hardware": "stm32-channel-request", + "features": [], + "protocols": ["mem2mem", "mem2per", "per2per"], + "devices": [ + {"family": ["l0", "l4"]}, + {"family": ["f0"], "name": ["91", "98"]}, + {"family": ["f0"], "name": ["30"], "size": ["c"]}, + ], + }, + { + "hardware": "stm32-channel", + "features": [], + "protocols": ["mem2mem", "mem2per", "per2per"], + "devices": "*", + }, + ], + } + ], + "iwdg": [ + { + "instances": "*", + "groups": [ + { + "hardware": "stm32", + "features": ["window"], + "protocols": [], + "devices": [{"family": ["f0", "f3", "f7", "g0", "g4", "l5", "u5", "wb"]}], + }, + {"hardware": "stm32", "features": [], "protocols": [], "devices": "*"}, + ], + } + ], + "spi": [ + { + "instances": "*", + "groups": [ + { + "hardware": "stm32", + "features": ["data-size", "nss-pulse", "fifo"], + "protocols": [], + "devices": [{"family": ["f0", "g0", "f3", "f7", "l4", "l5", "g4", "wb"]}], + }, + {"hardware": "stm32-extended", "features": [], "protocols": [], "devices": [{"family": ["h7", "u5"]}]}, + {"hardware": "stm32", "features": [], "protocols": [], "devices": "*"}, + ], + } + ], + "dac": [ + { + "instances": "*", + "groups": [ + {"hardware": "stm32", "features": [], "protocols": [], "devices": [{"family": ["f1"]}]}, + {"hardware": "stm32", "features": ["status"], "protocols": [], "devices": "*"}, + ], + } + ], + "dcmi": [{"instances": "*", "groups": [{"hardware": "stm32", "features": [], "protocols": [], "devices": "*"}]}], + "dsi": [{"instances": "*", "groups": [{"hardware": "stm32", "features": [], "protocols": [], "devices": "*"}]}], + "sdio": [{"instances": "*", "groups": [{"hardware": "stm32", "features": [], "protocols": [], "devices": "*"}]}], + "tim": [ + { + "instances": ["1", "8", "20"], + "groups": [{"hardware": "stm32-advanced", "features": [], "protocols": [], "devices": "*"}], + }, + { + "instances": ["2", "3", "4", "5"], + "groups": [{"hardware": "stm32-general-purpose", "features": [], "protocols": [], "devices": "*"}], + }, + { + "instances": ["9", "10", "11", "12", "13", "14", "15", "16", "17"], + "groups": [{"hardware": "stm32-general-purpose", "features": [], "protocols": [], "devices": "*"}], + }, + { + "instances": ["6", "7"], + "groups": [{"hardware": "stm32-basic", "features": [], "protocols": [], "devices": "*"}], + }, + ], + "sys": [ + { + "instances": "*", + "groups": [ + { + # Registers are called AFIO, not SYS! + "hardware": "stm32-f1", + "features": ["exti", "remap"], + "protocols": [], + "devices": [{"family": ["f1"]}], + }, + { + "hardware": "stm32", + "features": ["exti", "fpu", "ccm-wp", "cfgr2"], + "protocols": [], + "devices": [{"family": ["f3", "g4"]}], + }, + { + "hardware": "stm32", + "features": ["exti", "sram2-wp", "cfgr2", "imr"], + "protocols": [], + "devices": [{"family": ["l5", "wb"]}], + }, + { + "hardware": "stm32", + "features": ["exti", "cfgr2", "itline"], + "protocols": [], + "devices": [{"family": ["f0"], "name": ["91", "98"]}, {"family": ["g0"]}], + }, + {"hardware": "stm32", "features": ["exti", "cfgr2"], "protocols": [], "devices": [{"family": ["f0"]}]}, + {"hardware": "stm32", "features": ["exti"], "protocols": [], "devices": "*"}, + ], } ], - 'sys': [{ - 'instances': '*', - 'groups': [ - { - # Registers are called AFIO, not SYS! - 'hardware': 'stm32-f1', - 'features': ['exti', 'remap'], - 'protocols': [], - 'devices': [{'family': ['f1']}] - },{ - 'hardware': 'stm32', - 'features': ['exti', 'fpu', 'ccm-wp', 'cfgr2'], - 'protocols': [], - 'devices': [{'family': ['f3', 'g4']}] - },{ - 'hardware': 'stm32', - 'features': ['exti', 'sram2-wp', 'cfgr2', 'imr'], - 'protocols': [], - 'devices': [{'family': ['l5', 'wb']}] - },{ - 'hardware': 'stm32', - 'features': ['exti', 'cfgr2', 'itline'], - 'protocols': [], - 'devices': [{'family': ['f0'], 'name': ['91', '98']}, {'family': ['g0']}] - },{ - 'hardware': 'stm32', - 'features': ['exti', 'cfgr2'], - 'protocols': [], - 'devices': [{'family': ['f0']}] - },{ - 'hardware': 'stm32', - 'features': ['exti'], - 'protocols': [], - 'devices': '*' - } - ] - }], - 'dma2d': [{ - 'instances': '*', - 'groups': [ - { - 'hardware': 'stm32', - 'features': [], - 'protocols': ['2d', 'blitter'], - 'devices': '*' - } - ] - }], - 'rng': [{ - 'instances': '*', - 'groups': [ - { - 'hardware': 'stm32', - 'features': [], - 'protocols': [], - 'devices': '*' - } - ] - }], - 'i2c': [ + "dma2d": [ + { + "instances": "*", + "groups": [{"hardware": "stm32", "features": [], "protocols": ["2d", "blitter"], "devices": "*"}], + } + ], + "rng": [{"instances": "*", "groups": [{"hardware": "stm32", "features": [], "protocols": [], "devices": "*"}]}], + "i2c": [ { # F1/F2/F4/L1 standard I2C with SMBus support - 'instances': '*', - 'groups': [ + "instances": "*", + "groups": [ { # Some F4 have a digital noise filter - 'hardware': 'stm32', - 'features': ['dnf'], - 'protocols': ['i2c-v3.0', 'smb-v2.0', 'pmb-v1.1'], - 'devices': [{'family': ['f4'], 'name': ['27', '29', '37', '39', '46', '69', '79']}] - },{ - 'hardware': 'stm32', - 'features': [], - 'protocols': ['i2c-v3.0', 'smb-v2.0', 'pmb-v1.1'], - 'devices': [{'family': ['f1', 'f2', 'f4', 'l1']}] - } - ] - },{ + "hardware": "stm32", + "features": ["dnf"], + "protocols": ["i2c-v3.0", "smb-v2.0", "pmb-v1.1"], + "devices": [{"family": ["f4"], "name": ["27", "29", "37", "39", "46", "69", "79"]}], + }, + { + "hardware": "stm32", + "features": [], + "protocols": ["i2c-v3.0", "smb-v2.0", "pmb-v1.1"], + "devices": [{"family": ["f1", "f2", "f4", "l1"]}], + }, + ], + }, + { # F0/F3/F7/L0/L4/L4+/H7 extended I2C instance 2 with optional FM+ and SMBus support - 'instances': ['2'], - 'groups': [ + "instances": ["2"], + "groups": [ { # This hardware supports neither FM+ (1 Mhz) nor SMBus - 'hardware': 'stm32-extended', - 'features': ['dnf'], - 'protocols': ['i2c-v3.0'], - 'devices': [ - { - 'family': ['f0'], - 'name': ['30', '31', '38', '51', '58'] - },{ - 'family': ['f0'], - 'name': ['70'], - 'size': ['b'] - } - ] - },{ + "hardware": "stm32-extended", + "features": ["dnf"], + "protocols": ["i2c-v3.0"], + "devices": [ + {"family": ["f0"], "name": ["30", "31", "38", "51", "58"]}, + {"family": ["f0"], "name": ["70"], "size": ["b"]}, + ], + }, + { # This hardware supports FM+ (1 Mhz) but not SMBus - 'hardware': 'stm32-extended', - 'features': ['dnf', 'fmp'], - 'protocols': ['i2c-v3.0'], - 'devices': [{'family': ['f0', 'g0', 'l0']}] - },{ + "hardware": "stm32-extended", + "features": ["dnf", "fmp"], + "protocols": ["i2c-v3.0"], + "devices": [{"family": ["f0", "g0", "l0"]}], + }, + { # This hardware supports FM+ (1 Mhz) and SMBus - 'hardware': 'stm32-extended', - 'features': ['dnf', 'fmp'], - 'protocols': ['i2c-v3.0', 'smb-v2.0', 'pmb-v1.1'], - 'devices': [{'family': ['f3', 'f7', 'l4', 'l5', 'h7', 'g4', 'u5', 'wb']}] - } - ] - },{ + "hardware": "stm32-extended", + "features": ["dnf", "fmp"], + "protocols": ["i2c-v3.0", "smb-v2.0", "pmb-v1.1"], + "devices": [{"family": ["f3", "f7", "l4", "l5", "h7", "g4", "u5", "wb"]}], + }, + ], + }, + { # F0/F3/F7/L0/L4/L4+/H7 extended I2C with FM+ and SMBus support - 'instances': ['1', '3', '4'], - 'groups': [ + "instances": ["1", "3", "4"], + "groups": [ { # This hardware supports FM+ (1 Mhz) and SMBus - 'hardware': 'stm32-extended', - 'features': ['dnf', 'fmp'], - 'protocols': ['i2c-v3.0', 'smb-v2.0', 'pmb-v1.1'], - 'devices': [{'family': ['f0', 'g0', 'f3', 'f7', 'l0', 'l4', 'l5', 'h7', 'g4', 'u5', 'wb']}] + "hardware": "stm32-extended", + "features": ["dnf", "fmp"], + "protocols": ["i2c-v3.0", "smb-v2.0", "pmb-v1.1"], + "devices": [{"family": ["f0", "g0", "f3", "f7", "l0", "l4", "l5", "h7", "g4", "u5", "wb"]}], } - ] + ], + }, + ], + "uart": [ + { + "instances": "*", + "groups": [ + { + "hardware": "stm32-extended", + "features": ["wakeup"], + "protocols": ["uart"], + "devices": [{"family": ["f0", "f3"]}], + }, + { + "hardware": "stm32-extended", + "features": ["tcbgt"], + "protocols": ["uart"], + "devices": [ + { + "family": ["l4"], + "name": ["p5", "p7", "p9", "q5", "q7", "q9", "r5", "r7", "r9", "s5", "s7", "s9"], + }, + {"family": ["g0", "g4", "wb", "h7", "l5", "u5"]}, + ], + }, + { + "hardware": "stm32-extended", + "features": [], + "protocols": ["uart"], + "devices": [{"family": ["f7", "l4"]}], + }, + { + "hardware": "stm32", + "features": ["over8"], + "protocols": ["uart"], + "devices": [{"family": ["f2", "f4"]}], + }, + {"hardware": "stm32", "features": [], "protocols": ["uart"], "devices": "*"}, + ], + } + ], + "usart": [ + { + "instances": "*", + "groups": [ + { + "hardware": "stm32-extended", + "features": ["wakeup"], + "protocols": ["uart", "spi"], + "devices": [{"family": ["f0", "f3"]}], + }, + { + "hardware": "stm32-extended", + "features": ["tcbgt"], + "protocols": ["uart", "spi"], + "devices": [ + { + "family": ["l4"], + "name": ["p5", "p7", "p9", "q5", "q7", "q9", "r5", "r7", "r9", "s5", "s7", "s9"], + }, + {"family": ["g0", "g4", "wb", "h7", "l5", "u5"]}, + ], + }, + { + "hardware": "stm32-extended", + "features": [], + "protocols": ["uart", "spi"], + "devices": [{"family": ["f7", "l4"]}], + }, + { + "hardware": "stm32-extended", + "features": ["over8"], + "protocols": ["uart", "spi"], + "devices": [{"family": ["l0"]}], + }, + { + "hardware": "stm32", + "features": ["over8"], + "protocols": ["uart", "spi"], + "devices": [{"family": ["f2", "f4"]}], + }, + {"hardware": "stm32", "features": [], "protocols": ["uart", "spi"], "devices": "*"}, + ], + } + ], + "gpio": [ + { + "instances": "*", + "groups": [ + { + # The F1 remaps groups of pins + "hardware": "stm32-f1", + "features": [], + "protocols": ["digital-in", "digital-out", "open-drain", "exti"], + "devices": [{"family": ["f1"]}], + }, + { + # The rest remaps pins individually + "hardware": "stm32", + "features": [], + "protocols": ["digital-in", "digital-out", "open-drain", "exti"], + "devices": "*", + }, + ], } ], - 'uart': [{ - 'instances': '*', - 'groups': [ - { - 'hardware': 'stm32-extended', - 'features': ['wakeup'], - 'protocols': ['uart'], - 'devices': [{'family': ['f0', 'f3']}] - },{ - 'hardware': 'stm32-extended', - 'features': ['tcbgt'], - 'protocols': ['uart'], - 'devices': [{'family': ['l4'], 'name': ['p5', 'p7', 'p9', 'q5', 'q7', 'q9', 'r5', 'r7', 'r9', 's5', 's7', 's9']}, {'family': ['g0', 'g4', 'wb', 'h7', 'l5', 'u5']}] - },{ - 'hardware': 'stm32-extended', - 'features': [], - 'protocols': ['uart'], - 'devices': [{'family': ['f7', 'l4']}] - },{ - 'hardware': 'stm32', - 'features': ['over8'], - 'protocols': ['uart'], - 'devices': [{'family': ['f2', 'f4']}] - },{ - 'hardware': 'stm32', - 'features': [], - 'protocols': ['uart'], - 'devices': '*' - } - ] - }], - 'usart': [{ - 'instances': '*', - 'groups': [ - { - 'hardware': 'stm32-extended', - 'features': ['wakeup'], - 'protocols': ['uart', 'spi'], - 'devices': [{'family': ['f0', 'f3']}] - },{ - 'hardware': 'stm32-extended', - 'features': ['tcbgt'], - 'protocols': ['uart', 'spi'], - 'devices': [{'family': ['l4'], 'name': ['p5', 'p7', 'p9', 'q5', 'q7', 'q9', 'r5', 'r7', 'r9', 's5', 's7', 's9']}, {'family': ['g0', 'g4', 'wb', 'h7', 'l5', 'u5']}] - },{ - 'hardware': 'stm32-extended', - 'features': [], - 'protocols': ['uart', 'spi'], - 'devices': [{'family': ['f7', 'l4']}] - },{ - 'hardware': 'stm32-extended', - 'features': ['over8'], - 'protocols': ['uart', 'spi'], - 'devices': [{'family': ['l0']}] - },{ - 'hardware': 'stm32', - 'features': ['over8'], - 'protocols': ['uart', 'spi'], - 'devices': [{'family': ['f2', 'f4']}] - },{ - 'hardware': 'stm32', - 'features': [], - 'protocols': ['uart', 'spi'], - 'devices': '*' - } - ] - }], - 'gpio': [{ - 'instances': '*', - 'groups': [ - { - # The F1 remaps groups of pins - 'hardware': 'stm32-f1', - 'features': [], - 'protocols': ['digital-in', 'digital-out', 'open-drain', 'exti'], - 'devices': [{'family': ['f1']}] - },{ - # The rest remaps pins individually - 'hardware': 'stm32', - 'features': [], - 'protocols': ['digital-in', 'digital-out', 'open-drain', 'exti'], - 'devices': '*' - } - ] - }] } + def _get_index_for_id(merge_group, did): for group in merge_group: if all(did[key] in value for key, value in group.items()): return merge_group.index(group) return -1 + def getPeripheralData(did, module): name, inst, version = module if name in stm_peripherals: for instance_list in stm_peripherals[name]: - if instance_list['instances'] == '*' or inst[len(name):] in instance_list['instances']: - for group in instance_list['groups']: - if group['devices'] == '*' or _get_index_for_id(group['devices'], did) >= 0: - return (group['hardware'], group['features'], group['protocols']) + if instance_list["instances"] == "*" or inst[len(name) :] in instance_list["instances"]: + for group in instance_list["groups"]: + if group["devices"] == "*" or _get_index_for_id(group["devices"], did) >= 0: + return (group["hardware"], group["features"], group["protocols"]) - return ('stm32-' + version, [], []) + return ("stm32-" + version, [], []) diff --git a/src/modm_data/cubemx/stm32_data.py b/src/modm_data/cubemx/stm32_data.py index 0791911..62c9c00 100644 --- a/src/modm_data/cubemx/stm32_data.py +++ b/src/modm_data/cubemx/stm32_data.py @@ -5,12 +5,12 @@ LOGGER = logging.getLogger("dfg.stm.data") -ignored_devices = \ -[ +ignored_devices = [ "STM32U59", "STM32U5A", ] + def ignoreDevice(device_id: str) -> bool: for ignore in ignored_devices: if device_id.startswith(ignore): @@ -18,176 +18,167 @@ def ignoreDevice(device_id: str) -> bool: return False -stm32f1_gpio_remap = \ -{ +stm32f1_gpio_remap = { # (position % 32) -> local bit position # MAPR register - 'spi1': {'position': 0, 'mask': 1, 'mapping': [0, 1]}, - 'i2c1': {'position': 1, 'mask': 1, 'mapping': [0, 1]}, - 'usart1': {'position': 2, 'mask': 1, 'mapping': [0, 1]}, - 'usart2': {'position': 3, 'mask': 1, 'mapping': [0, 1]}, - 'usart3': {'position': 4, 'mask': 3, 'mapping': [0, 1, 3]}, - 'tim1': {'position': 6, 'mask': 3, 'mapping': [0, 1, 3]}, - 'tim2': {'position': 8, 'mask': 3, 'mapping': [0, 1, 2, 3]}, - 'tim3': {'position': 10, 'mask': 3, 'mapping': [0, 2, 3]}, - 'tim4': {'position': 12, 'mask': 1, 'mapping': [0, 1]}, - 'can': {'position': 13, 'mask': 3, 'mapping': [0, 2, 3]}, - 'can1': {'position': 13, 'mask': 3, 'mapping': [0, 2, 3]}, - 'pd01': {'position': 15, 'mask': 1, 'mapping': [0, 1]}, - 'tim5ch4': {'position': 16, 'mask': 1, 'mapping': [0, 1]}, - 'adc1etrginj': {'position': 17, 'mask': 1, 'mapping': [0, 1]}, - 'adc1etrgreg': {'position': 18, 'mask': 1, 'mapping': [0, 1]}, - 'adc2etrginj': {'position': 19, 'mask': 1, 'mapping': [0, 1]}, - 'adc2etrgreg': {'position': 20, 'mask': 1, 'mapping': [0, 1]}, - 'eth': {'position': 21, 'mask': 1, 'mapping': [0, 1]}, - 'can2': {'position': 22, 'mask': 1, 'mapping': [0, 1]}, - 'mii_rmii_sel': {'position': 23, 'mask': 1, 'mapping': [0, 1]}, - 'swj_cfg': {'position': 24, 'mask': 7, 'mapping': [0, 1, 2, 4]}, + "spi1": {"position": 0, "mask": 1, "mapping": [0, 1]}, + "i2c1": {"position": 1, "mask": 1, "mapping": [0, 1]}, + "usart1": {"position": 2, "mask": 1, "mapping": [0, 1]}, + "usart2": {"position": 3, "mask": 1, "mapping": [0, 1]}, + "usart3": {"position": 4, "mask": 3, "mapping": [0, 1, 3]}, + "tim1": {"position": 6, "mask": 3, "mapping": [0, 1, 3]}, + "tim2": {"position": 8, "mask": 3, "mapping": [0, 1, 2, 3]}, + "tim3": {"position": 10, "mask": 3, "mapping": [0, 2, 3]}, + "tim4": {"position": 12, "mask": 1, "mapping": [0, 1]}, + "can": {"position": 13, "mask": 3, "mapping": [0, 2, 3]}, + "can1": {"position": 13, "mask": 3, "mapping": [0, 2, 3]}, + "pd01": {"position": 15, "mask": 1, "mapping": [0, 1]}, + "tim5ch4": {"position": 16, "mask": 1, "mapping": [0, 1]}, + "adc1etrginj": {"position": 17, "mask": 1, "mapping": [0, 1]}, + "adc1etrgreg": {"position": 18, "mask": 1, "mapping": [0, 1]}, + "adc2etrginj": {"position": 19, "mask": 1, "mapping": [0, 1]}, + "adc2etrgreg": {"position": 20, "mask": 1, "mapping": [0, 1]}, + "eth": {"position": 21, "mask": 1, "mapping": [0, 1]}, + "can2": {"position": 22, "mask": 1, "mapping": [0, 1]}, + "mii_rmii_sel": {"position": 23, "mask": 1, "mapping": [0, 1]}, + "swj_cfg": {"position": 24, "mask": 7, "mapping": [0, 1, 2, 4]}, # position 27 is empty - 'spi3': {'position': 28, 'mask': 1, 'mapping': [0, 1]}, - 'i2s3': {'position': 28, 'mask': 1, 'mapping': [0, 1]}, - 'tim2itr1': {'position': 29, 'mask': 1, 'mapping': [0, 1]}, - 'ptp_pps': {'position': 30, 'mask': 1, 'mapping': [0, 1]}, + "spi3": {"position": 28, "mask": 1, "mapping": [0, 1]}, + "i2s3": {"position": 28, "mask": 1, "mapping": [0, 1]}, + "tim2itr1": {"position": 29, "mask": 1, "mapping": [0, 1]}, + "ptp_pps": {"position": 30, "mask": 1, "mapping": [0, 1]}, # position 31 is empty # MAPR2 register - 'tim15': {'position': 32, 'mask': 1, 'mapping': [0, 1]}, - 'tim16': {'position': 33, 'mask': 1, 'mapping': [0, 1]}, - 'tim17': {'position': 34, 'mask': 1, 'mapping': [0, 1]}, - 'cec': {'position': 35, 'mask': 1, 'mapping': [0, 1]}, - 'tim1_dma': {'position': 36, 'mask': 1, 'mapping': [0, 1]}, - 'tim9': {'position': 37, 'mask': 1, 'mapping': [0, 1]}, - 'tim10': {'position': 38, 'mask': 1, 'mapping': [0, 1]}, - 'tim11': {'position': 39, 'mask': 1, 'mapping': [0, 1]}, - 'tim13': {'position': 40, 'mask': 1, 'mapping': [0, 1]}, - 'tim14': {'position': 41, 'mask': 1, 'mapping': [0, 1]}, - 'fsmc_nadv': {'position': 42, 'mask': 1, 'mapping': [0, 1]}, - 'tim67_dac_dma':{'position': 43, 'mask': 1, 'mapping': [0, 1]}, - 'tim12': {'position': 44, 'mask': 1, 'mapping': [0, 1]}, - 'misc': {'position': 45, 'mask': 1, 'mapping': [0, 1]}, + "tim15": {"position": 32, "mask": 1, "mapping": [0, 1]}, + "tim16": {"position": 33, "mask": 1, "mapping": [0, 1]}, + "tim17": {"position": 34, "mask": 1, "mapping": [0, 1]}, + "cec": {"position": 35, "mask": 1, "mapping": [0, 1]}, + "tim1_dma": {"position": 36, "mask": 1, "mapping": [0, 1]}, + "tim9": {"position": 37, "mask": 1, "mapping": [0, 1]}, + "tim10": {"position": 38, "mask": 1, "mapping": [0, 1]}, + "tim11": {"position": 39, "mask": 1, "mapping": [0, 1]}, + "tim13": {"position": 40, "mask": 1, "mapping": [0, 1]}, + "tim14": {"position": 41, "mask": 1, "mapping": [0, 1]}, + "fsmc_nadv": {"position": 42, "mask": 1, "mapping": [0, 1]}, + "tim67_dac_dma": {"position": 43, "mask": 1, "mapping": [0, 1]}, + "tim12": {"position": 44, "mask": 1, "mapping": [0, 1]}, + "misc": {"position": 45, "mask": 1, "mapping": [0, 1]}, } + def getGpioRemapForModuleConfig(module, config): mmm = {} if module in stm32f1_gpio_remap: - mmm['mask'] = stm32f1_gpio_remap[module]['mask'] - mmm['position'] = stm32f1_gpio_remap[module]['position'] - mmm['mapping'] = stm32f1_gpio_remap[module]['mapping'][int(config)] + mmm["mask"] = stm32f1_gpio_remap[module]["mask"] + mmm["position"] = stm32f1_gpio_remap[module]["position"] + mmm["mapping"] = stm32f1_gpio_remap[module]["mapping"][int(config)] return mmm -stm32_flash_latency = \ -{ - 'f0': { - 1800: [24, 48] - }, - 'f1': [ - {'name': ['00'], 1800: [24]}, - {1800: [24, 48, 72]} - ], - 'f2': { +stm32_flash_latency = { + "f0": {1800: [24, 48]}, + "f1": [{"name": ["00"], 1800: [24]}, {1800: [24, 48, 72]}], + "f2": { 2700: [30, 60, 90, 120], 2400: [24, 48, 72, 96, 120], 2100: [18, 36, 54, 72, 90, 108, 120], - 1800: [16, 32, 48, 64, 80, 96, 112, 120] - }, - 'f3': { - 1800: [24, 48, 72] - }, - 'f4': [{ - 'name': ['10', '11', '12', '13', '23'], - 2700: [30, 60, 90, 100], - 2400: [24, 48, 72, 96, 100], - 2100: [18, 36, 54, 72, 90, 100], - 1800: [16, 32, 48, 64, 80, 96, 100] - },{ - 'name': ['01'], - 2700: [30, 60, 84], - 2400: [24, 48, 72, 84], - 2100: [18, 36, 54, 72, 84], - 1800: [16, 32, 48, 64, 80, 84] - },{ - 'name': ['05', '07', '15', '17'], - 2700: [30, 60, 90, 120, 150, 168], - 2400: [24, 48, 72, 96, 120, 144, 168], - 2100: [22, 44, 66, 88, 110, 132, 154, 168], - 1800: [20, 40, 60, 80, 100, 120, 140, 160] - },{ - 'name': ['27', '29', '37', '39', '46', '69', '79'], - 2700: [30, 60, 90, 120, 150, 180], - 2400: [24, 48, 72, 96, 120, 144, 168, 180], - 2100: [22, 44, 66, 88, 110, 132, 154, 176, 180], - 1800: [20, 40, 60, 80, 100, 120, 140, 160, 168] - }], - 'f7': { + 1800: [16, 32, 48, 64, 80, 96, 112, 120], + }, + "f3": {1800: [24, 48, 72]}, + "f4": [ + { + "name": ["10", "11", "12", "13", "23"], + 2700: [30, 60, 90, 100], + 2400: [24, 48, 72, 96, 100], + 2100: [18, 36, 54, 72, 90, 100], + 1800: [16, 32, 48, 64, 80, 96, 100], + }, + { + "name": ["01"], + 2700: [30, 60, 84], + 2400: [24, 48, 72, 84], + 2100: [18, 36, 54, 72, 84], + 1800: [16, 32, 48, 64, 80, 84], + }, + { + "name": ["05", "07", "15", "17"], + 2700: [30, 60, 90, 120, 150, 168], + 2400: [24, 48, 72, 96, 120, 144, 168], + 2100: [22, 44, 66, 88, 110, 132, 154, 168], + 1800: [20, 40, 60, 80, 100, 120, 140, 160], + }, + { + "name": ["27", "29", "37", "39", "46", "69", "79"], + 2700: [30, 60, 90, 120, 150, 180], + 2400: [24, 48, 72, 96, 120, 144, 168, 180], + 2100: [22, 44, 66, 88, 110, 132, 154, 176, 180], + 1800: [20, 40, 60, 80, 100, 120, 140, 160, 168], + }, + ], + "f7": { 2700: [30, 60, 90, 120, 150, 180, 216], 2400: [24, 48, 72, 96, 120, 144, 168, 192, 216], 2100: [22, 44, 66, 88, 110, 132, 154, 176, 198, 216], - 1800: [20, 40, 60, 80, 100, 120, 140, 160, 180] - }, - 'l0': { - 1650: [16, 32], - 1350: [8, 16], - 1050: [4.2] - }, - 'l1': [{ # Cat 1 - 'size': ['6', '8', 'b'], - 1800: [16, 32], - 1500: [8, 16], - 1200: [4.2, 8] - },{ # Cat 2,3,4,5,6 - 1800: [16, 32], - 1500: [8, 16], - 1200: [2.1, 4.2] - }], - 'l4': [{ # L4+ devices - 'name': ['r5', 'r7', 'r9', 's5', 's7', 's9', 'p5', 'q5'], - 1280: [20, 40, 60, 80, 100, 120], - 1200: [20, 40, 60, 80], - 1000: [8, 16, 26] - },{ # L4 devices - 1200: [16, 32, 48, 64, 80], - 1000: [6, 12, 18, 26] - }], - 'l5': { + 1800: [20, 40, 60, 80, 100, 120, 140, 160, 180], + }, + "l0": {1650: [16, 32], 1350: [8, 16], 1050: [4.2]}, + "l1": [ + { # Cat 1 + "size": ["6", "8", "b"], + 1800: [16, 32], + 1500: [8, 16], + 1200: [4.2, 8], + }, + { # Cat 2,3,4,5,6 + 1800: [16, 32], + 1500: [8, 16], + 1200: [2.1, 4.2], + }, + ], + "l4": [ + { # L4+ devices + "name": ["r5", "r7", "r9", "s5", "s7", "s9", "p5", "q5"], + 1280: [20, 40, 60, 80, 100, 120], + 1200: [20, 40, 60, 80], + 1000: [8, 16, 26], + }, + { # L4 devices + 1200: [16, 32, 48, 64, 80], + 1000: [6, 12, 18, 26], + }, + ], + "l5": { 1280: [20, 40, 60, 80, 110], # Vcore range 0 - 1200: [20, 40, 60, 80], # Vcore range 1 - 1000: [8, 16, 26], # Vcore range 2 - }, - 'g0': { - 1200: [24, 48, 64], - 1000: [8, 16] - }, - 'g4': { - 1280: [20, 40, 60, 80, 100, 120, 140, 160, 170], - 1000: [8, 16, 26] - }, - 'h7': [{ - 'name': ['23', '25', '30', '33', '35'], - 1260: [70, 140, 210, 275], - 1150: [67, 133, 200], - 1050: [50, 100, 150], - 950: [35, 70, 85], - },{ - 'name': ['a0', 'a3', 'b0', 'b3'], - 1250: [42, 84, 126, 168, 210, 252, 280], - 1150: [38, 76, 114, 152, 190, 225], - 1050: [34, 68, 102, 136, 160], - 950: [22, 44, 66, 88], - },{ # The remaining devices - 1260: [70, 140, 210, 225, 240], - 1150: [70, 140, 210, 225], - 1050: [55, 110, 165, 225], - 950: [45, 90, 135, 180, 225], - }], - 'wb': { - 1200: [18, 36, 54, 64], - 1000: [6, 12, 16] - }, - 'wl': { - 1200: [18, 36, 48], - 1000: [6, 12, 16] - }, - 'u5': { + 1200: [20, 40, 60, 80], # Vcore range 1 + 1000: [8, 16, 26], # Vcore range 2 + }, + "g0": {1200: [24, 48, 64], 1000: [8, 16]}, + "g4": {1280: [20, 40, 60, 80, 100, 120, 140, 160, 170], 1000: [8, 16, 26]}, + "h7": [ + { + "name": ["23", "25", "30", "33", "35"], + 1260: [70, 140, 210, 275], + 1150: [67, 133, 200], + 1050: [50, 100, 150], + 950: [35, 70, 85], + }, + { + "name": ["a0", "a3", "b0", "b3"], + 1250: [42, 84, 126, 168, 210, 252, 280], + 1150: [38, 76, 114, 152, 190, 225], + 1050: [34, 68, 102, 136, 160], + 950: [22, 44, 66, 88], + }, + { # The remaining devices + 1260: [70, 140, 210, 225, 240], + 1150: [70, 140, 210, 225], + 1050: [55, 110, 165, 225], + 950: [45, 90, 135, 180, 225], + }, + ], + "wb": {1200: [18, 36, 54, 64], 1000: [6, 12, 16]}, + "wl": {1200: [18, 36, 48], 1000: [6, 12, 16]}, + "u5": { 1200: [32, 64, 96, 128, 160], 1100: [30, 60, 90, 110], 1000: [24, 48, 55], @@ -195,155 +186,152 @@ def getGpioRemapForModuleConfig(module, config): }, } + def getFlashLatencyForDevice(did): lts = stm32_flash_latency.get(did.family) - if lts is None: return {}; # family not known + if lts is None: + return {} # family not known + # Convert MHz to Hz and filter out string keys - lconv = lambda l: {k:[int(f*1e6) for f in v] for k, v in l.items() if isinstance(k, int)} - if isinstance(lts, dict): return lconv(lts); # whole family uses same table + def lconv(lt): + return {k: [int(f * 1000000.0) for f in v] for k, v in lt.items() if isinstance(k, int)} + + if isinstance(lts, dict): + return lconv(lts) # whole family uses same table for lt in lts: # check if all conditions match if all(did[k] in v for k, v in lt.items() if isinstance(k, str)): - return lconv(lt) # return filtered table - return lconv(lts[-1]) # if non were found, return last table - - -stm32f3_dma_remap = \ -{ - 'dma1ch1': { - 'tim17_ch1': 'tim17_up', - 'tim17_up': {'position': 12, 'mask': 1, 'id': 0}, - }, - 'dma1ch2': { - 'adc2': {'position': 72, 'mask': 3, 'id': 2}, - 'i2c_tx': {'position': 70, 'mask': 3, 'id': 1}, - 'spi_rx': {'position': 64, 'mask': 3, 'id': 0}, # also 'id': 3 - }, - 'dma1ch3': { - 'dac1_ch1': 'tim6_up', - 'i2c_rx': {'position': 68, 'mask': 3, 'id': 1}, - 'spi_tx': {'position': 66, 'mask': 3, 'id': 0}, # also 'id': 3 - 'tim16_ch1': 'tim16_up', - 'tim16_up': {'position': 11, 'mask': 1, 'id': 0}, - 'tim6_up': {'position': 13, 'mask': 1, 'id': 1}, - }, - 'dma1ch4': { - 'adc2': {'position': 72, 'mask': 3, 'id': 3}, - 'dac1_ch2': 'tim7_up', - 'i2c_tx': {'position': 70, 'mask': 3, 'id': 2}, - 'spi_rx': {'position': 64, 'mask': 3, 'id': 1}, - 'tim7_up': {'position': 14, 'mask': 1, 'id': 1}, - }, - 'dma1ch5': { - 'dac2_ch1': 'tim18_up', - 'i2c_rx': {'position': 68, 'mask': 3, 'id': 2}, - 'spi_tx': {'position': 66, 'mask': 3, 'id': 1}, - 'tim18_up': {'position': 15, 'mask': 1, 'id': 1}, - }, - 'dma1ch6': { - 'i2c_tx': {'position': 70, 'mask': 3, 'id': 0}, # also 'id': 3 - 'spi_rx': {'position': 64, 'mask': 3, 'id': 2}, - }, - 'dma1ch7': { - 'i2c_rx': {'position': 68, 'mask': 3, 'id': 0}, # also 'id': 3 - 'spi_tx': {'position': 66, 'mask': 3, 'id': 2}, - 'tim17_ch1': 'tim17_up', - 'tim17_up': {'position': 12, 'mask': 1, 'id': 1}, - }, - - - 'dma2ch1': { - 'adc2': [{'position': 8, 'mask': 1, 'id': 0}, - {'position': 73, 'mask': 1, 'id': 0}], - 'adc4': {'position': 8, 'mask': 1, 'id': 0}, - }, - 'dma2ch3': { - 'adc2': [{'position': 8, 'mask': 1, 'id': 1}, - {'position': 73, 'mask': 1, 'id': 0}], - 'adc4': {'position': 8, 'mask': 1, 'id': 1}, - 'dac1_ch1': 'tim6_up', - 'tim6_up': {'position': 13, 'mask': 1, 'id': 0}, - }, - 'dma2ch4': { - 'dac1_ch2': 'tim7_up', - 'tim7_up': {'position': 14, 'mask': 1, 'id': 0}, - }, - 'dma2ch5': { - 'dac2_ch1': 'tim18_up', - 'tim18_up': {'position': 15, 'mask': 1, 'id': 0}, + return lconv(lt) # return filtered table + return lconv(lts[-1]) # if non were found, return last table + + +stm32f3_dma_remap = { + "dma1ch1": { + "tim17_ch1": "tim17_up", + "tim17_up": {"position": 12, "mask": 1, "id": 0}, + }, + "dma1ch2": { + "adc2": {"position": 72, "mask": 3, "id": 2}, + "i2c_tx": {"position": 70, "mask": 3, "id": 1}, + "spi_rx": {"position": 64, "mask": 3, "id": 0}, # also 'id': 3 + }, + "dma1ch3": { + "dac1_ch1": "tim6_up", + "i2c_rx": {"position": 68, "mask": 3, "id": 1}, + "spi_tx": {"position": 66, "mask": 3, "id": 0}, # also 'id': 3 + "tim16_ch1": "tim16_up", + "tim16_up": {"position": 11, "mask": 1, "id": 0}, + "tim6_up": {"position": 13, "mask": 1, "id": 1}, + }, + "dma1ch4": { + "adc2": {"position": 72, "mask": 3, "id": 3}, + "dac1_ch2": "tim7_up", + "i2c_tx": {"position": 70, "mask": 3, "id": 2}, + "spi_rx": {"position": 64, "mask": 3, "id": 1}, + "tim7_up": {"position": 14, "mask": 1, "id": 1}, + }, + "dma1ch5": { + "dac2_ch1": "tim18_up", + "i2c_rx": {"position": 68, "mask": 3, "id": 2}, + "spi_tx": {"position": 66, "mask": 3, "id": 1}, + "tim18_up": {"position": 15, "mask": 1, "id": 1}, + }, + "dma1ch6": { + "i2c_tx": {"position": 70, "mask": 3, "id": 0}, # also 'id': 3 + "spi_rx": {"position": 64, "mask": 3, "id": 2}, + }, + "dma1ch7": { + "i2c_rx": {"position": 68, "mask": 3, "id": 0}, # also 'id': 3 + "spi_tx": {"position": 66, "mask": 3, "id": 2}, + "tim17_ch1": "tim17_up", + "tim17_up": {"position": 12, "mask": 1, "id": 1}, + }, + "dma2ch1": { + "adc2": [{"position": 8, "mask": 1, "id": 0}, {"position": 73, "mask": 1, "id": 0}], + "adc4": {"position": 8, "mask": 1, "id": 0}, + }, + "dma2ch3": { + "adc2": [{"position": 8, "mask": 1, "id": 1}, {"position": 73, "mask": 1, "id": 0}], + "adc4": {"position": 8, "mask": 1, "id": 1}, + "dac1_ch1": "tim6_up", + "tim6_up": {"position": 13, "mask": 1, "id": 0}, + }, + "dma2ch4": { + "dac1_ch2": "tim7_up", + "tim7_up": {"position": 14, "mask": 1, "id": 0}, + }, + "dma2ch5": { + "dac2_ch1": "tim18_up", + "tim18_up": {"position": 15, "mask": 1, "id": 0}, }, } -stm32f0_dma_remap = \ -{ - 'dma1ch1': { - 'tim17_up': [{'position': 14, 'mask': 1, 'id': 1}, - {'position': 12, 'mask': 1, 'id': 0}], - 'tim17_ch1': 'tim17_up', - 'adc': {'position': 8, 'mask': 1, 'id': 0}, - }, - 'dma1ch2': { - 'tim1_ch1': {'position': 28, 'mask': 1, 'id': 0}, - 'i2c1_tx': {'position': 27, 'mask': 1, 'id': 0}, - 'usart3_tx': {'position': 26, 'mask': 1, 'id': 1}, - 'tim17_up': [{'position': 14, 'mask': 1, 'id': 1}, - {'position': 12, 'mask': 1, 'id': 1}], - 'tim17_ch1': 'tim17_up', - 'usart1_tx': {'position': 9, 'mask': 1, 'id': 0}, - 'adc': {'position': 8, 'mask': 1, 'id': 1}, - }, - 'dma1ch3': { - 'tim1_ch2': {'position': 28, 'mask': 1, 'id': 0}, - 'tim2_ch2': {'position': 29, 'mask': 1, 'id': 0}, - 'i2c1_rx': {'position': 27, 'mask': 1, 'id': 0}, - 'usart3_rx': {'position': 26, 'mask': 1, 'id': 1}, - 'tim16_up': [{'position': 13, 'mask': 1, 'id': 1}, - {'position': 11, 'mask': 1, 'id': 0}], - 'tim16_ch1': 'tim16_up', - 'usart1_rx': {'position': 10, 'mask': 1, 'id': 0}, - }, - 'dma1ch4': { - 'tim1_ch3': {'position': 28, 'mask': 1, 'id': 0}, - 'tim3_trig': {'position': 30, 'mask': 1, 'id': 0}, - 'tim3_ch1': 'tim3_trig', - 'tim2_ch4': {'position': 29, 'mask': 1, 'id': 0}, - 'usart2_tx': {'position': 25, 'mask': 1, 'id': 0}, - 'spi2_rx': {'position': 24, 'mask': 1, 'id': 0}, - 'tim16_up': [{'position': 13, 'mask': 1, 'id': 1}, - {'position': 11, 'mask': 1, 'id': 1}], - 'tim16_ch1': 'tim16_up', - 'usart1_tx': {'position': 9, 'mask': 1, 'id': 1}, - }, - 'dma1ch5': { - 'usart2_rx': {'position': 25, 'mask': 1, 'id': 0}, - 'spi2_tx': {'position': 24, 'mask': 1, 'id': 0}, - 'usart1_rx': {'position': 10, 'mask': 1, 'id': 1}, - }, - 'dma1ch6': { - 'tim3_trig': {'position': 30, 'mask': 1, 'id': 1}, - 'tim3_ch1': 'tim3_trig', - 'tim1_ch1': {'position': 28, 'mask': 1, 'id': 1}, - 'tim1_ch2': 'tim1_ch1', - 'tim1_ch3': 'tim1_ch1', - 'i2c1_tx': {'position': 27, 'mask': 1, 'id': 1}, - 'usart3_rx': {'position': 26, 'mask': 1, 'id': 0}, - 'usart2_rx': {'position': 25, 'mask': 1, 'id': 1}, - 'spi2_rx': {'position': 24, 'mask': 1, 'id': 1}, - 'tim16_up': {'position': 13, 'mask': 1, 'id': 1}, - 'tim16_ch1': 'tim16_up', - }, - 'dma1ch7': { - 'tim2_ch2': {'position': 29, 'mask': 1, 'id': 1}, - 'tim2_ch4': 'tim2_ch2', - 'i2c1_rx': {'position': 27, 'mask': 1, 'id': 1}, - 'usart3_tx': {'position': 26, 'mask': 1, 'id': 0}, - 'usart2_tx': {'position': 25, 'mask': 1, 'id': 1}, - 'spi2_tx': {'position': 24, 'mask': 1, 'id': 1}, - 'tim17_up': {'position': 14, 'mask': 1, 'id': 1}, - 'tim17_ch1': 'tim17_up', +stm32f0_dma_remap = { + "dma1ch1": { + "tim17_up": [{"position": 14, "mask": 1, "id": 1}, {"position": 12, "mask": 1, "id": 0}], + "tim17_ch1": "tim17_up", + "adc": {"position": 8, "mask": 1, "id": 0}, + }, + "dma1ch2": { + "tim1_ch1": {"position": 28, "mask": 1, "id": 0}, + "i2c1_tx": {"position": 27, "mask": 1, "id": 0}, + "usart3_tx": {"position": 26, "mask": 1, "id": 1}, + "tim17_up": [{"position": 14, "mask": 1, "id": 1}, {"position": 12, "mask": 1, "id": 1}], + "tim17_ch1": "tim17_up", + "usart1_tx": {"position": 9, "mask": 1, "id": 0}, + "adc": {"position": 8, "mask": 1, "id": 1}, + }, + "dma1ch3": { + "tim1_ch2": {"position": 28, "mask": 1, "id": 0}, + "tim2_ch2": {"position": 29, "mask": 1, "id": 0}, + "i2c1_rx": {"position": 27, "mask": 1, "id": 0}, + "usart3_rx": {"position": 26, "mask": 1, "id": 1}, + "tim16_up": [{"position": 13, "mask": 1, "id": 1}, {"position": 11, "mask": 1, "id": 0}], + "tim16_ch1": "tim16_up", + "usart1_rx": {"position": 10, "mask": 1, "id": 0}, + }, + "dma1ch4": { + "tim1_ch3": {"position": 28, "mask": 1, "id": 0}, + "tim3_trig": {"position": 30, "mask": 1, "id": 0}, + "tim3_ch1": "tim3_trig", + "tim2_ch4": {"position": 29, "mask": 1, "id": 0}, + "usart2_tx": {"position": 25, "mask": 1, "id": 0}, + "spi2_rx": {"position": 24, "mask": 1, "id": 0}, + "tim16_up": [{"position": 13, "mask": 1, "id": 1}, {"position": 11, "mask": 1, "id": 1}], + "tim16_ch1": "tim16_up", + "usart1_tx": {"position": 9, "mask": 1, "id": 1}, + }, + "dma1ch5": { + "usart2_rx": {"position": 25, "mask": 1, "id": 0}, + "spi2_tx": {"position": 24, "mask": 1, "id": 0}, + "usart1_rx": {"position": 10, "mask": 1, "id": 1}, + }, + "dma1ch6": { + "tim3_trig": {"position": 30, "mask": 1, "id": 1}, + "tim3_ch1": "tim3_trig", + "tim1_ch1": {"position": 28, "mask": 1, "id": 1}, + "tim1_ch2": "tim1_ch1", + "tim1_ch3": "tim1_ch1", + "i2c1_tx": {"position": 27, "mask": 1, "id": 1}, + "usart3_rx": {"position": 26, "mask": 1, "id": 0}, + "usart2_rx": {"position": 25, "mask": 1, "id": 1}, + "spi2_rx": {"position": 24, "mask": 1, "id": 1}, + "tim16_up": {"position": 13, "mask": 1, "id": 1}, + "tim16_ch1": "tim16_up", + }, + "dma1ch7": { + "tim2_ch2": {"position": 29, "mask": 1, "id": 1}, + "tim2_ch4": "tim2_ch2", + "i2c1_rx": {"position": 27, "mask": 1, "id": 1}, + "usart3_tx": {"position": 26, "mask": 1, "id": 0}, + "usart2_tx": {"position": 25, "mask": 1, "id": 1}, + "spi2_tx": {"position": 24, "mask": 1, "id": 1}, + "tim17_up": {"position": 14, "mask": 1, "id": 1}, + "tim17_ch1": "tim17_up", }, } + def getDmaRemap(did, dma, channel, driver, inst, signal): if did.family == "f0": remap = stm32f0_dma_remap @@ -352,8 +340,8 @@ def getDmaRemap(did, dma, channel, driver, inst, signal): else: return None - key1 = "dma{}ch{}".format(dma, channel) - key2 = (driver + inst if inst else "") + ("_{}".format(signal) if signal else "") + key1 = f"dma{dma}ch{channel}" + key2 = (driver + inst if inst else "") + (f"_{signal}" if signal else "") signals = remap.get(key1, {}) signal = signals.get(key2, None) @@ -367,368 +355,339 @@ def getDmaRemap(did, dma, channel, driver, inst, signal): signal = [signal] # print(key1, key2, signal) - assert( isinstance(signal, list) ) + assert isinstance(signal, list) return signal -stm32_memory = \ -{ - 'f0': { - 'start': { - 'flash': 0x08000000, - 'sram': 0x20000000 - }, - 'model': [ +stm32_memory = { + "f0": { + "start": {"flash": 0x08000000, "sram": 0x20000000}, + "model": [ { - 'name': ['30', '31', '38', '42', '48', '51', '58', '70', '71', '72', '78', '91', '98'], - 'memories': {'flash': 0, 'sram1': 0} + "name": ["30", "31", "38", "42", "48", "51", "58", "70", "71", "72", "78", "91", "98"], + "memories": {"flash": 0, "sram1": 0}, } - ] + ], }, - 'g0': { - 'start': { - 'flash': 0x08000000, - 'sram': 0x20000000 - }, - 'model': [ + "g0": { + "start": {"flash": 0x08000000, "sram": 0x20000000}, + "model": [ { - 'name': ['30', '31', '41', '50', '51', '61', '70', '71', '81', 'b0', 'b1', 'c0', 'c1'], - 'memories': {'flash': 0, 'sram1': 0} + "name": ["30", "31", "41", "50", "51", "61", "70", "71", "81", "b0", "b1", "c0", "c1"], + "memories": {"flash": 0, "sram1": 0}, } - ] + ], }, - 'g4': { - 'start': { - 'flash': 0x08000000, - 'ccm': 0x10000000, - 'sram': 0x20000000 - }, - 'model': [ - { - 'name': ['31', '41'], - 'memories': {'flash': 0, 'sram1': 0, 'sram2': 6*1024, 'ccm': 10*1024} - }, + "g4": { + "start": {"flash": 0x08000000, "ccm": 0x10000000, "sram": 0x20000000}, + "model": [ + {"name": ["31", "41"], "memories": {"flash": 0, "sram1": 0, "sram2": 6 * 1024, "ccm": 10 * 1024}}, + {"name": ["91", "a1"], "memories": {"flash": 0, "sram1": 0, "sram2": 16 * 1024, "ccm": 16 * 1024}}, { - 'name': ['91', 'a1'], - 'memories': {'flash': 0, 'sram1': 0, 'sram2': 16*1024, 'ccm': 16*1024} + "name": ["71", "73", "74", "83", "84"], + "memories": {"flash": 0, "sram1": 0, "sram2": 16 * 1024, "ccm": 32 * 1024}, }, - { - 'name': ['71', '73', '74', '83', '84'], - 'memories': {'flash': 0, 'sram1': 0, 'sram2': 16*1024, 'ccm': 32*1024} - } - ] + ], }, - 'f1': { - 'start': { - 'flash': 0x08000000, - 'sram': 0x20000000 - }, - 'model': [ - { - 'name': ['00', '01', '02', '03', '05', '07'], - 'memories': {'flash': 0, 'sram1': 0} - } - ] + "f1": { + "start": {"flash": 0x08000000, "sram": 0x20000000}, + "model": [{"name": ["00", "01", "02", "03", "05", "07"], "memories": {"flash": 0, "sram1": 0}}], }, - 'f2': { - 'start': { - 'flash': 0x08000000, - 'sram': 0x20000000 - }, - 'model': [ - { - 'name': ['05', '07', '15', '17'], - 'memories': {'flash': 0, 'sram1': 0, 'sram2': 16*1024} - } - ] + "f2": { + "start": {"flash": 0x08000000, "sram": 0x20000000}, + "model": [{"name": ["05", "07", "15", "17"], "memories": {"flash": 0, "sram1": 0, "sram2": 16 * 1024}}], }, - 'f3': { - 'start': { - 'flash': 0x08000000, - 'ccm': 0x10000000, - 'sram': 0x20000000 - }, - 'model': [ - { - 'name' : ['01', '02', '18', '78', '73'], - 'memories' : {'flash': 0, 'sram1' : 0} - }, + "f3": { + "start": {"flash": 0x08000000, "ccm": 0x10000000, "sram": 0x20000000}, + "model": [ + {"name": ["01", "02", "18", "78", "73"], "memories": {"flash": 0, "sram1": 0}}, { - 'name': ['03', '28', '34'], - 'size': ['4', '6', '8'], - 'memories': {'flash': 0, 'ccm': 4*1024, 'sram1': 0} + "name": ["03", "28", "34"], + "size": ["4", "6", "8"], + "memories": {"flash": 0, "ccm": 4 * 1024, "sram1": 0}, }, + {"name": ["03", "58"], "size": ["b", "c"], "memories": {"flash": 0, "ccm": 8 * 1024, "sram1": 0}}, + {"name": ["03", "98"], "size": ["d", "e"], "memories": {"flash": 0, "ccm": 16 * 1024, "sram1": 0}}, + ], + }, + "f4": { + "start": {"flash": 0x08000000, "ccm": 0x10000000, "sram": 0x20000000, "backup": 0x40024000}, + "model": [ + {"name": ["01", "10", "11", "12", "46"], "memories": {"flash": 0, "sram1": 0}}, { - 'name': ['03', '58'], - 'size': ['b', 'c'], - 'memories': {'flash': 0, 'ccm': 8*1024, 'sram1': 0} + "name": ["05", "07", "15", "17"], + "memories": {"flash": 0, "ccm": 64 * 1024, "sram1": 0, "sram2": 16 * 1024, "backup": 4 * 1024}, }, - { - 'name': ['03', '98'], - 'size': ['d', 'e'], - 'memories': {'flash': 0, 'ccm': 16*1024, 'sram1': 0} - } - ] - }, - 'f4': { - 'start': { - 'flash': 0x08000000, - 'ccm': 0x10000000, - 'sram': 0x20000000, - 'backup': 0x40024000 - }, - 'model': [ - { - 'name' : ['01', '10', '11', '12', '46'], - 'memories' : {'flash': 0, 'sram1' : 0} + {"name": ["13", "23"], "memories": {"flash": 0, "ccm": 64 * 1024, "sram1": 0, "backup": 4 * 1024}}, + { + "name": ["27", "29", "37", "39"], + "memories": { + "flash": 0, + "ccm": 64 * 1024, + "sram1": 0, + "sram2": 16 * 1024, + "sram3": 64 * 1024, + "backup": 4 * 1024, + }, }, { - 'name': ['05', '07', '15', '17'], - 'memories': {'flash': 0, 'ccm': 64*1024, 'sram1': 0, 'sram2': 16*1024, 'backup': 4*1024} + "name": ["69", "79"], + "memories": { + "flash": 0, + "ccm": 64 * 1024, + "sram1": 0, + "sram2": 32 * 1024, + "sram3": 128 * 1024, + "backup": 4 * 1024, + }, }, - { - 'name': ['13', '23'], - 'memories': {'flash': 0, 'ccm': 64*1024, 'sram1': 0, 'backup': 4*1024} + ], + }, + "f7": { + "start": { + "flash": 0x00200000, + "dtcm": 0x20000000, + "itcm": 0x00000000, + "sram": 0x20010000, + "backup": 0x40024000, + }, + "model": [ + { + "name": ["22", "32", "23", "30", "33", "45", "46", "50", "56"], + "memories": { + "flash": 0, + "itcm": 16 * 1024, + "dtcm": 64 * 1024, + "sram1": 0, + "sram2": 16 * 1024, + "backup": 4 * 1024, + }, }, { - 'name': ['27', '29', '37', '39'], - 'memories': {'flash': 0, 'ccm': 64*1024, 'sram1': 0, 'sram2': 16*1024, 'sram3': 64*1024, 'backup': 4*1024} + "name": ["65", "67", "68", "69", "77", "78", "79"], + "memories": { + "flash": 0, + "itcm": 16 * 1024, + "dtcm": 128 * 1024, + "sram1": 0, + "sram2": 16 * 1024, + "backup": 4 * 1024, + }, + "start": {"sram": 0x20020000}, # overwrite due to bigger dtcm size! }, - { - 'name': ['69', '79'], - 'memories': {'flash': 0, 'ccm': 64*1024, 'sram1': 0, 'sram2': 32*1024, 'sram3': 128*1024, 'backup': 4*1024} - } - ] - }, - 'f7': { - 'start': { - 'flash': 0x00200000, - 'dtcm': 0x20000000, - 'itcm': 0x00000000, - 'sram': 0x20010000, - 'backup': 0x40024000 + ], + }, + "h7": { + "start": { + "flash": 0x08000000, + "dtcm": 0x20000000, + "itcm": 0x00000000, + "d1_sram": 0x24000000, + "d2_sram": 0x30000000, + "d3_sram": 0x38000000, + "backup": 0x38800000, }, - 'model': [ - { - 'name': ['22', '32', '23', '30', '33', '45', '46', '50', '56'], - 'memories': {'flash': 0, 'itcm': 16*1024, 'dtcm': 64*1024, 'sram1': 0, 'sram2': 16*1024, 'backup': 4*1024} + "model": [ + { + "name": ["42"], + "memories": { + "flash": 0, + "itcm": 64 * 1024, + "dtcm": 128 * 1024, + "backup": 4 * 1024, + "d1_sram": 384 * 1024, + "d2_sram1": 32 * 1024, + "d2_sram2": 16 * 1024, + "d3_sram": 64 * 1024, + }, }, { - 'name': ['65', '67', '68', '69', '77', '78', '79'], - 'memories': {'flash': 0, 'itcm': 16*1024, 'dtcm': 128*1024, 'sram1': 0, 'sram2': 16*1024, 'backup': 4*1024}, - 'start': {'sram': 0x20020000} # overwrite due to bigger dtcm size! - } - ] - }, - 'h7': { - 'start': { - 'flash': 0x08000000, - 'dtcm': 0x20000000, - 'itcm': 0x00000000, - 'd1_sram': 0x24000000, - 'd2_sram': 0x30000000, - 'd3_sram': 0x38000000, - 'backup': 0x38800000 - }, - 'model': [ - { - 'name': ['42'], - 'memories': {'flash': 0, 'itcm': 64*1024, 'dtcm': 128*1024, 'backup': 4*1024, - 'd1_sram': 384*1024, - 'd2_sram1': 32*1024, 'd2_sram2': 16*1024, - 'd3_sram': 64*1024} + "name": ["23", "25", "30", "33", "35"], + "memories": { + "flash": 0, + "itcm": 64 * 1024, + "dtcm": 128 * 1024, + "backup": 4 * 1024, + "d1_sram": 320 * 1024, + "d2_sram1": 16 * 1024, + "d2_sram2": 16 * 1024, + "d3_sram": 16 * 1024, + }, }, { - 'name': ['23', '25', '30', '33', '35'], - 'memories': {'flash': 0, 'itcm': 64*1024, 'dtcm': 128*1024, 'backup': 4*1024, - 'd1_sram': 320*1024, - 'd2_sram1': 16*1024, 'd2_sram2': 16*1024, - 'd3_sram': 16*1024} + "name": ["40", "43", "50", "53"], + "memories": { + "flash": 0, + "itcm": 64 * 1024, + "dtcm": 128 * 1024, + "backup": 4 * 1024, + "d1_sram": 512 * 1024, + "d2_sram1": 128 * 1024, + "d2_sram2": 128 * 1024, + "d2_sram3": 32 * 1024, + "d3_sram": 64 * 1024, + }, }, { - 'name': ['40', '43', '50', '53'], - 'memories': {'flash': 0, 'itcm': 64*1024, 'dtcm': 128*1024, 'backup': 4*1024, - 'd1_sram': 512*1024, - 'd2_sram1': 128*1024, 'd2_sram2': 128*1024, 'd2_sram3': 32*1024, - 'd3_sram': 64*1024} + "name": ["45", "47", "55", "57"], + "core": ["m7"], + "memories": { + "flash": 0, + "itcm": 64 * 1024, + "dtcm": 128 * 1024, + "backup": 4 * 1024, + "d1_sram": 512 * 1024, + "d2_sram1": 128 * 1024, + "d2_sram2": 128 * 1024, + "d2_sram3": 32 * 1024, + "d3_sram": 64 * 1024, + }, }, { - 'name': ['45', '47', '55', '57'], - 'core': ['m7'], - 'memories': {'flash': 0, 'itcm': 64*1024, 'dtcm': 128*1024, 'backup': 4*1024, - 'd1_sram': 512*1024, - 'd2_sram1': 128*1024, 'd2_sram2': 128*1024, 'd2_sram3': 32*1024, - 'd3_sram': 64*1024} + "name": ["45", "47", "55", "57"], + "core": ["m4"], + "memories": { + "flash": 0, + "backup": 4 * 1024, + "d1_sram": 512 * 1024, + "d2_sram1": 128 * 1024, + "d2_sram2": 128 * 1024, + "d2_sram3": 32 * 1024, + "d3_sram": 64 * 1024, + }, }, { - 'name': ['45', '47', '55', '57'], - 'core': ['m4'], - 'memories': {'flash': 0, 'backup': 4*1024, - 'd1_sram': 512*1024, - 'd2_sram1': 128*1024, 'd2_sram2': 128*1024, 'd2_sram3': 32*1024, - 'd3_sram': 64*1024} + "name": ["a0", "a3", "b0", "b3"], + "memories": { + "flash": 0, + "itcm": 64 * 1024, + "dtcm": 128 * 1024, + "backup": 4 * 1024, + "d1_sram1": 256 * 1024, + "d1_sram2": 384 * 1024, + "d1_sram3": 384 * 1024, + "d2_sram1": 64 * 1024, + "d2_sram2": 64 * 1024, + "d3_sram": 32 * 1024, + }, }, - { - 'name': ['a0', 'a3', 'b0', 'b3'], - 'memories': {'flash': 0, 'itcm': 64*1024, 'dtcm': 128*1024, 'backup': 4*1024, - 'd1_sram1': 256*1024, 'd1_sram2': 384*1024, 'd1_sram3': 384*1024, - 'd2_sram1': 64*1024, 'd2_sram2': 64*1024, - 'd3_sram': 32*1024} - } - ] + ], }, - 'l0': { - 'start': { - 'flash': 0x08000000, - 'eeprom': 0x08080000, - 'sram': 0x20000000 - }, - 'model': [ + "l0": { + "start": {"flash": 0x08000000, "eeprom": 0x08080000, "sram": 0x20000000}, + "model": [ + {"name": ["10"], "size": ["4"], "memories": {"flash": 0, "sram1": 0, "eeprom": 128}}, + {"name": ["10"], "size": ["6", "8"], "memories": {"flash": 0, "sram1": 0, "eeprom": 256}}, { - 'name': ['10'], - 'size': ['4'], - 'memories': {'flash': 0, 'sram1': 0, 'eeprom': 128} - },{ - 'name': ['10'], - 'size': ['6', '8'], - 'memories': {'flash': 0, 'sram1': 0, 'eeprom': 256} - },{ # CAT1 - 'name': ['10', '11', '21'], - 'memories': {'flash': 0, 'sram1': 0, 'eeprom': 512} - },{ + "name": ["10", "11", "21"], + "memories": {"flash": 0, "sram1": 0, "eeprom": 512}, + }, + { # CAT2 - 'name': ['31', '41'], - 'memories': {'flash': 0, 'sram1': 0, 'eeprom': 1024} - },{ + "name": ["31", "41"], + "memories": {"flash": 0, "sram1": 0, "eeprom": 1024}, + }, + { # CAT3 - 'name': ['51', '52', '53', '62', '63'], - 'memories': {'flash': 0, 'sram1': 0, 'eeprom': 2*1024} - },{ + "name": ["51", "52", "53", "62", "63"], + "memories": {"flash": 0, "sram1": 0, "eeprom": 2 * 1024}, + }, + { # CAT5 - 'name': ['71', '72', '73', '81', '82', '83'], - 'memories': {'flash': 0, 'sram1': 0, 'eeprom': 6*1024} + "name": ["71", "72", "73", "81", "82", "83"], + "memories": {"flash": 0, "sram1": 0, "eeprom": 6 * 1024}, }, - ] + ], }, - 'l1': { - 'start': { - 'flash': 0x08000000, - 'eeprom': 0x08080000, - 'sram': 0x20000000 - }, - 'model': [ + "l1": { + "start": {"flash": 0x08000000, "eeprom": 0x08080000, "sram": 0x20000000}, + "model": [ { # CAT1 & 2 - 'name': ['00', '51', '52'], - 'size': ['6', '8', 'b'], - 'memories': {'flash': 0, 'sram1': 0, 'eeprom': 4*1024} - },{ + "name": ["00", "51", "52"], + "size": ["6", "8", "b"], + "memories": {"flash": 0, "sram1": 0, "eeprom": 4 * 1024}, + }, + { # CAT3 - 'name': ['00', '51', '52', '62'], - 'size': ['c'], - 'memories': {'flash': 0, 'sram1': 0, 'eeprom': 8*1024} - },{ + "name": ["00", "51", "52", "62"], + "size": ["c"], + "memories": {"flash": 0, "sram1": 0, "eeprom": 8 * 1024}, + }, + { # CAT4 - 'name': ['51', '52', '62'], - 'size': ['d'], - 'memories': {'flash': 0, 'sram1': 0, 'eeprom': 12*1024} - },{ + "name": ["51", "52", "62"], + "size": ["d"], + "memories": {"flash": 0, "sram1": 0, "eeprom": 12 * 1024}, + }, + { # CAT5 & 6 - 'name': ['51', '52', '62'], - 'size': ['e'], - 'memories': {'flash': 0, 'sram1': 0, 'eeprom': 16*1024} + "name": ["51", "52", "62"], + "size": ["e"], + "memories": {"flash": 0, "sram1": 0, "eeprom": 16 * 1024}, }, - ] + ], }, - 'l4': { - 'start': { - 'flash': 0x08000000, - 'ccm': 0x10000000, - 'sram': 0x20000000 - }, - 'model': [ + "l4": { + "start": {"flash": 0x08000000, "ccm": 0x10000000, "sram": 0x20000000}, + "model": [ + {"name": ["12", "22"], "memories": {"flash": 0, "sram1": 0, "ccm": 8 * 1024}}, + {"name": ["51", "71", "75", "76", "85", "86"], "memories": {"flash": 0, "sram1": 0, "ccm": 32 * 1024}}, { - 'name': ['12', '22'], - 'memories': {'flash': 0, 'sram1': 0, 'ccm': 8*1024} - },{ - 'name': ['51', '71', '75', '76', '85', '86'], - 'memories': {'flash': 0, 'sram1': 0, 'ccm': 32*1024} - },{ - 'name': ['31', '32', '33', '42', '43', '52', '62'], - 'memories': {'flash': 0, 'sram1': 0, 'ccm': 16*1024} - },{ - 'name': ['96', 'a6'], - 'memories': {'flash': 0, 'sram1': 0, 'ccm': 64*1024} + "name": ["31", "32", "33", "42", "43", "52", "62"], + "memories": {"flash": 0, "sram1": 0, "ccm": 16 * 1024}, }, + {"name": ["96", "a6"], "memories": {"flash": 0, "sram1": 0, "ccm": 64 * 1024}}, # Technically part of the STM32L4+ family { - 'name': ['r5', 'r7', 'r9', 's5', 's7', 's9'], - 'memories': {'flash': 0, 'sram1': 0, 'sram2': 64*1024, 'sram3': 384*1024} - },{ - 'name': ['p5', 'q5'], - 'memories': {'flash': 0, 'sram1': 0, 'sram2': 64*1024, 'sram3': 128*1024} - } - ] - }, - 'l5': { - 'start': { - 'flash': 0x08000000, - 'sram': 0x20000000 - }, - 'model': [ - { - 'name': ['52', '62'], - 'memories': {'flash': 0, 'sram1': 0, 'sram2': 64*1024} - } - ] - }, - 'wb': { - 'start': { - 'flash': 0x08000000, - 'sram': 0x20000000 - }, - 'model': [ - { - 'name': ['10', '15', '1m'], - 'memories': {'flash': 0, 'sram1': 0, 'sram2': 36*1024} - },{ - 'name': ['30', '35', '50', '55', '5m'], - 'memories': {'flash': 0, 'sram1': 0, 'sram2': 64*1024} - } - ] - }, - 'wl': { - 'start': { - 'flash': 0x08000000, - 'sram': 0x20000000 - }, - 'model': [ - { - 'name': ['54', '55', 'e4', 'e5'], - 'memories': {'flash': 0, 'sram1': 0, 'sram2': 32*1024} - } - ] - }, - 'u5': { - 'start': { - 'flash': 0x08000000, - 'sram1': 0x20000000, - 'sram2': 0x20030000, - 'sram3': 0x20040000, - 'sram4': 0x28000000, - 'bkpsram': 0x40036400, + "name": ["r5", "r7", "r9", "s5", "s7", "s9"], + "memories": {"flash": 0, "sram1": 0, "sram2": 64 * 1024, "sram3": 384 * 1024}, + }, + {"name": ["p5", "q5"], "memories": {"flash": 0, "sram1": 0, "sram2": 64 * 1024, "sram3": 128 * 1024}}, + ], + }, + "l5": { + "start": {"flash": 0x08000000, "sram": 0x20000000}, + "model": [{"name": ["52", "62"], "memories": {"flash": 0, "sram1": 0, "sram2": 64 * 1024}}], + }, + "wb": { + "start": {"flash": 0x08000000, "sram": 0x20000000}, + "model": [ + {"name": ["10", "15", "1m"], "memories": {"flash": 0, "sram1": 0, "sram2": 36 * 1024}}, + {"name": ["30", "35", "50", "55", "5m"], "memories": {"flash": 0, "sram1": 0, "sram2": 64 * 1024}}, + ], + }, + "wl": { + "start": {"flash": 0x08000000, "sram": 0x20000000}, + "model": [{"name": ["54", "55", "e4", "e5"], "memories": {"flash": 0, "sram1": 0, "sram2": 32 * 1024}}], + }, + "u5": { + "start": { + "flash": 0x08000000, + "sram1": 0x20000000, + "sram2": 0x20030000, + "sram3": 0x20040000, + "sram4": 0x28000000, + "bkpsram": 0x40036400, }, - 'model': [ - { - 'name': ['75', '85'], - 'memories': {'flash': 0, 'sram1': 192*1024, 'sram2': 64*1024, 'sram3': 512*1024, 'sram4': 16*1024, 'bkpsram': 2*1024} + "model": [ + { + "name": ["75", "85"], + "memories": { + "flash": 0, + "sram1": 192 * 1024, + "sram2": 64 * 1024, + "sram3": 512 * 1024, + "sram4": 16 * 1024, + "bkpsram": 2 * 1024, + }, } # ,{ # 'name': ['95', '99', 'a5', 'a9'], # This devices are not published yet... # 'memories': # TODO... # } - ] + ], }, } @@ -736,18 +695,19 @@ def getDmaRemap(did, dma, channel, driver, inst, signal): def getMemoryModel(device_id): mem_fam = stm32_memory[device_id.family] mem_model = None - for model in mem_fam['model']: - if all(device_id[k] in v for k, v in model.items() if k not in ['start', 'memories']): + for model in mem_fam["model"]: + if all(device_id[k] in v for k, v in model.items() if k not in ["start", "memories"]): mem_model = model break - if mem_model == None: - LOGGER.error("Memory model not found for device '{}'".format(device_id.string)) + if mem_model is None: + LOGGER.error(f"Memory model not found for device '{device_id.string}'") exit(1) - start = dict(mem_fam['start']) - memories = dict(mem_model['memories']) - start.update(mem_model.get('start', {})) + start = dict(mem_fam["start"]) + memories = dict(mem_model["memories"]) + start.update(mem_model.get("start", {})) return (start, memories) + def getMemoryForDevice(device_id, total_flash, total_ram): mem_start, mem_model = getMemoryModel(device_id) @@ -755,22 +715,23 @@ def getMemoryForDevice(device_id, total_flash, total_ram): mem_model["flash"] = total_flash # Correct RAM size - main_sram = next( (name for (name, size) in mem_model.items() if size == 0), None ) + main_sram = next((name for (name, size) in mem_model.items() if size == 0), None) if main_sram is not None: - main_sram_name = next( ram for ram in mem_start.keys() if main_sram.startswith(ram) ) + main_sram_name = next(ram for ram in mem_start.keys() if main_sram.startswith(ram)) # compute the size from total ram mem_model[main_sram] = total_ram main_sram_index = int(main_sram.split("sram")[-1]) if main_sram[-1].isdigit() else 0 for name, size in mem_model.items(): mem_index = int(name.split("sram")[-1]) if name[-1].isdigit() else 0 - if ((name.startswith(main_sram_name) and mem_index != main_sram_index) or - (device_id.family == "g4" and name.startswith("ccm"))): + if (name.startswith(main_sram_name) and mem_index != main_sram_index) or ( + device_id.family == "g4" and name.startswith("ccm") + ): mem_model[main_sram] -= size # Assemble flattened memories memories = [] for name, size in mem_model.items(): - sram_name = next( ram for ram in mem_start.keys() if name.startswith(ram) ) + sram_name = next(ram for ram in mem_start.keys() if name.startswith(ram)) index = int(name.split("sram")[-1]) if name[-1].isdigit() else 0 start = mem_start[sram_name] if index > 1: @@ -779,8 +740,6 @@ def getMemoryForDevice(device_id, total_flash, total_ram): mem_index = int(mem_name.split("sram")[-1]) if mem_name[-1].isdigit() else 0 if mem_name.startswith(sram_name) and mem_index < index: start += mem_size - memories.append( (name, start, size) ) + memories.append((name, start, size)) return memories - - diff --git a/src/modm_data/dl/__init__.py b/src/modm_data/dl/__init__.py index a409583..6abdc3d 100644 --- a/src/modm_data/dl/__init__.py +++ b/src/modm_data/dl/__init__.py @@ -2,3 +2,10 @@ # SPDX-License-Identifier: MPL-2.0 from . import stmicro +from .store import download_data, download_file + +__all__ = [ + "stmicro", + "download_data", + "download_file", +] diff --git a/src/modm_data/dl/stmicro/__init__.py b/src/modm_data/dl/stmicro/__init__.py index ce166af..256e261 100644 --- a/src/modm_data/dl/stmicro/__init__.py +++ b/src/modm_data/dl/stmicro/__init__.py @@ -1,8 +1,14 @@ # Copyright 2022, Niklas Hauser # SPDX-License-Identifier: MPL-2.0 -from .document import Document -from .document import load_remote_info, load_local_info -from .document import store_remote_info, store_local_info - +from .document import Document, load_remote_info, load_local_info, store_remote_info, store_local_info from .cubemx import download_cubemx + +__all__ = [ + "Document", + "load_remote_info", + "load_local_info", + "store_remote_info", + "store_local_info", + "download_cubemx", +] diff --git a/src/modm_data/dl/stmicro/__main__.py b/src/modm_data/dl/stmicro/__main__.py index 153b3ce..4675849 100644 --- a/src/modm_data/dl/stmicro/__main__.py +++ b/src/modm_data/dl/stmicro/__main__.py @@ -1,20 +1,18 @@ # Copyright 2022, Niklas Hauser # SPDX-License-Identifier: MPL-2.0 -import os -import sys import shutil import tempfile import argparse from pathlib import Path import modm_data -from modm_data.utils import ext_path from modm_data.dl.stmicro import download_cubemx from modm_data.dl.stmicro import load_remote_info, store_remote_info from modm_data.dl.stmicro import load_local_info, store_local_info, Document import logging + logging.basicConfig(level=logging.DEBUG) LOGGER = logging.getLogger("update") @@ -74,28 +72,16 @@ def download_pdfs(base_dir, with_download, new_pdfs=None): if __name__ == "__main__": parser = argparse.ArgumentParser() - parser.add_argument( - "--download", - action="store_true", - help="Download the data source.") - parser.add_argument( - "--directory", - type=Path, - help="Where to put the downloaded files.") + parser.add_argument("--download", action="store_true", help="Download the data source.") + parser.add_argument("--directory", type=Path, help="Where to put the downloaded files.") subparsers = parser.add_subparsers(title="Command", dest="command") pdf_parser = subparsers.add_parser("pdf", help="PDF documents.") - pdf_parser.add_argument( - "--new", - type=Path, - help="Dump the newly downloaded PDFs into this file.") + pdf_parser.add_argument("--new", type=Path, help="Dump the newly downloaded PDFs into this file.") cubemx_parser = subparsers.add_parser("cubemx", help="CubeMX database.") - cubemx_parser.add_argument( - "--patch", - action="store_true", - help="Apply the patch to the database.") + cubemx_parser.add_argument("--patch", action="store_true", help="Apply the patch to the database.") args = parser.parse_args() diff --git a/src/modm_data/dl/stmicro/cubemx.py b/src/modm_data/dl/stmicro/cubemx.py index b980030..dca6dec 100755 --- a/src/modm_data/dl/stmicro/cubemx.py +++ b/src/modm_data/dl/stmicro/cubemx.py @@ -2,12 +2,10 @@ # SPDX-License-Identifier: MPL-2.0 -import urllib.request import zipfile import shutil import re import io -import os import random import time import logging @@ -20,27 +18,19 @@ LOGGER = logging.getLogger(__name__) -_hdr = { - 'Accept': '*', - 'Accept-Encoding': '*', - 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.2 Safari/605.1.15', - 'Accept-Language': 'en-GB,en;q=0.9', - 'Connection': 'keep-alive', -} - def _dl(url): - cmd = f"curl '{url}' -L -s --max-time 120 -o - " + " ".join(f"-H '{k}: {v}'" for k,v in _hdr.items()) + cmd = f"curl '{url}' -L -s --max-time 120 -o - " + " ".join(f"-H '{k}: {v}'" for k, v in _hdr.items()) return subprocess.run(cmd, shell=True, stdout=subprocess.PIPE).stdout def download_cubemx(extraction_path: Path, with_download: bool = True, with_patch: bool = True) -> bool: # First check STMUpdaterDefinitions.xml from this zip update_url = "https://sw-center.st.com/packs/resource/utility/updaters.zip" - update_url2 = "https://www.ebuc23.com/s3/stm_test/software/utility/updaters.zip" + update_url2 = "https://www.ebuc23.com/s3/stm_test/software/utility/updaters.zip" # noqa: F841 # Then Release="MX.6.2.0" maps to this: -win, -lin, -mac cube_url = "https://sw-center.st.com/packs/resource/library/stm32cube_mx_v{}-lin.zip" - cube_url2 = "https://www.ebuc23.com/s3/stm_test/software/library/stm32cube_mx_v{}-lin.zip" + cube_url2 = "https://www.ebuc23.com/s3/stm_test/software/library/stm32cube_mx_v{}-lin.zip" # noqa: F841 if with_download: LOGGER.info("Downloading Update Info...") @@ -63,7 +53,7 @@ def download_cubemx(extraction_path: Path, with_download: bool = True, with_patc LOGGER.info("Downloading Database...") LOGGER.debug(cube_url.format(version)) - time.sleep(random.randrange(1,6)) + time.sleep(random.randrange(1, 6)) z = zipfile.ZipFile(io.BytesIO(_dl(cube_url.format(version)))) LOGGER.info("Extracting Database...") @@ -80,15 +70,14 @@ def download_cubemx(extraction_path: Path, with_download: bool = True, with_patc for file in Path(extraction_path).glob("**/*"): if str(file).endswith(".xml"): with file.open("r", newline=None, encoding="utf-8", errors="replace") as rfile: - content = [l.rstrip()+"\n" for l in rfile.readlines()] + content = [line.rstrip() + "\n" for line in rfile.readlines()] with file.open("w", encoding="utf-8") as wfile: wfile.writelines(content) if with_patch: LOGGER.info("Patching Database...") from . import data + return pkg_apply_patch(data, "cubemx.patch", extraction_path) return True - - diff --git a/src/modm_data/dl/stmicro/document.py b/src/modm_data/dl/stmicro/document.py index 6f90c30..82053e3 100644 --- a/src/modm_data/dl/stmicro/document.py +++ b/src/modm_data/dl/stmicro/document.py @@ -4,9 +4,7 @@ import json import logging from pathlib import Path -from functools import cached_property from ..store import download_file, download_data -from ...utils import ext_path LOGGER = logging.getLogger(__name__) @@ -87,10 +85,10 @@ def __hash__(self) -> int: # Technical docs for STMicro sensors "sensors": [ "SC444", - #SC1946", + # SC1946", "SC1449", "SC1288", - #SC1718", + # SC1718", "SC1448", "SC1922", "SC1316", @@ -103,15 +101,14 @@ def __hash__(self) -> int: "SC397", ], # Technical docs for STMicro hardware debug tools - "debug": [ - "SC2330" - ] + "debug": ["SC2330"], } _json_url_prefix = "https://www.st.com/bin/st/selectors/cxst/en.cxst-rs-grid.html/" _json_url_suffix = ".technical_literature.json" -_json_urls = {key: [_json_url_prefix + url + _json_url_suffix for url in urls] - for key, urls in _json_short_urls.items()} +_json_urls = { + key: [_json_url_prefix + url + _json_url_suffix for url in urls] for key, urls in _json_short_urls.items() +} _remote_info = "remote.json" _local_info = "local.json" @@ -119,10 +116,10 @@ def __hash__(self) -> int: def load_remote_info(base_dir: Path, use_cached: bool = False) -> list[dict]: info = base_dir / _remote_info if use_cached and info.exists(): - LOGGER.debug(f"Loading remote info from cache") + LOGGER.debug("Loading remote info from cache") docs = json.loads(info.read_text()) else: - LOGGER.info(f"Downloading remote info") + LOGGER.info("Downloading remote info") docs = [] for urls in _json_urls.values(): for url in urls: @@ -133,14 +130,13 @@ def load_remote_info(base_dir: Path, use_cached: bool = False) -> list[dict]: def store_remote_info(base_dir: Path, docs: list[dict]): info = base_dir / _remote_info info.parent.mkdir(parents=True, exist_ok=True) - info.write_text(json.dumps(sorted(docs, - key=lambda d: (d["title"], d["version"])), indent=4, sort_keys=True)) + info.write_text(json.dumps(sorted(docs, key=lambda d: (d["title"], d["version"])), indent=4, sort_keys=True)) def load_local_info(base_dir: Path) -> list[dict]: info = base_dir / _local_info if info.exists(): - LOGGER.debug(f"Loading local info from cache") + LOGGER.debug("Loading local info from cache") return json.loads(info.read_text()) return [] @@ -148,8 +144,7 @@ def load_local_info(base_dir: Path) -> list[dict]: def store_local_info(base_dir: Path, docs: list[dict]): info = base_dir / _local_info info.parent.mkdir(parents=True, exist_ok=True) - info.write_text(json.dumps(sorted(docs, - key=lambda d: (d["title"], d["version"])), indent=4, sort_keys=True)) + info.write_text(json.dumps(sorted(docs, key=lambda d: (d["title"], d["version"])), indent=4, sort_keys=True)) def sync_info(base_dir: Path, use_cached: bool = False) -> set[Document]: diff --git a/src/modm_data/dl/store.py b/src/modm_data/dl/store.py index 3d19b36..a24a73c 100644 --- a/src/modm_data/dl/store.py +++ b/src/modm_data/dl/store.py @@ -1,26 +1,22 @@ # Copyright 2022, Niklas Hauser # SPDX-License-Identifier: MPL-2.0 -import os -import shutil import logging -import tempfile import subprocess from pathlib import Path -from urllib.request import urlopen, Request LOGGER = logging.getLogger(__name__) _hdr = { - 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', - 'Sec-Fetch-Site': 'none', - 'Accept-Encoding': 'identity', - 'Sec-Fetch-Mode': 'navigate', - 'Host': 'www.st.com', - 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.2 Safari/605.1.15', - 'Accept-Language': 'en-GB,en;q=0.9', - 'Sec-Fetch-Dest': 'document', - 'Connection': 'keep-alive', + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "Sec-Fetch-Site": "none", + "Accept-Encoding": "identity", + "Sec-Fetch-Mode": "navigate", + "Host": "www.st.com", + "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.2 Safari/605.1.15", + "Accept-Language": "en-GB,en;q=0.9", + "Sec-Fetch-Dest": "document", + "Connection": "keep-alive", } @@ -34,7 +30,7 @@ def download_data(url: str, encoding: str = None, errors: str = None) -> str: :return: The data as a decoded string. """ LOGGER.debug(f"Downloading data from {url}") - cmd = f"curl '{url}' -L -s --max-time 120 -o - " + " ".join(f"-H '{k}: {v}'" for k,v in _hdr.items()) + cmd = f"curl '{url}' -L -s --max-time 120 -o - " + " ".join(f"-H '{k}: {v}'" for k, v in _hdr.items()) data = subprocess.run(cmd, shell=True, stdout=subprocess.PIPE).stdout return data.decode(encoding=encoding or "utf-8", errors=errors or "ignore") @@ -55,7 +51,7 @@ def download_file(url: str, path: Path, overwrite: bool = False) -> bool: if isinstance(path, Path): path.parent.mkdir(parents=True, exist_ok=True) LOGGER.debug(f"Downloading file from {url} to {path}") - cmd = f"curl '{url}' -L -s --max-time 60 -o {path} " + " ".join(f"-H '{k}: {v}'" for k,v in _hdr.items()) + cmd = f"curl '{url}' -L -s --max-time 60 -o {path} " + " ".join(f"-H '{k}: {v}'" for k, v in _hdr.items()) return subprocess.call(cmd, shell=True) == 0 # with tempfile.NamedTemporaryFile() as outfile: # os.system(f'wget -q --user-agent="{_hdr["User-Agent"]}" "{url}" -O {outfile.name}') diff --git a/src/modm_data/header2svd/__init__.py b/src/modm_data/header2svd/__init__.py index f838789..041edaa 100644 --- a/src/modm_data/header2svd/__init__.py +++ b/src/modm_data/header2svd/__init__.py @@ -7,3 +7,8 @@ from . import stmicro from .header import Header + +__all__ = [ + "stmicro", + "Header", +] diff --git a/src/modm_data/header2svd/header.py b/src/modm_data/header2svd/header.py index 697ca63..05ffb21 100644 --- a/src/modm_data/header2svd/header.py +++ b/src/modm_data/header2svd/header.py @@ -7,8 +7,8 @@ class Header: - CMSIS_PATH = root_path("ext/cmsis/header/CMSIS/Core/Include") - CACHE_HEADER = defaultdict(dict) + CMSIS_PATH = root_path("ext/cmsis/header/CMSIS/Core/Include") + _CACHE_HEADER = defaultdict(dict) def __init__(self, filename, substitutions=None): self.filename = filename @@ -18,15 +18,15 @@ def __init__(self, filename, substitutions=None): @property def _cache(self): - return Header.CACHE_HEADER[self.filename] + return Header._CACHE_HEADER[self.filename] @property def header(self): from CppHeaderParser import CppHeader + if "header" not in self._cache: content = self.filename.read_text(encoding="utf-8-sig", errors="replace") for pattern, subs in self.substitutions.items(): content = re.sub(pattern, subs, content, flags=(re.DOTALL | re.MULTILINE)) self._cache["header"] = CppHeader(content, "string") return self._cache["header"] - diff --git a/src/modm_data/header2svd/stmicro/__init__.py b/src/modm_data/header2svd/stmicro/__init__.py index 401983c..fc925f9 100644 --- a/src/modm_data/header2svd/stmicro/__init__.py +++ b/src/modm_data/header2svd/stmicro/__init__.py @@ -2,4 +2,11 @@ # SPDX-License-Identifier: MPL-2.0 from .header import Header, getDefineForDevice + from .tree import normalize_memory_map + +__all__ = [ + "Header", + "getDefineForDevice", + "normalize_memory_map", +] diff --git a/src/modm_data/header2svd/stmicro/__main__.py b/src/modm_data/header2svd/stmicro/__main__.py index 5b0fd3f..46b7b63 100644 --- a/src/modm_data/header2svd/stmicro/__main__.py +++ b/src/modm_data/header2svd/stmicro/__main__.py @@ -36,16 +36,19 @@ def main(): calls = [] for devices in header_devices: - call = f"python3 -m modm_data.header2svd.stmicro " \ - f"--device {' --device '.join(devices)} " \ - f"> log/stmicro/svd/header_{list(sorted(devices))[0]}.txt 2>&1" + call = ( + f"python3 -m modm_data.header2svd.stmicro " + f"--device {' --device '.join(devices)} " + f"> log/stmicro/svd/header_{list(sorted(devices))[0]}.txt 2>&1" + ) calls.append(call) # print(call) with ThreadPool() as pool: retvals = list(tqdm.tqdm(pool.imap(lambda c: subprocess.run(c, shell=True), calls), total=len(calls))) for retval, call in zip(retvals, calls): - if retval.returncode != 0: print(call) + if retval.returncode != 0: + print(call) return all(r.returncode == 0 for r in retvals) mmaps = defaultdict(list) @@ -55,7 +58,7 @@ def main(): device = did_from_string(device) header = Header(device) print(device.string, header.filename) - mmaptree = header.memory_map_tree # create cache entry + mmaptree = header.memory_map_tree # create cache entry mmaps[header._memory_map_key].append(device) headers[header._memory_map_key] = header diff --git a/src/modm_data/header2svd/stmicro/header.py b/src/modm_data/header2svd/stmicro/header.py index 3e41fb1..bd16241 100644 --- a/src/modm_data/header2svd/stmicro/header.py +++ b/src/modm_data/header2svd/stmicro/header.py @@ -35,7 +35,7 @@ def getDefineForDevice(device_id, familyDefines): # get all defines for this device name - devName = "STM32{}{}".format(device_id.family.upper(), device_id.name.upper()) + devName = f"STM32{device_id.family.upper()}{device_id.name.upper()}" # Map STM32F7x8 -> STM32F7x7 if device_id.family == "f7" and devName[8] == "8": @@ -54,7 +54,7 @@ def getDefineForDevice(device_id, familyDefines): if device_id.family == "h7": devNameMatch = devName + "xx" else: - devNameMatch = devName + "x{}".format(device_id.size.upper()) + devNameMatch = devName + f"x{device_id.size.upper()}" if device_id.family == "l1": # Map STM32L1xxQC and STM32L1xxZC -> STM32L162QCxA variants if device_id.pin in ["q", "z"] and device_id.size == "c": @@ -69,7 +69,7 @@ def getDefineForDevice(device_id, familyDefines): return define # now we match for the pin-id. - devNameMatch = devName + "{}x".format(device_id.pin.upper()) + devNameMatch = devName + f"{device_id.pin.upper()}x" for define in deviceDefines: if devNameMatch <= define: return define @@ -78,31 +78,32 @@ def getDefineForDevice(device_id, familyDefines): class Header(CmsisHeader): - HEADER_PATH = ext_path("stmicro/header") - CACHE_PATH = cache_path("cmsis/stm32") - CACHE_FAMILY = defaultdict(dict) - BUILTINS = { + _HEADER_PATH = ext_path("stmicro/header") + _CACHE_PATH = cache_path("cmsis/stm32") + _CACHE_FAMILY = defaultdict(dict) + _BUILTINS = { "const uint32_t": 4, "const uint16_t": 2, "const uint8_t": 1, "uint32_t": 4, "uint16_t": 2, - "uint8_t": 1, + "uint8_t": 1, } def __init__(self, did): self.did = did - self.family_folder = "stm32{}xx".format(self.did.family) - self.cmsis_folder = Header.HEADER_PATH / self.family_folder / "Include" - self.family_header_file = "{}.h".format(self.family_folder) + self.family_folder = f"stm32{self.did.family}xx" + self.cmsis_folder = Header._HEADER_PATH / self.family_folder / "Include" + self.family_header_file = f"{self.family_folder}.h" self.family_defines = self._get_family_defines() self.define = getDefineForDevice(self.did, self.family_defines) assert self.define is not None self.is_valid = self.define is not None - if not self.is_valid: return + if not self.is_valid: + return - self.header_file = "{}.h".format(self.define.lower()) + self.header_file = f"{self.define.lower()}.h" self.device_map = None substitutions = { # r"/\* +?(Legacy defines|Legacy aliases|Old .*? legacy purpose|Aliases for .*?) +?\*/.*?\n\n": "", @@ -134,78 +135,102 @@ def memory_map_tree(self): def interrupt_table(self): if "vectors" not in self._cache: interrupt_enum = [i["values"] for i in self.header.enums if i["name"] == "IRQn_Type"][0] - vectors = [{"position": int(str(i["value"]).replace(" ", "")), - "name": i["name"][:-5]} for i in interrupt_enum] + vectors = [ + {"position": int(str(i["value"]).replace(" ", "")), "name": i["name"][:-5]} for i in interrupt_enum + ] self._cache["vectors"] = vectors return self._cache["vectors"] def _get_family_defines(self): - if self.did.family not in Header.CACHE_FAMILY: + if self.did.family not in Header._CACHE_FAMILY: defines = [] - match = re.findall(r"if defined\((?PSTM32[A-Z].....)\)", (self.cmsis_folder / self.family_header_file).read_text(encoding="utf-8", errors="replace")) - if match: defines = match; - else: LOGGER.error("Cannot find family defines for {}!".format(self.did.string)); - Header.CACHE_FAMILY[self.did.family]["family_defines"] = defines - return Header.CACHE_FAMILY[self.did.family]["family_defines"] + match = re.findall( + r"if defined\((?PSTM32[A-Z].....)\)", + (self.cmsis_folder / self.family_header_file).read_text(encoding="utf-8", errors="replace"), + ) + if match: + defines = match + else: + LOGGER.error(f"Cannot find family defines for {self.did.string}!") + Header._CACHE_FAMILY[self.did.family]["family_defines"] = defines + return Header._CACHE_FAMILY[self.did.family]["family_defines"] def _get_filtered_defines(self): defines = {} # get all the non-empty defines for define in self.header.defines: if comment := re.search(r"/\* *(.*?) *\*/", define): - if "legacy" in comment.group(1): continue + if "legacy" in comment.group(1): + continue define = re.sub(r"/\*.*?\*/", "", define).strip() name, *parts = define.split(" ") - if not len(parts): continue - if any(i in name for i in ["("]): continue; - if any(name.endswith(i) for i in ["_IRQn", "_IRQHandler", "_SUPPORT", "_TypeDef"]): continue; - if any(name.startswith(i) for i in ["IS_"]): continue; - if name in ["FLASH_SIZE", "FLASH_BANK_SIZE", "COMP1_COMMON", "COMP12_COMMON_BASE", "OPAMP12_COMMON_BASE", "BL_ID"]: continue; + if not len(parts): + continue + if any(i in name for i in ["("]): + continue + if any(name.endswith(i) for i in ["_IRQn", "_IRQHandler", "_SUPPORT", "_TypeDef"]): + continue + if any(name.startswith(i) for i in ["IS_"]): + continue + if name in [ + "FLASH_SIZE", + "FLASH_BANK_SIZE", + "COMP1_COMMON", + "COMP12_COMMON_BASE", + "OPAMP12_COMMON_BASE", + "BL_ID", + ]: + continue defines[name] = "".join(parts[1:]).strip() return defines def _get_defines(self): from jinja2 import Environment + # create the destination directory - destination = (Header.CACHE_PATH / self.family_folder / self.header_file).with_suffix(".cpp").absolute() + destination = (Header._CACHE_PATH / self.family_folder / self.header_file).with_suffix(".cpp").absolute() core = self.did.get("core", "") - executable = destination.with_suffix("."+core if core else "") + executable = destination.with_suffix("." + core if core else "") defines = self._get_filtered_defines() if not executable.exists(): # generate the cpp file from the template - LOGGER.info("Generating {} ...".format(destination.name)) + LOGGER.info(f"Generating {destination.name} ...") substitutions = {"header": self.header_file, "defines": sorted(defines)} content = Environment().from_string(HEADER_TEMPLATE).render(substitutions) # write the cpp file into the cache destination.parent.mkdir(exist_ok=True, parents=True) destination.write_text(content) header_defines = [self.define] - if core: header_defines.append(f"CORE_C{core.upper()}=1") + if core: + header_defines.append(f"CORE_C{core.upper()}=1") # compile file into an executable includes = [str(Header.CMSIS_PATH.absolute()), str(self.cmsis_folder.absolute())] - gcc_command = ["g++", "-Wno-narrowing", "-fms-extensions", + gcc_command = [ + "g++", + "-Wno-narrowing", + "-fms-extensions", " ".join(f"-I{incl}" for incl in includes), " ".join(f"-D{define}" for define in header_defines), - "-o {}".format(executable), - str(destination) + f"-o {executable}", + str(destination), ] - LOGGER.info("Compiling {} ...".format(destination.name)) + LOGGER.info(f"Compiling {destination.name} ...") retval = subprocess.run(" ".join(gcc_command), shell=True) if retval.returncode: - LOGGER.error("Header compilation failed! {}".format(retval)); + LOGGER.error(f"Header compilation failed! {retval}") return None # execute the file - LOGGER.info("Running {} ...".format(executable.name)) + LOGGER.info(f"Running {executable.name} ...") retval = subprocess.run([str(executable)], stdout=subprocess.PIPE) if retval.returncode: - LOGGER.error("Header execution failed! {}".format(retval)); + LOGGER.error(f"Header execution failed! {retval}") return None # parse the printed values localv = {} exec(retval.stdout, globals(), localv) undefined = [d for d in defines if d not in localv["cpp_defines"]] if len(undefined): - LOGGER.warning("Undefined macros: {}".format(undefined)) + LOGGER.warning(f"Undefined macros: {undefined}") return localv["cpp_defines"] def _get_memmap(self): @@ -223,21 +248,29 @@ def _get_memmap(self): LOGGER.debug(f"Found peripheral ({typedef} *) {name} @ 0x{defines[name]:x}") # build the array containing the peripheral types - raw_types = {typedef: [(v["type"], v["name"], int(v["array_size"], 16 if v["array_size"].startswith("0x") else 10) if v["array"] else 0) - for v in values["properties"]["public"]] - for typedef, values in self.header.classes.items()} + raw_types = { + typedef: [ + ( + v["type"], + v["name"], + int(v["array_size"], 16 if v["array_size"].startswith("0x") else 10) if v["array"] else 0, + ) + for v in values["properties"]["public"] + ] + for typedef, values in self.header.classes.items() + } # function to recursively flatten the types def _flatten_type(typedef, result, prefix=""): - for (t, n, s) in raw_types[typedef]: - if t in Header.BUILTINS: - size = Header.BUILTINS[t] + for t, n, s in raw_types[typedef]: + if t in Header._BUILTINS: + size = Header._BUILTINS[t] name = None if n.upper().startswith("RESERVED") else n if s == 0: - result.append( (size, (prefix + name) if name else name) ) + result.append((size, (prefix + name) if name else name)) else: if not name: - result.append( (size * s, name) ) + result.append((size * s, name)) else: result.extend([(size, f"{prefix}{name}.{ii}") for ii in range(s)]) elif t in raw_types.keys(): @@ -245,9 +278,9 @@ def _flatten_type(typedef, result, prefix=""): _flatten_type(t, result, prefix) else: for ii in range(s): - _flatten_type(t, result, prefix + "{}.{}.".format(n, ii)) + _flatten_type(t, result, prefix + f"{n}.{ii}.") else: - LOGGER.error("Unknown type: {} ({} {})".format(t, n, s)) + LOGGER.error(f"Unknown type: {t} ({n} {s})") exit(1) # flatten all types @@ -277,53 +310,60 @@ def _flatten_type(typedef, result, prefix=""): prefix.append("{}_{}_".format(peri, "".join([r for r in sreg if r.isupper()]) + sreg[1])) prefix.append("{}_{}x_".format(peri, "".join([r for r in sreg if r.isupper()]))) # A bunch of aliases - if "FSMC_BTCR_" in prefix: prefix.extend(["FSMC_BCRx_", "FSMC_BTRx_"]) - if "ADC_TR_" in prefix: prefix.append("ADC_TR1_") - if "DBGMCU_APB1FZ_" in prefix: prefix.append("DBGMCU_APB1_FZ_") - if "DBGMCU_APB2FZ_" in prefix: prefix.append("DBGMCU_APB2_FZ_") + if "FSMC_BTCR_" in prefix: + prefix.extend(["FSMC_BCRx_", "FSMC_BTRx_"]) + if "ADC_TR_" in prefix: + prefix.append("ADC_TR1_") + if "DBGMCU_APB1FZ_" in prefix: + prefix.append("DBGMCU_APB1_FZ_") + if "DBGMCU_APB2FZ_" in prefix: + prefix.append("DBGMCU_APB2_FZ_") # if "FLASH_KEYR_" in prefix: prefix.extend(["FLASH_KEY1_", "FLASH_KEY2_"]) # if "FLASH_OPTKEYR_" in prefix: prefix.extend(["FLASH_OPTKEY1_", "FLASH_OPTKEY2_"]) - if "GPIO_AFR1_" in prefix: prefix.append("GPIO_AFRL_") - if "GPIO_AFR2_" in prefix: prefix.append("GPIO_AFRH_") - if "SAI_Block" in typedef: prefix = [p.replace("SAI_", "SAI_x") for p in prefix] + if "GPIO_AFR1_" in prefix: + prefix.append("GPIO_AFRL_") + if "GPIO_AFR2_" in prefix: + prefix.append("GPIO_AFRH_") + if "SAI_Block" in typedef: + prefix = [p.replace("SAI_", "SAI_x") for p in prefix] regmap = {} for p in prefix: keys = [d for d in defines.keys() if d.startswith(p)] seen_defines.extend(keys) - regmap.update({k.replace(p, ""):defines[k] for k in keys}) + regmap.update({k.replace(p, ""): defines[k] for k in keys}) if not len(regmap): - LOGGER.info("Empty: {:30} {}->{} ({} >> {})".format(typedef, peri, prefix, reg[1], sreg)) + LOGGER.info(f"Empty: {typedef:30} {peri}->{prefix} ({reg[1]} >> {sreg})") # convert macro names to positional arguments fields = sorted(list(set([r[:-4] for r in regmap if r.endswith("_Pos")]))) registers = {} for field in fields: - regs = {k:v for k,v in regmap.items() if k == field or k.startswith(field + "_")} + regs = {k: v for k, v in regmap.items() if k == field or k.startswith(field + "_")} val = regs.pop(field, None) pos = regs.pop(field + "_Pos", None) msk = regs.pop(field + "_Msk", None) if val is None: - LOGGER.warning("{} not found: {}".format(field, regs)) + LOGGER.warning(f"{field} not found: {regs}") continue if pos is None: - LOGGER.warning("{}_Pos not found: {}".format(field, regs)) + LOGGER.warning(f"{field}_Pos not found: {regs}") continue if msk is None: - LOGGER.warning("{}_Msk not found: {}".format(field, regs)) + LOGGER.warning(f"{field}_Msk not found: {regs}") continue - rem = {k.replace(field + "_", ""):v for k,v in regs.items()} + rem = {k.replace(field + "_", ""): v for k, v in regs.items()} mask = msk >> pos width = 0 - while(mask): + while mask: width += 1 mask >>= 1 registers[pos] = (field, width, msk, val, rem) # print(registers) # Store in map - matched_types[typedef].append( (position, reg[0], reg[1], registers) ) + matched_types[typedef].append((position, reg[0], reg[1], registers)) position += reg[0] # print the remaining @@ -332,15 +372,14 @@ def _flatten_type(typedef, result, prefix=""): peri = "_".join([t for t in typedef.split("_") if t.isupper()]) + "_" rem = [d for d in remaining_defines if d.startswith(peri)] if len(rem): - LOGGER.warning("Unassigned defines for ({} *) {}: {}".format(typedef, peri, len(rem))) + LOGGER.warning(f"Unassigned defines for ({typedef} *) {peri}: {len(rem)}") for d in rem: - LOGGER.info("{}: {}".format(d, defines[d])) + LOGGER.info(f"{d}: {defines[d]}") # for typedef, registers in matched_types.items(): # print(typedef) # for reg in registers: - # print(" {:03x}: {}".format(reg[0], reg[2])) - + # print(f" {reg[0]:03x}: {reg[2]}") device = svd.Device(self.did.string) for name, (typedef, address) in peripheral_map.items(): diff --git a/src/modm_data/header2svd/stmicro/tree.py b/src/modm_data/header2svd/stmicro/tree.py index 87f4318..560f272 100644 --- a/src/modm_data/header2svd/stmicro/tree.py +++ b/src/modm_data/header2svd/stmicro/tree.py @@ -4,9 +4,8 @@ import re import logging from anytree.search import findall, find_by_attr, findall_by_attr -from anytree import RenderTree from collections import defaultdict -from ...svd import * +from ...svd import Device, PeripheralType, Peripheral, Register, BitField LOGGER = logging.getLogger(__file__) @@ -28,7 +27,8 @@ def _normalize_subtypes(memtree, peripheral, *subtypes): tchannels = [] for dma, channels in dmamap.items(): tdma = find_by_attr(memtree, peripheral, maxlevel=2) - if not channels: continue + if not channels: + continue tchannel = find_by_attr(memtree, channels[0].type, maxlevel=2) tchannels.append(tchannel) for channel in channels: @@ -64,9 +64,11 @@ def _normalize_duplicates(memtree, infilter, outfilter): def _normalize_adc_common(memtree): adc = set(findall_by_attr(memtree, "ADC_TypeDef", name="type", maxlevel=2)) - if len(adc) != 1: return memtree + if len(adc) != 1: + return memtree common = set(findall_by_attr(memtree, "ADC_Common_TypeDef", name="type", maxlevel=2)) - if len(common) != 1: return memtree + if len(common) != 1: + return memtree adc, common = adc.pop(), common.pop() tadc = find_by_attr(memtree, "ADC_TypeDef", maxlevel=2) @@ -87,7 +89,6 @@ def _normalize_adc_common(memtree): def _normalize_i2sext(memtree): - ext = findall(memtree, lambda n: "ext" not in n.name, maxlevel=2) for common in findall(memtree, lambda n: "ext" in n.name, maxlevel=2): LOGGER.info(f"Removing aliased peripheral '{common}'!") common.parent = None @@ -101,15 +102,21 @@ def _normalize_dmamux(memtree): dmamux.type = "DMAMUX_TypeDef" if dmamuxs: PeripheralType("DMAMUX_TypeDef", parent=memtree) - memtree = _normalize_subtypes(memtree, "DMAMUX_TypeDef", - "DMAMUX_Channel_TypeDef", "DMAMUX_ChannelStatus_TypeDef", - "DMAMUX_RequestGen_TypeDef", "DMAMUX_RequestGenStatus_TypeDef") + memtree = _normalize_subtypes( + memtree, + "DMAMUX_TypeDef", + "DMAMUX_Channel_TypeDef", + "DMAMUX_ChannelStatus_TypeDef", + "DMAMUX_RequestGen_TypeDef", + "DMAMUX_RequestGenStatus_TypeDef", + ) return memtree def _normalize_dfsdm(memtree): channels = findall(memtree, lambda n: re.match(r"DFSDM\d_Channel0$", n.name), maxlevel=2) - if not channels: return memtree + if not channels: + return memtree PeripheralType("DFSDM_TypeDef", parent=memtree) for channel in channels: @@ -166,10 +173,8 @@ def normalize_memory_map(memtree): memtree = _normalize_dmamux(memtree) memtree = _normalize_adc_common(memtree) - memtree = _normalize_duplicates(memtree, - lambda n: "_COMMON" in n.name, lambda n: "_COMMON" not in n.name) - memtree = _normalize_duplicates(memtree, - lambda n: "OPAMP" == n.name, lambda n: re.match(r"OPAMP\d$", n.name)) + memtree = _normalize_duplicates(memtree, lambda n: "_COMMON" in n.name, lambda n: "_COMMON" not in n.name) + memtree = _normalize_duplicates(memtree, lambda n: "OPAMP" == n.name, lambda n: re.match(r"OPAMP\d$", n.name)) memtree = _normalize_instances(memtree) memtree = _normalize_order(memtree) diff --git a/src/modm_data/html/__init__.py b/src/modm_data/html/__init__.py index fc8b63d..ddf16c6 100644 --- a/src/modm_data/html/__init__.py +++ b/src/modm_data/html/__init__.py @@ -1,10 +1,21 @@ # Copyright 2022, Niklas Hauser # SPDX-License-Identifier: MPL-2.0 +from . import stmicro from .document import Document from .chapter import Chapter from .table import Table from .text import Text, Heading, replace, listify from .list import List -from . import stmicro +__all__ = [ + "stmicro", + "Document", + "Chapter", + "Table", + "Text", + "Heading", + "List", + "replace", + "listify", +] diff --git a/src/modm_data/html/chapter.py b/src/modm_data/html/chapter.py index 4f431ae..f30e03d 100644 --- a/src/modm_data/html/chapter.py +++ b/src/modm_data/html/chapter.py @@ -56,9 +56,13 @@ def _heading_objects(self, obj_type, pattern, **subs) -> list: current[1].append(item) if current[1]: heading_texts.append(tuple(current)) - if pattern is None: return heading_texts - return [ht for ht in heading_texts if re.search(pattern, - ht[0].text(**subs) if ht[0] is not None else "", re.IGNORECASE)] + if pattern is None: + return heading_texts + return [ + ht + for ht in heading_texts + if re.search(pattern, ht[0].text(**subs) if ht[0] is not None else "", re.IGNORECASE) + ] def heading_texts(self, pattern=None, **subs) -> list: return self._heading_objects(Text, pattern, **subs) @@ -68,7 +72,8 @@ def heading_tables(self, pattern=None, **subs) -> list: def tables(self, pattern: str = None, **subs) -> list[Table]: tables = [t for t in self.items if isinstance(t, Table)] - if pattern is None: return tables + if pattern is None: + return tables return [t for t in tables if re.search(pattern, t.caption(**subs), re.IGNORECASE)] def table(self, pattern: str) -> Table: diff --git a/src/modm_data/html/document.py b/src/modm_data/html/document.py index 4335fb5..7db34f3 100644 --- a/src/modm_data/html/document.py +++ b/src/modm_data/html/document.py @@ -1,7 +1,8 @@ # Copyright 2022, Niklas Hauser # SPDX-License-Identifier: MPL-2.0 -import re, os +import re +import os import logging from pathlib import Path from functools import cached_property @@ -32,8 +33,7 @@ def path_pdf(self) -> str: def chapters(self, pattern: str = None) -> list[Chapter]: if pattern is None: return list(self._chapters.values()) - return [c for name, c in self._chapters.items() - if re.search(pattern, name, re.IGNORECASE)] + return [c for name, c in self._chapters.items() if re.search(pattern, name, re.IGNORECASE)] def chapter(self, pattern: str) -> Chapter: chapters = self.chapters(pattern) diff --git a/src/modm_data/html/list.py b/src/modm_data/html/list.py index 5ff6cf9..7e688d1 100644 --- a/src/modm_data/html/list.py +++ b/src/modm_data/html/list.py @@ -1,9 +1,7 @@ # Copyright 2022, Niklas Hauser # SPDX-License-Identifier: MPL-2.0 -import re -from functools import cached_property -from .text import replace as html_replace, Text +from .text import Text class List(Text): diff --git a/src/modm_data/html/parser.py b/src/modm_data/html/parser.py index 54bd4ba..dd754db 100644 --- a/src/modm_data/html/parser.py +++ b/src/modm_data/html/parser.py @@ -2,15 +2,14 @@ # SPDX-License-Identifier: MPL-2.0 import re -import os.path import logging -from functools import cached_property from html.parser import HTMLParser from .table import Table, Cell from .text import Text, Heading LOGGER = logging.getLogger(__name__) + class Parser(HTMLParser): def __init__(self): super().__init__(convert_charrefs=True) @@ -49,8 +48,8 @@ def handle_starttag(self, tag, attrs): self._tx = -1 if self._table and tag in ["th", "td"]: self._tx += 1 - tys = next((a[1] for a in attrs if a[0] == "rowspan"), 1) - txs = next((a[1] for a in attrs if a[0] == "colspan"), 1) + tys = next((a[1] for a in attrs if a[0] == "rowspan"), 1) + txs = next((a[1] for a in attrs if a[0] == "colspan"), 1) self._cell = Cell(self._tx, self._ty, int(txs), int(tys), tag == "th") elif re.match(r"h[1-6]", tag): @@ -87,7 +86,7 @@ def handle_endtag(self, tag): elif self._table and tag == "table": if self._table._cells: self._table._normalize() - if self._table.size > (1,1) or self._table.cell(0,0).html != "(omitted)": + if self._table.size > (1, 1) or self._table.cell(0, 0).html != "(omitted)": self._items.append(self._table) self._table = None self._type = None @@ -98,6 +97,3 @@ def handle_endtag(self, tag): if self._collect_data and tag not in self._ignore_tags: self._data += f"" - - - diff --git a/src/modm_data/html/stmicro/__init__.py b/src/modm_data/html/stmicro/__init__.py index bca0b82..8eaf64b 100644 --- a/src/modm_data/html/stmicro/__init__.py +++ b/src/modm_data/html/stmicro/__init__.py @@ -4,6 +4,16 @@ from .datasheet_sensor import DatasheetSensor from .datasheet_stm32 import DatasheetStm32 from .reference import ReferenceManual -from .document import load_documents, load_document_devices -from .document import datasheet_for_device, reference_manual_for_device +from .document import load_documents, load_document_devices, datasheet_for_device, reference_manual_for_device from .helper import find_device_filter + +__all__ = [ + "DatasheetSensor", + "DatasheetStm32", + "ReferenceManual", + "load_documents", + "load_document_devices", + "datasheet_for_device", + "reference_manual_for_device", + "find_device_filter", +] diff --git a/src/modm_data/html/stmicro/datasheet_sensor.py b/src/modm_data/html/stmicro/datasheet_sensor.py index adbb57e..ffe8dea 100644 --- a/src/modm_data/html/stmicro/datasheet_sensor.py +++ b/src/modm_data/html/stmicro/datasheet_sensor.py @@ -1,19 +1,13 @@ # Copyright 2022, Niklas Hauser # SPDX-License-Identifier: MPL-2.0 -import re -import itertools -from pathlib import Path -from functools import cached_property, cache -from collections import defaultdict +from functools import cache -from .helper import split_device_filter, split_package -from ...html.text import ReDict -import modm_data.html as html +from ..document import Document -class DatasheetSensor(html.Document): +class DatasheetSensor(Document): def __init__(self, path: str): super().__init__(path) diff --git a/src/modm_data/html/stmicro/datasheet_stm32.py b/src/modm_data/html/stmicro/datasheet_stm32.py index c77f16d..52967ad 100644 --- a/src/modm_data/html/stmicro/datasheet_stm32.py +++ b/src/modm_data/html/stmicro/datasheet_stm32.py @@ -2,19 +2,17 @@ # SPDX-License-Identifier: MPL-2.0 import re -import itertools -from pathlib import Path from functools import cached_property from collections import defaultdict from .helper import split_device_filter, split_package -from ...html.text import ReDict +from ...html.text import ReDict, listify as html_listify, replace as html_replace from ...owl import DeviceIdentifier from ...owl.stmicro import did_from_string -import modm_data.html as html +from ..document import Document -class DatasheetStm32(html.Document): +class DatasheetStm32(Document): def __init__(self, path: str): super().__init__(path) self._id = {} @@ -34,8 +32,9 @@ def devices(self) -> list[DeviceIdentifier]: ndevices = [] if tables := chapter.tables("Device summary"): for row in tables[0].cell_rows("number"): - ndevices.extend(c.text(br=" ", sup=" ").replace(",", " ").strip() - for cells in row.values() for c in cells) + ndevices.extend( + c.text(br=" ", sup=" ").replace(",", " ").strip() for cells in row.values() for c in cells + ) ndevices = [d for dev in ndevices for d in dev.split(" ") if "STM32" in d] else: # Check all uncaptioned tables for "Product summary" domain @@ -43,7 +42,7 @@ def devices(self) -> list[DeviceIdentifier]: if table.domains_x("Product +summary"): for row in table.cell_rows(): for cells in row.values(): - ndevices.extend(html.listify(cells[-1].text(br=" ", sup=" "))) + ndevices.extend(html_listify(cells[-1].text(br=" ", sup=" "))) break else: # print(chapter._path.relative_to(Path().cwd())) @@ -108,7 +107,7 @@ def devices(self) -> list[DeviceIdentifier]: for row in tables[0].cell_rows("code"): for cells in row.values(): for cell in cells: - devices.extend(html.listify(cell.text())) + devices.extend(html_listify(cell.text())) # convert to proper device identifiers dids = set() @@ -133,7 +132,8 @@ def _chapter_pins(self): @property def _table_pinout(self): tables = self._chapter_pins.tables( - "pin( +and +ball|/ball| +assignment +and)? +(definition|description)", br=" ") + "pin( +and +ball|/ball| +assignment +and)? +(definition|description)", br=" " + ) if not tables: raise ValueError(f"Unable to find pin definition table in chapter! {self._chapter_pins._relpath}") return tables[0] @@ -149,10 +149,11 @@ def _tables_alternate_functions(self): def packages(self): domains = self._table_pinout.domains_x(r"num(
)?ber|pins?:|pin/ball +name|:WLCSP.*?", br="
") if not domains: - raise ValueError(f"Unable to find package domains in table! {self._chapter_pins._relpath} {tables[0].caption()}") + raise ValueError(f"Unable to find package domains in table! {self._chapter_pins._relpath}") packages = [] for domain in domains: - if domain == "Pin (UFQFPN48):Number": continue + if domain == "Pin (UFQFPN48):Number": + continue ndpackage = domain.split(":")[-1].replace("UFBGA/TFBGA64", "UFBGA64/TFBGA64") ndpackage = ndpackage.replace("LPQF", "LQFP").replace("TSSPOP", "TSSOP") ndpackage = ndpackage.replace("UFBG100", "UFBGA100").replace("UBGA", "UFBGA") @@ -175,7 +176,7 @@ def packages(self): if "DS13311" in self.name or "DS13312" in self.name: ndpackage = ndpackage.replace("+SMPS", "") if (match := re.search(r":(STM32.*?):", domain)) is not None: - devs = html.listify(match.group(1)) + devs = html_listify(match.group(1)) ndpackage += "+" + "|".join(d.replace("x", ".") for d in devs) ndpackage, *filters = ndpackage.split("+") filters = ("+" + "+".join(filters)) if filters else "" @@ -199,12 +200,21 @@ def packages(self): def packages_pins(self): # Add pinouts and signals pin_replace = {r"sup| |D?NC|/$|\(.*?\)|.*?connected.*": "", "–": "-", r"VREF_\+": "VREF+"} - add_replace = {r"[- ]|\(.*?\)|.*?selection.*|.*?reset.*": "", - r"OPAMP": ",OPAMP", r"COMP": ",COMP", r"OPAMP,1": "OPAMP1"} - afs_replace = {r"[- ]|\(.*?\)": "", "LCD_SEG": ",LCD_SEG"} + add_replace = { + r"[- ]|\(.*?\)|.*?selection.*|.*?reset.*": "", + r"OPAMP": ",OPAMP", + r"COMP": ",COMP", + r"OPAMP,1": "OPAMP1", + } + afs_replace = {r"[- ]|\(.*?\)": "", "LCD_SEG": ",LCD_SEG"} # noqa: F841 pos_replace = {r'[“”\-"]|NC|\(.*?\)': ""} - sig_replace = {r"[- ]|\(.*?\)": "", r"(MOSI|SCK|NSS)I2S": r"\1,I2S", "_µM": "_M", - r"(CH\d)TIM": r"\1,TIM", r"_([RT]XD\d?|DV|EN|CLK)ETH_": r"_\1,ETH_"} + sig_replace = { + r"[- ]|\(.*?\)": "", + r"(MOSI|SCK|NSS)I2S": r"\1,I2S", + "_µM": "_M", + r"(CH\d)TIM": r"\1,TIM", + r"_([RT]XD\d?|DV|EN|CLK)ETH_": r"_\1,ETH_", + } data_packages = defaultdict(list) data_pins = defaultdict(dict) @@ -213,8 +223,9 @@ def packages_pins(self): # Import the pin definitions incl. additional function for row in self._table_pinout.cell_rows(br="
"): pin_name = row.match_value("pin +name|:name")[0].text(**pin_replace).strip() - if not pin_name: continue - ios = row.match_value("I ?/ ?O")[0].text(**{"-":""}) + if not pin_name: + continue + ios = row.match_value("I ?/ ?O")[0].text(**{"-": ""}) ptype = row.match_value("type")[0].text() # Hack to make fix the PD0/PD1 pins if pin_name.startswith("OSC") and (remap := row.match_value("remap")): @@ -222,28 +233,30 @@ def packages_pins(self): pin_name = f"{pin}-{pin_name}" data_pin = data_pins[pin_name] - if ios: data_pin["structure"] = ios - if ptype: data_pin["type"] = ptype + if ios: + data_pin["structure"] = ios + if ptype: + data_pin["type"] = ptype if ptype == "I/O" and "STM32F1" not in self.device_family: - signals = html.listify(row.match_value("additional")[0].text(**add_replace)) + signals = html_listify(row.match_value("additional")[0].text(**add_replace)) data_pin["additional"] = set(signals) for domain, package_name in packages: - if ppos := html.listify(row[domain][0].text(**pos_replace)): - data_packages[package_name].append( (pin_name, ppos) ) + if ppos := html_listify(row[domain][0].text(**pos_replace)): + data_packages[package_name].append((pin_name, ppos)) # Import the alternate functions for table in self._tables_alternate_functions: cells = table.domains("port|pin +name").cells(r"AF(IO)?\d+") for pin, afs in cells.items(): - name = html.replace(pin.split(":")[-1], **pin_replace) + name = html_replace(pin.split(":")[-1], **pin_replace) data_pin = data_pins[name] if "alternate" not in data_pin: data_pin["alternate"] = defaultdict(list) for af, csignals in afs.items(): af = int(re.search(r"AF(IO)?(\d{1,2})", af).group(2)) for csignal in csignals: - signals = html.listify(csignal.text(**sig_replace)) + signals = html_listify(csignal.text(**sig_replace)) data_pin["alternate"][af].extend(signals) return data_packages, data_pins diff --git a/src/modm_data/html/stmicro/document.py b/src/modm_data/html/stmicro/document.py index 3fa7649..9f29b3d 100644 --- a/src/modm_data/html/stmicro/document.py +++ b/src/modm_data/html/stmicro/document.py @@ -3,7 +3,7 @@ import json from collections import defaultdict -from ...html import Document +from ..document import Document from ...utils import cache_path, ext_path from .datasheet_stm32 import DatasheetStm32 from .datasheet_sensor import DatasheetSensor @@ -20,14 +20,19 @@ def load_documents() -> list: documents = defaultdict(dict) for path in sorted(ext_path("stmicro/html").glob("*-v*")): # This doc is parsed wrongly since it has a DRAFT background - if "DS12960-v5" in path.stem: continue + if "DS12960-v5" in path.stem: + continue # This doc has a preliminary ordering information STM32WBA52CGU6TR - if "DS14127" in path.stem: continue + if "DS14127" in path.stem: + continue doc = Document(path) if "DS" in doc.name and (chap := doc.chapters("chapter 0")): # FIXME: Better detection that DS13252 is a STM32WB55 module, not a chip! - if any("STM32" in h.html for h in chap[0].headings()) and \ - "DS13252" not in doc.name and "DS14096" not in doc.name: + if ( + any("STM32" in h.html for h in chap[0].headings()) + and "DS13252" not in doc.name + and "DS14096" not in doc.name + ): documents[doc.name][doc.version] = DatasheetStm32(path) else: documents[doc.name][doc.version] = DatasheetSensor(path) @@ -36,15 +41,16 @@ def load_documents() -> list: return documents -def load_document_devices(use_cached=True) -> tuple[dict[DeviceIdentifier, DatasheetStm32], - dict[DeviceIdentifier, ReferenceManual]]: +def load_document_devices( + use_cached=True, +) -> tuple[dict[DeviceIdentifier, DatasheetStm32], dict[DeviceIdentifier, ReferenceManual]]: global DOCUMENT_CACHE if DOCUMENT_CACHE is not None: return DOCUMENT_CACHE global MAP_DEVICE_DOC_FILE if MAP_DEVICE_DOC_FILE.exists() and use_cached: - with MAP_DEVICE_DOC_FILE.open('r', encoding='utf-8') as fh: + with MAP_DEVICE_DOC_FILE.open("r", encoding="utf-8") as fh: json_data = json.load(fh) docs = {} @@ -52,10 +58,8 @@ def load_document_devices(use_cached=True) -> tuple[dict[DeviceIdentifier, Datas docs[path] = DatasheetStm32(path) for path in set(json_data["rm"].values()): docs[path] = ReferenceManual(path) - datasheets = {did_from_string(did): docs[path] - for did, path in json_data["ds"].items()} - reference_manuals = {did_from_string(did): docs[path] - for did, path in json_data["rm"].items()} + datasheets = {did_from_string(did): docs[path] for did, path in json_data["ds"].items()} + reference_manuals = {did_from_string(did): docs[path] for did, path in json_data["rm"].items()} else: dss = defaultdict(set) rms = defaultdict(set) @@ -78,7 +82,7 @@ def load_document_devices(use_cached=True) -> tuple[dict[DeviceIdentifier, Datas for dev, docs in dss.items(): if len(docs) != 1: raise ValueError(f"One device points to multiple datasheets! {dev} -> {docs}") - datasheets = {did:list(ds)[0] for did, ds in dss.items()} + datasheets = {did: list(ds)[0] for did, ds in dss.items()} # print(len(datasheets.keys()), sorted(list(d.string for d in datasheets.keys()))) manuals = defaultdict(set) @@ -92,15 +96,15 @@ def load_document_devices(use_cached=True) -> tuple[dict[DeviceIdentifier, Datas if len(docs) != 1: raise ValueError(f"One device points to multiple reference manuals! {dev} -> {docs}") - reference_manuals = {did:list(rm)[0] for did, rm in manuals.items()} + reference_manuals = {did: list(rm)[0] for did, rm in manuals.items()} # Cache the results for the next call json_data = { "ds": {did.string: str(doc.path) for did, doc in datasheets.items()}, - "rm": {did.string: str(doc.path) for did, doc in reference_manuals.items()} + "rm": {did.string: str(doc.path) for did, doc in reference_manuals.items()}, } MAP_DEVICE_DOC_FILE.parent.mkdir(parents=True, exist_ok=True) - with MAP_DEVICE_DOC_FILE.open('w', encoding='utf-8') as fh: + with MAP_DEVICE_DOC_FILE.open("w", encoding="utf-8") as fh: json.dump(json_data, fh, indent=4) DOCUMENT_CACHE = (datasheets, reference_manuals) diff --git a/src/modm_data/html/stmicro/helper.py b/src/modm_data/html/stmicro/helper.py index 0a06ea4..7b46a3e 100644 --- a/src/modm_data/html/stmicro/helper.py +++ b/src/modm_data/html/stmicro/helper.py @@ -16,7 +16,7 @@ def split_device_filter(device_filter: str) -> list[str]: if len(parts := device_filter.split("/")) >= 2: base = parts[0] devices.append(base) - base = base[:-len(parts[1])] + base = base[: -len(parts[1])] for part in parts[1:]: devices.append(base + part) else: diff --git a/src/modm_data/html/stmicro/reference.py b/src/modm_data/html/stmicro/reference.py index 1ee869b..7d6f8fa 100644 --- a/src/modm_data/html/stmicro/reference.py +++ b/src/modm_data/html/stmicro/reference.py @@ -5,10 +5,11 @@ from functools import cached_property, cache from collections import defaultdict import modm_data.html as html -from .helper import split_device_filter, device_filter_from +from .helper import device_filter_from +from ..document import Document -class ReferenceManual(html.Document): +class ReferenceManual(Document): def __init__(self, path: str): super().__init__(path) @@ -35,7 +36,7 @@ def devices(self) -> list[str]: if len(parts := device.split("/")) >= 2: base = parts[0] devices.append(base) - base = base[:-len(parts[1])] + base = base[: -len(parts[1])] for part in parts[1:]: devices.append(base + part) else: @@ -69,20 +70,19 @@ def flash_latencies(self): if any(d.startswith("STM32L1") for d in self.devices): # See Table 13 and Table 26 return { - "stm32l1...[68b]": { # Cat 1 + "stm32l1...[68b]": { # Cat 1 1800: [16, 32], 1500: [8, 16], - 1200: [2.1, 4.2] + 1200: [2.1, 4.2], }, - "": { # Cat 2,3,4,5,6 + "": { # Cat 2,3,4,5,6 1800: [16, 32], 1500: [8, 16], - 1200: [4.2, 8] - } + 1200: [4.2, 8], + }, } - if (any(d.startswith("STM32F0") for d in self.devices) or - any(d.startswith("STM32F3") for d in self.devices)): + if any(d.startswith("STM32F0") for d in self.devices) or any(d.startswith("STM32F3") for d in self.devices): # Attempt to find the FLASH_ACR register and look at the LATENCY descriptions chapter = self.chapter(r"flash") headings = chapter.heading_tables(r"\(FLASH_ACR\)") @@ -107,7 +107,7 @@ def flash_latencies(self): for wait_state in row.match_keys("wait +state"): max_frequency = float(re.search(r"(.*?) +MHz", row[wait_state][0].text()).group(1)) table_data[min_voltage].append(max_frequency) - return {"": {k:sorted(v) for k,v in table_data.items()}} + return {"": {k: sorted(v) for k, v in table_data.items()}} ws_tables = {} for table in chapter.tables(r"wait states"): @@ -118,12 +118,10 @@ def flash_latencies(self): min_voltage = int(float(vrange.group(1)) * 1000) else: vrange = re.search(r"Range(\d[bn]?)", vkey.replace(" ", "")) - min_voltage = {"0": 1280, "1b": 1280, - "1n": 1200, "1": 1200, - "2": 1000}[vrange.group(1)] + min_voltage = {"0": 1280, "1b": 1280, "1n": 1200, "1": 1200, "2": 1000}[vrange.group(1)] max_frequency = row[vkey][0].text( - **{r"-": "", r".*?CLK.*?(\d+).*": r"\1", - r".*?;(\d+) *MHz.*": r"\1", r".*?(\d+).*": r"\1"}) + **{r"-": "", r".*?CLK.*?(\d+).*": r"\1", r".*?;(\d+) *MHz.*": r"\1", r".*?(\d+).*": r"\1"} + ) if max_frequency: table_data[min_voltage].append(float(max_frequency)) dfilter = device_filter_from(table.caption()) @@ -134,17 +132,26 @@ def flash_latencies(self): assert ws_tables return ws_tables - - @cached_property def vector_tables(self): - name_replace = {"p": r"\1,", r" +": "", r"\(.*?\)|_IRQn|_$|^-$|[Rr]eserved": "", - r"\+|and": ",", r"_(\w+)(LSE|BLE|I2C\d)_": r"_\1,\2_", - r"EXTI\[(\d+):(\d+)\]": r"EXTI\1_\2", r"\[(\d+):(\d+)\]": r"\2_\1"} - capt_replace = {"br": " ", r" +": " ", r"\((Cat\.\d.*?devices)\)": r"\1", r"\(.*?\)": "", - r"for +connectivity +line +devices": "for STM32F105/7", - r"for +XL\-density +devices": "for STM32F10xxF/G", - r"for +other +STM32F10xxx +devices": "for the rest",} + name_replace = { + "p": r"\1,", + r" +": "", + r"\(.*?\)|_IRQn|_$|^-$|[Rr]eserved": "", + r"\+|and": ",", + r"_(\w+)(LSE|BLE|I2C\d)_": r"_\1,\2_", + r"EXTI\[(\d+):(\d+)\]": r"EXTI\1_\2", + r"\[(\d+):(\d+)\]": r"\2_\1", + } + capt_replace = { + "br": " ", + r" +": " ", + r"\((Cat\.\d.*?devices)\)": r"\1", + r"\(.*?\)": "", + r"for +connectivity +line +devices": "for STM32F105/7", + r"for +XL\-density +devices": "for STM32F10xxF/G", + r"for +other +STM32F10xxx +devices": "for the rest", + } chapter = next(c for c in self.chapters(r"nvic|interrupt") if "exti" not in c.name) tables = chapter.tables(r"vector +table|list +of +vector|NVIC|CPU") @@ -157,7 +164,7 @@ def vector_tables(self): if len(tables) > 1: # Create the right table filter if (core := re.search(r"CPU(\d)|NVIC(\d)", caption)) is not None: - table_name += f":Core={core.group(1)}" # Multi-core device + table_name += f":Core={core.group(1)}" # Multi-core device elif devices := device_filter_from(caption): table_name += f":Device={devices}" elif categories := re.findall(r"Cat\.(\d)", caption): @@ -176,27 +183,39 @@ def vector_tables(self): @cache def peripheral_maps(self, chapter, assert_table=True): off_replace = {r" +": "", "0x000x00": "0x00", "to": "-", "×": "*", r"\(\d+\)": ""} - dom_replace = {r"Register +size": "Bit position"} + dom_replace = {r"Register +size": "Bit position"} # noqa: F841 reg_replace = { - r" +|\.+": "", r"\(COM(\d)\)": r"_COM\1", + r" +|\.+": "", + r"\(COM(\d)\)": r"_COM\1", r"^[Rr]es$||0x[\da-fA-FXx]+|\(.*?\)|-": "", - r"(?i)reserved|resetvalue.*": "", "enabled": "_EN", "disabled": "_DIS", - r"(?i)Outputcomparemode": "_Output", "(?i)Inputcapturemode": "_Input", "mode": "", - r"^TG_FS_": "OTG_FS_", "toRTC": "RTC", "SPI2S_": "SPI_", - r"andTIM\d+_.*": "", r"x=[\d,]+": ""} + r"(?i)reserved|resetvalue.*": "", + "enabled": "_EN", + "disabled": "_DIS", + r"(?i)Outputcomparemode": "_Output", + "(?i)Inputcapturemode": "_Input", + "mode": "", + r"^TG_FS_": "OTG_FS_", + "toRTC": "RTC", + "SPI2S_": "SPI_", + r"andTIM\d+_.*": "", + r"x=[\d,]+": "", + } fld_replace = { - r"\] +\d+(th|rd|nd|st)": "]", r" +|\.+|\[.*?\]|\[?\d+:\d+\]?|\(.*?\)|-|^[\dXx]+$|%|__|:0\]": "", + r"\] +\d+(th|rd|nd|st)": "]", + r" +|\.+|\[.*?\]|\[?\d+:\d+\]?|\(.*?\)|-|^[\dXx]+$|%|__|:0\]": "", r"Dataregister|Independentdataregister": "DATA", r"Framefilterreg0.*": "FRAME_FILTER_REG", r"[Rr]es(erved)?|[Rr]egular|x_x(bits)?|NotAvailable|RefertoSection\d+:Comparator": "", r"Sampletimebits|Injectedchannelsequence|channelsequence|conversioninregularsequencebits": "", - r"conversioninsequencebits|conversionininjectedsequencebits|or|first|second|third|fourth": ""} + r"conversioninsequencebits|conversionininjectedsequencebits|or|first|second|third|fourth": "", + } bit_replace = {r".*:": ""} glo_replace = {r"[Rr]eserved": ""} print(chapter._relpath) tables = chapter.tables(r"register +map") - if assert_table: assert tables + if assert_table: + assert tables peripheral_data = {} for table in tables: @@ -208,17 +227,21 @@ def peripheral_maps(self, chapter, assert_table=True): register_data = {} for row in table.cell_rows(): - rkey = next(k for k in row.match_keys("register") if not "size" in k) + rkey = next(k for k in row.match_keys("register") if "size" not in k) register = row[rkey][0].text(**reg_replace).strip() - if not register: continue + if not register: + continue offset = row.match_value(r"off-?set|addr")[0].text(**off_replace) - if not offset: continue + if not offset: + continue field_data = {} for bits in row.match_keys(r"^[\d-]+$|.*?:[\d-]+$"): field = row[bits][0].text(**fld_replace).strip() - if not field: continue + if not field: + continue bits = sorted(html.listify(html.replace(bits, **bit_replace), r"-")) - if len(bits) == 2: bits = range(int(bits[0]), int(bits[1])) + if len(bits) == 2: + bits = range(int(bits[0]), int(bits[1])) for bit in bits: bit = int(bit) field_data[bit] = field @@ -228,8 +251,8 @@ def peripheral_maps(self, chapter, assert_table=True): peripheral_data[caption] = (heading, register_data) if peripheral_data and all("HRTIM" in ca for ca in peripheral_data): - caption, heading = next((c,p) for c, (p, _) in peripheral_data.items()) - all_registers = {k:v for (_, values) in peripheral_data.values() for k,v in values.items()} + caption, heading = next((c, p) for c, (p, _) in peripheral_data.items()) + all_registers = {k: v for (_, values) in peripheral_data.values() for k, v in values.items()} peripheral_data = {caption: (heading, all_registers)} instance_offsets = {} @@ -245,16 +268,30 @@ def peripheral_maps(self, chapter, assert_table=True): @cached_property def peripherals(self): per_replace = { - r" +": "", r".*?\(([A-Z]+|DMA2D)\).*?": r"\1", + r" +": "", + r".*?\(([A-Z]+|DMA2D)\).*?": r"\1", r"Reserved|Port|Power|Registers|Reset|\(.*?\)|_REG": "", - r"(\d)/I2S\d": r"\1", r"/I2S|CANMessageRAM|Cortex-M4|I2S\dext|^GPV$": "", - r"Ethernet": "ETH", r"Flash": "FLASH", r"(?i).*ETHERNET.*": "ETH", - r"(?i)Firewall": "FW", r"HDMI-|": "", "SPDIF-RX": "SPDIFRX", - r"SPI2S2": "SPI2", "Tamper": "TAMP", "TT-FDCAN": "FDCAN", - r"USBOTG([FH])S": r"USB_OTG_\1S", "LCD-TFT": "LTDC", "DSIHOST": "DSI", - "TIMER": "TIM", r"^VREF$": "VREFBUF", "DelayBlock": "DLYB", - "I/O": "M", "I/O": "", "DAC1/2": "DAC12", - r"[a-z]": ""} + r"(\d)/I2S\d": r"\1", + r"/I2S|CANMessageRAM|Cortex-M4|I2S\dext|^GPV$": "", + r"Ethernet": "ETH", + r"Flash": "FLASH", + r"(?i).*ETHERNET.*": "ETH", + r"(?i)Firewall": "FW", + r"HDMI-|": "", + "SPDIF-RX": "SPDIFRX", + r"SPI2S2": "SPI2", + "Tamper": "TAMP", + "TT-FDCAN": "FDCAN", + r"USBOTG([FH])S": r"USB_OTG_\1S", + "LCD-TFT": "LTDC", + "DSIHOST": "DSI", + "TIMER": "TIM", + r"^VREF$": "VREFBUF", + "DelayBlock": "DLYB", + "I/O": "", + "DAC1/2": "DAC12", + r"[a-z]": "", + } adr_replace = {r" |\(\d\)": ""} sct_replace = {r"-": ""} hdr_replace = {r"Peripheral +register +map": "map"} @@ -279,17 +316,21 @@ def peripherals(self): if regmaps: regmap = regmaps[0].text(**sct_replace).strip() sections = re.findall(r"Section +(\d+\.\d+(\.\d+)?)", regmap) - if not sections: continue + if not sections: + continue sections = [s[0] for s in sections] else: sections = [] names = html.listify(row.match_value("peri")[0].text(**per_replace), r"[-\&\+/,]") - if not names: continue + if not names: + continue address = row.match_value("address")[0].text(**adr_replace) address_min = int(address.split("-")[0], 16) address_max = int(address.split("-")[1], 16) bus = row.match_value("bus")[0].text(**bus_replace).strip() - peripherals[table.caption()].append( (names, address_min, address_max, bus, sections) ) - print(f"{','.join(names):20} @[{hex(address_min)}, {hex(address_max)}] {bus:4} -> {', '.join(sections)}") + peripherals[table.caption()].append((names, address_min, address_max, bus, sections)) + print( + f"{','.join(names):20} @[{hex(address_min)}, {hex(address_max)}] {bus:4} -> {', '.join(sections)}" + ) return peripherals diff --git a/src/modm_data/html/table.py b/src/modm_data/html/table.py index 8053908..0975e42 100644 --- a/src/modm_data/html/table.py +++ b/src/modm_data/html/table.py @@ -2,24 +2,23 @@ # SPDX-License-Identifier: MPL-2.0 import re -from functools import cached_property from collections import defaultdict -from .text import replace as html_replace, ReDict, Text +from .text import ReDict, Text, Heading class Cell(Text): - def __init__(self, x: int, y: int, xs: int=1, ys: int=1, head: bool=False): + def __init__(self, x: int, y: int, xs: int = 1, ys: int = 1, head: bool = False): super().__init__() self._span = (xs, ys) self._pos = [(x, y)] self._head = head @property - def span(self) -> tuple[int,int]: + def span(self) -> tuple[int, int]: return self._span @property - def positions(self) -> list[tuple[int,int]]: + def positions(self) -> list[tuple[int, int]]: return self._pos @property @@ -69,7 +68,7 @@ def cells(self, pattern_x: str, pattern_y: str = None, **subs) -> list[Cell]: # print(x, y, dom_x, dom_y) cells[dom_y][dom_x].add(self._table.cell(x, y)) - return ReDict({k:ReDict({vk:list(vv) for vk,vv in v.items()}) for k,v in cells.items()}) + return ReDict({k: ReDict({vk: list(vv) for vk, vv in v.items()}) for k, v in cells.items()}) def __repr__(self) -> str: return f"Domains({self.domains_x()}, {self.domains_y()})" @@ -79,7 +78,7 @@ class Table: def __init__(self, heading=None): self._heading = heading or Heading("") self._cells = [] - self._size = (0,0) + self._size = (0, 0) self._grid = None self._hrows = 0 self._caption = Text("") @@ -160,12 +159,9 @@ def rows(self) -> int: return self._size[1] @property - def size(self) -> tuple[int,int]: + def size(self) -> tuple[int, int]: return self._size - def cell(self, x, y) -> Cell: - return self._grid[y][x] - def _normalize(self): xsize = sum(c._span[0] for c in self._cells if c._pos[0][1] == 0) ysize = max(c._pos[0][1] + c._span[1] for c in self._cells) @@ -193,5 +189,3 @@ def render(self): for x in range(self.columns): print(self.cell(x, y).text()[:15] if self.cell(x, y) is not None else "None", end="\t") print() - - diff --git a/src/modm_data/html/text.py b/src/modm_data/html/text.py index 4c474a1..b7e6293 100644 --- a/src/modm_data/html/text.py +++ b/src/modm_data/html/text.py @@ -5,11 +5,7 @@ def replace(html, **substitutions) -> str: - subs = { - "u": "*", "i": "*", "b": "*", - "sub": "*", "sup": "*", - "br": "*", "p": "*" - } + subs = {"u": "*", "i": "*", "b": "*", "sub": "*", "sup": "*", "br": "*", "p": "*"} subs.update(substitutions) for tag, replacement in subs.items(): if tag in {"u", "i", "b", "p", "br", "sup", "sub"}: @@ -27,7 +23,8 @@ def replace(html, **substitutions) -> str: def listify(text, pattern=None, strip=True) -> list[str]: - if pattern is None: pattern = " |,|/|
" + if pattern is None: + pattern = " |,|/|
" text = re.split(pattern, text) if strip: return [t.strip() for t in text if t.strip()] @@ -49,8 +46,10 @@ def match_values(self, pattern, **subs) -> list: def match_key(self, pattern, default=None, **subs) -> str: keys = self.match_keys(pattern, **subs) - if default is None: assert len(keys) == 1 - if len(keys) != 1: return default + if default is None: + assert len(keys) == 1 + if len(keys) != 1: + return default return keys[0] def match_keys(self, pattern, **subs) -> list: diff --git a/src/modm_data/html2owl/__init__.py b/src/modm_data/html2owl/__init__.py index 705ef65..6ff35ff 100644 --- a/src/modm_data/html2owl/__init__.py +++ b/src/modm_data/html2owl/__init__.py @@ -6,3 +6,5 @@ """ from . import stmicro + +__all__ = ["stmicro"] diff --git a/src/modm_data/html2owl/stmicro/__main__.py b/src/modm_data/html2owl/stmicro/__main__.py index 0d11c32..5c2ba60 100644 --- a/src/modm_data/html2owl/stmicro/__main__.py +++ b/src/modm_data/html2owl/stmicro/__main__.py @@ -1,12 +1,10 @@ # Copyright 2022, Niklas Hauser # SPDX-License-Identifier: MPL-2.0 -import re import tqdm import argparse import subprocess from pathlib import Path -from collections import defaultdict from multiprocessing.pool import ThreadPool from modm_data.html.stmicro import DatasheetStm32, ReferenceManual, load_documents @@ -30,12 +28,12 @@ def main(): elif isinstance(doc, ReferenceManual): docs.append(doc) - calls = [f"python3 -m modm_data.html2owl.stmicro " - f"--document {doc.path}" for doc in docs] + calls = [f"python3 -m modm_data.html2owl.stmicro " f"--document {doc.path}" for doc in docs] with ThreadPool() as pool: retvals = list(tqdm.tqdm(pool.imap(lambda c: subprocess.run(c, shell=True), calls), total=len(calls))) for retval, call in zip(retvals, calls): - if retval.returncode != 0: print(call) + if retval.returncode != 0: + print(call) return all(r.returncode == 0 for r in retvals) path = Path(args.document).absolute() @@ -44,9 +42,11 @@ def main(): elif path.stem.startswith("RM"): doc = ReferenceManual(path) - print(doc.path_pdf.relative_to(Path().cwd()), - doc.path.relative_to(Path().cwd()), - f"ext/stmicro/owl/{doc.fullname}.owl") + print( + doc.path_pdf.relative_to(Path().cwd()), + doc.path.relative_to(Path().cwd()), + f"ext/stmicro/owl/{doc.fullname}.owl", + ) owl_from_doc(doc) Store("stmicro", "stm32").save(doc.fullname) diff --git a/src/modm_data/html2svd/__init__.py b/src/modm_data/html2svd/__init__.py index 1708076..8518d90 100644 --- a/src/modm_data/html2svd/__init__.py +++ b/src/modm_data/html2svd/__init__.py @@ -6,3 +6,5 @@ """ from . import stmicro + +__all__ = ["stmicro"] diff --git a/src/modm_data/html2svd/stmicro/__init__.py b/src/modm_data/html2svd/stmicro/__init__.py index ee08e88..d39e9e7 100644 --- a/src/modm_data/html2svd/stmicro/__init__.py +++ b/src/modm_data/html2svd/stmicro/__init__.py @@ -3,3 +3,8 @@ from .reference import memory_map_from_reference_manual from .datasheet import memory_map_from_datasheet + +__all__ = [ + "memory_map_from_reference_manual", + "memory_map_from_datasheet", +] diff --git a/src/modm_data/html2svd/stmicro/__main__.py b/src/modm_data/html2svd/stmicro/__main__.py index 9f2270c..af34ba5 100644 --- a/src/modm_data/html2svd/stmicro/__main__.py +++ b/src/modm_data/html2svd/stmicro/__main__.py @@ -1,7 +1,6 @@ # Copyright 2022, Niklas Hauser # SPDX-License-Identifier: MPL-2.0 -import re import tqdm import argparse import subprocess @@ -31,20 +30,22 @@ def main(): docs.append(doc) Path("log/stmicro/svd").mkdir(exist_ok=True, parents=True) - calls = [f"python3 -m modm_data.html2svd.stmicro --stm32 {doc.path} " - f"> log/stmicro/svd/html_{doc.name}.txt 2>&1" for doc in docs] + calls = [ + f"python3 -m modm_data.html2svd.stmicro --stm32 {doc.path} " f"> log/stmicro/svd/html_{doc.name}.txt 2>&1" + for doc in docs + ] with ThreadPool() as pool: retvals = list(tqdm.tqdm(pool.imap(lambda c: subprocess.run(c, shell=True), calls), total=len(calls))) for retval, call in zip(retvals, calls): - if retval.returncode != 0: print(call) + if retval.returncode != 0: + print(call) return all(r.returncode == 0 for r in retvals) if args.stm32: doc = ReferenceManual(args.stm32.absolute()) elif args.sensor: doc = DatasheetSensor(args.sensor.absolute()) - print(doc.path_pdf.relative_to(Path().cwd()), - doc.path.relative_to(Path().cwd())) + print(doc.path_pdf.relative_to(Path().cwd()), doc.path.relative_to(Path().cwd())) if args.stm32: mmaptrees = memory_map_from_reference_manual(doc) diff --git a/src/modm_data/html2svd/stmicro/datasheet.py b/src/modm_data/html2svd/stmicro/datasheet.py index 35e93f4..4a79047 100644 --- a/src/modm_data/html2svd/stmicro/datasheet.py +++ b/src/modm_data/html2svd/stmicro/datasheet.py @@ -2,15 +2,14 @@ # SPDX-License-Identifier: MPL-2.0 import re -from functools import cached_property from collections import defaultdict from anytree import RenderTree from ...html.stmicro.helper import split_device_filter -from ...svd import * +from ...svd import Device, Peripheral, PeripheralType, Register, BitField from ...header2svd.stmicro.tree import _normalize_order -from ...cubemx import cubemx_device_list from ...html import replace as html_replace +from ...cubemx import cubemx_device_list def _deduplicate_bit_fields(bit_fields): @@ -33,11 +32,11 @@ def _peripheral_map_to_tree(chapter, peripheral_maps): peripheral_trees = [] for caption, (heading, register_map) in peripheral_maps.items(): print(caption) - if match := re.search(f"OTG_[FH]S", caption): + if match := re.search("OTG_[FH]S", caption): replace_name = peripheral_name = "OTG" - elif match := re.search(f"JPEG", caption): + elif match := re.search("JPEG", caption): replace_name = peripheral_name = "JPEG" - elif match := re.search(f"CCU ", caption): + elif match := re.search("CCU ", caption): peripheral_name = "CANCCU" replace_name = "FDCAN_CCU" else: @@ -52,7 +51,8 @@ def _peripheral_map_to_tree(chapter, peripheral_maps): elif len(peripheral_names) > 1: print(f"Multiple peripheral names detected: {peripheral_names}") - if peripheral_name == "M7": continue + if peripheral_name == "M7": + continue # Some chapters have multiple tables for multiple instances filters = defaultdict(set) instances = set() @@ -90,8 +90,7 @@ def _peripheral_map_to_tree(chapter, peripheral_maps): elif "low medium high and xl density" in chapter.name: filters["devices"].add("STM32F10[123]") - peripheral_type = PeripheralType(peripheral_name, _chapter=chapter, - filters=dict(filters), section=heading) + peripheral_type = PeripheralType(peripheral_name, _chapter=chapter, filters=dict(filters), section=heading) for rname, (offset, bitfields) in register_map.items(): filters = {} if replace_name: @@ -102,7 +101,7 @@ def _peripheral_map_to_tree(chapter, peripheral_maps): nrname = rname.replace(replace_name + "_", "") if len(rname) == len(nrname) and "_" in rname: instance = rname.split("_")[0] - filters["instances"] = {instance+"$"} + filters["instances"] = {instance + "$"} nrname = rname.replace(instance + "_", "") print(instance, nrname) rname = nrname @@ -112,8 +111,10 @@ def _peripheral_map_to_tree(chapter, peripheral_maps): elif match := re.match("(.*?)low,medium,highandXLdensitydevices", rname): rname = match.group(1) filters["devices"] = {r"STM32F10[123]"} - try: offset = int(offset, 16) - except: pass + try: + offset = int(offset, 16) + except ValueError: + pass register_type = Register(rname, offset, filters=filters, parent=peripheral_type) fields = [BitField(field, bit) for bit, field in bitfields.items()] register_type.children = _deduplicate_bit_fields(fields) @@ -127,16 +128,20 @@ def _expand_register_offsets(peripheral_trees): for peripheral in peripheral_trees: unexpanded = defaultdict(list) for register in peripheral.children: - if (isinstance(register.offset, str) or - ("CAN" in peripheral.name and "F1R2" in register.name) or - ("GFXMMU" in peripheral.name and "LUT0L" in register.name) or - ("GFXMMU" in peripheral.name and "LUT0H" in register.name) or - ("HSEM" in peripheral.name and "R1" in register.name)): + if ( + isinstance(register.offset, str) + or ("CAN" in peripheral.name and "F1R2" in register.name) + or ("GFXMMU" in peripheral.name and "LUT0L" in register.name) + or ("GFXMMU" in peripheral.name and "LUT0H" in register.name) + or ("HSEM" in peripheral.name and "R1" in register.name) + ): unexpanded[str(register.offset)].append(register) for offsets, registers in unexpanded.items(): print(offsets, registers) - conv = lambda i: int(i, 16) + def conv(i): + return int(i, 16) + # if match := re.search(r"x=([\d,]+)", registers[0].name): # offsets = [offsets] * len(match.group(1).split(",")) if any(pat in offsets for pat in ["x=", "channelnumber"]): @@ -157,40 +162,56 @@ def _expand_register_offsets(peripheral_trees): print(formula, offsets, orange) elif "-" in offsets: omin, omax = list(map(conv, offsets.split("-"))) - offsets = enumerate(range(omin, omax+1, 4)) + offsets = enumerate(range(omin, omax + 1, 4)) elif "or" in offsets: offsets = enumerate(list(map(conv, offsets.split("or")))) elif "F1R2" in registers[0].name: - offsets = enumerate(range(int(offsets), int(offsets)+4*25*2+1, 4)) + offsets = enumerate(range(int(offsets), int(offsets) + 4 * 25 * 2 + 1, 4)) elif "LUT0" in registers[0].name: - offsets = enumerate(range(int(offsets), int(offsets)+4*2044+1, 8)) + offsets = enumerate(range(int(offsets), int(offsets) + 4 * 2044 + 1, 8)) elif "HSEM" in peripheral.name: print(offsets) - offsets = enumerate(range(int(offsets), int(offsets)+4*29+1, 4)) + offsets = enumerate(range(int(offsets), int(offsets) + 4 * 29 + 1, 4)) else: print(f"Unknown expansion format for {offsets}!") return False fields = registers[0].children if all(re.match(r"BKP\d+R", r.name) for r in registers): - name_template = lambda i: f"BKP{i}R" + + def name_template(i): + return f"BKP{i}R" elif "SAI" in peripheral.name: - name_template = lambda i: f"{registers[0].name[1:]}{chr(i+ord('A'))}" + + def name_template(i): + return f"{registers[0].name[1:]}{chr(i + ord('A'))}" elif "HRTIM" in peripheral.name: - name_template = lambda i: registers[0].name.replace("x", chr(i+ord('A'))) + + def name_template(i): + return registers[0].name.replace("x", chr(i + ord("A"))) elif "CAN" in peripheral.name: - name_template = lambda i: f"F{(i+3)//2}R{(i+1)%2+1}" + + def name_template(i): + return f"F{(i + 3) // 2}R{(i + 1) % 2 + 1}" elif "GFXMMU" in peripheral.name: - name_template = lambda i: f"LUT{i}{registers[0].name[-1]}" + + def name_template(i): + return f"LUT{i}{registers[0].name[-1]}" elif "HSEM" in peripheral.name: - name_template = lambda i: f"{registers[0].name[:-1]}{i+1}" + + def name_template(i): + return f"{registers[0].name[:-1]}{i + 1}" elif len(registers) == 1: # if "x=" in registers[0].name: # name_template = lambda i: f"{registers[0].name.split('x=')[0]}.{i}" if "x" in registers[0].name: - name_template = lambda i: registers[0].name.replace("x", str(i)) + + def name_template(i): + return registers[0].name.replace("x", str(i)) else: - name_template = lambda i: f"{registers[0].name}.{i}" + + def name_template(i): + return f"{registers[0].name}.{i}" else: print(f"Unknown expansion pattern for {registers}!") return False @@ -219,14 +240,13 @@ def _link_instance_to_type(ds, peripheral_types, instance_offsets): if devices: filters["devices"].update(d.replace("x", ".") for d in devices) - for (names, amin, amax, bus, sections) in locations: + for names, amin, amax, bus, sections in locations: for name in names: ptypes = [t for tname, types in peripheral_types.items() for t in types if tname == name] if not ptypes: ptypes = [t for tname, types in peripheral_types.items() for t in types if tname in name] if not ptypes: - ptypes = [t for tname, types in peripheral_types.items() - for t in types if t.section in sections] + ptypes = [t for tname, types in peripheral_types.items() for t in types if t.section in sections] if not ptypes and name.startswith("UART"): ptypes = [t for tname, types in peripheral_types.items() for t in types if tname == "USART"] if not ptypes and "BKP" == name: @@ -237,7 +257,8 @@ def _link_instance_to_type(ds, peripheral_types, instance_offsets): print(f"Available sections are {nsections}.") exit(1) offsets = [v for k, v in instance_offsets.items() if re.search(k, name)] - if offsets: amin += offsets[0] + if offsets: + amin += offsets[0] p = Peripheral(name, ptypes, amin, filters=dict(filters), sections=sections) peripherals.add(p) return peripherals @@ -259,14 +280,21 @@ def _normalize_instances(memtree, peripherals, device): continue ptypes = peripheral.type if len(ptypes) > 1: - ptypes = [ptype for ptype in sorted(peripheral.type, key=lambda p: -len(p.filters)) - if _resolve_filters(ptype.filters, instances=peripheral.name, devices=device.string)] + ptypes = [ + ptype + for ptype in sorted(peripheral.type, key=lambda p: -len(p.filters)) + if _resolve_filters(ptype.filters, instances=peripheral.name, devices=device.string) + ] if len(ptypes) > 1 and any(p.filters for p in ptypes): ptypes = [p for p in ptypes if p.filters] if len(ptypes) > 1: - nptypes = [p for p in ptypes if any(p.section.startswith(per) or per.startswith(p.section) - for per in peripheral.sections)] - if nptypes: ptypes = nptypes + nptypes = [ + p + for p in ptypes + if any(p.section.startswith(per) or per.startswith(p.section) for per in peripheral.sections) + ] + if nptypes: + ptypes = nptypes for pname in ["DMAMUX", "BDMA", "OCTOSPI"]: if len(ptypes) > 1 and pname in peripheral.name: ptypes = [p for p in ptypes if pname in p.name] @@ -276,15 +304,17 @@ def _normalize_instances(memtree, peripherals, device): continue ptype = ptypes[0] - nper = Peripheral(peripheral.name, ptype, peripheral.address, - filters=peripheral.filters, parent=memtree) + nper = Peripheral(peripheral.name, ptype, peripheral.address, filters=peripheral.filters, parent=memtree) rmap = defaultdict(list) for treg in ptype.children: rmap[treg.name].append(treg) for name, tregs in rmap.items(): - regs = [reg for reg in sorted(tregs, key=lambda p: -len(p.filters)) - if _resolve_filters(reg.filters, instances=peripheral.name, devices=device.string)] + regs = [ + reg + for reg in sorted(tregs, key=lambda p: -len(p.filters)) + if _resolve_filters(reg.filters, instances=peripheral.name, devices=device.string) + ] if len(regs) > 1 and any(r.filters for r in regs): regs = [r for r in regs if r.filters] if len(regs) != 1: @@ -293,14 +323,13 @@ def _normalize_instances(memtree, peripherals, device): continue treg = regs[0] if _resolve_filters(treg.filters, devices=device.string, instances=nper.name): - preg = Register(treg.name, offset=treg.offset, width=treg.width, - filters=treg.filters, parent=nper) + preg = Register(treg.name, offset=treg.offset, width=treg.width, filters=treg.filters, parent=nper) for tbit in treg.children: BitField(tbit.name, tbit.position, tbit.width, parent=preg) def _build_device_trees(ds, peripheral_types, instance_offsets): - devices = ds.filter_devices(modm_device_list()) + devices = ds.filter_devices(cubemx_device_list()) memtrees = [] for device in devices: @@ -332,28 +361,24 @@ def memory_map_from_datasheet(ds): register = ds.chapter(r"chapter +\d+ +register +mapping") table = register.tables("register")[0] print(table) - registers = {} for row in table.cell_rows(): cname = row.match_value("name")[0].text() ctype = row.match_value("type")[0].text() caddr = row.match_value(r"address.*?hex")[0].text() cvalue = row.match_value(r"default")[0].text() ccomment = row.match_value(r"comment")[0].text() - if not ctype: continue + if not ctype: + continue cvalue = int(cvalue, 2) if cvalue.isdigit() else None print(cname, ctype, int(caddr, 16), cvalue, ccomment) - - - - exit(1) peripheral_types = defaultdict(set) instance_offsets = {} - for chapter in all_chapters: + for chapter in all_chapters: # noqa: F821 print() - peripheral_maps, peripheral_offsets = ds.peripheral_maps(chapter, assert_table=chapter in type_chapters) + peripheral_maps, peripheral_offsets = ds.peripheral_maps(chapter, assert_table=chapter in type_chapters) # noqa: F821 instance_offsets.update(peripheral_offsets) peripheral_maps = _peripheral_map_to_tree(chapter, peripheral_maps) if not _expand_register_offsets(peripheral_maps): @@ -369,7 +394,6 @@ def memory_map_from_datasheet(ds): print(pmap.section, pmap._chapter._relpath) print(RenderTree(pmap, maxlevel=2)) - memtrees = _build_device_trees(ds, peripheral_types, instance_offsets) # for tree in memtrees: # print(RenderTree(tree, maxlevel=2)) diff --git a/src/modm_data/html2svd/stmicro/reference.py b/src/modm_data/html2svd/stmicro/reference.py index 8455ca3..611a941 100644 --- a/src/modm_data/html2svd/stmicro/reference.py +++ b/src/modm_data/html2svd/stmicro/reference.py @@ -2,15 +2,14 @@ # SPDX-License-Identifier: MPL-2.0 import re -from functools import cached_property from collections import defaultdict from anytree import RenderTree from ...html.stmicro.helper import split_device_filter -from ...svd import * +from ...svd import Device, PeripheralType, Peripheral, Register, BitField from ...header2svd.stmicro.tree import _normalize_order -from ...cubemx import cubemx_device_list from ...html import replace as html_replace +from ...cubemx import cubemx_device_list def _deduplicate_bit_fields(bit_fields): @@ -33,11 +32,11 @@ def _peripheral_map_to_tree(chapter, peripheral_maps): peripheral_trees = [] for caption, (heading, register_map) in peripheral_maps.items(): print(caption) - if match := re.search(f"OTG_[FH]S", caption): + if match := re.search("OTG_[FH]S", caption): replace_name = peripheral_name = "OTG" - elif match := re.search(f"JPEG", caption): + elif match := re.search("JPEG", caption): replace_name = peripheral_name = "JPEG" - elif match := re.search(f"CCU ", caption): + elif match := re.search("CCU ", caption): peripheral_name = "CANCCU" replace_name = "FDCAN_CCU" else: @@ -52,7 +51,8 @@ def _peripheral_map_to_tree(chapter, peripheral_maps): elif len(peripheral_names) > 1: print(f"Multiple peripheral names detected: {peripheral_names}") - if peripheral_name == "M7": continue + if peripheral_name == "M7": + continue # Some chapters have multiple tables for multiple instances filters = defaultdict(set) instances = set() @@ -90,8 +90,7 @@ def _peripheral_map_to_tree(chapter, peripheral_maps): elif "low medium high and xl density" in chapter.name: filters["devices"].add("STM32F10[123]") - peripheral_type = PeripheralType(peripheral_name, _chapter=chapter, - filters=dict(filters), section=heading) + peripheral_type = PeripheralType(peripheral_name, _chapter=chapter, filters=dict(filters), section=heading) for rname, (offset, bitfields) in register_map.items(): filters = {} if replace_name: @@ -102,7 +101,7 @@ def _peripheral_map_to_tree(chapter, peripheral_maps): nrname = rname.replace(replace_name + "_", "") if len(rname) == len(nrname) and "_" in rname: instance = rname.split("_")[0] - filters["instances"] = {instance+"$"} + filters["instances"] = {instance + "$"} nrname = rname.replace(instance + "_", "") print(instance, nrname) rname = nrname @@ -112,8 +111,10 @@ def _peripheral_map_to_tree(chapter, peripheral_maps): elif match := re.match("(.*?)low,medium,highandXLdensitydevices", rname): rname = match.group(1) filters["devices"] = {r"STM32F10[123]"} - try: offset = int(offset, 16) - except: pass + try: + offset = int(offset, 16) + except ValueError: + pass register_type = Register(rname, offset, filters=filters, parent=peripheral_type) fields = [BitField(field, bit) for bit, field in bitfields.items()] register_type.children = _deduplicate_bit_fields(fields) @@ -127,16 +128,20 @@ def _expand_register_offsets(peripheral_trees): for peripheral in peripheral_trees: unexpanded = defaultdict(list) for register in peripheral.children: - if (isinstance(register.offset, str) or - ("CAN" in peripheral.name and "F1R2" in register.name) or - ("GFXMMU" in peripheral.name and "LUT0L" in register.name) or - ("GFXMMU" in peripheral.name and "LUT0H" in register.name) or - ("HSEM" in peripheral.name and "R1" in register.name)): + if ( + isinstance(register.offset, str) + or ("CAN" in peripheral.name and "F1R2" in register.name) + or ("GFXMMU" in peripheral.name and "LUT0L" in register.name) + or ("GFXMMU" in peripheral.name and "LUT0H" in register.name) + or ("HSEM" in peripheral.name and "R1" in register.name) + ): unexpanded[str(register.offset)].append(register) for offsets, registers in unexpanded.items(): print(offsets, registers) - conv = lambda i: int(i, 16) + def conv(i): + return int(i, 16) + # if match := re.search(r"x=([\d,]+)", registers[0].name): # offsets = [offsets] * len(match.group(1).split(",")) if any(pat in offsets for pat in ["x=", "channelnumber"]): @@ -157,40 +162,56 @@ def _expand_register_offsets(peripheral_trees): print(formula, offsets, orange) elif "-" in offsets: omin, omax = list(map(conv, offsets.split("-"))) - offsets = enumerate(range(omin, omax+1, 4)) + offsets = enumerate(range(omin, omax + 1, 4)) elif "or" in offsets: offsets = enumerate(list(map(conv, offsets.split("or")))) elif "F1R2" in registers[0].name: - offsets = enumerate(range(int(offsets), int(offsets)+4*25*2+1, 4)) + offsets = enumerate(range(int(offsets), int(offsets) + 4 * 25 * 2 + 1, 4)) elif "LUT0" in registers[0].name: - offsets = enumerate(range(int(offsets), int(offsets)+4*2044+1, 8)) + offsets = enumerate(range(int(offsets), int(offsets) + 4 * 2044 + 1, 8)) elif "HSEM" in peripheral.name: print(offsets) - offsets = enumerate(range(int(offsets), int(offsets)+4*29+1, 4)) + offsets = enumerate(range(int(offsets), int(offsets) + 4 * 29 + 1, 4)) else: print(f"Unknown expansion format for {offsets}!") return False fields = registers[0].children if all(re.match(r"BKP\d+R", r.name) for r in registers): - name_template = lambda i: f"BKP{i}R" + + def name_template(i): + return f"BKP{i}R" elif "SAI" in peripheral.name: - name_template = lambda i: f"{registers[0].name[1:]}{chr(i+ord('A'))}" + + def name_template(i): + return f"{registers[0].name[1:]}{chr(i + ord('A'))}" elif "HRTIM" in peripheral.name: - name_template = lambda i: registers[0].name.replace("x", chr(i+ord('A'))) + + def name_template(i): + return registers[0].name.replace("x", chr(i + ord("A"))) elif "CAN" in peripheral.name: - name_template = lambda i: f"F{(i+3)//2}R{(i+1)%2+1}" + + def name_template(i): + return f"F{(i + 3) // 2}R{(i + 1) % 2 + 1}" elif "GFXMMU" in peripheral.name: - name_template = lambda i: f"LUT{i}{registers[0].name[-1]}" + + def name_template(i): + return f"LUT{i}{registers[0].name[-1]}" elif "HSEM" in peripheral.name: - name_template = lambda i: f"{registers[0].name[:-1]}{i+1}" + + def name_template(i): + return f"{registers[0].name[:-1]}{i + 1}" elif len(registers) == 1: # if "x=" in registers[0].name: # name_template = lambda i: f"{registers[0].name.split('x=')[0]}.{i}" if "x" in registers[0].name: - name_template = lambda i: registers[0].name.replace("x", str(i)) + + def name_template(i): + return registers[0].name.replace("x", str(i)) else: - name_template = lambda i: f"{registers[0].name}.{i}" + + def name_template(i): + return f"{registers[0].name}.{i}" else: print(f"Unknown expansion pattern for {registers}!") return False @@ -219,14 +240,13 @@ def _link_instance_to_type(rm, peripheral_types, instance_offsets): if devices: filters["devices"].update(d.replace("x", ".") for d in devices) - for (names, amin, amax, bus, sections) in locations: + for names, amin, amax, bus, sections in locations: for name in names: ptypes = [t for tname, types in peripheral_types.items() for t in types if tname == name] if not ptypes: ptypes = [t for tname, types in peripheral_types.items() for t in types if tname in name] if not ptypes: - ptypes = [t for tname, types in peripheral_types.items() - for t in types if t.section in sections] + ptypes = [t for tname, types in peripheral_types.items() for t in types if t.section in sections] if not ptypes and name.startswith("UART"): ptypes = [t for tname, types in peripheral_types.items() for t in types if tname == "USART"] if not ptypes and "BKP" == name: @@ -237,7 +257,8 @@ def _link_instance_to_type(rm, peripheral_types, instance_offsets): print(f"Available sections are {nsections}.") exit(1) offsets = [v for k, v in instance_offsets.items() if re.search(k, name)] - if offsets: amin += offsets[0] + if offsets: + amin += offsets[0] p = Peripheral(name, ptypes, amin, filters=dict(filters), sections=sections) peripherals.add(p) return peripherals @@ -259,14 +280,21 @@ def _normalize_instances(memtree, peripherals, device): continue ptypes = peripheral.type if len(ptypes) > 1: - ptypes = [ptype for ptype in sorted(peripheral.type, key=lambda p: -len(p.filters)) - if _resolve_filters(ptype.filters, instances=peripheral.name, devices=device.string)] + ptypes = [ + ptype + for ptype in sorted(peripheral.type, key=lambda p: -len(p.filters)) + if _resolve_filters(ptype.filters, instances=peripheral.name, devices=device.string) + ] if len(ptypes) > 1 and any(p.filters for p in ptypes): ptypes = [p for p in ptypes if p.filters] if len(ptypes) > 1: - nptypes = [p for p in ptypes if any(p.section.startswith(per) or per.startswith(p.section) - for per in peripheral.sections)] - if nptypes: ptypes = nptypes + nptypes = [ + p + for p in ptypes + if any(p.section.startswith(per) or per.startswith(p.section) for per in peripheral.sections) + ] + if nptypes: + ptypes = nptypes for pname in ["DMAMUX", "BDMA", "OCTOSPI"]: if len(ptypes) > 1 and pname in peripheral.name: ptypes = [p for p in ptypes if pname in p.name] @@ -276,15 +304,17 @@ def _normalize_instances(memtree, peripherals, device): continue ptype = ptypes[0] - nper = Peripheral(peripheral.name, ptype, peripheral.address, - filters=peripheral.filters, parent=memtree) + nper = Peripheral(peripheral.name, ptype, peripheral.address, filters=peripheral.filters, parent=memtree) rmap = defaultdict(list) for treg in ptype.children: rmap[treg.name].append(treg) for name, tregs in rmap.items(): - regs = [reg for reg in sorted(tregs, key=lambda p: -len(p.filters)) - if _resolve_filters(reg.filters, instances=peripheral.name, devices=device.string)] + regs = [ + reg + for reg in sorted(tregs, key=lambda p: -len(p.filters)) + if _resolve_filters(reg.filters, instances=peripheral.name, devices=device.string) + ] if len(regs) > 1 and any(r.filters for r in regs): regs = [r for r in regs if r.filters] if len(regs) != 1: @@ -293,14 +323,13 @@ def _normalize_instances(memtree, peripherals, device): continue treg = regs[0] if _resolve_filters(treg.filters, devices=device.string, instances=nper.name): - preg = Register(treg.name, offset=treg.offset, width=treg.width, - filters=treg.filters, parent=nper) + preg = Register(treg.name, offset=treg.offset, width=treg.width, filters=treg.filters, parent=nper) for tbit in treg.children: BitField(tbit.name, tbit.position, tbit.width, parent=preg) def _build_device_trees(rm, peripheral_types, instance_offsets): - devices = rm.filter_devices(modm_device_list()) + devices = rm.filter_devices(cubemx_device_list()) memtrees = [] for device in devices: @@ -334,8 +363,9 @@ def memory_map_from_reference_manual(rm): return [] all_chapters = rm.chapters() - type_chapters = {rm.chapter(f"chapter {s.split('.')[0]} ") for pers in rm.peripherals.values() - for locs in pers for s in locs[4]} + type_chapters = { + rm.chapter(f"chapter {s.split('.')[0]} ") for pers in rm.peripherals.values() for locs in pers for s in locs[4] + } peripheral_types = defaultdict(set) instance_offsets = {} for chapter in all_chapters: @@ -356,7 +386,6 @@ def memory_map_from_reference_manual(rm): print(pmap.section, pmap._chapter._relpath) print(RenderTree(pmap, maxlevel=2)) - memtrees = _build_device_trees(rm, peripheral_types, instance_offsets) # for tree in memtrees: # print(RenderTree(tree, maxlevel=2)) diff --git a/src/modm_data/owl/__init__.py b/src/modm_data/owl/__init__.py index 02c7fe3..21d0205 100644 --- a/src/modm_data/owl/__init__.py +++ b/src/modm_data/owl/__init__.py @@ -4,3 +4,9 @@ from .store import Store from .identifier import DeviceIdentifier from . import stmicro + +__all__ = [ + "stmicro", + "Store", + "DeviceIdentifier", +] diff --git a/src/modm_data/owl/identifier.py b/src/modm_data/owl/identifier.py index 63a0401..b9017cd 100644 --- a/src/modm_data/owl/identifier.py +++ b/src/modm_data/owl/identifier.py @@ -18,7 +18,8 @@ def __init__(self, naming_schema=None): def _ustring(self): if self.__ustring is None: self.__ustring = "".join([k + self._properties[k] for k in sorted(self._properties.keys())]) - if self.naming_schema: self.__ustring += self.naming_schema; + if self.naming_schema: + self.__ustring += self.naming_schema return self.__ustring def copy(self): @@ -39,8 +40,7 @@ def string(self): raise ValueError("Naming schema is missing!") # Use the naming schema to generate the string if self.__string is None: - self.__string = string.Formatter().vformat( - self.naming_schema, (), defaultdict(str, **self._properties)) + self.__string = string.Formatter().vformat(self.naming_schema, (), defaultdict(str, **self._properties)) return self.__string def set(self, key, value): @@ -58,7 +58,7 @@ def __getitem__(self, key): def __getattr__(self, attr): val = self.get(attr, None) if val is None: - raise AttributeError("'{}' has no property '{}'".format(repr(self), attr)) + raise AttributeError(f"'{self!r}' has no property '{attr}'") return val def __eq__(self, other): @@ -76,4 +76,4 @@ def __str__(self): return self.string def __repr__(self): - return self.string if self.naming_schema else "DeviceId({})".format(self._ustring) + return self.string if self.naming_schema else f"DeviceId({self._ustring})" diff --git a/src/modm_data/owl/stmicro/__init__.py b/src/modm_data/owl/stmicro/__init__.py index abc8d9c..8ee4f5c 100644 --- a/src/modm_data/owl/stmicro/__init__.py +++ b/src/modm_data/owl/stmicro/__init__.py @@ -4,4 +4,26 @@ from .device import owls, owl_devices, owl_device, load_owl_device from .identifier import did_from_string from .ontology import create_ontology -from .model import * +from .model import ( + owl_from_datasheet, + owl_from_reference_manual, + owl_from_doc, + owl_from_did, + owl_from_cubemx, + owl_from_header, +) + +__all__ = [ + "owls", + "owl_devices", + "owl_device", + "load_owl_device", + "did_from_string", + "create_ontology", + "owl_from_datasheet", + "owl_from_reference_manual", + "owl_from_doc", + "owl_from_did", + "owl_from_cubemx", + "owl_from_header", +] diff --git a/src/modm_data/owl/stmicro/device.py b/src/modm_data/owl/stmicro/device.py index 44dcc70..6c8f715 100644 --- a/src/modm_data/owl/stmicro/device.py +++ b/src/modm_data/owl/stmicro/device.py @@ -1,4 +1,3 @@ - # Copyright 2022, Niklas Hauser # SPDX-License-Identifier: MPL-2.0 @@ -32,10 +31,10 @@ def owl_devices(): _OWL_MAPPING[name] = ds _OWL_MAPPING_FILE.parent.mkdir(parents=True, exist_ok=True) - with _OWL_MAPPING_FILE.open('w', encoding='utf-8') as fh: + with _OWL_MAPPING_FILE.open("w", encoding="utf-8") as fh: json.dump(_OWL_MAPPING, fh, indent=4) else: - with _OWL_MAPPING_FILE.open('r', encoding='utf-8') as fh: + with _OWL_MAPPING_FILE.open("r", encoding="utf-8") as fh: _OWL_MAPPING = json.load(fh) return _OWL_MAPPING diff --git a/src/modm_data/owl/stmicro/model.py b/src/modm_data/owl/stmicro/model.py index d3c8fae..7c54c7b 100644 --- a/src/modm_data/owl/stmicro/model.py +++ b/src/modm_data/owl/stmicro/model.py @@ -53,7 +53,7 @@ def owl_from_cubemx(onto, data): # Add internal memories for memory in data["memories"]: - omem = onto.Memory("Memory_" + memory['name'].upper()) + omem = onto.Memory("Memory_" + memory["name"].upper()) odid.hasMemory.append(omem) omem.hasName = memory["name"].upper() omem.hasMemoryStartAddress = int(memory["start"], 16) @@ -61,7 +61,7 @@ def owl_from_cubemx(onto, data): omem.hasMemoryAccess = memory["access"] # Add the peripherals and their type - for (pbase, name, version, ptype, features, stype) in data["modules"]: + for pbase, name, version, ptype, features, stype in data["modules"]: oper = onto.Peripheral("Peripheral_" + name.upper()) odid.hasPeripheral.append(oper) oper.hasName = name.upper() @@ -89,7 +89,7 @@ def owl_from_cubemx(onto, data): onto.pinPosition[opack, onto.hasPin, opin].append(pin["position"]) # Add alternate and additional functions to pins - for (port, number, signals) in data["gpios"]: + for port, number, signals in data["gpios"]: opin = io_pins[(port, int(number))] for signal in signals: peripheral = (signal["driver"] or "").upper() + (signal["instance"] or "") @@ -100,7 +100,8 @@ def owl_from_cubemx(onto, data): osig.hasPeripheral.append(onto.Peripheral("Peripheral_" + peripheral)) osig.hasName = name opin.hasSignal.append(osig) - if af: onto.alternateFunction[opin, onto.hasSignal, osig].append(int(af)) + if af: + onto.alternateFunction[opin, onto.hasSignal, osig].append(int(af)) def owl_from_header(onto, header): diff --git a/src/modm_data/owl/stmicro/ontology.py b/src/modm_data/owl/stmicro/ontology.py index 620fc76..b6c97a3 100644 --- a/src/modm_data/owl/stmicro/ontology.py +++ b/src/modm_data/owl/stmicro/ontology.py @@ -1,10 +1,10 @@ # Copyright 2022, Niklas Hauser # SPDX-License-Identifier: MPL-2.0 -import re import owlready2 as owl from .. import Store + class KeyDict: def __init__(self, values): self.values = values @@ -12,7 +12,7 @@ def __init__(self, values): def __getattr__(self, attr): val = self.values.get(attr, None) if val is None: - raise AttributeError("'{}' has no property '{}'".format(repr(self), attr)) + raise AttributeError(f"'{self!r}' has no property '{attr}'") return val diff --git a/src/modm_data/owl/store.py b/src/modm_data/owl/store.py index 727683e..a1990e8 100644 --- a/src/modm_data/owl/store.py +++ b/src/modm_data/owl/store.py @@ -3,8 +3,6 @@ from ..utils import ext_path import owlready2 as owl -import lxml.etree as ET -from pathlib import Path XSLT_SORT = r""" @@ -54,13 +52,15 @@ def namespace(self, name): return self.ontology.get_namespace(f"{self.vendor}/{name}") def load(self, name=None): - if name is None: name = "ontology" + if name is None: + name = "ontology" fileobj = open(self._path / f"{name}.owl", "rb") self.ontology.load(only_local=True, fileobj=fileobj, reload=True) def save(self, name=None): self._path.mkdir(exist_ok=True, parents=True) - if name is None: name = "ontology" + if name is None: + name = "ontology" file = str(self._path / f"{name}.owl") self.ontology.save(file=file) @@ -81,4 +81,3 @@ def clear(self): def __repr__(self) -> str: return f"Store({self.vendor}/{self.device})" - diff --git a/src/modm_data/pdf/__init__.py b/src/modm_data/pdf/__init__.py index aa8d6b4..ac442fd 100644 --- a/src/modm_data/pdf/__init__.py +++ b/src/modm_data/pdf/__init__.py @@ -19,3 +19,16 @@ from .path import Path from .image import Image from .render import render_page_pdf +from .structure import Structure + +__all__ = [ + "Document", + "Page", + "Character", + "ObjLink", + "WebLink", + "Path", + "Image", + "Structure", + "render_page_pdf", +] diff --git a/src/modm_data/pdf/character.py b/src/modm_data/pdf/character.py index 27287c1..2140a0e 100644 --- a/src/modm_data/pdf/character.py +++ b/src/modm_data/pdf/character.py @@ -29,8 +29,10 @@ class Character: This class contains all information about a single character in the PDF page. """ + class RenderMode(Enum): """Tells the PDF viewer how to render this character glyph.""" + UNKNOWN = -1 FILL = 0 STROKE = 1 @@ -41,7 +43,7 @@ class RenderMode(Enum): FILL_STROKE_CLIP = 6 CLIP = 7 - def __init__(self, page: "modm_data.pdf.page.Page", index: int): + def __init__(self, page: "modm_data.pdf.page.Page", index: int): # noqa: F821 """ :param page: The page containing the character. :param index: The index of the character. @@ -54,15 +56,14 @@ def __init__(self, page: "modm_data.pdf.page.Page", index: int): self.unicode: int = pp.raw.FPDFText_GetUnicode(self._text, self._index) """The unicode value of the character.""" - self.objlink: "modm_data.pdf.link.ObjLink" = None + self.objlink: "modm_data.pdf.link.ObjLink" = None # noqa: F821 """The object link of this character or `None`""" - self.weblink: "modm_data.pdf.link.WebLink" = None + self.weblink: "modm_data.pdf.link.WebLink" = None # noqa: F821 """The web link of this character or `None`""" bbox = Rectangle(*self._text.get_charbox(self._index, loose=True)) if self._page.rotation: - bbox = Rectangle(bbox.p0.y, self._page.height - bbox.p1.x, - bbox.p1.y, self._page.height - bbox.p0.x) + bbox = Rectangle(bbox.p0.y, self._page.height - bbox.p1.x, bbox.p1.y, self._page.height - bbox.p0.x) self._bbox = bbox def _font_flags(self) -> tuple[str, int]: @@ -107,8 +108,7 @@ def tbbox(self) -> Rectangle: """The tight bounding box of the character.""" tbbox = Rectangle(*self._text.get_charbox(self._index)) if self._page.rotation: - tbbox = Rectangle(tbbox.p0.y, self._page.height - tbbox.p1.x, - tbbox.p1.y, self._page.height - tbbox.p0.x) + tbbox = Rectangle(tbbox.p0.y, self._page.height - tbbox.p1.x, tbbox.p1.y, self._page.height - tbbox.p0.x) return tbbox @property @@ -142,7 +142,7 @@ def render_mode(self) -> RenderMode: def rotation(self) -> int: """The rotation of the character in degrees modulo 360.""" # Special case for vertical text in rotated pages - if self._page.rotation == 90 and self._rotation == 0 and self.unicode not in {0x20, 0xa, 0xd}: + if self._page.rotation == 90 and self._rotation == 0 and self.unicode not in {0x20, 0xA, 0xD}: return 90 if self._page.rotation and self._rotation: return (self._page.rotation + self._rotation) % 360 @@ -187,15 +187,17 @@ def descr(self) -> str: char = chr(self.unicode) if not char.isprintable(): char = hex(self.unicode) - return f"Chr({char}, {self.size}, {self.weight}, {self.rotation}, " \ - f"{self.render_mode}, {self.font}, {hex(self.flags)}, " \ - f"{self.fill}, {self.stroke}, {repr(self.bbox)})" + return ( + f"Chr({char}, {self.size}, {self.weight}, {self.rotation}, " + f"{self.render_mode}, {self.font}, {hex(self.flags)}, " + f"{self.fill}, {self.stroke}, {repr(self.bbox)})" + ) def __str__(self) -> str: return self.char def __repr__(self) -> str: char = chr(self.unicode) - escape = {0xa: "\\n", 0xd: "\\r", 0x9: "\\t", 0x20: "␣"} + escape = {0xA: "\\n", 0xD: "\\r", 0x9: "\\t", 0x20: "␣"} char = escape.get(self.unicode, char if char.isprintable() else hex(self.unicode)) return char diff --git a/src/modm_data/pdf/document.py b/src/modm_data/pdf/document.py index 58917af..3e6c7f7 100644 --- a/src/modm_data/pdf/document.py +++ b/src/modm_data/pdf/document.py @@ -11,7 +11,6 @@ correct page class from `page()` function. """ - import ctypes import logging import pypdfium2 as pp @@ -30,7 +29,8 @@ def __hash__(self) -> int: return hash(f"{self.page_index}+{self.title}") def __eq__(self, other) -> bool: - if not isinstance(other, type(self)): return NotImplemented + if not isinstance(other, type(self)): + return NotImplemented return self.page_index == other.page_index and self.title == other.title def __repr__(self) -> str: @@ -42,6 +42,7 @@ class Document(pp.PdfDocument): This class is a convenience wrapper with caching around the high-level APIs of pypdfium. """ + def __init__(self, path: Path, autoclose: bool = False): """ :param path: Path to the PDF to open. @@ -65,9 +66,9 @@ def destinations(self) -> Iterator[tuple[int, str]]: for ii in range(pp.raw.FPDF_CountNamedDests(self)): length = pp.raw.FPDF_GetNamedDest(self, ii, 0, 0) clength = ctypes.c_long(length) - cbuffer = ctypes.create_string_buffer(length*2) + cbuffer = ctypes.create_string_buffer(length * 2) dest = pp.raw.FPDF_GetNamedDest(self, ii, cbuffer, clength) - name = cbuffer.raw[:clength.value*2].decode("utf-16-le").rstrip("\x00") + name = cbuffer.raw[: clength.value * 2].decode("utf-16-le").rstrip("\x00") page = pp.raw.FPDFDest_GetDestPageIndex(self, dest) yield (page, name) @@ -81,9 +82,15 @@ def toc(self) -> list[pp.PdfOutlineItem]: # Sometimes the TOC contains duplicates so we must use a set last_page_index = 0 for toc in self.get_toc(): - outline = _OutlineItem(toc.level, toc.title, toc.is_closed, - toc.n_kids, toc.page_index or last_page_index, - toc.view_mode, toc.view_pos) + outline = _OutlineItem( + toc.level, + toc.title, + toc.is_closed, + toc.n_kids, + toc.page_index or last_page_index, + toc.view_mode, + toc.view_pos, + ) last_page_index = toc.page_index or last_page_index tocs.add(outline) return list(sorted(list(tocs), key=lambda o: (o.page_index, o.level, o.title))) diff --git a/src/modm_data/pdf/image.py b/src/modm_data/pdf/image.py index 24a4041..4c88b81 100644 --- a/src/modm_data/pdf/image.py +++ b/src/modm_data/pdf/image.py @@ -22,6 +22,7 @@ class Image(pp.PdfImage): .. note:: Images are currently ignored. """ + # Overwrite the PdfPageObject.__new__ function def __new__(cls, *args, **kwargs): return object.__new__(cls) @@ -57,8 +58,7 @@ def bbox(self) -> Rectangle: """The bounding box of the image.""" bbox = Rectangle(*self.get_pos()) if self.page.rotation: - bbox = Rectangle(bbox.p0.y, self.page.height - bbox.p1.x, - bbox.p1.y, self.page.height - bbox.p0.x) + bbox = Rectangle(bbox.p0.y, self.page.height - bbox.p1.x, bbox.p1.y, self.page.height - bbox.p0.x) return bbox @cached_property @@ -79,8 +79,12 @@ def lines(self) -> list[Line]: (For compatibility with `Path.lines`.) """ p = self.points - return [Line(p[0], p[1], p[1].type, 0), Line(p[1], p[2], p[2].type, 0), - Line(p[2], p[3], p[3].type, 0), Line(p[3], p[0], p[0].type, 0)] + return [ + Line(p[0], p[1], p[1].type, 0), + Line(p[1], p[2], p[2].type, 0), + Line(p[2], p[3], p[3].type, 0), + Line(p[3], p[0], p[0].type, 0), + ] def __repr__(self) -> str: return f"I{self.bbox}" diff --git a/src/modm_data/pdf/link.py b/src/modm_data/pdf/link.py index 1e4a8ac..f4d43df 100644 --- a/src/modm_data/pdf/link.py +++ b/src/modm_data/pdf/link.py @@ -12,7 +12,6 @@ and `modm_data.pdf.page.Page.weblinks` properties. """ -import copy import ctypes from functools import cached_property import pypdfium2 as pp @@ -21,7 +20,8 @@ class ObjLink: """A link to a PDF object giving the bounding box and destination page.""" - def __init__(self, page: "modm_data.pdf.Page", link: pp.raw.FPDF_LINK): + + def __init__(self, page: "modm_data.pdf.Page", link: pp.raw.FPDF_LINK): # noqa: F821 """ :param page: Page containing the link, used to compute bounding box. :param link: Raw link object. @@ -33,8 +33,7 @@ def __init__(self, page: "modm_data.pdf.Page", link: pp.raw.FPDF_LINK): assert pp.raw.FPDFLink_GetAnnotRect(link, bbox) bbox = Rectangle(bbox) if page.rotation: - bbox = Rectangle(bbox.p0.y, page.height - bbox.p1.x, - bbox.p1.y, page.height - bbox.p0.x) + bbox = Rectangle(bbox.p0.y, page.height - bbox.p1.x, bbox.p1.y, page.height - bbox.p0.x) self.bbox: Rectangle = bbox """Bounding box of the link source""" @@ -49,7 +48,8 @@ def __repr__(self) -> str: class WebLink: """A weblink object giving the bounding box and destination URL.""" - def __init__(self, page: "modm_data.pdf.Page", index: int): + + def __init__(self, page: "modm_data.pdf.Page", index: int): # noqa: F821 """ :param page: Page containing the link, used to compute bounding box. :param index: 0-index of the weblink object. @@ -73,9 +73,10 @@ def bboxes(self) -> list[Rectangle]: assert pp.raw.FPDFLink_GetRect(self._link, self._index, ii, x0, y1, x1, y0) bboxes.append(Rectangle(x0.value, y0.value, x1.value, y1.value)) if self._page.rotation: - bboxes = [Rectangle(bbox.p0.y, self._page.height - bbox.p1.x, - bbox.p1.y, self._page.height - bbox.p0.x) - for bbox in bboxes] + bboxes = [ + Rectangle(bbox.p0.y, self._page.height - bbox.p1.x, bbox.p1.y, self._page.height - bbox.p0.x) + for bbox in bboxes + ] return bboxes @cached_property diff --git a/src/modm_data/pdf/page.py b/src/modm_data/pdf/page.py index 2beb50c..a363bce 100644 --- a/src/modm_data/pdf/page.py +++ b/src/modm_data/pdf/page.py @@ -32,7 +32,8 @@ class Page(pp.PdfPage): It also fixes missing bounding boxes for rotates characters on page load, as well as allow searching for characters in an area instead of just text. """ - def __init__(self, document: "modm_data.pdf.Document", index: int): + + def __init__(self, document: "modm_data.pdf.Document", index: int): # noqa: F821 """ :param document: a PDF document. :param index: 0-index page number. @@ -162,8 +163,7 @@ def find(self, string: str, case_sensitive: bool = True) -> Iterator[Character]: :param case_sensitive: Ignore case if false. :return: yields the characters found. """ - searcher = self._text.search(string, match_case=case_sensitive, - match_whole_word=True, consecutive=True) + searcher = self._text.search(string, match_case=case_sensitive, match_whole_word=True, consecutive=True) while idx := searcher.get_next(): chars = [self.char(ii) for ii in range(idx[0], idx[0] + idx[1])] yield chars @@ -178,8 +178,9 @@ def images(self) -> list[Image]: """All images.""" return [Image(o) for o in self.get_objects([pp.raw.FPDF_PAGEOBJ_IMAGE])] - def graphic_clusters(self, predicate: Callable[[Path | Image], bool] = None, - absolute_tolerance: float = None) -> list[tuple[Rectangle, list[Path]]]: + def graphic_clusters( + self, predicate: Callable[[Path | Image], bool] = None, absolute_tolerance: float = None + ) -> list[tuple[Rectangle, list[Path]]]: if absolute_tolerance is None: absolute_tolerance = min(self.width, self.height) * 0.01 @@ -193,7 +194,7 @@ def graphic_clusters(self, predicate: Callable[[Path | Image], bool] = None, filtered_paths.append(image) regions = [] - for path in sorted(filtered_paths, key=lambda l: l.bbox.y): + for path in sorted(filtered_paths, key=lambda path: path.bbox.y): for reg in regions: if reg.overlaps(path.bbox.bottom, path.bbox.top, absolute_tolerance): # They overlap, so merge them @@ -206,7 +207,7 @@ def graphic_clusters(self, predicate: Callable[[Path | Image], bool] = None, # Now collect horizontal region inside each vertical region for yreg in regions: - for path in sorted(filtered_paths, key=lambda l: l.bbox.x): + for path in sorted(filtered_paths, key=lambda path: path.bbox.x): # check if horizontal line is contained in vregion if yreg.contains(path.bbox.y, absolute_tolerance): for xreg in yreg.subregions: @@ -235,7 +236,6 @@ def graphic_clusters(self, predicate: Callable[[Path | Image], bool] = None, return sorted(clusters, key=lambda c: (-c[0].y, c[0].x)) - def _link_characters(self): if self._linked: return @@ -267,15 +267,15 @@ def _key(char): height = round(char.tbbox.height, 1) width = round(char.tbbox.width, 1) return f"{char.font} {char.unicode} {height} {width}" + fix_chars = [] for char in self.chars: if not char._bbox.width or not char._bbox.height: if char._rotation: fix_chars.append(char) - elif char.unicode not in {0xa, 0xd}: + elif char.unicode not in {0xA, 0xD}: fix_chars.append(char) - elif (char.unicode not in {0xa, 0xd} and not char._rotation and - _key(char) not in self.pdf._bbox_cache): + elif char.unicode not in {0xA, 0xD} and not char._rotation and _key(char) not in self.pdf._bbox_cache: bbox = char._bbox.translated(-char.origin).rotated(self.rotation + char._rotation) self.pdf._bbox_cache[_key(char)] = (char, bbox) # print("->", _key(char), char.descr(), char.height, char.rotation, char._rotation, self.rotation) @@ -286,5 +286,5 @@ def _key(char): _, bbox = bbox bbox = bbox.rotated(-self.rotation - char._rotation).translated(char.origin) char._bbox = bbox - elif char.unicode not in {0x20, 0xa, 0xd}: + elif char.unicode not in {0x20, 0xA, 0xD}: _LOGGER.debug(f"Unable to fix bbox for {char.descr()}!") diff --git a/src/modm_data/pdf/path.py b/src/modm_data/pdf/path.py index bf59f28..1dee9a9 100644 --- a/src/modm_data/pdf/path.py +++ b/src/modm_data/pdf/path.py @@ -23,20 +23,24 @@ class Path(pp.PdfObject): You must construct the paths by calling `modm_data.pdf.page.Page.paths`. """ + class Type(Enum): """Path Type""" + LINE = 0 BEZIER = 1 MOVE = 2 class Cap(Enum): """Path Cap Type""" + BUTT = 0 ROUND = 1 PROJECTING_SQUARE = 2 class Join(Enum): """Path Join Type""" + MITER = 0 ROUND = 1 BEVEL = 2 @@ -102,13 +106,12 @@ def bbox(self) -> Rectangle: The bounding is only approximated using the control points! Therefore bezier curves will likely have a larger bounding box. """ - l, b = ctypes.c_float(), ctypes.c_float() - r, t = ctypes.c_float(), ctypes.c_float() - assert pp.raw.FPDFPageObj_GetBounds(self, l, b, r, t) - bbox = Rectangle(l.value, b.value, r.value, t.value) + left, bottom = ctypes.c_float(), ctypes.c_float() + right, top = ctypes.c_float(), ctypes.c_float() + assert pp.raw.FPDFPageObj_GetBounds(self, left, bottom, right, top) + bbox = Rectangle(left.value, bottom.value, right.value, top.value) if self.page.rotation: - bbox = Rectangle(bbox.p0.y, self.page.height - bbox.p1.x, - bbox.p1.y, self.page.height - bbox.p0.x) + bbox = Rectangle(bbox.p0.y, self.page.height - bbox.p1.x, bbox.p1.y, self.page.height - bbox.p0.x) return bbox @cached_property @@ -140,8 +143,10 @@ def points(self) -> list[Point]: def lines(self) -> list[Line]: """List of lines between the path points.""" points = self.points - return [Line(points[ii], points[ii + 1], width=self.width, - type=points[ii + 1].type) for ii in range(len(points) - 1)] + return [ + Line(points[ii], points[ii + 1], width=self.width, type=points[ii + 1].type) + for ii in range(len(points) - 1) + ] def __repr__(self) -> str: points = ",".join(repr(p) for p in self.points) diff --git a/src/modm_data/pdf/render.py b/src/modm_data/pdf/render.py index 29d5c41..f4cf967 100644 --- a/src/modm_data/pdf/render.py +++ b/src/modm_data/pdf/render.py @@ -1,54 +1,54 @@ # Copyright 2022, Niklas Hauser # SPDX-License-Identifier: MPL-2.0 -import math -import ctypes from ..utils import VLine, HLine import pypdfium2 as pp + def _vline(pageobj, rotation, x, y0, y1, **kw): _line(pageobj, rotation, VLine(x, y0, y1), **kw) + def _hline(pageobj, rotation, y, x0, x1, **kw): _line(pageobj, rotation, HLine(y, x0, x1), **kw) + def _line(pageobj, rotation, line, **kw): if rotation: - obj = pp.raw.FPDFPageObj_CreateNewPath(height - line.p0.y, line.p0.x) - assert pp.raw.FPDFPath_LineTo(obj, height - line.p1.y, line.p1.x) + obj = pp.raw.FPDFPageObj_CreateNewPath(pageobj.height - line.p0.y, line.p0.x) + assert pp.raw.FPDFPath_LineTo(obj, pageobj.height - line.p1.y, line.p1.x) else: obj = pp.raw.FPDFPageObj_CreateNewPath(line.p0.x, line.p0.y) assert pp.raw.FPDFPath_LineTo(obj, line.p1.x, line.p1.y) if fill := kw.get("fill"): - assert pp.raw.FPDFPageObj_SetFillColor(obj, (fill >> 16) & 0xff, (fill >> 8) & 0xff, fill & 0xff, 0xC0) + assert pp.raw.FPDFPageObj_SetFillColor(obj, (fill >> 16) & 0xFF, (fill >> 8) & 0xFF, fill & 0xFF, 0xC0) if stroke := kw.get("stroke"): - assert pp.raw.FPDFPageObj_SetStrokeColor(obj, (stroke >> 16) & 0xff, (stroke >> 8) & 0xff, stroke & 0xff, 0xC0) + assert pp.raw.FPDFPageObj_SetStrokeColor(obj, (stroke >> 16) & 0xFF, (stroke >> 8) & 0xFF, stroke & 0xFF, 0xC0) if width := kw.get("width"): assert pp.raw.FPDFPageObj_SetStrokeWidth(obj, width) - assert pp.raw.FPDFPath_SetDrawMode(obj, 1 if kw.get("fill") else 0, - kw.get("width") is not None) + assert pp.raw.FPDFPath_SetDrawMode(obj, 1 if kw.get("fill") else 0, kw.get("width") is not None) pp.raw.FPDFPage_InsertObject(pageobj, obj) + def _rect(pageobj, rotation, rect, **kw): if rotation: obj = pp.raw.FPDFPageObj_CreateNewRect( - height - rect.bottom - rect.height, rect.left, rect.height, rect.width) + pageobj.height - rect.bottom - rect.height, rect.left, rect.height, rect.width + ) else: obj = pp.raw.FPDFPageObj_CreateNewRect(rect.left, rect.bottom, rect.width, rect.height) if fill := kw.get("fill"): - assert pp.raw.FPDFPageObj_SetFillColor(obj, (fill >> 16) & 0xff, (fill >> 8) & 0xff, fill & 0xff, 0xC0) + assert pp.raw.FPDFPageObj_SetFillColor(obj, (fill >> 16) & 0xFF, (fill >> 8) & 0xFF, fill & 0xFF, 0xC0) if stroke := kw.get("stroke"): - assert pp.raw.FPDFPageObj_SetStrokeColor(obj, (stroke >> 16) & 0xff, (stroke >> 8) & 0xff, stroke & 0xff, 0xC0) + assert pp.raw.FPDFPageObj_SetStrokeColor(obj, (stroke >> 16) & 0xFF, (stroke >> 8) & 0xFF, stroke & 0xFF, 0xC0) if width := kw.get("width"): assert pp.raw.FPDFPageObj_SetStrokeWidth(obj, width) - assert pp.raw.FPDFPath_SetDrawMode(obj, 1 if kw.get("fill") else 0, - kw.get("width") is not None) + assert pp.raw.FPDFPath_SetDrawMode(obj, 1 if kw.get("fill") else 0, kw.get("width") is not None) pp.raw.FPDFPage_InsertObject(pageobj, obj) - -def render_page_pdf(doc, page, new_doc = None, index = 0): - width, height = page.width, page.height +def render_page_pdf(doc, page, new_doc=None, index=0): + _, height = page.width, page.height if new_doc is None: new_doc = pp.raw.FPDF_CreateNewDocument() @@ -59,20 +59,26 @@ def render_page_pdf(doc, page, new_doc = None, index = 0): for path in page.paths: p0 = path.points[0] - if rotation: obj = pp.raw.FPDFPageObj_CreateNewPath(height - p0.y, p0.x) - else: obj = pp.raw.FPDFPageObj_CreateNewPath(p0.x, p0.y) - assert pp.raw.FPDFPageObj_SetStrokeColor(obj, 0,0,0xff, 0xC0) + if rotation: + obj = pp.raw.FPDFPageObj_CreateNewPath(height - p0.y, p0.x) + else: + obj = pp.raw.FPDFPageObj_CreateNewPath(p0.x, p0.y) + assert pp.raw.FPDFPageObj_SetStrokeColor(obj, 0, 0, 0xFF, 0xC0) assert pp.raw.FPDFPageObj_SetStrokeWidth(obj, 0.25) assert pp.raw.FPDFPageObj_SetLineJoin(obj, pp.raw.FPDF_LINEJOIN_ROUND) assert pp.raw.FPDFPageObj_SetLineCap(obj, pp.raw.FPDF_LINECAP_ROUND) assert pp.raw.FPDFPath_SetDrawMode(obj, 0, True) for point in path.points[1:]: if point.type == path.Type.MOVE: - if rotation: assert pp.raw.FPDFPath_MoveTo(obj, height - point.y, point.x) - else: assert pp.raw.FPDFPath_MoveTo(obj, point.x, point.y) + if rotation: + assert pp.raw.FPDFPath_MoveTo(obj, height - point.y, point.x) + else: + assert pp.raw.FPDFPath_MoveTo(obj, point.x, point.y) else: - if rotation: assert pp.raw.FPDFPath_LineTo(obj, height - point.y, point.x) - else: assert pp.raw.FPDFPath_LineTo(obj, point.x, point.y) + if rotation: + assert pp.raw.FPDFPath_LineTo(obj, height - point.y, point.x) + else: + assert pp.raw.FPDFPath_LineTo(obj, point.x, point.y) pp.raw.FPDFPage_InsertObject(new_page, obj) for bbox, _ in page.graphic_clusters(): @@ -83,14 +89,30 @@ def render_page_pdf(doc, page, new_doc = None, index = 0): for link in page.weblinks: for bbox in link.bboxes: - _rect(new_page, rotation, bbox, width=0.75, stroke=0x00ff00) + _rect(new_page, rotation, bbox, width=0.75, stroke=0x00FF00) for char in page.chars: - color = 0x0000ff + color = 0x0000FF if char.bbox.width: - _rect(new_page, rotation, char.bbox, width=0.5, stroke=0xff0000) - _vline(new_page, rotation, char.bbox.midpoint.x, char.bbox.midpoint.y - 1, char.bbox.midpoint.y + 1, width=0.25, stroke=0xff0000) - _hline(new_page, rotation, char.bbox.midpoint.y, char.bbox.midpoint.x - 1, char.bbox.midpoint.x + 1, width=0.25, stroke=0xff0000) + _rect(new_page, rotation, char.bbox, width=0.5, stroke=0xFF0000) + _vline( + new_page, + rotation, + char.bbox.midpoint.x, + char.bbox.midpoint.y - 1, + char.bbox.midpoint.y + 1, + width=0.25, + stroke=0xFF0000, + ) + _hline( + new_page, + rotation, + char.bbox.midpoint.y, + char.bbox.midpoint.x - 1, + char.bbox.midpoint.x + 1, + width=0.25, + stroke=0xFF0000, + ) color = 0x000000 _vline(new_page, rotation, char.origin.x, char.origin.y - 1, char.origin.y + 1, width=0.25, stroke=color) _hline(new_page, rotation, char.origin.y, char.origin.x - 1, char.origin.x + 1, width=0.25, stroke=color) diff --git a/src/modm_data/pdf/structure.py b/src/modm_data/pdf/structure.py index d978a45..00199bd 100644 --- a/src/modm_data/pdf/structure.py +++ b/src/modm_data/pdf/structure.py @@ -28,9 +28,8 @@ class Structure: This class is a convenience wrapper around [the pdfium structtree methods]( https://pdfium.googlesource.com/pdfium/+/main/public/fpdf_structtree.h). """ - def __init__(self, page: "modm_data.pdf.page.Page", - element: pp.raw.FPDF_STRUCTELEMENT, - parent: "Structure" = None): + + def __init__(self, page: "modm_data.pdf.page.Page", element: pp.raw.FPDF_STRUCTELEMENT, parent: "Structure" = None): # noqa: F821 self._page = page self._element = element self.parent: Structure = weakref.ref(parent) if parent else None @@ -88,7 +87,7 @@ def marked_ids(self) -> list[int]: return ids @cached_property - def attributes(self) -> dict[str, str|bool|float]: + def attributes(self) -> dict[str, str | bool | float]: """ All attributes of this structure element as a dictionary. @@ -102,7 +101,7 @@ def attributes(self) -> dict[str, str|bool|float]: for aindex in range(pp.raw.FPDF_StructElement_Attr_GetCount(attr)): # Get the name clength = ctypes.c_ulong(0) - cname = ctypes.create_string_buffer(1) # workaround to get length + cname = ctypes.create_string_buffer(1) # workaround to get length assert pp.raw.FPDF_StructElement_Attr_GetName(attr, aindex, cname, 0, clength) cname = ctypes.create_string_buffer(clength.value) assert pp.raw.FPDF_StructElement_Attr_GetName(attr, aindex, cname, clength, clength) @@ -126,9 +125,9 @@ def attributes(self) -> dict[str, str|bool|float]: case pp.raw.FPDF_OBJECT_STRING | pp.raw.FPDF_OBJECT_NAME: assert pp.raw.FPDF_StructElement_Attr_GetStringValue(attr, cname, 0, 0, clength) - cattrname = ctypes.create_string_buffer(clength.value*2) + cattrname = ctypes.create_string_buffer(clength.value * 2) assert pp.raw.FPDF_StructElement_Attr_GetStringValue(attr, cname, cattrname, clength, clength) - kv[name] = cattrname.raw.decode("utf-16-le", errors="ignore")[:clength.value-1] + kv[name] = cattrname.raw.decode("utf-16-le", errors="ignore")[: clength.value - 1] # FIXME: FPDF_OBJECT_ARRAY is not a blob, but no other APIs are exposed? # case pp.raw.FPDF_OBJECT_ARRAY: @@ -138,7 +137,7 @@ def attributes(self) -> dict[str, str|bool|float]: # kv[name] = cblob.raw case pp.raw.FPDF_OBJECT_ARRAY: - kv[name] = f"[?]" + kv[name] = "[?]" case _: kv[name] = f"[unknown={atype}?]" @@ -169,11 +168,16 @@ def descr(self, indent=0) -> str: def __repr__(self) -> str: values = [] - if self.type: values.append(f"type={self.type}") - if self.title: values.append(f"title={self.title}") - if self.actual_text: values.append(f"act_text={self.actual_text}") - if self.alt_text: values.append(f"alt_text={self.alt_text}") - if self.id: values.append(f"id={self.id}") + if self.type: + values.append(f"type={self.type}") + if self.title: + values.append(f"title={self.title}") + if self.actual_text: + values.append(f"act_text={self.actual_text}") + if self.alt_text: + values.append(f"alt_text={self.alt_text}") + if self.id: + values.append(f"id={self.id}") values += [f"mid={i}" for i in self.marked_ids] values += [f"{k}={v}" for k, v in self.attributes.items()] return f"S({','.join(map(str, values))})" diff --git a/src/modm_data/pdf2html/__init__.py b/src/modm_data/pdf2html/__init__.py index c272980..16fcea6 100644 --- a/src/modm_data/pdf2html/__init__.py +++ b/src/modm_data/pdf2html/__init__.py @@ -9,3 +9,25 @@ from .render import render_page_pdf from .convert import convert, patch from .html import format_document, write_html + +from . import ast +from . import cell +from . import figure +from . import line +from . import page +from . import table + +__all__ = [ + "stmicro", + "render_page_pdf", + "convert", + "patch", + "format_document", + "write_html", + "ast", + "cell", + "figure", + "line", + "page", + "table", +] diff --git a/src/modm_data/pdf2html/ast.py b/src/modm_data/pdf2html/ast.py index ee252c4..8743692 100644 --- a/src/modm_data/pdf2html/ast.py +++ b/src/modm_data/pdf2html/ast.py @@ -6,7 +6,7 @@ from anytree import RenderTree, Node from collections import defaultdict from ..utils import Rectangle, ReversePreOrderIter -from .table import VirtualTable, TableCell +from .table import VirtualTable, Cell _LOGGER = logging.getLogger(__name__) @@ -29,36 +29,42 @@ def merge_area(document: Node, area: Node, debug: bool = False) -> Node: document._end = document if not area.children: return document - if debug: _LOGGER.debug() + if debug: + _LOGGER.debug() def _find_end(node): # Find the last leaf node but skip lines, paragraphs, captions/tables/figures - return next((c for c in ReversePreOrderIter(node) - if any(c.name.startswith(name) for name in {"head", "list", "note"})), - next(ReversePreOrderIter(node), node)) + return next( + (c for c in ReversePreOrderIter(node) if any(c.name.startswith(name) for name in {"head", "list", "note"})), + next(ReversePreOrderIter(node), node), + ) + def _find_ancestor(filter_): - if filter_(document._end): return document._end - return next((c for c in document._end.iter_path_reverse() - if filter_(c)), document.root) + if filter_(document._end): + return document._end + return next((c for c in document._end.iter_path_reverse() if filter_(c)), document.root) area = _normalize_area(area) - if debug: _LOGGER.debug(RenderTree(area)) + if debug: + _LOGGER.debug(RenderTree(area)) children = area.children # All area nodes up to the next top-level element must now be # xpos-aligned with the previous area's last leaf node - connect_index = next((ii for ii, c in enumerate(children) - if c.name.startswith("head")), len(children)) + connect_index = next((ii for ii, c in enumerate(children) if c.name.startswith("head")), len(children)) x_em = area.page._spacing["x_em"] - if debug: _LOGGER.debug("area=", area, "connect_index=", connect_index) + if debug: + _LOGGER.debug("area=", area, "connect_index=", connect_index) # Align these children with the last leaf node xpos for child in children[:connect_index]: if any(child.name.startswith(name) for name in {"list"}): # Find the node that is left of the current node but not too far left - host = _find_ancestor(lambda c: -4 * x_em < (c.xpos - child.xpos) < -x_em or - c.name.startswith("head")) - elif (child.name == "para" and document._end.name == "note" and - child.children[0].obj.contains_font("Italic", "Oblique")): + host = _find_ancestor(lambda c: -4 * x_em < (c.xpos - child.xpos) < -x_em or c.name.startswith("head")) + elif ( + child.name == "para" + and document._end.name == "note" + and child.children[0].obj.contains_font("Italic", "Oblique") + ): host = document._end else: # Insert underneath the next heading @@ -67,7 +73,9 @@ def _find_ancestor(filter_): child.parent = host document._end = _find_end(document) if debug: - _LOGGER.debug(f"{child=}", ) + _LOGGER.debug( + f"{child=}", + ) _LOGGER.debug(f"{host=}") _LOGGER.debug(f"end={document._end}") _LOGGER.debug() @@ -75,7 +83,7 @@ def _find_ancestor(filter_): # Add the remaining top-level children to connect index node if connect_index < len(children): children[connect_index].parent = document - for child in children[connect_index + 1:]: + for child in children[connect_index + 1 :]: child.parent = children[connect_index] document._end = _find_end(document) @@ -109,8 +117,7 @@ def normalize_lists(node: Node) -> Node: for llist in lists: # Insert a new list group node and redirect all children to it if llist[0].name.startswith("list"): - nlist = Node(llist[0].name, obj=llist[0].obj, - start=llist[0].value, xpos=llist[0].xpos) + nlist = Node(llist[0].name, obj=llist[0].obj, start=llist[0].value, xpos=llist[0].xpos) for lnode in llist: lnode.name = "element" lnode.parent = nlist @@ -158,7 +165,7 @@ def normalize_captions(document: Node) -> Node: for caption in captions: cindex = caption.parent.children.index(caption) # Find the next table for this caption within 5 nodes - for sibling in caption.parent.children[cindex:cindex + 6]: + for sibling in caption.parent.children[cindex : cindex + 6]: if sibling.name == caption._type: caption.parent = sibling sibling.number = caption.number @@ -187,15 +194,14 @@ def normalize_headings(document: Node) -> Node: def normalize_registers(document: Node) -> Node: bits_list = [] sections = anytree.search.findall(document, filter_=lambda n: n.name == "section") - for section in (sections + (document,)): + for section in sections + (document,): new_children = [] bits = None for child in section.children: if child.name == "bit": # Insert a new bits group node and redirect all children to it if bits is None or bits._page != child._page: - bits = Node("table", xpos=child.xpos, obj=None, - _type="bits", _width=1, _page=child._page) + bits = Node("table", xpos=child.xpos, obj=None, _type="bits", _width=1, _page=child._page) new_children.append(bits) bits_list.append(bits) child.parent = bits @@ -215,14 +221,16 @@ def normalize_registers(document: Node) -> Node: bottom = next(c.obj.bbox.bottom for c in reversed(bit.descendants) if c.name == "line") # Left table cell contains Bits left_bbox = Rectangle(bit._left, bottom, bit._middle, top) - cells.append(TableCell(None, (ypos, 0), left_bbox, (1,1,1,1), is_simple=True)) + cells.append(Cell(None, (ypos, 0), left_bbox, (1, 1, 1, 1), is_simple=True)) # Right cell contains description right_bbox = Rectangle(bit._middle, bottom, bit._right, top) - cells.append(TableCell(None, (ypos, 1), right_bbox, (1,1,1,1))) - tbbox = Rectangle(min(c.bbox.left for c in cells), - min(c.bbox.bottom for c in cells), - max(c.bbox.right for c in cells), - max(c.bbox.top for c in cells)) + cells.append(Cell(None, (ypos, 1), right_bbox, (1, 1, 1, 1))) + tbbox = Rectangle( + min(c.bbox.left for c in cells), + min(c.bbox.bottom for c in cells), + max(c.bbox.right for c in cells), + max(c.bbox.top for c in cells), + ) bits.obj = VirtualTable(bits._page, tbbox, cells, "bitfield") return document @@ -247,7 +255,7 @@ def _push(): sections = anytree.search.findall(document, filter_=lambda n: n.name == "section") last_number = 0 - for section in (sections + (document,)): + for section in sections + (document,): current_rtables = [] current_bitstables = [] for child in section.children: @@ -317,7 +325,7 @@ def normalize_chapters(document: Node) -> Node: if heading.name == "head1": chapter_name = "0 " + chapter_name filename = chapter_name.lower().translate(cleaner) - chapters.append( (chapter_name, filename, document.children[idx0:idx1 + 1]) ) + chapters.append((chapter_name, filename, document.children[idx0 : idx1 + 1])) for title, filename, nodes in chapters: chapter = Node("chapter", title=title, _filename=filename, parent=document) diff --git a/src/modm_data/pdf2html/cell.py b/src/modm_data/pdf2html/cell.py index 2c051eb..3ec0c5c 100644 --- a/src/modm_data/pdf2html/cell.py +++ b/src/modm_data/pdf2html/cell.py @@ -3,29 +3,32 @@ from functools import cached_property from anytree import Node -from ..utils import Rectangle +from dataclasses import dataclass +from ..utils import Rectangle, Point from .line import CharLine -class TableCell: - class Borders: - """The four borders of a Cell""" - def __init__(self, l, b, r, t): - self.l = l - self.b = b - self.r = r - self.t = t +@dataclass +class Borders: + """The four borders of a `Cell`""" - def __init__(self, table, position, bbox, borders, is_simple=False): + left: bool + bottom: bool + right: bool + top: bool + + +class Cell: + def __init__(self, table, position: Point, bbox: Rectangle, borders: Borders, is_simple: bool = False): self._table = table self._bboxes = [bbox] - self.b = borders + self._is_simple = is_simple + self.borders: Borders = borders """Borders of the cell""" - self.positions = [position] + self.positions: list[Point] = [position] """Index positions of the cell""" - self.is_header = False + self.is_header: bool = False """Is this cell a header?""" - self._is_simple = is_simple def _merge(self, other): self.positions.extend(other.positions) @@ -74,16 +77,19 @@ def yspan(self) -> int: @cached_property def rotation(self) -> int: """The rotation of the cell text.""" - if not self.lines: return 0 + if not self.lines: + return 0 return self.lines[0].rotation @cached_property def bbox(self) -> Rectangle: """The tight bounding box of this cell.""" - return Rectangle(min(bbox.left for bbox in self._bboxes), - min(bbox.bottom for bbox in self._bboxes), - max(bbox.right for bbox in self._bboxes), - max(bbox.top for bbox in self._bboxes)) + return Rectangle( + min(bbox.left for bbox in self._bboxes), + min(bbox.bottom for bbox in self._bboxes), + max(bbox.right for bbox in self._bboxes), + max(bbox.top for bbox in self._bboxes), + ) @cached_property def lines(self) -> list[CharLine]: @@ -107,19 +113,22 @@ def is_left_aligned(self) -> bool: @cached_property def ast(self) -> Node: """The abstract syntax tree of the cell without graphics.""" - ast = self._table._page.ast_in_area(self.bbox, with_graphics=False, - ignore_xpos=not self.is_left_aligned, - with_bits=False, with_notes=False) + ast = self._table._page.ast_in_area( + self.bbox, with_graphics=False, ignore_xpos=not self.is_left_aligned, with_bits=False, with_notes=False + ) ast.name = "cell" return ast def __repr__(self) -> str: positions = ",".join(f"({p[1]},{p[0]})" for p in self.positions) borders = "" - if self.b.l: borders += "[" - if self.b.b: borders += "_" - if self.b.t: borders += "^" - if self.b.r: borders += "]" + if self.borders.left: + borders += "[" + if self.borders.bottom: + borders += "_" + if self.borders.top: + borders += "^" + if self.borders.right: + borders += "]" start = "CellH" if self.is_header else "Cell" return start + f"[{positions}] {borders}" - diff --git a/src/modm_data/pdf2html/convert.py b/src/modm_data/pdf2html/convert.py index 62504f7..c65d95e 100644 --- a/src/modm_data/pdf2html/convert.py +++ b/src/modm_data/pdf2html/convert.py @@ -5,17 +5,25 @@ from .html import format_document, write_html from .render import render_page_pdf -from ..utils import pkg_apply_patch, pkg_file_exists +from ..utils import pkg_apply_patch, pkg_file_exists, apply_patch from .ast import merge_area from pathlib import Path import pypdfium2 as pp -import subprocess -def convert(doc, page_range, output_path, format_chapters=False, pretty=True, - render_html=True, render_pdf=False, render_all=False, - show_ast=False, show_tree=False, show_tags=False) -> bool: - +def convert( + doc, + page_range, + output_path, + format_chapters=False, + pretty=True, + render_html=True, + render_pdf=False, + render_all=False, + show_ast=False, + show_tree=False, + show_tags=False, +) -> bool: document = None debug_doc = None debug_index = 0 @@ -43,7 +51,7 @@ def convert(doc, page_range, output_path, format_chapters=False, pretty=True, debug_index += 1 if render_pdf: - with open(f"debug_{output_path.stem}.pdf", 'wb') as file_handle: + with open(f"debug_{output_path.stem}.pdf", "wb") as file_handle: pp.PdfDocument(debug_doc).save(file_handle) if show_tree or render_html: diff --git a/src/modm_data/pdf2html/figure.py b/src/modm_data/pdf2html/figure.py index 45a168f..ded747c 100644 --- a/src/modm_data/pdf2html/figure.py +++ b/src/modm_data/pdf2html/figure.py @@ -1,7 +1,6 @@ # Copyright 2022, Niklas Hauser # SPDX-License-Identifier: MPL-2.0 -import math from ..utils import Rectangle diff --git a/src/modm_data/pdf2html/html.py b/src/modm_data/pdf2html/html.py index 8db89a8..c9bc148 100644 --- a/src/modm_data/pdf2html/html.py +++ b/src/modm_data/pdf2html/html.py @@ -4,12 +4,12 @@ import logging from lxml import etree import anytree -from anytree import RenderTree from ..utils import list_strip from .ast import normalize_lines, normalize_lists, normalize_paragraphs _LOGGER = logging.getLogger(__name__) + def _format_html_figure(xmlnode, figurenode): tnode = etree.Element("table") tnode.set("width", f"{int(figurenode._width * 50)}%") @@ -84,8 +84,9 @@ def _format_html_table(xmlnode, tablenode): cell_doc = normalize_lists(cell_doc) cell_doc = normalize_paragraphs(cell_doc) # _LOGGER.debug(RenderTree(cell_doc)) - _format_html(xynodespan, cell_doc, with_newlines=True, - ignore_formatting={"bold"} if cell.is_header else None) + _format_html( + xynodespan, cell_doc, with_newlines=True, ignore_formatting={"bold"} if cell.is_header else None + ) def _format_char(node, state, chars, ignore): @@ -96,9 +97,10 @@ def _format_char(node, state, chars, ignore): "bold": False, "underline": False, } - if state is None: state = NOFMT + if state is None: + state = NOFMT char = chars[0] - if char["char"] in {'\r'}: + if char["char"] in {"\r"}: return (True, node, state) # print(node, state, char["char"]) @@ -110,7 +112,7 @@ def _format_char(node, state, chars, ignore): if not diffs: prev_name = node.children[-1].name if node.children else None # print(node) - if prev_name != "newline" and char["char"] == '\n': + if prev_name != "newline" and char["char"] == "\n": # if not (prev_name == "chars" and node.children[-1].chars[-1] == " "): anytree.Node("newline", parent=node) elif prev_name != "chars": @@ -136,21 +138,22 @@ def _format_lines(textnode, ignore, with_newlines, with_start): chars = [] for line in textnode.children: if line.name == "line": - for char in line.obj.chars[0 if with_start else line.start:]: - if not with_newlines and char.unicode in {0xa, 0xd}: + for char in line.obj.chars[0 if with_start else line.start :]: + if not with_newlines and char.unicode in {0xA, 0xD}: continue chars.append(char_props(line.obj, char)) - if with_newlines and chars[-1]["char"] not in {'\n'}: + if with_newlines and chars[-1]["char"] not in {"\n"}: char = char_props(line.obj, line.obj.chars[-1]) - char["char"] = '\n' + char["char"] = "\n" chars.append(char) - chars = list_strip(chars, lambda c: c["char"] in {' ', '\n'}) + chars = list_strip(chars, lambda c: c["char"] in {" ", "\n"}) state = None node = formatn while chars: popchar, node, state = _format_char(node, state, chars, ignore) - if popchar: chars.pop(0) + if popchar: + chars.pop(0) return formatn @@ -173,7 +176,8 @@ def _format_html_fmt(xmlnode, treenode, tail=False): return (tail, xmlnode) else: # print(f"sub {treenode.name}") - if tail: xmlnode = xmlnode.getparent() + if tail: + xmlnode = xmlnode.getparent() subnode = etree.SubElement(xmlnode, CONV[treenode.name]) tail = False iternode = subnode @@ -193,8 +197,7 @@ def _format_html_text(xmlnode, treenode, ignore=None, with_newlines=False, with_ # print(etree.tostring(xmlnode, pretty_print=True).decode("utf-8")) -def _format_html(xmlnode, treenode, ignore_formatting=None, - with_newlines=False, with_start=True): +def _format_html(xmlnode, treenode, ignore_formatting=None, with_newlines=False, with_start=True): if ignore_formatting is None: ignore_formatting = set() # print(xmlnode, treenode.name) @@ -232,9 +235,9 @@ def _format_html(xmlnode, treenode, ignore_formatting=None, _format_html_figure(xmlnode, treenode) return - elif treenode.name == "bits": - _format_html_bits(xmlnode, treenode) - return + # elif treenode.name == "bits": + # _format_html_bits(xmlnode, treenode) + # return elif treenode.name.startswith("list"): if treenode.name[4] in {"b", "s"}: diff --git a/src/modm_data/pdf2html/line.py b/src/modm_data/pdf2html/line.py index 31d6e0e..091cb04 100644 --- a/src/modm_data/pdf2html/line.py +++ b/src/modm_data/pdf2html/line.py @@ -23,10 +23,12 @@ def content(self) -> str: @cached_property def bbox(self) -> Rectangle: - return Rectangle(min(c.bbox.left for c in self.chars), - min(c.bbox.bottom for c in self.chars), - max(c.bbox.right for c in self.chars), - max(c.bbox.top for c in self.chars)) + return Rectangle( + min(c.bbox.left for c in self.chars), + min(c.bbox.bottom for c in self.chars), + max(c.bbox.right for c in self.chars), + max(c.bbox.top for c in self.chars), + ) class CharLine: @@ -34,10 +36,18 @@ class CharLine: A line of characters with super- and sub-script chars merged into. """ - def __init__(self, page, chars: list, bottom: float, - origin: float, top: float, - height: float = None, rotation: int = 0, - offset: float = 0, sort_origin: float = None): + def __init__( + self, + page, + chars: list, + bottom: float, + origin: float, + top: float, + height: float = None, + rotation: int = 0, + offset: float = 0, + sort_origin: float = None, + ): self._page = page self.chars = chars self.bottom = bottom @@ -51,10 +61,12 @@ def __init__(self, page, chars: list, bottom: float, @cached_property def bbox(self) -> Rectangle: """Bounding box of the character line""" - return Rectangle(min(c.bbox.left for c in self.chars), - min(c.bbox.bottom for c in self.chars), - max(c.bbox.right for c in self.chars), - max(c.bbox.top for c in self.chars)) + return Rectangle( + min(c.bbox.left for c in self.chars), + min(c.bbox.bottom for c in self.chars), + max(c.bbox.right for c in self.chars), + max(c.bbox.top for c in self.chars), + ) @cached_property def fonts(self) -> set[str]: @@ -75,6 +87,7 @@ def content(self) -> str: def clusters(self, absolute_tolerance: float = None) -> list[CharCluster]: """Find clusters of characters in a line separated by `absolute_tolerance`.""" + def _cluster(clusters, chars): if chars: clusters.append(CharCluster(self, chars)) @@ -89,7 +102,7 @@ def _cluster(clusters, chars): if next_char.bbox.left - last_char.bbox.right < absolute_tolerance: # Keep this char in the current cluster current_chars.append(next_char) - if next_char.unicode not in {0x20, 0xa, 0xd}: + if next_char.unicode not in {0x20, 0xA, 0xD}: last_char = next_char else: # Larger spacing detected, create a new cluster diff --git a/src/modm_data/pdf2html/page.py b/src/modm_data/pdf2html/page.py index 33f687a..b926be9 100644 --- a/src/modm_data/pdf2html/page.py +++ b/src/modm_data/pdf2html/page.py @@ -1,19 +1,16 @@ # Copyright 2022, Niklas Hauser # SPDX-License-Identifier: MPL-2.0 -import re -import math import logging -import textwrap import statistics from typing import Callable -from functools import cached_property, cache, reduce +from functools import cached_property from collections import defaultdict from .table import Table from .figure import Figure from .line import CharLine -from ..utils import HLine, VLine, Rectangle, Region -from ..pdf import Path, Image, Page as PdfPage, Character +from ..utils import Rectangle, Region +from ..pdf import Page as PdfPage, Character from anytree import Node @@ -53,16 +50,24 @@ def _spacing(self) -> dict[str, float]: def _line_size(self, line: CharLine) -> str: rsize = line.height - if rsize >= 17.5: return "h1" - elif rsize >= 15.5: return "h2" - elif rsize >= 13.5: return "h3" - elif rsize >= 11.4: return "h4" - elif rsize >= 8.5: return "n" - else: return "fn" + if rsize >= 17.5: + return "h1" + elif rsize >= 15.5: + return "h2" + elif rsize >= 13.5: + return "h3" + elif rsize >= 11.4: + return "h4" + elif rsize >= 8.5: + return "n" + else: + return "fn" def _colors(self, color: int) -> str: - if 0xff <= color <= 0xff: return "black" - if 0xffffffff <= color <= 0xffffffff: return "white" + if 0xFF <= color <= 0xFF: + return "black" + if 0xFFFFFFFF <= color <= 0xFFFFFFFF: + return "white" return "unknown" @cached_property @@ -70,9 +75,10 @@ def _areas(self) -> dict[str, list[Rectangle] | Rectangle]: content = Rectangle(0.1, 0.1, 0.9, 0.9) areas = {"content": [content]} scaled_areas = {} + def _s(r): - return Rectangle(r.left * self.width, r.bottom * self.height, - r.right * self.width, r.top * self.height) + return Rectangle(r.left * self.width, r.bottom * self.height, r.right * self.width, r.top * self.height) + for name, area in areas.items(): scaled_areas[name] = [_s(r) for r in area] if isinstance(area, list) else _s(area) return scaled_areas @@ -107,17 +113,21 @@ def text_in_named_area(self, name: str, check_length: bool = True) -> str | None :param check_length: assert that the text has a length. :return: the concatenated text of the named area(s) or `None` if area not found. """ - if name not in self._areas: return None + if name not in self._areas: + return None text = "" areas = self._areas[name] - if not isinstance(areas, list): areas = [areas] - for area in areas: text += self.text_in_area(area) - if check_length: assert text + if not isinstance(areas, list): + areas = [areas] + for area in areas: + text += self.text_in_area(area) + if check_length: + assert text return text - def charlines_in_area(self, area: Rectangle, - predicate: Callable[[Character], bool] = None, - rtol: float = None) -> list[CharLine]: + def charlines_in_area( + self, area: Rectangle, predicate: Callable[[Character], bool] = None, rtol: float = None + ) -> list[CharLine]: """ Coalesce the characters in the area and predicate into lines. @@ -135,7 +145,8 @@ def charlines_in_area(self, area: Rectangle, :param rtol: Relative tolerance to separate lines vertically or use `sc` spacing by default. :return: A list of character lines sorted by x or y position. """ - if rtol is None: rtol = self._spacing["sc"] + if rtol is None: + rtol = self._spacing["sc"] # Split all chars into lines based on rounded origin origin_lines_y = defaultdict(list) origin_lines_x = defaultdict(list) @@ -144,12 +155,13 @@ def charlines_in_area(self, area: Rectangle, if predicate is not None and not predicate(char): continue cunicode = self._unicode_filter(char.unicode) - if cunicode is None: continue + if cunicode is None: + continue char.unicode = cunicode - if char.unicode < 32 and char.unicode not in {0xa}: + if char.unicode < 32 and char.unicode not in {0xA}: continue # Ignore characters without width that are not spaces - if not char.width and char.unicode not in {0xa, 0xd, 0x20}: + if not char.width and char.unicode not in {0xA, 0xD, 0x20}: _LOGGER.error(f"Unknown char width for {char}: {char.bbox}") # Split up the chars depending on the orientation if 45 < char.rotation <= 135 or 225 < char.rotation <= 315: @@ -163,32 +175,38 @@ def charlines_in_area(self, area: Rectangle, bbox_lines_y = [] for chars in origin_lines_y.values(): # Remove lines with whitespace only - if all(c.unicode in {0xa, 0xd, 0x20} for c in chars): + if all(c.unicode in {0xA, 0xD, 0x20} for c in chars): continue origin = statistics.fmean(c.origin.y for c in chars) - line = CharLine(self, chars, - min(c.bbox.bottom for c in chars), - origin, - max(c.bbox.top for c in chars), - max(c.height for c in chars), - sort_origin=self.height - origin) + line = CharLine( + self, + chars, + min(c.bbox.bottom for c in chars), + origin, + max(c.bbox.top for c in chars), + max(c.height for c in chars), + sort_origin=self.height - origin, + ) bbox_lines_y.append(line) # print(line, line.top, line.origin, line.bottom, line.height) - bbox_lines = sorted(bbox_lines_y, key=lambda l: l._sort_origin) + bbox_lines = sorted(bbox_lines_y, key=lambda line: line._sort_origin) bbox_lines_x = [] for chars in origin_lines_x.values(): # Remove lines with whitespace only - if all(c.unicode in {0xa, 0xd, 0x20} for c in chars): + if all(c.unicode in {0xA, 0xD, 0x20} for c in chars): continue - line = CharLine(self, chars, - min(c.bbox.left for c in chars), - statistics.fmean(c.origin.x for c in chars), - max(c.bbox.right for c in chars), - max(c.width for c in chars), - 270 if sum(c.rotation for c in chars) <= 135 * len(chars) else 90) + line = CharLine( + self, + chars, + min(c.bbox.left for c in chars), + statistics.fmean(c.origin.x for c in chars), + max(c.bbox.right for c in chars), + max(c.width for c in chars), + 270 if sum(c.rotation for c in chars) <= 135 * len(chars) else 90, + ) bbox_lines_x.append(line) - bbox_lines += sorted(bbox_lines_x, key=lambda l: l._sort_origin) + bbox_lines += sorted(bbox_lines_x, key=lambda line: line._sort_origin) if not bbox_lines: return [] @@ -200,8 +218,7 @@ def charlines_in_area(self, area: Rectangle, for next_line in bbox_lines[1:]: height = max(current_line.height, next_line.height) # Calculate overlap via normalize origin (increasing with line index) - if ((current_line._sort_origin + rtol * height) > - (next_line._sort_origin - rtol * height)): + if (current_line._sort_origin + rtol * height) > (next_line._sort_origin - rtol * height): # if line.rotation or self.rotation: # # The next line overlaps this one, we merge the shorter line # # (typically super- and subscript) into taller line @@ -209,10 +226,16 @@ def charlines_in_area(self, area: Rectangle, # else: use_current = current_line.height >= next_line.height line = current_line if use_current else next_line - current_line = CharLine(self, current_line.chars + next_line.chars, - line.bottom, line.origin, line.top, - height, line.rotation, - sort_origin=line._sort_origin) + current_line = CharLine( + self, + current_line.chars + next_line.chars, + line.bottom, + line.origin, + line.top, + height, + line.rotation, + sort_origin=line._sort_origin, + ) else: # The next line does not overlap the current line merged_lines.append(current_line) @@ -224,29 +247,43 @@ def charlines_in_area(self, area: Rectangle, sorted_lines = [] for line in merged_lines: if line.rotation == 90: + def sort_key(char): - if char.unicode in {0xa, 0xd}: + if char.unicode in {0xA, 0xD}: return char.tbbox.midpoint.y - 1e9 return char.tbbox.midpoint.y elif line.rotation == 270: + def sort_key(char): - if char.unicode in {0xa, 0xd}: + if char.unicode in {0xA, 0xD}: return -char.tbbox.midpoint.y + 1e9 return -char.tbbox.midpoint.y else: + def sort_key(char): - if char.unicode in {0xa, 0xd}: + if char.unicode in {0xA, 0xD}: return char.origin.x + 1e9 return char.origin.x - sorted_lines.append(CharLine(self, sorted(line.chars, key=sort_key), - line.bottom, line.origin, - line.top, line.height, - line.rotation, area.left, - sort_origin=line._sort_origin)) + + sorted_lines.append( + CharLine( + self, + sorted(line.chars, key=sort_key), + line.bottom, + line.origin, + line.top, + line.height, + line.rotation, + area.left, + sort_origin=line._sort_origin, + ) + ) return sorted_lines - def graphic_bboxes_in_area(self, area: Rectangle, with_graphics: bool = True) -> list[tuple[Rectangle, Table | Figure | None]]: + def graphic_bboxes_in_area( + self, area: Rectangle, with_graphics: bool = True + ) -> list[tuple[Rectangle, Table | Figure | None]]: """ Coalesce the graphics in the area into full width bounding boxes. @@ -305,7 +342,10 @@ def objects_in_area(self, area: Rectangle, with_graphics: bool = True) -> list[C objects += self.charlines_in_area(narea) else: oarea = obj.bbox.joined(obj.cbbox) if obj.cbbox else obj.bbox - predicate = lambda c: not obj.bbox.contains(c.origin) + + def predicate(c): + return not obj.bbox.contains(c.origin) + lines = self.charlines_in_area(oarea, predicate) # print(obj, oarea, lines, [line.content for line in lines]) objects += list(sorted(lines + [obj], key=lambda o: (-o.bbox.y, o.bbox.x))) diff --git a/src/modm_data/pdf2html/render.py b/src/modm_data/pdf2html/render.py index 526eb61..efbbee5 100644 --- a/src/modm_data/pdf2html/render.py +++ b/src/modm_data/pdf2html/render.py @@ -1,15 +1,12 @@ # Copyright 2022, Niklas Hauser # SPDX-License-Identifier: MPL-2.0 -import math import pypdfium2 as pp -from ..utils import VLine, HLine from ..pdf.render import render_page_pdf as pdf_render_page_pdf from ..pdf.render import _vline, _hline, _line, _rect - -def render_page_pdf(doc, page, new_doc = None, index = 0): +def render_page_pdf(doc, page, new_doc=None, index=0): """ @@ -25,14 +22,14 @@ def render_page_pdf(doc, page, new_doc = None, index = 0): if False: for ii in range(20): - _vline(new_page, rotation, page.width * ii / 20, 0, page.height, width=1, stroke="black") - _hline(new_page, rotation, page.height * ii / 20, 0, page.width, width=1, stroke="black") + _vline(new_page, rotation, width * ii / 20, 0, height, width=1, stroke="black") + _hline(new_page, rotation, height * ii / 20, 0, width, width=1, stroke="black") # for name, distance in page._spacing.items(): # if name.startswith("x_"): - # _vline(new_page, rotation, distance, 0, page.height, width=0.5, stroke=0xFFA500) + # _vline(new_page, rotation, distance, 0, height, width=0.5, stroke=0xFFA500) # else: - # _hline(new_page, rotation, distance, 0, page.width, width=0.5, stroke=0xFFA500) + # _hline(new_page, rotation, distance, 0, width, width=0.5, stroke=0xFFA500) for name, area in page._areas.items(): if isinstance(area, list): @@ -45,34 +42,50 @@ def render_page_pdf(doc, page, new_doc = None, index = 0): if obj.cbbox is not None: _rect(new_page, rotation, obj.cbbox, width=2, stroke=0x9ACD32) if obj.bbox is not None: - _rect(new_page, rotation, obj.bbox, width=2, stroke=0x00ff00) + _rect(new_page, rotation, obj.bbox, width=2, stroke=0x00FF00) for table in page.content_tables: - _rect(new_page, rotation, table.bbox, width=1.5, stroke=0x0000ff) + _rect(new_page, rotation, table.bbox, width=1.5, stroke=0x0000FF) for lines in table._xgrid.values(): for line in lines: - _line(new_page, rotation, line, width=0.75, stroke=0x0000ff) + _line(new_page, rotation, line, width=0.75, stroke=0x0000FF) for lines in table._ygrid.values(): for line in lines: - _line(new_page, rotation, line, width=0.75, stroke=0x0000ff) + _line(new_page, rotation, line, width=0.75, stroke=0x0000FF) for cell in table.cells: for line in cell.lines: for cluster in line.clusters(): _rect(new_page, rotation, cluster.bbox, width=0.33, stroke=0x808080) if cell.b.l: - _vline(new_page, rotation, cell.bbox.left, cell.bbox.bottom, cell.bbox.top, - width=cell.b.l, stroke=0xff0000) + _vline( + new_page, rotation, cell.bbox.left, cell.bbox.bottom, cell.bbox.top, width=cell.b.l, stroke=0xFF0000 + ) if cell.b.r: - _vline(new_page, rotation, cell.bbox.right, cell.bbox.bottom, cell.bbox.top, - width=cell.b.r, stroke=0x0000ff) + _vline( + new_page, + rotation, + cell.bbox.right, + cell.bbox.bottom, + cell.bbox.top, + width=cell.b.r, + stroke=0x0000FF, + ) if cell.b.b: - _hline(new_page, rotation, cell.bbox.bottom, cell.bbox.left, cell.bbox.right, - width=cell.b.b, stroke=0x00ff00) + _hline( + new_page, + rotation, + cell.bbox.bottom, + cell.bbox.left, + cell.bbox.right, + width=cell.b.b, + stroke=0x00FF00, + ) if cell.b.t: - _hline(new_page, rotation, cell.bbox.top, cell.bbox.left, cell.bbox.right, - width=cell.b.t, stroke=0x808080) + _hline( + new_page, rotation, cell.bbox.top, cell.bbox.left, cell.bbox.right, width=cell.b.t, stroke=0x808080 + ) assert pp.raw.FPDFPage_GenerateContent(new_page) pp.raw.FPDF_ClosePage(new_page) diff --git a/src/modm_data/pdf2html/stmicro/__init__.py b/src/modm_data/pdf2html/stmicro/__init__.py index fd9ce58..6b20557 100644 --- a/src/modm_data/pdf2html/stmicro/__init__.py +++ b/src/modm_data/pdf2html/stmicro/__init__.py @@ -3,3 +3,6 @@ from .document import Document +from .page import Page + +__all__ = ["Document", "Page"] diff --git a/src/modm_data/pdf2html/stmicro/__main__.py b/src/modm_data/pdf2html/stmicro/__main__.py index 208e2f6..e0f4563 100644 --- a/src/modm_data/pdf2html/stmicro/__main__.py +++ b/src/modm_data/pdf2html/stmicro/__main__.py @@ -11,8 +11,10 @@ from .. import convert, patch + def main(): import modm_data + parser = argparse.ArgumentParser() parser.add_argument("--document", type=Path) parser.add_argument("--output", type=str, default="") @@ -49,13 +51,12 @@ def main(): output_path = Path(args.output) output_path.parent.mkdir(parents=True, exist_ok=True) - document = None if args.parallel: log = Path(f"log/stmicro/html/{doc.name}.txt") log.parent.mkdir(exist_ok=True, parents=True) - with log.open('w') as logfile: + with log.open("w") as logfile: print(doc.page_count, doc.metadata, doc.is_tagged, file=logfile) - output_dir = (output_path.parent / output_path.stem) + output_dir = output_path.parent / output_path.stem output_dir.mkdir(parents=True, exist_ok=True) dests = [(0, "introduction")] for toc in doc.toc: @@ -72,22 +73,36 @@ def main(): ranges = [(p0, p1, t0) for (p0, t0), (p1, t1) in zip(dests, dests[1:]) if p0 != p1] calls = [] for ii, (p0, p1, title) in enumerate(ranges): - call = f"python3 -m modm_data.pdf2html.stmicro " \ - f"--document {args.document} --range {p0 + 1}:{p1} --html " \ - f"--output {output_dir}/chapter_{ii}_{title}.html" + call = ( + f"python3 -m modm_data.pdf2html.stmicro " + f"--document {args.document} --range {p0 + 1}:{p1} --html " + f"--output {output_dir}/chapter_{ii}_{title}.html" + ) calls.append(call + f" >> {log} 2>&1") print(call, file=logfile) with ThreadPool() as pool: retvals = list(tqdm.tqdm(pool.imap(lambda c: subprocess.run(c, shell=True), calls), total=len(calls))) for retval, call in zip(retvals, calls): - if retval.returncode != 0: print(call) + if retval.returncode != 0: + print(call) if all(r.returncode == 0 for r in retvals): from . import data + return patch(doc, data, output_dir) return False - return convert(doc, page_range, output_path, format_chapters=args.chapters, - render_html=args.html, render_pdf=args.pdf, render_all=args.all, - show_ast=args.ast, show_tree=args.tree, show_tags=args.tags) + return convert( + doc, + page_range, + output_path, + format_chapters=args.chapters, + render_html=args.html, + render_pdf=args.pdf, + render_all=args.all, + show_ast=args.ast, + show_tree=args.tree, + show_tags=args.tags, + ) + exit(0 if main() else 1) diff --git a/src/modm_data/pdf2html/stmicro/document.py b/src/modm_data/pdf2html/stmicro/document.py index 3931033..97654c4 100644 --- a/src/modm_data/pdf2html/stmicro/document.py +++ b/src/modm_data/pdf2html/stmicro/document.py @@ -2,14 +2,16 @@ # SPDX-License-Identifier: MPL-2.0 import logging +from anytree import RenderTree from .page import Page as StmPage from ...pdf import Document as PdfDocument from ..ast import normalize_lines, normalize_captions, normalize_lists from ..ast import normalize_paragraphs, normalize_headings, normalize_registers -from ..ast import normalize_tables, normalize_chapters +from ..ast import normalize_tables _LOGGER = logging.getLogger(__name__) + def _debug(func, indata, debug=0): _LOGGER.debug(func.__name__) if debug == -1: @@ -34,7 +36,6 @@ def _normalize_document(document): return document - class Document(PdfDocument): def __init__(self, path: str): super().__init__(path) diff --git a/src/modm_data/pdf2html/stmicro/page.py b/src/modm_data/pdf2html/stmicro/page.py index 68cec3c..6540f95 100644 --- a/src/modm_data/pdf2html/stmicro/page.py +++ b/src/modm_data/pdf2html/stmicro/page.py @@ -2,17 +2,14 @@ # SPDX-License-Identifier: MPL-2.0 import re -import math import logging -import textwrap -import statistics -from functools import cached_property, cache, reduce +from functools import cached_property, reduce from collections import defaultdict from ..table import Table from ..figure import Figure from ..line import CharLine -from ...utils import HLine, VLine, Rectangle, Region -from ...pdf import Path, Image, Page as PdfPage +from ...utils import HLine, VLine, Rectangle +from ...pdf import Image from ..page import Page as BasePage from anytree import Node @@ -29,10 +26,10 @@ def is_compatible(document) -> bool: def _areas_black_white(page) -> dict: def _scale(r): if page.rotation: - return Rectangle(r.bottom * page.width, (1 - r.right) * page.height, - r.top * page.width, (1 - r.left) * page.height) - return Rectangle(r.left * page.width, r.bottom * page.height, - r.right * page.width, r.top * page.height) + return Rectangle( + r.bottom * page.width, (1 - r.right) * page.height, r.top * page.width, (1 - r.left) * page.height + ) + return Rectangle(r.left * page.width, r.bottom * page.height, r.right * page.width, r.top * page.height) bottom_left = Rectangle(0.1, 0.1, 0.3, 0.12) bottom_middle = Rectangle(0.3, 0.1, 0.7, 0.12) @@ -61,8 +58,11 @@ def _scale(r): # Recognize the two column design of the Datasheets with a big table underneath if page.index < 3 and "DS" in page.pdf.name: # Find a wide path that would denote the beginning of a table - top_rect = [p.bbox.top / page.height for p in page.paths - if _scale(content).contains(p.bbox) and p.bbox.width > page.width * 0.75] + top_rect = [ + p.bbox.top / page.height + for p in page.paths + if _scale(content).contains(p.bbox) and p.bbox.width > page.width * 0.75 + ] if top_rect: # offset for table label just above it ybottom = max(*top_rect) + 0.0175 @@ -75,9 +75,9 @@ def _scale(r): text_middle = page.text_in_area(_scale(mr)) text_bullets = page.text_in_area(_scale(br)) text_hyphens = page.text_in_area(_scale(hr)) - if (not text_middle and - (any(c in text_bullets for c in {"•", chr(61623)}) or - any(c in text_hyphens for c in {"-"}))): + if not text_middle and ( + any(c in text_bullets for c in {"•", chr(61623)}) or any(c in text_hyphens for c in {"-"}) + ): areas["middle_bullets"] = br areas["middle_hyphens"] = hr all_content = all_content[:-1] @@ -98,8 +98,7 @@ def _scale(r): def _areas_blue_gray(page) -> dict: def _scale(r): - return Rectangle(r.left * page.width, r.bottom * page.height, - r.right * page.width, r.top * page.height) + return Rectangle(r.left * page.width, r.bottom * page.height, r.right * page.width, r.top * page.height) # This template doesn't use rotated pages, instead uses # hardcoded rotated page dimensions @@ -111,18 +110,13 @@ def _scale(r): content = Rectangle(0.025, 0.05, 0.975, 0.89 if page.index else 0.81) bottom_left = Rectangle(0, 0, 0.4, 0.05) top_right = Rectangle(0.3, 0.9025, 0.95, 0.9175) - areas = { - "id": bottom_left, - "top": top_right, - "all_content": content, - "content": [] - } + areas = {"id": bottom_left, "top": top_right, "all_content": content, "content": []} if page.index == 0: areas["content"] = [ # Document device string Rectangle(0.4, 0.91, 0.95, 0.95), # Document description string - Rectangle(0.05, 0.81, 0.95, 0.86) + Rectangle(0.05, 0.81, 0.95, 0.86), ] if page.index < 10: # Contains only a table with product summary @@ -169,16 +163,18 @@ def _spacing_black_white(page) -> dict: } if page.rotation: content = 0.14 - spacing.update({ - "x_em": 0.01 * page.height, - "y_em": 0.01 * page.width, - "x_left": content * page.width, - "x_right": (1 - content) * page.width, - "x_content": 0.2075 * page.width, - "y_tline": 0.005 * page.width, - "lh": 1.2, - "sc": 0.4, - }) + spacing.update( + { + "x_em": 0.01 * page.height, + "y_em": 0.01 * page.width, + "x_left": content * page.width, + "x_right": (1 - content) * page.width, + "x_content": 0.2075 * page.width, + "y_tline": 0.005 * page.width, + "lh": 1.2, + "sc": 0.4, + } + ) return spacing | _spacing_special(page) @@ -202,74 +198,91 @@ def _spacing_blue_gray(page) -> dict: "th": 0.33, } if page.rotation: - spacing.update({ - "x_em": 0.01 * page.height, - "y_em": 0.01 * page.width, - "x_left": 0.05 * page.width, - "x_right": (1 - 0.16) * page.width, - "x_content": 0.2075 * page.width, - "y_tline": 0.005 * page.width, - "lh": 1.6, - "sc": 0.2, - }) + spacing.update( + { + "x_em": 0.01 * page.height, + "y_em": 0.01 * page.width, + "x_left": 0.05 * page.width, + "x_right": (1 - 0.16) * page.width, + "x_content": 0.2075 * page.width, + "y_tline": 0.005 * page.width, + "lh": 1.6, + "sc": 0.2, + } + ) return spacing | _spacing_special(page) def _spacing_special(page) -> dict: # Patches to detect the header cells correctly - if ((page.pdf.name == "DS12930-v1" and page.index in range(90, 106)) or - (page.pdf.name == "DS12931-v1" and page.index in range(89, 105))): + if (page.pdf.name == "DS12930-v1" and page.index in range(90, 106)) or ( + page.pdf.name == "DS12931-v1" and page.index in range(89, 105) + ): return {"th": 0.1} - if ((page.pdf.name == "RM0453-v2" and page.index in [1354]) or - (page.pdf.name == "RM0456-v2" and page.index in [2881]) or - (page.pdf.name == "RM0456-v3" and page.index in [2880]) or - (page.pdf.name == "RM0461-v4" and page.index in [1246])): + if ( + (page.pdf.name == "RM0453-v2" and page.index in [1354]) + or (page.pdf.name == "RM0456-v2" and page.index in [2881]) + or (page.pdf.name == "RM0456-v3" and page.index in [2880]) + or (page.pdf.name == "RM0461-v4" and page.index in [1246]) + ): return {"th": 0.5} - if ((page.pdf.name == "RM0456-v2" and page.index in [3005])): + if page.pdf.name == "RM0456-v2" and page.index in [3005]: return {"th": 0.52} return {} def _linesize_black_white(line: CharLine) -> str: rsize = line.height - if rsize >= 17.5: return "h1" - elif rsize >= 15.5: return "h2" - elif rsize >= 13.5: return "h3" - elif rsize >= 11.4: return "h4" - elif rsize >= 8.5: return "n" - else: return "fn" + if rsize >= 17.5: + return "h1" + elif rsize >= 15.5: + return "h2" + elif rsize >= 13.5: + return "h3" + elif rsize >= 11.4: + return "h4" + elif rsize >= 8.5: + return "n" + else: + return "fn" def _linesize_blue_gray(line: CharLine) -> str: rsize = round(line.height) - if rsize >= 16: return "h1" - elif rsize >= 14: return "h2" - elif rsize >= 12: return "h3" - elif rsize >= 10: return "h4" - elif rsize >= 7: return "n" - else: return "fn" + if rsize >= 16: + return "h1" + elif rsize >= 14: + return "h2" + elif rsize >= 12: + return "h3" + elif rsize >= 10: + return "h4" + elif rsize >= 7: + return "n" + else: + return "fn" def _colors_black_white(color: int) -> str: - if 0xff <= color <= 0xff: + if 0xFF <= color <= 0xFF: return "black" - if 0xffffffff <= color <= 0xffffffff: + if 0xFFFFFFFF <= color <= 0xFFFFFFFF: return "white" return "unknown" def _colors_blue_gray(color: int) -> str: - if 0xff <= color <= 0xff: + if 0xFF <= color <= 0xFF: return "black" - if 0xffffffff <= color <= 0xffffffff: + if 0xFFFFFFFF <= color <= 0xFFFFFFFF: return "white" - if 0xb9c4caff <= color <= 0xb9c4caff: + if 0xB9C4CAFF <= color <= 0xB9C4CAFF: return "gray" - if 0x1f81afff <= color <= 0x1f81afff: + if 0x1F81AFFF <= color <= 0x1F81AFFF: return "lightblue" - if 0x2052ff <= color <= 0x2052ff: + if 0x2052FF <= color <= 0x2052FF: return "darkblue" - if 0x39a9dcff <= color <= 0x39a9dcff: + if 0x39A9DCFF <= color <= 0x39A9DCFF: return "blue" return "unknown" @@ -299,10 +312,13 @@ def __init__(self, document, index: int): def _unicode_filter(self, code: int) -> int: # Ignore Carriage Return characters and ® (superscript issues) - if code in {0xd, ord("®")}: return None + if code in {0xD, ord("®")}: + return None # Correct some weird unicode stuffing choices - if code in {2}: return ord("-") - if code in {61623, 61664}: return ord("•") + if code in {2}: + return ord("-") + if code in {61623, 61664}: + return ord("•") return code @cached_property @@ -328,9 +344,15 @@ def content_ast(self) -> list: if "DS" in self.pdf.name: # FIXME: Terrible hack to get the ordering information table fixed # Should be done in the AST as a rewrite similar to bit table rewrite with VirtualTable - order_page = next((item.page_index for item in self.pdf.toc if item.level == 0 and - re.search("ordering +information|part +numbering", item.title, re.IGNORECASE)), -1) - with_graphics = (order_page != self.index) + order_page = next( + ( + item.page_index + for item in self.pdf.toc + if item.level == 0 and re.search("ordering +information|part +numbering", item.title, re.IGNORECASE) + ), + -1, + ) + with_graphics = order_page != self.index for area in self._areas["content"]: ast.append(self.ast_in_area(area, with_graphics=with_graphics)) # Add a page node to the first leaf to keep track of where a page starts @@ -341,8 +363,8 @@ def content_ast(self) -> list: def graphics_in_area(self, area: Rectangle) -> list[Table | Figure]: # Find all graphic clusters in this area em = self._spacing["y_em"] - large_area = area.offset_x(em/2) - graphic_clusters = self.graphic_clusters(lambda p: large_area.contains(p.bbox), em/2) + large_area = area.offset_x(em / 2) + graphic_clusters = self.graphic_clusters(lambda p: large_area.contains(p.bbox), em / 2) # for bbox, paths in raw_graphic_clusters: # # Some docs have large DRAFT chars in the background # if any(path.fill == 0xe6e6e6ff and path.stroke == 0xff for path in paths): @@ -367,9 +389,10 @@ def graphics_in_area(self, area: Rectangle) -> list[Table | Figure]: bottom, top, height = chars[0].bbox.bottom, chars[0].bbox.top, chars[0].height # Find the graphic associated with this caption - graphic = next(((b, p) for b, p in graphic_clusters - if b.bottom <= bottom and - left <= b.left and b.right <= right), None) + graphic = next( + ((b, p) for b, p in graphic_clusters if b.bottom <= bottom and left <= b.left and b.right <= right), + None, + ) if graphic is None: _LOGGER.error(f"Graphic cluster not found for caption {''.join(c.char for c in chars)}") continue @@ -395,7 +418,7 @@ def graphics_in_area(self, area: Rectangle) -> list[Table | Figure]: graphics = [] for b, p in graphic_clusters: if gbbox.overlaps(b): - graphics.append((b,p)) + graphics.append((b, p)) for g in graphics: graphic_clusters.remove(g) gbbox = [cluster[0] for cluster in graphics] @@ -414,8 +437,10 @@ def graphics_in_area(self, area: Rectangle) -> list[Table | Figure]: elif "Table" in phrase: graphic_clusters.remove(graphic) gbbox, paths = graphic - if (self._template == "black_white" and - sum(1 for path in paths if path.count == 2) >= len(paths) / 2): + if ( + self._template == "black_white" + and sum(1 for path in paths if path.count == 2) >= len(paths) / 2 + ): otype += "_lines" categories.append((otype, cbbox, gbbox, paths)) @@ -427,8 +452,7 @@ def graphics_in_area(self, area: Rectangle) -> list[Table | Figure]: if any(isinstance(p, Image) for p in paths): category = "figure" elif self._template == "blue_gray": - if all(self._colors(path.stroke) == "gray" or - self._colors(path.fill) == "darkblue" for path in paths): + if all(self._colors(path.stroke) == "gray" or self._colors(path.fill) == "darkblue" for path in paths): category = "table" else: category = "figure" @@ -437,10 +461,11 @@ def graphics_in_area(self, area: Rectangle) -> list[Table | Figure]: # shapes with others are implicitly rendered with stroked lines stroked_table_lines = sum(1 for path in paths if path.count == 2) >= len(paths) / 2 is_table = stroked_table_lines or all( - [any(p.isclose(pp) for pp in path.bbox.points) - for p in path.points].count(True) >= len(path.points) * 2 / 3 - for path in paths) - if (len(paths) > 1 and is_table): + [any(p.isclose(pp) for pp in path.bbox.points) for p in path.points].count(True) + >= len(path.points) * 2 / 3 + for path in paths + ) + if len(paths) > 1 and is_table: category = "table" if stroked_table_lines: category += "_lines" @@ -450,7 +475,7 @@ def graphics_in_area(self, area: Rectangle) -> list[Table | Figure]: if "table" in category: # Check if there are only numbers on top of the table cbbox = Rectangle(gbbox.left, gbbox.top, gbbox.right, gbbox.top + self._spacing["y_em"]) - nchars = [c for c in self.chars_in_area(cbbox) if c.unicode not in {0x20, 0xa, 0xd}] + nchars = [c for c in self.chars_in_area(cbbox) if c.unicode not in {0x20, 0xA, 0xD}] if nchars and sum(1 if c.char.isnumeric() else 0 for c in nchars) >= len(nchars) / 3: # This is a register table with invisible top borders! @@ -512,19 +537,23 @@ def graphics_in_area(self, area: Rectangle) -> list[Table | Figure]: ylines.append(HLine(bbox.bottom, bbox.left, bbox.right, 0.1)) ylines.append(HLine(bbox.top, bbox.left, bbox.right, 0.1)) if yhlines: - yhlines.sort(key=lambda l: l.p0.y) + yhlines.sort(key=lambda line: line.p0.y) ylines.append(yhlines[0]) if not xlines or not ylines: continue - table = Table(self, graphics_bbox, xlines, ylines, caption_bbox, - is_register="register" in otype) + table = Table(self, graphics_bbox, xlines, ylines, caption_bbox, is_register="register" in otype) objects.append(table) return objects - def ast_in_area(self, area: Rectangle, with_graphics: bool = True, - ignore_xpos: bool = False, with_bits: bool = True, - with_notes: bool = True) -> Node: + def ast_in_area( + self, + area: Rectangle, + with_graphics: bool = True, + ignore_xpos: bool = False, + with_bits: bool = True, + with_notes: bool = True, + ) -> Node: x_em = self._spacing["x_em"] spacing_content = self._spacing["x_content"] lh_factor = self._spacing["lh"] @@ -550,11 +579,17 @@ def parent_name(current): # Tables should remain in their current hierarchy regardless of indentation if isinstance(obj, (Table, Figure)): - current = next((c for c in current.iter_path_reverse() - if c.name.startswith("head")), root) + current = next((c for c in current.iter_path_reverse() if c.name.startswith("head")), root) name = "figure" if isinstance(obj, Figure) else "table" - Node(name, parent=current, obj=obj, xpos=xpos, number=-1, - _width=obj.bbox.width / area.width, _type=obj._type) + Node( + name, + parent=current, + obj=obj, + xpos=xpos, + number=-1, + _width=obj.bbox.width / area.width, + _type=obj._type, + ) ypos = obj.bbox.bottom # Lines of text need to be carefully checked for indentation @@ -566,14 +601,14 @@ def parent_name(current): linesize = self._line_size(obj) # Check when the note has finished (=> paragraphs without italic) - if (parent_name(current) == "note" and - ((current.parent.type == "note" and not obj.contains_font(current.parent._font)) or - (current.parent.type in {"caution", "warning"} and newlines >= 2))): + if parent_name(current) == "note" and ( + (current.parent.type == "note" and not obj.contains_font(current.parent._font)) + or (current.parent.type in {"caution", "warning"} and newlines >= 2) + ): current = current.parent.parent # Check when the list ends into something indented far too right - elif (parent_name(current).startswith("list") - and (xpos - current.xpos) >= 2 * x_em): + elif parent_name(current).startswith("list") and (xpos - current.xpos) >= 2 * x_em: current = current.parent.parent # print(obj.fonts, ypos, xpos, current.xpos, f"{obj.height:.2f}", content) @@ -581,12 +616,13 @@ def parent_name(current): # Check if line is a heading, which may be multi-line, so we must # be careful not to nest them, but group them properly # Headings are always inserted into the root note! - if linesize.startswith("h1") or (linesize.startswith("h") and - xpos < (spacing_content + 2 * x_em) and "Bold" in obj.chars[0].font): + if linesize.startswith("h1") or ( + linesize.startswith("h") and xpos < (spacing_content + 2 * x_em) and "Bold" in obj.chars[0].font + ): if (match := re.match(r"^ *(\d+(\.\d+)?(\.\d+)?) *", content)) is not None: start = min(len(match.group(0)), len(obj.chars) - 1) marker = match.group(1) - size = marker.count('.') + 2 + size = marker.count(".") + 2 else: start = 0 marker = None @@ -596,12 +632,13 @@ def parent_name(current): if parent_name(current) != name or newlines > 2: content_start = start xpos = round(obj.chars[content_start].bbox.left) - current = Node(name, parent=root, obj=obj, xpos=xpos, - size=size, marker=marker) + current = Node(name, parent=root, obj=obj, xpos=xpos, size=size, marker=marker) current = Node("para", parent=current, obj=obj, xpos=current.xpos) # Check if the line is a note and deal with the indentation correctly - elif with_notes and (match := re.match(r" *([Nn]ote|[Cc]aution|[Ww]arning):? \d?", content)) is not None: + elif ( + with_notes and (match := re.match(r" *([Nn]ote|[Cc]aution|[Ww]arning):? \d?", content)) is not None + ): content_start = min(len(match.group(0)), len(obj.chars) - 1) # print(obj.fonts) # Correct xposition only if the Note: string is very far left @@ -609,20 +646,33 @@ def parent_name(current): xpos = round(obj.chars[content_start].bbox.left) # Prevent nesting of notes, they should only be listed if parent_name(current) == "note": - current = current.parent.parent + current = current.parent.parent current = unindent(xpos, current, 2) - current = Node("note", parent=current, obj=obj, xpos=xpos, - type=match.group(1).lower(), _font=obj.chars[content_start].font) + current = Node( + "note", + parent=current, + obj=obj, + xpos=xpos, + type=match.group(1).lower(), + _font=obj.chars[content_start].font, + ) current = Node("para", parent=current, obj=obj, xpos=current.xpos) # Check if line is Table or Figure caption - elif with_graphics and ((match := re.match(r" *([Tt]able|[Ff]igure) ?(\d+)\.? ?", content)) is not None - and "Bold" in obj.chars[0].font): + elif with_graphics and ( + (match := re.match(r" *([Tt]able|[Ff]igure) ?(\d+)\.? ?", content)) is not None + and "Bold" in obj.chars[0].font + ): content_start = min(len(match.group(0)), len(obj.chars) - 1) - current = next((c for c in current.iter_path_reverse() - if c.name.startswith("head")), root) - current = Node("caption", parent=current, obj=obj, xpos=xpos, - _type=match.group(1).lower(), number=int(match.group(2))) + current = next((c for c in current.iter_path_reverse() if c.name.startswith("head")), root) + current = Node( + "caption", + parent=current, + obj=obj, + xpos=xpos, + _type=match.group(1).lower(), + number=int(match.group(2)), + ) current = Node("para", parent=current, obj=obj, xpos=current.xpos) # Check if line is list and group them according to indentation @@ -632,8 +682,10 @@ def parent_name(current): xpos = round(obj.chars[content_start].bbox.left) name = "listb" value = lcontent[0] - if value in {"–", "-"}: name = "lists" - elif value.isalpha(): name = "lista" + if value in {"–", "-"}: + name = "lists" + elif value.isalpha(): + name = "lista" elif value.isnumeric(): name = "listn" value = int(match.group(2)) @@ -648,8 +700,12 @@ def parent_name(current): else: # Default back to the regex if "Reserved" not in content: - _LOGGER.warning(f"Fallback to Regex length for Bit pattern '{content}'!\nFonts: {obj.fonts}") - content_start = re.match(r" *([Bb]ytes? *.+? *)?(B[uio]t)( *\d+:?|s *(\d+ *([:-] *\d+ *)? *,? *)+) *", content) + _LOGGER.warning( + f"Fallback to Regex length for Bit pattern '{content}'!\nFonts: {obj.fonts}" + ) + content_start = re.match( + r" *([Bb]ytes? *.+? *)?(B[uio]t)( *\d+:?|s *(\d+ *([:-] *\d+ *)? *,? *)+) *", content + ) if content_start is None: _LOGGER.error(f"Unable to match Bit regex at all! '{content}'!") content_start = 0 @@ -659,12 +715,19 @@ def parent_name(current): _LOGGER.error(f"Missing content start (=0)! '{content}'!") content_start = min(content_start, len(obj.chars) - 1) - current = next((c for c in current.iter_path_reverse() - if c.name.startswith("head")), root) + current = next((c for c in current.iter_path_reverse() if c.name.startswith("head")), root) middle = obj.chars[content_start].bbox.left xpos = round(middle) - current = Node("bit", parent=current, obj=obj, xpos=xpos, _page=self, - _middle=middle, _left=area.left, _right=area.right) + current = Node( + "bit", + parent=current, + obj=obj, + xpos=xpos, + _page=self, + _middle=middle, + _left=area.left, + _right=area.right, + ) current = Node("para", parent=current, obj=obj, xpos=current.xpos) # Check if this is a new paragraph @@ -675,15 +738,13 @@ def parent_name(current): xpos = current.parent.xpos # Prevent multiline current = unindent(xpos, current, newlines) - current = Node("para", parent=current, obj=obj, - xpos=xpos if current.is_root else current.xpos) + current = Node("para", parent=current, obj=obj, xpos=xpos if current.is_root else current.xpos) - elif (parent_name(current) not in {"caption", "bit", "area"}): + elif parent_name(current) not in {"caption", "bit", "area"}: current = unindent(xpos, current, newlines) # Add the actual line - Node("line", parent=current, obj=obj, xpos=xpos, - start=content_start, str=content[content_start:50]) + Node("line", parent=current, obj=obj, xpos=xpos, start=content_start, str=content[content_start:50]) ypos = obj.origin diff --git a/src/modm_data/pdf2html/table.py b/src/modm_data/pdf2html/table.py index 6aa0995..dc4d1ad 100644 --- a/src/modm_data/pdf2html/table.py +++ b/src/modm_data/pdf2html/table.py @@ -6,14 +6,15 @@ from functools import cached_property from collections import defaultdict from ..utils import HLine, VLine, Rectangle -from .cell import TableCell +from .cell import Cell, Borders _LOGGER = logging.getLogger(__name__) class Table: - def __init__(self, page, bbox: Rectangle, xlines: list, ylines: list, - cbbox: Rectangle = None, is_register: bool = False): + def __init__( + self, page, bbox: Rectangle, xlines: list, ylines: list, cbbox: Rectangle = None, is_register: bool = False + ): self._page = page self._spacing = page._spacing self.bbox = bbox @@ -33,8 +34,9 @@ def _cluster(lines, key): grid[current].append(line) last = key(line) return grid - xgrid = _cluster(xlines, lambda l: l.p0.x) - ygrid = _cluster(ylines, lambda l: l.p0.y) + + xgrid = _cluster(xlines, lambda line: line.p0.x) + ygrid = _cluster(ylines, lambda line: line.p0.y) if is_register: self._type = "register" @@ -51,16 +53,17 @@ def _cluster(lines, key): # Find the positions of the second row of numbers if len(ygrid) > 2: for yi, (ypos0, ypos1) in enumerate(zip(sorted(ygrid), sorted(ygrid)[1:])): - nbbox = Rectangle(self.bbox.left, ygrid[ypos0][0].p0.y, - self.bbox.right, ygrid[ypos1][0].p0.y) + nbbox = Rectangle(self.bbox.left, ygrid[ypos0][0].p0.y, self.bbox.right, ygrid[ypos1][0].p0.y) if lines := self._page.charlines_in_area(nbbox): - if all(c.char.isnumeric() or c.unicode in {0x20, 0xa, 0xd} for c in lines[0].chars): + if all(c.char.isnumeric() or c.unicode in {0x20, 0xA, 0xD} for c in lines[0].chars): if not len(cluster := lines[0].clusters(self._page._spacing["x_em"] / 2)) % 16: clusters.append((cluster, nbbox)) self._bit_headers = len(ygrid) - yi - 1 else: self.grid = (len(cluster), 0) - _LOGGER.warning(f"Second bit pattern does not have 16 or 32 clusters! {self} ({self._page})") + _LOGGER.warning( + f"Second bit pattern does not have 16 or 32 clusters! {self} ({self._page})" + ) break # Merge these clusters to find their positions @@ -71,8 +74,14 @@ def _cluster(lines, key): # Now close the lines in between for cleft, cright in zip(cluster, cluster[1:]): # find a line between the clusters - xpos = next(((x, xgrid[x][0].p0.x) for x in sorted(xgrid) - if cleft.bbox.right < xgrid[x][0].p0.x < cright.bbox.left), None) + xpos = next( + ( + (x, xgrid[x][0].p0.x) + for x in sorted(xgrid) + if cleft.bbox.right < xgrid[x][0].p0.x < cright.bbox.left + ), + None, + ) # Didn't find one, we must add one manually if xpos is None: xpos = (cleft.bbox.right + cright.bbox.left) / 2 @@ -83,10 +92,8 @@ def _cluster(lines, key): ygrid[self.bbox.top].append(HLine(self.bbox.top, self.bbox.left, self.bbox.right)) # Fix the position keys properly - self._xgrid = {int(round(statistics.fmean(m.p0.x for m in l))): l - for l in xgrid.values()} - self._ygrid = {int(round(statistics.fmean(m.p0.y for m in l))): l - for l in ygrid.values()} + self._xgrid = {int(round(statistics.fmean(m.p0.x for m in line))): line for line in xgrid.values()} + self._ygrid = {int(round(statistics.fmean(m.p0.y for m in line))): line for line in ygrid.values()} # Map the positions to integers self._xpos = list(sorted(self._xgrid)) self._ypos = list(sorted(self._ygrid)) @@ -94,8 +101,7 @@ def _cluster(lines, key): self.grid = (len(self._xpos) - 1, len(self._ypos) - 1) self._cells = None - def _cell_borders(self, x: int, y: int, bbox: Rectangle, - mask: int = 0b1111) -> tuple[int, int, int, int]: + def _cell_borders(self, x: int, y: int, bbox: Rectangle, mask: int = 0b1111) -> tuple[int, int, int, int]: # left, bottom, right, top borders = [0, 0, 0, 0] mp = bbox.midpoint @@ -124,38 +130,37 @@ def _cell_borders(self, x: int, y: int, bbox: Rectangle, assert line.width break - return TableCell.Borders(*borders) + return Borders(*borders) def _fix_borders(self, cells, x: int, y: int): # We are looking at the 9 neighbors around the cells - cell = cells[(x, y)] - c = cells[(x, y)].b - r = cells[(x + 1, y)].b if cells[(x + 1, y)] is not None else TableCell.Borders(0, 0, 1, 0) - t = cells[(x, y + 1)].b if cells[(x, y + 1)] is not None else TableCell.Borders(0, 1, 0, 0) + c = cells[(x, y)].borders + r = cells[(x + 1, y)].borders if cells[(x + 1, y)] is not None else Borders(0, 0, 1, 0) + t = cells[(x, y + 1)].borders if cells[(x, y + 1)] is not None else Borders(0, 1, 0, 0) - # if (not c.t and csand c.r and c.b) and "Reset value" in cell.content: - # c.t = 1 + # if (not c.top and c.left and c.right and c.bottom) and "Reset value" in cell.content: + # c.top = 1 # Open at the top into a span - if (not c.t and c.r) and (not t.r or not t.l): - c.t = 1 - t.b = 1 + if (not c.top and c.right) and (not t.right or not t.left): + c.top = 1 + t.bottom = 1 # Open at the top and self is a span - if (not c.t and not c.r) and (t.r and t.l): - c.t = 1 - t.b = 1 + if (not c.top and not c.right) and (t.right and t.left): + c.top = 1 + t.bottom = 1 # Open to the right into a span - if (not c.r and c.t) and (not r.t or not r.b): - c.r = 1 - r.l = 1 + if (not c.right and c.top) and (not r.top or not r.bottom): + c.right = 1 + r.left = 1 # Open to the right and self is a span - if (not c.r and not c.t) and (r.t and r.b): - c.r = 1 - r.l = 1 + if (not c.right and not c.top) and (r.top and r.bottom): + c.right = 1 + r.left = 1 @property - def cells(self) -> list[TableCell]: + def cells(self) -> list[Cell]: if self._cells is None: if self.grid < (1, 1): self._cells = [] @@ -167,8 +172,7 @@ def cells(self) -> list[TableCell]: for xi, (x0, x1) in enumerate(zip(self._xpos, self._xpos[1:])): bbox = Rectangle(x0, y0, x1, y1) borders = self._cell_borders(xi, yi, bbox, 0b1111) - cells[(xi, yi)] = TableCell(self, (self.grid[1] - 1 - yi, xi), - bbox, borders, self._type == "register") + cells[(xi, yi)] = Cell(self, (self.grid[1] - 1 - yi, xi), bbox, borders, self._type == "register") # Fix table cell borders via consistency checks for yi in range(self.grid[1]): @@ -181,17 +185,18 @@ def _merge(px, py, x, y): return # print(cells[(x, y)]) # Right border is open - if not cells[(x, y)].b.r: + if not cells[(x, y)].borders.right: if cells[(x + 1, y)] is not None: cells[(px, py)]._merge(cells[(x + 1, y)]) _merge(px, py, x + 1, y) cells[(x + 1, y)] = None # Top border is open - if not cells[(x, y)].b.t: + if not cells[(x, y)].borders.top: if cells[(x, y + 1)] is not None: cells[(px, py)]._merge(cells[(x, y + 1)]) _merge(px, py, x, y + 1) cells[(x, y + 1)] = None + # Start merging in bottom left cell for yi in range(self.grid[1]): for xi in range(self.grid[0]): @@ -201,15 +206,21 @@ def _merge(px, py, x, y): y_header_pos = self.grid[1] if self._type != "register": if self.grid[1] > 1: - line_widths = {round(line.width, 1) for llist in self._ygrid.values() - for line in llist if line.width != 0.1} # magic width of virtual borders + line_widths = { + round(line.width, 1) for llist in self._ygrid.values() for line in llist if line.width != 0.1 + } # magic width of virtual borders if line_widths: line_width_max = max(line_widths) * 0.9 if min(line_widths) < line_width_max: # Find the first thick line starting from the top - y_header_pos = next((yi for yi, ypos in reversed(list(enumerate(self._ypos))) - if any(line.width > line_width_max for line in self._ygrid[ypos])), - y_header_pos) + y_header_pos = next( + ( + yi + for yi, ypos in reversed(list(enumerate(self._ypos))) + if any(line.width > line_width_max for line in self._ygrid[ypos]) + ), + y_header_pos, + ) # Map all the header is_bold = [] @@ -221,7 +232,8 @@ def _merge(px, py, x, y): bbox = cell.bbox else: bbox = bbox.joined(cell.bbox) - if bbox is None: continue + if bbox is None: + continue chars = self._page.chars_in_area(bbox) is_bold_pct = sum(1 if "Bold" in c.font else 0 for c in chars) / len(chars) if chars else 1 is_bold.append((yi, is_bold_pct > self._spacing["th"])) @@ -229,7 +241,8 @@ def _merge(px, py, x, y): # Some tables have no bold cells at all if all(not b[1] for b in is_bold): # Special case for two row tables without bold headers, but a bold line inbetween - if self.grid[1] == 2 and y_header_pos == 1: y_header_pos = 2 + if self.grid[1] == 2 and y_header_pos == 1: + y_header_pos = 2 else: if y_header_pos < self.grid[1]: # Find the lowest bold row starting from bold line @@ -237,7 +250,8 @@ def _merge(px, py, x, y): else: # Find the lowest bold row starting from the top for b in reversed(is_bold): - if not b[1]: break + if not b[1]: + break y_header_pos = b[0] # Tell the header cells @@ -355,25 +369,30 @@ def _move_cells(cells, own_xpos): if insert_only: src = merged_xpos[ii - 1] dsts = merged_xpos[ii:] - if debug: print(f"{src}->{dsts} I") + if debug: + print(f"{src}->{dsts} I") for cell in cells: _insert_cells(cell, src, dsts, True) break else: src = own_xpos[ii] - dsts = merged_xpos[ii:ii + 1] - if debug: print(f"{src}->{dsts} M") + dsts = merged_xpos[ii : ii + 1] + if debug: + print(f"{src}->{dsts} M") for cell in cells: _insert_cells(cell, src, dsts, False) - if debug: print() + if debug: + print() if self_xpos != merged_xpos: if debug: print(f"====== Self: x={self_xhead}->{merged_xhead} xpos={self_xpos}->{merged_xpos}") _move_cells(self.cells, self_xpos) if other_xpos != merged_xpos: if debug: - print(f"====== Other: x={other_xhead}->{merged_xhead} xpos={other_xheaders[other_xhead]}->{merged_xheaders[merged_xhead]}") + print( + f"====== Other: x={other_xhead}->{merged_xhead} xpos={other_xheaders[other_xhead]}->{merged_xheaders[merged_xhead]}" + ) _move_cells(other.cells, other_xpos) if debug: print() @@ -396,7 +415,9 @@ def _move_cells(cells, own_xpos): def append_side(self, other, expand=False) -> bool: if self.grid[1] != other.grid[1]: if expand: - _LOGGER.debug(f"Expanding bottom cells to match height: {self} ({self._page}) + {other} ({other._page})") + _LOGGER.debug( + f"Expanding bottom cells to match height: {self} ({self._page}) + {other} ({other._page})" + ) ymin = min(self.grid[1], other.grid[1]) ymax = max(self.grid[1], other.grid[1]) etable = other if self.grid[1] > other.grid[1] else self diff --git a/src/modm_data/svd/__init__.py b/src/modm_data/svd/__init__.py index a8abd5d..af5d163 100644 --- a/src/modm_data/svd/__init__.py +++ b/src/modm_data/svd/__init__.py @@ -2,7 +2,19 @@ # SPDX-License-Identifier: MPL-2.0 from . import stmicro - -from .model import * +from .model import Device, PeripheralType, Peripheral, Register, BitField, compare_device_trees from .write import format_svd, write_svd from .read import read_svd + +__all__ = [ + "stmicro", + "Device", + "PeripheralType", + "Peripheral", + "Register", + "BitField", + "compare_device_trees", + "format_svd", + "write_svd", + "read_svd", +] diff --git a/src/modm_data/svd/model.py b/src/modm_data/svd/model.py index f01f7d9..92c8cab 100644 --- a/src/modm_data/svd/model.py +++ b/src/modm_data/svd/model.py @@ -122,12 +122,13 @@ def _compare_trees(left, right): return False if not left.children: return True - return all(_compare_trees(l, r) for l,r in zip(left.children, right.children)) + return all(_compare_trees(left, right) for left, right in zip(left.children, right.children)) def compare_device_trees(left, right): assert isinstance(left, Device) and isinstance(right, Device) - if len(left.children) != len(right.children): return False - if not left.children: return True - return all(_compare_trees(l, r) for l,r in zip(left.children, right.children)) - + if len(left.children) != len(right.children): + return False + if not left.children: + return True + return all(_compare_trees(left, right) for left, right in zip(left.children, right.children)) diff --git a/src/modm_data/svd/read.py b/src/modm_data/svd/read.py index 38ca1b8..fbebbdc 100644 --- a/src/modm_data/svd/read.py +++ b/src/modm_data/svd/read.py @@ -1,7 +1,7 @@ # Copyright 2022, Niklas Hauser # SPDX-License-Identifier: MPL-2.0 -from .model import * +from .model import Device, Peripheral, Register, BitField def read_svd(path) -> Device: @@ -13,11 +13,11 @@ def read_svd(path) -> Device: for peripheral in pdev.peripherals: ptype = peripheral.get_derived_from() - if ptype is not None: ptype = ptype.name + if ptype is not None: + ptype = ptype.name nper = Peripheral(peripheral.name, ptype, peripheral.base_address, parent=device) for register in peripheral.registers: - nreg = Register(register.name, register.address_offset, - register.size//8, parent=nper) + nreg = Register(register.name, register.address_offset, register.size // 8, parent=nper) for field in register.fields: BitField(field.name, field.bit_offset, field.bit_width, parent=nreg) diff --git a/src/modm_data/svd/stmicro/__init__.py b/src/modm_data/svd/stmicro/__init__.py index 26f774a..ad096ec 100644 --- a/src/modm_data/svd/stmicro/__init__.py +++ b/src/modm_data/svd/stmicro/__init__.py @@ -2,3 +2,5 @@ # SPDX-License-Identifier: MPL-2.0 from .device import svd_file_devices, svd_device_files + +__all__ = ["svd_file_devices", "svd_device_files"] diff --git a/src/modm_data/svd/stmicro/device.py b/src/modm_data/svd/stmicro/device.py index bb41043..93daa2e 100644 --- a/src/modm_data/svd/stmicro/device.py +++ b/src/modm_data/svd/stmicro/device.py @@ -30,12 +30,11 @@ def svd_file_devices() -> dict[Path, list[DeviceIdentifier]]: hd_files[file] = devices files = list(root_path("ext/cmsis/svd/data/STMicro/").glob("*.svd")) + def _sortkey(file): name = re.sub(r"x+$", "", file.stem) - return (-len(name), - name.count("x"), - name.index("x") if "x" in name else 0, - name) + return (-len(name), name.count("x"), name.index("x") if "x" in name else 0, name) + files.sort(key=_sortkey) def _match(file, devices): @@ -50,17 +49,19 @@ def _match(file, devices): mdevices -= set(devices) cm_files[file] = devices - filefmt = {"cm": {str(p):[str(d) for d in v] for p,v in cm_files.items()}, - "hd": {str(p):[str(d) for d in v] for p,v in hd_files.items()}, - "rm": {str(p):[str(d) for d in v] for p,v in rm_files.items()}} - with _SVD_MAP_FILE.open('w', encoding='utf-8') as fh: + filefmt = { + "cm": {str(p): [str(d) for d in v] for p, v in cm_files.items()}, + "hd": {str(p): [str(d) for d in v] for p, v in hd_files.items()}, + "rm": {str(p): [str(d) for d in v] for p, v in rm_files.items()}, + } + with _SVD_MAP_FILE.open("w", encoding="utf-8") as fh: json.dump(filefmt, fh, indent=4) else: - with _SVD_MAP_FILE.open('r', encoding='utf-8') as fh: + with _SVD_MAP_FILE.open("r", encoding="utf-8") as fh: cache = json.load(fh) - rm_files = {Path(p):[did_from_string(d) for d in v] for p,v in cache["rm"].items()} - hd_files = {Path(p):[did_from_string(d) for d in v] for p,v in cache["hd"].items()} - cm_files = {Path(p):[did_from_string(d) for d in v] for p,v in cache["cm"].items()} + rm_files = {Path(p): [did_from_string(d) for d in v] for p, v in cache["rm"].items()} + hd_files = {Path(p): [did_from_string(d) for d in v] for p, v in cache["hd"].items()} + cm_files = {Path(p): [did_from_string(d) for d in v] for p, v in cache["cm"].items()} _SVD_FILES = (hd_files, cm_files, rm_files) @@ -80,4 +81,3 @@ def _remap(files, key): _remap(cm_files, "cm") _remap(rm_files, "rm") return device_files - diff --git a/src/modm_data/svd/write.py b/src/modm_data/svd/write.py index cb6c35b..f2c2c38 100644 --- a/src/modm_data/svd/write.py +++ b/src/modm_data/svd/write.py @@ -2,12 +2,13 @@ # SPDX-License-Identifier: MPL-2.0 from lxml import etree -from .model import * +from .model import Device, Peripheral, Register, BitField def _add_element(node, tag, text=None): element = etree.Element(tag) - if text is not None: element.text = str(text) + if text is not None: + element.text = str(text) node.append(element) return element @@ -55,7 +56,6 @@ def _format_bit_field(xmlnode, treenode): def _format_svd(xmlnode, treenode): - current = xmlnode if isinstance(treenode, Device): current = _format_device(xmlnode, treenode) @@ -90,16 +90,4 @@ def format_svd(register_tree): def write_svd(svd, path, pretty=True): with open(path, "wb") as file: - svd.write(file, pretty_print=pretty, - doctype='') - - -def read_svd(path) -> Device: - - parser = SVDParser.for_xml_file(path) - for peripheral in parser.get_device().peripherals: - print("%s @ 0x%08x" % (peripheral.name, peripheral.base_address)) - - with open(path, "wb") as file: - svd.write(file, pretty_print=pretty, - doctype='') + svd.write(file, pretty_print=pretty, doctype='') diff --git a/src/modm_data/utils/__init__.py b/src/modm_data/utils/__init__.py index 288f4c9..7dac16b 100644 --- a/src/modm_data/utils/__init__.py +++ b/src/modm_data/utils/__init__.py @@ -2,7 +2,28 @@ # SPDX-License-Identifier: MPL-2.0 from .math import Point, Line, VLine, HLine, Rectangle, Region -from .helper import * +from .helper import list_lstrip, list_rstrip, list_strip, pkg_file_exists, pkg_apply_patch, apply_patch from .anytree import ReversePreOrderIter -from .path import * +from .path import root_path, ext_path, cache_path, patch_path from .xml import XmlReader + +__all__ = [ + "Point", + "Line", + "VLine", + "HLine", + "Rectangle", + "Region", + "list_lstrip", + "list_rstrip", + "list_strip", + "pkg_file_exists", + "pkg_apply_patch", + "apply_patch", + "ReversePreOrderIter", + "root_path", + "ext_path", + "cache_path", + "patch_path", + "XmlReader", +] diff --git a/src/modm_data/utils/anytree.py b/src/modm_data/utils/anytree.py index d3c7bb6..5c77485 100644 --- a/src/modm_data/utils/anytree.py +++ b/src/modm_data/utils/anytree.py @@ -10,8 +10,7 @@ def _iter(children, filter_, stop, maxlevel): for child_ in reversed(children): if not AbstractIter._abort_at_level(2, maxlevel): descendantmaxlevel = maxlevel - 1 if maxlevel else None - yield from ReversePreOrderIter._iter( - child_.children, filter_, stop, descendantmaxlevel) + yield from ReversePreOrderIter._iter(child_.children, filter_, stop, descendantmaxlevel) if stop(child_): continue if filter_(child_): diff --git a/src/modm_data/utils/helper.py b/src/modm_data/utils/helper.py index cd98304..69f7191 100644 --- a/src/modm_data/utils/helper.py +++ b/src/modm_data/utils/helper.py @@ -5,6 +5,7 @@ from pathlib import Path from importlib.resources import files, as_file + def list_lstrip(input_values: list, strip_fn) -> list: ii = 0 for value in input_values: @@ -22,7 +23,7 @@ def list_rstrip(input_values: list, strip_fn) -> list: ii += 1 else: break - return input_values[:len(input_values) - ii] + return input_values[: len(input_values) - ii] def list_strip(input_values: list, strip_fn) -> list: @@ -39,11 +40,19 @@ def pkg_apply_patch(pkg, patch: Path, base_dir: Path) -> bool: def apply_patch(patch_file: Path, base_dir: Path) -> bool: - cmds = ["patch", - "--strip=1", "--silent", "--ignore-whitespace", - "--reject-file=-", "--forward", "--posix", - "--directory", Path(base_dir).absolute(), - "--input", Path(patch_file).absolute()] + cmds = [ + "patch", + "--strip=1", + "--silent", + "--ignore-whitespace", + "--reject-file=-", + "--forward", + "--posix", + "--directory", + Path(base_dir).absolute(), + "--input", + Path(patch_file).absolute(), + ] if subprocess.run(cmds + ["--dry-run"]).returncode: return False return subprocess.run(cmds).returncode == 0 diff --git a/src/modm_data/utils/math.py b/src/modm_data/utils/math.py index 76744be..54ee444 100644 --- a/src/modm_data/utils/math.py +++ b/src/modm_data/utils/math.py @@ -20,8 +20,9 @@ def __init__(self, *xy, type: Enum = None): self.type = type def isclose(self, other, rtol: float = 1e-09, atol: float = 0.0) -> bool: - return (math.isclose(self.x, other.x, rel_tol=rtol, abs_tol=atol) and - math.isclose(self.y, other.y, rel_tol=rtol, abs_tol=atol)) + return math.isclose(self.x, other.x, rel_tol=rtol, abs_tol=atol) and math.isclose( + self.y, other.y, rel_tol=rtol, abs_tol=atol + ) def distance_squared(self, other) -> float: return math.pow(self.x - other.x, 2) + math.pow(self.y - other.y, 2) @@ -67,14 +68,12 @@ def __init__(self, *r, width: float = None, type: Enum = None): @cached_property def bbox(self): - return Rectangle(min(self.p0.x, self.p1.x), - min(self.p0.y, self.p1.y), - max(self.p0.x, self.p1.x), - max(self.p0.y, self.p1.y)) + return Rectangle( + min(self.p0.x, self.p1.x), min(self.p0.y, self.p1.y), max(self.p0.x, self.p1.x), max(self.p0.y, self.p1.y) + ) def isclose(self, other, rtol: float = 1e-09, atol: float = 0.0) -> bool: - return (self.p0.isclose(other.p0, rtol, atol) and - self.p1.isclose(other.p1, rtol, atol)) + return self.p0.isclose(other.p0, rtol, atol) and self.p1.isclose(other.p1, rtol, atol) def contains(self, point, atol: float = 0.0) -> bool: # if the point lies on the line (A-C---B), the distance A-C + C-B = A-B @@ -100,14 +99,17 @@ def specialize(self): def __repr__(self) -> str: data = [repr(self.p0), repr(self.p1)] - if self.width: data += [f"{self.width:.1f}"] - if self.type is not None: data += [self.type.name] + if self.width: + data += [f"{self.width:.1f}"] + if self.type is not None: + data += [self.type.name] return f"<{','.join(data)}>" class VLine(Line): def __init__(self, x: float, y0: float, y1: float, width: float = None): - if y0 > y1: y0, y1 = y1, y0 + if y0 > y1: + y0, y1 = y1, y0 super().__init__(Point(x, y0), Point(x, y1), width=width) self.length = y1 - y0 @@ -120,13 +122,15 @@ def __repr__(self) -> str: y0 = f"{self.p0.y:.1f}" if isinstance(self.p0.y, float) else self.p0.y y1 = f"{self.p1.y:.1f}" if isinstance(self.p1.y, float) else self.p1.y out = f"" class HLine(Line): def __init__(self, y: float, x0: float, x1: float, width: float = None): - if x0 > x1: x0, x1 = x1, x0 + if x0 > x1: + x0, x1 = x1, x0 super().__init__(Point(x0, y), Point(x1, y), width=width) self.length = x1 - x0 @@ -139,7 +143,8 @@ def __repr__(self) -> str: x0 = f"{self.p0.x:.1f}" if isinstance(self.p0.x, float) else self.p0.x x1 = f"{self.p1.x:.1f}" if isinstance(self.p1.x, float) else self.p1.x out = f"" @@ -182,19 +187,21 @@ def __init__(self, *r): def contains(self, other) -> bool: if isinstance(other, Point): - return (self.bottom <= other.y <= self.top and - self.left <= other.x <= self.right) + return self.bottom <= other.y <= self.top and self.left <= other.x <= self.right # Comparing y-axis first may be faster for "content areas filtering" # when doing subparsing of page content (like in tables) - return (self.bottom <= other.bottom and other.top <= self.top and - self.left <= other.left and other.right <= self.right) + return ( + self.bottom <= other.bottom + and other.top <= self.top + and self.left <= other.left + and other.right <= self.right + ) def overlaps(self, other) -> bool: return self.contains(other.p0) or self.contains(other.p1) def isclose(self, other, rtol: float = 1e-09, atol: float = 0.0) -> bool: - return (self.p0.isclose(other.p0, rtol, atol) and - self.p1.isclose(other.p1, rtol, atol)) + return self.p0.isclose(other.p0, rtol, atol) and self.p1.isclose(other.p1, rtol, atol) @cached_property def midpoint(self) -> Point: @@ -202,42 +209,45 @@ def midpoint(self) -> Point: @cached_property def points(self) -> list[Point]: - return [self.p0, Point(self.right, self.bottom), - self.p1, Point(self.left, self.top)] + return [self.p0, Point(self.right, self.bottom), self.p1, Point(self.left, self.top)] def offset(self, offset): - return Rectangle(self.p0.x - offset, self.p0.y - offset, - self.p1.x + offset, self.p1.y + offset) + return Rectangle(self.p0.x - offset, self.p0.y - offset, self.p1.x + offset, self.p1.y + offset) def offset_x(self, offset): - return Rectangle(self.p0.x - offset, self.p0.y, - self.p1.x + offset, self.p1.y) + return Rectangle(self.p0.x - offset, self.p0.y, self.p1.x + offset, self.p1.y) def offset_y(self, offset): - return Rectangle(self.p0.x, self.p0.y - offset, - self.p1.x, self.p1.y + offset) + return Rectangle(self.p0.x, self.p0.y - offset, self.p1.x, self.p1.y + offset) def translated(self, point): - return Rectangle(self.p0.x + point.x, self.p0.y + point.y, - self.p1.x + point.x, self.p1.y + point.y) + return Rectangle(self.p0.x + point.x, self.p0.y + point.y, self.p1.x + point.x, self.p1.y + point.y) def rotated(self, rotation): cos = math.cos(math.radians(rotation)) sin = math.sin(math.radians(rotation)) - return Rectangle(self.p0.x * cos - self.p0.y * sin, - self.p0.x * sin + self.p0.y * cos, - self.p1.x * cos - self.p1.y * sin, - self.p1.x * sin + self.p1.y * cos) + return Rectangle( + self.p0.x * cos - self.p0.y * sin, + self.p0.x * sin + self.p0.y * cos, + self.p1.x * cos - self.p1.y * sin, + self.p1.x * sin + self.p1.y * cos, + ) def joined(self, other): - return Rectangle(min(self.p0.x, other.p0.x), - min(self.p0.y, other.p0.y), - max(self.p1.x, other.p1.x), - max(self.p1.y, other.p1.y)) + return Rectangle( + min(self.p0.x, other.p0.x), + min(self.p0.y, other.p0.y), + max(self.p1.x, other.p1.x), + max(self.p1.y, other.p1.y), + ) def round(self, accuracy=0): - return Rectangle(round(self.p0.x, accuracy), round(self.p0.y, accuracy), - round(self.p1.x, accuracy), round(self.p1.y, accuracy)) + return Rectangle( + round(self.p0.x, accuracy), + round(self.p0.y, accuracy), + round(self.p1.x, accuracy), + round(self.p1.y, accuracy), + ) def __hash__(self): return hash(self.p0) + hash(self.p1) @@ -248,14 +258,16 @@ def __repr__(self) -> str: class Region: def __init__(self, v0, v1, obj=None): - if v0 > v1: v0, v1 = v1, v0 + if v0 > v1: + v0, v1 = v1, v0 self.v0 = v0 self.v1 = v1 self.objs = [] if obj is None else [obj] self.subregions = [] def overlaps(self, o0, o1, atol=0) -> bool: - if o0 > o1: o0, o1 = o1, o0 + if o0 > o1: + o0, o1 = o1, o0 # if reg top is lower then o0 if (self.v1 + atol) <= o0: return False diff --git a/src/modm_data/utils/xml.py b/src/modm_data/utils/xml.py index d5fb525..954f828 100644 --- a/src/modm_data/utils/xml.py +++ b/src/modm_data/utils/xml.py @@ -10,7 +10,7 @@ from pathlib import Path LOGGER = logging.getLogger(__file__) -PARSER = etree.XMLParser(ns_clean=True, recover=True, encoding='utf-8') +PARSER = etree.XMLParser(ns_clean=True, recover=True, encoding="utf-8") class XmlReader: @@ -24,7 +24,7 @@ def __repr__(self): def _openDeviceXML(self, filename): LOGGER.debug("Opening XML file '%s'", os.path.basename(filename)) xml_file = Path(filename).read_text("utf-8", errors="replace") - xml_file = re.sub(' xmlns="[^"]+"', '', xml_file, count=1).encode("utf-8") + xml_file = re.sub(' xmlns="[^"]+"', "", xml_file, count=1).encode("utf-8") xmltree = etree.fromstring(xml_file, parser=PARSER) return xmltree @@ -38,8 +38,8 @@ def queryTree(self, query): response = None try: response = self.tree.xpath(query) - except: - LOGGER.error(f"Query failed for '{query}'", ) + except Exception: + LOGGER.error(f"Query failed for '{query}'") return response diff --git a/tools/marimo/search.py b/tools/marimo/search.py index a45e847..4c70bfe 100644 --- a/tools/marimo/search.py +++ b/tools/marimo/search.py @@ -26,7 +26,8 @@ def __(mo): @app.cell def __(__file__, filter_chapter, filter_document, filter_table, mo): - import lxml.html, lxml.etree + import lxml.html + import lxml.etree import re from pathlib import Path import pandas as pd diff --git a/tools/scripts/search_html.py b/tools/scripts/search_html.py index a89b519..ec423b4 100644 --- a/tools/scripts/search_html.py +++ b/tools/scripts/search_html.py @@ -121,7 +121,7 @@ def main(): chanode = anytree.Node("chapter", parent=docnode, obj=chapter) for table in tables: print(table, table.caption()) - tabnode = anytree.Node("table", parent=chanode, obj=table) + anytree.Node("table", parent=chanode, obj=table) html = format_document(rootnode) with open(Path(args.html), "wb") as f: diff --git a/tools/scripts/synchronize_docs.py b/tools/scripts/synchronize_docs.py index 121cb29..43a5e7d 100755 --- a/tools/scripts/synchronize_docs.py +++ b/tools/scripts/synchronize_docs.py @@ -1,12 +1,11 @@ # Copyright 2022, Niklas Hauser # SPDX-License-Identifier: MPL-2.0 -import re, os, sys, subprocess, shutil +import re +import sys +import subprocess from pathlib import Path from jinja2 import Environment -from os import listdir -from os.path import isfile, join, abspath -from collections import defaultdict TABLE_TEMPLATE = \ r""" @@ -55,7 +54,7 @@ def extract(text, key): def format_table(items, width, align=None): subs = {"items": items, "width": width} - if align: subs["align"] = align; + if align: subs["align"] = align return Environment().from_string(TABLE_TEMPLATE).render(subs) def template(path_in, path_out, substitutions):