[topgen] Add class to load a complete topcfg properly

At the moment, topgen simplies serialized the complete topcfg using
hjson.dumps() but this poses a few problems. The biggest one is that
it is a mix of manual conversion to dict (when calling as_dict()
in the various topgen functions) and automatic (when calling _asdict()
from hjson.dumps). Furthermore, it turns out that we are missing some
fields that probably were never added to _asdict().

This commit introduces a new class (CompleteTopCfg) whose sole purpose
is to take the produced Hjson and reconstruct an in-memory topcfg that
is *exactly* equivalent (in Pythonic types) to the ones that was dumped.
This requires to sometimes reconstruct some classes, sometimes not.
Classes that need to be reconstruct get a new method (fromdict) and
the CompleteTopCfg does as much automatic deserializing as possible,
then some manual fixing.

There is also a small tweak to how clock groups are handled: in
alert_lpgs, the clock groups are all elements of the clocks.groups
but because of the naive serializing, a full copy is made. This
commit introduces GroupProxy that behaves like a Group for all
purposes but serializes only to the group name. This makes the
resulting hjson clearer and the deserialization easier.

Since this process is quite fragile, when topgen is running it will
actually check that this works by dumping the Hjson, reloading it
with the CompleteTopCfg and then checking that the two are equivalent.

Signed-off-by: Amaury Pouly <[email protected]>

util/reggen/inter_signal.py

@@ -90,8 +90,21 @@ def _asdict(self) -> Dict[str, object]:
         ret['width'] = self.width
         if self.default is not None:
             ret['default'] = self.default
+        ret['class'] = 'InterSignal'  # This will let fromdict() know it has to create the class
 
         return ret
 
+    @classmethod
+    def fromdict(cls, item: Dict[str, object]) -> object:
+        if 'class' not in item or item['class'] == 'InterSignal':
+            return item
+        item["package"] = item.get("package", None)
+        item["default"] = item.get("default", None)
+        item["signal_type"] = item["type"]
+        del item["type"]
+        c = cls.__new__(cls)
+        c.__dict__.update(**item)
+        return c
+
     def as_dict(self) -> Dict[str, object]:
         return self._asdict()

util/reggen/params.py

@@ -55,8 +55,29 @@ def as_dict(self) -> Dict[str, object]:
         rd['type'] = self.param_type
         if self.unpacked_dimensions is not None:
             rd['unpacked_dimensions'] = self.unpacked_dimensions
+        # topgen sometimes manually adds a 'name_top' field after creation.
+        if getattr(self, "name_top", None):
+            rd['name_top'] = self.name_top
         return rd
 
+    def _asdict(self) -> Dict[str, object]:
+        # Add an attribute to distinguished between manual serialization (as_dict())
+        # or automatic by hjson.
+        d = self.as_dict()
+        d['class'] = self.__class__.__name__
+        return d
+
+    @classmethod
+    def fromdict(cls, param: Dict[str, object]) -> object:
+        param['desc'] = param.get('desc', None)
+        param['unpacked_dimensions'] = param.get('unpacked_dimensions', None)
+        del param['class']
+        param['param_type'] = param['type']
+        del param['type']
+        c = cls.__new__(cls)
+        c.__dict__.update(**param)
+        return c
+
 
 class LocalParam(BaseParam):
     def __init__(self,
@@ -82,6 +103,12 @@ def as_dict(self) -> Dict[str, object]:
         rd['default'] = self.value
         return rd
 
+    @classmethod
+    def fromdict(cls, param: Dict[str, object]) -> object:
+        assert param['local']
+        del param['local']
+        return super().fromdict.__func__(cls, param)
+
 
 class Parameter(BaseParam):
     def __init__(self,
@@ -104,6 +131,12 @@ def as_dict(self) -> Dict[str, object]:
         rd['expose'] = 'true' if self.expose else 'false'
         return rd
 
+    @classmethod
+    def fromdict(cls, param: Dict[str, object]) -> object:
+        param['local'] = param['local'] == 'true'
+        param['expose'] = param['expose'] == 'true'
+        return super().fromdict.__func__(cls, param)
+
 
 class RandParameter(BaseParam):
     def __init__(self,
@@ -129,6 +162,10 @@ def as_dict(self) -> Dict[str, object]:
         rd['randtype'] = self.randtype
         return rd
 
+    @classmethod
+    def fromdict(cls, param: Dict[str, object]) -> object:
+        return super().fromdict.__func__(cls, param)
+
 
 class MemSizeParameter(BaseParam):
     def __init__(self,

util/topgen.py

@@ -39,6 +39,7 @@
 from topgen.resets import Resets
 from topgen.rust import TopGenRust
 from topgen.top import Top
+from topgen.topcfg import CompleteTopCfg
 
 # Common header for generated files
 warnhdr = """//
@@ -90,6 +91,9 @@ def ipgen_render(template_name: str, topname: str, params: Dict[str, object],
         log.error(e.verbose_str())
         sys.exit(1)
 
+    # Remote extra topname
+    params.pop("topname")
+
 
 def generate_top(top: Dict[str, object], name_to_block: Dict[str, IpBlock],
                  tpl_filename: str, **kwargs: Dict[str, object]) -> None:
@@ -460,12 +464,6 @@ def generate_flash(topcfg: Dict[str, object], out_path: Path) -> None:
         return
 
     params = vars(flash_mems[0]["memory"]["mem"]["config"])
-    # Additional parameters not provided in the top config.
-    params.update({
-        "metadata_width": 12,
-        "info_types": 3,
-        "infos_per_bank": [10, 1, 2]
-    })
 
     params.pop('base_addrs', None)
     ipgen_render("flash_ctrl", topname, params, out_path)
@@ -822,6 +820,12 @@ def _process_top(
     return completecfg, name_to_block, name_to_hjson
 
 
+def test_topcfg_loader(genhjson_path: Path, completecfg: Dict[str, object]):
+    loaded_cfg = CompleteTopCfg.from_path(genhjson_path)
+
+    CompleteTopCfg.check_equivalent(completecfg, loaded_cfg)
+
+
 def _check_countermeasures(completecfg: Dict[str, object],
                            name_to_block: Dict[str, IpBlock],
                            name_to_hjson: Dict[str, Path]) -> bool:
@@ -1106,6 +1110,10 @@ def main():
     genhjson_path.write_text(genhdr + gencmd +
                              hjson.dumps(completecfg, for_json=True, default=vars) + '\n')
 
+    # We also run a sanity check on the topcfg loader to make sure that it roundtrips
+    # correctly when loading.
+    test_topcfg_loader(genhjson_path, completecfg)
+
     # Generate Rust toplevel definitions
     if not args.no_rust:
         generate_rust(topname, completecfg, name_to_block, out_path.resolve(),
@@ -1287,9 +1295,9 @@ def render_template(template_path: str, rendered_path: Path,
 
         # Auto-generate tests in "sw/device/tests/autogen" area.
         gencmd = warnhdr + GENCMD.format(top_name=top_name)
-        for fname in ["plic_all_irqs_test.c", "alert_test.c", "BUILD"]:
+        for fname in ["plic_all_irqs_test.c", "alert_test.c"]:
             outfile = SRCTREE_TOP / "sw/device/tests/autogen" / fname
-            render_template(TOPGEN_TEMPLATE_PATH / f"{fname}.tpl",
+            render_template(TOPGEN_TEMPLATE_PATH / ".." / ".." / "autogen_tests" / "templates" / f"{fname}.tpl",
                             outfile,
                             helper=c_helper,
                             gencmd=gencmd)

util/topgen/clocks.py

@@ -20,13 +20,17 @@ def _bool_to_yn(val: bool) -> str:
     return 'yes' if val else 'no'
 
 
+def _bool_from_yn(val: str) -> bool:
+    return val == 'yes'
+
+
 def _to_int(val: object) -> int:
     if isinstance(val, int):
         return val
     return int(str(val))
 
 
-def _check_choices(val: str, what: str, choices: List[str]) -> str:
+def _check_choices(val: str, what: str, choices: list[str]) -> str:
     if val in choices:
         return val
     raise ValueError(
@@ -64,6 +68,14 @@ def _asdict(self) -> Dict[str, object]:
             'ref': self.ref
         }
 
+    @classmethod
+    def fromdict(cls, src: Dict[str, object]) -> object:
+        src['aon'] = _bool_from_yn(src['aon'])
+        src['freq'] = int(src['freq'])
+        c = cls.__new__(cls)
+        c.__dict__.update(**src)
+        return c
+
 
 class DerivedSourceClock(SourceClock):
     '''A derived source clock (divided down from some other clock).'''
@@ -80,6 +92,12 @@ def _asdict(self) -> Dict[str, object]:
         ret['src'] = self.src.name
         return ret
 
+    @classmethod
+    def fromdict(cls, src: Dict[str, object], clocks: Dict[str, SourceClock]) -> object:
+        src['div'] = int(src['div'])
+        src['src'] = clocks[src['src']]
+        return super().fromdict.__func__(cls, src)
+
 
 class ClockSignal:
     '''A clock signal in the design.'''
@@ -139,6 +157,42 @@ def _asdict(self) -> Dict[str, object]:
              for name, sig in self.clocks.items()}
         }
 
+    @classmethod
+    def fromdict(cls, src: Dict[str, object], clocks: Dict[str, SourceClock]) -> object:
+        src['unique'] = _bool_from_yn(src['unique'])
+        src['clocks'] = {
+            name: ClockSignal(name, clocks[sig_src_name])
+            for (name, sig_src_name) in src['clocks'].items()
+        }
+        c = cls.__new__(cls)
+        c.__dict__.update(**src)
+        return c
+
+
+class GroupProxy:
+    """
+    The sole purpose of this class is to use an object as if it's an object
+    but when serializing, we only print the group name instead of the definition.
+    """
+    def __init__(self, grp):
+        self._grp = grp
+
+    def __getattr__(self, name):
+        return getattr(self._grp, name)
+
+    def _asdict(self):
+        return {
+            "name": self._grp.name
+        }
+
+    @classmethod
+    def fromdict(cls, proxy: Dict[str, object], clocks: object) -> object:
+        proxy['_grp'] = clocks.groups[proxy["name"]]
+        del proxy["name"]
+        c = cls.__new__(cls)
+        c.__dict__.update(**proxy)
+        return c
+
 
 class TypedClocks(NamedTuple):
     # External clocks that are consumed only inside the clkmgr and are fed from
@@ -170,7 +224,7 @@ class TypedClocks(NamedTuple):
     # division, sorted by name. This doesn't include clock sources that are
     # only used to derive divided clocks (we might gate the divided clocks, but
     # don't bother gating the upstream source).
-    rg_srcs: List[str]
+    rg_srcs: list[str]
 
     # A diction of the clock families.
     # The key for each is root clock, while the list contains all the clocks
@@ -225,6 +279,9 @@ def __init__(self, raw: List[object]):
     def _asdict(self) -> Dict[str, object]:
         return self.clks
 
+    def _check_field(self) -> list[str]:
+        return ["clkc"]
+
 
 class Clocks:
     '''Clock connections for the chip.'''
@@ -265,6 +322,25 @@ def _asdict(self) -> Dict[str, object]:
             'groups': list(self.groups.values())
         }
 
+    @classmethod
+    def fromdict(cls, clocks: Dict[str, object]) -> object:
+        clocks['srcs'] = {
+            src["name"]: SourceClock.fromdict(src) for src in clocks['srcs']
+        }
+        clocks['derived_srcs'] = {
+            src["name"]: DerivedSourceClock.fromdict(src, clocks['srcs'])
+            for src in clocks['derived_srcs']
+        }
+        clocks['all_srcs'] = clocks['srcs'].copy()
+        clocks['all_srcs'].update(clocks['derived_srcs'])
+        all_clocks = clocks['all_srcs']
+        clocks['groups'] = {
+            src["name"]: Group.fromdict(src, all_clocks) for src in clocks['groups']
+        }
+        c = cls.__new__(cls)
+        c.__dict__.update(clocks)
+        return c
+
     def add_clock_to_group(self, grp: Group, clk_name: str,
                            src_name: str) -> ClockSignal:
         src = self.all_srcs.get(src_name)
@@ -280,7 +356,7 @@ def get_clock_by_name(self, name: str) -> object:
             raise ValueError(f'{name} is not a valid clock')
         return ret
 
-    def reset_signals(self) -> List[str]:
+    def reset_signals(self) -> list[str]:
         '''Return the list of clock reset signal names.
 
         These signals are inputs to the clock manager (from the reset
@@ -364,7 +440,7 @@ def make_clock_to_group(self) -> Dict[str, Group]:
                 c2g[clk_name] = grp
         return c2g
 
-    def all_derived_srcs(self) -> List[str]:
+    def all_derived_srcs(self) -> list[str]:
         '''Return a list of all the clocks used as the source for derived clocks'''
 
         srcs = []

util/topgen/merge.py

@@ -10,7 +10,7 @@
 from typing import Dict, List, Union, Tuple
 
 from topgen import lib, secure_prng
-from .clocks import Clocks
+from .clocks import Clocks, GroupProxy
 from .resets import Resets
 from reggen.ip_block import IpBlock
 from reggen.params import LocalParam, Parameter, RandParameter, MemSizeParameter
@@ -875,8 +875,9 @@ def create_alert_lpgs(top, name_to_block: Dict[str, IpBlock]):
         else:
             clk = clk.split(".")[-1]
 
-            # Discover what clock group we are related to
-            clock_group = clock_groups[clk]
+            # Discover what clock group we are related to.
+            # We create a proxy so that it is only serialized by name.
+            clock_group = GroupProxy(clock_groups[clk])
 
             # using this info, we can create an LPG identifier
             # and uniquify it via a dict.
@@ -896,7 +897,7 @@ def create_alert_lpgs(top, name_to_block: Dict[str, IpBlock]):
                 'clock_group': None if unmanaged_clock else clock_group,
                 'clock_connection': clock,
                 'unmanaged_clock': unmanaged_clock,
-                'reset_connection': primary_reset
+                'reset_connection': primary_reset,
             })
             num_lpg += 1
 

util/topgen/resets.py

@@ -50,9 +50,16 @@ def _asdict(self) -> Dict[str, object]:
             'domains': self.domains,
             'shadowed': self.shadowed,
             'sw': self.sw,
-            'path': self.path
+            'path': self.path,
+            'shadow_path': self.shadow_path,
         }
 
+        if getattr(self, "shadow_lpg_path", None):
+            ret['shadow_lpg_path'] = self.shadow_lpg_path
+
+        if getattr(self, "lpg_path", None):
+            ret['lpg_path'] = self.lpg_path
+
         if self.parent:
             ret['parent'] = self.parent
 
@@ -61,6 +68,17 @@ def _asdict(self) -> Dict[str, object]:
 
         return ret
 
+    @classmethod
+    def fromdict(cls, item: Dict[str, object], clocks: Clocks) -> object:
+        # rst_type can be None which is serialized as "null", also the name is different
+        item["rst_type"] = None if item["type"] == "null" else item["type"]
+        del item["type"]
+        item["clock"] = clocks.get_clock_by_name(item["clock"])
+        item["parent"] = item.get("parent", "")
+        c = cls.__new__(cls)
+        c.__dict__.update(**item)
+        return c
+
 
 class Resets:
     '''Resets for the chip'''
@@ -86,6 +104,16 @@ def _asdict(self) -> Dict[str, object]:
 
         return ret
 
+    @classmethod
+    def fromdict(cls, resets: Dict[str, object], clocks: Clocks) -> object:
+        # Reconstruct dict.
+        resets['nodes'] = {
+            node["name"]: ResetItem.fromdict(node, clocks) for node in resets['nodes']
+        }
+        c = cls.__new__(cls)
+        c.__dict__.update(**resets)
+        return c
+
     def get_reset_by_name(self, name: str) -> ResetItem:
 
         ret = self.nodes.get(name, None)