improved inheritance

improved inheritance
centralized inheritance calls
remove extra import
fixed lint issues
improved yaml_merge_2_fusesoc_merge()
lint

fusesoc/capi2/inheritance.py

@@ -0,0 +1,39 @@
+# Copyright FuseSoC contributors
+# Licensed under the 2-Clause BSD License, see LICENSE for details.
+# SPDX-License-Identifier: BSD-2-Clause
+
+import copy
+import re
+
+from fusesoc import utils
+
+
+class Inheritance:
+    MERGE_OPERATOR = "<<__FUSESOC_MERGE_OVERLOAD__<<"
+
+    def yaml_merge_2_fusesoc_merge(capi):
+        """
+        Replace YAML merge key operator (<<) with FuseSoC merge operator
+        """
+        yaml_merge_pattern = (
+            r"((?:\n|{|\{\s*(?:[^{}]*\{[^{}]*\})*[^{}]*\},)\s*)<<(?=\s*:)"
+        )
+        while re.search(yaml_merge_pattern, capi):
+            capi = re.sub(yaml_merge_pattern, r"\1" + Inheritance.MERGE_OPERATOR, capi)
+        return capi
+
+    def elaborate_inheritance(capi):
+        if not isinstance(capi, dict):
+            return capi
+
+        for key, value in capi.items():
+            if isinstance(value, dict):
+                capi[key] = Inheritance.elaborate_inheritance(copy.deepcopy(value))
+
+        parent = capi.pop(Inheritance.MERGE_OPERATOR, {})
+        if isinstance(parent, dict):
+            capi = utils.merge_dict(parent, capi, concat_list_appends_only=True)
+        else:
+            raise SyntaxError("Invalid use of inheritance operator")
+
+        return capi

fusesoc/utils.py

@@ -16,6 +16,8 @@
     from yaml import SafeDumper as YamlDumper
     from yaml import SafeLoader as YamlLoader
 
+from fusesoc.capi2.inheritance import Inheritance
+
 logger = logging.getLogger(__name__)
 
 
@@ -159,15 +161,18 @@ def yaml_fread(filepath, resolve_env_vars=False, remove_preamble=False):
     with open(filepath) as f:
         if remove_preamble:
             f.readline()
-        return yaml_read(f, resolve_env_vars)
+        return yaml_read(f.read(), resolve_env_vars)
 
 
 def yaml_read(data, resolve_env_vars=False):
     try:
+        data = Inheritance.yaml_merge_2_fusesoc_merge(data)
+        capi_data = {}
         if resolve_env_vars:
-            return yaml.load(os.path.expandvars(data.read()), Loader=YamlLoader)
+            capi_data = yaml.load(os.path.expandvars(data), Loader=YamlLoader)
         else:
-            return yaml.load(data, Loader=YamlLoader)
+            capi_data = yaml.load(data, Loader=YamlLoader)
+        return Inheritance.elaborate_inheritance(capi_data)
     except (yaml.parser.ParserError, yaml.scanner.ScannerError) as e:
         raise SyntaxError(str(e))
 
@@ -176,11 +181,13 @@ def yaml_dump(data):
     return yaml.dump(data)
 
 
-def merge_dict(d1, d2):
+def merge_dict(d1, d2, concat_list_appends_only=False):
     for key, value in d2.items():
         if isinstance(value, dict):
             d1[key] = merge_dict(d1.get(key, {}), value)
-        elif isinstance(value, list):
+        elif isinstance(value, list) and (
+            not concat_list_appends_only or key.endswith("_append")
+        ):
             d1[key] = d1.get(key, []) + value
         else:
             d1[key] = value

tests/capi2_cores/parser/inheritance.core

@@ -0,0 +1,59 @@
+CAPI=2:
+# Copyright FuseSoC contributors
+# Licensed under the 2-Clause BSD License, see LICENSE for details.
+# SPDX-License-Identifier: BSD-2-Clause
+
+name: ::inheritance:0
+filesets:
+  fileset_a:
+    files:
+      - 1.txt
+      - 2.txt
+      - 3.txt
+  fileset_b:
+    files:
+      - 4.txt
+      - 5.txt
+      - 6.txt
+  fileset_c:
+    files:
+      - 7.txt
+      - 8.txt
+      - 9.txt
+
+targets:
+  default: &default
+    filesets:
+      - fileset_a
+  child: &child
+    <<: *default
+    filesets_append:
+      - fileset_b
+  grandchild: &grandchild
+    <<: *child
+    filesets_append:
+      - fileset_c
+  child2: &child2
+    <<: *default
+    filesets:
+      - fileset_b
+    filesets_append:
+      - fileset_c
+
+  subfield: &subfield
+    tools:
+      verilator:
+        mode: cc
+        verilator_options:
+          - --timing
+  subfield_child:
+    <<: *subfield
+    tools:
+      verilator:
+        mode: lint-only
+
+  merge_test1: {<<: *default}
+  merge_test2<<: {tools: {2<<: {}}}
+  <<merge_test3: {tools: {<<3: {}}}
+  merge_test4: {tools: {t4: {t44: <<4, <<: *default}}, <<: *default}
+  merge_test5: {<<__FUSESOC_MERGE_OVERLOAD__<<: *default}

tests/test_capi2.py

@@ -734,3 +734,57 @@ def test_syntax_error():
     with pytest.raises(SyntaxError) as excinfo:
         parser.read(core_file)
     assert "did not find expected node content" in str(excinfo.value)
+
+
+def test_inheritance():
+    import os
+
+    from fusesoc.capi2.coreparser import Core2Parser
+
+    core_file = os.path.join(tests_dir, "capi2_cores", "parser", "inheritance.core")
+
+    parser = Core2Parser()
+    assert parser.get_version() == 2
+    assert parser.get_preamble() == "CAPI=2:"
+
+    expected = {
+        "name": "::inheritance:0",
+        "filesets": {
+            "fileset_a": {"files": ["1.txt", "2.txt", "3.txt"]},
+            "fileset_b": {"files": ["4.txt", "5.txt", "6.txt"]},
+            "fileset_c": {"files": ["7.txt", "8.txt", "9.txt"]},
+        },
+        "targets": {
+            "default": {"filesets": ["fileset_a"]},
+            "child": {"filesets": ["fileset_a"], "filesets_append": ["fileset_b"]},
+            "grandchild": {
+                "filesets": ["fileset_a"],
+                "filesets_append": ["fileset_b", "fileset_c"],
+            },
+            "child2": {"filesets": ["fileset_b"], "filesets_append": ["fileset_c"]},
+            "subfield": {
+                "tools": {
+                    "verilator": {"mode": "cc", "verilator_options": ["--timing"]}
+                }
+            },
+            "subfield_child": {
+                "tools": {
+                    "verilator": {
+                        "mode": "lint-only",
+                        "verilator_options": ["--timing"],
+                    }
+                }
+            },
+            "merge_test1": {"filesets": ["fileset_a"]},
+            "merge_test2<<": {"tools": {"2<<": {}}},
+            "<<merge_test3": {"tools": {"<<3": {}}},
+            "merge_test4": {
+                "tools": {"t4": {"t44": "<<4", "filesets": ["fileset_a"]}},
+                "filesets": ["fileset_a"],
+            },
+            "merge_test5": {"filesets": ["fileset_a"]},
+        },
+    }
+
+    capi2_data = parser.read(core_file)
+    assert expected == capi2_data