refactor(codegen/docs): add type hints

.github/workflows/commit.yml

@@ -133,7 +133,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ ubuntu-latest, macos-latest, windows-latest ]
-        python-version: [ 3.8, 3.9, "3.10", "3.11", "3.12" ]
+        python-version: [ 3.9, "3.10", "3.11", "3.12" ]
     defaults:
       run:
         shell: bash -l {0}
@@ -181,10 +181,13 @@ jobs:
         working-directory: autotest
         run: |
           pytest -v -m="not example" -n=auto --cov=flopy --cov-append --cov-report=xml --durations=0 --keep-failed=.failed --dist loadfile
-          coverage report
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
+      - name: Report coverage
+        working-directory: autotest
+        run: coverage report
+
       - name: Upload failed test outputs
         uses: actions/upload-artifact@v4
         if: failure()

autotest/test_codegen.py

@@ -0,0 +1,162 @@
+import traceback
+from ast import Assign, ClassDef, expr
+from ast import parse as parse_ast
+from pprint import pformat
+from shutil import copytree
+from typing import List, Union
+from warnings import warn
+
+import pytest
+from modflow_devtools.misc import run_cmd
+
+from autotest.conftest import get_project_root_path
+from flopy.mf6.utils.codegen.context import get_context_names
+from flopy.mf6.utils.codegen.dfn import Dfn
+from flopy.mf6.utils.codegen.make import (
+    DfnName,
+    make_all,
+    make_context,
+    make_contexts,
+    make_targets,
+)
+
+PROJ_ROOT = get_project_root_path()
+MF6_PATH = PROJ_ROOT / "flopy" / "mf6"
+TGT_PATH = MF6_PATH / "modflow"
+DFN_PATH = MF6_PATH / "data" / "dfn"
+DFN_NAMES = [
+    dfn.stem
+    for dfn in DFN_PATH.glob("*.dfn")
+    if dfn.stem not in ["common", "flopy"]
+]
+
+
+@pytest.mark.parametrize(
+    "dfn, n_flat, n_params", [("gwf-ic", 2, 2), ("prt-prp", 40, 18)]
+)
+def test_make_context(dfn, n_flat, n_params):
+    with open(DFN_PATH / "common.dfn") as f:
+        commonvars = Dfn.load(f)
+
+    with open(DFN_PATH / f"{dfn}.dfn") as f:
+        dfn = DfnName(*dfn.split("-"))
+        definition = Dfn.load(f, name=dfn)
+
+    context_names = get_context_names(dfn)
+    context_name = context_names[0]
+    context = make_context(context_name, definition, commonvars)
+    assert len(context_names) == 1
+    assert len(context.variables) == n_params
+    assert len(context.definition.metadata) == n_flat + 1  # +1 for metadata
+
+
+@pytest.mark.skip(reason="TODO")
+@pytest.mark.parametrize("dfn_name", ["gwf-ic", "prt-prp", "gwf-nam"])
+def test_make_contexts(dfn_name):
+    with open(DFN_PATH / "common.dfn") as f:
+        common = Dfn.load(f)
+
+    # TODO
+
+
+@pytest.mark.parametrize("dfn_name", DFN_NAMES)
+def test_make_targets(dfn_name, function_tmpdir):
+    with open(DFN_PATH / "common.dfn") as f:
+        common = Dfn.load(f)
+
+    with open(DFN_PATH / f"{dfn_name}.dfn", "r") as f:
+        dfn_name = DfnName(*dfn_name.split("-"))
+        dfn = Dfn.load(f, name=dfn_name)
+
+    make_targets(dfn, function_tmpdir, common=common)
+    for ctx_name in get_context_names(dfn_name):
+        run_cmd("ruff", "format", function_tmpdir, verbose=True)
+        run_cmd("ruff", "check", "--fix", function_tmpdir, verbose=True)
+        assert (function_tmpdir / ctx_name.target).is_file()
+
+
+def test_make_all(function_tmpdir):
+    make_all(DFN_PATH, function_tmpdir, verbose=True)
+
+
+def compare_ast(
+    node1: Union[expr, List[expr]], node2: Union[expr, List[expr]]
+) -> bool:
+    t1 = type(node1)
+    t2 = type(node2)
+    if t1 is not t2:
+        print(f"type mismatch: {t1} != {t2}")
+        return False
+
+    if t1 is ClassDef:
+        assert t2 is ClassDef
+        assert node1.name == node2.name
+        for base1, base2 in zip(node1.bases, node2.bases):
+
+            def _id(b):
+                attrs = ["id", "name", "attr"]
+                for attr in attrs:
+                    try:
+                        return getattr(b, attr)
+                    except:
+                        pass
+                return None
+
+            assert _id(base1) == _id(base2)
+
+        body1, body2 = node1.body, node2.body
+        assert len(body1) == len(body2), f"body mismatch in {node1.name}"
+
+        for b1, b2 in zip(body1, body2):
+            if isinstance(b1, Assign):
+                assert isinstance(b2, Assign)
+                b1tgts = set(sorted([t.id for t in b1.targets]))
+                b2tgts = set(sorted([t.id for t in b2.targets]))
+                diff = b1tgts ^ b2tgts
+                if any(diff):
+                    warn(
+                        f"assignment targets don't match in {node1.name}\n"
+                        f"=> symmetric difference:\n{pformat(diff)}\n"
+                        f"=> prev - test:\n{pformat(b1tgts - b2tgts)}\n"
+                        f"=> test - prev:\n{pformat(b2tgts - b1tgts)}\n"
+                    )
+
+
+def test_equivalence(function_tmpdir):
+    prev_dir = function_tmpdir / "prev"
+    test_dir = function_tmpdir / "test"
+    test_dir.mkdir()
+    copytree(TGT_PATH, prev_dir)
+    make_all(DFN_PATH, test_dir, verbose=True)
+    prev_files = list(prev_dir.glob("*.py"))
+    test_files = list(test_dir.glob("*.py"))
+    prev_names = set([p.name for p in prev_files])
+    test_names = set([p.name for p in test_files])
+    diff = prev_names ^ test_names
+    assert not any(diff), (
+        f"previous files don't match test files\n"
+        f"=> symmetric difference:\n{pformat(diff)}\n"
+        f"=> prev - test:\n{pformat(prev_names - test_names)}\n"
+        f"=> test - prev:\n{pformat(test_names - prev_names)}\n"
+    )
+    for prev_file, test_file in zip(prev_files, test_files):
+        prev = parse_ast(open(prev_file).read())
+        try:
+            test = parse_ast(open(test_file).read())
+        except:
+            raise ValueError(
+                f"Failed to parse {test_file}: {traceback.format_exc()}"
+            )
+        prev_classes = [n for n in prev.body if isinstance(n, ClassDef)]
+        test_classes = [n for n in test.body if isinstance(n, ClassDef)]
+        prev_clsnames = set([c.name for c in prev_classes])
+        test_clsnames = set([c.name for c in test_classes])
+        diff = prev_clsnames ^ test_clsnames
+        assert not any(diff), (
+            f"previous classes don't match test classes in {test_file.name}\n"
+            f"=> symmetric difference:\n{pformat(diff)}\n"
+            f"=> prev - test:\n{pformat(prev_clsnames - test_clsnames)}\n"
+            f"=> test - prev:\n{pformat(test_clsnames - prev_clsnames)}\n"
+        )
+        for prev_cls, test_cls in zip(prev_classes, test_classes):
+            compare_ast(prev_cls, test_cls)

autotest/test_dfn.py

@@ -0,0 +1,22 @@
+import pytest
+
+from autotest.conftest import get_project_root_path
+from flopy.mf6.utils.codegen.dfn import Dfn
+from flopy.mf6.utils.codegen.make import DfnName
+
+PROJ_ROOT = get_project_root_path()
+MF6_PATH = PROJ_ROOT / "flopy" / "mf6"
+TGT_PATH = MF6_PATH / "modflow"
+DFN_PATH = MF6_PATH / "data" / "dfn"
+DFN_NAMES = [
+    dfn.stem
+    for dfn in DFN_PATH.glob("*.dfn")
+    if dfn.stem not in ["common", "flopy"]
+]
+
+
+@pytest.mark.parametrize("dfn_name", DFN_NAMES)
+def test_load_dfn(dfn_name):
+    dfn_path = DFN_PATH / f"{dfn_name}.dfn"
+    with open(dfn_path, "r") as f:
+        dfn = Dfn.load(f, name=DfnName(*dfn_name.split("-")))

docs/mf6_dev_guide.md

@@ -10,38 +10,44 @@ FPMF6 uses meta-data files located in flopy/mf6/data/dfn to define the model and
 
 All meta-data can be accessed from the flopy.mf6.data.mfstructure.MFStructure class.  This is a singleton class, meaning only one instance of this class can be created.  The class contains a sim_struct attribute (which is a flopy.mf6.data.mfstructure.MFSimulationStructure object) which contains all of the meta-data for all package files.  Meta-data is stored in a structured format. MFSimulationStructure contains MFModelStructure and MFInputFileStructure objects, which contain the meta-data for each model type and each "simulation-level" package (tdis, ims, ...).  MFModelStructure contains model specific meta-data and a MFInputFileStructure object for each package in that model.  MFInputFileStructure contains package specific meta-data and a MFBlockStructure object for each block contained in the package file.  MFBlockStructure contains block specific meta-data and a MFDataStructure object for each data structure defined in the block, and MFDataStructure contains data structure specific meta-data and a MFDataItemStructure object for each data item contained in the data structure.  Data structures define the structure of data that is naturally grouped together, for example, the data in a numpy recarray.  Data item structures define the structure of specific pieces of data, for example, a single column of a numpy recarray.  The meta-data defined in these classes provides all the information FloPy needs to read and write MODFLOW 6 package and name files, create the Flopy interface, and check the data for various constraints.
 
-
-***
-MFStructure --+ MFSimulationStructure --+ MFModelStructure --+ MFInputFileStructure --+ MFBlockStructure --+ MFDataStructure --+ MFDataItemStructure
-
-Figure 1: FPMF6 generic data structure classes.  Lines connecting classes show a relationship defined between the two connected classes.  A "*" next to the class means that the  class is a sub-class of the connected class.  A "+" next to the class means that the class is contained within the connected class.
-***
+```mermaid
+classDiagram
+    MFStructure --* "1" MFSimulationStructure : has
+    MFSimulationStructure --* "1+" MFModelStructure : has
+    MFModelStructure --* "1" MFInputFileStructure : has
+    MFInputFileStructure --* "1+" MFBlockStructure : has
+    MFBlockStructure --* "1+" MFDataStructure : has
+    MFDataStructure --* "1+" MFDataItemStructure : has
+```
+
+Figure 1: Generic data structure hierarchy.  Connections show composition relationships.
 
 Package and Data Base Classes
 -----------------------------------------------
 
 The package and data classes are related as shown below in figure 2.  On the top of the figure 2 is the MFPackage class, which is the base class for all packages.  MFPackage contains generic methods for building data objects and reading and writing the package to a file.  MFPackage contains a MFInputFileStructure object that defines how the data is structured in the package file.  MFPackage also contains a dictionary of blocks (MFBlock).  The MFBlock class is a generic class used to represent a block within a package.  MFBlock contains a MFBlockStructure object that defines how the data in the block is structured.  MFBlock also contains a dictionary of data objects (subclasses of MFData) contained in the block and a list of block headers (MFBlockHeader) for that block.  Block headers contain the block's name and optionally data items (eg. iprn).
 
 
-***
-MFPackage --+ MFBlock --+ MFData
-
-MFPackage --+ MFInputFileStructure
-
-MFBlock --+ MFBlockStructure
-
-MFData --+ MFDataStructure
-
-MFData --* MFArray --* MFTransientArray
-
-MFData --* MFList --* MFTransientList
-
-MFData --* MFScalar --* MFTransientScalar
-
-MFTransientData --* MFTransientArray, MFTransientList, MFTransientScalar
+```mermaid
+classDiagram
+
+MFPackage --* "1+" MFBlock : has
+MFBlock --* "1+" MFData : has
+MFPackage --* "1" MFInputFileStructure : has
+MFBlock --* "1" MFBlockStructure : has
+MFData --* "1" MFDataStructure : has
+MFData --|> MFArray
+MFArray --|> MFTransientArray
+MFData --|> MFList
+MFList --|> MFTransientList
+MFData --|> MFScalar
+MFScalar --|> MFTransientScalar
+MFTransientData --|> MFTransientArray
+MFTransientData --|> MFTransientList
+MFTransientData --|> MFTransientScalar
+```
 
 Figure 2:  FPMF6 package and data classes.  Lines connecting classes show a relationship defined between the two connected classes.  A "*" next to the class means that the  class is a sub-class of the connected class.  A "+" next to the class means that the class is contained within the connected class.
-***
 
 There are three main types of data, MFList, MFArray, and MFScalar data.  All three of these data types are derived from the MFData abstract base class.  MFList data is the type of data stored in a spreadsheet with different column headings.  For example, the data describing a flow barrier are of type MFList.  MFList data is stored in numpy recarrays.  MFArray data is data of a single type (eg. all integer values).  For example, the model's HK values are of type MFArray.  MFArrays are stored in numpy ndarrays.  MFScalar data is a single data item.  Most MFScalar data are options.  All MFData subclasses contain an MFDataStructure object that defines the expected structure and types of the data.
 

flopy/mf6/data/dfn/exg-gwfgwf.dfn

@@ -61,6 +61,7 @@ name cvoptions
 type record variablecv dewatered
 reader urword
 optional true
+class_attr false
 longname vertical conductance options
 description none
 

flopy/mf6/data/dfn/gwe-lke.dfn

@@ -442,7 +442,7 @@ description real or character value that defines the temperature of external inf
 
 block period
 name auxiliaryrecord
-type record auxiliary auxname auxval
+type record aux auxname auxval
 shape
 tagged
 in_record true
@@ -451,7 +451,7 @@ longname
 description
 
 block period
-name auxiliary
+name aux
 type keyword
 shape
 in_record true

flopy/mf6/data/dfn/gwe-mwe.dfn

@@ -408,7 +408,7 @@ description real or character value that defines the injection solute temperatur
 
 block period
 name auxiliaryrecord
-type record auxiliary auxname auxval
+type record aux auxname auxval
 shape
 tagged
 in_record true
@@ -417,7 +417,7 @@ longname
 description
 
 block period
-name auxiliary
+name aux
 type keyword
 shape
 in_record true

flopy/mf6/data/dfn/gwe-sfe.dfn

@@ -441,7 +441,7 @@ description real or character value that defines the temperature of inflow $(^{\
 
 block period
 name auxiliaryrecord
-type record auxiliary auxname auxval
+type record aux auxname auxval
 shape
 tagged
 in_record true
@@ -450,7 +450,7 @@ longname
 description
 
 block period
-name auxiliary
+name aux
 type keyword
 shape
 in_record true

flopy/mf6/data/dfn/gwe-uze.dfn

@@ -399,7 +399,7 @@ description real or character value that states what fraction of the simulated u
 
 block period
 name auxiliaryrecord
-type record auxiliary auxname auxval
+type record aux auxname auxval
 shape
 tagged
 in_record true
@@ -408,7 +408,7 @@ longname
 description
 
 block period
-name auxiliary
+name aux
 type keyword
 shape
 in_record true

flopy/mf6/data/dfn/gwf-lak.dfn

@@ -845,7 +845,7 @@ description real or character value that defines the bed slope for the lake outl
 
 block period
 name auxiliaryrecord
-type record auxiliary auxname auxval
+type record aux auxname auxval
 shape
 tagged
 in_record true
@@ -854,7 +854,7 @@ longname
 description
 
 block period
-name auxiliary
+name aux
 type keyword
 shape
 in_record true

flopy/mf6/data/dfn/gwf-maw.dfn

@@ -723,7 +723,7 @@ description height above the pump elevation (SCALING\_LENGTH).  If the simulated
 
 block period
 name auxiliaryrecord
-type record auxiliary auxname auxval
+type record aux auxname auxval
 shape
 tagged
 in_record true
@@ -732,7 +732,7 @@ longname
 description
 
 block period
-name auxiliary
+name aux
 type keyword
 shape
 in_record true