fix: compile ABI-based dependencies at better time (#130)

.pre-commit-config.yaml

@@ -10,18 +10,18 @@ repos:
       - id: isort
 
 -   repo: https://github.com/psf/black
-    rev: 24.4.2
+    rev: 24.8.0
     hooks:
       - id: black
         name: black
 
 -   repo: https://github.com/pycqa/flake8
-    rev: 7.1.0
+    rev: 7.1.1
     hooks:
     -   id: flake8
 
 -   repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.11.0
+    rev: v1.11.2
     hooks:
     -   id: mypy
         additional_dependencies: [types-setuptools, pydantic==1.10.4]

ape_vyper/compiler.py

@@ -492,7 +492,25 @@ def _get_imports(
                     dep_key = prefix.split(os.path.sep)[0]
                     dependency_name = prefix.split(os.path.sep)[0]
                     filestem = prefix.replace(f"{dependency_name}{os.path.sep}", "")
-                    if dep_key in dependencies:
+                    found = False
+                    if dependency_name:
+                        # Attempt looking up dependency from site-packages.
+                        if res := _lookup_source_from_site_packages(dependency_name, filestem):
+                            source_path, imported_project = res
+                            import_source_id = str(source_path)
+                            # Also include imports of imports.
+                            sub_imports = self._get_imports(
+                                (source_path,),
+                                project=imported_project,
+                                handled=handled,
+                            )
+                            for sub_import_ls in sub_imports.values():
+                                import_map[source_id].extend(sub_import_ls)
+
+                            is_local = False
+                            found = True
+
+                    if not found and dep_key in dependencies:
                         for version_str, dep_project in pm.dependencies[dependency_name].items():
                             dependency = pm.dependencies.get_dependency(
                                 dependency_name, version_str
@@ -502,9 +520,29 @@ def _get_imports(
                             dependency_source_prefix = (
                                 f"{get_relative_path(contracts_path, dep_project.path)}"
                             )
-                            source_id_stem = f"{dependency_source_prefix}{os.path.sep}{filestem}"
-                            for ext in (".vy", ".json"):
+                            source_id_stem = (
+                                f"{dependency_source_prefix}{os.path.sep}{filestem}".lstrip(
+                                    f"{os.path.sep}."
+                                )
+                            )
+                            for ext in (".vy", ".vyi", ".json"):
                                 if f"{source_id_stem}{ext}" in dep_project.sources:
+                                    # Dependency located.
+                                    if not dependency.project.manifest.contract_types:
+                                        # In this case, the dependency *must* be compiled
+                                        # so the ABIs can be found later on.
+                                        try:
+                                            dependency.compile()
+                                        except Exception as err:
+                                            # Compiling failed. Try to continue anyway to get
+                                            # a better error from the Vyper compiler, in case
+                                            # something else is wrong.
+                                            logger.warning(
+                                                f"Failed to compile dependency '{dependency.name}' "
+                                                f"@ '{dependency.version}'.\n"
+                                                f"Reason: {err}"
+                                            )
+
                                     import_source_id = os.path.sep.join(
                                         (path_id, version_str, f"{source_id_stem}{ext}")
                                     )
@@ -520,21 +558,6 @@ def _get_imports(
 
                                     is_local = False
                                     break
-                    elif dependency_name:
-                        # Attempt looking up dependency from site-packages.
-                        if res := _lookup_source_from_site_packages(dependency_name, filestem):
-                            source_path, imported_project = res
-                            import_source_id = str(source_path)
-                            # Also include imports of imports.
-                            sub_imports = self._get_imports(
-                                (source_path,),
-                                project=imported_project,
-                                handled=handled,
-                            )
-                            for sub_import_ls in sub_imports.values():
-                                import_map[source_id].extend(sub_import_ls)
-
-                            is_local = False
 
                 if is_local and local_prefix is not None and local_path is not None:
                     import_source_id = f"{local_prefix}{ext}"
@@ -668,16 +691,6 @@ def get_dependencies(
                 continue
 
             handled.add(dep_id)
-
-            try:
-                dependency.compile()
-            except Exception as err:
-                logger.warning(
-                    f"Failed to compile dependency '{dependency.name}' @ '{dependency.version}'.\n"
-                    f"Reason: {err}"
-                )
-                continue
-
             dependencies[remapping.key] = dependency.project
 
         # Add auto-remapped dependencies.
@@ -689,16 +702,6 @@ def get_dependencies(
                 continue
 
             handled.add(dep_id)
-
-            try:
-                dependency.compile()
-            except Exception as err:
-                logger.warning(
-                    f"Failed to compile dependency '{dependency.name}' @ '{dependency.version}'.\n"
-                    f"Reason: {err}"
-                )
-                continue
-
             dependencies[dependency.name] = dependency.project
 
         return dependencies

setup.py

@@ -11,10 +11,10 @@
         "snekmate",  # Python package-sources integration testing
     ],
     "lint": [
-        "black>=24.4.2,<25",  # Auto-formatter and linter
-        "mypy>=1.11.0,<2",  # Static type analyzer
+        "black>=24.8.0,<25",  # Auto-formatter and linter
+        "mypy>=1.11.2,<2",  # Static type analyzer
         "types-setuptools",  # Needed due to mypy typeshed
-        "flake8>=7.1.0,<8",  # Style linter
+        "flake8>=7.1.1,<8",  # Style linter
         "isort>=5.13.2",  # Import sorting linter
         "mdformat>=0.7.17",  # Auto-formatter for markdown
         "mdformat-gfm>=0.3.5",  # Needed for formatting GitHub-flavored markdown

tests/ape-config.yaml

@@ -5,3 +5,10 @@ contracts_folder: contracts/passing_contracts
 dependencies:
   - name: exampledependency
     local: ./ExampleDependency
+
+  # NOTE: Snekmate does not need to be listed here since
+  # it is installed in site-packages. However, we include it
+  # to show it doesn't cause problems when included.
+  - python: snekmate
+    config_override:
+      contracts_folder: .

tests/test_compiler.py

@@ -378,10 +378,21 @@ def test_compile_parse_dev_messages(compiler, dev_revert_source, project):
 
 
 def test_get_imports(compiler, project):
+    # Ensure the dependency starts off un-compiled so we can show this
+    # is the point at which it will be compiled. We make sure to only
+    # compile when we know it is a JSON interface based dependency
+    # and not a site-package or relative-path based dependency.
+    dependency = project.dependencies["exampledependency"]["local"]
+    dependency.manifest.contract_types = {}
+
     vyper_files = [
         x for x in project.contracts_folder.iterdir() if x.is_file() and x.suffix == ".vy"
     ]
     actual = compiler.get_imports(vyper_files, project=project)
+
+    # The dependency should have gotten compiled!
+    assert dependency.manifest.contract_types
+
     prefix = "contracts/passing_contracts"
     builtin_import = "vyper/interfaces/ERC20.json"
     local_import = "IFace.vy"
@@ -745,3 +756,19 @@ def test_flatten_contract_04(project, compiler):
     version = compiler._source_vyper_version(source_code)
     vvm.install_vyper(str(version))
     vvm.compile_source(source_code, base_path=project.path, vyper_version=version)
+
+
+def test_get_import_remapping(project, compiler):
+    dependency = project.dependencies["exampledependency"]["local"]
+    dependency.manifest.contract_types = {}
+
+    # Getting import remapping does not compile on its own!
+    # This is important because we don't necessarily want to
+    # compile every dependency, only the ones with imports
+    # that indicate this.
+    actual = compiler.get_import_remapping(project=project)
+    assert actual == {}
+
+    dependency.load_contracts()
+    actual = compiler.get_import_remapping(project=project)
+    assert "exampledependency/Dependency.json" in actual