Merge pull request #1931 from apache/tristan/junction-source-mirrors

Implement `pip` and `junction` origin loading for SourceMirror plugins

src/buildstream/_frontend/app.py

@@ -282,6 +282,7 @@ def initialized(self, *, session_name=None):
                     self.context,
                     cli_options=self._main_options["option"],
                     default_mirror=self._main_options.get("default_mirror"),
+                    fetch_subprojects=self.stream.fetch_subprojects,
                 )
             except LoadError as e:
 

src/buildstream/_pluginfactory/pluginoriginjunction.py

@@ -41,6 +41,8 @@ def get_plugin_paths(self, kind, plugin_type):
             factory = project.source_factory
         elif plugin_type == PluginType.ELEMENT:
             factory = project.element_factory
+        elif plugin_type == PluginType.SOURCE_MIRROR:
+            factory = project.source_mirror_factory
         else:
             assert False, "unreachable"
 
@@ -66,7 +68,7 @@ def get_plugin_paths(self, kind, plugin_type):
             # subproject.
             #
             raise PluginError(
-                "{}: project '{}' referred to by junction '{}' does not declare any {} plugin kind: '{}'".format(
+                "{}: project '{}' referred to by junction '{}' does not declare a {} plugin named: '{}'".format(
                     self.provenance_node.get_provenance(), project.name, self._junction, plugin_type, kind
                 ),
                 reason="junction-plugin-not-found",

src/buildstream/_pluginfactory/pluginoriginpip.py

@@ -48,6 +48,8 @@ def get_plugin_paths(self, kind, plugin_type):
             entrypoint_group = "buildstream.plugins.sources"
         elif plugin_type == PluginType.ELEMENT:
             entrypoint_group = "buildstream.plugins.elements"
+        elif plugin_type == PluginType.SOURCE_MIRROR:
+            entrypoint_group = "buildstream.plugins.sourcemirrors"
         else:
             assert False, "unreachable"
 
@@ -84,8 +86,8 @@ def get_plugin_paths(self, kind, plugin_type):
 
         if package is None:
             raise PluginError(
-                "{}: Pip package {} does not contain a plugin named '{}'".format(
-                    self.provenance_node.get_provenance(), self._package_name, kind
+                "{}: Pip package {} does not contain a {} plugin named '{}'".format(
+                    self.provenance_node.get_provenance(), self._package_name, plugin_type, kind
                 ),
                 reason="plugin-not-found",
             )

src/buildstream/_project.py

@@ -90,6 +90,7 @@ def __init__(
         parent_loader: Optional[Loader] = None,
         provenance_node: Optional[ProvenanceInformation] = None,
         search_for_project: bool = True,
+        fetch_subprojects=None
     ):
         #
         # Public members
@@ -161,6 +162,7 @@ def __init__(
             self.load_context = parent_loader.load_context
         else:
             self.load_context = LoadContext(self._context)
+            self.load_context.set_fetch_subprojects(fetch_subprojects)
 
         if search_for_project:
             self.directory, self._invoked_from_workspace_element = self._find_project_dir(directory)
@@ -1100,6 +1102,9 @@ def _load_pass(self, config, output, *, ignore_unknown=False):
         # Override default_mirror if not set by command-line
         output.default_mirror = self._default_mirror or overrides.get_str("default-mirror", default=None)
 
+        # Source url aliases
+        output._aliases = config.get_mapping("aliases", default={})
+
         # First try mirrors specified in user configuration, user configuration
         # is allowed to completely disable mirrors by specifying an empty list,
         # so we check for a None value here too.
@@ -1121,9 +1126,6 @@ def _load_pass(self, config, output, *, ignore_unknown=False):
             if not output.default_mirror:
                 output.default_mirror = mirror.name
 
-        # Source url aliases
-        output._aliases = config.get_mapping("aliases", default={})
-
         # Perform variable substitutions in source aliases
         variables.expand(output._aliases)
 

src/buildstream/_stream.py

@@ -127,8 +127,6 @@ def cleanup(self):
     def set_project(self, project):
         assert self._project is None
         self._project = project
-        if self._project:
-            self._project.load_context.set_fetch_subprojects(self._fetch_subprojects)
 
     # load_selection()
     #
@@ -1323,6 +1321,22 @@ def retry_job(self, action_name: str, unique_id: str) -> None:
         assert queue
         queue.enqueue([element])
 
+    # fetch_subprojects()
+    #
+    # Fetch subprojects as part of the project and element loading process.
+    #
+    # This is passed to the Project in order to handle loading of subprojects
+    #
+    # Args:
+    #    junctions (list of Element): The junctions to fetch
+    #
+    def fetch_subprojects(self, junctions):
+        self._reset()
+        queue = FetchQueue(self._scheduler)
+        queue.enqueue(junctions)
+        self.queues = [queue]
+        self._run()
+
     #############################################################
     #                    Private Methods                        #
     #############################################################
@@ -1343,20 +1357,6 @@ def _assert_project(self, message: str) -> None:
                 message, detail="No project.conf or active workspace was located", reason="project-not-loaded"
             )
 
-    # _fetch_subprojects()
-    #
-    # Fetch subprojects as part of the project and element loading process.
-    #
-    # Args:
-    #    junctions (list of Element): The junctions to fetch
-    #
-    def _fetch_subprojects(self, junctions):
-        self._reset()
-        queue = FetchQueue(self._scheduler)
-        queue.enqueue(junctions)
-        self.queues = [queue]
-        self._run()
-
     # _load_artifacts()
     #
     # Loads artifacts from target artifact refs

tests/frontend/mirror.py

@@ -26,6 +26,7 @@
 from buildstream._testing import cli  # pylint: disable=unused-import
 
 from tests.testutils.repo.git import Git
+from tests.testutils.repo.tar import Tar
 from tests.testutils.site import pip_sample_packages  # pylint: disable=unused-import
 from tests.testutils.site import SAMPLE_PACKAGES_SKIP_REASON
 
@@ -817,7 +818,8 @@ def test_mirror_expand_project_and_toplevel_root(cli, tmpdir):
 #
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.usefixtures("datafiles")
-def test_source_mirror_plugin(cli, tmpdir):
+@pytest.mark.parametrize("origin", [("local"), ("junction"), ("pip")])
+def test_source_mirror_plugin(cli, tmpdir, origin):
     output_file = os.path.join(str(tmpdir), "output.txt")
     project_dir = str(tmpdir)
     element_dir = os.path.join(project_dir, "elements")
@@ -827,6 +829,36 @@ def test_source_mirror_plugin(cli, tmpdir):
     element = generate_element(output_file)
     _yaml.roundtrip_dump(element, element_path)
 
+    def source_mirror_plugin_origin():
+        if origin == "local":
+            return {"origin": "local", "path": "sourcemirrors", "source-mirrors": ["mirror"]}
+        elif origin == "pip":
+            return {
+                "origin": "pip",
+                "package-name": "sample-plugins>=1.2",
+                "source-mirrors": ["mirror"],
+            }
+        elif origin == "junction":
+            # For junction loading, just copy in the sample-plugins into a subdir and
+            # create a local junction
+            sample_plugins_dir = os.path.join(TOP_DIR, "..", "plugins", "sample-plugins")
+            sample_plugins_copy_dir = os.path.join(project_dir, "sample-plugins-copy")
+            junction_file = os.path.join(element_dir, "sample-plugins.bst")
+
+            shutil.copytree(sample_plugins_dir, sample_plugins_copy_dir)
+
+            _yaml.roundtrip_dump(
+                {"kind": "junction", "sources": [{"kind": "local", "path": "sample-plugins-copy"}]}, junction_file
+            )
+
+            return {
+                "origin": "junction",
+                "junction": "sample-plugins.bst",
+                "source-mirrors": ["mirror"],
+            }
+        else:
+            assert False
+
     project_file = os.path.join(project_dir, "project.conf")
     project = {
         "name": "test",
@@ -870,7 +902,7 @@ def test_source_mirror_plugin(cli, tmpdir):
         ],
         "plugins": [
             {"origin": "local", "path": "sources", "sources": ["fetch_source"]},
-            {"origin": "local", "path": "sourcemirrors", "source-mirrors": ["mirror"]},
+            source_mirror_plugin_origin(),
         ],
     }
 
@@ -1098,3 +1130,107 @@ def map_alias_valid(x):
                 assert "Fetch RAB/repo2 succeeded from RAB/repo2" in contents
             else:
                 assert "Fetch bar:repo2 succeeded from RAB/repo2" in contents
+
+
+# Test the behavior of loading a SourceMirror plugin across a junction,
+# when the cross junction SourceMirror object has a mirror.
+#
+# Check what happens when the mirror does not need to be exercized (success)
+#
+# Check what happens when the mirror needs to be exercised in order to obtain
+# the mirror plugin itself (failure) and check the failure mode.
+#
+#
+@pytest.mark.parametrize("fetch_source", [("all"), ("mirrors")], ids=["normal", "circular"])
+def test_source_mirror_circular_junction(cli, tmpdir, fetch_source):
+    project_dir = str(tmpdir)
+    element_dir = os.path.join(project_dir, "elements")
+    os.makedirs(element_dir, exist_ok=True)
+
+    cli.configure({"fetch": {"source": fetch_source}})
+
+    # Generate a 2 tar repos with the sample plugins
+    #
+    sample_plugins_dir = os.path.join(TOP_DIR, "..", "plugins", "sample-plugins")
+    base_sample_plugins_repodir = os.path.join(str(tmpdir), "base_sample_plugins")
+    base_sample_plugins_repo = Tar(base_sample_plugins_repodir)
+    base_sample_plugins_ref = base_sample_plugins_repo.create(sample_plugins_dir)
+    mirror_sample_plugins_repodir = os.path.join(str(tmpdir), "mirror_sample_plugins")
+    mirror_sample_plugins_repo = Tar(mirror_sample_plugins_repodir)
+
+    # Don't expect determinism from python tar, just copy over the Tar repo file
+    # and we need to use the same ref for both.
+    shutil.copyfile(
+        os.path.join(base_sample_plugins_repo.repo, "file.tar.gz"),
+        os.path.join(mirror_sample_plugins_repo.repo, "file.tar.gz"),
+    )
+
+    # Generate junction for sample plugins
+    #
+    sample_plugins_junction = {
+        "kind": "junction",
+        "sources": [
+            {
+                "kind": "tar",
+                "url": "samplemirror:file.tar.gz",
+                "ref": base_sample_plugins_ref,
+            }
+        ],
+    }
+    element_path = os.path.join(element_dir, "sample-plugins.bst")
+    _yaml.roundtrip_dump(sample_plugins_junction, element_path)
+
+    # Generate project.conf
+    #
+    project_file = os.path.join(project_dir, "project.conf")
+    project = {
+        "name": "test",
+        "min-version": "2.0",
+        "element-path": "elements",
+        "aliases": {
+            "samplemirror": "file://" + base_sample_plugins_repo.repo + "/",
+        },
+        "mirrors": [
+            {
+                "name": "alternative",
+                "kind": "mirror",
+                "config": {
+                    "aliases": {
+                        "samplemirror": ["file://" + mirror_sample_plugins_repo.repo + "/"],
+                    },
+                },
+            },
+        ],
+        "plugins": [
+            {"origin": "junction", "junction": "sample-plugins.bst", "source-mirrors": ["mirror"]},
+        ],
+    }
+    _yaml.roundtrip_dump(project, project_file)
+
+    # Make a silly element
+    element = {"kind": "import", "sources": [{"kind": "local", "path": "project.conf"}]}
+    element_path = os.path.join(element_dir, "test.bst")
+    _yaml.roundtrip_dump(element, element_path)
+
+    result = cli.run(project=project_dir, args=["show", "test.bst"])
+
+    if fetch_source == "all":
+        result.assert_success()
+    elif fetch_source == "mirrors":
+        #
+        # This error looks like this:
+        #
+        #   Error loading project: tar source at sample-plugins.bst [line 3 column 2]: No fetch URI found for alias 'samplemirror'
+        #
+        #       Check fetch controls in your user configuration
+        #
+        # This is not 100% ideal, as we could theoretically have Source.mark_download_url() detect
+        # the case that we are currently instantiating the specific SourceMirror plugin required
+        # to resolve the URL needed to obtain the same said SourceMirror plugin, and report
+        # something about this being a circular dependency error.
+        #
+        # However, this would be fairly complex to reason about in the code, especially considering
+        # the source alias redirects, and the possibility that a subproject's source mirror is being
+        # redirected to a parent project's aliases and corresponding mirrors.
+        #
+        result.assert_main_error(ErrorDomain.SOURCE, "missing-source-alias-target")

tests/plugins/sample-plugins/project.conf

@@ -13,3 +13,8 @@ plugins:
   sources:
   - git
   - sample
+
+- origin: local
+  path: src/sample_plugins/sourcemirrors
+  source-mirrors:
+  - mirror

tests/plugins/sample-plugins/setup.py

@@ -33,6 +33,9 @@
             "sample = sample_plugins.sources.sample",
             "git = sample_plugins.sources.git",
         ],
+        "buildstream.plugins.sourcemirrors": [
+            "mirror = sample_plugins.sourcemirrors.mirror",
+        ],
     },
     zip_safe=False,
 )

tests/plugins/sample-plugins/src/sample_plugins/sourcemirrors/mirror.py

@@ -0,0 +1,36 @@
+from typing import Optional, Dict, Any
+
+from buildstream import SourceMirror, MappingNode
+
+
+# This mirror plugin basically implements the default behavior
+# by loading the alias definitions as custom "config" configuration
+# instead, and implementing the translate_url method.
+#
+class Sample(SourceMirror):
+    def configure(self, node):
+        node.validate_keys(["aliases"])
+
+        self.aliases = {}
+
+        aliases = node.get_mapping("aliases")
+        for alias_name, url_list in aliases.items():
+            self.aliases[alias_name] = url_list.as_str_list()
+
+        self.set_supported_aliases(self.aliases.keys())
+
+    def translate_url(
+        self,
+        *,
+        alias: str,
+        alias_url: str,
+        source_url: str,
+        extra_data: Optional[Dict[str, Any]],
+    ) -> str:
+        return self.aliases[alias][0] + source_url
+
+
+# Plugin entry point
+def setup():
+
+    return Sample