Merge pull request #6 from jdoiro3/dev

Imported Repos Development Feature

.github/workflows/test.yml

@@ -10,23 +10,22 @@ on:
 
 jobs:
   test:
-    name: Test on ${{ matrix.os }} with Python ${{ matrix.python-version }}
-    runs-on: ${{ matrix.os }}
+      name: Test on Python ${{ matrix.python-version }}
+      runs-on: ubuntu-latest
 
-    strategy:
-      matrix:
-        os: [ubuntu-latest, windows-latest, macOS-latest]
-        python-version: [3, 3.6, 3.7]
+      strategy:
+        matrix:
+          python-version: [3, 3.6, 3.7]
 
-    steps:
-      - uses: actions/checkout@master
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@master
-        with:
-          python-version: ${{ matrix.python-version }}
-      - name: Install Bats
-        run: sudo ./__tests__/setup-bats.sh
-      - name: Install Python dependencies
-        run: sudo ./__tests__/setup-pip.sh
-      - name: Linter, Integration and Unit Tests
-        run: sudo ./__tests__/test-ci.sh
+      steps:
+        - uses: actions/checkout@master
+        - name: Set up Python ${{ matrix.python-version }}
+          uses: actions/setup-python@master
+          with:
+            python-version: ${{ matrix.python-version }}
+        - name: Install Bats
+          run: sudo ./__tests__/setup-bats.sh
+        - name: Install Python dependencies
+          run: sudo ./__tests__/setup-pip.sh
+        - name: Linter, Integration and Unit Tests
+          run: sudo ./__tests__/test-ci.sh

README.md

@@ -166,30 +166,34 @@ If you want to use the plugin within Azure Pipelines or Github Actions, you'll n
   displayName: 'Build MkDocs Site'
 ```
 
-### α Development in Imported Repos (Alpha)
+### β Development in Imported Repos (Beta)
 
-For `mkdocs serve` to work properly in another repo (a repo that is imported into a main site), you will need to add the multirepo plugin within the *imported* repo, including the following configuration.
+For `mkdocs serve` to work properly in an imported repo (a repo that is imported in the parent site), *you will need to add the multirepo plugin within the imported repo as well as the parent repo*, including the following configuration.
 
 > Notes:
-> - You will also need to have `plugins` the main repo (the repo what imports other repos) uses installed within your local `venv`.
+> - You will also need to have `plugins` and `packages` the parent repo uses installed within your local `venv`.
 > - See documentation on the [set](https://git-scm.com/docs/git-sparse-checkout#Documentation/git-sparse-checkout.txt-emsetem) git command for `sparse-checkout` if you are confused with what `dirs` can contain.
 
 ```yml
-site_name: My Docs
-
 plugins:
   multirepo:
     imported_repo: true
-    url: [url to main repo]
+    url: https://github.com/squidfunk/mkdocs-material
+    section_name: Backstage
     # directories and files needed for building the site
-    dirs: ["overrides/*", "mkdocs.yml"]
-    custom_dir: overrides # overrides directory
+    # any path in docs will be included. For example, index.md is the
+    # homepage of the parent site
+    dirs: ["material/*", "mkdocs.yml", "docs/index.md"]
+    custom_dir: material
     yml_file: mkdocs.yml # this can also be a relative path
-    branch: dev
+    branch: master
 ```
 
-Engineers can now run `mkdocs serve` within their local repo, using the main site's configuration, custom theming and features.
+Writers can now run `mkdocs serve` within their local repo, using the main site's configuration, custom theming and features. This means all development is distributed, without technical writers having to switch repos.
+
+![imported repo serve example](assets/imported-repo-serve.gif)
 
+![site image](assets/backstage-material-theme.png)
 
 ## Running Tests
 

mkdocs_multirepo_plugin/plugin.py

@@ -3,14 +3,15 @@
 from mkdocs.structure.files import get_files, Files
 from mkdocs.config import config_options
 from mkdocs.theme import Theme
-from mkdocs.config import Config, defaults
+from mkdocs.config import Config, config_options
 from .structure import (
-    Repo, DocsRepo, parse_import, parse_repo_url, batch_import
+    Repo, DocsRepo, parse_import, parse_repo_url, batch_import, resolve_nav_paths
     )
 from .util import ImportDocsException, log, get_src_path_root, asyncio_run
 from pathlib import Path
 from copy import deepcopy
 import shutil
+import tempfile
 
 IMPORT_STATEMENT = "!import"
 
@@ -27,58 +28,71 @@ class MultirepoPlugin(BasePlugin):
         ("yml_file", config_options.Type(str, default=None)),
         ("imported_repo", config_options.Type(bool, default=False)),
         ("dirs", config_options.Type(list, default=[])),
-        ("branch", config_options.Type(str, default=None))
+        ("branch", config_options.Type(str, default=None)),
+        ("section_name", config_options.Type(str, default="Imported Docs"))
     )
 
     def __init__(self):
         self.temp_dir: Path = None
         self.repos: Dict[str, DocsRepo] = {}
 
+    def set_temp_dir(self, dir):
+        self.temp_dir = dir
+
+    def setup_imported_repo(self, config: Config):
+        temp_dir = tempfile.mkdtemp(prefix="multirepo_")
+        temp_section_dir = Path(temp_dir) / "docs" / self.config.get("section_name")
+        temp_section_dir.mkdir(parents=True)
+        shutil.copytree(config.get('docs_dir'), str(temp_section_dir), dirs_exist_ok=True)
+        return temp_dir
+
     def handle_imported_repo(self, config: Config) -> Config:
         """Imports necessary files for serving site in an imported repo"""
-        repo = Repo(
-            "importee", self.config.get("url"), self.config.get("branch"),
-            self.temp_dir
-        )
-        repo.import_config_files(self.config.get("dirs"))
-        new_config = repo.load_config(self.config.get("yml_file"))
-        # remove nav
+        temp_dir = Path(self.setup_imported_repo(config))
+        parent_repo = Repo("importee", self.config.get("url"), self.config.get("branch"), temp_dir)
+        asyncio_run(parent_repo.import_config_files(self.config.get("dirs")))
+        shutil.copytree(str(parent_repo.location / "docs"), str(temp_dir / "docs"), dirs_exist_ok=True)
+
+        new_config = parent_repo.load_config(self.config.get("yml_file"))
+        # remove parent nav
         if "nav" in new_config:
             del new_config["nav"]
         # update plugins
         if "plugins" in new_config:
             plugins_copy = deepcopy(new_config["plugins"])
             for p in plugins_copy:
                 if "search" in p:
-                    log.info("removing search")
+                    log.info("Multirepo removing search")
                     new_config["plugins"].remove(p)
                 if "multirepo" in p:
                     new_config["plugins"].remove(p)
-                    new_config["plugins"].append({"multirepo": {"imported_repo": True}})
             # validate and provide PluginCollection object to config
             plugins = config_options.Plugins()
             plugin_collection = plugins.validate(new_config.get("plugins"))
+            plugin_collection["multirepo"] = self
             new_config["plugins"] = plugin_collection
         # update theme
         if "theme" in new_config:
             del new_config["theme"]["custom_dir"]
             new_config["theme"] = Theme(
-                custom_dir=str(repo.location / self.config.get("custom_dir")),
+                custom_dir=str(parent_repo.location / self.config.get("custom_dir")),
                 **new_config["theme"]
             )
+        # update docs dir to point to temp_dir
+        new_config["docs_dir"] = str(temp_dir / "docs")
+        # resolve the nav paths
+        if config.get("nav"):
+            resolve_nav_paths(config.get("nav"), self.config.get("section_name"))
         # update this repo's config with the main repo's config
         config.update(new_config)
-        # needs to be a dict
-        new_config = dict(config)
+        # update markdown externsions
+        option = config_options.MarkdownExtensions()
+        config['markdown_extensions'] = option.validate(config['markdown_extensions'])
         # update dev address
         dev_addr = config_options.IpAddress()
-        addr = dev_addr.validate(new_config.get("dev_addr"))
+        addr = dev_addr.validate(new_config.get("dev_addr") or '127.0.0.1:8000')
         config["dev_addr"] = (addr.host, addr.port)
-        # create new config object
-        config = Config(defaults.get_schema())
-        config.load_dict(new_config)
-        config.validate()
-        return config
+        return config, temp_dir
 
     def handle_nav_based_import(self, config: Config) -> Config:
         """Imports documentation in other repos based on nav configuration"""
@@ -131,13 +145,15 @@ def handle_repos_based_import(self, config: Config, repos: list) -> Config:
         return config
 
     def on_config(self, config: Config) -> Config:
-        docs_dir = Path(config.get('docs_dir'))
-        self.temp_dir = docs_dir.parent / self.config.get("temp_dir")
-        if not self.temp_dir.is_dir():
-            self.temp_dir.mkdir()
         if self.config.get("imported_repo"):
-            return self.handle_imported_repo(config)
+            config, temp_dir = self.handle_imported_repo(config)
+            self.set_temp_dir(temp_dir)
+            return config
         else:
+            docs_dir = Path(config.get('docs_dir'))
+            self.set_temp_dir(docs_dir.parent / self.config.get("temp_dir"))
+            if not self.temp_dir.is_dir():
+                self.temp_dir.mkdir()
             repos = self.config.get("repos")
             if not config.get('nav') and not repos:
                 return config
@@ -174,10 +190,14 @@ def on_nav(self, nav, config: Config, files: Files):
             return nav
 
     def on_post_build(self, config: Config) -> None:
-        if self.config.get("imported_repo") and self.config.get("cleanup"):
+        if self.config.get("imported_repo"):
+            config["docs_dir"] = "docs"
+            shutil.rmtree(str(self.temp_dir))
+        elif self.temp_dir and self.config.get("cleanup"):
+            temp_dir = self.config.get("temp_dir")
+            log.info(f"Multirepo plugin is cleaning up {temp_dir}/")
+            shutil.rmtree(str(self.temp_dir))
+
+    def on_build_error(self, error):
+        if self.temp_dir:
             shutil.rmtree(str(self.temp_dir))
-        else:
-            if self.temp_dir and self.config.get("cleanup"):
-                temp_dir = self.config.get("temp_dir")
-                log.info(f"Multirepo plugin is cleaning up {temp_dir}/")
-                shutil.rmtree(str(self.temp_dir))

mkdocs_multirepo_plugin/structure.py

@@ -71,11 +71,11 @@ async def sparse_clone(self, dirs: List[str]) -> Tuple[str, str]:
             stdout = await execute_bash_script("sparse_clone_old.sh", args, self.temp_dir)
         return stdout
 
-    def import_config_files(self, dirs: List[str]) -> subprocess.CompletedProcess:
+    async def import_config_files(self, dirs: List[str]) -> subprocess.CompletedProcess:
         """Imports directories needed for building the site
         the list of dirs might include: mkdocs.yml, overrides/*, etc"""
         self.temp_dir.mkdir(exist_ok=True)
-        return self.sparse_clone(dirs)
+        return await self.sparse_clone(dirs)
 
     def delete_repo(self) -> None:
         """Deletes the repo from the temp directory"""

setup.py

@@ -11,7 +11,7 @@
         'mkdocs_multirepo_plugin/scripts/sparse_clone_old.sh',
         'mkdocs_multirepo_plugin/scripts/mv_docs_up.sh'
         ],
-    version="0.2.9",
+    version="0.3.0",
     author="Joseph Doiron",
     author_email="[email protected]",
     description="Build documentation in multiple repos into one site.",
@@ -27,7 +27,6 @@
         "asyncio",
         "tqdm"
         ],
-    extras_require={"test": ["pytest>=4.0", "pytest-cov"]},
     include_package_data=True,
     zip_safe=False,
     entry_points={