diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 54a9b41..edcf284 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,23 +10,22 @@ on: jobs: test: - name: Test on ${{ matrix.os }} with Python ${{ matrix.python-version }} - runs-on: ${{ matrix.os }} + name: Test on Python ${{ matrix.python-version }} + runs-on: ubuntu-latest - strategy: - matrix: - os: [ubuntu-latest, windows-latest, macOS-latest] - python-version: [3, 3.6, 3.7] + strategy: + matrix: + python-version: [3, 3.6, 3.7] - steps: - - uses: actions/checkout@master - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@master - with: - python-version: ${{ matrix.python-version }} - - name: Install Bats - run: sudo ./__tests__/setup-bats.sh - - name: Install Python dependencies - run: sudo ./__tests__/setup-pip.sh - - name: Linter, Integration and Unit Tests - run: sudo ./__tests__/test-ci.sh + steps: + - uses: actions/checkout@master + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@master + with: + python-version: ${{ matrix.python-version }} + - name: Install Bats + run: sudo ./__tests__/setup-bats.sh + - name: Install Python dependencies + run: sudo ./__tests__/setup-pip.sh + - name: Linter, Integration and Unit Tests + run: sudo ./__tests__/test-ci.sh diff --git a/README.md b/README.md index 4a4fedd..86f5fb6 100644 --- a/README.md +++ b/README.md @@ -166,30 +166,34 @@ If you want to use the plugin within Azure Pipelines or Github Actions, you'll n displayName: 'Build MkDocs Site' ``` -### α Development in Imported Repos (Alpha) +### β Development in Imported Repos (Beta) -For `mkdocs serve` to work properly in another repo (a repo that is imported into a main site), you will need to add the multirepo plugin within the *imported* repo, including the following configuration. +For `mkdocs serve` to work properly in an imported repo (a repo that is imported in the parent site), *you will need to add the multirepo plugin within the imported repo as well as the parent repo*, including the following configuration. > Notes: -> - You will also need to have `plugins` the main repo (the repo what imports other repos) uses installed within your local `venv`. +> - You will also need to have `plugins` and `packages` the parent repo uses installed within your local `venv`. > - See documentation on the [set](https://git-scm.com/docs/git-sparse-checkout#Documentation/git-sparse-checkout.txt-emsetem) git command for `sparse-checkout` if you are confused with what `dirs` can contain. ```yml -site_name: My Docs - plugins: multirepo: imported_repo: true - url: [url to main repo] + url: https://github.com/squidfunk/mkdocs-material + section_name: Backstage # directories and files needed for building the site - dirs: ["overrides/*", "mkdocs.yml"] - custom_dir: overrides # overrides directory + # any path in docs will be included. For example, index.md is the + # homepage of the parent site + dirs: ["material/*", "mkdocs.yml", "docs/index.md"] + custom_dir: material yml_file: mkdocs.yml # this can also be a relative path - branch: dev + branch: master ``` -Engineers can now run `mkdocs serve` within their local repo, using the main site's configuration, custom theming and features. +Writers can now run `mkdocs serve` within their local repo, using the main site's configuration, custom theming and features. This means all development is distributed, without technical writers having to switch repos. + +![imported repo serve example](assets/imported-repo-serve.gif) +![site image](assets/backstage-material-theme.png) ## Running Tests diff --git a/assets/backstage-material-theme.png b/assets/backstage-material-theme.png new file mode 100644 index 0000000..956f906 Binary files /dev/null and b/assets/backstage-material-theme.png differ diff --git a/assets/imported-repo-serve.gif b/assets/imported-repo-serve.gif new file mode 100644 index 0000000..cf5446d Binary files /dev/null and b/assets/imported-repo-serve.gif differ diff --git a/mkdocs_multirepo_plugin/plugin.py b/mkdocs_multirepo_plugin/plugin.py index a8b2c58..59b926e 100644 --- a/mkdocs_multirepo_plugin/plugin.py +++ b/mkdocs_multirepo_plugin/plugin.py @@ -3,14 +3,15 @@ from mkdocs.structure.files import get_files, Files from mkdocs.config import config_options from mkdocs.theme import Theme -from mkdocs.config import Config, defaults +from mkdocs.config import Config, config_options from .structure import ( - Repo, DocsRepo, parse_import, parse_repo_url, batch_import + Repo, DocsRepo, parse_import, parse_repo_url, batch_import, resolve_nav_paths ) from .util import ImportDocsException, log, get_src_path_root, asyncio_run from pathlib import Path from copy import deepcopy import shutil +import tempfile IMPORT_STATEMENT = "!import" @@ -27,22 +28,33 @@ class MultirepoPlugin(BasePlugin): ("yml_file", config_options.Type(str, default=None)), ("imported_repo", config_options.Type(bool, default=False)), ("dirs", config_options.Type(list, default=[])), - ("branch", config_options.Type(str, default=None)) + ("branch", config_options.Type(str, default=None)), + ("section_name", config_options.Type(str, default="Imported Docs")) ) def __init__(self): self.temp_dir: Path = None self.repos: Dict[str, DocsRepo] = {} + def set_temp_dir(self, dir): + self.temp_dir = dir + + def setup_imported_repo(self, config: Config): + temp_dir = tempfile.mkdtemp(prefix="multirepo_") + temp_section_dir = Path(temp_dir) / "docs" / self.config.get("section_name") + temp_section_dir.mkdir(parents=True) + shutil.copytree(config.get('docs_dir'), str(temp_section_dir), dirs_exist_ok=True) + return temp_dir + def handle_imported_repo(self, config: Config) -> Config: """Imports necessary files for serving site in an imported repo""" - repo = Repo( - "importee", self.config.get("url"), self.config.get("branch"), - self.temp_dir - ) - repo.import_config_files(self.config.get("dirs")) - new_config = repo.load_config(self.config.get("yml_file")) - # remove nav + temp_dir = Path(self.setup_imported_repo(config)) + parent_repo = Repo("importee", self.config.get("url"), self.config.get("branch"), temp_dir) + asyncio_run(parent_repo.import_config_files(self.config.get("dirs"))) + shutil.copytree(str(parent_repo.location / "docs"), str(temp_dir / "docs"), dirs_exist_ok=True) + + new_config = parent_repo.load_config(self.config.get("yml_file")) + # remove parent nav if "nav" in new_config: del new_config["nav"] # update plugins @@ -50,35 +62,37 @@ def handle_imported_repo(self, config: Config) -> Config: plugins_copy = deepcopy(new_config["plugins"]) for p in plugins_copy: if "search" in p: - log.info("removing search") + log.info("Multirepo removing search") new_config["plugins"].remove(p) if "multirepo" in p: new_config["plugins"].remove(p) - new_config["plugins"].append({"multirepo": {"imported_repo": True}}) # validate and provide PluginCollection object to config plugins = config_options.Plugins() plugin_collection = plugins.validate(new_config.get("plugins")) + plugin_collection["multirepo"] = self new_config["plugins"] = plugin_collection # update theme if "theme" in new_config: del new_config["theme"]["custom_dir"] new_config["theme"] = Theme( - custom_dir=str(repo.location / self.config.get("custom_dir")), + custom_dir=str(parent_repo.location / self.config.get("custom_dir")), **new_config["theme"] ) + # update docs dir to point to temp_dir + new_config["docs_dir"] = str(temp_dir / "docs") + # resolve the nav paths + if config.get("nav"): + resolve_nav_paths(config.get("nav"), self.config.get("section_name")) # update this repo's config with the main repo's config config.update(new_config) - # needs to be a dict - new_config = dict(config) + # update markdown externsions + option = config_options.MarkdownExtensions() + config['markdown_extensions'] = option.validate(config['markdown_extensions']) # update dev address dev_addr = config_options.IpAddress() - addr = dev_addr.validate(new_config.get("dev_addr")) + addr = dev_addr.validate(new_config.get("dev_addr") or '127.0.0.1:8000') config["dev_addr"] = (addr.host, addr.port) - # create new config object - config = Config(defaults.get_schema()) - config.load_dict(new_config) - config.validate() - return config + return config, temp_dir def handle_nav_based_import(self, config: Config) -> Config: """Imports documentation in other repos based on nav configuration""" @@ -131,13 +145,15 @@ def handle_repos_based_import(self, config: Config, repos: list) -> Config: return config def on_config(self, config: Config) -> Config: - docs_dir = Path(config.get('docs_dir')) - self.temp_dir = docs_dir.parent / self.config.get("temp_dir") - if not self.temp_dir.is_dir(): - self.temp_dir.mkdir() if self.config.get("imported_repo"): - return self.handle_imported_repo(config) + config, temp_dir = self.handle_imported_repo(config) + self.set_temp_dir(temp_dir) + return config else: + docs_dir = Path(config.get('docs_dir')) + self.set_temp_dir(docs_dir.parent / self.config.get("temp_dir")) + if not self.temp_dir.is_dir(): + self.temp_dir.mkdir() repos = self.config.get("repos") if not config.get('nav') and not repos: return config @@ -174,10 +190,14 @@ def on_nav(self, nav, config: Config, files: Files): return nav def on_post_build(self, config: Config) -> None: - if self.config.get("imported_repo") and self.config.get("cleanup"): + if self.config.get("imported_repo"): + config["docs_dir"] = "docs" + shutil.rmtree(str(self.temp_dir)) + elif self.temp_dir and self.config.get("cleanup"): + temp_dir = self.config.get("temp_dir") + log.info(f"Multirepo plugin is cleaning up {temp_dir}/") + shutil.rmtree(str(self.temp_dir)) + + def on_build_error(self, error): + if self.temp_dir: shutil.rmtree(str(self.temp_dir)) - else: - if self.temp_dir and self.config.get("cleanup"): - temp_dir = self.config.get("temp_dir") - log.info(f"Multirepo plugin is cleaning up {temp_dir}/") - shutil.rmtree(str(self.temp_dir)) diff --git a/mkdocs_multirepo_plugin/structure.py b/mkdocs_multirepo_plugin/structure.py index c4383e0..ddcde55 100644 --- a/mkdocs_multirepo_plugin/structure.py +++ b/mkdocs_multirepo_plugin/structure.py @@ -71,11 +71,11 @@ async def sparse_clone(self, dirs: List[str]) -> Tuple[str, str]: stdout = await execute_bash_script("sparse_clone_old.sh", args, self.temp_dir) return stdout - def import_config_files(self, dirs: List[str]) -> subprocess.CompletedProcess: + async def import_config_files(self, dirs: List[str]) -> subprocess.CompletedProcess: """Imports directories needed for building the site the list of dirs might include: mkdocs.yml, overrides/*, etc""" self.temp_dir.mkdir(exist_ok=True) - return self.sparse_clone(dirs) + return await self.sparse_clone(dirs) def delete_repo(self) -> None: """Deletes the repo from the temp directory""" diff --git a/setup.py b/setup.py index 7e3a84f..4bd2fbd 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ 'mkdocs_multirepo_plugin/scripts/sparse_clone_old.sh', 'mkdocs_multirepo_plugin/scripts/mv_docs_up.sh' ], - version="0.2.9", + version="0.3.0", author="Joseph Doiron", author_email="josephdoiron1234@yahoo.com", description="Build documentation in multiple repos into one site.", @@ -27,7 +27,6 @@ "asyncio", "tqdm" ], - extras_require={"test": ["pytest>=4.0", "pytest-cov"]}, include_package_data=True, zip_safe=False, entry_points={