From 3ba59eb1382e2b669f98fafaeac3b27efaf9e101 Mon Sep 17 00:00:00 2001 From: Joan Puigcerver Date: Mon, 25 Nov 2024 11:04:32 +0100 Subject: [PATCH] Feature: Data Sources (#1) Multiple data sources can be defined. --- README.md | 4 +- docs/configuration/index.md | 18 ++++-- docs/index.md | 4 +- mkdocs.yml | 11 ++-- mkdocs_data_plugin/__init__.py | 3 +- mkdocs_data_plugin/plugin.py | 64 ++++++++++++------- tests/docs/fruits.yml | 3 + ...st_data_template.md => test_dir_source.md} | 2 +- tests/docs/test_file_source.md | 7 ++ tests/test_config.py | 8 +-- tests/test_data_template.py | 27 +++++++- tests/test_load_data.py | 61 ++++++++++++++++-- 12 files changed, 158 insertions(+), 54 deletions(-) create mode 100644 tests/docs/fruits.yml rename tests/docs/{test_data_template.md => test_dir_source.md} (89%) create mode 100644 tests/docs/test_file_source.md diff --git a/README.md b/README.md index 5bee78e..045acdb 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # mkdocs-data-plugin __MkDocs Data Plugin__ is a plugin for [MkDocs](https://www.mkdocs.org/) that allows -reading data from separate external markup files and use it in your Markdown pages. +reading data from markup files and use it in your Markdown pages. Currently supported formats: @@ -28,7 +28,7 @@ plugins: ## Overview When using this plugin, you can define data in YAML or JSON files -in a separate directory and reference it in your Markdown files. +in a separate directory and reference them in your Markdown files. ```txt root/ diff --git a/docs/configuration/index.md b/docs/configuration/index.md index fb0f5cc..453cc33 100644 --- a/docs/configuration/index.md +++ b/docs/configuration/index.md @@ -2,18 +2,26 @@ This plugin provides the following configuration options. -## Data Directory -The `data_dir` option specifies the directory where data files are stored. By default, this is set to `data`. +## Data Sources +The `sources` option is a dictionary that defines the data sources to be used. +Each key-value pair represents a source name and its corresponding directory path. + +A source can reference directories or files. +If a directory is specified, all files in the directory and its subdirectories will be loaded. + +By default, a single data source named `data` is defined with the directory path `data`. ```yaml plugins: - - data: - data_dir: data + - sources: + data: data # default + foo: docs/foo.yml ``` !!! tip It is recommended to configure the `watch` option in the `mkdocs.yml` file - to automatically reload the site when data files are modified. + to automatically reload the site when files in the data sources outside + the `docs` directory are modified. ```yaml watch: diff --git a/docs/index.md b/docs/index.md index 8b24411..48e9080 100644 --- a/docs/index.md +++ b/docs/index.md @@ -4,11 +4,11 @@ title: Home # MkDocs Data Plugin __MkDocs Data Plugin__ is a plugin for [MkDocs](https://www.mkdocs.org/) that allows -reading data from separate external markup files and use it in your Markdown pages. +reading data from markup files and use it in your Markdown pages. ## Overview When using this plugin, you can define data in YAML or JSON files -in a separate directory and reference it in your Markdown pages. +in a separate directory and reference them in your Markdown pages. ```txt root/ diff --git a/mkdocs.yml b/mkdocs.yml index 747decf..7d2cecd 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -28,8 +28,8 @@ theme: # Palette toggle for light mode - media: "(prefers-color-scheme: light)" - primary: deep purple - accent: deep purple + primary: lime + accent: lime scheme: default toggle: icon: material/toggle-switch @@ -37,8 +37,8 @@ theme: # Palette toggle for dark mode - media: "(prefers-color-scheme: dark)" - primary: deep purple - accent: deep purple + primary: lime + accent: lime scheme: slate toggle: icon: material/toggle-switch-off @@ -66,7 +66,8 @@ plugins: use_anchor_titles: true - data - macros - - social + - social: + enabled: !ENV [CI, false] - tags - git-revision-date-localized diff --git a/mkdocs_data_plugin/__init__.py b/mkdocs_data_plugin/__init__.py index ee7bcb8..181e9f0 100644 --- a/mkdocs_data_plugin/__init__.py +++ b/mkdocs_data_plugin/__init__.py @@ -1,2 +1 @@ -version = "0.1.0" -__version__ = "0.1.0" +version = "0.2.0" diff --git a/mkdocs_data_plugin/plugin.py b/mkdocs_data_plugin/plugin.py index 6fdfb5c..de3bf02 100644 --- a/mkdocs_data_plugin/plugin.py +++ b/mkdocs_data_plugin/plugin.py @@ -11,35 +11,40 @@ class DataPluginConfig(base.Config): - data_dir = c.Type(str, default='data') + sources = c.Type(dict, default={'data': 'data'}) class DataPlugin(BasePlugin[DataPluginConfig]): def __init__(self): - self.data = {} + self.sources = {} self.processors = {'.yml': yaml.safe_load, '.yaml': yaml.safe_load, '.json': json.load} - def set_data(self, keys, value): + def load_sources(self): """ - Set a value in the data attribute. + Load all sources from the config file and load the data from the files. """ - data = self.data - for key in keys[:-1]: - data = data.setdefault(key, {}) - data[keys[-1]] = value + for source, path in self.config['sources'].items(): + if not os.path.exists(path): + log.warning(f"Mapping path '{path}' not found. Skipping.") + elif os.path.isdir(path): + self.load_folder(source, path) + else: + value = self.load_file(path) + self.update_data(source, [], value) - def on_config(self, config: MkDocsConfig): - self.load_data(self.config.data_dir) - - macros_plugin = config.plugins.get('macros') - if macros_plugin: - macros_plugin.register_variables({'data': self.data}) + def update_data(self, source: str, keys: list, value: any): + """ + Update the sources data with the given value. + """ + if len(keys) == 0: + self.sources[source] = value else: - log.warning( - "The macros plugin is not installed. The `data` variable won't be available in pages." - ) + data = self.sources.setdefault(source, {}) + for key in keys[:-1]: + data = data.setdefault(key, {}) + data[keys[-1]] = value - def load_data(self, path: str): + def load_folder(self, source: str, path: str): """ Iterate over all files in the data directory and load them into the data attribute. @@ -47,24 +52,37 @@ def load_data(self, path: str): for root, _, files in os.walk(path): keys = [] - if root != self.config.data_dir: - directory = os.path.relpath(root, self.config.data_dir) + if root != path: + directory = os.path.relpath(root, path) keys = directory.split(os.sep) for file in files: value = self.load_file(os.path.join(root, file)) filename, _ = os.path.splitext(file) - self.set_data(keys + [filename], value) + self.update_data(source, keys + [filename], value) def load_file(self, path: str): """ - Load a file and return its content. + Loads a file and processes it with the appropriate processor. """ _, extension = os.path.splitext(path) with open(path, 'r') as file: return self.processors[extension](file) + def on_config(self, config: MkDocsConfig): + self.load_sources() + + macros_plugin = config.plugins.get('macros') + if macros_plugin: + for source, data in self.sources.items(): + macros_plugin.register_variables({source: data}) + else: + log.warning( + "The macros plugin is not installed. The `data` variable won't be available in pages." + ) + def on_page_context(self, context, page, config, nav): - context['data'] = self.data + for source, data in self.sources.items(): + context[source] = data return context diff --git a/tests/docs/fruits.yml b/tests/docs/fruits.yml new file mode 100644 index 0000000..36978f4 --- /dev/null +++ b/tests/docs/fruits.yml @@ -0,0 +1,3 @@ +- Apple +- Banana +- Strawberry diff --git a/tests/docs/test_data_template.md b/tests/docs/test_dir_source.md similarity index 89% rename from tests/docs/test_data_template.md rename to tests/docs/test_dir_source.md index 982ae3d..f06b5a4 100644 --- a/tests/docs/test_data_template.md +++ b/tests/docs/test_dir_source.md @@ -1,5 +1,5 @@ --- -title: "Test Data Template" +title: "Test Directory Source" --- - data.a: `{{ data.a }}` diff --git a/tests/docs/test_file_source.md b/tests/docs/test_file_source.md new file mode 100644 index 0000000..4dd3c56 --- /dev/null +++ b/tests/docs/test_file_source.md @@ -0,0 +1,7 @@ +--- +title: "Test File Source" +--- + +{% for fruit in fruits %} +- `{{ fruit }}` +{% endfor %} diff --git a/tests/test_config.py b/tests/test_config.py index cf8f0c6..13090df 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -6,10 +6,10 @@ def test_config_default_values(): plugin = DataPlugin() plugin.load_config({}) - assert plugin.config.data_dir == 'data' + assert plugin.config.sources == {'data': 'data'} -def test_config_data_dir(): +def test_config_sources(): plugin = DataPlugin() - plugin.load_config({'data_dir': 'other_data'}) - assert plugin.config.data_dir == 'other_data' + plugin.load_config({'sources': {'data': 'other_data'}}) + assert plugin.config.sources == {'data': 'other_data'} diff --git a/tests/test_data_template.py b/tests/test_data_template.py index c2b3fd4..4120ca1 100644 --- a/tests/test_data_template.py +++ b/tests/test_data_template.py @@ -4,19 +4,19 @@ from mkdocs.config.base import load_config -def test_loads_files(): +def test_dir_source_in_markdown_file(): mkdocs_config = load_config( "tests/mkdocs.yml", plugins={ "macros": {}, - "data": {'data_dir': 'tests/data'}, + "data": {'sources': {'data': 'tests/data'}}, }, ) build(mkdocs_config) site_dir = mkdocs_config["site_dir"] - with open(site_dir+'/test_data_template/index.html') as f: + with open(site_dir+'/test_dir_source/index.html') as f: data_loaded = re.findall(r"([^<]*)", f.read()) print(data_loaded) assert(data_loaded == [ @@ -29,3 +29,24 @@ def test_loads_files(): "text", # data/dir2/c.yml -> c2 ]) +def test_file_source_in_markdown_file(): + mkdocs_config = load_config( + "tests/mkdocs.yml", + plugins={ + "macros": {}, + "data": {'sources': {'fruits': 'tests/docs/fruits.yml'}}, + }, + ) + + build(mkdocs_config) + site_dir = mkdocs_config["site_dir"] + + with open(site_dir+'/test_file_source/index.html') as f: + data_loaded = re.findall(r"([^<]*)", f.read()) + print(data_loaded) + assert(data_loaded == [ + "Apple", + "Banana", + "Strawberry", + ]) + diff --git a/tests/test_load_data.py b/tests/test_load_data.py index 304192b..92b99a4 100644 --- a/tests/test_load_data.py +++ b/tests/test_load_data.py @@ -2,34 +2,33 @@ from mkdocs.config.base import load_config -def test_empty_data_is_empty_dict(): +def test_inexistent_source_is_skipped(): mkdocs_config = load_config( "tests/mkdocs.yml", plugins={ - "data": {'data_dir': 'tests/inexistent'} + "data": {'sources': {'data': 'tests/inexistent'}} }, ) build(mkdocs_config) dataPlugin = mkdocs_config["plugins"]["data"] - data = dataPlugin.data - assert data == {} + assert "data" not in dataPlugin.sources -def test_loads_files(): +def test_folder_source(): mkdocs_config = load_config( "tests/mkdocs.yml", plugins={ - "data": {'data_dir': 'tests/data'} + "data": {'sources': {'data': 'tests/data'}} }, ) build(mkdocs_config) dataPlugin = mkdocs_config["plugins"]["data"] - data = dataPlugin.data + data = dataPlugin.sources["data"] assert data == { "a": { @@ -49,3 +48,51 @@ def test_loads_files(): }, }, } + +def test_folder_source_slash(): + mkdocs_config = load_config( + "tests/mkdocs.yml", + plugins={ + "data": {'sources': {'data': 'tests/data/'}} + }, + ) + + build(mkdocs_config) + + dataPlugin = mkdocs_config["plugins"]["data"] + data = dataPlugin.sources["data"] + + assert data == { + "a": { + "a1": 1, + "a2": "text", + }, + "dir1": { + "b": { + "b1": 2, + "b2": "text", + }, + }, + "dir2": { + "c": { + "c1": 3, + "c2": "text", + }, + }, + } + +def test_file_source(): + mkdocs_config = load_config( + "tests/mkdocs.yml", + plugins={ + "data": {'sources': {'fruits': 'tests/docs/fruits.yml'}} + }, + ) + + build(mkdocs_config) + + dataPlugin = mkdocs_config["plugins"]["data"] + data = dataPlugin.sources["fruits"] + print(dataPlugin.sources) + + assert data == ['Apple', 'Banana', 'Strawberry']