217 lucene index to store meta data (#220)

* #217 simple persistence * #217 added most query methods * #217 added most query methods * #217 add missing queries, allow names for query operations * #217 add paging and sorting * #217 sorting and some refactorings * #217 add support for custom query operations * #217 remove odl secondary index * #217 make persistens configurable * remove data * update gitignore * update gitignore * #217 refactorings * #217 refactor sonar issues * #217 refactor sonar issues * #217 refactor sonar issues * #217 refactor sonar issues * #217 refactor sonar issues * #217 refactor sonar issues * #217 refactor sonar issues * #217 refactor sonar issues * #217 refactor folder names * update dependencies * optimize imports * update readme * update readme * fix issue with folder in folder --------- Co-authored-by: Thorsten Marx <[email protected]>
CondationCMS · Jul 3, 2024 · 198b337 · 198b337
1 parent 524bcc1
commit 198b337
Show file tree

Hide file tree

Showing 49 changed files with 2,241 additions and 779 deletions.
diff --git a/.gitignore b/.gitignore
@@ -40,3 +40,4 @@ cms-server/cms.pid
 /distribution/temp/
 /distribution/build/
 /distribution/dist/
+/cms-server/hosts/features/data/
diff --git a/README.md b/README.md
@@ -6,15 +6,27 @@ see wiki for more information: [wiki](https://github.com/thmarx/cms/wiki)
 
 # changelog
 
-## 5.0.0
+## 5.1.0
 
-* **FEATURE** Correction for the shortcode syntax [#216](https://github.com/thmarx/cms/issues/216)
+* **FEATURE** Persitent index for metadata [#217](https://github.com/thmarx/cms/issues/217)  
+	**Attention**: Refactoring of the page objects requires a migration of your templates
+* **FEATURE** Correction for the shortcode syntax [#216](https://github.com/thmarx/cms/issues/216)  
 	The comma to separate the parameters is not needed anymore.
 * **FEATURE** Markdown support for multiline list items [#215](https://github.com/thmarx/cms/issues/215)
 * **FEATURE** New taxonomies added at runtime are reloaded when using the cli comman *host reload* [#213](https://github.com/thmarx/cms/issues/213)
 * **FEATURE** introduce more hooks [#218](https://github.com/thmarx/cms/issues/218)
 
 
+### Migration
+
+#### Query
+
+In template code the property total of the page object has been renamed to totalPages.
+
+#### ShortCodes
+
+Shortcodes have to been changed from [code param1="",param2="" /] to [code param1="" param2="" /]
+
 ## 5.0.0
 
 * **BREAKING CHANGE** Introduce module registry [#205](https://github.com/thmarx/cms/issues/205)

diff --git a/cms-api/src/main/java/com/github/thmarx/cms/api/SiteProperties.java b/cms-api/src/main/java/com/github/thmarx/cms/api/SiteProperties.java
@@ -63,6 +63,10 @@ public String id () {
 	public String theme () {
 		return (String) properties.get("theme");
 	}
+
+	public boolean peristentIndex () {
+		return (boolean)getSubMap("index").getOrDefault("persistent", false);
+	}
 
 	public Locale locale () {
 		if (properties.containsKey("language")) {

diff --git a/cms-api/src/main/java/com/github/thmarx/cms/api/db/ContentNode.java b/cms-api/src/main/java/com/github/thmarx/cms/api/db/ContentNode.java
@@ -28,6 +28,7 @@
 import com.github.thmarx.cms.api.feature.features.SitePropertiesFeature;
 import com.github.thmarx.cms.api.utils.MapUtil;
 import com.github.thmarx.cms.api.utils.SectionUtil;
+import java.io.Serializable;
 import java.time.Instant;
 import java.time.LocalDate;
 import java.util.Date;
@@ -40,7 +41,7 @@
  * @author t.marx
  */
 public record ContentNode(String uri, String name, Map<String, Object> data,
-		boolean directory, Map<String, ContentNode> children, LocalDate lastmodified) {
+		boolean directory, Map<String, ContentNode> children, LocalDate lastmodified) implements Serializable {
 
 	public ContentNode(String uri, String name, Map<String, Object> data, boolean directory, Map<String, ContentNode> children) {
 		this(uri, name, data, directory, children, LocalDate.now());

diff --git a/cms-api/src/main/java/com/github/thmarx/cms/api/db/Page.java b/cms-api/src/main/java/com/github/thmarx/cms/api/db/Page.java
@@ -37,11 +37,27 @@
 @NoArgsConstructor
 public class Page<T> {
 
-	public static final Page EMPTY = new Page(0, 0, 1, Collections.EMPTY_LIST);
+	public static final Page EMPTY = new Page(0, 0, 0, 1, Collections.EMPTY_LIST);
 
-	private int size;
-	private long total;
+	/**
+	 * Total number of items
+	 */
+	private long totalItems;
+	/**
+	 * Total number of items per page
+	 */
+	private long pageSize;
+	/**
+	 * Total number of pages
+	 */
+	private long totalPages;
+	/**
+	 * Number of the current page
+	 */
 	private int page;
 
+	/**
+	 * Items of the current page;
+	 */
 	private List<T> items;
 }
diff --git a/.../cms/filesystem/index/IndexProviding.java → ...ithub/thmarx/cms/api/utils/FileUtils.java b/.../cms/filesystem/index/IndexProviding.java → ...ithub/thmarx/cms/api/utils/FileUtils.java
@@ -1,10 +1,10 @@
-package com.github.thmarx.cms.filesystem.index;
+package com.github.thmarx.cms.api.utils;
 
 /*-
  * #%L
- * cms-filesystem
+ * cms-api
  * %%
- * Copyright (C) 2023 Marx-Software
+ * Copyright (C) 2023 - 2024 Marx-Software
  * %%
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as
@@ -22,14 +22,24 @@
  * #L%
  */
 
-import com.github.thmarx.cms.api.db.ContentNode;
-import java.util.function.Function;
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Comparator;
 
 /**
  *
- * @author t.marx
+ * @author thmar
  */
-public interface IndexProviding {
+public class FileUtils {
+
+	private FileUtils () {}
 
-	public SecondaryIndex<?> getOrCreateIndex (final String field, Function<ContentNode, Object> indexFunction);
+	public static void deleteFolder(Path pathToBeDeleted) throws IOException {
+		Files.walk(pathToBeDeleted)
+				.sorted(Comparator.reverseOrder())
+				.map(Path::toFile)
+				.forEach(File::delete);
+	}
 }
diff --git a/cms-content/src/main/java/com/github/thmarx/cms/content/TaxonomyResolver.java b/cms-content/src/main/java/com/github/thmarx/cms/content/TaxonomyResolver.java
@@ -92,7 +92,7 @@ public Optional<TaxonomyResponse> getTaxonomyResponse(final RequestContext conte
 			if (value.isPresent()) {
 				resultPage = new Page<>();
 				resultPage.setPage(page);
-				resultPage.setSize(size);
+				resultPage.setPageSize(size);
 
 				template = taxonomy.getSingleTemplate();
 				meta.put(Constants.MetaFields.TITLE, taxonomy.getTitle() + " - " + taxonomy.getValueTitle(value.get()));
@@ -101,7 +101,8 @@ public Optional<TaxonomyResponse> getTaxonomyResponse(final RequestContext conte
 					return contentNodeMapper.toListNode(node, context);
 				}).filter(node -> node != null).toList();
 				resultPage.setItems(nodes);
-				resultPage.setTotal(contentPage.getTotal());	
+				resultPage.setTotalItems(contentPage.getTotalItems());	
+				resultPage.setTotalPages(contentPage.getTotalPages());
 			} else {
 				meta.put(Constants.MetaFields.TITLE, taxonomy.getTitle());
 			}

diff --git a/cms-filesystem/.gitignore b/cms-filesystem/.gitignore
@@ -0,0 +1 @@
+src/test/resources/data/
diff --git a/cms-filesystem/pom.xml b/cms-filesystem/pom.xml
@@ -10,6 +10,20 @@
 	<packaging>jar</packaging>
 
 	<dependencies>
+
+		<dependency>
+			<groupId>org.apache.lucene</groupId>
+			<artifactId>lucene-core</artifactId>
+		</dependency>
+		<dependency>
+			<groupId>org.apache.lucene</groupId>
+			<artifactId>lucene-analysis-common</artifactId>
+		</dependency>
+		<dependency>
+			<groupId>com.h2database</groupId>
+			<artifactId>h2-mvstore</artifactId>
+		</dependency>
+
 		<dependency>
 			<groupId>com.github.thmarx.cms</groupId>
 			<artifactId>cms-api</artifactId>

diff --git a/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/FileDB.java b/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/FileDB.java
@@ -53,8 +53,12 @@ public class FileDB implements DB {
 	private FileTaxonomies taxonomies;
 
 	public void init () throws IOException {
+		init(MetaData.Type.MEMORY);
+	}
+
+	public void init (MetaData.Type metaDataType) throws IOException {
 		fileSystem = new FileSystem(hostBaseDirectory, eventBus, contentParser);
-		fileSystem.init();
+		fileSystem.init(metaDataType);
 
 		content = new FileContent(fileSystem);
 

diff --git a/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/FileSystem.java b/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/FileSystem.java
@@ -21,9 +21,11 @@
  * <http://www.gnu.org/licenses/gpl-3.0.html>.
  * #L%
  */
+import com.github.thmarx.cms.filesystem.metadata.memory.MemoryMetaData;
 import com.github.thmarx.cms.api.ModuleFileSystem;
 import com.github.thmarx.cms.api.Constants;
 import com.github.thmarx.cms.api.db.ContentNode;
+import com.github.thmarx.cms.api.db.ContentQuery;
 import com.github.thmarx.cms.api.db.DBFileSystem;
 import com.github.thmarx.cms.api.eventbus.EventBus;
 import com.github.thmarx.cms.api.eventbus.events.ContentChangedEvent;
@@ -32,7 +34,8 @@
 import com.github.thmarx.cms.api.eventbus.events.ReIndexContentMetaDataEvent;
 import com.github.thmarx.cms.api.eventbus.events.TemplateChangedEvent;
 import com.github.thmarx.cms.api.utils.PathUtil;
-import com.github.thmarx.cms.filesystem.query.Query;
+import com.github.thmarx.cms.filesystem.metadata.AbstractMetaData;
+import com.github.thmarx.cms.filesystem.metadata.persistent.PersistentMetaData;
 import java.io.IOException;
 import java.nio.charset.Charset;
 import java.nio.charset.StandardCharsets;
@@ -70,29 +73,19 @@ public class FileSystem implements ModuleFileSystem, DBFileSystem {
 	private Path contentBase;
 
 	@Getter
-	private final MetaData metaData = new MetaData();
+	private MetaData metaData;
 
 	@Override
 	public Path base () {
 		return hostBaseDirectory;
 	}
 
-	public <T> Query<T> query(final BiFunction<ContentNode, Integer, T> nodeMapper) {
-		return new Query(new ArrayList<>(metaData.nodes().values()), metaData, nodeMapper);
+	public <T> ContentQuery<T> query(final BiFunction<ContentNode, Integer, T> nodeMapper) {
+		return metaData.query(nodeMapper);
 	}
 
-	public <T> Query<T> query(final String startURI, final BiFunction<ContentNode, Integer, T> nodeMapper) {
-
-		final String uri;
-		if (startURI.startsWith("/")) {
-			uri = startURI.substring(1);
-		} else {
-			uri = startURI;
-		}
-
-		var nodes = metaData.nodes().values().stream().filter(node -> node.uri().startsWith(uri)).toList();
-
-		return new Query(nodes, metaData, nodeMapper);
+	public <T> ContentQuery<T> query(final String startURI, final BiFunction<ContentNode, Integer, T> nodeMapper) {
+		return metaData.query(startURI, nodeMapper);
 	}
 
 	public boolean isVisible(final String uri) {
@@ -101,7 +94,7 @@ public boolean isVisible(final String uri) {
 			return false;
 		}
 		var n = node.get();
-		return MetaData.isVisible(n);
+		return AbstractMetaData.isVisible(n);
 	}
 
 	@Override
@@ -147,12 +140,12 @@ public List<String> loadLines(final Path file, final Charset charset) throws IOE
 
 	public List<ContentNode> listDirectories(final Path base, final String start) {
 		var startPath = base.resolve(start);
-		String folder = PathUtil.toRelativePath(startPath, contentBase).toString();
+		String folder = PathUtil.toRelativePath(startPath, contentBase);
 
 		List<ContentNode> nodes = new ArrayList<>();
 
 		if ("".equals(folder)) {
-			metaData.tree().values()
+			metaData.getTree().values()
 					.stream()
 					.filter(node -> node.isDirectory())
 					.forEach((node) -> {
@@ -163,7 +156,7 @@ public List<ContentNode> listDirectories(final Path base, final String start) {
 			var parts = folder.split("\\/");
 			for (var part : parts) {
 				if (node == null) {
-					node = metaData.tree().get(part);
+					node = metaData.getTree().get(part);
 				} else {
 					node = node.children().get(part);
 				}
@@ -177,7 +170,7 @@ public List<ContentNode> listDirectories(final Path base, final String start) {
 						});
 			}
 		} else {
-			metaData.tree().get(folder).children().values()
+			metaData.getTree().get(folder).children().values()
 					.stream()
 					.filter(node -> node.isDirectory())
 					.forEach((node) -> {
@@ -191,7 +184,7 @@ public List<ContentNode> listDirectories(final Path base, final String start) {
 	public List<ContentNode> listContent(final Path base, final String start) {
 		var startPath = base.resolve(start);
 
-		String folder = PathUtil.toRelativePath(startPath, contentBase).toString();
+		String folder = PathUtil.toRelativePath(startPath, contentBase);
 
 		if ("".equals(folder)) {
 			return metaData.listChildren("");
@@ -202,7 +195,7 @@ public List<ContentNode> listContent(final Path base, final String start) {
 	}
 
 	public List<ContentNode> listSections(final Path contentFile) {
-		String folder = PathUtil.toRelativePath(contentFile, contentBase).toString();
+		String folder = PathUtil.toRelativePath(contentFile, contentBase);
 		String filename = contentFile.getFileName().toString();
 		filename = filename.substring(0, filename.length() - 3);
 
@@ -212,7 +205,7 @@ public List<ContentNode> listSections(final Path contentFile) {
 		final Pattern isOrderedSectionOf = Constants.SECTION_ORDERED_OF_PATTERN.apply(filename);
 
 		if ("".equals(folder)) {
-			metaData.tree().values()
+			metaData.getTree().values()
 					.stream()
 					.filter(node -> !node.isHidden())
 					.filter(node -> node.isPublished())
@@ -239,7 +232,6 @@ public List<ContentNode> listSections(final Path contentFile) {
 							nodes.add(node);
 						});
 			}
-
 		}
 
 		return nodes;
@@ -267,7 +259,18 @@ private void addOrUpdateMetaData(Path file) {
 	}
 
 	public void init() throws IOException {
+		init(MetaData.Type.MEMORY);
+	}
+
+	public void init(MetaData.Type metaDataType) throws IOException {
 		log.debug("init filesystem");
+
+		if (MetaData.Type.PERSISTENT.equals(metaDataType)) {
+			this.metaData = new PersistentMetaData(this.hostBaseDirectory);
+		} else {
+			this.metaData = new MemoryMetaData();
+		}
+		this.metaData.open();
 
 		this.contentBase = resolve("content/");
 		var templateBase = resolve("templates/");