From 198b33729bc89a14a530a477cc801a0519efde16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thorsten=20Marx=20=E3=8B=A1?= Date: Wed, 3 Jul 2024 10:37:08 +0200 Subject: [PATCH] 217 lucene index to store meta data (#220) * #217 simple persistence * #217 added most query methods * #217 added most query methods * #217 add missing queries, allow names for query operations * #217 add paging and sorting * #217 sorting and some refactorings * #217 add support for custom query operations * #217 remove odl secondary index * #217 make persistens configurable * remove data * update gitignore * update gitignore * #217 refactorings * #217 refactor sonar issues * #217 refactor sonar issues * #217 refactor sonar issues * #217 refactor sonar issues * #217 refactor sonar issues * #217 refactor sonar issues * #217 refactor sonar issues * #217 refactor sonar issues * #217 refactor folder names * update dependencies * optimize imports * update readme * update readme * fix issue with folder in folder --------- Co-authored-by: Thorsten Marx --- .gitignore | 1 + README.md | 16 +- .../github/thmarx/cms/api/SiteProperties.java | 4 + .../github/thmarx/cms/api/db/ContentNode.java | 3 +- .../com/github/thmarx/cms/api/db/Page.java | 22 +- .../thmarx/cms/api/utils/FileUtils.java | 26 +- .../thmarx/cms/content/TaxonomyResolver.java | 5 +- cms-filesystem/.gitignore | 1 + cms-filesystem/pom.xml | 14 + .../github/thmarx/cms/filesystem/FileDB.java | 6 +- .../thmarx/cms/filesystem/FileSystem.java | 53 +-- .../thmarx/cms/filesystem/MetaData.java | 191 ++--------- .../cms/filesystem/index/SecondaryIndex.java | 77 ----- .../filesystem/metadata/AbstractMetaData.java | 175 ++++++++++ .../metadata/memory/MemoryMetaData.java | 106 ++++++ .../metadata/memory/MemoryQuery.java | 204 ++++++++++++ .../memory}/QueryContext.java | 13 +- .../filesystem/metadata/memory/QueryUtil.java | 97 ++++++ .../metadata/persistent/DocumentHelper.java | 103 ++++++ .../metadata/persistent/LuceneIndex.java | 180 +++++++++++ .../metadata/persistent/LuceneQuery.java | 303 ++++++++++++++++++ .../persistent/PersistentMetaData.java | 143 +++++++++ .../metadata/persistent/QueryHelper.java | 295 +++++++++++++++++ .../metadata/persistent/utils/FlattenMap.java | 49 +++ .../query/ExcerptMapperFunction.java | 2 +- .../metadata/query/ExtendableQuery.java | 56 ++++ .../{ => metadata}/query/Filter.java | 5 +- .../query/Queries.java} | 111 ++----- .../thmarx/cms/filesystem/query/Query.java | 217 ------------- .../filesystem/AbstractFileSystemTest.java | 107 +++++++ .../thmarx/cms/filesystem/FileSystemTest.java | 29 +- .../filesystem/PresistentFileSystemTest.java | 73 +++++ .../filesystem/index/SecondaryIndexTest.java | 55 ---- .../persistent/utils/FlattenMapTest.java | 92 ++++++ .../cms/filesystem/query/QueryPerfTest.java | 35 +- .../cms/filesystem/query/QueryTest.java | 74 ++--- .../src/test/resources/content/index.md | 2 +- .../src/test/resources/content/test/test1.md | 4 +- cms-server/.gitignore | 1 + cms-server/hosts/features/site.yaml | 2 + cms-server/pom.xml | 2 - .../thmarx/cms/server/configs/SiteModule.java | 9 +- .../com/github/thmarx/cms/SectionsTest.java | 2 +- .../cms/content/views/ViewParserTest.java | 2 - .../thmarx/cms/utils/NodeUtilNGTest.java | 2 +- cms-server/themes/test/templates/blog.html | 2 +- .../functions/list/NodeListFunction.java | 4 +- .../functions/query/QueryFunction.java | 9 +- pom.xml | 36 ++- 49 files changed, 2241 insertions(+), 779 deletions(-) rename cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/index/IndexProviding.java => cms-api/src/main/java/com/github/thmarx/cms/api/utils/FileUtils.java (59%) create mode 100644 cms-filesystem/.gitignore delete mode 100644 cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/index/SecondaryIndex.java create mode 100644 cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/metadata/AbstractMetaData.java create mode 100644 cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/metadata/memory/MemoryMetaData.java create mode 100644 cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/metadata/memory/MemoryQuery.java rename cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/{query => metadata/memory}/QueryContext.java (78%) create mode 100644 cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/metadata/memory/QueryUtil.java create mode 100644 cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/metadata/persistent/DocumentHelper.java create mode 100644 cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/metadata/persistent/LuceneIndex.java create mode 100644 cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/metadata/persistent/LuceneQuery.java create mode 100644 cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/metadata/persistent/PersistentMetaData.java create mode 100644 cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/metadata/persistent/QueryHelper.java create mode 100644 cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/metadata/persistent/utils/FlattenMap.java rename cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/{ => metadata}/query/ExcerptMapperFunction.java (95%) create mode 100644 cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/metadata/query/ExtendableQuery.java rename cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/{ => metadata}/query/Filter.java (86%) rename cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/{query/QueryUtil.java => metadata/query/Queries.java} (59%) delete mode 100644 cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/query/Query.java create mode 100644 cms-filesystem/src/test/java/com/github/thmarx/cms/filesystem/AbstractFileSystemTest.java create mode 100644 cms-filesystem/src/test/java/com/github/thmarx/cms/filesystem/PresistentFileSystemTest.java delete mode 100644 cms-filesystem/src/test/java/com/github/thmarx/cms/filesystem/index/SecondaryIndexTest.java create mode 100644 cms-filesystem/src/test/java/com/github/thmarx/cms/filesystem/persistent/utils/FlattenMapTest.java create mode 100644 cms-server/.gitignore diff --git a/.gitignore b/.gitignore index 664bdb83..f301e4aa 100644 --- a/.gitignore +++ b/.gitignore @@ -40,3 +40,4 @@ cms-server/cms.pid /distribution/temp/ /distribution/build/ /distribution/dist/ +/cms-server/hosts/features/data/ diff --git a/README.md b/README.md index 6f0a694b..7e19f09c 100644 --- a/README.md +++ b/README.md @@ -6,15 +6,27 @@ see wiki for more information: [wiki](https://github.com/thmarx/cms/wiki) # changelog -## 5.0.0 +## 5.1.0 -* **FEATURE** Correction for the shortcode syntax [#216](https://github.com/thmarx/cms/issues/216) +* **FEATURE** Persitent index for metadata [#217](https://github.com/thmarx/cms/issues/217) + **Attention**: Refactoring of the page objects requires a migration of your templates +* **FEATURE** Correction for the shortcode syntax [#216](https://github.com/thmarx/cms/issues/216) The comma to separate the parameters is not needed anymore. * **FEATURE** Markdown support for multiline list items [#215](https://github.com/thmarx/cms/issues/215) * **FEATURE** New taxonomies added at runtime are reloaded when using the cli comman *host reload* [#213](https://github.com/thmarx/cms/issues/213) * **FEATURE** introduce more hooks [#218](https://github.com/thmarx/cms/issues/218) +### Migration + +#### Query + +In template code the property total of the page object has been renamed to totalPages. + +#### ShortCodes + +Shortcodes have to been changed from [code param1="",param2="" /] to [code param1="" param2="" /] + ## 5.0.0 * **BREAKING CHANGE** Introduce module registry [#205](https://github.com/thmarx/cms/issues/205) diff --git a/cms-api/src/main/java/com/github/thmarx/cms/api/SiteProperties.java b/cms-api/src/main/java/com/github/thmarx/cms/api/SiteProperties.java index 2832cce4..93fd9179 100644 --- a/cms-api/src/main/java/com/github/thmarx/cms/api/SiteProperties.java +++ b/cms-api/src/main/java/com/github/thmarx/cms/api/SiteProperties.java @@ -63,6 +63,10 @@ public String id () { public String theme () { return (String) properties.get("theme"); } + + public boolean peristentIndex () { + return (boolean)getSubMap("index").getOrDefault("persistent", false); + } public Locale locale () { if (properties.containsKey("language")) { diff --git a/cms-api/src/main/java/com/github/thmarx/cms/api/db/ContentNode.java b/cms-api/src/main/java/com/github/thmarx/cms/api/db/ContentNode.java index d84e1d57..5ccc4002 100644 --- a/cms-api/src/main/java/com/github/thmarx/cms/api/db/ContentNode.java +++ b/cms-api/src/main/java/com/github/thmarx/cms/api/db/ContentNode.java @@ -28,6 +28,7 @@ import com.github.thmarx.cms.api.feature.features.SitePropertiesFeature; import com.github.thmarx.cms.api.utils.MapUtil; import com.github.thmarx.cms.api.utils.SectionUtil; +import java.io.Serializable; import java.time.Instant; import java.time.LocalDate; import java.util.Date; @@ -40,7 +41,7 @@ * @author t.marx */ public record ContentNode(String uri, String name, Map data, - boolean directory, Map children, LocalDate lastmodified) { + boolean directory, Map children, LocalDate lastmodified) implements Serializable { public ContentNode(String uri, String name, Map data, boolean directory, Map children) { this(uri, name, data, directory, children, LocalDate.now()); diff --git a/cms-api/src/main/java/com/github/thmarx/cms/api/db/Page.java b/cms-api/src/main/java/com/github/thmarx/cms/api/db/Page.java index 44a13191..10043bd5 100644 --- a/cms-api/src/main/java/com/github/thmarx/cms/api/db/Page.java +++ b/cms-api/src/main/java/com/github/thmarx/cms/api/db/Page.java @@ -37,11 +37,27 @@ @NoArgsConstructor public class Page { - public static final Page EMPTY = new Page(0, 0, 1, Collections.EMPTY_LIST); + public static final Page EMPTY = new Page(0, 0, 0, 1, Collections.EMPTY_LIST); - private int size; - private long total; + /** + * Total number of items + */ + private long totalItems; + /** + * Total number of items per page + */ + private long pageSize; + /** + * Total number of pages + */ + private long totalPages; + /** + * Number of the current page + */ private int page; + /** + * Items of the current page; + */ private List items; } diff --git a/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/index/IndexProviding.java b/cms-api/src/main/java/com/github/thmarx/cms/api/utils/FileUtils.java similarity index 59% rename from cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/index/IndexProviding.java rename to cms-api/src/main/java/com/github/thmarx/cms/api/utils/FileUtils.java index 7a8a98e7..f1d634d2 100644 --- a/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/index/IndexProviding.java +++ b/cms-api/src/main/java/com/github/thmarx/cms/api/utils/FileUtils.java @@ -1,10 +1,10 @@ -package com.github.thmarx.cms.filesystem.index; +package com.github.thmarx.cms.api.utils; /*- * #%L - * cms-filesystem + * cms-api * %% - * Copyright (C) 2023 Marx-Software + * Copyright (C) 2023 - 2024 Marx-Software * %% * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as @@ -22,14 +22,24 @@ * #L% */ -import com.github.thmarx.cms.api.db.ContentNode; -import java.util.function.Function; +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Comparator; /** * - * @author t.marx + * @author thmar */ -public interface IndexProviding { +public class FileUtils { + + private FileUtils () {} - public SecondaryIndex getOrCreateIndex (final String field, Function indexFunction); + public static void deleteFolder(Path pathToBeDeleted) throws IOException { + Files.walk(pathToBeDeleted) + .sorted(Comparator.reverseOrder()) + .map(Path::toFile) + .forEach(File::delete); + } } diff --git a/cms-content/src/main/java/com/github/thmarx/cms/content/TaxonomyResolver.java b/cms-content/src/main/java/com/github/thmarx/cms/content/TaxonomyResolver.java index b9e0f656..187e9fe4 100644 --- a/cms-content/src/main/java/com/github/thmarx/cms/content/TaxonomyResolver.java +++ b/cms-content/src/main/java/com/github/thmarx/cms/content/TaxonomyResolver.java @@ -92,7 +92,7 @@ public Optional getTaxonomyResponse(final RequestContext conte if (value.isPresent()) { resultPage = new Page<>(); resultPage.setPage(page); - resultPage.setSize(size); + resultPage.setPageSize(size); template = taxonomy.getSingleTemplate(); meta.put(Constants.MetaFields.TITLE, taxonomy.getTitle() + " - " + taxonomy.getValueTitle(value.get())); @@ -101,7 +101,8 @@ public Optional getTaxonomyResponse(final RequestContext conte return contentNodeMapper.toListNode(node, context); }).filter(node -> node != null).toList(); resultPage.setItems(nodes); - resultPage.setTotal(contentPage.getTotal()); + resultPage.setTotalItems(contentPage.getTotalItems()); + resultPage.setTotalPages(contentPage.getTotalPages()); } else { meta.put(Constants.MetaFields.TITLE, taxonomy.getTitle()); } diff --git a/cms-filesystem/.gitignore b/cms-filesystem/.gitignore new file mode 100644 index 00000000..fb254a98 --- /dev/null +++ b/cms-filesystem/.gitignore @@ -0,0 +1 @@ +src/test/resources/data/ \ No newline at end of file diff --git a/cms-filesystem/pom.xml b/cms-filesystem/pom.xml index 91b2b592..9c464861 100644 --- a/cms-filesystem/pom.xml +++ b/cms-filesystem/pom.xml @@ -10,6 +10,20 @@ jar + + + org.apache.lucene + lucene-core + + + org.apache.lucene + lucene-analysis-common + + + com.h2database + h2-mvstore + + com.github.thmarx.cms cms-api diff --git a/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/FileDB.java b/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/FileDB.java index 4a49ff2e..4e6f64c1 100644 --- a/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/FileDB.java +++ b/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/FileDB.java @@ -53,8 +53,12 @@ public class FileDB implements DB { private FileTaxonomies taxonomies; public void init () throws IOException { + init(MetaData.Type.MEMORY); + } + + public void init (MetaData.Type metaDataType) throws IOException { fileSystem = new FileSystem(hostBaseDirectory, eventBus, contentParser); - fileSystem.init(); + fileSystem.init(metaDataType); content = new FileContent(fileSystem); diff --git a/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/FileSystem.java b/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/FileSystem.java index 81e8a820..17ecb7ba 100644 --- a/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/FileSystem.java +++ b/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/FileSystem.java @@ -21,9 +21,11 @@ * . * #L% */ +import com.github.thmarx.cms.filesystem.metadata.memory.MemoryMetaData; import com.github.thmarx.cms.api.ModuleFileSystem; import com.github.thmarx.cms.api.Constants; import com.github.thmarx.cms.api.db.ContentNode; +import com.github.thmarx.cms.api.db.ContentQuery; import com.github.thmarx.cms.api.db.DBFileSystem; import com.github.thmarx.cms.api.eventbus.EventBus; import com.github.thmarx.cms.api.eventbus.events.ContentChangedEvent; @@ -32,7 +34,8 @@ import com.github.thmarx.cms.api.eventbus.events.ReIndexContentMetaDataEvent; import com.github.thmarx.cms.api.eventbus.events.TemplateChangedEvent; import com.github.thmarx.cms.api.utils.PathUtil; -import com.github.thmarx.cms.filesystem.query.Query; +import com.github.thmarx.cms.filesystem.metadata.AbstractMetaData; +import com.github.thmarx.cms.filesystem.metadata.persistent.PersistentMetaData; import java.io.IOException; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; @@ -70,29 +73,19 @@ public class FileSystem implements ModuleFileSystem, DBFileSystem { private Path contentBase; @Getter - private final MetaData metaData = new MetaData(); + private MetaData metaData; @Override public Path base () { return hostBaseDirectory; } - public Query query(final BiFunction nodeMapper) { - return new Query(new ArrayList<>(metaData.nodes().values()), metaData, nodeMapper); + public ContentQuery query(final BiFunction nodeMapper) { + return metaData.query(nodeMapper); } - public Query query(final String startURI, final BiFunction nodeMapper) { - - final String uri; - if (startURI.startsWith("/")) { - uri = startURI.substring(1); - } else { - uri = startURI; - } - - var nodes = metaData.nodes().values().stream().filter(node -> node.uri().startsWith(uri)).toList(); - - return new Query(nodes, metaData, nodeMapper); + public ContentQuery query(final String startURI, final BiFunction nodeMapper) { + return metaData.query(startURI, nodeMapper); } public boolean isVisible(final String uri) { @@ -101,7 +94,7 @@ public boolean isVisible(final String uri) { return false; } var n = node.get(); - return MetaData.isVisible(n); + return AbstractMetaData.isVisible(n); } @Override @@ -147,12 +140,12 @@ public List loadLines(final Path file, final Charset charset) throws IOE public List listDirectories(final Path base, final String start) { var startPath = base.resolve(start); - String folder = PathUtil.toRelativePath(startPath, contentBase).toString(); + String folder = PathUtil.toRelativePath(startPath, contentBase); List nodes = new ArrayList<>(); if ("".equals(folder)) { - metaData.tree().values() + metaData.getTree().values() .stream() .filter(node -> node.isDirectory()) .forEach((node) -> { @@ -163,7 +156,7 @@ public List listDirectories(final Path base, final String start) { var parts = folder.split("\\/"); for (var part : parts) { if (node == null) { - node = metaData.tree().get(part); + node = metaData.getTree().get(part); } else { node = node.children().get(part); } @@ -177,7 +170,7 @@ public List listDirectories(final Path base, final String start) { }); } } else { - metaData.tree().get(folder).children().values() + metaData.getTree().get(folder).children().values() .stream() .filter(node -> node.isDirectory()) .forEach((node) -> { @@ -191,7 +184,7 @@ public List listDirectories(final Path base, final String start) { public List listContent(final Path base, final String start) { var startPath = base.resolve(start); - String folder = PathUtil.toRelativePath(startPath, contentBase).toString(); + String folder = PathUtil.toRelativePath(startPath, contentBase); if ("".equals(folder)) { return metaData.listChildren(""); @@ -202,7 +195,7 @@ public List listContent(final Path base, final String start) { } public List listSections(final Path contentFile) { - String folder = PathUtil.toRelativePath(contentFile, contentBase).toString(); + String folder = PathUtil.toRelativePath(contentFile, contentBase); String filename = contentFile.getFileName().toString(); filename = filename.substring(0, filename.length() - 3); @@ -212,7 +205,7 @@ public List listSections(final Path contentFile) { final Pattern isOrderedSectionOf = Constants.SECTION_ORDERED_OF_PATTERN.apply(filename); if ("".equals(folder)) { - metaData.tree().values() + metaData.getTree().values() .stream() .filter(node -> !node.isHidden()) .filter(node -> node.isPublished()) @@ -239,7 +232,6 @@ public List listSections(final Path contentFile) { nodes.add(node); }); } - } return nodes; @@ -267,7 +259,18 @@ private void addOrUpdateMetaData(Path file) { } public void init() throws IOException { + init(MetaData.Type.MEMORY); + } + + public void init(MetaData.Type metaDataType) throws IOException { log.debug("init filesystem"); + + if (MetaData.Type.PERSISTENT.equals(metaDataType)) { + this.metaData = new PersistentMetaData(this.hostBaseDirectory); + } else { + this.metaData = new MemoryMetaData(); + } + this.metaData.open(); this.contentBase = resolve("content/"); var templateBase = resolve("templates/"); diff --git a/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/MetaData.java b/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/MetaData.java index bce8569e..664a094c 100644 --- a/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/MetaData.java +++ b/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/MetaData.java @@ -2,9 +2,9 @@ /*- * #%L - * cms-server + * cms-filesystem * %% - * Copyright (C) 2023 Marx-Software + * Copyright (C) 2023 - 2024 Marx-Software * %% * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as @@ -22,193 +22,44 @@ * #L% */ -import com.github.thmarx.cms.api.Constants; import com.github.thmarx.cms.api.db.ContentNode; -import com.github.thmarx.cms.filesystem.index.IndexProviding; -import com.github.thmarx.cms.filesystem.index.SecondaryIndex; -import com.google.common.base.Strings; +import com.github.thmarx.cms.api.db.ContentQuery; +import java.io.IOException; import java.time.LocalDate; -import java.util.Arrays; -import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.atomic.AtomicReference; -import java.util.function.Function; -import java.util.stream.Collectors; -import java.util.stream.Stream; +import java.util.function.BiFunction; /** * * @author t.marx */ -public class MetaData implements IndexProviding { - - private ConcurrentMap nodes = new ConcurrentHashMap<>(); - - private ConcurrentMap tree = new ConcurrentHashMap<>(); - - private ConcurrentMap> secondaryIndexes = new ConcurrentHashMap<>(); - - @Override - public SecondaryIndex getOrCreateIndex (final String field, Function indexFunction) { - - if (!secondaryIndexes.containsKey(field)) { - var index = SecondaryIndex.builder() - .indexFunction(indexFunction) - .build(); - index.addAll(nodes.values()); - secondaryIndexes.put(field, index); - } - - return secondaryIndexes.get(field); - } +public interface MetaData { - void clear() { - nodes.clear(); - tree.clear(); - secondaryIndexes.clear(); + public enum Type { + MEMORY, PERSISTENT } - ConcurrentMap nodes() { - return new ConcurrentHashMap<>(nodes); - } + void open () throws IOException; + void close () throws IOException; - ConcurrentMap tree() { - return new ConcurrentHashMap<>(tree); - } + void addFile(final String uri, final Map data, final LocalDate lastModified); - public void createDirectory(final String uri) { - if (Strings.isNullOrEmpty(uri)) { - return; - } - var parts = uri.split(Constants.SPLIT_PATH_PATTERN); - ContentNode n = new ContentNode(uri, parts[parts.length - 1], Map.of(), true); + Optional byUri(final String uri); - Optional parentFolder; - if (parts.length == 1) { - parentFolder = getFolder(uri); - } else { - var parentPath = Arrays.copyOfRange(parts, 0, parts.length - 1); - var parentUri = String.join("/", parentPath); - parentFolder = getFolder(parentUri); - } + void createDirectory(final String uri); - if (parentFolder.isPresent()) { - parentFolder.get().children().put(n.name(), n); - } else { - tree.put(n.name(), n); - } - } + Optional findFolder(String uri); - public List listChildren(String uri) { - if ("".equals(uri)) { - return tree.values().stream() - .filter(node -> !node.isHidden()) - .map(this::mapToIndex) - .filter(node -> node != null) - .filter(MetaData::isVisible) - .collect(Collectors.toList()); - - } else { - Optional findFolder = findFolder(uri); - if (findFolder.isPresent()) { - return findFolder.get().children().values() - .stream() - .filter(node -> !node.isHidden()) - .map(this::mapToIndex) - .filter(node -> node != null) - .filter(MetaData::isVisible) - .collect(Collectors.toList()); - } - } - return Collections.emptyList(); - } - - protected ContentNode mapToIndex(ContentNode node) { - if (node.isDirectory()) { - var tempNode = node.children().entrySet().stream().filter((entry) - -> entry.getKey().equals("index.md") - ).findFirst(); - if (tempNode.isPresent()) { - return tempNode.get().getValue(); - } - return null; - } else { - return node; - } - } - - public static boolean isVisible (ContentNode node) { - return node != null - // check if some parent is hidden - && !node.uri().startsWith(".") && !node.uri().contains("/.") - && node.isPublished() - && !node.isHidden() - && !node.isSection(); - } + List listChildren(String uri); - public Optional findFolder(String uri) { - return getFolder(uri); - } - - private Optional getFolder(String uri) { - var parts = uri.split(Constants.SPLIT_PATH_PATTERN); - - final AtomicReference folder = new AtomicReference<>(null); - Stream.of(parts).forEach(part -> { - if (part.endsWith(".md")) { - return; - } - if (folder.get() == null) { - folder.set(tree.get(part)); - } else { - folder.set(folder.get().children().get(part)); - } - }); - return Optional.ofNullable(folder.get()); - } - - public void addFile(final String uri, final Map data, final LocalDate lastModified) { - - var parts = uri.split(Constants.SPLIT_PATH_PATTERN); - final ContentNode node = new ContentNode(uri, parts[parts.length - 1], data, lastModified); - - nodes.put(uri, node); - - var folder = getFolder(uri); - if (folder.isPresent()) { - folder.get().children().put(node.name(), node); - } else { - tree.put(node.name(), node); - } - - secondaryIndexes.values().forEach(index -> index.add(node)); - } - - public Optional byUri(final String uri) { - if (!nodes.containsKey(uri)) { - return Optional.empty(); - } - return Optional.of(nodes.get(uri)); - } - - void remove(String uri) { - var node = nodes.remove(uri); - - var folder = getFolder(uri); - var parts = uri.split(Constants.SPLIT_PATH_PATTERN); - var name = parts[parts.length - 1]; - if (folder.isPresent()) { - folder.get().children().remove(name); - } else { - tree.remove(name); - } - - secondaryIndexes.values().forEach(index -> index.remove(node)); - } + void clear (); + + Map getNodes(); + Map getTree(); + ContentQuery query(final BiFunction nodeMapper); + ContentQuery query(final String startURI, final BiFunction nodeMapper); } diff --git a/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/index/SecondaryIndex.java b/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/index/SecondaryIndex.java deleted file mode 100644 index 5dacb5e3..00000000 --- a/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/index/SecondaryIndex.java +++ /dev/null @@ -1,77 +0,0 @@ -package com.github.thmarx.cms.filesystem.index; - -/*- - * #%L - * cms-filesystem - * %% - * Copyright (C) 2023 Marx-Software - * %% - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public - * License along with this program. If not, see - * . - * #L% - */ - -import com.github.thmarx.cms.api.db.ContentNode; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.NavigableMap; -import java.util.TreeMap; -import java.util.concurrent.CopyOnWriteArrayList; -import java.util.function.Function; -import lombok.Builder; - -/** - * - * @author t.marx - * @param - */ -@Builder -public class SecondaryIndex { - - private final Function indexFunction; - - private final NavigableMap> index = new TreeMap<>(); - - public void clear () { - index.clear(); - } - - public void addAll (final Collection nodes) { - nodes.forEach(this::add); - } - - public void add(final ContentNode node) { - T value = indexFunction.apply(node); - if (!index.containsKey(value)) { - index.put(value, new ArrayList<>()); - } - index.get(value).add(node.uri()); - } - - public void remove(final ContentNode node) { - T value = indexFunction.apply(node); - if (index.containsKey(value)) { - index.get(value).remove(node.uri()); - } - } - - public boolean eq (final ContentNode node, final T value) { - if (index.containsKey(value)) { - return index.get(value).contains(node.uri()); - } - return false; - } -} diff --git a/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/metadata/AbstractMetaData.java b/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/metadata/AbstractMetaData.java new file mode 100644 index 00000000..6fa8d564 --- /dev/null +++ b/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/metadata/AbstractMetaData.java @@ -0,0 +1,175 @@ +package com.github.thmarx.cms.filesystem.metadata; + +/*- + * #%L + * cms-filesystem + * %% + * Copyright (C) 2023 - 2024 Marx-Software + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this program. If not, see + * . + * #L% + */ + +import com.github.thmarx.cms.api.Constants; +import com.github.thmarx.cms.api.db.ContentNode; +import com.github.thmarx.cms.filesystem.MetaData; +import com.google.common.base.Strings; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Stream; + +/** + * + * @author t.marx + */ +public abstract class AbstractMetaData implements MetaData { + + protected ConcurrentMap nodes; + + protected ConcurrentMap tree; + + @Override + public void clear() { + if (nodes != null) { + nodes.clear(); + tree.clear(); + } + } + + @Override + public ConcurrentMap getNodes() { + if (nodes == null) { + return new ConcurrentHashMap<>(); + } + return new ConcurrentHashMap<>(nodes); + } + + @Override + public ConcurrentMap getTree() { + if (tree == null) { + return new ConcurrentHashMap<>(); + } + return new ConcurrentHashMap<>(tree); + } + + public static boolean isVisible (ContentNode node) { + return node != null + // check if some parent is hidden + && !node.uri().startsWith(".") && !node.uri().contains("/.") + && node.isPublished() + && !node.isHidden() + && !node.isSection(); + } + + @Override + public Optional byUri(String uri) { + if (!nodes.containsKey(uri)) { + return Optional.empty(); + } + return Optional.of(nodes.get(uri)); + } + + @Override + public Optional findFolder(String uri) { + return getFolder(uri); + } + + protected Optional getFolder(String uri) { + var parts = uri.split(Constants.SPLIT_PATH_PATTERN); + + final AtomicReference folder = new AtomicReference<>(null); + Stream.of(parts).forEach(part -> { + if (part.endsWith(".md")) { + return; + } + if (folder.get() == null) { + folder.set(getTree().get(part)); + } else { + folder.set(folder.get().children().get(part)); + } + }); + return Optional.ofNullable(folder.get()); + } + + @Override + public void createDirectory(String uri) { + if (Strings.isNullOrEmpty(uri)) { + return; + } + var parts = uri.split(Constants.SPLIT_PATH_PATTERN); + ContentNode n = new ContentNode(uri, parts[parts.length - 1], Map.of(), true); + + Optional parentFolder; + if (parts.length == 1) { + //parentFolder = getFolder(uri); + parentFolder = Optional.empty(); + } else { + var parentPath = Arrays.copyOfRange(parts, 0, parts.length - 1); + var parentUri = String.join("/", parentPath); + parentFolder = getFolder(parentUri); + } + + if (parentFolder.isPresent()) { + parentFolder.get().children().put(n.name(), n); + } else { + tree.put(n.name(), n); + } + } + + @Override + public List listChildren(String uri) { + if ("".equals(uri)) { + return getTree().values().stream() + .filter(node -> !node.isHidden()) + .map(this::mapToIndex) + .filter(node -> node != null) + .filter(AbstractMetaData::isVisible) + .toList(); + + } else { + Optional findFolder = findFolder(uri); + if (findFolder.isPresent()) { + return findFolder.get().children().values() + .stream() + .filter(node -> !node.isHidden()) + .map(this::mapToIndex) + .filter(node -> node != null) + .filter(AbstractMetaData::isVisible) + .toList(); + } + } + return Collections.emptyList(); + } + + protected ContentNode mapToIndex(ContentNode node) { + if (node.isDirectory()) { + var tempNode = node.children().entrySet().stream().filter((entry) + -> entry.getKey().equals("index.md") + ).findFirst(); + if (tempNode.isPresent()) { + return tempNode.get().getValue(); + } + return null; + } else { + return node; + } + } +} diff --git a/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/metadata/memory/MemoryMetaData.java b/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/metadata/memory/MemoryMetaData.java new file mode 100644 index 00000000..da2e82ec --- /dev/null +++ b/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/metadata/memory/MemoryMetaData.java @@ -0,0 +1,106 @@ +package com.github.thmarx.cms.filesystem.metadata.memory; + +/*- + * #%L + * cms-server + * %% + * Copyright (C) 2023 Marx-Software + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this program. If not, see + * . + * #L% + */ + +import com.github.thmarx.cms.api.Constants; +import com.github.thmarx.cms.api.db.ContentNode; +import com.github.thmarx.cms.api.db.ContentQuery; +import com.github.thmarx.cms.filesystem.metadata.AbstractMetaData; +import java.io.IOException; +import java.time.LocalDate; +import java.util.ArrayList; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.BiFunction; + +/** + * + * @author t.marx + */ +public class MemoryMetaData extends AbstractMetaData { + + @Override + public void addFile(final String uri, final Map data, final LocalDate lastModified) { + + var parts = uri.split(Constants.SPLIT_PATH_PATTERN); + final ContentNode node = new ContentNode(uri, parts[parts.length - 1], data, lastModified); + + nodes.put(uri, node); + + var folder = getFolder(uri); + if (folder.isPresent()) { + folder.get().children().put(node.name(), node); + } else { + tree.put(node.name(), node); + } + } + + void remove(String uri) { + nodes.remove(uri); + + var folder = getFolder(uri); + var parts = uri.split(Constants.SPLIT_PATH_PATTERN); + var name = parts[parts.length - 1]; + if (folder.isPresent()) { + folder.get().children().remove(name); + } else { + tree.remove(name); + } + } + + @Override + public void open() throws IOException { + if (nodes == null) { + nodes = new ConcurrentHashMap<>(); + tree = new ConcurrentHashMap<>(); + } + } + + @Override + public void close() throws IOException { + nodes = null; + tree = null; + } + + @Override + public ContentQuery query(final BiFunction nodeMapper) { + return new MemoryQuery(new ArrayList<>(nodes.values()), nodeMapper); + } + + @Override + public ContentQuery query(final String startURI, final BiFunction nodeMapper) { + + final String uri; + if (startURI.startsWith("/")) { + uri = startURI.substring(1); + } else { + uri = startURI; + } + + var filtered = getNodes().values().stream().filter(node -> node.uri().startsWith(uri)).toList(); + + return new MemoryQuery(filtered, nodeMapper); + } + + +} diff --git a/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/metadata/memory/MemoryQuery.java b/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/metadata/memory/MemoryQuery.java new file mode 100644 index 00000000..34092cd9 --- /dev/null +++ b/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/metadata/memory/MemoryQuery.java @@ -0,0 +1,204 @@ +package com.github.thmarx.cms.filesystem.metadata.memory; + +/*- + * #%L + * cms-filesystem + * %% + * Copyright (C) 2023 Marx-Software + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this program. If not, see + * . + * #L% + */ +import com.github.thmarx.cms.api.Constants; +import com.github.thmarx.cms.api.db.ContentQuery; +import com.github.thmarx.cms.api.db.ContentNode; +import com.github.thmarx.cms.api.db.Page; +import static com.github.thmarx.cms.filesystem.metadata.memory.QueryUtil.filtered; +import static com.github.thmarx.cms.filesystem.metadata.memory.QueryUtil.sorted; +import com.github.thmarx.cms.api.utils.NodeUtil; +import com.github.thmarx.cms.filesystem.metadata.AbstractMetaData; +import com.github.thmarx.cms.filesystem.metadata.query.ExcerptMapperFunction; +import com.github.thmarx.cms.filesystem.metadata.query.Queries; +import com.github.thmarx.cms.filesystem.metadata.query.ExtendableQuery; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.function.BiFunction; +import java.util.stream.Stream; + +/** + * + * @author t.marx + * @param + */ +public class MemoryQuery extends ExtendableQuery { + + private QueryContext context; + + public MemoryQuery(Collection nodes, BiFunction nodeMapper) { + this(nodes.stream(), new ExcerptMapperFunction<>(nodeMapper)); + } + + public MemoryQuery(Stream nodes, ExcerptMapperFunction nodeMapper) { + this(new QueryContext<>(nodes, nodeMapper, Constants.DEFAULT_CONTENT_TYPE)); + } + + public MemoryQuery(QueryContext context) { + this.context = context; + } + + @Override + public MemoryQuery excerpt(final long excerptLength) { + context.getNodeMapper().setExcerpt((int)excerptLength); + return this; + } + + @Override + public MemoryQuery where(final String field, final Object value) { + return where(field, Queries.Operator.EQ, value); + } + + @Override + public MemoryQuery where(final String field, final String operator, final Object value) { + if (Queries.isDefaultOperation(operator)) { + return where(field, Queries.operator4String(operator), value); + } else if (getContext().getQueryOperations().containsKey(operator)) { + return new MemoryQuery<>(QueryUtil.filter_extension(context, field, value, getContext().getQueryOperations().get(operator))); + } + throw new IllegalArgumentException("unknown operator " + operator); + } + + @Override + public MemoryQuery whereContains(final String field, final Object value) { + return where(field, Queries.Operator.CONTAINS, value); + } + + @Override + public MemoryQuery whereNotContains(final String field, final Object value) { + return where(field, Queries.Operator.CONTAINS_NOT, value); + } + + @Override + public MemoryQuery whereIn(final String field, final Object... value) { + return where(field, Queries.Operator.IN, value); + } + + @Override + public MemoryQuery whereNotIn(final String field, final Object... value) { + return where(field, Queries.Operator.NOT_IN, value); + } + + @Override + public MemoryQuery whereIn(final String field, final List value) { + return where(field, Queries.Operator.IN, value); + } + + @Override + public MemoryQuery whereNotIn(final String field, final List value) { + return where(field, Queries.Operator.NOT_IN, value); + } + + private MemoryQuery where(final String field, final Queries.Operator operator, final Object value) { + return new MemoryQuery<>(filtered(context, field, value, operator)); + } + + + @Override + public MemoryQuery html() { + context.setContentType(Constants.ContentTypes.HTML); + return new MemoryQuery<>(context); + } + + @Override + public MemoryQuery json() { + context.setContentType(Constants.ContentTypes.JSON); + return new MemoryQuery<>(context); + } + + @Override + public MemoryQuery contentType(String contentType) { + context.setContentType(contentType); + return new MemoryQuery<>(context); + } + + @Override + public List get() { + return context.getNodes() + .filter(NodeUtil.contentTypeFiler(context.getContentType())) + .filter(node -> !node.isDirectory()) + .filter(AbstractMetaData::isVisible) + .map(context.getNodeMapper()) + .toList(); + } + + public Page page(final Object page, final Object size) { + int i_page = Constants.DEFAULT_PAGE; + int i_size = Constants.DEFAULT_PAGE_SIZE; + if (page instanceof Integer || page instanceof Long) { + i_page = ((Number) page).intValue(); + } else if (page instanceof String string) { + i_page = Integer.parseInt(string); + } + if (size instanceof Integer || size instanceof Long) { + i_size = ((Number) size).intValue(); + } else if (size instanceof String string) { + i_size = Integer.parseInt(string); + } + return page((int) i_page, (int) i_size); + } + + @Override + public Page page(final long page, final long pageSize) { + long offset = (page - 1) * pageSize; + + var filteredNodes = context.getNodes() + .filter(NodeUtil.contentTypeFiler(context.getContentType())) + .filter(node -> !node.isDirectory()) + .filter(AbstractMetaData::isVisible) + .toList(); + + var totalItems = filteredNodes.size(); + + var filteredTargetNodes = filteredNodes.stream() + .skip(offset) + .limit(pageSize) + .map(context.getNodeMapper()) + .toList(); + + int totalPages = (int) Math.ceil((float) totalItems / pageSize); + return new Page<>(totalItems, pageSize, totalPages, (int)page, filteredTargetNodes); + } + + @Override + public Sort orderby(final String field) { + return new Sort<>(field, context); + } + + @Override + public Map> groupby(final String field) { + return QueryUtil.groupby(context.getNodes(), field); + } + + public static record Sort(String field, QueryContext context) implements ContentQuery.Sort { + + public MemoryQuery asc() { + return new MemoryQuery(sorted(context, field, true)); + } + + public MemoryQuery desc() { + return new MemoryQuery(sorted(context, field, false)); + } + } +} diff --git a/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/query/QueryContext.java b/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/metadata/memory/QueryContext.java similarity index 78% rename from cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/query/QueryContext.java rename to cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/metadata/memory/QueryContext.java index 7fd929a7..12a31b77 100644 --- a/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/query/QueryContext.java +++ b/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/metadata/memory/QueryContext.java @@ -1,4 +1,4 @@ -package com.github.thmarx.cms.filesystem.query; +package com.github.thmarx.cms.filesystem.metadata.memory; /*- * #%L @@ -24,9 +24,7 @@ import com.github.thmarx.cms.api.Constants; import com.github.thmarx.cms.api.db.ContentNode; -import com.github.thmarx.cms.filesystem.index.IndexProviding; -import java.util.Map; -import java.util.function.BiPredicate; +import com.github.thmarx.cms.filesystem.metadata.query.ExcerptMapperFunction; import java.util.stream.Stream; import lombok.AllArgsConstructor; import lombok.Data; @@ -35,6 +33,7 @@ /** * * @author t.marx + * @param */ @Data @AllArgsConstructor @@ -45,11 +44,5 @@ public class QueryContext { private ExcerptMapperFunction nodeMapper; - private IndexProviding indexProviding; - - private boolean useSecondaryIndex = false; - private String contentType = Constants.DEFAULT_CONTENT_TYPE; - - private Map> queryOperations; } diff --git a/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/metadata/memory/QueryUtil.java b/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/metadata/memory/QueryUtil.java new file mode 100644 index 00000000..afa749a2 --- /dev/null +++ b/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/metadata/memory/QueryUtil.java @@ -0,0 +1,97 @@ +package com.github.thmarx.cms.filesystem.metadata.memory; + +/*- + * #%L + * cms-filesystem + * %% + * Copyright (C) 2023 Marx-Software + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this program. If not, see + * . + * #L% + */ +import com.github.thmarx.cms.api.db.ContentNode; +import com.github.thmarx.cms.api.utils.MapUtil; +import com.github.thmarx.cms.filesystem.metadata.query.Queries; +import java.util.List; +import java.util.Map; +import java.util.function.BiPredicate; +import java.util.function.Predicate; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import lombok.extern.slf4j.Slf4j; + +/** + * + * @author t.marx + */ +@Slf4j +public final class QueryUtil { + + + + public static Map> groupby(final Stream nodes, final String field) { + return nodes.collect(Collectors.groupingBy((node) -> MapUtil.getValue(node.data(), field))); + } + + protected static QueryContext sorted(final QueryContext context, final String field, final boolean asc) { + + var tempNodes = context.getNodes().sorted( + (node1, node2) -> { + var value1 = MapUtil.getValue(node1.data(), field); + var value2 = MapUtil.getValue(node2.data(), field); + + return Queries.compare(value1, value2); + } + ).toList(); + + if (!asc) { + tempNodes = tempNodes.reversed(); + } + + context.setNodes(tempNodes.stream()); + + return context; + } + + public static QueryContext filtered(final QueryContext context, final String field, final Object value, final Queries.Operator operator) { + context.setNodes(context.getNodes().filter(createPredicate(field, value, operator))); + return context; + } + + private static Predicate createPredicate(final String field, final Object value, final Queries.Operator operator) { + return (node) -> { + var node_value = MapUtil.getValue(node.data(), field); + + if (node_value == null) { + return false; + } + + if (Queries.filters.containsKey(operator)) { + return Queries.filters.get(operator).matches(node_value, value); + } + + log.error("unknown operation " + operator.name()); + return false; + }; + } + + protected static QueryContext filter_extension(final QueryContext context, final String field, final Object value, final BiPredicate predicate) { + context.setNodes(context.getNodes().filter(Queries.createExtensionPredicate(field, value, predicate))); + return context; + } + + + +} diff --git a/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/metadata/persistent/DocumentHelper.java b/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/metadata/persistent/DocumentHelper.java new file mode 100644 index 00000000..d3ff8833 --- /dev/null +++ b/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/metadata/persistent/DocumentHelper.java @@ -0,0 +1,103 @@ +package com.github.thmarx.cms.filesystem.metadata.persistent; + +/*- + * #%L + * cms-filesystem + * %% + * Copyright (C) 2023 - 2024 Marx-Software + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this program. If not, see + * . + * #L% + */ + +import com.github.thmarx.cms.filesystem.metadata.persistent.utils.FlattenMap; +import java.util.List; +import java.util.Map; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.DoubleField; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.FloatField; +import org.apache.lucene.document.IntField; +import org.apache.lucene.document.LongField; +import org.apache.lucene.document.SortedNumericDocValuesField; +import org.apache.lucene.document.SortedSetDocValuesField; +import org.apache.lucene.document.StringField; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.NumericUtils; + +/** + * + * @author t.marx + */ +public class DocumentHelper { + public static void addData(final Document document, Map data) { + var flatten = FlattenMap.flattenMap(data); + + flatten.entrySet().forEach(entry -> { + + switch (entry.getValue()) { + case List listValue -> + handleList(document, entry.getKey(), listValue); + default -> { + addValue(document, entry.getKey(), entry.getValue()); + } + } + }); + } + + private static void handleList(Document document, String name, List list) { + list.forEach(item -> addValue(document, name, item)); + } + + private static void addValue(Document document, String name, Object value) { + switch (value) { + case String stringValue -> { + document.add(new StringField(name, stringValue, Field.Store.NO)); +// document.add(new SortedSetDocValuesField(name, new BytesRef(stringValue))); + } + case Integer intValue -> { + document.add(new IntField(name, intValue, Field.Store.NO)); +// document.add(new SortedNumericDocValuesField(name, intValue)); + } + case Long longValue -> { + document.add(new LongField(name, longValue, Field.Store.NO)); +// document.add(new SortedNumericDocValuesField(name, longValue)); + } + case Float floatValue -> { + document.add(new FloatField(name, floatValue, Field.Store.NO)); +// document.add(new SortedNumericDocValuesField(name, NumericUtils.floatToSortableInt(floatValue))); + } + case Double doubleValue -> { + document.add(new DoubleField(name, doubleValue, Field.Store.NO)); +// document.add(new SortedNumericDocValuesField(name, NumericUtils.doubleToSortableLong(doubleValue))); + } + case Boolean booleanValue -> { + var intValue = booleanValue ? 1 : 0; + document.add( + new IntField( + name, + intValue, + Field.Store.NO + ) + ); +// document.add(new SortedNumericDocValuesField(name, intValue)); + } + case List listValue -> + handleList(document, name, listValue); + default -> { + } + } + } +} diff --git a/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/metadata/persistent/LuceneIndex.java b/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/metadata/persistent/LuceneIndex.java new file mode 100644 index 00000000..1b3db43d --- /dev/null +++ b/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/metadata/persistent/LuceneIndex.java @@ -0,0 +1,180 @@ +package com.github.thmarx.cms.filesystem.metadata.persistent; + +/*- + * #%L + * cms-filesystem + * %% + * Copyright (C) 2023 - 2024 Marx-Software + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this program. If not, see + * . + * #L% + */ +import com.github.thmarx.cms.api.utils.FileUtils; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import lombok.extern.slf4j.Slf4j; +import org.apache.lucene.analysis.core.KeywordAnalyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.SearcherFactory; +import org.apache.lucene.search.SearcherManager; +import org.apache.lucene.search.Sort; +import org.apache.lucene.search.SortField; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FSDirectory; +import org.apache.lucene.store.NRTCachingDirectory; + +/** + * + * @author t.marx + */ +@Slf4j +public class LuceneIndex implements AutoCloseable { + + private Directory directory; + private IndexWriter writer = null; + + private SearcherManager nrt_manager; + private NRTCachingDirectory nrt_index; + + @Override + public void close() throws Exception { + if (nrt_manager != null) { + nrt_manager.close(); + + writer.commit(); + writer.close(); + directory.close(); + } + } + + public void commit() throws IOException { + writer.flush(); + writer.commit(); + nrt_manager.maybeRefresh(); + } + + void add(Document document) throws IOException { + writer.addDocument(document); + commit(); + } + + void update(Term term, Document document) throws IOException { + writer.updateDocument(term, document); + commit(); + } + + void delete(Query query) throws IOException { + writer.deleteDocuments(query); + commit(); + } + + List query(Query query, Sort sort) throws IOException { + IndexSearcher searcher = nrt_manager.acquire(); + try { + var topDocs = searcher.search(query, Integer.MAX_VALUE, sort); + + List result = new ArrayList<>(); + for (var scoreDoc : topDocs.scoreDocs) { + result.add(searcher.storedFields().document(scoreDoc.doc)); + } + + return result; + } catch (IOException e) { + log.error("", e); + } finally { + nrt_manager.release(searcher); + } + return Collections.emptyList(); + } + + List query(Query query) throws IOException { + IndexSearcher searcher = nrt_manager.acquire(); + try { + var topDocs = searcher.search(query, Integer.MAX_VALUE); + + List result = new ArrayList<>(); + for (var scoreDoc : topDocs.scoreDocs) { + result.add(searcher.storedFields().document(scoreDoc.doc)); + } + + return result; + } catch (IOException e) { + log.error("", e); + } finally { + nrt_manager.release(searcher); + } + return Collections.emptyList(); + } + + public void open(Path path) throws IOException { + if (Files.exists(path)) { + FileUtils.deleteFolder(path); + } + Files.createDirectories(path); + + this.directory = FSDirectory.open(path); + IndexWriterConfig indexWriterConfig = new IndexWriterConfig(new KeywordAnalyzer()); + indexWriterConfig.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND); + indexWriterConfig.setCommitOnClose(true); + nrt_index = new NRTCachingDirectory(directory, 5.0, 60.0); + writer = new IndexWriter(nrt_index, indexWriterConfig); + + final SearcherFactory sf = new SearcherFactory(); + nrt_manager = new SearcherManager(writer, true, true, sf); + } + + SortField.Type getFieldType(String fieldName) throws IOException { + IndexSearcher searcher = nrt_manager.acquire(); + try { + + var fieldInfos = FieldInfos.getMergedFieldInfos(searcher.getIndexReader()); + + var fieldInfo = fieldInfos.fieldInfo(fieldName); + if (fieldInfo == null) { + return null; + } + + return switch (fieldInfo.getDocValuesType()) { + case NUMERIC -> SortField.Type.INT; + case BINARY -> SortField.Type.STRING; + case SORTED -> SortField.Type.STRING; + case SORTED_NUMERIC -> SortField.Type.INT; + case SORTED_SET -> SortField.Type.STRING; + default -> null; + }; // Prüfen Sie, ob das Feld integer, long, float oder double ist + // Diese Information ist nicht direkt in FieldInfo verfügbar, + // Sie müssen dies möglicherweise aus dem Kontext wissen + // Hier nehmen wir an, dass es ein Integer-Feld ist + // Ähnlich wie bei NUMERIC müssen Sie den genauen Typ kennen + // Hier nehmen wir an, dass es ein Integer-Feld ist + } catch (Exception e) { + log.error("", e); + } finally { + nrt_manager.release(searcher); + } + return null; + } +} diff --git a/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/metadata/persistent/LuceneQuery.java b/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/metadata/persistent/LuceneQuery.java new file mode 100644 index 00000000..c17a9ce1 --- /dev/null +++ b/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/metadata/persistent/LuceneQuery.java @@ -0,0 +1,303 @@ +package com.github.thmarx.cms.filesystem.metadata.persistent; + +/*- + * #%L + * cms-filesystem + * %% + * Copyright (C) 2023 - 2024 Marx-Software + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this program. If not, see + * . + * #L% + */ +import com.github.thmarx.cms.api.Constants; +import com.github.thmarx.cms.api.db.ContentNode; +import com.github.thmarx.cms.api.db.ContentQuery; +import com.github.thmarx.cms.api.db.Page; +import com.github.thmarx.cms.filesystem.MetaData; +import com.github.thmarx.cms.filesystem.metadata.AbstractMetaData; +import com.github.thmarx.cms.filesystem.metadata.memory.QueryUtil; +import com.github.thmarx.cms.filesystem.metadata.query.ExcerptMapperFunction; +import com.github.thmarx.cms.filesystem.metadata.query.Queries; +import com.github.thmarx.cms.filesystem.metadata.query.ExtendableQuery; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.function.Predicate; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.lucene.document.Document; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.BooleanClause; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.PrefixQuery; +import org.apache.lucene.search.TermQuery; + +/** + * + * @author t.marx + * @param + */ +@Slf4j +@RequiredArgsConstructor +public class LuceneQuery extends ExtendableQuery implements ContentQuery.Sort { + + private final LuceneIndex index; + private final MetaData metaData; + private final ExcerptMapperFunction nodeMapper; + + private String contentType = Constants.DEFAULT_CONTENT_TYPE; + + private final BooleanQuery.Builder queryBuilder = new BooleanQuery.Builder(); + + enum Order { + ASC, DESC; + } + + private Order sortOrder = Order.ASC; + private Optional orderByField = Optional.empty(); + + private Optional startUri = Optional.empty(); + + private List> extensionOperations = new ArrayList<>(); + + public LuceneQuery( + final String startUri, + final LuceneIndex index, + final MetaData metaData, + final ExcerptMapperFunction nodeMapper) { + this(index, metaData, nodeMapper); + this.startUri = Optional.ofNullable(startUri); + } + + @Override + public ContentQuery excerpt(long excerptLength) { + nodeMapper.setExcerpt((int) excerptLength); + return this; + } + + public Page page(final Object page, final Object size) { + int i_page = Constants.DEFAULT_PAGE; + int i_size = Constants.DEFAULT_PAGE_SIZE; + if (page instanceof Integer || page instanceof Long) { + i_page = ((Number) page).intValue(); + } else if (page instanceof String string) { + i_page = Integer.parseInt(string); + } + if (size instanceof Integer || size instanceof Long) { + i_size = ((Number) size).intValue(); + } else if (size instanceof String string) { + i_size = Integer.parseInt(string); + } + return page((int) i_page, (int) i_size); + } + + @Override + public Page page(long page, long size) { + + long offset = (page - 1) * size; + + var result = queryNodes(); + + var filteredTargetNodes = result.nodes.stream() + .skip(offset) + .limit(size) + .toList(); + + int totalPages = (int) Math.ceil((float) result.total / size); + return new Page<>(result.total, size, totalPages, (int) page, filteredTargetNodes); + } + + @Override + public List get() { + return queryNodes().nodes; + } + + private List queryContentNodes() { + queryBuilder.add(new TermQuery(new Term("content.type", contentType)), BooleanClause.Occur.MUST); + if (startUri.isPresent()) { + queryBuilder.add(new PrefixQuery(new Term("_uri", startUri.get())), BooleanClause.Occur.FILTER); + } + + try { + List result = index.query(queryBuilder.build()); + + var contentNodes = result.stream() + .map(document -> document.get("_uri")) + .map(metaData::byUri) + .filter(Optional::isPresent) + .map(Optional::get) + .filter(node -> !node.isDirectory()) + .filter(AbstractMetaData::isVisible) + .toList(); + + if (!extensionOperations.isEmpty()) { + contentNodes = contentNodes.stream() + .filter((node) -> { + return extensionOperations.stream() + .map(predicate -> predicate.test(node)) + .filter(value -> !value) + .count() == 0; + }) + .toList(); + } + return contentNodes; + } catch (IOException ex) { + log.error("", ex); + } + return Collections.emptyList(); + } + + private NodeResult queryNodes() { + var contentNodes = queryContentNodes(); + + var mappedContentNodes = contentNodes.stream() + .map(nodeMapper) + .toList(); + + var total = contentNodes.size(); + + return new NodeResult<>(total, mappedContentNodes); + } + + @Override + public Map> groupby(String field) { + var nodes = queryContentNodes(); + return QueryUtil.groupby(nodes.stream(), field); + } + + @Override + public Sort orderby(String field) { + this.orderByField = Optional.ofNullable(field); + return this; + } + + @Override + public ContentQuery json() { + this.contentType = Constants.ContentTypes.JSON; + return this; + } + + @Override + public ContentQuery html() { + this.contentType = Constants.ContentTypes.HTML; + return this; + } + + @Override + public ContentQuery contentType(String contentType) { + this.contentType = contentType; + return this; + } + + @Override + public ContentQuery where(String field, Object value) { + return where(field, Queries.Operator.EQ, value); + } + + @Override + public ContentQuery where(String field, String operator, Object value) { + if (Queries.isDefaultOperation(operator)) { + return where(field, Queries.operator4String(operator), value); + } else if (getContext().getQueryOperations().containsKey(operator)) { + extensionOperations.add( + (Predicate) Queries.createExtensionPredicate( + field, + value, + getContext().getQueryOperations().get(operator) + )); + return this; + } + throw new IllegalArgumentException("unknown operator " + operator); + } + + @Override + public ContentQuery whereContains(String field, Object value) { + return where(field, Queries.Operator.CONTAINS, value); + } + + @Override + public ContentQuery whereNotContains(String field, Object value) { + return where(field, Queries.Operator.CONTAINS_NOT, value); + } + + @Override + public ContentQuery whereIn(String field, Object... value) { + return where(field, Queries.Operator.IN, value); + } + + @Override + public ContentQuery whereIn(String field, List value) { + return where(field, Queries.Operator.IN, value); + } + + @Override + public ContentQuery whereNotIn(String field, Object... value) { + return where(field, Queries.Operator.NOT_IN, value); + } + + @Override + public ContentQuery whereNotIn(String field, List value) { + return where(field, Queries.Operator.NOT_IN, value); + } + + private ContentQuery where(final String field, final Queries.Operator operator, final Object value) { + + QueryHelper.exists(queryBuilder, field, value); + + switch (operator) { + case EQ -> + QueryHelper.eq(queryBuilder, field, value, BooleanClause.Occur.MUST); + case NOT_EQ -> + QueryHelper.eq(queryBuilder, field, value, BooleanClause.Occur.MUST_NOT); + case CONTAINS -> + QueryHelper.contains(queryBuilder, field, value, BooleanClause.Occur.MUST); + case CONTAINS_NOT -> + QueryHelper.contains(queryBuilder, field, value, BooleanClause.Occur.MUST_NOT); + case IN -> + QueryHelper.in(queryBuilder, field, value, BooleanClause.Occur.MUST); + case NOT_IN -> + QueryHelper.in(queryBuilder, field, value, BooleanClause.Occur.MUST_NOT); + case LT -> + QueryHelper.lt(queryBuilder, field, value); + case LTE -> + QueryHelper.lte(queryBuilder, field, value); + case GT -> + QueryHelper.gt(queryBuilder, field, value); + case GTE -> + QueryHelper.gte(queryBuilder, field, value); + } + + return this; + } + + @Override + public ContentQuery asc() { + this.sortOrder = Order.ASC; + return this; + } + + @Override + public ContentQuery desc() { + this.sortOrder = Order.DESC; + return this; + } + + private record NodeResult(int total, List nodes) { + + } +} diff --git a/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/metadata/persistent/PersistentMetaData.java b/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/metadata/persistent/PersistentMetaData.java new file mode 100644 index 00000000..252467ee --- /dev/null +++ b/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/metadata/persistent/PersistentMetaData.java @@ -0,0 +1,143 @@ +package com.github.thmarx.cms.filesystem.metadata.persistent; + +/*- + * #%L + * cms-filesystem + * %% + * Copyright (C) 2023 - 2024 Marx-Software + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this program. If not, see + * . + * #L% + */ +import com.github.thmarx.cms.api.Constants; +import com.github.thmarx.cms.api.db.ContentNode; +import com.github.thmarx.cms.api.db.ContentQuery; +import com.github.thmarx.cms.filesystem.metadata.AbstractMetaData; +import com.github.thmarx.cms.filesystem.metadata.query.ExcerptMapperFunction; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.LocalDate; +import java.util.Map; +import java.util.function.BiFunction; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.StringField; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.MatchAllDocsQuery; +import org.h2.mvstore.MVStore; + +/** + * + * @author t.marx + */ +@Slf4j +@RequiredArgsConstructor +public class PersistentMetaData extends AbstractMetaData implements AutoCloseable { + + private final Path hostPath; + + private LuceneIndex index; + private MVStore store; + + @Override + public void open() throws IOException { + + Files.createDirectories(hostPath.resolve("data/metadata/store")); + Files.createDirectories(hostPath.resolve("data/metadata/index")); + + index = new LuceneIndex(); + index.open(hostPath.resolve("data/metadata/index")); + + store = MVStore.open(hostPath.resolve("data/metadata/store/data.db").toString()); + + nodes = store.openMap("nodes"); + tree = store.openMap("tree"); + } + + @Override + public void close() throws IOException { + try { + if (index != null) { + index.close(); + } + if (store != null) { + store.close(); + } + } catch (Exception ex) { + throw new IOException(ex); + } + } + + @Override + public void addFile(String uri, Map data, LocalDate lastModified) { + + var parts = uri.split(Constants.SPLIT_PATH_PATTERN); + final ContentNode node = new ContentNode(uri, parts[parts.length - 1], data, lastModified); + + nodes.put(uri, node); + + var folder = getFolder(uri); + if (folder.isPresent()) { + folder.get().children().put(node.name(), node); + } else { + tree.put(node.name(), node); + } + + Document document = new Document(); + document.add(new StringField("_uri", uri, Field.Store.YES)); + //document.add(new StringField("_source", GSON.toJson(node), Field.Store.NO)); + + DocumentHelper.addData(document, data); + + document.add(new StringField("content.type", node.contentType(), Field.Store.NO)); + + try { + this.index.update(new Term("_uri", uri), document); + } catch (IOException ex) { + log.error("", ex); + } + } + + @Override + public void clear() { + super.clear(); + try { + index.delete(new MatchAllDocsQuery()); + } catch (IOException ex) { + log.error("", ex); + } + } + + @Override + public ContentQuery query(final BiFunction nodeMapper) { + return new LuceneQuery<>(this.index, this, new ExcerptMapperFunction<>(nodeMapper)); + } + + @Override + public ContentQuery query(final String startURI, final BiFunction nodeMapper) { + + final String uri; + if (startURI.startsWith("/")) { + uri = startURI.substring(1); + } else { + uri = startURI; + } + return new LuceneQuery<>(uri, this.index, this, new ExcerptMapperFunction<>(nodeMapper)); + } + +} diff --git a/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/metadata/persistent/QueryHelper.java b/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/metadata/persistent/QueryHelper.java new file mode 100644 index 00000000..2991c4c6 --- /dev/null +++ b/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/metadata/persistent/QueryHelper.java @@ -0,0 +1,295 @@ +package com.github.thmarx.cms.filesystem.metadata.persistent; + +/*- + * #%L + * cms-filesystem + * %% + * Copyright (C) 2023 - 2024 Marx-Software + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this program. If not, see + * . + * #L% + */ + +import com.github.thmarx.cms.api.db.ContentNode; +import com.github.thmarx.cms.api.utils.MapUtil; +import com.github.thmarx.cms.filesystem.metadata.query.Queries; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import lombok.extern.slf4j.Slf4j; +import org.apache.lucene.document.DoubleField; +import org.apache.lucene.document.FloatField; +import org.apache.lucene.document.IntField; +import org.apache.lucene.document.LongField; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.BooleanClause; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.TermRangeQuery; + +/** + * + * @author t.marx + */ +@Slf4j +public class QueryHelper { + + public static void exists (BooleanQuery.Builder queryBuilder, String field, Object value) { + if (value.getClass().isArray()) { + value = ((Object[])value)[1]; + } + if (value instanceof String) { + queryBuilder.add( + TermRangeQuery.newStringRange(field, null, null, true, true), + BooleanClause.Occur.FILTER + ); + } else if (value instanceof Float) { + queryBuilder.add( + FloatField.newRangeQuery(field, Float.MIN_VALUE, Float.MAX_VALUE), + BooleanClause.Occur.FILTER + ); + } else if (value instanceof Double) { + queryBuilder.add( + DoubleField.newRangeQuery(field, Double.MIN_VALUE, Double.MAX_VALUE), + BooleanClause.Occur.FILTER + ); + } else if (value instanceof Integer) { + queryBuilder.add( + IntField.newRangeQuery(field, Integer.MIN_VALUE, Integer.MAX_VALUE), + BooleanClause.Occur.FILTER + ); + } else if (value instanceof Long) { + queryBuilder.add( + LongField.newRangeQuery(field, Long.MIN_VALUE, Long.MAX_VALUE), + BooleanClause.Occur.FILTER + ); + } else if (value instanceof Boolean) { + queryBuilder.add( + IntField.newRangeQuery(field, Integer.MIN_VALUE, Integer.MAX_VALUE), + BooleanClause.Occur.FILTER + ); + } + } + + public static void lt(BooleanQuery.Builder queryBuilder, String field, Object value) { + switch (value) { + case Integer numberValue -> { + queryBuilder.add( + IntField.newRangeQuery(field, Integer.MIN_VALUE, numberValue - 1), + BooleanClause.Occur.MUST + ); + } + case Long numberValue -> { + queryBuilder.add( + LongField.newRangeQuery(field, Integer.MIN_VALUE, numberValue - 1), + BooleanClause.Occur.MUST + ); + } + case Float numberValue -> { + queryBuilder.add( + FloatField.newRangeQuery(field, Integer.MIN_VALUE, numberValue - 0.0001f), + BooleanClause.Occur.MUST + ); + } + case Double numberValue -> { + queryBuilder.add( + DoubleField.newRangeQuery(field, Integer.MIN_VALUE, numberValue - 0.0001), + BooleanClause.Occur.MUST + ); + } + default -> { + } + } + } + + public static void lte(BooleanQuery.Builder queryBuilder, String field, Object value) { + switch (value) { + case Integer numberValue -> { + queryBuilder.add( + IntField.newRangeQuery(field, Integer.MIN_VALUE, numberValue), + BooleanClause.Occur.MUST + ); + } + case Long numberValue -> { + queryBuilder.add( + LongField.newRangeQuery(field, Integer.MIN_VALUE, numberValue), + BooleanClause.Occur.MUST + ); + } + case Float numberValue -> { + queryBuilder.add( + FloatField.newRangeQuery(field, Integer.MIN_VALUE, numberValue), + BooleanClause.Occur.MUST + ); + } + case Double numberValue -> { + queryBuilder.add( + DoubleField.newRangeQuery(field, Integer.MIN_VALUE, numberValue), + BooleanClause.Occur.MUST + ); + } + default -> { + } + } + } + + public static void gt(BooleanQuery.Builder queryBuilder, String field, Object value) { + switch (value) { + case Integer numberValue -> { + queryBuilder.add( + IntField.newRangeQuery(field, numberValue + 1, Integer.MAX_VALUE), + BooleanClause.Occur.MUST + ); + } + case Long numberValue -> { + queryBuilder.add( + LongField.newRangeQuery(field, numberValue + 1, Long.MAX_VALUE), + BooleanClause.Occur.MUST + ); + } + case Float numberValue -> { + queryBuilder.add( + FloatField.newRangeQuery(field, numberValue + 0.0001f, Float.MAX_VALUE), + BooleanClause.Occur.MUST + ); + } + case Double numberValue -> { + queryBuilder.add( + DoubleField.newRangeQuery(field, numberValue + 0.0001, Double.MAX_VALUE), + BooleanClause.Occur.MUST + ); + } + default -> { + } + } + } + + public static void gte(BooleanQuery.Builder queryBuilder, String field, Object value) { + switch (value) { + case Integer numberValue -> { + queryBuilder.add( + IntField.newRangeQuery(field, numberValue, Integer.MAX_VALUE), + BooleanClause.Occur.MUST + ); + } + case Long numberValue -> { + queryBuilder.add( + LongField.newRangeQuery(field, numberValue, Long.MAX_VALUE), + BooleanClause.Occur.MUST + ); + } + case Float numberValue -> { + queryBuilder.add( + FloatField.newRangeQuery(field, numberValue, Float.MAX_VALUE), + BooleanClause.Occur.MUST + ); + } + case Double numberValue -> { + queryBuilder.add( + DoubleField.newRangeQuery(field, numberValue, Double.MAX_VALUE), + BooleanClause.Occur.MUST + ); + } + default -> { + } + } + } + + public static void eq(BooleanQuery.Builder queryBuilder, String field, Object value, BooleanClause.Occur occur) { + var query = toQuery(field, value); + if (query != null) { + queryBuilder.add(query, occur); + } + } + + public static void contains(BooleanQuery.Builder queryBuilder, String field, Object value, BooleanClause.Occur occur) { + var query = toQuery(field, value); + if (query != null) { + queryBuilder.add( + TermRangeQuery.newStringRange(field, null, null, true, true), + BooleanClause.Occur.FILTER + ); + queryBuilder.add(query, occur); + } + } + + public static void in(BooleanQuery.Builder queryBuilder, String field, Object value, BooleanClause.Occur occur) { + if (value == null) { + log.warn("value is null"); + return; + } + if (!(value instanceof List || value.getClass().isArray())) { + log.warn("value is not of type list"); + return; + } + + BooleanQuery.Builder inBuilder = new BooleanQuery.Builder(); + + List listValues = Collections.emptyList(); + if (value instanceof List) { + listValues = (List) value; + } else if (value.getClass().isArray()) { + listValues = Arrays.asList((Object[]) value); + } + + listValues.forEach(item -> { + inBuilder.add(toQuery(field, item), BooleanClause.Occur.SHOULD); + }); + + var inQuery = inBuilder.build(); + if (!inQuery.clauses().isEmpty()) { + queryBuilder.add(inQuery, occur); + } + } + + private static Query toQuery(final String field, final Object value) { + if (value instanceof String stringValue) { + return new TermQuery(new Term(field, stringValue)); + } else if (value instanceof Boolean booleanValue) { + return IntField.newExactQuery( + field, + booleanValue ? 1 : 0 + ); + } else if (value instanceof Integer numberValue) { + return IntField.newExactQuery(field, numberValue); + } else if (value instanceof Long numberValue) { + return LongField.newExactQuery(field, numberValue); + } else if (value instanceof Float numberValue) { + return FloatField.newExactQuery(field, numberValue); + } else if (value instanceof Double numberValue) { + return DoubleField.newExactQuery(field, numberValue); + } + return null; + } + + protected static List sorted(final List nodes, final String field, final boolean asc) { + + var tempNodes = nodes.stream().sorted( + (node1, node2) -> { + var value1 = MapUtil.getValue(((ContentNode)node1).data(), field); + var value2 = MapUtil.getValue(((ContentNode)node2).data(), field); + + return Queries.compare(value1, value2); + } + ).toList(); + + if (!asc) { + tempNodes = tempNodes.reversed(); + } + + return tempNodes; + } +} diff --git a/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/metadata/persistent/utils/FlattenMap.java b/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/metadata/persistent/utils/FlattenMap.java new file mode 100644 index 00000000..42df776e --- /dev/null +++ b/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/metadata/persistent/utils/FlattenMap.java @@ -0,0 +1,49 @@ +package com.github.thmarx.cms.filesystem.metadata.persistent.utils; + +/*- + * #%L + * cms-filesystem + * %% + * Copyright (C) 2023 - 2024 Marx-Software + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this program. If not, see + * . + * #L% + */ + +import java.util.HashMap; +import java.util.Map; + +public class FlattenMap { + + public static Map flattenMap(Map map) { + Map result = new HashMap<>(); + flattenMap("", map, result); + return result; + } + + private static void flattenMap(String prefix, Map map, Map result) { + for (Map.Entry entry : map.entrySet()) { + String key = prefix.isEmpty() ? entry.getKey() : prefix + "." + entry.getKey(); + Object value = entry.getValue(); + + if (value instanceof Map) { + // Rekursion für verschachtelte Maps + flattenMap(key, (Map) value, result); + } else { + result.put(key, value); + } + } + } +} diff --git a/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/query/ExcerptMapperFunction.java b/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/metadata/query/ExcerptMapperFunction.java similarity index 95% rename from cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/query/ExcerptMapperFunction.java rename to cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/metadata/query/ExcerptMapperFunction.java index 18ca6bd8..6002855a 100644 --- a/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/query/ExcerptMapperFunction.java +++ b/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/metadata/query/ExcerptMapperFunction.java @@ -1,4 +1,4 @@ -package com.github.thmarx.cms.filesystem.query; +package com.github.thmarx.cms.filesystem.metadata.query; /*- * #%L diff --git a/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/metadata/query/ExtendableQuery.java b/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/metadata/query/ExtendableQuery.java new file mode 100644 index 00000000..efe3cec3 --- /dev/null +++ b/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/metadata/query/ExtendableQuery.java @@ -0,0 +1,56 @@ +package com.github.thmarx.cms.filesystem.metadata.query; + +import com.github.thmarx.cms.api.db.ContentQuery; +import java.util.HashMap; +import java.util.Map; +import java.util.function.BiPredicate; +import lombok.Data; +import lombok.Getter; + +/*- + * #%L + * cms-filesystem + * %% + * Copyright (C) 2023 - 2024 Marx-Software + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this program. If not, see + * . + * #L% + */ + + +/** + * + * @author t.marx + */ +public abstract class ExtendableQuery implements ContentQuery { + + @Getter + private final Context context = new Context(); + + public ContentQuery addCustomOperators (String operator, BiPredicate queryOperations) { + context.queryOperations.put(operator, queryOperations); + return this; + } + + public ContentQuery addAllCustomOperators (Map> queryOperations) { + context.queryOperations.putAll(queryOperations); + return this; + } + + @Data + public static class Context { + private final Map> queryOperations = new HashMap<>(); + } +} diff --git a/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/query/Filter.java b/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/metadata/query/Filter.java similarity index 86% rename from cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/query/Filter.java rename to cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/metadata/query/Filter.java index 38e9f6fa..3a43d87b 100644 --- a/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/query/Filter.java +++ b/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/metadata/query/Filter.java @@ -1,4 +1,4 @@ -package com.github.thmarx.cms.filesystem.query; +package com.github.thmarx.cms.filesystem.metadata.query; /*- * #%L @@ -22,9 +22,6 @@ * #L% */ -import com.github.thmarx.cms.api.db.ContentNode; -import java.util.function.Predicate; - /** * * @author thmar diff --git a/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/query/QueryUtil.java b/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/metadata/query/Queries.java similarity index 59% rename from cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/query/QueryUtil.java rename to cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/metadata/query/Queries.java index 8fa5c809..b4bf21d8 100644 --- a/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/query/QueryUtil.java +++ b/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/metadata/query/Queries.java @@ -1,10 +1,10 @@ -package com.github.thmarx.cms.filesystem.query; +package com.github.thmarx.cms.filesystem.metadata.query; /*- * #%L * cms-filesystem * %% - * Copyright (C) 2023 Marx-Software + * Copyright (C) 2023 - 2024 Marx-Software * %% * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as @@ -21,9 +21,9 @@ * . * #L% */ + import com.github.thmarx.cms.api.db.ContentNode; import com.github.thmarx.cms.api.utils.MapUtil; -import com.github.thmarx.cms.filesystem.index.SecondaryIndex; import com.google.common.base.Strings; import java.util.Arrays; import java.util.Collections; @@ -34,17 +34,12 @@ import java.util.Objects; import java.util.function.BiPredicate; import java.util.function.Predicate; -import java.util.stream.Collectors; -import java.util.stream.Stream; -import lombok.extern.slf4j.Slf4j; /** * * @author t.marx */ -@Slf4j -public final class QueryUtil { - +public class Queries { public static enum Operator { CONTAINS, CONTAINS_NOT, @@ -58,7 +53,7 @@ public static enum Operator { LTE; } - private static Map filters = new HashMap<>(); + public final static Map filters = new HashMap<>(); static { filters.put(Operator.EQ, (node_value, value) -> Objects.equals(node_value, value)); @@ -89,9 +84,15 @@ public static enum Operator { }); } - private static List operations = List.of( - "=" , "!=", ">", ">=", "<", "<=", - "in", "not in", "contains", "not contains" + private static final List operations = List.of( + "=" , "eq", + "!=", "not eq", + ">", "gt", + ">=", "gte", + "<", "lt", + "<=", "lte", + "in", "not in", + "contains", "not contains" ); public static boolean isDefaultOperation (final String operation) { return operations.contains(operation); @@ -104,16 +105,28 @@ public static Operator operator4String(final String operator) { return switch (operator) { case "=" -> Operator.EQ; + case "eq" -> + Operator.EQ; case "!=" -> Operator.NOT_EQ; + case "not eq" -> + Operator.NOT_EQ; case ">" -> Operator.GT; + case "gt" -> + Operator.GT; case ">=" -> Operator.GTE; + case "gte" -> + Operator.GTE; case "<" -> Operator.LT; + case "lt" -> + Operator.LT; case "<=" -> Operator.LTE; + case "lte" -> + Operator.LTE; case "in" -> Operator.IN; case "not in" -> @@ -126,32 +139,8 @@ public static Operator operator4String(final String operator) { throw new IllegalArgumentException("unknown operator " + operator); }; } - - protected static Map> groupby(final Stream nodes, final String field) { - return nodes.collect(Collectors.groupingBy((node) -> MapUtil.getValue(node.data(), field))); - } - - protected static QueryContext sorted(final QueryContext context, final String field, final boolean asc) { - - var tempNodes = context.getNodes().sorted( - (node1, node2) -> { - var value1 = MapUtil.getValue(node1.data(), field); - var value2 = MapUtil.getValue(node2.data(), field); - - return compare(value1, value2); - } - ).toList(); - - if (!asc) { - tempNodes = tempNodes.reversed(); - } - - context.setNodes(tempNodes.stream()); - - return context; - } - - private static int compare(Object o1, Object o2) { + + public static int compare(Object o1, Object o2) { if (Objects.equals(o1, o2)) { return 0; } @@ -184,47 +173,8 @@ private static int compare(Object o1, Object o2) { return 0; } - - protected static QueryContext filteredWithIndex(final QueryContext context, final String field, final Object value, final Operator operator) { - - if (Operator.EQ.equals(operator)) { - SecondaryIndex index = (SecondaryIndex) context.getIndexProviding().getOrCreateIndex(field, node -> MapUtil.getValue(node.data(), field)); - context.setNodes(context.getNodes().filter(node -> index.eq(node, value))); - return context; - } else { - context.setNodes(context.getNodes().filter(createPredicate(field, value, operator))); - return context; - } - } - - protected static QueryContext filtered(final QueryContext context, final String field, final Object value, final Operator operator) { - context.setNodes(context.getNodes().filter(createPredicate(field, value, operator))); - return context; - } - - private static Predicate createPredicate(final String field, final Object value, final Operator operator) { - return (node) -> { - var node_value = MapUtil.getValue(node.data(), field); - - if (node_value == null) { - return false; - } - - if (filters.containsKey(operator)) { - return filters.get(operator).matches(node_value, value); - } - - log.error("unknown operation " + operator.name()); - return false; - }; - } - - protected static QueryContext filter_extension(final QueryContext context, final String field, final Object value, final BiPredicate predicate) { - context.setNodes(context.getNodes().filter(createExtensionPredicate(field, value, predicate))); - return context; - } - - private static Predicate createExtensionPredicate(final String field, final Object value, final BiPredicate predicate) { + + public static Predicate createExtensionPredicate(final String field, final Object value, final BiPredicate predicate) { return (node) -> { var node_value = MapUtil.getValue(node.data(), field); @@ -235,5 +185,4 @@ private static Predicate createExtensionPredicate(final Str return predicate.test(node_value, value); }; } - } diff --git a/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/query/Query.java b/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/query/Query.java deleted file mode 100644 index 9b6157ea..00000000 --- a/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/query/Query.java +++ /dev/null @@ -1,217 +0,0 @@ -package com.github.thmarx.cms.filesystem.query; - -/*- - * #%L - * cms-filesystem - * %% - * Copyright (C) 2023 Marx-Software - * %% - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public - * License along with this program. If not, see - * . - * #L% - */ -import com.github.thmarx.cms.api.Constants; -import com.github.thmarx.cms.api.db.ContentQuery; -import com.github.thmarx.cms.api.db.ContentNode; -import com.github.thmarx.cms.api.db.Page; -import com.github.thmarx.cms.filesystem.MetaData; -import com.github.thmarx.cms.filesystem.index.IndexProviding; -import static com.github.thmarx.cms.filesystem.query.QueryUtil.filtered; -import static com.github.thmarx.cms.filesystem.query.QueryUtil.filteredWithIndex; -import static com.github.thmarx.cms.filesystem.query.QueryUtil.sorted; -import com.github.thmarx.cms.api.utils.NodeUtil; -import java.util.Collection; -import java.util.List; -import java.util.Map; -import java.util.function.BiFunction; -import java.util.function.BiPredicate; -import java.util.stream.Stream; - -/** - * - * @author t.marx - * @param - */ -public class Query implements ContentQuery { - - private QueryContext context; - - public Query(Collection nodes, IndexProviding indexProviding, BiFunction nodeMapper) { - this(nodes.stream(), indexProviding, new ExcerptMapperFunction<>(nodeMapper)); - } - - public Query(Stream nodes, IndexProviding indexProviding, ExcerptMapperFunction nodeMapper) { - this(new QueryContext(nodes, nodeMapper, indexProviding, false, Constants.DEFAULT_CONTENT_TYPE, Map.of())); - } - - public Query(QueryContext context) { - this.context = context; - } - - public Query setCustomOperators (Map> queryOperations) { - context.setQueryOperations(queryOperations); - return this; - } - - @Override - public Query excerpt(final long excerptLength) { - context.getNodeMapper().setExcerpt((int)excerptLength); - return this; - } - - @Override - public Query where(final String field, final Object value) { - return where(field, QueryUtil.Operator.EQ, value); - } - - @Override - public Query where(final String field, final String operator, final Object value) { - if (QueryUtil.isDefaultOperation(operator)) { - return where(field, QueryUtil.operator4String(operator), value); - } else if (context.getQueryOperations().containsKey(operator)) { - return new Query<>(QueryUtil.filter_extension(context, field, value, context.getQueryOperations().get(operator))); - } - throw new IllegalArgumentException("unknown operator " + operator); - } - - @Override - public Query whereContains(final String field, final Object value) { - return where(field, QueryUtil.Operator.CONTAINS, value); - } - - @Override - public Query whereNotContains(final String field, final Object value) { - return where(field, QueryUtil.Operator.CONTAINS_NOT, value); - } - - @Override - public Query whereIn(final String field, final Object... value) { - return where(field, QueryUtil.Operator.IN, value); - } - - @Override - public Query whereNotIn(final String field, final Object... value) { - return where(field, QueryUtil.Operator.NOT_IN, value); - } - - @Override - public Query whereIn(final String field, final List value) { - return where(field, QueryUtil.Operator.IN, value); - } - - @Override - public Query whereNotIn(final String field, final List value) { - return where(field, QueryUtil.Operator.NOT_IN, value); - } - - private Query where(final String field, final QueryUtil.Operator operator, final Object value) { - if (context.isUseSecondaryIndex()) { - return new Query(filteredWithIndex(context, field, value, operator)); - } else { - return new Query(filtered(context, field, value, operator)); - } - } - - public Query enableSecondaryIndex() { - context.setUseSecondaryIndex(true); - return new Query<>(context); - } - - @Override - public Query html() { - context.setContentType(Constants.ContentTypes.HTML); - return new Query<>(context); - } - - @Override - public Query json() { - context.setContentType(Constants.ContentTypes.JSON); - return new Query<>(context); - } - - @Override - public Query contentType(String contentType) { - context.setContentType(contentType); - return new Query<>(context); - } - - @Override - public List get() { - return context.getNodes() - .filter(NodeUtil.contentTypeFiler(context.getContentType())) - .filter(node -> !node.isDirectory()) - .filter(MetaData::isVisible) - .map(context.getNodeMapper()) - .toList(); - } - - public Page page(final Object page, final Object size) { - int i_page = Constants.DEFAULT_PAGE; - int i_size = Constants.DEFAULT_PAGE_SIZE; - if (page instanceof Integer || page instanceof Long) { - i_page = ((Number) page).intValue(); - } else if (page instanceof String string) { - i_page = Integer.parseInt(string); - } - if (size instanceof Integer || size instanceof Long) { - i_size = ((Number) size).intValue(); - } else if (size instanceof String string) { - i_size = Integer.parseInt(string); - } - return page((int) i_page, (int) i_size); - } - - @Override - public Page page(final long page, final long size) { - long offset = (page - 1) * size; - - var filteredNodes = context.getNodes() - .filter(NodeUtil.contentTypeFiler(context.getContentType())) - .filter(node -> !node.isDirectory()) - .filter(MetaData::isVisible) - .toList(); - - var total = filteredNodes.size(); - - var filteredTargetNodes = filteredNodes.stream() - .skip(offset) - .limit(size) - .map(context.getNodeMapper()) - .toList(); - - int totalPages = (int) Math.ceil((float) total / size); - return new Page(filteredNodes.size(), totalPages, (int)page, filteredTargetNodes); - } - - @Override - public Sort orderby(final String field) { - return new Sort(field, context); - } - - @Override - public Map> groupby(final String field) { - return QueryUtil.groupby(context.getNodes(), field); - } - - public static record Sort(String field, QueryContext context) implements ContentQuery.Sort { - - public Query asc() { - return new Query(sorted(context, field, true)); - } - - public Query desc() { - return new Query(sorted(context, field, false)); - } - } -} diff --git a/cms-filesystem/src/test/java/com/github/thmarx/cms/filesystem/AbstractFileSystemTest.java b/cms-filesystem/src/test/java/com/github/thmarx/cms/filesystem/AbstractFileSystemTest.java new file mode 100644 index 00000000..8adb5165 --- /dev/null +++ b/cms-filesystem/src/test/java/com/github/thmarx/cms/filesystem/AbstractFileSystemTest.java @@ -0,0 +1,107 @@ +package com.github.thmarx.cms.filesystem; + +/*- + * #%L + * cms-filesystem + * %% + * Copyright (C) 2023 - 2024 Marx-Software + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this program. If not, see + * . + * #L% + */ + +import com.github.thmarx.cms.filesystem.metadata.query.ExtendableQuery; +import java.io.IOException; +import java.util.Map; +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.Test; + +/** + * + * @author t.marx + */ +public abstract class AbstractFileSystemTest { + + protected abstract FileSystem getFileSystem (); + + @Test + public void test_seconday_index() throws IOException { + +// var dimension = fileSystem.createDimension("featured", (ContentNode node) -> node.data().containsKey("featured") ? (Boolean) node.data().get("featured") : false, Boolean.class); + +// Assertions.assertThat(dimension.filter(Boolean.TRUE)).hasSize(2); +// Assertions.assertThat(dimension.filter(Boolean.FALSE)).hasSize(1); + } + + @Test + public void test_query() throws IOException { + var nodes = getFileSystem().query((node, i) -> node).where("featured", true).get(); + Assertions.assertThat(nodes).hasSize(2); + } + + @Test + public void test_query_in() throws IOException { + var nodes = getFileSystem().query((node, i) -> node).whereIn("name", "test1", "test2").get(); + Assertions.assertThat(nodes).hasSize(2); + } + + @Test + public void test_query_not_in() throws IOException { + var nodes = getFileSystem().query((node, i) -> node).whereNotIn("name", "test1", "test2").get(); + Assertions.assertThat(nodes).hasSize(0); + } + @Test + public void test_query_contains() throws IOException { + var nodes = getFileSystem().query((node, i) -> node).whereContains("taxonomy.tags", "eins").get(); + Assertions.assertThat(nodes).hasSize(1); + } + + @Test + public void test_query_contains_not() throws IOException { + var nodes = getFileSystem().query((node, i) -> node).whereNotContains("taxonomy.tags", "eins").get(); + Assertions.assertThat(nodes).hasSize(1); + } + + @Test + public void test_lt_lte() throws IOException { + var nodes = getFileSystem().query((node, i) -> node).where("number2", "<", 5).get(); + Assertions.assertThat(nodes).hasSize(0); + + nodes = getFileSystem().query((node, i) -> node).where("number2", "lte", 5).get(); + Assertions.assertThat(nodes).hasSize(1); + } + + @Test + public void test_query_with_start_uri() throws IOException { + + var nodes = getFileSystem().query("/test", (node, i) -> node).where("featured", true).get(); + + Assertions.assertThat(nodes).hasSize(1); + Assertions.assertThat(nodes.getFirst().uri()).isEqualTo("test/test1.md"); + } + + @Test + public void test_custom_operation() throws IOException { + + var query = getFileSystem().query((node, i) -> node); + var nodes = query.get(); + Assertions.assertThat(nodes).hasSize(3); + + query = getFileSystem().query((node, i) -> node); + ((ExtendableQuery)query).addCustomOperators("none", (value1, value2) -> false); + nodes = query.where("featured", "none").get(); + Assertions.assertThat(nodes).hasSize(0); + } +} diff --git a/cms-filesystem/src/test/java/com/github/thmarx/cms/filesystem/FileSystemTest.java b/cms-filesystem/src/test/java/com/github/thmarx/cms/filesystem/FileSystemTest.java index b545332c..9a21ae6d 100644 --- a/cms-filesystem/src/test/java/com/github/thmarx/cms/filesystem/FileSystemTest.java +++ b/cms-filesystem/src/test/java/com/github/thmarx/cms/filesystem/FileSystemTest.java @@ -36,7 +36,7 @@ * * @author t.marx */ -public class FileSystemTest { +public class FileSystemTest extends AbstractFileSystemTest { static FileSystem fileSystem; @@ -60,29 +60,8 @@ static void shutdown () { fileSystem.shutdown(); } - @Test - public void test_seconday_index() throws IOException { - -// var dimension = fileSystem.createDimension("featured", (ContentNode node) -> node.data().containsKey("featured") ? (Boolean) node.data().get("featured") : false, Boolean.class); - -// Assertions.assertThat(dimension.filter(Boolean.TRUE)).hasSize(2); -// Assertions.assertThat(dimension.filter(Boolean.FALSE)).hasSize(1); - } - - @Test - public void test_query() throws IOException { - - var nodes = fileSystem.query((node, i) -> node).where("featured", true).get(); - - Assertions.assertThat(nodes).hasSize(2); - } - - @Test - public void test_query_with_start_uri() throws IOException { - - var nodes = fileSystem.query("/test", (node, i) -> node).where("featured", true).get(); - - Assertions.assertThat(nodes).hasSize(1); - Assertions.assertThat(nodes.getFirst().uri()).isEqualTo("test/test1.md"); + @Override + protected FileSystem getFileSystem() { + return fileSystem; } } diff --git a/cms-filesystem/src/test/java/com/github/thmarx/cms/filesystem/PresistentFileSystemTest.java b/cms-filesystem/src/test/java/com/github/thmarx/cms/filesystem/PresistentFileSystemTest.java new file mode 100644 index 00000000..e9dcb049 --- /dev/null +++ b/cms-filesystem/src/test/java/com/github/thmarx/cms/filesystem/PresistentFileSystemTest.java @@ -0,0 +1,73 @@ +package com.github.thmarx.cms.filesystem; + +/*- + * #%L + * cms-filesystem + * %% + * Copyright (C) 2023 Marx-Software + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this program. If not, see + * . + * #L% + */ +import com.github.thmarx.cms.api.eventbus.EventBus; +import com.github.thmarx.cms.api.utils.FileUtils; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; +import org.yaml.snakeyaml.Yaml; + +/** + * + * @author t.marx + */ +public class PresistentFileSystemTest extends AbstractFileSystemTest { + + static FileSystem fileSystem; + + + @BeforeAll + static void setup() throws IOException { + + var eventBus = Mockito.mock(EventBus.class); + + fileSystem = new FileSystem(Path.of("src/test/resources"), eventBus, (file) -> { + try { + return new Yaml().load(Files.readString(file)); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + fileSystem.init(MetaData.Type.PERSISTENT); + } + + @AfterAll + static void shutdown () throws IOException { + fileSystem.shutdown(); + + if (Files.exists(Path.of("src/test/resources/data"))) { + FileUtils.deleteFolder(Path.of("src/test/resources/data")); + } + } + + @Override + protected FileSystem getFileSystem() { + return fileSystem; + } +} diff --git a/cms-filesystem/src/test/java/com/github/thmarx/cms/filesystem/index/SecondaryIndexTest.java b/cms-filesystem/src/test/java/com/github/thmarx/cms/filesystem/index/SecondaryIndexTest.java deleted file mode 100644 index 9098a5b6..00000000 --- a/cms-filesystem/src/test/java/com/github/thmarx/cms/filesystem/index/SecondaryIndexTest.java +++ /dev/null @@ -1,55 +0,0 @@ -package com.github.thmarx.cms.filesystem.index; - -/*- - * #%L - * cms-filesystem - * %% - * Copyright (C) 2023 Marx-Software - * %% - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public - * License along with this program. If not, see - * . - * #L% - */ - -import com.github.thmarx.cms.api.db.ContentNode; -import com.github.thmarx.cms.api.utils.MapUtil; -import java.util.Map; -import org.assertj.core.api.Assertions; -import org.junit.jupiter.api.Test; - -/** - * - * @author t.marx - */ -public class SecondaryIndexTest { - - @Test - public void testSomeMethod() { - SecondaryIndex index = SecondaryIndex.builder() - .indexFunction(node -> MapUtil.getValue(node.data(), "name", "")) - .build(); - final ContentNode node1 = new ContentNode("node1", "Eins", Map.of("name", "Eins")); - final ContentNode node2 = new ContentNode("node2", "Eins", Map.of("name", "Eins")); - final ContentNode node3 = new ContentNode("node3", "Eins", Map.of("name", "Zwei")); - - index.add(node1); - index.add(node2); - index.add(node3); - - Assertions.assertThat(index.eq(node1, "Eins")).isTrue(); - Assertions.assertThat(index.eq(node2, "Eins")).isTrue(); - Assertions.assertThat(index.eq(node3, "Eins")).isFalse(); - } - -} diff --git a/cms-filesystem/src/test/java/com/github/thmarx/cms/filesystem/persistent/utils/FlattenMapTest.java b/cms-filesystem/src/test/java/com/github/thmarx/cms/filesystem/persistent/utils/FlattenMapTest.java new file mode 100644 index 00000000..2de6f391 --- /dev/null +++ b/cms-filesystem/src/test/java/com/github/thmarx/cms/filesystem/persistent/utils/FlattenMapTest.java @@ -0,0 +1,92 @@ +package com.github.thmarx.cms.filesystem.persistent.utils; + +/*- + * #%L + * cms-filesystem + * %% + * Copyright (C) 2023 - 2024 Marx-Software + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this program. If not, see + * . + * #L% + */ + +import com.github.thmarx.cms.filesystem.metadata.persistent.utils.FlattenMap; +import org.junit.jupiter.api.Test; + +import java.util.HashMap; +import java.util.Map; +import org.assertj.core.api.Assertions; + + +public class FlattenMapTest { + + @Test + public void testFlattenMap() { + // Beispielinput + Map nestedMap = new HashMap<>(); + Map nestedLevel1 = new HashMap<>(); + Map nestedLevel2 = new HashMap<>(); + + nestedLevel2.put("key3", "value3"); + nestedLevel1.put("key2", nestedLevel2); + nestedMap.put("key1", nestedLevel1); + nestedMap.put("key4", "value4"); + + // Erwartete flache Map + Map expectedFlatMap = new HashMap<>(); + expectedFlatMap.put("key1.key2.key3", "value3"); + expectedFlatMap.put("key4", "value4"); + + // Flache Map erzeugen + Map actualFlatMap = FlattenMap.flattenMap(nestedMap); + + // Überprüfen, ob die flache Map korrekt ist + Assertions.assertThat(actualFlatMap).isEqualTo(expectedFlatMap); + } + + @Test + public void testFlattenMapWithEmptyMap() { + // Leere Map + Map emptyMap = new HashMap<>(); + + // Erwartete flache Map + Map expectedFlatMap = new HashMap<>(); + + // Flache Map erzeugen + Map actualFlatMap = FlattenMap.flattenMap(emptyMap); + + // Überprüfen, ob die flache Map korrekt ist + Assertions.assertThat(actualFlatMap).isEqualTo(expectedFlatMap); + } + + @Test + public void testFlattenMapWithSingleLevelMap() { + // Einfache Map ohne Verschachtelung + Map singleLevelMap = new HashMap<>(); + singleLevelMap.put("key1", "value1"); + singleLevelMap.put("key2", "value2"); + + // Erwartete flache Map + Map expectedFlatMap = new HashMap<>(); + expectedFlatMap.put("key1", "value1"); + expectedFlatMap.put("key2", "value2"); + + // Flache Map erzeugen + Map actualFlatMap = FlattenMap.flattenMap(singleLevelMap); + + // Überprüfen, ob die flache Map korrekt ist + Assertions.assertThat(actualFlatMap).isEqualTo(expectedFlatMap); + } +} diff --git a/cms-filesystem/src/test/java/com/github/thmarx/cms/filesystem/query/QueryPerfTest.java b/cms-filesystem/src/test/java/com/github/thmarx/cms/filesystem/query/QueryPerfTest.java index c854a9e3..b0178c43 100644 --- a/cms-filesystem/src/test/java/com/github/thmarx/cms/filesystem/query/QueryPerfTest.java +++ b/cms-filesystem/src/test/java/com/github/thmarx/cms/filesystem/query/QueryPerfTest.java @@ -21,10 +21,9 @@ * . * #L% */ +import com.github.thmarx.cms.filesystem.metadata.memory.MemoryQuery; import com.github.thmarx.cms.api.Constants; import com.github.thmarx.cms.api.db.ContentNode; -import com.github.thmarx.cms.filesystem.index.IndexProviding; -import com.github.thmarx.cms.filesystem.index.SecondaryIndex; import java.time.Instant; import java.time.temporal.ChronoUnit; import java.util.ArrayList; @@ -51,26 +50,6 @@ public class QueryPerfTest { private static int COUNT = 1000; - private static IndexProviding indexProviding = new IndexProviding() { - private ConcurrentMap> secondaryIndexes = new ConcurrentHashMap<>(); - - @Override - public SecondaryIndex getOrCreateIndex(String field, Function indexFunction) { - System.out.println("check field: " + field); - System.out.println(secondaryIndexes.keySet()); - if (!secondaryIndexes.containsKey(field)) { - System.out.println("build index"); - var index = SecondaryIndex.builder() - .indexFunction(indexFunction) - .build(); - index.addAll(nodes); - secondaryIndexes.put(field, index); - } - - return secondaryIndexes.get(field); - } - }; - @BeforeAll public static void setup() { System.out.println("build elements"); @@ -86,11 +65,9 @@ public static void setup() { } } - protected Query createQuery(boolean useIndex) { - var query = new Query<>(nodes, indexProviding, (node, i) -> node); - if (useIndex) { - query.enableSecondaryIndex(); - } + protected MemoryQuery createQuery() { + var query = new MemoryQuery<>(nodes, (node, i) -> node); + return query; } @@ -102,7 +79,7 @@ class NoIndex { public void test_no_index() { System.out.println("run tests without index"); - Query query = createQuery(false); + MemoryQuery query = createQuery(); var nodes = query.where("article.featured", true).get(); Assertions.assertThat(nodes).hasSize(COUNT / 2); @@ -128,7 +105,7 @@ class WithIndex { public void test_use_index() { System.out.println("run tests with index"); - Query query = createQuery(true); + MemoryQuery query = createQuery(); var nodes = query.where("article.featured", true).get(); Assertions.assertThat(nodes).hasSize(COUNT / 2); diff --git a/cms-filesystem/src/test/java/com/github/thmarx/cms/filesystem/query/QueryTest.java b/cms-filesystem/src/test/java/com/github/thmarx/cms/filesystem/query/QueryTest.java index 952aff88..1404d199 100644 --- a/cms-filesystem/src/test/java/com/github/thmarx/cms/filesystem/query/QueryTest.java +++ b/cms-filesystem/src/test/java/com/github/thmarx/cms/filesystem/query/QueryTest.java @@ -21,10 +21,9 @@ * . * #L% */ +import com.github.thmarx.cms.filesystem.metadata.memory.MemoryQuery; import com.github.thmarx.cms.api.Constants; import com.github.thmarx.cms.api.db.ContentNode; -import com.github.thmarx.cms.filesystem.index.IndexProviding; -import com.github.thmarx.cms.filesystem.index.SecondaryIndex; import java.time.Instant; import java.time.temporal.ChronoUnit; import java.util.ArrayList; @@ -47,23 +46,6 @@ public class QueryTest { private static Collection nodes; - private IndexProviding indexProviding = new IndexProviding() { - private ConcurrentMap> secondaryIndexes = new ConcurrentHashMap<>(); - - @Override - public SecondaryIndex getOrCreateIndex(String field, Function indexFunction) { - if (!secondaryIndexes.containsKey(field)) { - var index = SecondaryIndex.builder() - .indexFunction(indexFunction) - .build(); - index.addAll(nodes); - secondaryIndexes.put(field, index); - } - - return secondaryIndexes.get(field); - } - }; - @BeforeAll public static void setup() { nodes = new ArrayList<>(); @@ -100,25 +82,23 @@ public static void setup() { nodes.add(node); } - protected Query createQuery() { - var query = new Query<>(nodes, indexProviding, (node, i) -> node); - query.setCustomOperators(Map.of( - "none", (node_value, value) -> false - )); + protected MemoryQuery createQuery() { + var query = new MemoryQuery<>(nodes, (node, i) -> node); + query.addCustomOperators("none", (node_value, value) -> false); return query; } @Test public void test_custom_operator() { - Query query = createQuery(); + MemoryQuery query = createQuery(); var nodes = query.where("featured", "none", true).get(); Assertions.assertThat(nodes).hasSize(0); } @Test public void test_eq() { - Query query = createQuery(); + MemoryQuery query = createQuery(); var nodes = query.where("featured", true).get(); Assertions.assertThat(nodes).hasSize(1); Assertions.assertThat(nodes.getFirst().uri()).isEqualTo("/2"); @@ -131,7 +111,7 @@ public void test_eq() { @Test public void test_not() { - Query query = createQuery(); + MemoryQuery query = createQuery(); var nodes = query.where("featured", "!=", true).get(); Assertions.assertThat(nodes).hasSize(2); Assertions.assertThat(nodes.stream().map(ContentNode::uri).toList()).contains("/test1", "/test2"); @@ -139,14 +119,14 @@ public void test_not() { @Test public void test_data() { - Query query = createQuery(); + MemoryQuery query = createQuery(); var nodes = query.get(); Assertions.assertThat(nodes).hasSize(3); } @Test public void test_sort_asc() { - Query query = createQuery(); + MemoryQuery query = createQuery(); var nodes = query.where("featured", false).orderby("index").asc().get(); Assertions.assertThat(nodes).hasSize(2); Assertions.assertThat(nodes.get(0).uri()).isEqualTo("/test1"); @@ -155,7 +135,7 @@ public void test_sort_asc() { @Test public void test_sort_desc() { - Query query = createQuery(); + MemoryQuery query = createQuery(); var nodes = query.where("featured", false).orderby("index").desc().get(); Assertions.assertThat(nodes).hasSize(2); Assertions.assertThat(nodes.get(0).uri()).isEqualTo("/test2"); @@ -164,7 +144,7 @@ public void test_sort_desc() { @Test public void test_offset_0() { - Query query = createQuery(); + MemoryQuery query = createQuery(); var page = query.where("featured", false).orderby("index").desc().page(1, 1); Assertions.assertThat(page.getItems()).hasSize(1); Assertions.assertThat(page.getItems().get(0).uri()).isEqualTo("/test2"); @@ -172,7 +152,7 @@ public void test_offset_0() { @Test public void test_offset_1() { - Query query = createQuery(); + MemoryQuery query = createQuery(); var page = query.where("featured", false).orderby("index").desc().page(2, 1); Assertions.assertThat(page.getItems()).hasSize(1); Assertions.assertThat(page.getItems().getFirst().uri()).isEqualTo("/test1"); @@ -180,7 +160,7 @@ public void test_offset_1() { @Test public void test_contains() { - Query query = createQuery(); + MemoryQuery query = createQuery(); var nodes = query.whereContains("tags", "one").get(); Assertions.assertThat(nodes).hasSize(1); Assertions.assertThat(nodes.get(0).uri()).isEqualTo("/test2"); @@ -188,7 +168,7 @@ public void test_contains() { @Test public void test_contains_operator() { - Query query = createQuery(); + MemoryQuery query = createQuery(); var nodes = query.where("tags", "contains", "one").get(); Assertions.assertThat(nodes).hasSize(1); Assertions.assertThat(nodes.get(0).uri()).isEqualTo("/test2"); @@ -196,7 +176,7 @@ public void test_contains_operator() { @Test public void test_not_contains() { - Query query = createQuery(); + MemoryQuery query = createQuery(); var nodes = query.whereNotContains("tags", "one").get(); Assertions.assertThat(nodes).hasSize(1); Assertions.assertThat(nodes.get(0).uri()).isEqualTo("/test1"); @@ -204,7 +184,7 @@ public void test_not_contains() { @Test public void test_not_contains_operator() { - Query query = createQuery(); + MemoryQuery query = createQuery(); var nodes = query.where("tags", "not contains", "one").get(); Assertions.assertThat(nodes).hasSize(1); Assertions.assertThat(nodes.get(0).uri()).isEqualTo("/test1"); @@ -212,7 +192,7 @@ public void test_not_contains_operator() { @Test public void test_gt() { - Query query = createQuery(); + MemoryQuery query = createQuery(); var nodes = query.where("index", ">", 1).get(); Assertions.assertThat(nodes).hasSize(1); Assertions.assertThat(nodes.get(0).uri()).isEqualTo("/test2"); @@ -220,7 +200,7 @@ public void test_gt() { @Test public void test_gte() { - Query query = createQuery(); + MemoryQuery query = createQuery(); var nodes = query.where("index", ">=", 2).get(); Assertions.assertThat(nodes).hasSize(1); Assertions.assertThat(nodes.get(0).uri()).isEqualTo("/test2"); @@ -228,7 +208,7 @@ public void test_gte() { @Test public void test_lt() { - Query query = createQuery(); + MemoryQuery query = createQuery(); var nodes = query.where("index", "<", 2).get(); Assertions.assertThat(nodes).hasSize(1); Assertions.assertThat(nodes.get(0).uri()).isEqualTo("/test1"); @@ -236,7 +216,7 @@ public void test_lt() { @Test public void test_lte() { - Query query = createQuery(); + MemoryQuery query = createQuery(); var nodes = query.where("index", "<=", 1).get(); Assertions.assertThat(nodes).hasSize(1); Assertions.assertThat(nodes.get(0).uri()).isEqualTo("/test1"); @@ -244,7 +224,7 @@ public void test_lte() { @Test public void test_group_by() { - Query query = createQuery(); + MemoryQuery query = createQuery(); var nodes = query.groupby("featured"); Assertions.assertThat(nodes).hasSize(2); Assertions.assertThat(nodes).containsKeys(true, false); @@ -254,7 +234,7 @@ public void test_group_by() { @Test public void test_in() { - Query query = createQuery(); + MemoryQuery query = createQuery(); var nodes = query.whereIn("index", 1, 2).get(); Assertions.assertThat(nodes).hasSize(2); Assertions.assertThat(nodes.stream().map(ContentNode::uri).toList()).contains("/test1", "/test2"); @@ -262,7 +242,7 @@ public void test_in() { @Test public void test_in_operator() { - Query query = createQuery(); + MemoryQuery query = createQuery(); var nodes = query.where("index", "in", List.of(1, 2)).get(); Assertions.assertThat(nodes).hasSize(2); Assertions.assertThat(nodes.stream().map(ContentNode::uri).toList()).contains("/test1", "/test2"); @@ -275,7 +255,7 @@ public void test_in_operator() { @Test public void test_not_in() { - Query query = createQuery(); + MemoryQuery query = createQuery(); var nodes = query.whereNotIn("index", 1, 3, 4).get(); Assertions.assertThat(nodes).hasSize(1); Assertions.assertThat(nodes.stream().map(ContentNode::uri).toList()).contains("/test2"); @@ -283,7 +263,7 @@ public void test_not_in() { @Test public void test_not_in_operator() { - Query query = createQuery(); + MemoryQuery query = createQuery(); var nodes = query.where("index", "not in", List.of(1, 3, 4)).get(); Assertions.assertThat(nodes).hasSize(1); Assertions.assertThat(nodes.stream().map(ContentNode::uri).toList()).contains("/test2"); @@ -296,7 +276,7 @@ public void test_not_in_operator() { @Test public void test_json() { - Query query = createQuery(); + MemoryQuery query = createQuery(); var nodes = query.json().page(1, 1).getItems(); Assertions.assertThat(nodes).hasSize(1); Assertions.assertThat(nodes.get(0).uri()).isEqualTo("/json"); @@ -304,7 +284,7 @@ public void test_json() { @Test public void test_where_not_null() { - Query query = createQuery(); + MemoryQuery query = createQuery(); var nodes = query.where("index", "!=", null).get(); Assertions.assertThat(nodes).hasSize(2); diff --git a/cms-filesystem/src/test/resources/content/index.md b/cms-filesystem/src/test/resources/content/index.md index 77ec3cb9..605899c6 100644 --- a/cms-filesystem/src/test/resources/content/index.md +++ b/cms-filesystem/src/test/resources/content/index.md @@ -1 +1 @@ -featured: true +featured: true \ No newline at end of file diff --git a/cms-filesystem/src/test/resources/content/test/test1.md b/cms-filesystem/src/test/resources/content/test/test1.md index 14cac3ef..795372c8 100644 --- a/cms-filesystem/src/test/resources/content/test/test1.md +++ b/cms-filesystem/src/test/resources/content/test/test1.md @@ -1,4 +1,6 @@ name: test1 featured: true taxonomy: - tags: [eins, zwei] \ No newline at end of file + tags: [eins, zwei] +number1: 1 +number2: 5 \ No newline at end of file diff --git a/cms-server/.gitignore b/cms-server/.gitignore new file mode 100644 index 00000000..ad80b637 --- /dev/null +++ b/cms-server/.gitignore @@ -0,0 +1 @@ +hosts/feature/data/ \ No newline at end of file diff --git a/cms-server/hosts/features/site.yaml b/cms-server/hosts/features/site.yaml index 142e9cd1..0d302961 100644 --- a/cms-server/hosts/features/site.yaml +++ b/cms-server/hosts/features/site.yaml @@ -5,6 +5,8 @@ baseurl: "http://localhost2:1010" language: en test: Hallo theme: test +index: + persistent: true modules: active: - forms-module diff --git a/cms-server/pom.xml b/cms-server/pom.xml index e51e2b84..59632e39 100644 --- a/cms-server/pom.xml +++ b/cms-server/pom.xml @@ -108,12 +108,10 @@ info.picocli picocli - 4.7.6 org.semver4j semver4j - 5.3.0 diff --git a/cms-server/src/main/java/com/github/thmarx/cms/server/configs/SiteModule.java b/cms-server/src/main/java/com/github/thmarx/cms/server/configs/SiteModule.java index 78dddf1d..55e07321 100644 --- a/cms-server/src/main/java/com/github/thmarx/cms/server/configs/SiteModule.java +++ b/cms-server/src/main/java/com/github/thmarx/cms/server/configs/SiteModule.java @@ -49,6 +49,7 @@ import com.github.thmarx.cms.eventbus.DefaultEventBus; import com.github.thmarx.cms.extensions.ExtensionManager; import com.github.thmarx.cms.filesystem.FileDB; +import com.github.thmarx.cms.filesystem.MetaData; import com.github.thmarx.cms.template.functions.taxonomy.TaxonomyFunction; import com.github.thmarx.cms.media.FileMediaService; import com.github.thmarx.cms.media.SiteMediaManager; @@ -163,7 +164,7 @@ public MessageSource messages(SiteProperties site, DB db) throws IOException { @Provides @Singleton - public DB fileDb(ContentParser contentParser, Configuration configuration, EventBus eventBus) throws IOException { + public DB fileDb(SiteProperties site, ContentParser contentParser, Configuration configuration, EventBus eventBus) throws IOException { var db = new FileDB(hostBase, eventBus, (file) -> { try { return contentParser.parseMeta(file); @@ -172,7 +173,11 @@ public DB fileDb(ContentParser contentParser, Configuration configuration, Event throw new RuntimeException(ioe); } }, configuration); - db.init(); + if (site.peristentIndex()) { + db.init(MetaData.Type.PERSISTENT); + } else { + db.init(); + } return db; } diff --git a/cms-server/src/test/java/com/github/thmarx/cms/SectionsTest.java b/cms-server/src/test/java/com/github/thmarx/cms/SectionsTest.java index 651042be..af1b36d5 100644 --- a/cms-server/src/test/java/com/github/thmarx/cms/SectionsTest.java +++ b/cms-server/src/test/java/com/github/thmarx/cms/SectionsTest.java @@ -27,7 +27,7 @@ import com.github.thmarx.cms.api.configuration.Configuration; import com.github.thmarx.cms.api.db.ContentNode; import com.github.thmarx.cms.eventbus.DefaultEventBus; -import com.github.thmarx.cms.filesystem.MetaData; +import com.github.thmarx.cms.filesystem.metadata.memory.MemoryMetaData; import com.github.thmarx.cms.api.markdown.MarkdownRenderer; import com.github.thmarx.cms.api.template.TemplateEngine; import com.github.thmarx.cms.content.Section; diff --git a/cms-server/src/test/java/com/github/thmarx/cms/content/views/ViewParserTest.java b/cms-server/src/test/java/com/github/thmarx/cms/content/views/ViewParserTest.java index 6535ea7f..f79e799c 100644 --- a/cms-server/src/test/java/com/github/thmarx/cms/content/views/ViewParserTest.java +++ b/cms-server/src/test/java/com/github/thmarx/cms/content/views/ViewParserTest.java @@ -29,8 +29,6 @@ import com.github.thmarx.cms.api.markdown.MarkdownRenderer; import com.github.thmarx.cms.api.request.RequestContext; import com.github.thmarx.cms.content.DefaultContentParser; -import com.github.thmarx.cms.content.views.ViewParser; -import com.github.thmarx.cms.content.views.ViewParser; import com.github.thmarx.cms.eventbus.DefaultEventBus; import com.github.thmarx.cms.filesystem.FileDB; import java.io.IOException; diff --git a/cms-server/src/test/java/com/github/thmarx/cms/utils/NodeUtilNGTest.java b/cms-server/src/test/java/com/github/thmarx/cms/utils/NodeUtilNGTest.java index 0e923b3e..36d1f158 100644 --- a/cms-server/src/test/java/com/github/thmarx/cms/utils/NodeUtilNGTest.java +++ b/cms-server/src/test/java/com/github/thmarx/cms/utils/NodeUtilNGTest.java @@ -25,7 +25,7 @@ import com.github.thmarx.cms.api.utils.NodeUtil; import com.github.thmarx.cms.api.Constants; import com.github.thmarx.cms.api.db.ContentNode; -import com.github.thmarx.cms.filesystem.MetaData; +import com.github.thmarx.cms.filesystem.metadata.memory.MemoryMetaData; import java.util.Map; import org.assertj.core.api.Assertions; import org.junit.jupiter.api.Test; diff --git a/cms-server/themes/test/templates/blog.html b/cms-server/themes/test/templates/blog.html index bf0b875a..85c75e2b 100644 --- a/cms-server/themes/test/templates/blog.html +++ b/cms-server/themes/test/templates/blog.html @@ -34,7 +34,7 @@

> Previous -
  • Next diff --git a/cms-template/src/main/java/com/github/thmarx/cms/template/functions/list/NodeListFunction.java b/cms-template/src/main/java/com/github/thmarx/cms/template/functions/list/NodeListFunction.java index 99feea36..947046aa 100644 --- a/cms-template/src/main/java/com/github/thmarx/cms/template/functions/list/NodeListFunction.java +++ b/cms-template/src/main/java/com/github/thmarx/cms/template/functions/list/NodeListFunction.java @@ -147,7 +147,7 @@ Page getNodes(final String start, int page, int size, int excerptLengt }); int totalPages = (int) Math.ceil((float) total / size); - return new Page(navNodes.size(), totalPages, page, navNodes); + return new Page<>(total, navNodes.size(), totalPages, page, navNodes); } else { return getNodesFromBase(baseNode, path, page, size, comparator, nodeFilter); @@ -200,7 +200,7 @@ public Page getNodesFromBase(final Path base, final String start, fina }); int totalPages = (int) Math.ceil((float) total / pageSize); - return new Page(pageSize, totalPages, page, nodes); + return new Page(total, pageSize, totalPages, page, nodes); } catch (Exception ex) { log.error(null, ex); } diff --git a/cms-template/src/main/java/com/github/thmarx/cms/template/functions/query/QueryFunction.java b/cms-template/src/main/java/com/github/thmarx/cms/template/functions/query/QueryFunction.java index 6f351aa7..cd7343ac 100644 --- a/cms-template/src/main/java/com/github/thmarx/cms/template/functions/query/QueryFunction.java +++ b/cms-template/src/main/java/com/github/thmarx/cms/template/functions/query/QueryFunction.java @@ -30,7 +30,8 @@ import com.github.thmarx.cms.template.functions.AbstractCurrentNodeFunction; import com.github.thmarx.cms.api.model.ListNode; import com.github.thmarx.cms.api.request.RequestContext; -import com.github.thmarx.cms.filesystem.query.Query; +import com.github.thmarx.cms.filesystem.metadata.memory.MemoryQuery; +import com.github.thmarx.cms.filesystem.metadata.query.ExtendableQuery; import com.google.common.base.Strings; import java.nio.file.Path; import java.util.HashMap; @@ -82,7 +83,7 @@ private BiFunction nodeMapper() { public ContentQuery create() { var query = db.getContent().query(nodeMapper()); - ((Query)query).setCustomOperators(extendedQueryOperations); + ((ExtendableQuery)query).addAllCustomOperators(extendedQueryOperations); if (!Strings.isNullOrEmpty(contentType)) { query.contentType(contentType); @@ -92,7 +93,9 @@ public ContentQuery create() { public ContentQuery create(final String startUri) { var query = db.getContent().query(startUri, nodeMapper()); - ((Query)query).setCustomOperators(extendedQueryOperations); + + ((ExtendableQuery)query).addAllCustomOperators(extendedQueryOperations); + if (!Strings.isNullOrEmpty(contentType)) { query.contentType(contentType); } diff --git a/pom.xml b/pom.xml index 1c60464d..91c25624 100644 --- a/pom.xml +++ b/pom.xml @@ -26,7 +26,7 @@ cms-content cms-extensions cms-auth - integration-tests + integration-tests 2023 @@ -122,7 +122,7 @@ com.google.guava guava - 33.2.0-jre + 33.2.1-jre com.google.inject @@ -130,6 +130,22 @@ 7.0.0 + + org.apache.lucene + lucene-core + 9.11.1 + + + org.apache.lucene + lucene-analysis-common + 9.11.1 + + + com.h2database + h2-mvstore + 2.2.224 + + org.graalvm.polyglot polyglot @@ -180,7 +196,7 @@ org.assertj assertj-core - 3.25.3 + 3.26.0 test @@ -207,6 +223,16 @@ awaitility 4.2.1 + + info.picocli + picocli + 4.7.6 + + + org.semver4j + semver4j + 5.3.0 + @@ -292,7 +318,7 @@ org.apache.maven.plugins maven-surefire-plugin - 3.2.5 + 3.3.0 @@ -318,7 +344,7 @@ org.apache.maven.plugins maven-deploy-plugin - 3.1.1 + 3.1.2