From ac56496fcb954178d82c763096a2009ec4445847 Mon Sep 17 00:00:00 2001 From: Silvan Heller Date: Mon, 7 Feb 2022 15:31:50 +0100 Subject: [PATCH] New Cottontail API, PolyphenyDB (#224) * Upgraded to new Cottontail API * New Polypheny-DB Connector * New DB Abstractions * LSC Import Code which needed the new Cottontail API Co-authored-by: Ralph Gasser Co-authored-by: Loris Sauter Former-commit-id: c450c76aeb9f127f7b13842915e523daf3d1b608 --- .github/workflows/gradle.yml | 10 +- build.gradle | 2 +- .../org/vitrivr/cineast/api/APIEndpoint.java | 3 +- .../org/vitrivr/cineast/api/GRPCEndpoint.java | 1 - cineast-core/build.gradle | 5 +- .../cineast/core/config/DatabaseConfig.java | 168 +----- .../vitrivr/cineast/core/db/DBSelector.java | 3 + .../cineast/core/db/DBSelectorSupplier.java | 8 +- .../vitrivr/cineast/core/db/DataSource.java | 109 ++++ .../vitrivr/cineast/core/db/NoDBWriter.java | 5 + .../cineast/core/db/PersistencyWriter.java | 18 +- .../core/db/PersistencyWriterSupplier.java | 6 +- .../core/db/adampro/ADAMproEntityCreator.java | 1 + .../db/adampro/ADAMproMessageBuilder.java | 2 +- .../core/db/adampro/ADAMproSelector.java | 1 + .../db/adampro/ADAMproStreamingSelector.java | 1 + .../core/db/adampro/ADAMproWrapper.java | 1 + .../core/db/adampro/ADAMproWriter.java | 9 +- .../db/adampro/AbstractADAMproSelector.java | 3 +- .../cottontaildb/CottontailEntityCreator.java | 61 ++- .../db/cottontaildb/CottontailSelector.java | 182 +++---- .../db/cottontaildb/CottontailWrapper.java | 92 ++-- .../db/cottontaildb/CottontailWriter.java | 29 +- .../db/dao/reader/AbstractEntityReader.java | 2 - .../writer/AbstractBatchedEntityWriter.java | 10 +- .../core/db/dao/writer/BatchedTagWriter.java | 6 +- .../dao/writer/MediaObjectMetadataWriter.java | 8 +- .../core/db/dao/writer/MediaObjectWriter.java | 2 +- .../writer/MediaSegmentMetadataWriter.java | 8 +- .../db/dao/writer/MediaSegmentWriter.java | 6 +- ...veTypeProviderFeatureDescriptorWriter.java | 6 +- .../db/dao/writer/SimpleBitSetWriter.java | 4 +- .../writer/SimpleFeatureDescriptorWriter.java | 6 +- ...SimpleFulltextFeatureDescriptorWriter.java | 7 +- .../cineast/core/db/json/JsonFileWriter.java | 5 + .../core/db/memory/InMemoryWriter.java | 5 + .../db/polypheny/PolyphenyEntityCreator.java | 290 ++++++++++ .../core/db/polypheny/PolyphenySelector.java | 498 ++++++++++++++++++ .../core/db/polypheny/PolyphenyWrapper.java | 75 +++ .../core/db/polypheny/PolyphenyWriter.java | 173 ++++++ .../extraction/ExtractionContextProvider.java | 11 +- .../core/features/AverageColorRaster.java | 5 +- .../features/AverageColorRasterReduced11.java | 2 +- .../features/AverageColorRasterReduced15.java | 2 +- .../cineast/core/features/DCTImageHash.java | 5 +- .../core/features/ForegroundBoundingBox.java | 5 +- .../core/features/MedianColorRaster.java | 2 +- .../cineast/core/features/STMP7EH.java | 2 +- .../cineast/core/features/SegmentTags.java | 4 +- .../AbstractCodebookFeatureModule.java | 9 +- .../abstracts/AbstractFeatureModule.java | 6 +- .../abstracts/AbstractTextRetriever.java | 5 +- .../abstracts/MetadataFeatureModule.java | 4 +- .../abstracts/SubDivMotionHistogram.java | 2 +- .../exporter/AudioSegmentExporter.java | 2 +- .../exporter/AudioSpectogramExporter.java | 2 +- .../exporter/AudioWaveformExporter.java | 2 +- .../core/features/exporter/CENSExporter.java | 2 +- .../features/exporter/ChromagramExporter.java | 2 +- .../core/features/exporter/FrameExporter.java | 2 +- .../exporter/Model3DThumbnailExporter.java | 2 +- .../exporter/MotionFrameExporter.java | 2 +- .../exporter/MotionHistoryImageExporter.java | 2 +- .../exporter/RepresentativeFrameExporter.java | 2 +- .../exporter/ShotDescriptorExporter.java | 2 +- .../exporter/ShotThumbnailsExporter.java | 2 +- .../core/features/extractor/Extractor.java | 2 +- .../core/features/retriever/Retriever.java | 4 +- .../core/db/DBBooleanIntegrationTest.java | 133 +++-- .../cineast/core/db/DBIntegrationTest.java | 168 +++--- .../core/db/IntegrationDBProvider.java | 3 +- .../CottontailBooleanIntegrationTest.java | 7 +- .../CottontailIntegrationDBProvider.java | 30 +- .../CottontailIntegrationTest.java | 12 +- .../cottontaildb/CottontailMetadataTest.java | 16 +- .../cineast/core/db/dao/MetadataTest.java | 99 ++-- .../PolyphenyBooleanIntegrationTest.java | 57 ++ .../PolyphenyIntegrationDBProvider.java | 64 +++ .../polyphenydb/PolyphenyIntegrationTest.java | 113 ++++ .../db/polyphenydb/PolyphenyMetadataTest.java | 38 ++ .../cineast/standalone/cli/CineastCli.java | 3 +- .../standalone/cli/ExtractionCommand.java | 4 +- .../cineast/standalone/cli/ImportCommand.java | 4 +- .../cli/OptimizeEntitiesCommand.java | 11 +- .../cli/db/LSC21TemporalUpdateCommand.java | 97 +++- .../cli/db/PolyphenyBenchmarkCommand.java | 303 +++++++++++ .../standalone/config/IngestConfig.java | 8 +- .../importer/handlers/DataImportHandler.java | 29 +- .../importer/lsc2020/LSCUtilities.java | 11 +- .../v3c1analysis/ColorlabelImportHandler.java | 2 +- .../run/GenericExtractionItemHandler.java | 14 +- .../runtime/ExtractionPipeline.java | 4 +- cineast.json | 5 +- gradle.properties | 6 +- 94 files changed, 2427 insertions(+), 739 deletions(-) create mode 100644 cineast-core/src/main/java/org/vitrivr/cineast/core/db/DataSource.java create mode 100644 cineast-core/src/main/java/org/vitrivr/cineast/core/db/polypheny/PolyphenyEntityCreator.java create mode 100644 cineast-core/src/main/java/org/vitrivr/cineast/core/db/polypheny/PolyphenySelector.java create mode 100644 cineast-core/src/main/java/org/vitrivr/cineast/core/db/polypheny/PolyphenyWrapper.java create mode 100644 cineast-core/src/main/java/org/vitrivr/cineast/core/db/polypheny/PolyphenyWriter.java create mode 100644 cineast-core/src/test/java/org/vitrivr/cineast/core/db/polyphenydb/PolyphenyBooleanIntegrationTest.java create mode 100644 cineast-core/src/test/java/org/vitrivr/cineast/core/db/polyphenydb/PolyphenyIntegrationDBProvider.java create mode 100644 cineast-core/src/test/java/org/vitrivr/cineast/core/db/polyphenydb/PolyphenyIntegrationTest.java create mode 100644 cineast-core/src/test/java/org/vitrivr/cineast/core/db/polyphenydb/PolyphenyMetadataTest.java create mode 100644 cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/db/PolyphenyBenchmarkCommand.java diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index 119334db0..24eed03d1 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -17,16 +17,24 @@ jobs: # Setup Cottontail DB service container services: cottontail: - image: vitrivr/cottontaildb:0.12.11 + image: vitrivr/cottontaildb:0.13.3 ports: - 1865:1865 options: -it + polypheny-ci: + image: silvanheller/polypheny-vitrivr-ci:0.4.1 + ports: + - 8070:8070 + - 13137:13137 + - 20591:20591 # Start actual job. steps: - uses: actions/checkout@v2 - name: nc connection test run: nc -zv 127.0.0.1 1865 + - name: polypheny connection test + run: nc -zv 127.0.0.1 13137 - name: Set up JDK 11 uses: actions/setup-java@v2 with: diff --git a/build.gradle b/build.gradle index 3cc107580..542d5ee3c 100644 --- a/build.gradle +++ b/build.gradle @@ -11,7 +11,7 @@ allprojects { group = 'org.vitrivr' /* Our current version, on dev branch this should always be release+1-SNAPSHOT */ - version = '3.6.4' + version = '3.7.0' apply plugin: 'java-library' apply plugin: 'maven-publish' diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/APIEndpoint.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/APIEndpoint.java index e13ce1a7c..3acab9eef 100644 --- a/cineast-api/src/main/java/org/vitrivr/cineast/api/APIEndpoint.java +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/APIEndpoint.java @@ -93,8 +93,7 @@ public class APIEndpoint { /** * The retrieval logic used to retrieve */ - public static ContinuousRetrievalLogic retrievalLogic = new ContinuousRetrievalLogic( - Config.sharedConfig().getDatabase()); + public static ContinuousRetrievalLogic retrievalLogic = new ContinuousRetrievalLogic(Config.sharedConfig().getDatabase()); /** * The single instance of this class diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/GRPCEndpoint.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/GRPCEndpoint.java index 7f36da99b..5aa047600 100644 --- a/cineast-api/src/main/java/org/vitrivr/cineast/api/GRPCEndpoint.java +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/GRPCEndpoint.java @@ -34,7 +34,6 @@ public static void start() { try { server.start(); } catch (IOException e) { - e.printStackTrace(); } } diff --git a/cineast-core/build.gradle b/cineast-core/build.gradle index da5136500..379daf889 100644 --- a/cineast-core/build.gradle +++ b/cineast-core/build.gradle @@ -98,9 +98,12 @@ signing { dependencies { /** THE Cottontail DB proto dependency */ - api group: 'org.vitrivr', name: 'cottontaildb-proto', version: version_cottontail_proto + api group: 'org.vitrivr', name: 'cottontaildb-proto', version: version_cottontaildb_proto api group: 'org.jetbrains.kotlin', name: 'kotlin-stdlib', version: version_kotlin_stdlib + /** THE Polypheny dependency */ + api group: "org.polypheny", name: "polypheny-jdbc-driver", version: version_polypheny + /** THE ADAMpro proto dependency (legacy) */ api group: 'org.vitrivr', name: 'adampro-proto', version: version_adampro diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/config/DatabaseConfig.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/config/DatabaseConfig.java index 3c49a0c2e..b925c3984 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/config/DatabaseConfig.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/config/DatabaseConfig.java @@ -2,31 +2,16 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import java.io.File; import java.util.function.Supplier; +import org.vitrivr.cineast.core.db.DBSelector; import org.vitrivr.cineast.core.db.DBSelectorSupplier; -import org.vitrivr.cineast.core.db.NoDBSelector; -import org.vitrivr.cineast.core.db.NoDBWriter; +import org.vitrivr.cineast.core.db.DataSource; +import org.vitrivr.cineast.core.db.PersistencyWriter; import org.vitrivr.cineast.core.db.PersistencyWriterSupplier; -import org.vitrivr.cineast.core.db.adampro.ADAMproEntityCreator; -import org.vitrivr.cineast.core.db.adampro.ADAMproSelector; -import org.vitrivr.cineast.core.db.adampro.ADAMproStreamingSelector; -import org.vitrivr.cineast.core.db.adampro.ADAMproWrapper; -import org.vitrivr.cineast.core.db.adampro.ADAMproWriter; -import org.vitrivr.cineast.core.db.cottontaildb.CottontailEntityCreator; -import org.vitrivr.cineast.core.db.cottontaildb.CottontailSelector; -import org.vitrivr.cineast.core.db.cottontaildb.CottontailWrapper; -import org.vitrivr.cineast.core.db.cottontaildb.CottontailWriter; -import org.vitrivr.cineast.core.db.json.JsonFileWriter; -import org.vitrivr.cineast.core.db.json.JsonSelector; -import org.vitrivr.cineast.core.db.memory.InMemoryEntityCreator; -import org.vitrivr.cineast.core.db.memory.InMemoryWriter; import org.vitrivr.cineast.core.db.setup.EntityCreator; -import org.vitrivr.cineast.core.db.setup.NoEntityCreator; public final class DatabaseConfig { - /** * Default value for batchsize. */ @@ -34,59 +19,17 @@ public final class DatabaseConfig { public static final String DEFAULT_HOST = "localhost"; public static final int DEFAULT_PORT = 5890; public static final boolean DEFAULT_PLAINTEXT = true; - public static final boolean SINGLE_CONNECTION = true; private String host = DEFAULT_HOST; private int port = DEFAULT_PORT; private boolean plaintext = DEFAULT_PLAINTEXT; - private Writer writer = Writer.COTTONTAIL; - private Selector selector = Selector.COTTONTAIL; + private DataSource writer = DataSource.COTTONTAIL; + private DataSource selector = DataSource.COTTONTAIL; private Integer batchsize = DEFAULT_BATCH_SIZE; - private static final PersistencyWriterSupplier NO_WRITER_SUPPLY = NoDBWriter::new; - - private static final DBSelectorSupplier NO_SELECTOR_SUPPLY = NoDBSelector::new; - - private static final Supplier NO_CREATOR_SUPPLY = NoEntityCreator::new; - - - public enum Writer { - NONE, - JSON, - ADAMPRO, - COTTONTAIL, - INMEMORY - } - - public enum Selector { - NONE, - JSON, - ADAMPRO, - ADAMPROSTREAM, - COTTONTAIL, - INMEMORY - } - - private ADAMproWrapper adaMproWrapper = null; - - private synchronized void ensureAdamProWrapper() { - if (this.adaMproWrapper == null) { - this.adaMproWrapper = new ADAMproWrapper(this); - } - } - - private CottontailWrapper cottontailWrapper = null; - - private synchronized void ensureCottontailWrapper() { - if (this.cottontailWrapper == null) { - this.cottontailWrapper = new CottontailWrapper(this, true); - } - } - @JsonCreator public DatabaseConfig() { - } @JsonProperty @@ -105,7 +48,6 @@ public void setHost(String host) { public int getPort() { return this.port; } - public void setPort(int port) { if (port < 1 || port > 65535) { throw new IllegalArgumentException(port + " is outside of valid port range"); @@ -124,7 +66,7 @@ public void setPlaintext(boolean plaintext) { @JsonProperty public Integer getBatchsize() { - return batchsize; + return this.batchsize; } public void setBatchsize(Integer batchsize) { @@ -132,110 +74,32 @@ public void setBatchsize(Integer batchsize) { } @JsonProperty - public Writer getWriter() { + public DataSource getWriter() { return this.writer; } - public void setWriter(Writer writer) { + public void setWriter(DataSource writer) { this.writer = writer; } @JsonProperty - public Selector getSelector() { + public DataSource getSelector() { return this.selector; } - public void setSelector(Selector selector) { + public void setSelector(DataSource selector) { this.selector = selector; } - public synchronized PersistencyWriterSupplier getWriterSupplier() { - switch (this.writer) { - case NONE: - return NO_WRITER_SUPPLY; - case ADAMPRO: { - if (SINGLE_CONNECTION) { - ensureAdamProWrapper(); - return () -> new ADAMproWriter(this.adaMproWrapper); - } - return () -> new ADAMproWriter(new ADAMproWrapper(this)); - } - case JSON: { - return () -> new JsonFileWriter(new File(this.host)); - } - case COTTONTAIL: { - if (SINGLE_CONNECTION) { - ensureCottontailWrapper(); - return () -> new CottontailWriter(this.cottontailWrapper); - } - return () -> new CottontailWriter(new CottontailWrapper(this, false)); - } - case INMEMORY: { - return InMemoryWriter::new; - } - default: - throw new IllegalStateException("No supplier for writer " + this.writer); - - } + public PersistencyWriterSupplier getWriterSupplier() { + return this.writer.getWriterSupplier(this); } - public synchronized DBSelectorSupplier getSelectorSupplier() { - switch (this.selector) { - case ADAMPRO: { - if (SINGLE_CONNECTION) { - ensureAdamProWrapper(); - return () -> new ADAMproSelector(this.adaMproWrapper); - } - return () -> new ADAMproSelector(new ADAMproWrapper(this)); - } - case ADAMPROSTREAM: { - if (SINGLE_CONNECTION) { - ensureAdamProWrapper(); - return () -> new ADAMproStreamingSelector(this.adaMproWrapper); - } - return () -> new ADAMproStreamingSelector(new ADAMproWrapper(this)); - } - case JSON: { - return () -> new JsonSelector(new File(this.host)); - } - case NONE: - return NO_SELECTOR_SUPPLY; - case COTTONTAIL: { - if (SINGLE_CONNECTION) { - ensureCottontailWrapper(); - return () -> new CottontailSelector(this.cottontailWrapper); - } - return () -> new CottontailSelector(new CottontailWrapper(this, false)); - } - default: - throw new IllegalStateException("No supplier for selector " + this.selector); - - } + public Supplier getEntityCreatorSupplier() { + return this.writer.getEntityCreatorSupplier(this); } - public synchronized Supplier getEntityCreatorSupplier() { - switch (this.selector) { - case ADAMPRO: - case ADAMPROSTREAM: { - if (SINGLE_CONNECTION) { - ensureAdamProWrapper(); - return () -> new ADAMproEntityCreator(this.adaMproWrapper); - } - return () -> new ADAMproEntityCreator(new ADAMproWrapper(this)); - } - case NONE: - return NO_CREATOR_SUPPLY; - case COTTONTAIL: { - if (SINGLE_CONNECTION) { - ensureCottontailWrapper(); - return () -> new CottontailEntityCreator(this.cottontailWrapper); - } - return () -> new CottontailEntityCreator(new CottontailWrapper(this, false)); - } - case INMEMORY: - return InMemoryEntityCreator::new; - default: - throw new IllegalStateException("No supplier for EntityCreator " + this.selector); - } + public DBSelectorSupplier getSelectorSupplier() { + return this.writer.getSelectorSupplier(this); } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/DBSelector.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/DBSelector.java index e08f3cdd1..82174ecda 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/DBSelector.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/DBSelector.java @@ -16,6 +16,7 @@ import java.util.stream.Collectors; import org.apache.commons.lang3.tuple.Triple; import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.vitrivr.cineast.core.config.ReadableQueryConfig; import org.vitrivr.cineast.core.data.distance.DistanceElement; import org.vitrivr.cineast.core.data.providers.primitive.FloatArrayTypeProvider; @@ -26,6 +27,8 @@ public interface DBSelector extends Closeable { + Logger LOGGER = LogManager.getLogger(); + boolean open(String name); void close(); diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/DBSelectorSupplier.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/DBSelectorSupplier.java index 4aa070519..b83ef1c59 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/DBSelectorSupplier.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/DBSelectorSupplier.java @@ -2,6 +2,12 @@ import java.util.function.Supplier; -public interface DBSelectorSupplier extends Supplier { +/** + * This interface describes a {@link Supplier} for {@link DBSelector}s. + * + * Important: This class is required because of signature clashes in {@link org.vitrivr.cineast.core.features.abstracts.AbstractFeatureModule} due + * to type erasure! + */ +public interface DBSelectorSupplier extends Supplier{ } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/DataSource.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/DataSource.java new file mode 100644 index 000000000..1428f1178 --- /dev/null +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/DataSource.java @@ -0,0 +1,109 @@ +package org.vitrivr.cineast.core.db; + +import java.io.File; +import java.util.function.Supplier; +import org.vitrivr.cineast.core.config.DatabaseConfig; +import org.vitrivr.cineast.core.db.adampro.ADAMproEntityCreator; +import org.vitrivr.cineast.core.db.adampro.ADAMproSelector; +import org.vitrivr.cineast.core.db.adampro.ADAMproWrapper; +import org.vitrivr.cineast.core.db.adampro.ADAMproWriter; +import org.vitrivr.cineast.core.db.cottontaildb.CottontailEntityCreator; +import org.vitrivr.cineast.core.db.cottontaildb.CottontailSelector; +import org.vitrivr.cineast.core.db.cottontaildb.CottontailWrapper; +import org.vitrivr.cineast.core.db.cottontaildb.CottontailWriter; +import org.vitrivr.cineast.core.db.json.JsonFileWriter; +import org.vitrivr.cineast.core.db.json.JsonSelector; +import org.vitrivr.cineast.core.db.memory.InMemoryEntityCreator; +import org.vitrivr.cineast.core.db.polypheny.PolyphenyEntityCreator; +import org.vitrivr.cineast.core.db.polypheny.PolyphenySelector; +import org.vitrivr.cineast.core.db.polypheny.PolyphenyWrapper; +import org.vitrivr.cineast.core.db.polypheny.PolyphenyWriter; +import org.vitrivr.cineast.core.db.setup.EntityCreator; +import org.vitrivr.cineast.core.db.setup.NoEntityCreator; + +/** + * Enumaration of all {@link DataSource}s available to Cineast. + * + * @author Ralph Gasser + * @version 1.0.0 + */ +public enum DataSource { + NONE, + JSON, + COTTONTAIL, + POLYPHENY, + INMEMORY, + + @Deprecated + ADAMPRO; + + /** + * Returns a new {@link PersistencyWriterSupplier} + * + * @param config The {@link DatabaseConfig} to use. + * @return Resulting {@link PersistencyWriterSupplier} + */ + public PersistencyWriterSupplier getWriterSupplier(DatabaseConfig config) { + switch (this) { + case NONE: + return NoDBWriter::new; + case COTTONTAIL: + return () -> new CottontailWriter(new CottontailWrapper(config.getHost(), config.getPort()), config.getBatchsize()); + case POLYPHENY: + return () -> new PolyphenyWriter(new PolyphenyWrapper(config.getHost(), config.getPort()), config.getBatchsize()); + case JSON: + return () -> new JsonFileWriter(new File(config.getHost())); + case ADAMPRO: + return () -> new ADAMproWriter(new ADAMproWrapper(config)); + default: + throw new IllegalStateException("No supplier for " + this + " selector."); + } + } + + /** + * Returns a new {@link DBSelectorSupplier} + * + * @param config The {@link DatabaseConfig} to use. + * @return Resulting {@link DBSelectorSupplier} + */ + public DBSelectorSupplier getSelectorSupplier(DatabaseConfig config) { + switch (this) { + case NONE: + return NoDBSelector::new; + case COTTONTAIL: + return () -> new CottontailSelector(new CottontailWrapper(config.getHost(), config.getPort())); + case POLYPHENY: + return () -> new PolyphenySelector(new PolyphenyWrapper(config.getHost(), config.getPort())); + case JSON: + return () -> new JsonSelector(new File(config.getHost())); + case ADAMPRO: + return () -> new ADAMproSelector(new ADAMproWrapper(config)); + default: + throw new IllegalStateException("No supplier for " + this + " selector."); + } + } + + /** + * Returns a new {@link Supplier} for an {@link EntityCreator} + * + * @param config The {@link DatabaseConfig} to use. + * @return Resulting {@link Supplier} + */ + public Supplier getEntityCreatorSupplier(DatabaseConfig config) { + switch (this) { + case NONE: + return NoEntityCreator::new; + case INMEMORY: + return InMemoryEntityCreator::new; + case COTTONTAIL: + return () -> new CottontailEntityCreator(new CottontailWrapper(config.getHost(), config.getPort())); + case POLYPHENY: + return () -> new PolyphenyEntityCreator(new PolyphenyWrapper(config.getHost(), config.getPort())); + case ADAMPRO: + return () -> new ADAMproEntityCreator(new ADAMproWrapper(config)); + default: + throw new IllegalStateException("No supplier for " + this + " entity creator."); + } + } +} + diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/NoDBWriter.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/NoDBWriter.java index 36c4f4a65..b96bbc295 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/NoDBWriter.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/NoDBWriter.java @@ -37,4 +37,9 @@ public boolean persist(List tuples) { public Object getPersistentRepresentation(PersistentTuple tuple) { return new Object(); } + + @Override + public int supportedBatchSize() { + return 0; + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/PersistencyWriter.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/PersistencyWriter.java index 297c84a3b..379785210 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/PersistencyWriter.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/PersistencyWriter.java @@ -2,16 +2,23 @@ import java.io.Closeable; import java.util.List; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; public interface PersistencyWriter extends Closeable { /** - * @return true if the writer was successfully opened + * Logger instance used for logging. */ - boolean open(String name); + Logger LOGGER = LogManager.getLogger(); void close(); + /** + * @return true if the writer was successfully opened + */ + boolean open(String name); + boolean idExists(String id); boolean exists(String key, String value); @@ -25,4 +32,11 @@ public interface PersistencyWriter extends Closeable { boolean persist(List tuples); R getPersistentRepresentation(PersistentTuple tuple); + + /** + * Batch size supported when inserting data. This is merely a hint to upper system components, that batching is supported. + * + * @return The supported batch size when inserting data. + */ + int supportedBatchSize(); } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/PersistencyWriterSupplier.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/PersistencyWriterSupplier.java index d644f948e..5d1dc46b3 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/PersistencyWriterSupplier.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/PersistencyWriterSupplier.java @@ -2,6 +2,10 @@ import java.util.function.Supplier; +/** + * This interface describes a {@link Supplier} for {@link PersistencyWriter}s. + * + * Important: This class is required because of signature clashes in {@link org.vitrivr.cineast.core.features.abstracts.AbstractFeatureModule} due to type erasure! + */ public interface PersistencyWriterSupplier extends Supplier> { - } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/adampro/ADAMproEntityCreator.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/adampro/ADAMproEntityCreator.java index b6cfa3468..d78b53ccd 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/adampro/ADAMproEntityCreator.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/adampro/ADAMproEntityCreator.java @@ -19,6 +19,7 @@ import org.vitrivr.cineast.core.db.setup.EntityCreator; import org.vitrivr.cineast.core.db.setup.EntityDefinition; +@Deprecated public class ADAMproEntityCreator implements EntityCreator { /** diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/adampro/ADAMproMessageBuilder.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/adampro/ADAMproMessageBuilder.java index b80527fc9..714a1ae1b 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/adampro/ADAMproMessageBuilder.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/adampro/ADAMproMessageBuilder.java @@ -29,7 +29,7 @@ import org.vitrivr.cineast.core.db.DataMessageConverter; import org.vitrivr.cineast.core.db.RelationalOperator; - +@Deprecated public class ADAMproMessageBuilder { /** diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/adampro/ADAMproSelector.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/adampro/ADAMproSelector.java index 7fbbe7512..df9188293 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/adampro/ADAMproSelector.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/adampro/ADAMproSelector.java @@ -36,6 +36,7 @@ import org.vitrivr.cineast.core.db.RelationalOperator; import org.vitrivr.cineast.core.util.LogHelper; +@Deprecated public class ADAMproSelector extends AbstractADAMproSelector { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/adampro/ADAMproStreamingSelector.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/adampro/ADAMproStreamingSelector.java index a61ee133f..af6cdb2ce 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/adampro/ADAMproStreamingSelector.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/adampro/ADAMproStreamingSelector.java @@ -32,6 +32,7 @@ import org.vitrivr.cineast.core.db.DataMessageConverter; import org.vitrivr.cineast.core.db.RelationalOperator; +@Deprecated public class ADAMproStreamingSelector extends AbstractADAMproSelector { private static final Logger LOGGER = LogManager.getLogger(); diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/adampro/ADAMproWrapper.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/adampro/ADAMproWrapper.java index 099f68f1d..0acc30dd2 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/adampro/ADAMproWrapper.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/adampro/ADAMproWrapper.java @@ -36,6 +36,7 @@ import org.vitrivr.cineast.core.config.DatabaseConfig; import org.vitrivr.cineast.core.util.LogHelper; +@Deprecated public class ADAMproWrapper implements AutoCloseable { private static final Logger LOGGER = LogManager.getLogger(); diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/adampro/ADAMproWriter.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/adampro/ADAMproWriter.java index 3f9ae759c..60ee38883 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/adampro/ADAMproWriter.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/adampro/ADAMproWriter.java @@ -20,6 +20,7 @@ import org.vitrivr.cineast.core.db.PersistentTuple; import org.vitrivr.cineast.core.util.LogHelper; +@Deprecated public class ADAMproWriter extends ProtobufTupleGenerator { private static final Logger LOGGER = LogManager.getLogger(); @@ -45,8 +46,7 @@ public boolean open(String name) { } @Override - public void close() { - } + public void close() {} @Override public boolean exists(String key, String value) { @@ -120,5 +120,10 @@ public boolean persist(List tuples) { return true; } + @Override + public int supportedBatchSize() { + return 1; + } + } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/adampro/AbstractADAMproSelector.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/adampro/AbstractADAMproSelector.java index 1d8e6d081..a38c085a9 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/adampro/AbstractADAMproSelector.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/adampro/AbstractADAMproSelector.java @@ -25,6 +25,7 @@ import org.vitrivr.cineast.core.db.DataMessageConverter; import org.vitrivr.cineast.core.db.RelationalOperator; +@Deprecated public abstract class AbstractADAMproSelector implements DBSelector { /** @@ -42,7 +43,7 @@ public abstract class AbstractADAMproSelector implements DBSelector { */ protected FromMessage fromMessage; - final ADAMproWrapper adampro; + protected final ADAMproWrapper adampro; public AbstractADAMproSelector(ADAMproWrapper wrapper) { this.adampro = wrapper; diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailEntityCreator.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailEntityCreator.java index 47c11e144..6fc97f424 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailEntityCreator.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailEntityCreator.java @@ -11,7 +11,6 @@ import java.util.HashMap; import java.util.Objects; import java.util.Optional; -import org.vitrivr.cineast.core.config.DatabaseConfig; import org.vitrivr.cineast.core.data.entities.MediaObjectDescriptor; import org.vitrivr.cineast.core.data.entities.MediaObjectMetadataDescriptor; import org.vitrivr.cineast.core.data.entities.MediaSegmentDescriptor; @@ -19,7 +18,8 @@ import org.vitrivr.cineast.core.db.dao.reader.TagReader; import org.vitrivr.cineast.core.db.setup.AttributeDefinition; import org.vitrivr.cineast.core.db.setup.EntityCreator; -import org.vitrivr.cottontail.client.TupleIterator; +import org.vitrivr.cottontail.client.iterators.Tuple; +import org.vitrivr.cottontail.client.iterators.TupleIterator; import org.vitrivr.cottontail.client.language.basics.Constants; import org.vitrivr.cottontail.client.language.basics.Type; import org.vitrivr.cottontail.client.language.ddl.AboutEntity; @@ -40,11 +40,6 @@ public final class CottontailEntityCreator implements EntityCreator { */ private final CottontailWrapper cottontail; - public CottontailEntityCreator(DatabaseConfig config) { - this.cottontail = new CottontailWrapper(config, false); - init(); - } - public CottontailEntityCreator(CottontailWrapper cottontailWrapper) { this.cottontail = cottontailWrapper; init(); @@ -56,18 +51,18 @@ public CottontailEntityCreator(CottontailWrapper cottontailWrapper) { private void init() { final long txId = this.cottontail.client.begin(); try { - final ListSchemas list = new ListSchemas(); - final TupleIterator iterator = this.cottontail.client.list(list, txId); + final ListSchemas list = new ListSchemas().txId(txId); + final TupleIterator iterator = this.cottontail.client.list(list); boolean exists = false; while (iterator.hasNext()) { - TupleIterator.Tuple next = iterator.next(); + Tuple next = iterator.next(); if (Objects.equals(next.asString(Constants.COLUMN_NAME_DBO), CottontailWrapper.FQN_CINEAST_SCHEMA)) { exists = true; break; } } if (!exists) { - this.cottontail.client.create(new CreateSchema(CottontailWrapper.CINEAST_SCHEMA), txId); + this.cottontail.client.create(new CreateSchema(CottontailWrapper.CINEAST_SCHEMA).txId(txId)); } this.cottontail.client.commit(txId); } catch (StatusRuntimeException e) { @@ -85,8 +80,9 @@ public boolean createTagEntity() { final CreateEntity create = new CreateEntity(entityName) .column(TagReader.TAG_ID_COLUMNNAME, Type.STRING, -1, false) .column(TagReader.TAG_NAME_COLUMNNAME, Type.STRING, -1, false) - .column(TagReader.TAG_DESCRIPTION_COLUMNNAME, Type.STRING, -1, false); - this.cottontail.client.create(create, txId); + .column(TagReader.TAG_DESCRIPTION_COLUMNNAME, Type.STRING, -1, false) + .txId(txId); + this.cottontail.client.create(create); /* tag ids should be unique */ this.createIndex(entityName, TagReader.TAG_ID_COLUMNNAME, IndexType.HASH_UQ, txId); @@ -113,8 +109,9 @@ public boolean createMultiMediaObjectsEntity() { .column(MediaObjectDescriptor.FIELDNAMES[0], Type.STRING, -1, false) .column(MediaObjectDescriptor.FIELDNAMES[1], Type.INTEGER, -1, false) .column(MediaObjectDescriptor.FIELDNAMES[2], Type.STRING, -1, false) - .column(MediaObjectDescriptor.FIELDNAMES[3], Type.STRING, -1, false); - this.cottontail.client.create(entity, txId); + .column(MediaObjectDescriptor.FIELDNAMES[3], Type.STRING, -1, false) + .txId(txId); + this.cottontail.client.create(entity); /* Create index. */ this.createIndex(entityName, MediaObjectDescriptor.FIELDNAMES[0], IndexType.HASH_UQ, txId); @@ -139,8 +136,9 @@ public boolean createSegmentEntity() { .column(MediaSegmentDescriptor.FIELDNAMES[3], Type.INTEGER, -1, false) .column(MediaSegmentDescriptor.FIELDNAMES[4], Type.INTEGER, -1, false) .column(MediaSegmentDescriptor.FIELDNAMES[5], Type.DOUBLE, -1, false) - .column(MediaSegmentDescriptor.FIELDNAMES[6], Type.DOUBLE, -1, false); - this.cottontail.client.create(entity, txId); + .column(MediaSegmentDescriptor.FIELDNAMES[6], Type.DOUBLE, -1, false) + .txId(txId); + this.cottontail.client.create(entity); /* Create indexes. */ this.createIndex(entityName, MediaSegmentDescriptor.FIELDNAMES[0], IndexType.HASH_UQ, txId); @@ -163,8 +161,9 @@ public boolean createMetadataEntity(String tableName) { .column(MediaObjectMetadataDescriptor.FIELDNAMES[0], Type.STRING, -1, false) .column(MediaObjectMetadataDescriptor.FIELDNAMES[1], Type.STRING, -1, false) .column(MediaObjectMetadataDescriptor.FIELDNAMES[2], Type.STRING, -1, false) - .column(MediaObjectMetadataDescriptor.FIELDNAMES[3], Type.STRING, -1, false); - this.cottontail.client.create(entity, txId); + .column(MediaObjectMetadataDescriptor.FIELDNAMES[3], Type.STRING, -1, false) + .txId(txId); + this.cottontail.client.create(entity); /* Create Index. */ this.createIndex(entityName, MediaObjectMetadataDescriptor.FIELDNAMES[0], IndexType.HASH, txId); @@ -186,8 +185,9 @@ public boolean createSegmentMetadataEntity(String tableName) { .column(MediaSegmentMetadataDescriptor.FIELDNAMES[0], Type.STRING, -1, false) .column(MediaSegmentMetadataDescriptor.FIELDNAMES[1], Type.STRING, -1, false) .column(MediaSegmentMetadataDescriptor.FIELDNAMES[2], Type.STRING, -1, false) - .column(MediaSegmentMetadataDescriptor.FIELDNAMES[3], Type.STRING, -1, false); - this.cottontail.client.create(entity, txId); + .column(MediaSegmentMetadataDescriptor.FIELDNAMES[3], Type.STRING, -1, false) + .txId(txId); + this.cottontail.client.create(entity); /* Create Index. */ this.createIndex(entityName, MediaSegmentMetadataDescriptor.FIELDNAMES[0], IndexType.HASH, txId); @@ -238,16 +238,15 @@ public boolean createEntity(org.vitrivr.cineast.core.db.setup.EntityDefinition d try { /* Create entity. */ final String entityName = CottontailWrapper.CINEAST_SCHEMA + "." + def.getEntityName(); - final CreateEntity entity = new CreateEntity(entityName); + final CreateEntity entity = new CreateEntity(entityName).txId(txId); for (AttributeDefinition attribute : def.getAttributes()) { int length = -1; if ((attribute.getType() == VECTOR || attribute.getType() == BITSET) && attribute.getLength() > 0) { length = attribute.getLength(); } entity.column(attribute.getName(), mapAttributeType(attribute.getType()), length, false); - } - this.cottontail.client.create(entity, txId); + this.cottontail.client.create(entity); /* Create Index. */ for (AttributeDefinition attribute : def.getAttributes()) { @@ -287,7 +286,7 @@ public boolean createHashNonUniqueIndex(String entityName, String column) { public boolean existsEntity(String entityName) { final AboutEntity about = new AboutEntity(this.cottontail.fqnInput(entityName)); try { - final TupleIterator results = this.cottontail.client.about(about, null); + final TupleIterator results = this.cottontail.client.about(about); return results.hasNext(); } catch (StatusRuntimeException e) { return false; @@ -299,7 +298,7 @@ public boolean dropEntity(String entityName) { final long txId = this.cottontail.client.begin(); try { final String fqn = CottontailWrapper.CINEAST_SCHEMA + "." + entityName; - this.cottontail.client.drop(new DropEntity(fqn), txId); + this.cottontail.client.drop(new DropEntity(fqn).txId(txId)); this.cottontail.client.commit(txId); return true; } catch (StatusRuntimeException e) { @@ -322,9 +321,13 @@ public static Type mapAttributeType(AttributeDefinition.AttributeType type) { case VECTOR: return Type.FLOAT_VECTOR; case BITSET: - return Type.BOOL_VECTOR; + return Type.BOOLEAN_VECTOR; case FLOAT: return Type.FLOAT; + /*case GEOGRAPHY: + return Type.GEOGRAPHY; + case GEOMETRY: + return Type.GEOMETRY;*/ case INT: return Type.INTEGER; case LONG: @@ -340,7 +343,7 @@ public static Type mapAttributeType(AttributeDefinition.AttributeType type) { private void createIndex(String entityName, String attribute, IndexType type, long txId) { final String indexName = entityName + ".idx_" + attribute + "_" + type.name().toLowerCase(); - final CreateIndex index = new CreateIndex(indexName, type).column(entityName + "." + attribute); - this.cottontail.client.create(index, txId); + final CreateIndex index = new CreateIndex(indexName, type).column(entityName + "." + attribute).txId(txId); + this.cottontail.client.create(index); } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailSelector.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailSelector.java index 5445eab29..2a30b2775 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailSelector.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailSelector.java @@ -5,7 +5,6 @@ import static org.vitrivr.cineast.core.util.CineastConstants.GENERIC_ID_COLUMN_QUALIFIER; import static org.vitrivr.cineast.core.util.CineastConstants.KEY_COL_NAME; -import com.google.common.collect.Lists; import io.grpc.StatusRuntimeException; import java.util.ArrayList; import java.util.Arrays; @@ -21,8 +20,6 @@ import java.util.stream.StreamSupport; import org.apache.commons.lang3.time.StopWatch; import org.apache.commons.lang3.tuple.Triple; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import org.vitrivr.cineast.core.config.ReadableQueryConfig; import org.vitrivr.cineast.core.config.ReadableQueryConfig.Distance; import org.vitrivr.cineast.core.data.distance.DistanceElement; @@ -30,20 +27,19 @@ import org.vitrivr.cineast.core.db.DBSelector; import org.vitrivr.cineast.core.db.RelationalOperator; import org.vitrivr.cineast.core.db.dao.MetadataAccessSpecification; -import org.vitrivr.cottontail.client.TupleIterator; +import org.vitrivr.cottontail.client.iterators.Tuple; +import org.vitrivr.cottontail.client.iterators.TupleIterator; +import org.vitrivr.cottontail.client.language.basics.Direction; +import org.vitrivr.cottontail.client.language.basics.Distances; import org.vitrivr.cottontail.client.language.ddl.AboutEntity; import org.vitrivr.cottontail.client.language.dql.Query; import org.vitrivr.cottontail.client.language.extensions.And; import org.vitrivr.cottontail.client.language.extensions.Literal; import org.vitrivr.cottontail.client.language.extensions.Or; import org.vitrivr.cottontail.client.language.extensions.Predicate; -import org.vitrivr.cottontail.grpc.CottontailGrpc; -import org.vitrivr.cottontail.grpc.CottontailGrpc.Knn; public final class CottontailSelector implements DBSelector { - private static final Logger LOGGER = LogManager.getLogger(); - /** * Internal reference to the {@link CottontailWrapper} used by this {@link CottontailSelector}. */ @@ -65,18 +61,16 @@ public boolean open(String name) { } @Override - public void close() { - this.cottontail.close(); - } + public void close() { /* No op. */ } /** * if {@link ReadableQueryConfig#getRelevantSegmentIds()} is null, the where-clause will be left empty */ @Override public List getNearestNeighboursGeneric(int k, float[] vector, String column, Class distanceElementClass, ReadableQueryConfig config) { - final Query query = knn(k, vector, column, config).select(GENERIC_ID_COLUMN_QUALIFIER, DB_DISTANCE_VALUE_QUALIFIER); + final Query query = knn(k, vector, column, config); try { - return handleNearestNeighbourResponse(this.cottontail.client.query(query, null), distanceElementClass); + return handleNearestNeighbourResponse(this.cottontail.client.query(query), distanceElementClass); } catch (StatusRuntimeException e) { LOGGER.warn("Error occurred during query execution in getNearestNeighboursGeneric(): {}", e.getMessage()); return new ArrayList<>(0); @@ -90,9 +84,9 @@ public List getBatchedNearestNeighbours(int k, Li @Override public List> getNearestNeighbourRows(int k, float[] vector, String column, ReadableQueryConfig config) { - final Query query = knn(k, vector, column, config).select("*"); + final Query query = knn(k, vector, column, config); try { - return processResults(this.cottontail.client.query(query, null)); + return processResults(this.cottontail.client.query(query)); } catch (StatusRuntimeException e) { LOGGER.warn("Error occurred during query execution in getNearestNeighbourRows(): {}", e.getMessage()); return new ArrayList<>(0); @@ -101,12 +95,12 @@ public List> getNearestNeighbourRows(int k, f @Override public List getFeatureVectors(String fieldName, PrimitiveTypeProvider value, String vectorName) { - final Query query = new Query(this.fqn).select(vectorName).where(new Literal(fieldName, "==", value.toObject())); + final Query query = new Query(this.fqn).select(vectorName, null).where(new Literal(fieldName, "==", value.toObject())); try { - final TupleIterator results = this.cottontail.client.query(query, null); + final TupleIterator results = this.cottontail.client.query(query); final List _return = new LinkedList<>(); while (results.hasNext()) { - final TupleIterator.Tuple t = results.next(); + final Tuple t = results.next(); _return.add(t.asFloatVector(vectorName)); } return _return; @@ -118,9 +112,9 @@ public List getFeatureVectors(String fieldName, PrimitiveTypeProvider v @Override public List getFeatureVectorsGeneric(String fieldName, PrimitiveTypeProvider value, String vectorName) { - final Query query = new Query(this.fqn).select(vectorName).where(new Literal(fieldName, "==", value.toObject())); + final Query query = new Query(this.fqn).select(vectorName, null).where(new Literal(fieldName, "==", value.toObject())); try { - return toSingleCol(this.cottontail.client.query(query, null), vectorName); + return toSingleCol(this.cottontail.client.query(query), vectorName); } catch (StatusRuntimeException e) { LOGGER.warn("Error occurred during query execution in getFeatureVectorsGeneric(): {}", e.getMessage()); return new ArrayList<>(0); @@ -130,9 +124,9 @@ public List getFeatureVectorsGeneric(String fieldName, Pr @Override public List> getRows(String fieldName, Iterable values) { final Object[] mapped = StreamSupport.stream(values.spliterator(), false).map(PrimitiveTypeProvider::toObject).toArray(); - final Query query = new Query(this.fqn).select("*").where(new Literal(fieldName, "IN", mapped)); + final Query query = new Query(this.fqn).select("*", null).where(new Literal(fieldName, "IN", mapped)); try { - return processResults(this.cottontail.client.query(query, null)); + return processResults(this.cottontail.client.query(query)); } catch (StatusRuntimeException e) { LOGGER.warn("Error occurred during query execution in getRows(): {}", e.getMessage()); return new ArrayList<>(0); @@ -142,9 +136,9 @@ public List> getRows(String fieldName, Iterab @Override public List> getRows(String fieldName, List values) { final Object[] mapped = values.toArray(); - final Query query = new Query(this.fqn).select("*").where(new Literal(fieldName, "IN", mapped)); + final Query query = new Query(this.fqn).select("*", null).where(new Literal(fieldName, "IN", mapped)); try { - return processResults(this.cottontail.client.query(query, null)); + return processResults(this.cottontail.client.query(query)); } catch (StatusRuntimeException e) { LOGGER.warn("Error occurred during query execution in getRows(): {}", e.getMessage()); return new ArrayList<>(0); @@ -155,9 +149,9 @@ public List> getRows(String fieldName, List> getRows(String fieldName, RelationalOperator operator, Iterable values) { final Object[] mapped = StreamSupport.stream(values.spliterator(), false).map(PrimitiveTypeProvider::toObject).toArray(); final String op = toOperator(operator); - final Query query = new Query(this.fqn).select("*").where(new Literal(fieldName, op, mapped)); + final Query query = new Query(this.fqn).select("*", null).where(new Literal(fieldName, op, mapped)); try { - return processResults(this.cottontail.client.query(query, null)); + return processResults(this.cottontail.client.query(query)); } catch (StatusRuntimeException e) { LOGGER.warn("Error occurred during query execution in getRows(): {}", e.getMessage()); return new ArrayList<>(0); @@ -167,30 +161,22 @@ public List> getRows(String fieldName, Relati @Override public List> getFulltextRows(int rows, String fieldname, ReadableQueryConfig queryConfig, String... terms) { /* Prepare plain query. */ - final Query query = new Query(this.fqn).select( - new kotlin.Pair<>("*", null) - ); + final String predicate = Arrays.stream(terms).map(String::trim).collect(Collectors.joining(" OR ")); + final Query query = new Query(this.fqn) + .select("*", null) + .fulltext(fieldname, predicate, DB_DISTANCE_VALUE_QUALIFIER); /* Process predicates. */ - final List atomics = Arrays.stream(terms).map(t -> new Literal(fieldname, "MATCH", t)).collect(Collectors.toList()); - final Optional predicates = atomics.stream().reduce(Or::new); if (queryConfig != null && !queryConfig.getRelevantSegmentIds().isEmpty()) { final Set relevant = queryConfig.getRelevantSegmentIds(); final Literal segmentIds = new Literal(GENERIC_ID_COLUMN_QUALIFIER, "IN", relevant.toArray()); - if (predicates.isPresent()) { - query.where(new And(segmentIds, predicates.get())); - } else { - query.where(segmentIds); - } + query.where(new And(segmentIds, new Literal(DB_DISTANCE_VALUE_QUALIFIER, ">", 0.0))); } else { - predicates.ifPresent(query::where); + query.where(new Literal(DB_DISTANCE_VALUE_QUALIFIER, ">", 0.0)); } - Map mappings = new HashMap<>(); - mappings.put("score", DB_DISTANCE_VALUE_QUALIFIER); - try { - return processResults(this.cottontail.client.query(query, null), mappings); + return processResults(this.cottontail.client.query(query)); } catch (StatusRuntimeException e) { LOGGER.warn("Error occurred during query execution in getFulltextRows(): {}", e.getMessage()); return new ArrayList<>(0); @@ -202,9 +188,11 @@ public List> getRowsAND(List> getRowsAND(List predicates = atomics.stream().reduce(And::new); if (qc != null && !qc.getRelevantSegmentIds().isEmpty()) { final Set relevant = qc.getRelevantSegmentIds(); @@ -224,8 +214,9 @@ public List> getRowsAND(List(0); @@ -234,25 +225,23 @@ public List> getRowsAND(List> getMetadataBySpec(List spec) { - Query query = new Query(this.fqn); - query.select("*"); - Optional predicates = generateQueryFromMetadataSpec(spec); + final Query query = new Query(this.fqn).select("*", null); + final Optional predicates = generateQueryFromMetadataSpec(spec); predicates.ifPresent(query::where); - return processResults(this.cottontail.client.query(query, null)); + return processResults(this.cottontail.client.query(query)); } @Override public List> getMetadataByIdAndSpec(List ids, List spec, String idColName) { - Query query = new Query(this.fqn); - query.select("*"); - Optional predicates = generateQueryFromMetadataSpec(spec); + final Query query = new Query(this.fqn).select("*", null); + final Optional predicates = generateQueryFromMetadataSpec(spec); final Literal segmentIds = new Literal(idColName, "IN", ids.toArray()); if (predicates.isPresent()) { query.where(new And(segmentIds, predicates.get())); } else { query.where(segmentIds); } - return processResults(this.cottontail.client.query(query, null)); + return processResults(this.cottontail.client.query(query)); } @@ -268,7 +257,7 @@ public Optional generateQueryFromMetadataSpec(List> reduce = atomics.stream().reduce((res, el) -> { + final Optional> reduce = atomics.stream().reduce((res, el) -> { if (!res.isPresent() && !el.isPresent()) { return Optional.empty(); } @@ -286,9 +275,9 @@ public Optional generateQueryFromMetadataSpec(List getAll(String column) { - final Query query = new Query(this.fqn).select(column); + final Query query = new Query(this.fqn).select(column, null); try { - return toSingleCol(this.cottontail.client.query(query, null), column); + return toSingleCol(this.cottontail.client.query(query), column); } catch (StatusRuntimeException e) { LOGGER.warn("Error occurred during query execution in getAll(): {}", e.getMessage()); return new ArrayList<>(0); @@ -297,12 +286,15 @@ public List getAll(String column) { @Override public List> getAll(List columns, int limit) { - final Query query = new Query(this.fqn).select(columns.toArray(new String[]{})); + final Query query = new Query(this.fqn); + for (String c : columns) { + query.select(c, null); + } if (limit > 0) { query.limit(limit); } try { - return processResults(this.cottontail.client.query(query, null)); + return processResults(this.cottontail.client.query(query)); } catch (StatusRuntimeException e) { LOGGER.warn("Error occurred during query execution in getAll(): {}", e.getMessage()); return new ArrayList<>(0); @@ -311,9 +303,9 @@ public List> getAll(List columns, int @Override public List getUniqueValues(String column) { - final Query query = new Query(this.fqn).distinct(column); + final Query query = new Query(this.fqn).distinct(column, null); try { - return toSingleCol(this.cottontail.client.query(query, null), column); + return toSingleCol(this.cottontail.client.query(query), column); } catch (StatusRuntimeException e) { LOGGER.warn("Error occurred during query execution in getUniqueValues(): {}", e.getMessage()); return new ArrayList<>(0); @@ -321,12 +313,12 @@ public List getUniqueValues(String column) { } public Map countDistinctValues(String column) { - final Query query = new Query(this.fqn).select("*"); + final Query query = new Query(this.fqn).select("*", null); final Map count = new HashMap<>(); try { - final TupleIterator results = this.cottontail.client.query(query, null); + final TupleIterator results = this.cottontail.client.query(query); while (results.hasNext()) { - final TupleIterator.Tuple t = results.next(); + final Tuple t = results.next(); count.merge(t.asString(column), 1, (old, one) -> old + 1); } return count; @@ -338,9 +330,9 @@ public Map countDistinctValues(String column) { @Override public List> getAll() { - final Query query = new Query(this.fqn).select("*"); + final Query query = new Query(this.fqn).select("*", null); try { - return processResults(this.cottontail.client.query(query, null)); + return processResults(this.cottontail.client.query(query)); } catch (StatusRuntimeException e) { LOGGER.warn("Error occurred during query execution in getAll(): {}", e.getMessage()); return new ArrayList<>(0); @@ -350,10 +342,10 @@ public List> getAll() { @Override public boolean existsEntity(String name) { final AboutEntity about = new AboutEntity(this.cottontail.fqnInput(name)); - try { - final TupleIterator results = this.cottontail.client.about(about, null); + try (final TupleIterator results = this.cottontail.client.about(about)) { return results.hasNext(); - } catch (StatusRuntimeException e) { + } catch (Exception e) { + LOGGER.error("Failed to close TupleIterator!", e); return false; } } @@ -367,10 +359,6 @@ public boolean ping() { } } - List toLiteralList(String s) { - return Lists.newArrayList(CottontailGrpc.Literal.newBuilder().setStringData(s).build()); - } - /** * Creates and returns a basic {@link Query} object for the given kNN parameters. * @@ -381,17 +369,13 @@ List toLiteralList(String s) { * @return {@link Query} */ private Query knn(int k, float[] vector, String column, ReadableQueryConfig config) { - final Optional weights = config.getDistanceWeights(); final Set relevant = config.getRelevantSegmentIds(); - final String distance = toDistance(config.getDistance().orElse(Distance.manhattan)); - final Query query = new Query(this.fqn); - - /* Add weights (optional). */ - if (weights.isPresent()) { - query.knn(column, k, distance, vector, weights.get()); - } else { - query.knn(column, k, distance, vector, null); - } + final Distances distance = toDistance(config.getDistance().orElse(Distance.manhattan)); + final Query query = new Query(this.fqn) + .select(GENERIC_ID_COLUMN_QUALIFIER, null) + .distance(column, vector, distance, DB_DISTANCE_VALUE_QUALIFIER) + .order(DB_DISTANCE_VALUE_QUALIFIER, Direction.ASC) + .limit(k); /* Add relevant segments (optional). */ if (!relevant.isEmpty()) { @@ -404,16 +388,12 @@ private Query knn(int k, float[] vector, String column, ReadableQueryConfig conf private static List> processResults(TupleIterator results, Map mappings) { final List> _return = new LinkedList<>(); final StopWatch watch = StopWatch.createStarted(); - final Collection columns = results.getColumns(); + final Collection columns = results.getSimpleNames(); while (results.hasNext()) { - final TupleIterator.Tuple t = results.next(); + final Tuple t = results.next(); final Map map = new HashMap<>(results.getNumberOfColumns()); for (String c : columns) { - if (mappings.containsKey(c)) { - map.put(mappings.get(c), PrimitiveTypeProvider.fromObject(t.get(c))); - } else { - map.put(c, PrimitiveTypeProvider.fromObject(t.get(c))); - } + map.put(mappings.getOrDefault(c, c), PrimitiveTypeProvider.fromObject(t.get(c))); } _return.add(map); } @@ -441,7 +421,7 @@ private static List> processResults(TupleIter private List toSingleCol(TupleIterator results, String colName) { final List _return = new LinkedList<>(); while (results.hasNext()) { - final TupleIterator.Tuple t = results.next(); + final Tuple t = results.next(); _return.add(PrimitiveTypeProvider.fromObject(t.get(colName))); } return _return; @@ -458,9 +438,9 @@ private static List handleNearestNeighbourRespons final List result = new LinkedList<>(); while (response.hasNext()) { try { - final TupleIterator.Tuple t = response.next(); - final String id = t.get(GENERIC_ID_COLUMN_QUALIFIER).toString(); /* This should be fine. */ - double distance = t.asDouble(DB_DISTANCE_VALUE_QUALIFIER); + final Tuple t = response.next(); + final String id = t.asString(GENERIC_ID_COLUMN_QUALIFIER); + double distance = t.asDouble(DB_DISTANCE_VALUE_QUALIFIER); /* This should be fine. */ T e = DistanceElement.create(distanceElementClass, id, distance); result.add(e); } catch (NullPointerException e) { @@ -476,23 +456,23 @@ private static List handleNearestNeighbourRespons * @param distance {@link Distance} to convert. * @return {@link String} Name of Cottontail DB distance. */ - private static String toDistance(Distance distance) { + private static Distances toDistance(Distance distance) { switch (distance) { case manhattan: - return Knn.Distance.L1.toString(); + return Distances.L1; case euclidean: - return Knn.Distance.L2.toString(); + return Distances.L2; case squaredeuclidean: - return Knn.Distance.L2SQUARED.toString(); + return Distances.L2SQUARED; case chisquared: - return Knn.Distance.CHISQUARED.toString(); + return Distances.CHISQUARED; case cosine: - return Knn.Distance.COSINE.toString(); + return Distances.COSINE; case haversine: - return Knn.Distance.HAVERSINE.toString(); + return Distances.HAVERSINE; default: LOGGER.error("distance '{}' not supported by cottontail", distance); - throw new IllegalArgumentException("Distance '" + distance.toString() + "' not supported by Cottontail DB."); + throw new IllegalArgumentException("Distance '" + distance + "' not supported by Cottontail DB."); } } @@ -531,7 +511,7 @@ private static String toOperator(RelationalOperator op) { case IN: return "IN"; default: - throw new IllegalArgumentException("Operator '" + op.toString() + "' not supported by Cottontail DB."); + throw new IllegalArgumentException("Operator '" + op + "' not supported by Cottontail DB."); } } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailWrapper.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailWrapper.java index 65a83585d..93f7ed01e 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailWrapper.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailWrapper.java @@ -1,13 +1,16 @@ package org.vitrivr.cineast.core.db.cottontaildb; +import io.grpc.ConnectivityState; import io.grpc.ManagedChannel; import io.grpc.netty.NettyChannelBuilder; +import java.util.HashMap; +import java.util.Map; import java.util.concurrent.TimeUnit; +import kotlin.jvm.Synchronized; import org.apache.commons.lang3.time.StopWatch; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.vitrivr.cineast.core.config.DatabaseConfig; -import org.vitrivr.cottontail.client.stub.SimpleClient; +import org.vitrivr.cottontail.client.SimpleClient; public final class CottontailWrapper implements AutoCloseable { @@ -18,19 +21,62 @@ public final class CottontailWrapper implements AutoCloseable { public static final String FQN_CINEAST_SCHEMA = WARREN_PREFIX + "." + CINEAST_SCHEMA; /** - * The {@link ManagedChannel} used for communication. + * Internal connection pool to re-use managed channels. */ - public final ManagedChannel channel; + private static final Map POOL = new HashMap<>(); /** - * The {@link SimpleClient} instance that facilitates access to Cottontail DB. + * Returns a {@link ManagedChannel} object for the given database configuration. + *

+ * Tries to re-use existing {@link ManagedChannel} objects. Currently, {@link ManagedChannel} are kept alive as long as Cineast runs. + * + * @param host Hostname of the Cottontail DB server. + * @param port Port of the Cottontail DB server. + * @return {@link ManagedChannel} */ - public final SimpleClient client; + @Synchronized + private static ManagedChannel sharedChannel(String host, int port) { + final String key = host + ":" + port; + ManagedChannel channel = POOL.get(key); + if (channel != null) { + final ConnectivityState state = channel.getState(true); + if (state == ConnectivityState.TRANSIENT_FAILURE || state == ConnectivityState.SHUTDOWN) { + channel.shutdownNow(); /* Close old channel. */ + channel = createChannel(host, port); + POOL.put(key, channel); + } + } else { + channel = createChannel(host, port); + POOL.put(key, channel); + } + return channel; + } + + /** + * Returns a new {@link ManagedChannel} object for the given database configuration. + * + * @param host Hostname of the Cottontail DB server. + * @param port Port of the Cottontail DB server. + * @return {@link ManagedChannel} + */ + private static ManagedChannel createChannel(String host, int port) { + final StopWatch watch = StopWatch.createStarted(); + LOGGER.debug("Starting to connect to Cottontail DB at {}:{}", host, port); + final NettyChannelBuilder builder = NettyChannelBuilder.forAddress(host, port).usePlaintext(); + final ManagedChannel channel = builder.build(); + Runtime.getRuntime().addShutdownHook(new Thread(() -> { + LOGGER.info("Closing connection to Cottontail DB."); + channel.shutdownNow(); + })); + watch.stop(); + LOGGER.info("Connected to Cottontail DB in {} ms at {}:{}", watch.getTime(TimeUnit.MILLISECONDS), host, port); + return channel; + } /** - * Flag indicating that his {@link CottontailWrapper}'s {@link ManagedChannel} should be kept open. + * The {@link SimpleClient} instance that facilitates access to Cottontail DB. */ - public final boolean keepOpen; + public final SimpleClient client; public String fqnInput(String entity) { return CINEAST_SCHEMA + "." + entity; @@ -40,24 +86,15 @@ public String fqnOutput(String entity) { return FQN_CINEAST_SCHEMA + "." + entity; } - public CottontailWrapper(DatabaseConfig config, boolean keepOpen) { + public CottontailWrapper(String host, int port) { StopWatch watch = StopWatch.createStarted(); - this.keepOpen = keepOpen; - LOGGER.debug("Starting to connect to cottontail at {}:{}", config.getHost(), config.getPort()); - final NettyChannelBuilder builder = NettyChannelBuilder - .forAddress(config.getHost(), config.getPort()); - if (config.getPlaintext()) { - builder.usePlaintext(); - } - this.channel = builder.build(); - this.client = new SimpleClient(this.channel); - + this.client = new SimpleClient(sharedChannel(host, port)); boolean pingSuccessful = this.client.ping(); watch.stop(); if (pingSuccessful) { - LOGGER.info("Connected to Cottontail in {} ms at {}:{}", watch.getTime(TimeUnit.MILLISECONDS), config.getHost(), config.getPort()); + LOGGER.info("Connected to Cottontail in {} ms at {}:{}", watch.getTime(TimeUnit.MILLISECONDS), host, port); } else { - LOGGER.warn("Could not connect to Cottontail at {}:{}", config.getHost(), config.getPort()); + LOGGER.warn("Could not connect to Cottontail at {}:{}", host, port); } } @@ -65,16 +102,5 @@ public CottontailWrapper(DatabaseConfig config, boolean keepOpen) { * Closes this {@link CottontailWrapper}. */ @Override - public void close() { - if (!this.keepOpen) { - LOGGER.info("Closing connection to Cottontail DB."); - try { - this.channel.shutdown(); - this.channel.awaitTermination(5000, TimeUnit.MILLISECONDS); - } catch (InterruptedException e) { - LOGGER - .warn("Thread was interrupted while waiting for gRPC channel to close (timeout = 5s)."); - } - } - } + public void close() { /* No op. */ } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailWriter.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailWriter.java index eb5a16812..3f7c8eed3 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailWriter.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailWriter.java @@ -1,13 +1,11 @@ package org.vitrivr.cineast.core.db.cottontaildb; - import io.grpc.StatusRuntimeException; import java.util.List; -import org.apache.logging.log4j.LogManager; import org.vitrivr.cineast.core.data.ReadableFloatVector; import org.vitrivr.cineast.core.db.AbstractPersistencyWriter; import org.vitrivr.cineast.core.db.PersistentTuple; -import org.vitrivr.cottontail.client.TupleIterator; +import org.vitrivr.cottontail.client.iterators.TupleIterator; import org.vitrivr.cottontail.client.language.basics.Constants; import org.vitrivr.cottontail.client.language.dml.BatchInsert; import org.vitrivr.cottontail.client.language.dml.Insert; @@ -21,15 +19,17 @@ public final class CottontailWriter extends AbstractPersistencyWriter { */ private final CottontailWrapper cottontail; - private static final org.apache.logging.log4j.Logger LOGGER = LogManager.getLogger(); - /** * The fully qualified name of the entity handled by this {@link CottontailWriter}. */ private String fqn; - public CottontailWriter(CottontailWrapper wrapper) { + /** The batch size to use for INSERTS. */ + private final int batchSize; + + public CottontailWriter(CottontailWrapper wrapper, int batchSize) { this.cottontail = wrapper; + this.batchSize = batchSize; } @Override @@ -39,14 +39,12 @@ public boolean open(String name) { } @Override - public void close() { - this.cottontail.close(); - } + public void close() { /* No op */ } @Override public boolean exists(String key, String value) { final Query query = new Query(this.fqn).exists().where(new Literal(key, "=", value)); - final TupleIterator results = this.cottontail.client.query(query, null); + final TupleIterator results = this.cottontail.client.query(query); final Boolean b = results.next().asBoolean("exists"); if (b != null) { return b; @@ -61,7 +59,7 @@ public boolean persist(List tuples) { int size = tuples.size(); final long txId = this.cottontail.client.begin(); try { - BatchInsert insert = new BatchInsert().into(this.fqn).columns(this.names); + BatchInsert insert = new BatchInsert().into(this.fqn).columns(this.names).txId(txId); while (!tuples.isEmpty()) { final PersistentTuple tuple = tuples.remove(0); final Object[] values = tuple.getElements().stream().map(o -> { @@ -74,13 +72,13 @@ public boolean persist(List tuples) { insert.append(values); if (insert.size() >= Constants.MAX_PAGE_SIZE_BYTES) { LOGGER.trace("Inserting msg of size {} into {}", insert.size(), this.fqn); - this.cottontail.client.insert(insert, txId); + this.cottontail.client.insert(insert); insert = new BatchInsert().into(this.fqn).columns(this.names); } } if (insert.getBuilder().getInsertsCount() > 0) { LOGGER.trace("Inserting msg of size {} into {}", insert.size(), this.fqn); - this.cottontail.client.insert(insert, txId); + this.cottontail.client.insert(insert); } this.cottontail.client.commit(txId); long stop = System.currentTimeMillis(); @@ -105,4 +103,9 @@ public Insert getPersistentRepresentation(PersistentTuple tuple) { } return insert; } + + @Override + public int supportedBatchSize() { + return this.batchSize; + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/reader/AbstractEntityReader.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/reader/AbstractEntityReader.java index 29a4ea117..dd4690480 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/reader/AbstractEntityReader.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/reader/AbstractEntityReader.java @@ -27,6 +27,4 @@ public AbstractEntityReader(DBSelector selector) { public void close() { this.selector.close(); } - - } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/writer/AbstractBatchedEntityWriter.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/writer/AbstractBatchedEntityWriter.java index 673db91af..3eff34396 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/writer/AbstractBatchedEntityWriter.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/writer/AbstractBatchedEntityWriter.java @@ -21,19 +21,17 @@ public abstract class AbstractBatchedEntityWriter implements Closeable { */ protected PersistencyWriter writer; + /** Flag indicating whether inserts should be batched in memory and submitted all at once. */ private final boolean batch; - protected AbstractBatchedEntityWriter(PersistencyWriter writer, int batchsize, boolean init) { - this.batch = batchsize > 1; + protected AbstractBatchedEntityWriter(PersistencyWriter writer) { + this.batch = writer.supportedBatchSize() > 1; if (this.batch) { - this.buffer = new ArrayBlockingQueue<>(batchsize); + this.buffer = new ArrayBlockingQueue<>(writer.supportedBatchSize()); } else { this.buffer = null; //not used } this.writer = writer; - if (init) { - this.init(); - } } protected abstract void init(); diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/writer/BatchedTagWriter.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/writer/BatchedTagWriter.java index d42c88e75..067bc4278 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/writer/BatchedTagWriter.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/writer/BatchedTagWriter.java @@ -12,11 +12,7 @@ public class BatchedTagWriter extends AbstractBatchedEntityWriter { private final String entityname; public BatchedTagWriter(PersistencyWriter writer, String entityname) { - this(writer, entityname, 1); - } - - public BatchedTagWriter(PersistencyWriter writer, String entityname, int batchsize) { - super(writer, batchsize, false); + super(writer); this.entityname = entityname; this.init(); } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/writer/MediaObjectMetadataWriter.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/writer/MediaObjectMetadataWriter.java index f000ae3e4..a2bc34998 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/writer/MediaObjectMetadataWriter.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/writer/MediaObjectMetadataWriter.java @@ -10,12 +10,12 @@ public class MediaObjectMetadataWriter extends AbstractBatchedEntityWriter writer, int batchsize) { - this(writer, batchsize, MediaObjectMetadataDescriptor.ENTITY); + public MediaObjectMetadataWriter(PersistencyWriter writer) { + this(writer, MediaObjectMetadataDescriptor.ENTITY); } - public MediaObjectMetadataWriter(PersistencyWriter writer, int batchsize, String tableName) { - super(writer, batchsize, false); + public MediaObjectMetadataWriter(PersistencyWriter writer, String tableName) { + super(writer); this.tableName = tableName; this.init(); } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/writer/MediaObjectWriter.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/writer/MediaObjectWriter.java index ac1f8fa0a..002e3c1f6 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/writer/MediaObjectWriter.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/writer/MediaObjectWriter.java @@ -8,7 +8,7 @@ public class MediaObjectWriter extends AbstractBatchedEntityWriter { public MediaObjectWriter(PersistencyWriter writer) { - super(writer, 1, true); + super(writer); } /** diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/writer/MediaSegmentMetadataWriter.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/writer/MediaSegmentMetadataWriter.java index 9f103c155..1a0551f2e 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/writer/MediaSegmentMetadataWriter.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/writer/MediaSegmentMetadataWriter.java @@ -9,12 +9,8 @@ public class MediaSegmentMetadataWriter extends AbstractBatchedEntityWriter writer, int batchsize) { - this(writer, batchsize, MediaSegmentMetadataDescriptor.ENTITY); - } - - public MediaSegmentMetadataWriter(PersistencyWriter writer, int batchsize, String testSegMetaTableName) { - super(writer, batchsize, false); + public MediaSegmentMetadataWriter(PersistencyWriter writer, String testSegMetaTableName) { + super(writer); this.tableName = testSegMetaTableName; this.init(); } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/writer/MediaSegmentWriter.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/writer/MediaSegmentWriter.java index 52fbde5ac..fd4e70487 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/writer/MediaSegmentWriter.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/writer/MediaSegmentWriter.java @@ -8,11 +8,7 @@ public class MediaSegmentWriter extends AbstractBatchedEntityWriter { public MediaSegmentWriter(PersistencyWriter writer) { - super(writer, 1, true); - } - - public MediaSegmentWriter(PersistencyWriter writer, int batchsize) { - super(writer, batchsize, true); + super(writer); } @Override diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/writer/PrimitiveTypeProviderFeatureDescriptorWriter.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/writer/PrimitiveTypeProviderFeatureDescriptorWriter.java index 9fa49d3cc..d5afbd7c8 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/writer/PrimitiveTypeProviderFeatureDescriptorWriter.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/writer/PrimitiveTypeProviderFeatureDescriptorWriter.java @@ -9,11 +9,7 @@ public class PrimitiveTypeProviderFeatureDescriptorWriter extends AbstractBatche private final String entityname; public PrimitiveTypeProviderFeatureDescriptorWriter(PersistencyWriter writer, String entityname) { - this(writer, entityname, 1); - } - - public PrimitiveTypeProviderFeatureDescriptorWriter(PersistencyWriter writer, String entityname, int batchsize) { - super(writer, batchsize, false); + super(writer); this.entityname = entityname; this.init(); } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/writer/SimpleBitSetWriter.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/writer/SimpleBitSetWriter.java index 4e4e16ae5..845d57bb0 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/writer/SimpleBitSetWriter.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/writer/SimpleBitSetWriter.java @@ -10,8 +10,8 @@ public class SimpleBitSetWriter extends AbstractBatchedEntityWriter writer, int batchsize, String entityName) { - super(writer, batchsize, false); + public SimpleBitSetWriter(PersistencyWriter writer, String entityName) { + super(writer); this.entityName = entityName; this.init(); } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/writer/SimpleFeatureDescriptorWriter.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/writer/SimpleFeatureDescriptorWriter.java index cdc07fcf8..a0c3aea03 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/writer/SimpleFeatureDescriptorWriter.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/writer/SimpleFeatureDescriptorWriter.java @@ -11,11 +11,7 @@ public class SimpleFeatureDescriptorWriter extends AbstractBatchedEntityWriter writer, String entityname) { - this(writer, entityname, 1); - } - - public SimpleFeatureDescriptorWriter(PersistencyWriter writer, String entityname, int batchsize) { - super(writer, batchsize, false); + super(writer); this.entityname = entityname; this.init(); } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/writer/SimpleFulltextFeatureDescriptorWriter.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/writer/SimpleFulltextFeatureDescriptorWriter.java index dafef55a2..9e9127af3 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/writer/SimpleFulltextFeatureDescriptorWriter.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/writer/SimpleFulltextFeatureDescriptorWriter.java @@ -9,13 +9,8 @@ public class SimpleFulltextFeatureDescriptorWriter extends AbstractBatchedEntity private final String entityname; public SimpleFulltextFeatureDescriptorWriter(PersistencyWriter writer, String entityname) { - this(writer, entityname, 1); - } - - public SimpleFulltextFeatureDescriptorWriter(PersistencyWriter writer, String entityname, int batchsize) { - super(writer, batchsize, false); + super(writer); this.entityname = entityname; - this.init(); } @Override diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/json/JsonFileWriter.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/json/JsonFileWriter.java index 4edb4207e..7ceb7bfcc 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/json/JsonFileWriter.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/json/JsonFileWriter.java @@ -122,6 +122,11 @@ public JsonObject getPersistentRepresentation(PersistentTuple tuple) { return _return; } + @Override + public int supportedBatchSize() { + return 1; + } + private static JsonArray toArray(boolean[] arr) { JsonArray jarr = new JsonArray(); for (int i = 0; i < arr.length; ++i) { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/memory/InMemoryWriter.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/memory/InMemoryWriter.java index 3f4a17b77..79957b359 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/memory/InMemoryWriter.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/memory/InMemoryWriter.java @@ -58,4 +58,9 @@ public boolean persist(List tuples) { public PersistentTuple getPersistentRepresentation(PersistentTuple tuple) { return tuple; } + + @Override + public int supportedBatchSize() { + return 1; + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/polypheny/PolyphenyEntityCreator.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/polypheny/PolyphenyEntityCreator.java new file mode 100644 index 000000000..6d55ac415 --- /dev/null +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/polypheny/PolyphenyEntityCreator.java @@ -0,0 +1,290 @@ +package org.vitrivr.cineast.core.db.polypheny; + +import static org.vitrivr.cineast.core.db.setup.AttributeDefinition.AttributeType.VECTOR; +import static org.vitrivr.cineast.core.util.CineastConstants.GENERIC_ID_COLUMN_QUALIFIER; + +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Arrays; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import org.vitrivr.cineast.core.data.entities.MediaObjectDescriptor; +import org.vitrivr.cineast.core.data.entities.MediaObjectMetadataDescriptor; +import org.vitrivr.cineast.core.data.entities.MediaSegmentDescriptor; +import org.vitrivr.cineast.core.data.entities.MediaSegmentMetadataDescriptor; +import org.vitrivr.cineast.core.db.setup.AttributeDefinition; +import org.vitrivr.cineast.core.db.setup.EntityCreator; +import org.vitrivr.cineast.core.db.setup.EntityDefinition; + +/** + * An {@link EntityCreator} implementation used to create the Cineast dada model in Polypheny DB. + * + * @author Ralph Gasser + * @version 1.0.0 + */ +public final class PolyphenyEntityCreator implements EntityCreator { + + /** + * Internal reference to the {@link PolyphenyWrapper} used by this {@link PolyphenyEntityCreator}. + */ + private final PolyphenyWrapper wrapper; + + /** + * Hint used to indicate primary key fields. + */ + private final static String PK_HINT = "pk"; + + /** + * Hint used to indicate that a field is nullable. + */ + private final static String NULLABLE_HINT = "nullable"; + + /** + * Constructor + * + * @param cottontailWrapper The {@link PolyphenyWrapper} to create this {@link PolyphenyEntityCreator} with. + */ + public PolyphenyEntityCreator(PolyphenyWrapper cottontailWrapper) { + this.wrapper = cottontailWrapper; + init(); + } + + /** + * Makes sure that schema 'cineast' is available. + */ + private void init() { + try (final Statement stmt = this.wrapper.connection.createStatement()) { + stmt.execute("CREATE SCHEMA IF NOT EXISTS " + PolyphenyWrapper.CINEAST_SCHEMA); + } catch (SQLException e) { + LOGGER.error("Error occurred during schema initialization", e); + } + } + + @Override + public boolean createMultiMediaObjectsEntity() { + final String entityName = PolyphenyWrapper.CINEAST_SCHEMA + "." + MediaObjectDescriptor.ENTITY; + try (final Statement stmt = this.wrapper.connection.createStatement()) { + stmt.execute("CREATE TABLE " + entityName + " (" + + MediaObjectDescriptor.FIELDNAMES[0] + " VARCHAR(255) NOT NULL," + /* object_id */ + MediaObjectDescriptor.FIELDNAMES[1] + " INT NOT NULL," + + MediaObjectDescriptor.FIELDNAMES[2] + " VARCHAR(255) NOT NULL ," + + MediaObjectDescriptor.FIELDNAMES[3] + " VARCHAR(255) NOT NULL," + + "PRIMARY KEY (" + MediaObjectDescriptor.FIELDNAMES[0] + ")" + + ") ON STORE " + PolyphenyWrapper.STORE_NAME_POSTGRESQL); + + /* TODO: Create index. */ + return true; + } catch (SQLException e) { + LOGGER.error("Error occurred while creating entity {}: {}", entityName, e); + return false; + } + } + + @Override + public boolean createSegmentEntity() { + final String entityName = PolyphenyWrapper.CINEAST_SCHEMA + "." + MediaSegmentDescriptor.ENTITY; + try (final Statement stmt = this.wrapper.connection.createStatement()) { + stmt.execute("CREATE TABLE " + entityName + " (" + + MediaSegmentDescriptor.FIELDNAMES[0] + " VARCHAR(255) NOT NULL," + /* segment_id */ + MediaSegmentDescriptor.FIELDNAMES[1] + " VARCHAR(255) NOT NULL," + /* object_id */ + MediaSegmentDescriptor.FIELDNAMES[2] + " INT NOT NULL ," + + MediaSegmentDescriptor.FIELDNAMES[3] + " INT NOT NULL," + + MediaSegmentDescriptor.FIELDNAMES[4] + " INT NOT NULL," + + MediaSegmentDescriptor.FIELDNAMES[5] + " REAL NOT NULL," + + MediaSegmentDescriptor.FIELDNAMES[6] + " REAL NOT NULL," + + "PRIMARY KEY (" + MediaSegmentDescriptor.FIELDNAMES[0] + ")" + + ") ON STORE " + PolyphenyWrapper.STORE_NAME_POSTGRESQL); + + /* TODO: Create index on segment_id and object_id. */ + return true; + } catch (SQLException e) { + LOGGER.error("Error occurred while creating entity {}: {}", entityName, e); + return false; + } + } + + @Override + public boolean createMetadataEntity(String tableName) { + final String entityName = PolyphenyWrapper.CINEAST_SCHEMA + "." + tableName; + try (final Statement stmt = this.wrapper.connection.createStatement()) { + stmt.execute("CREATE TABLE " + entityName + " (" + + MediaObjectMetadataDescriptor.FIELDNAMES[0] + " VARCHAR(255) NOT NULL," + /* object_id */ + MediaObjectMetadataDescriptor.FIELDNAMES[1] + " VARCHAR(255) NOT NULL," + /* domain */ + MediaObjectMetadataDescriptor.FIELDNAMES[2] + " VARCHAR(255) NOT NULL," + /* key */ + "\"" + MediaObjectMetadataDescriptor.FIELDNAMES[3] + "\"" + " VARCHAR(255) NOT NULL," + /* value */ + "PRIMARY KEY (" + MediaObjectMetadataDescriptor.FIELDNAMES[0] + "," + MediaObjectMetadataDescriptor.FIELDNAMES[1] + "," + MediaObjectMetadataDescriptor.FIELDNAMES[2] + ")" + + ") ON STORE " + PolyphenyWrapper.STORE_NAME_POSTGRESQL); + + /* TODO: Create index on object_id and domain. */ + return true; + } catch (SQLException e) { + LOGGER.error("Error occurred while creating entity {}: {}", entityName, e); + return false; + } + } + + @Override + public boolean createSegmentMetadataEntity(String tableName) { + final String entityName = PolyphenyWrapper.CINEAST_SCHEMA + "." + tableName; + try (final Statement stmt = this.wrapper.connection.createStatement()) { + stmt.execute("CREATE TABLE " + entityName + " (" + + MediaSegmentMetadataDescriptor.FIELDNAMES[0] + " VARCHAR(255) NOT NULL," + /* segment_id */ + MediaSegmentMetadataDescriptor.FIELDNAMES[1] + " VARCHAR(255) NOT NULL," + /* domain */ + MediaSegmentMetadataDescriptor.FIELDNAMES[2] + " VARCHAR(255) NOT NULL ," + /* key */ + "\"" + MediaObjectMetadataDescriptor.FIELDNAMES[3] + "\"" + " VARCHAR(255) NOT NULL," + /* value */ + "PRIMARY KEY (" + MediaSegmentMetadataDescriptor.FIELDNAMES[0] + "," + MediaSegmentMetadataDescriptor.FIELDNAMES[1] + "," + MediaSegmentMetadataDescriptor.FIELDNAMES[2] + ")" + + ") ON STORE " + PolyphenyWrapper.STORE_NAME_POSTGRESQL); + + /* TODO: Create index on object_id and domain. */ + return true; + } catch (SQLException e) { + LOGGER.error("Error occurred while creating entity {}: {}", entityName, e); + return false; + } + } + + @Override + public boolean createFeatureEntity(String featureEntityName, boolean unique, int length, String... featureNames) { + final AttributeDefinition[] attributes = Arrays.stream(featureNames).map(s -> new AttributeDefinition(s, VECTOR, length)).toArray(AttributeDefinition[]::new); + return this.createFeatureEntity(featureEntityName, unique, attributes); + } + + @Override + public boolean createFeatureEntity(String featureEntityName, boolean unique, AttributeDefinition... attributes) { + final AttributeDefinition[] extended = new AttributeDefinition[attributes.length + 1]; + final HashMap hints = new HashMap<>(2); + if (unique) { + hints.put(PK_HINT, Boolean.TRUE.toString()); + hints.put(NULLABLE_HINT, Boolean.FALSE.toString()); + } + extended[0] = new AttributeDefinition(GENERIC_ID_COLUMN_QUALIFIER, AttributeDefinition.AttributeType.STRING, hints); + System.arraycopy(attributes, 0, extended, 1, attributes.length); + return this.createEntity(featureEntityName, extended); + } + + @Override + public boolean createIdEntity(String entityName, AttributeDefinition... attributes) { + final AttributeDefinition[] extended = new AttributeDefinition[attributes.length + 1]; + final HashMap hints = new HashMap<>(2); + hints.put(PK_HINT, Boolean.TRUE.toString()); + hints.put(NULLABLE_HINT, Boolean.FALSE.toString()); + extended[0] = new AttributeDefinition(GENERIC_ID_COLUMN_QUALIFIER, AttributeDefinition.AttributeType.STRING, hints); + System.arraycopy(attributes, 0, extended, 1, attributes.length); + return this.createEntity(entityName, extended); + } + + @Override + public boolean createEntity(String entityName, AttributeDefinition... attributes) { + return this.createEntity(new org.vitrivr.cineast.core.db.setup.EntityDefinition.EntityDefinitionBuilder(entityName).withAttributes(attributes).build()); + } + + @Override + public boolean createEntity(EntityDefinition entityDefinition) { + final String entityFullName = PolyphenyWrapper.CINEAST_SCHEMA + "." + entityDefinition.getEntityName(); + try (final Statement stmt = this.wrapper.connection.createStatement()) { + final StringBuilder builder = new StringBuilder("CREATE TABLE " + entityFullName + " ("); + String store = PolyphenyWrapper.STORE_NAME_POSTGRESQL; + List pk = new LinkedList<>(); + int index = 0; + for (AttributeDefinition attribute : entityDefinition.getAttributes()) { + switch (attribute.getType()) { + case BOOLEAN: + builder.append(attribute.getName() + " BOOLEAN "); + break; + case DOUBLE: + builder.append(attribute.getName() + " DOUBLE "); + break; + case FLOAT: + builder.append(attribute.getName() + " REAL "); + break; + case INT: + builder.append(attribute.getName() + " INT "); + break; + case LONG: + builder.append(attribute.getName() + " LONG "); + break; + case STRING: + builder.append(attribute.getName() + " VARCHAR(255) "); + break; + case TEXT: + builder.append(attribute.getName() + " VARCHAR(65535) "); + break; + case VECTOR: + builder.append(attribute.getName() + " REAL ARRAY(1," + attribute.getLength() + ") "); + store = PolyphenyWrapper.STORE_NAME_COTTONTAIL; + break; + case BITSET: + builder.append(attribute.getName() + " BOOLEAN ARRAY(1," + attribute.getLength() + ") "); + store = PolyphenyWrapper.STORE_NAME_COTTONTAIL; + break; + default: + throw new RuntimeException("Type " + attribute.getType() + " has no matching analogue in Cottontail DB"); + } + + if (attribute.getHint(PK_HINT).map(h -> h.equals(Boolean.TRUE.toString())).orElse(false)) { + pk.add(attribute.getName()); + builder.append("NOT NULL"); + } else { + if (attribute.getHint(NULLABLE_HINT).map(h -> h.equals(Boolean.TRUE.toString())).orElse(true)) { + builder.append("NULL"); + } else { + builder.append("NOT NULL"); + } + } + + if ((index++) < entityDefinition.getAttributes().size() - 1) { + builder.append(", "); + } + } + + /* Add PRIMARY KEY definition. */ + if (!pk.isEmpty()) { + builder.append(", PRIMARY KEY("); + builder.append(String.join(",", pk)); + builder.append(")"); + } + + /* Add ON STORE. */ + builder.append(") ON STORE " + store); + stmt.execute(builder.toString()); + return true; + } catch (SQLException e) { + LOGGER.error("Error occurred while creating entity {}: {}", entityFullName, e); + return false; + } + } + + @Override + public boolean existsEntity(String entityName) { + final String entityFullName = PolyphenyWrapper.CINEAST_SCHEMA + "." + entityName; + try (final Statement stmt = this.wrapper.connection.createStatement()) { + stmt.executeQuery("SELECT * FROM " + entityFullName); + return true; + } catch (SQLException e) { + return false; + } + } + + @Override + public boolean dropEntity(String entityName) { + final String entityFullName = PolyphenyWrapper.CINEAST_SCHEMA + "." + entityName; + try (final Statement stmt = this.wrapper.connection.createStatement()) { + stmt.execute("DROP TABLE IF EXISTS " + entityFullName); + return true; + } catch (SQLException e) { + LOGGER.error("Error occurred while creating entity {}: {}", entityName, e); + return false; + } + } + + @Override + public boolean createHashNonUniqueIndex(String entityName, String column) { + return false; + } + + @Override + public void close() { + this.wrapper.close(); + } +} diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/polypheny/PolyphenySelector.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/polypheny/PolyphenySelector.java new file mode 100644 index 000000000..c0138e031 --- /dev/null +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/polypheny/PolyphenySelector.java @@ -0,0 +1,498 @@ +package org.vitrivr.cineast.core.db.polypheny; + +import static org.vitrivr.cineast.core.util.CineastConstants.GENERIC_ID_COLUMN_QUALIFIER; + +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; +import org.apache.commons.lang3.time.StopWatch; +import org.apache.commons.lang3.tuple.Triple; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.vitrivr.cineast.core.config.ReadableQueryConfig; +import org.vitrivr.cineast.core.config.ReadableQueryConfig.Distance; +import org.vitrivr.cineast.core.data.distance.DistanceElement; +import org.vitrivr.cineast.core.data.providers.primitive.BooleanProviderImpl; +import org.vitrivr.cineast.core.data.providers.primitive.ByteProviderImpl; +import org.vitrivr.cineast.core.data.providers.primitive.DoubleProviderImpl; +import org.vitrivr.cineast.core.data.providers.primitive.FloatProviderImpl; +import org.vitrivr.cineast.core.data.providers.primitive.IntProviderImpl; +import org.vitrivr.cineast.core.data.providers.primitive.LongProviderImpl; +import org.vitrivr.cineast.core.data.providers.primitive.PrimitiveTypeProvider; +import org.vitrivr.cineast.core.data.providers.primitive.ShortProviderImpl; +import org.vitrivr.cineast.core.data.providers.primitive.StringProviderImpl; +import org.vitrivr.cineast.core.db.DBSelector; +import org.vitrivr.cineast.core.db.RelationalOperator; + +/** + * A {@link DBSelector} implementation used to read data from Polypheny DB. + * + * @author Ralph Gasser + * @version 1.0.0 + */ +public final class PolyphenySelector implements DBSelector { + + private static final Logger LOGGER = LogManager.getLogger(); + + /** + * Internal reference to the {@link PolyphenyWrapper} used by this {@link PolyphenyEntityCreator}. + */ + private final PolyphenyWrapper wrapper; + + /** + * The fully qualified name of the entity handled by this {@link PolyphenySelector}. + */ + private String fqn; + + public PolyphenySelector(PolyphenyWrapper wrapper) { + this.wrapper = wrapper; + } + + @Override + public boolean open(String name) { + this.fqn = this.wrapper.fqnInput(name); + return false; + } + + @Override + public void close() { + this.wrapper.close(); + } + + @Override + public List getNearestNeighboursGeneric(int k, float[] vector, String column, Class distanceElementClass, ReadableQueryConfig config) { + final Distance distance = config.getDistance().orElse(Distance.euclidean); + try (final PreparedStatement statement = this.wrapper.connection.prepareStatement("SELECT id, distance(" + column + "," + toVectorString(vector) + ",'" + toName(distance) + "') as dist FROM " + this.fqn + " ORDER BY dist ASC LIMIT " + k)) { + /* Execute query and return results. */ + try (final ResultSet rs = statement.executeQuery()) { + return processResults(rs).stream().map(e -> { + final String id = e.get(GENERIC_ID_COLUMN_QUALIFIER).getString(); + double d = e.get("dist").getDouble(); /* This should be fine. */ + return DistanceElement.create(distanceElementClass, id, d); + }).collect(Collectors.toList()); + } + } catch (SQLException e) { + LOGGER.error("Error occurred during query execution in getNearestNeighboursGeneric(): {}", e.getMessage()); + return new ArrayList<>(0); + } + } + + @Override + public List getBatchedNearestNeighbours(int k, List vectors, String column, Class distanceElementClass, List configs) { + LOGGER.warn("Error occurred during query execution in getBatchedNearestNeighbours(): Not supported"); + return new ArrayList<>(0); + } + + @Override + public List> getNearestNeighbourRows(int k, float[] vector, String column, ReadableQueryConfig config) { + final Distance distance = config.getDistance().orElse(Distance.euclidean); + + try (final PreparedStatement statement = this.wrapper.connection.prepareStatement("SELECT id, distance(" + column + "," + toVectorString(vector) + ",'" + toName(distance) + "') as dist FROM " + this.fqn + " ORDER BY dist ASC LIMIT " + k)) { + /* Execute query and return results. */ + try (final ResultSet rs = statement.executeQuery()) { + return processResults(rs); + } + } catch (SQLException e) { + LOGGER.error("Error occurred during query execution in getNearestNeighbourRows(): {}", e.getMessage()); + return new ArrayList<>(0); + } + } + + @Override + public List getFeatureVectors(String fieldName, PrimitiveTypeProvider value, String vectorName) { + try (final PreparedStatement statement = this.prepareStatement(fieldName, RelationalOperator.EQ, List.of(value))) { + /* Execute query and return results. */ + final List _return = new LinkedList<>(); + try (final ResultSet rs = statement.executeQuery()) { + while (rs.next()) { + final Object converted = rs.getArray(vectorName).getArray(); + if (converted instanceof float[]) { + _return.add((float[]) converted); + } + } + } + return _return; + } catch (SQLException e) { + LOGGER.warn("Error occurred during query execution in getFeatureVectors(): {}", e.getMessage()); + return new ArrayList<>(0); + } + } + + @Override + public List> getRows(String fieldName, Iterable values) { + final Object[] mapped = StreamSupport.stream(values.spliterator(), false).map(PrimitiveTypeProvider::toObject).toArray(); + if (mapped.length == 0) { + return new ArrayList<>(0); + } + try (final PreparedStatement statement = this.prepareInStatement(fieldName, values)) { + /* Execute query and return results. */ + try (final ResultSet rs = statement.executeQuery()) { + return processResults(rs); + } + } catch (SQLException e) { + LOGGER.warn("Error occurred during query execution in getRows(): {}", e.getMessage()); + return new ArrayList<>(0); + } + } + + @Override + public List> getFulltextRows(int rows, String fieldname, ReadableQueryConfig queryConfig, String... terms) { + LOGGER.warn("Error occurred during query execution in getFulltextRows(): Not supported"); + return new ArrayList<>(0); + } + + @Override + public List> getRows(String fieldName, RelationalOperator operator, Iterable values) { + try (final PreparedStatement statement = this.prepareStatement(fieldName, operator, values)) { + try (final ResultSet rs = statement.executeQuery()) { + return processResults(rs); + } + } catch (SQLException e) { + LOGGER.warn("Error occurred during query execution in getRows(): {}", e.getMessage()); + return new ArrayList<>(0); + } + } + + /** + * Performs a boolean lookup based on multiple conditions, linked with AND. Each element of the list specifies one of the conditions - left middle right, i.e. id IN (1, 5, 7) + * + * @param conditions conditions which will be linked by AND + * @param identifier column upon which the retain operation will be performed if the database layer does not support compound boolean retrieval. + * @param projection Which columns shall be selected + */ + @Override + public List> getRowsAND(List>> conditions, String identifier, List projection, ReadableQueryConfig qc) { + final StringBuilder conditionBuilder = new StringBuilder(); + final LinkedList values = new LinkedList<>(); + int i = 0; + for (Triple> condition: conditions) { + if (i++ > 0) conditionBuilder.append(" AND "); + conditionBuilder.append(condition.getLeft()); + conditionBuilder.append(" "); + conditionBuilder.append(toPredicate(condition.getMiddle())); + if (condition.getMiddle() == RelationalOperator.IN) { + int j = 0; + conditionBuilder.append("("); + for (PrimitiveTypeProvider v : condition.getRight()) { + if (j++ > 0) conditionBuilder.append(","); + if (v instanceof StringProviderImpl) { + conditionBuilder.append("'"); + conditionBuilder.append(v.getString()); + conditionBuilder.append("'"); + } else { + conditionBuilder.append(v.getString()); + } + } + conditionBuilder.append(")"); + } else { + values.addAll(condition.getRight()); + } + } + + try (final PreparedStatement statement = this.wrapper.connection.prepareStatement("SELECT * FROM " + this.fqn + " WHERE " + conditionBuilder)) { + int k = 1; + for (PrimitiveTypeProvider v : values) { + this.bindScalarValue(k++, v, statement); + } + try (final ResultSet rs = statement.executeQuery()) { + return processResults(rs); + } + } catch (SQLException e) { + LOGGER.warn("Error occurred during query execution in getRowsAND(): {}", e.getMessage()); + return new ArrayList<>(0); + } + } + + @Override + public List> getAll(List columns, int limit) { + try (final Statement statement = this.wrapper.connection.createStatement()) { + if (limit > 0) { + try (final ResultSet rs = statement.executeQuery("SELECT " + String.join(",", columns) + " FROM " + this.fqn + " LIMIT " + limit)) { + return processResults(rs); + } + } else { + try (final ResultSet rs = statement.executeQuery("SELECT " + String.join(",", columns) + " FROM " + this.fqn)) { + return processResults(rs); + } + } + } catch (SQLException e) { + LOGGER.warn("Error occurred during query execution in getAll(): {}", e.getMessage()); + return new ArrayList<>(0); + } + } + + @Override + public List getAll(String column) { + try (final Statement statement = this.wrapper.connection.createStatement()) { + try (final ResultSet rs = statement.executeQuery("SELECT " + column + " FROM " + this.fqn)) { + return processSingleColumnResult(rs); + } + } catch (SQLException e) { + LOGGER.warn("Error occurred during query execution in getAll(): {}", e.getMessage()); + return new ArrayList<>(0); + } + } + + @Override + public List> getAll() { + try (final Statement statement = this.wrapper.connection.createStatement()) { + try (final ResultSet rs = statement.executeQuery("SELECT * FROM " + this.fqn)) { + return processResults(rs); + } + } catch (SQLException e) { + LOGGER.warn("Error occurred during query execution in getAll(): {}", e.getMessage()); + return new ArrayList<>(0); + } + } + + @Override + public boolean existsEntity(String name) { + try { + try (final ResultSet rs = this.wrapper.connection.getMetaData().getTables("%", PolyphenyWrapper.CINEAST_SCHEMA, name, null)) { + return rs.next(); + } + } catch (SQLException e) { + LOGGER.warn("Error occurred during query execution in existsEntity(): {}", e.getMessage()); + return false; + } + } + + @Override + public boolean ping() { + try (final Statement statement = this.wrapper.connection.createStatement()) { + try (final ResultSet rs = statement.executeQuery("SELECT 1")) { + return rs.next(); + } + } catch (SQLException e) { + LOGGER.warn("Error occurred during query execution in ping(): {}", e.getMessage()); + return false; + } + } + + /** + * Converts a {@link ResultSet} response generated by Polypheny DB into a {@link List} {@link PrimitiveTypeProvider}s that contain the results of a single column query. + * + * @param results {@link ResultSet} to gather the results from. + * @return {@link List} of {@link Map}s that contains the results. + * @throws SQLException If result set processing fails. + */ + private static List processSingleColumnResult(ResultSet results) throws SQLException { + final List _return = new LinkedList<>(); + while (results.next()) { + _return.add(PrimitiveTypeProvider.fromObject(results.getObject(1))); + } + return _return; + } + + /** + * Converts a {@link ResultSet} response generated by Polypheny DB into a {@link List} of column to {@link PrimitiveTypeProvider}s that contain the results of a query. + * + * @param results The {@link ResultSet} to convert. + * @return A {@link} of column name to {@link PrimitiveTypeProvider} mappings for each row. + * @throws SQLException If result set processing fails. + */ + private static List> processResults(ResultSet results) throws SQLException { + final List> _return = new LinkedList<>(); + final StopWatch watch = StopWatch.createStarted(); + final ResultSetMetaData rsmd = results.getMetaData(); + while (results.next()) { + final Map map = new HashMap<>(rsmd.getColumnCount()); + for (int index = 1; index <= rsmd.getColumnCount(); index++) { + final String label = rsmd.getColumnLabel(index); + map.put(label, PrimitiveTypeProvider.fromObject(results.getObject(index))); + } + _return.add(map); + } + LOGGER.trace("Processed {} results in {} ms", _return.size(), watch.getTime(TimeUnit.MILLISECONDS)); + return _return; + } + + /** + * Converts a flat vector to a string representation usable by Polypheny DB. + * + * @param vector {@link Distance} The float vector to convert. + * @return The resulting name. + */ + private static String toVectorString(float[] vector) { + final StringBuilder arrayString = new StringBuilder("ARRAY["); + int i = 0; + for (float v : vector) { + if (i++ > 0) { + arrayString.append(","); + } + arrayString.append(v); + } + arrayString.append("]"); + return arrayString.toString(); + } + + /** + * Converts a {@link Distance} to a name usable by Polypheny DB. + * + * @param distance {@link Distance} The distance to convert. + * @return The resulting name. + */ + private static String toName(Distance distance) { + switch (distance) { + case chisquared: + return "ChiSquared"; + case cosine: + return "Cosine"; + case manhattan: + return "L1"; + case euclidean: + return "L2"; + case squaredeuclidean: + return "L2 squared"; + default: + throw new IllegalArgumentException("Distance " + distance.name() + " is not supported by Polypheny DB."); + } + } + + /** + * Converts a Cineast {@link RelationalOperator} into the corresponding Cottontail DB representation. + * + * @param op {@link RelationalOperator} to convert. + * @return {@link String} representing the predicate part of a query. + */ + private static String toPredicate(RelationalOperator op) { + switch (op) { + case EQ: + return "= ?"; + case NEQ: + return "!= ?"; + case GEQ: + return ">= ?"; + case LEQ: + return "<= ?"; + case GREATER: + return "> ?"; + case LESS: + return "< ?"; + case BETWEEN: + return "BETWEEN ? AND ?"; + case LIKE: + return "LIKE ?"; + case NLIKE: + return "NOT LIKE ?"; + case ISNULL: + return "IS NULL"; + case ISNOTNULL: + return "IS NOT NULL"; + case IN: + return "IN "; + default: + throw new IllegalArgumentException("Operator '" + op + "' not supported by Cottontail DB."); + } + } + + /** + * Binds a scalar value to a {@link PreparedStatement}. + * + * @param index Index of the placeholder to bind to. + * @param value {@link PrimitiveTypeProvider} of values to bind. + * @param statement {@link PreparedStatement} to bind values to. + */ + private void bindScalarValue(int index, PrimitiveTypeProvider value, PreparedStatement statement) throws SQLException { + /* Bind values. */ + if (value instanceof DoubleProviderImpl) { + statement.setDouble(index, value.getDouble()); + } else if (value instanceof FloatProviderImpl) { + statement.setFloat(index, value.getFloat()); + } else if (value instanceof LongProviderImpl) { + statement.setLong(index, value.getLong()); + } else if (value instanceof IntProviderImpl) { + statement.setInt(index, value.getInt()); + } else if (value instanceof ShortProviderImpl) { + statement.setShort(index, value.getShort()); + } else if (value instanceof ByteProviderImpl) { + statement.setByte(index, value.getByte()); + } else if (value instanceof BooleanProviderImpl) { + statement.setBoolean(index, value.getBoolean()); + } else if (value instanceof StringProviderImpl) { + statement.setString(index, value.getString()); + } else { + LOGGER.warn("Error occurred during query execution in getRows(): {} not supported as parameter for IN query.", value); + } + } + + /** + * Prepares a prepared statement {@link PreparedStatement} for a query with a single IN predicate. + * + * @param fieldName Name of the field that should be queried. + * @param values Values to use in query. + * @return {@link PreparedStatement} + */ + private PreparedStatement prepareStatement(String fieldName, RelationalOperator operator, Iterable values) throws SQLException { + final Object[] mapped; + final PreparedStatement statement; + switch (operator) { + case ISNOTNULL: + case ISNULL: + return this.wrapper.connection.prepareStatement("SELECT * FROM " + this.fqn + " WHERE " + fieldName + " " + toPredicate(operator)); + case EQ: + case NEQ: + case GEQ: + case LEQ: + case GREATER: + case LESS: + case LIKE: + case NLIKE: + mapped = StreamSupport.stream(values.spliterator(), false).limit(1).toArray(); + statement = this.wrapper.connection.prepareStatement("SELECT * FROM " + this.fqn + " WHERE " + fieldName + " " + toPredicate(operator)); + this.bindScalarValue(1, (PrimitiveTypeProvider) mapped[0], statement); + return statement; + case BETWEEN: + mapped = StreamSupport.stream(values.spliterator(), false).limit(2).toArray(); + statement = this.wrapper.connection.prepareStatement("SELECT * FROM " + this.fqn + " WHERE " + fieldName + " " + toPredicate(operator)); + this.bindScalarValue(1, (PrimitiveTypeProvider) mapped[0], statement); + this.bindScalarValue(2, (PrimitiveTypeProvider) mapped[1], statement); + return statement; + case IN: + return this.prepareInStatement(fieldName, values); + default: + throw new IllegalArgumentException("Operator '" + operator + "' not supported by Cottontail DB."); + } + } + + /** + * Prepares a prepared statement {@link PreparedStatement} for a query with a single IN predicate. + * + * @param fieldName Name of the field that should be queried. + * @param values Values to use in query. + * @return {@link PreparedStatement} + * @throws SQLException + */ + private PreparedStatement prepareInStatement(String fieldName, Iterable values) throws SQLException { + /* Prepare query (apparently, JDBC doesn't support value binding for IN predicates).*/ + final StringBuilder stringStatement = new StringBuilder("SELECT * FROM " + this.fqn + " WHERE " + fieldName + " IN ("); + int index = 0; + for (PrimitiveTypeProvider v : values) { + if (index++ > 0) { + stringStatement.append(","); + } + if (v instanceof StringProviderImpl) { + stringStatement.append("'"); + stringStatement.append(v.getString()); + stringStatement.append("'"); + } else { + stringStatement.append(v.getString()); + } + } + stringStatement.append(")"); + + return this.wrapper.connection.prepareStatement(stringStatement.toString()); + } +} diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/polypheny/PolyphenyWrapper.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/polypheny/PolyphenyWrapper.java new file mode 100644 index 000000000..60c5d9b7f --- /dev/null +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/polypheny/PolyphenyWrapper.java @@ -0,0 +1,75 @@ +package org.vitrivr.cineast.core.db.polypheny; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.util.Properties; +import java.util.concurrent.TimeUnit; +import org.apache.commons.lang3.time.StopWatch; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +/** + * A wrapper class that exposes the Polypheny DB JDBC {@link Connection} used by Cineast. + * + * @author Ralph Gasser + * @version 1.0.0 + */ +public final class PolyphenyWrapper implements AutoCloseable { + + /** + * {@link Logger} used by this PolyphenyWrapper. + */ + private static final Logger LOGGER = LogManager.getLogger(); + + /** + * Name of the cineast schema in Polypheny DB. + */ + public static final String CINEAST_SCHEMA = "cineast"; + + /** + * Store name PostgreSQL instances. + */ + public static final String STORE_NAME_POSTGRESQL = "postgresql"; + + /** + * Store name Cottontail DB instances. + */ + public static final String STORE_NAME_COTTONTAIL = "cottontaildb"; + + /** + * The JDBC {@link Connection} used to communicate with Polypheny DB. + */ + final Connection connection; + + public PolyphenyWrapper(String host, int port) { + StopWatch watch = StopWatch.createStarted(); + LOGGER.debug("Starting to connect to Polypheny DB at {}:{}", host, port); + /* Try to instantiate Polypheny driver. */ + try { + Class.forName("org.polypheny.jdbc.Driver"); /* Make sure, driver was loaded. */ + final Properties properties = new Properties(); + properties.put("username", "pa"); /* TODO: Could be configurable :-) */ + this.connection = DriverManager.getConnection(String.format("jdbc:polypheny:http://%s/", host, port), properties); + } catch (ClassNotFoundException | SQLException e) { + throw new IllegalStateException("Failed to initialize JDBC connection to Polypheny DB due to error: " + e.getMessage()); + } + + watch.stop(); + LOGGER.debug("Connected to Polypheny DB in {} ms at {}", watch.getTime(TimeUnit.MILLISECONDS), host); + } + + public String fqnInput(String entity) { + return CINEAST_SCHEMA + "." + entity; + } + + @Override + public void close() { + try { + LOGGER.debug("Closing JDBC connection to Polypheny DB."); + this.connection.close(); + } catch (SQLException e) { + LOGGER.error("Closing JDBC connection to Polypheny DB failed: {}", e.getMessage()); + } + } +} diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/polypheny/PolyphenyWriter.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/polypheny/PolyphenyWriter.java new file mode 100644 index 000000000..22fc50b80 --- /dev/null +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/polypheny/PolyphenyWriter.java @@ -0,0 +1,173 @@ +package org.vitrivr.cineast.core.db.polypheny; + +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Types; +import java.util.List; +import org.vitrivr.cineast.core.data.ReadableFloatVector; +import org.vitrivr.cineast.core.db.AbstractPersistencyWriter; +import org.vitrivr.cineast.core.db.PersistentTuple; + +/** + * A {@link org.vitrivr.cineast.core.db.PersistencyWriter} implementation used to write data to Polypheny DB. + * + * @author Ralph Gasser + * @version 1.0.0 + */ +public final class PolyphenyWriter extends AbstractPersistencyWriter { + + /** + * Internal reference to the {@link PolyphenyWrapper} used by this {@link PolyphenyWriter}. + */ + private final PolyphenyWrapper wrapper; + + /** + * The fully qualified name of the entity handled by this {@link PolyphenyWriter}. + */ + private String fqn; + + /** The batch size to use for INSERTS; this is simply a property exposed. Only batched persistency writers make use of batched INSERTS. */ + private final int batchSize; + + + public PolyphenyWriter(PolyphenyWrapper wrapper, int batchSize) { + this.wrapper = wrapper; + this.batchSize = batchSize; + } + + @Override + public boolean open(String name) { + this.fqn = this.wrapper.fqnInput(name); + return true; + } + + @Override + public void close() { + this.wrapper.close(); + } + + @Override + public boolean exists(String key, String value) { + try (final PreparedStatement stmt = this.wrapper.connection.prepareStatement("SELECT COUNT(*) FROM " + this.fqn + " WHERE " + key + " = ?")) { + stmt.setString(1, value); + final ResultSet rs = stmt.executeQuery(); + if (rs.next()) { + return rs.getInt(1) > 0; + } else { + return false; + } + } catch (SQLException e) { + LOGGER.error("Error occurred while executing query {}: {}", this.fqn, e); + return false; + } + } + + @Override + public boolean persist(List tuples) { + long start = System.currentTimeMillis(); + int size = tuples.size(); + final String query = this.createInsertStatement(tuples.get(0)); + try (final PreparedStatement stmt = this.wrapper.connection.prepareStatement(query)) { + while (!tuples.isEmpty()) { + bindInsertStatement(stmt, tuples.remove(0)); + stmt.addBatch(); + } + stmt.executeBatch(); /* Execute INSERTs. */ + this.wrapper.connection.commit(); + long stop = System.currentTimeMillis(); + LOGGER.trace("Completed insert of {} elements in {} ms", size, stop - start); + return true; + } catch (SQLException e) { + LOGGER.error("Error occurred while executing INSERT on {}: {}.", this.fqn, e.getMessage()); + return false; + } + } + + @Override + public PreparedStatement getPersistentRepresentation(PersistentTuple tuple) { + final String query = this.createInsertStatement(tuple); + try (final PreparedStatement stmt = this.wrapper.connection.prepareStatement(query)) { + return bindInsertStatement(stmt, tuple); + } catch (SQLException e) { + LOGGER.error("Error occurred while constructing INSERT on {}: {}", this.fqn, e); + return null; + } + } + + @Override + public int supportedBatchSize() { + return this.batchSize; + } + + /** + * Constructs and returns an INSERT query for a persistent tuple. + * + * @param tuple The {@link PersistentTuple} to convert. + * @return Resulting INSERT query as string. + */ + private String createInsertStatement(PersistentTuple tuple) { + final StringBuilder insert = new StringBuilder("INSERT INTO ").append(this.fqn).append(" ("); + final StringBuilder values = new StringBuilder(") VALUES ("); + int index = 0; + for (Object ignored : tuple.getElements()) { + insert.append("\"").append(this.names[index++]).append("\""); + values.append("?"); + if ((index) < tuple.getElements().size()) { + insert.append(","); + values.append(","); + } + } + values.append(")"); + return insert.append(values).toString(); + } + + /** + * Binds the provided {@link PersistentTuple} to the given {@link PreparedStatement}. + * + * @param stmt The {@link PreparedStatement} to bind values to. + * @param tuple The {@link PersistentTuple} containing the values. + * @return True on success, false otherwise. + */ + private PreparedStatement bindInsertStatement(PreparedStatement stmt, PersistentTuple tuple) throws SQLException { + int index = 1; + for (Object o : tuple.getElements()) { + if (o instanceof Long) { + stmt.setLong(index++, (Long) o); + } else if (o instanceof Integer) { + stmt.setInt(index++, (Integer) o); + } else if (o instanceof Float) { + stmt.setFloat(index++, (Float) o); + } else if (o instanceof Double) { + stmt.setDouble(index++, (Double) o); + } else if (o instanceof Boolean) { + stmt.setBoolean(index++, (Boolean) o); + } else if (o instanceof String) { + stmt.setString(index++, (String) o); + } else if (o instanceof float[]) { + final Object[] list = new Object[((float[]) o).length]; + for (int i = 0; i < ((float[]) o).length; i++) { + list[i] = ((float[]) o)[i]; + } + stmt.setArray(index++, this.wrapper.connection.createArrayOf("REAL", list)); + } else if (o instanceof int[]) { + final Object[] list = new Object[((int[]) o).length]; + for (int i = 0; i < ((int[]) o).length; i++) { + list[i] = ((int[]) o)[i]; + } + stmt.setArray(index++, this.wrapper.connection.createArrayOf("INTEGER", list)); + } else if (o instanceof ReadableFloatVector) { + final Object[] list = new Object[((ReadableFloatVector) o).getElementCount()]; + for (int i = 0; i < ((ReadableFloatVector) o).getElementCount(); i++) { + list[i] = ((ReadableFloatVector) o).getElement(i); + } + stmt.setArray(index++, this.wrapper.connection.createArrayOf("REAL", list)); + } else if (o == null) { + stmt.setNull(index++, Types.NULL); + } else { + LOGGER.error("Error occurred while binding value to INSERT on {}: Encountered unsupported value {}.", this.fqn, o.toString()); + } + } + return stmt; + } +} diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/ExtractionContextProvider.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/ExtractionContextProvider.java index 2effd4238..c12e47473 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/ExtractionContextProvider.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/ExtractionContextProvider.java @@ -4,10 +4,13 @@ import java.nio.file.Path; import java.util.List; import java.util.Optional; +import java.util.function.Supplier; import org.vitrivr.cineast.core.config.CacheConfig; import org.vitrivr.cineast.core.config.IdConfig; import org.vitrivr.cineast.core.data.MediaType; +import org.vitrivr.cineast.core.db.DBSelector; import org.vitrivr.cineast.core.db.DBSelectorSupplier; +import org.vitrivr.cineast.core.db.PersistencyWriter; import org.vitrivr.cineast.core.db.PersistencyWriterSupplier; import org.vitrivr.cineast.core.extraction.idgenerator.ObjectIdGenerator; import org.vitrivr.cineast.core.extraction.metadata.MetadataExtractor; @@ -98,16 +101,16 @@ default Optional relPath() { IdConfig.ExistenceCheck existenceCheck(); /** - * Returns the PersistencyWriterSupplier that can be used during the extraction run to obtain PersistencyWriter instance. + * Returns the {@link PersistencyWriterSupplier} that can be used during the extraction run to obtain {@link PersistencyWriter} instance. * - * @return PersistencyWriterSupplier instance used obtain a PersistencyWriter. + * @return {@link PersistencyWriterSupplier} instance used obtain a {@link PersistencyWriter}. */ PersistencyWriterSupplier persistencyWriter(); /** - * Returns the DBSelectorSupplier that can be used during the extraction run to obtain a DBSelector instance. + * Returns the {@link DBSelectorSupplier} that can be used during the extraction run to obtain a {@link DBSelector} instance. * - * @return DBSelectorSupplier instance used obtain a DBSelector. + * @return {@link DBSelectorSupplier} instance used obtain a {@link DBSelector}. */ DBSelectorSupplier persistencyReader(); diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/AverageColorRaster.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/AverageColorRaster.java index d614be5fb..ff33f21fb 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/AverageColorRaster.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/AverageColorRaster.java @@ -25,6 +25,7 @@ import org.vitrivr.cineast.core.data.score.ScoreElement; import org.vitrivr.cineast.core.data.score.SegmentScoreElement; import org.vitrivr.cineast.core.data.segments.SegmentContainer; +import org.vitrivr.cineast.core.db.PersistencyWriter; import org.vitrivr.cineast.core.db.PersistencyWriterSupplier; import org.vitrivr.cineast.core.db.PersistentTuple; import org.vitrivr.cineast.core.db.setup.AttributeDefinition; @@ -43,8 +44,8 @@ public AverageColorRaster() { } @Override - public void init(PersistencyWriterSupplier supply, int batchSize) { - super.init(supply, batchSize); + public void init(PersistencyWriterSupplier supply) { + super.init(supply); this.phandler.setFieldNames(GENERIC_ID_COLUMN_QUALIFIER, "hist", "raster"); } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/AverageColorRasterReduced11.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/AverageColorRasterReduced11.java index 2bc6b43d0..ef6b2158a 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/AverageColorRasterReduced11.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/AverageColorRasterReduced11.java @@ -14,7 +14,7 @@ public class AverageColorRasterReduced11 extends AverageColorRaster { @Override - public void init(PersistencyWriterSupplier supply, int batchSize) { + public void init(PersistencyWriterSupplier supply) { /* TODO: Respect batchSize. */ this.phandler = supply.get(); this.phandler.open("features_AverageColorRasterReduced11"); diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/AverageColorRasterReduced15.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/AverageColorRasterReduced15.java index fdc90329b..3c69ae1c8 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/AverageColorRasterReduced15.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/AverageColorRasterReduced15.java @@ -14,7 +14,7 @@ public class AverageColorRasterReduced15 extends AverageColorRaster { @Override - public void init(PersistencyWriterSupplier supply, int batchSize) { + public void init(PersistencyWriterSupplier supply) { /* TODO: Respect batchSize. */ this.phandler = supply.get(); this.phandler.open("features_AverageColorRasterReduced15"); diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/DCTImageHash.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/DCTImageHash.java index 955afee3c..6384c3a08 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/DCTImageHash.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/DCTImageHash.java @@ -23,6 +23,7 @@ import org.vitrivr.cineast.core.data.raw.images.MultiImage; import org.vitrivr.cineast.core.data.score.ScoreElement; import org.vitrivr.cineast.core.data.segments.SegmentContainer; +import org.vitrivr.cineast.core.db.PersistencyWriter; import org.vitrivr.cineast.core.db.PersistencyWriterSupplier; import org.vitrivr.cineast.core.db.dao.writer.SimpleBitSetWriter; import org.vitrivr.cineast.core.db.setup.AttributeDefinition; @@ -124,9 +125,9 @@ public void processSegment(SegmentContainer shot) { } @Override - public void init(PersistencyWriterSupplier phandlerSupply, int batchSize) { + public void init(PersistencyWriterSupplier phandlerSupply) { this.phandler = phandlerSupply.get(); - this.writer = new SimpleBitSetWriter(this.phandler, batchSize, this.tableName); + this.writer = new SimpleBitSetWriter(this.phandler, this.tableName); } @Override diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/ForegroundBoundingBox.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/ForegroundBoundingBox.java index 707b10fc1..9590aa3be 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/ForegroundBoundingBox.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/ForegroundBoundingBox.java @@ -15,6 +15,7 @@ import org.vitrivr.cineast.core.data.frames.VideoFrame; import org.vitrivr.cineast.core.data.score.ScoreElement; import org.vitrivr.cineast.core.data.segments.SegmentContainer; +import org.vitrivr.cineast.core.db.PersistencyWriter; import org.vitrivr.cineast.core.db.PersistencyWriterSupplier; import org.vitrivr.cineast.core.db.PersistentTuple; import org.vitrivr.cineast.core.db.setup.AttributeDefinition; @@ -67,8 +68,8 @@ public void initalizePersistentLayer(Supplier supply) { } @Override - public void init(PersistencyWriterSupplier phandlerSupply, int batchSize) { - super.init(phandlerSupply, batchSize); + public void init(PersistencyWriterSupplier phandlerSupply) { + super.init(phandlerSupply); this.phandler.setFieldNames(GENERIC_ID_COLUMN_QUALIFIER, "frame", "bbox"); } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/MedianColorRaster.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/MedianColorRaster.java index f3387f706..db6fe367a 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/MedianColorRaster.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/MedianColorRaster.java @@ -13,7 +13,7 @@ public class MedianColorRaster extends AverageColorRaster { @Override - public void init(PersistencyWriterSupplier supply, int batchSize) { + public void init(PersistencyWriterSupplier supply) { /* TODO: Respect batchSize. */ this.phandler = supply.get(); this.phandler.open("features_MedianColorRaster"); diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/STMP7EH.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/STMP7EH.java index b6f78010e..35d2b0aa8 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/STMP7EH.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/STMP7EH.java @@ -19,7 +19,7 @@ public STMP7EH() { } @Override - public void init(PersistencyWriterSupplier supply, int batchSize) { + public void init(PersistencyWriterSupplier supply) { this.phandler = supply.get(); this.phandler.open("features_STMP7EH"); } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SegmentTags.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SegmentTags.java index 336fb05e6..53a4283a5 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SegmentTags.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SegmentTags.java @@ -187,9 +187,9 @@ public List getSimilar(String segmentId, ReadableQueryConfig qc) { } @Override - public void init(PersistencyWriterSupplier phandlerSupply, int batchSize) { + public void init(PersistencyWriterSupplier phandlerSupply) { this.phandler = phandlerSupply.get(); - this.writer = new BatchedTagWriter(this.phandler, SEGMENT_TAGS_TABLE_NAME, batchSize); + this.writer = new BatchedTagWriter(this.phandler, SEGMENT_TAGS_TABLE_NAME); } @Override diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/AbstractCodebookFeatureModule.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/AbstractCodebookFeatureModule.java index 39548e4da..ec564bd48 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/AbstractCodebookFeatureModule.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/AbstractCodebookFeatureModule.java @@ -7,10 +7,13 @@ import boofcv.struct.feature.TupleDesc_F64; import boofcv.struct.image.GrayF32; import java.util.List; +import java.util.function.Supplier; import org.ddogleg.clustering.AssignCluster; import org.vitrivr.cineast.core.config.QueryConfig; import org.vitrivr.cineast.core.config.ReadableQueryConfig; +import org.vitrivr.cineast.core.db.DBSelector; import org.vitrivr.cineast.core.db.DBSelectorSupplier; +import org.vitrivr.cineast.core.db.PersistencyWriter; import org.vitrivr.cineast.core.db.PersistencyWriterSupplier; /** @@ -42,15 +45,15 @@ protected AbstractCodebookFeatureModule(String tableName, float maxDist, int vec * Initializer for Extraction - must load the codebook. */ @Override - public final void init(PersistencyWriterSupplier phandlerSupply, int batchSize) { - super.init(phandlerSupply, batchSize); + public final void init(PersistencyWriterSupplier phandlerSupply) { + super.init(phandlerSupply); /* Load the Codebook. */ this.assignment = UtilIO.load(CODEBOOK_FOLDER + this.codebook()); } /** - * Initializer for Retrieval - must load the codebook.selectorSupply + * Initializer for Retrieval - must load the codebook. */ @Override public final void init(DBSelectorSupplier selectorSupply) { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/AbstractFeatureModule.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/AbstractFeatureModule.java index 7b12c5740..7938b310a 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/AbstractFeatureModule.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/AbstractFeatureModule.java @@ -61,10 +61,10 @@ public List getTableNames() { } @Override - public void init(PersistencyWriterSupplier phandlerSupply, int batchSize) { + public void init(PersistencyWriterSupplier phandlerSupply) { this.phandler = phandlerSupply.get(); - this.writer = new SimpleFeatureDescriptorWriter(this.phandler, this.tableName, batchSize); - this.primitiveWriter = new PrimitiveTypeProviderFeatureDescriptorWriter(this.phandler, this.tableName, batchSize); + this.writer = new SimpleFeatureDescriptorWriter(this.phandler, this.tableName); + this.primitiveWriter = new PrimitiveTypeProviderFeatureDescriptorWriter(this.phandler, this.tableName); } @Override diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/AbstractTextRetriever.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/AbstractTextRetriever.java index d91cefb7f..cb06aa87b 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/AbstractTextRetriever.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/AbstractTextRetriever.java @@ -26,6 +26,7 @@ import org.vitrivr.cineast.core.data.segments.SegmentContainer; import org.vitrivr.cineast.core.db.DBSelector; import org.vitrivr.cineast.core.db.DBSelectorSupplier; +import org.vitrivr.cineast.core.db.PersistencyWriter; import org.vitrivr.cineast.core.db.PersistencyWriterSupplier; import org.vitrivr.cineast.core.db.dao.writer.SimpleFulltextFeatureDescriptorWriter; import org.vitrivr.cineast.core.db.setup.AttributeDefinition; @@ -76,8 +77,8 @@ public void init(DBSelectorSupplier selectorSupply) { } @Override - public void init(PersistencyWriterSupplier phandlerSupply, int batchSize) { - this.writer = new SimpleFulltextFeatureDescriptorWriter(phandlerSupply.get(), this.tableName, batchSize); + public void init(PersistencyWriterSupplier phandlerSupply) { + this.writer = new SimpleFulltextFeatureDescriptorWriter(phandlerSupply.get(), this.tableName); } @Override diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/MetadataFeatureModule.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/MetadataFeatureModule.java index f32a80058..81657704a 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/MetadataFeatureModule.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/MetadataFeatureModule.java @@ -97,10 +97,10 @@ public void dropPersistentLayer(Supplier supply) { supply.get().dropEntity(this.featureEntityName()); } - public void init(PersistencyWriterSupplier supply, int batchSize) { + public void init(PersistencyWriterSupplier supply) { init(); //from MetadataFeatureExtractor PersistencyWriter writer = supply.get(); - this.featureWriter = new SimpleFeatureDescriptorWriter(writer, this.featureEntityName(), batchSize); + this.featureWriter = new SimpleFeatureDescriptorWriter(writer, this.featureEntityName()); this.featureWriter.init(); } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/SubDivMotionHistogram.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/SubDivMotionHistogram.java index 8901a5419..4809cccbd 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/SubDivMotionHistogram.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/SubDivMotionHistogram.java @@ -22,7 +22,7 @@ protected SubDivMotionHistogram(String tableName, String fieldName, double maxDi } @Override - public void init(PersistencyWriterSupplier supply, int batchSize) { + public void init(PersistencyWriterSupplier supply) { this.phandler = supply.get(); this.phandler.open(this.tableName); this.phandler.setFieldNames(GENERIC_ID_COLUMN_QUALIFIER, "sums", "hist"); diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/AudioSegmentExporter.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/AudioSegmentExporter.java index 24d76627c..9c08ed0dd 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/AudioSegmentExporter.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/AudioSegmentExporter.java @@ -125,7 +125,7 @@ private void writeWaveHeader(ByteBuffer buffer, float samplingrate, short channe @Override - public void init(PersistencyWriterSupplier phandlerSupply, int batchSize) { /* Nothing to init. */ } + public void init(PersistencyWriterSupplier phandlerSupply) { /* Nothing to init. */ } @Override public void finish() { /* Nothing to finish. */} diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/AudioSpectogramExporter.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/AudioSpectogramExporter.java index dbe2f0b45..7185576af 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/AudioSpectogramExporter.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/AudioSpectogramExporter.java @@ -111,7 +111,7 @@ public void processSegment(SegmentContainer shot) { } @Override - public void init(PersistencyWriterSupplier phandlerSupplier, int batchSize) { /* Noting to init. */} + public void init(PersistencyWriterSupplier phandlerSupplier) { /* Noting to init. */} @Override public void finish() { /* Nothing to finish. */} diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/AudioWaveformExporter.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/AudioWaveformExporter.java index beb386fcf..e8eccae2b 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/AudioWaveformExporter.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/AudioWaveformExporter.java @@ -150,7 +150,7 @@ public void processSegment(SegmentContainer shot) { } @Override - public void init(PersistencyWriterSupplier phandlerSupply, int batchSize) { /* Noting to init. */} + public void init(PersistencyWriterSupplier phandlerSupply) { /* Noting to init. */} @Override public void finish() { /* Nothing to finish. */} diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/CENSExporter.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/CENSExporter.java index 65396b04b..efaa25967 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/CENSExporter.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/CENSExporter.java @@ -103,7 +103,7 @@ public void processSegment(SegmentContainer shot) { } @Override - public void init(PersistencyWriterSupplier phandlerSupply, int batchSize) { /* Noting to init. */} + public void init(PersistencyWriterSupplier phandlerSupply) { /* Noting to init. */} @Override public void finish() { /* Nothing to finish. */} diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/ChromagramExporter.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/ChromagramExporter.java index 230992d0a..5f17c6bf6 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/ChromagramExporter.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/ChromagramExporter.java @@ -98,7 +98,7 @@ public void processSegment(SegmentContainer shot) { } @Override - public void init(PersistencyWriterSupplier phandlerSupply, int batchSize) { /* Noting to init. */} + public void init(PersistencyWriterSupplier phandlerSupply) { /* Noting to init. */} @Override public void finish() { /* Nothing to finish. */} diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/FrameExporter.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/FrameExporter.java index 9ced9f960..1a75fe433 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/FrameExporter.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/FrameExporter.java @@ -52,7 +52,7 @@ public FrameExporter(HashMap properties) { @Override - public void init(PersistencyWriterSupplier phandlerSupply, int batchSize) { + public void init(PersistencyWriterSupplier phandlerSupply) { if (!this.folder.exists()) { this.folder.mkdirs(); } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/Model3DThumbnailExporter.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/Model3DThumbnailExporter.java index 5ce93570f..d1821144a 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/Model3DThumbnailExporter.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/Model3DThumbnailExporter.java @@ -133,7 +133,7 @@ public void processSegment(SegmentContainer shot) { } @Override - public void init(PersistencyWriterSupplier phandlerSupply, int batchSize) { /* Noting to init. */} + public void init(PersistencyWriterSupplier phandlerSupply) { /* Noting to init. */} @Override public void finish() { /* Nothing to finish. */} diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/MotionFrameExporter.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/MotionFrameExporter.java index 5a517feb0..c2813af17 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/MotionFrameExporter.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/MotionFrameExporter.java @@ -49,7 +49,7 @@ public MotionFrameExporter(HashMap properties) { @Override - public void init(PersistencyWriterSupplier phandlerSupply, int batchSize) { + public void init(PersistencyWriterSupplier phandlerSupply) { if (!folder.exists()) { folder.mkdirs(); } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/MotionHistoryImageExporter.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/MotionHistoryImageExporter.java index 60606e85d..6776b520b 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/MotionHistoryImageExporter.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/MotionHistoryImageExporter.java @@ -49,7 +49,7 @@ public MotionHistoryImageExporter(HashMap properties) { } @Override - public void init(PersistencyWriterSupplier phandlerSupply, int batchSize) { + public void init(PersistencyWriterSupplier phandlerSupply) { if (!this.folder.exists()) { this.folder.mkdirs(); } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/RepresentativeFrameExporter.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/RepresentativeFrameExporter.java index 789d023b3..77f6693bf 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/RepresentativeFrameExporter.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/RepresentativeFrameExporter.java @@ -51,7 +51,7 @@ public RepresentativeFrameExporter(HashMap properties) { } @Override - public void init(PersistencyWriterSupplier supply, int batchSize) { + public void init(PersistencyWriterSupplier supply) { this.phandler = supply.get(); this.phandler.open("cineast_representativeframes"); this.phandler.setFieldNames(GENERIC_ID_COLUMN_QUALIFIER, "frame"); diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/ShotDescriptorExporter.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/ShotDescriptorExporter.java index 5a4bf7e50..0e8d32e18 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/ShotDescriptorExporter.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/ShotDescriptorExporter.java @@ -45,7 +45,7 @@ public ShotDescriptorExporter(HashMap properties) { } @Override - public void init(PersistencyWriterSupplier supply, int batchSize) { + public void init(PersistencyWriterSupplier supply) { if (!folder.exists()) { folder.mkdirs(); } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/ShotThumbnailsExporter.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/ShotThumbnailsExporter.java index e8b0340da..c6e73a73e 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/ShotThumbnailsExporter.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/ShotThumbnailsExporter.java @@ -57,7 +57,7 @@ public ShotThumbnailsExporter(HashMap properties) { } @Override - public void init(PersistencyWriterSupplier supply, int batchSize) { + public void init(PersistencyWriterSupplier supply) { if (!this.folder.exists()) { this.folder.mkdirs(); } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/extractor/Extractor.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/extractor/Extractor.java index 7c2c76eaf..18af2ee30 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/extractor/Extractor.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/extractor/Extractor.java @@ -6,7 +6,7 @@ public interface Extractor extends PersistentOperator { - void init(PersistencyWriterSupplier phandlerSupply, int batchSize); + void init(PersistencyWriterSupplier phandlerSupply); void processSegment(SegmentContainer shot); diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/retriever/Retriever.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/retriever/Retriever.java index 9cc55ebcf..998057b19 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/retriever/Retriever.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/retriever/Retriever.java @@ -2,9 +2,11 @@ import java.util.ArrayList; import java.util.List; +import java.util.function.Supplier; import org.vitrivr.cineast.core.config.ReadableQueryConfig; import org.vitrivr.cineast.core.data.score.ScoreElement; import org.vitrivr.cineast.core.data.segments.SegmentContainer; +import org.vitrivr.cineast.core.db.DBSelector; import org.vitrivr.cineast.core.db.DBSelectorSupplier; import org.vitrivr.cineast.core.db.PersistentOperator; @@ -24,7 +26,5 @@ default List getSimilar(List segmentIds, ReadableQueryConf return _return; } - ; - void finish(); } diff --git a/cineast-core/src/test/java/org/vitrivr/cineast/core/db/DBBooleanIntegrationTest.java b/cineast-core/src/test/java/org/vitrivr/cineast/core/db/DBBooleanIntegrationTest.java index bfe2a2f65..c01df330f 100644 --- a/cineast-core/src/test/java/org/vitrivr/cineast/core/db/DBBooleanIntegrationTest.java +++ b/cineast-core/src/test/java/org/vitrivr/cineast/core/db/DBBooleanIntegrationTest.java @@ -14,10 +14,13 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.hamcrest.MatcherAssert; +import org.junit.Ignore; +import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; @@ -48,21 +51,21 @@ @TestInstance(Lifecycle.PER_CLASS) public abstract class DBBooleanIntegrationTest { - private static final int TABLE_CARD = 20; - private DBSelector selector; - private String testTableName; - private PersistencyWriter writer; - private EntityCreator ec; - private QueryConfig queryConfig; + protected static final int TABLE_CARD = 20; + protected DBSelector selector; + protected String testTableName; + protected PersistencyWriter writer; + protected EntityCreator ec; + protected QueryConfig queryConfig; - private static final String ID_COL_NAME = "id"; - private static final String DATA_COL_NAME_1 = "data_1_string_data_id"; - private static final String DATA_COL_NAME_2 = "data_2_id_negative"; - private static final String DATA_COL_NAME_3 = "data_3_id_plus_card"; + protected static final String ID_COL_NAME = "id"; + protected static final String DATA_COL_NAME_1 = "data_1_string_data_id"; + protected static final String DATA_COL_NAME_2 = "data_2_id_negative"; + protected static final String DATA_COL_NAME_3 = "data_3_id_plus_card"; - private IntegrationDBProvider provider; + protected IntegrationDBProvider provider; - private static final Logger LOGGER = LogManager.getLogger(); + protected static final Logger LOGGER = LogManager.getLogger(); @BeforeAll void checkConnection() { @@ -71,17 +74,14 @@ void checkConnection() { LOGGER.info("Trying to establish connection to Database"); assumeTrue(selector.ping(), "Connection to database could not be established"); LOGGER.info("Connection to Database established"); - } - - @BeforeEach - void setupTest() { - provider = provider(); - selector = provider.getSelector(); - assumeTrue(selector.ping(), "Connection to database could not be established"); writer = provider.getPersistencyWriter(); ec = provider.getEntityCreator(); testTableName = getTestTableName(); queryConfig = getQueryConfig(); + } + + @BeforeEach + void setupTest() { dropTables(); createTables(); fillData(); @@ -90,9 +90,12 @@ void setupTest() { @AfterEach void tearDownTest() { - CineastIOUtils.closeQuietly(writer, selector); dropTables(); - CineastIOUtils.closeQuietly(ec); + } + + @AfterAll + void tearDownAll() { + CineastIOUtils.closeQuietly(this.provider); } protected String getTestTableName() { @@ -104,8 +107,7 @@ protected String getTestTableName() { protected abstract IntegrationDBProvider provider(); protected QueryConfig getQueryConfig() { - QueryConfig qc = new QueryConfig("test-" + RandomStringUtils.randomNumeric(4), new ArrayList<>()); - return qc; + return new QueryConfig("test-" + RandomStringUtils.randomNumeric(4), new ArrayList<>()); } protected void dropTables() { @@ -122,7 +124,6 @@ protected void dropTables() { * Create table for boolean retrieval */ protected void createTables() { - // TODO Text retrieval fails on windows ec.createEntity(testTableName, new AttributeDefinition(ID_COL_NAME, AttributeType.INT), new AttributeDefinition(DATA_COL_NAME_1, AttributeType.TEXT), new AttributeDefinition(DATA_COL_NAME_2, AttributeType.INT), new AttributeDefinition(DATA_COL_NAME_3, AttributeType.INT)); } @@ -130,46 +131,35 @@ protected void fillData() { writer.open(testTableName); writer.setFieldNames(ID_COL_NAME, DATA_COL_NAME_1, DATA_COL_NAME_2, DATA_COL_NAME_3); List vectors = new ArrayList<>(); - for (int i = 0; i < TABLE_CARD; i++) { vectors.add(writer.generateTuple(i, "string-data-" + i, -i, i + TABLE_CARD)); - //vectors.add(writer.generateTuple(i, i, -i, i + TABLE_CARD)); } writer.persist(vectors); } @Test @DisplayName("Ping the database") - void ping() { + public void ping() { Assertions.assertTrue(provider.getSelector().ping()); } @Test @DisplayName("Verify entity creation") - void entitiesExist() { + public void entitiesExist() { Assertions.assertTrue(selector.existsEntity(testTableName)); } @Test @DisplayName("Verify element count") - void count() { + public void count() { selector.open(testTableName); Assertions.assertEquals(TABLE_CARD, selector.getAll().size()); Assertions.assertEquals(TABLE_CARD, selector.getAll(DATA_COL_NAME_1).size()); } - @Test - @DisplayName("Verify elements exist by id") - void entriesExistById() { - writer.open(testTableName); - for (int i = 0; i < TABLE_CARD; i++) { - Assertions.assertTrue(writer.idExists(String.valueOf(i))); - } - } - @Test @DisplayName("Verify unique value count") - void uniqueValueCountCorrect() { + public void uniqueValueCountCorrect() { selector.open(testTableName); Assertions.assertEquals(TABLE_CARD, selector.getUniqueValues(ID_COL_NAME).size()); Assertions.assertEquals(TABLE_CARD, selector.getUniqueValues(DATA_COL_NAME_1).size()); @@ -179,11 +169,10 @@ void uniqueValueCountCorrect() { @Test @DisplayName("test fulltext query") - void testFulltextQuery() { - selector.open(testTableName); - // test latest entry + public void testFulltextQuery() { + this.selector.open(testTableName); int idToCheck = TABLE_CARD - 1; - List> result = selector.getFulltextRows(1, DATA_COL_NAME_1, queryConfig, "string-data-" + idToCheck); + final List> result = selector.getFulltextRows(1, DATA_COL_NAME_1, queryConfig, "string-data-" + idToCheck); Assertions.assertEquals(result.get(0).get(DATA_COL_NAME_1).getString(), "string-data-" + idToCheck); Assertions.assertEquals(result.get(0).get(DATA_COL_NAME_2).getInt(), -idToCheck); Assertions.assertEquals(result.get(0).get(DATA_COL_NAME_3).getInt(), (idToCheck + TABLE_CARD)); @@ -191,7 +180,7 @@ void testFulltextQuery() { @Test @DisplayName("test IN() query") - void testInQuery() { + public void testInQuery() { selector.open(testTableName); // test latest entry int idToCheck = TABLE_CARD - 1; @@ -206,64 +195,60 @@ void testInQuery() { @Test @DisplayName("test BETWEEN() query") - void testBetweenQuery() { - selector.open(testTableName); + public void testBetweenQuery() { + this.selector.open(testTableName); int idToCheck = TABLE_CARD - 1; - List values = new ArrayList<>(); + final List values = new ArrayList<>(); values.add(PrimitiveTypeProvider.fromObject(-idToCheck)); values.add(PrimitiveTypeProvider.fromObject(-(idToCheck - 1))); - List> result = selector.getRows(DATA_COL_NAME_2, RelationalOperator.BETWEEN, values); + final List> result = selector.getRows(DATA_COL_NAME_2, RelationalOperator.BETWEEN, values); MatcherAssert.assertThat(result.stream().map(el -> el.get(ID_COL_NAME).getString()).collect(Collectors.toList()), hasItem(String.valueOf(idToCheck))); MatcherAssert.assertThat(result.stream().map(el -> el.get(ID_COL_NAME).getString()).collect(Collectors.toList()), hasItem(String.valueOf(idToCheck - 1))); } @Test @DisplayName("test Greater() query") - void testGreaterQuery() { - selector.open(testTableName); - // query for greater than second-highest value - int idToCheck = TABLE_CARD - 2; - // cardinality should be higher than 2 - if (idToCheck < -1) { - Assertions.fail(); - } - List values = new ArrayList<>(); + public void testGreaterQuery() { + this.selector.open(testTableName); + int idToCheck = TABLE_CARD - 2; // query for greater than second-highest value + if (idToCheck < -1) Assertions.fail(); // cardinality should be higher than 2 + final List values = new ArrayList<>(); values.add(PrimitiveTypeProvider.fromObject(idToCheck)); - List> result = selector.getRows(ID_COL_NAME, RelationalOperator.GREATER, values); + final List> result = selector.getRows(ID_COL_NAME, RelationalOperator.GREATER, values); MatcherAssert.assertThat(result.stream().map(el -> el.get(ID_COL_NAME).getString()).collect(Collectors.toList()), hasItem(String.valueOf(TABLE_CARD - 1))); } @Test @DisplayName("test Less() query") - void testLessQuery() { + public void testLessQuery() { selector.open(testTableName); int idToCheck = 1; - List values = new ArrayList<>(); + final List values = new ArrayList<>(); values.add(PrimitiveTypeProvider.fromObject(idToCheck)); - List> result = selector.getRows(ID_COL_NAME, RelationalOperator.LESS, values); + final List> result = selector.getRows(ID_COL_NAME, RelationalOperator.LESS, values); MatcherAssert.assertThat(result.stream().map(el -> el.get(ID_COL_NAME).getString()).collect(Collectors.toList()), hasItem(String.valueOf(0))); } @Test @DisplayName("test BETWEEN() AND BETWEEN() query") - void testBetweenANDBetweenQuery() { + public void testBetweenANDBetweenQuery() { selector.open(testTableName); int idToCheck = TABLE_CARD / 2; //The result range is between x and x+1 - List values1 = new ArrayList<>(); + final List values1 = new ArrayList<>(); values1.add(PrimitiveTypeProvider.fromObject(idToCheck)); values1.add(PrimitiveTypeProvider.fromObject(idToCheck + 1)); - Triple> element1 = new ImmutableTriple<>(ID_COL_NAME, RelationalOperator.BETWEEN, values1); + final Triple> element1 = new ImmutableTriple<>(ID_COL_NAME, RelationalOperator.BETWEEN, values1); //The result range is between x+1 and x+2 - List values2 = new ArrayList<>(); + final List values2 = new ArrayList<>(); values2.add(PrimitiveTypeProvider.fromObject(idToCheck + 1)); values2.add(PrimitiveTypeProvider.fromObject(idToCheck + 2)); - Triple> element2 = new ImmutableTriple<>(ID_COL_NAME, RelationalOperator.BETWEEN, values2); + final Triple> element2 = new ImmutableTriple<>(ID_COL_NAME, RelationalOperator.BETWEEN, values2); - List> result = selector.getRowsAND(asList(element1, element2), ID_COL_NAME, asList(ID_COL_NAME), null); + final List> result = selector.getRowsAND(asList(element1, element2), ID_COL_NAME, asList(ID_COL_NAME), null); MatcherAssert.assertThat(result.stream().map(el -> el.get(ID_COL_NAME).getString()).collect(Collectors.toList()), hasItem(String.valueOf(idToCheck + 1))); Assertions.assertEquals(1, result.size()); @@ -272,23 +257,23 @@ void testBetweenANDBetweenQuery() { @Test @DisplayName("test BETWEEN() AND IN() query") - void testBetweenANDInQuery() { + public void testBetweenANDInQuery() { selector.open(testTableName); int idToCheck = TABLE_CARD / 2; //The result is between x-2 and x+2 - List values1 = new ArrayList<>(); - values1.add(PrimitiveTypeProvider.fromObject(idToCheck + 2)); + final List values1 = new ArrayList<>(); values1.add(PrimitiveTypeProvider.fromObject(idToCheck - 2)); - Triple> element1 = new ImmutableTriple<>(ID_COL_NAME, RelationalOperator.BETWEEN, values1); + values1.add(PrimitiveTypeProvider.fromObject(idToCheck + 2)); + final Triple> element1 = new ImmutableTriple<>(ID_COL_NAME, RelationalOperator.BETWEEN, values1); //The result range is either x+1 or x-1 - List values2 = new ArrayList<>(); + final List values2 = new ArrayList<>(); values2.add(PrimitiveTypeProvider.fromObject(idToCheck + 1)); values2.add(PrimitiveTypeProvider.fromObject(idToCheck - 1)); - Triple> element2 = new ImmutableTriple<>(ID_COL_NAME, RelationalOperator.IN, values2); + final Triple> element2 = new ImmutableTriple<>(ID_COL_NAME, RelationalOperator.IN, values2); - List> result = selector.getRowsAND(asList(element1, element2), ID_COL_NAME, asList(ID_COL_NAME), null); + final List> result = selector.getRowsAND(asList(element1, element2), ID_COL_NAME, asList(ID_COL_NAME), null); MatcherAssert.assertThat(result.stream().map(el -> el.get(ID_COL_NAME).getString()).collect(Collectors.toList()), hasItem(String.valueOf(idToCheck - 1))); MatcherAssert.assertThat(result.stream().map(el -> el.get(ID_COL_NAME).getString()).collect(Collectors.toList()), hasItem(String.valueOf(idToCheck + 1))); diff --git a/cineast-core/src/test/java/org/vitrivr/cineast/core/db/DBIntegrationTest.java b/cineast-core/src/test/java/org/vitrivr/cineast/core/db/DBIntegrationTest.java index 769f269a3..50ca3a469 100644 --- a/cineast-core/src/test/java/org/vitrivr/cineast/core/db/DBIntegrationTest.java +++ b/cineast-core/src/test/java/org/vitrivr/cineast/core/db/DBIntegrationTest.java @@ -6,6 +6,7 @@ import com.google.gson.Gson; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; @@ -14,6 +15,7 @@ import org.apache.commons.lang3.RandomStringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; @@ -27,8 +29,11 @@ import org.vitrivr.cineast.core.config.ReadableQueryConfig; import org.vitrivr.cineast.core.config.ReadableQueryConfig.Distance; import org.vitrivr.cineast.core.data.distance.SegmentDistanceElement; +import org.vitrivr.cineast.core.data.providers.primitive.IntTypeProvider; +import org.vitrivr.cineast.core.data.providers.primitive.LongTypeProvider; import org.vitrivr.cineast.core.data.providers.primitive.PrimitiveTypeProvider; import org.vitrivr.cineast.core.data.providers.primitive.StringTypeProvider; +import org.vitrivr.cineast.core.db.polypheny.PolyphenyWrapper; import org.vitrivr.cineast.core.db.setup.AttributeDefinition; import org.vitrivr.cineast.core.db.setup.AttributeDefinition.AttributeType; import org.vitrivr.cineast.core.db.setup.EntityCreator; @@ -42,23 +47,23 @@ @TestInstance(Lifecycle.PER_CLASS) public abstract class DBIntegrationTest { - private DBSelector selector; - private String testTextTableName; - private String testVectorTableName; - private PersistencyWriter writer; - private EntityCreator ec; - private QueryConfig queryConfig; - private static final String ID_COL_NAME = "id"; - private static final int VECTOR_ELEMENT_COUNT = 11; - private static final int MAX_VECTOR_ID = 10; - private static final int TEXT_ELEMENT_COUNT = 8; - private static final int MAX_TEXT_ID = 7; + protected DBSelector selector; + protected String testTextTableName; + protected String testVectorTableName; + protected PersistencyWriter writer; + protected EntityCreator ec; + protected QueryConfig queryConfig; + protected static final String ID_COL_NAME = "id"; + protected static final int VECTOR_ELEMENT_COUNT = 11; + protected static final int MAX_VECTOR_ID = 10; + protected static final int TEXT_ELEMENT_COUNT = 8; + protected static final int MAX_TEXT_ID = 7; /** * This is not called "feature" by design as it avoid the storage-layers doing optimization by col name */ - private static final String FEATURE_VECTOR_COL_NAME = "vector"; - private static final String TEXT_COL_NAME = "text"; - private static final Logger LOGGER = LogManager.getLogger(); + protected static final String FEATURE_VECTOR_COL_NAME = "vector"; + protected static final String TEXT_COL_NAME = "text"; + protected static final Logger LOGGER = LogManager.getLogger(); private IntegrationDBProvider provider; @@ -68,19 +73,16 @@ void checkConnection() { selector = provider.getSelector(); LOGGER.info("Trying to establish connection to Database"); assumeTrue(selector.ping(), "Connection to database could not be established"); - LOGGER.info("Connection to Database established"); - } - - @BeforeEach - void setupTest() { - provider = provider(); - selector = provider.getSelector(); - assumeTrue(selector.ping(), "Connection to database could not be established"); + LOGGER.info("Connection to database established"); writer = provider.getPersistencyWriter(); ec = provider.getEntityCreator(); testTextTableName = getTestTextTableName(); testVectorTableName = getTestVectorTableName(); queryConfig = getQueryConfig(); + } + + @BeforeEach + void setupTest() { dropTables(); createTables(); fillVectorData(); @@ -88,6 +90,16 @@ void setupTest() { finishSetup(); } + @AfterEach + void tearDownTest() { + dropTables(); + } + + @AfterAll + void tearDownAll() { + CineastIOUtils.closeQuietly(this.provider); + } + protected abstract void finishSetup(); protected QueryConfig getQueryConfig() { @@ -98,13 +110,6 @@ protected QueryConfig getQueryConfig() { protected abstract IntegrationDBProvider provider(); - @AfterEach - void tearDownTest() { - CineastIOUtils.closeQuietly(writer, selector); - dropTables(); - CineastIOUtils.closeQuietly(ec); - } - private static final int HELLO_WORLD_ID = MAX_TEXT_ID - 7; private static final int HELLA_WORLD_ID = MAX_TEXT_ID - 6; private static final int SINGLE_ID = MAX_TEXT_ID - 5; @@ -118,14 +123,14 @@ protected void fillTextData() { writer.open(testTextTableName); writer.setFieldNames("id", "text"); List vectors = new ArrayList<>(); - vectors.add(writer.generateTuple(HELLO_WORLD_ID, "hello world")); - vectors.add(writer.generateTuple(HELLA_WORLD_ID, "hella world")); - vectors.add(writer.generateTuple(SINGLE_ID, "single")); - vectors.add(writer.generateTuple(DOUBLE_ID, "double")); - vectors.add(writer.generateTuple(HELLO_ID, "hello")); - vectors.add(writer.generateTuple(DUPLICATE_ID, "duplicate")); - vectors.add(writer.generateTuple(WORLD_ID, "world")); - vectors.add(writer.generateTuple(HELLO_WORLD_MY_NAME_IS_CINEAST_ID, "hello world my name is cineast")); + vectors.add(writer.generateTuple(String.valueOf(HELLO_WORLD_ID), "hello world")); + vectors.add(writer.generateTuple(String.valueOf(HELLA_WORLD_ID), "hella world")); + vectors.add(writer.generateTuple(String.valueOf(SINGLE_ID), "single")); + vectors.add(writer.generateTuple(String.valueOf(DOUBLE_ID), "double")); + vectors.add(writer.generateTuple(String.valueOf(HELLO_ID), "hello")); + vectors.add(writer.generateTuple(String.valueOf(DUPLICATE_ID), "duplicate")); + vectors.add(writer.generateTuple(String.valueOf(WORLD_ID), "world")); + vectors.add(writer.generateTuple(String.valueOf(HELLO_WORLD_MY_NAME_IS_CINEAST_ID), "hello world my name is cineast")); writer.persist(vectors); } @@ -142,10 +147,10 @@ protected void fillVectorData() { vector[0] = i; vector[1] = 1; vector[2] = 0; - vectors.add(writer.generateTuple(i, vector)); + vectors.add(writer.generateTuple(String.valueOf(i), vector)); } - /** We write a second vector with the same id in the db */ - vectors.add(writer.generateTuple(0, new float[]{0, 0, 0})); + /* We write a second vector with the same id in the db */ + vectors.add(writer.generateTuple(String.valueOf(0), new float[]{0, 0, 0})); writer.persist(vectors); } @@ -153,8 +158,8 @@ protected void fillVectorData() { * Create both a table for vector retrieval & text retrieval */ protected void createTables() { - ec.createEntity(testTextTableName, new AttributeDefinition(ID_COL_NAME, AttributeType.INT), new AttributeDefinition(TEXT_COL_NAME, AttributeType.TEXT)); - ec.createEntity(testVectorTableName, new AttributeDefinition(ID_COL_NAME, AttributeType.LONG), new AttributeDefinition(FEATURE_VECTOR_COL_NAME, AttributeType.VECTOR, 3)); + this.ec.createEntity(testTextTableName, new AttributeDefinition(ID_COL_NAME, AttributeType.STRING), new AttributeDefinition(TEXT_COL_NAME, AttributeType.TEXT)); + this.ec.createEntity(testVectorTableName, new AttributeDefinition(ID_COL_NAME, AttributeType.STRING), new AttributeDefinition(FEATURE_VECTOR_COL_NAME, AttributeType.VECTOR, 3)); } protected void dropTables() { @@ -205,11 +210,11 @@ void count() { @Test @DisplayName("Verify elements exist by id") void entriesExistById() { - writer.open(testVectorTableName); + this.writer.open(testVectorTableName); for (int i = 0; i < MAX_VECTOR_ID; i++) { Assertions.assertTrue(writer.idExists(String.valueOf(i))); } - writer.open(testTextTableName); + this.writer.open(testTextTableName); for (int i = 0; i < MAX_TEXT_ID; i++) { Assertions.assertTrue(writer.idExists(String.valueOf(i))); } @@ -219,8 +224,8 @@ void entriesExistById() { @Test @DisplayName("get multiple feature vectors") void getFeatureVectors() { - selector.open(testVectorTableName); - List vectors = selector.getFeatureVectorsGeneric(ID_COL_NAME, new StringTypeProvider("0"), FEATURE_VECTOR_COL_NAME); + this.selector.open(testVectorTableName); + final List vectors = this.selector.getFeatureVectorsGeneric(ID_COL_NAME, new StringTypeProvider("0"), FEATURE_VECTOR_COL_NAME); Assertions.assertTrue((Arrays.equals(PrimitiveTypeProvider.getSafeFloatArray(vectors.get(0)), new float[]{0, 0, 0}) | Arrays.equals(PrimitiveTypeProvider.getSafeFloatArray(vectors.get(0)), new float[]{0, 1, 0}))); Assertions.assertTrue((Arrays.equals(PrimitiveTypeProvider.getSafeFloatArray(vectors.get(1)), new float[]{0, 0, 0}) | Arrays.equals(PrimitiveTypeProvider.getSafeFloatArray(vectors.get(1)), new float[]{0, 1, 0}))); } @@ -236,8 +241,11 @@ void knnSearch() { Assertions.assertTrue(result.get(1).getSegmentId().equals("0") || result.get(2).getSegmentId().equals("0")); } + /** + * TODO: Currently not supported in Cottontail DB v0.12.0. Re-activate, once support is back. + */ @Test - @Disabled /* TODO: Currently not supported in Cottontail DB v0.12.0. Re-activate, once support is back. */ + @Disabled @DisplayName("Batched KNN search") void batchedKnnSearch() { selector.open(testVectorTableName); @@ -285,13 +293,13 @@ private void checkContains(List> results, Str */ @Test @DisplayName("Text: One el, no quotes") - void textRetrievalSingleLike() { + public void textRetrievalSingleLike() { selector.open(testTextTableName); List> results = selector.getFulltextRows(10, TEXT_COL_NAME, queryConfig, "hello"); Assertions.assertEquals(3, results.size()); - checkContains(results, ID_COL_NAME, val -> val.getInt() == HELLO_WORLD_ID); - checkContains(results, ID_COL_NAME, val -> val.getInt() == HELLO_ID); - checkContains(results, ID_COL_NAME, val -> val.getInt() == HELLO_WORLD_MY_NAME_IS_CINEAST_ID); + checkContains(results, ID_COL_NAME, val -> val.getString().equals(String.valueOf(HELLO_WORLD_ID))); + checkContains(results, ID_COL_NAME, val -> val.getString().equals(String.valueOf(HELLO_ID))); + checkContains(results, ID_COL_NAME, val -> val.getString().equals(String.valueOf(HELLO_WORLD_MY_NAME_IS_CINEAST_ID))); } /** @@ -299,11 +307,11 @@ void textRetrievalSingleLike() { */ @Test @DisplayName("Text: two words, inverted, no quotes") - void textRetrievalSingleTwoWordsLike() { + public void textRetrievalSingleTwoWordsLike() { selector.open(testTextTableName); List> results = selector.getFulltextRows(10, TEXT_COL_NAME, queryConfig, "name my"); Assertions.assertEquals(1, results.size()); - checkContains(results, ID_COL_NAME, val -> val.getInt() == HELLO_WORLD_MY_NAME_IS_CINEAST_ID); + checkContains(results, ID_COL_NAME, val -> val.getString().equals(String.valueOf(HELLO_WORLD_MY_NAME_IS_CINEAST_ID))); } @@ -312,12 +320,12 @@ void textRetrievalSingleTwoWordsLike() { */ @Test @DisplayName("Text: One el (two words), quotes") - void textRetrievalSingleTwoWordsQuotedLike() { + public void textRetrievalSingleTwoWordsQuotedLike() { selector.open(testTextTableName); List> results = selector.getFulltextRows(10, TEXT_COL_NAME, queryConfig, "\"hello world\""); Assertions.assertEquals(2, results.size()); - checkContains(results, ID_COL_NAME, val -> val.getInt() == HELLO_WORLD_ID); - checkContains(results, ID_COL_NAME, val -> val.getInt() == HELLO_WORLD_MY_NAME_IS_CINEAST_ID); + checkContains(results, ID_COL_NAME, val -> val.getString().equals(String.valueOf(HELLO_WORLD_ID))); + checkContains(results, ID_COL_NAME, val -> val.getString().equals(String.valueOf(HELLO_WORLD_MY_NAME_IS_CINEAST_ID))); } /** @@ -325,50 +333,50 @@ void textRetrievalSingleTwoWordsQuotedLike() { */ @Test @DisplayName("Text: One el, one word, Fuzzy") - void testRetrievalSingleFuzzy() { + public void testRetrievalSingleFuzzy() { selector.open(testTextTableName); List> results = selector.getFulltextRows(10, TEXT_COL_NAME, queryConfig, "hello~1"); Assertions.assertEquals(4, results.size()); - checkContains(results, ID_COL_NAME, val -> val.getInt() == HELLO_WORLD_ID); - checkContains(results, ID_COL_NAME, val -> val.getInt() == HELLA_WORLD_ID); - checkContains(results, ID_COL_NAME, val -> val.getInt() == HELLO_ID); - checkContains(results, ID_COL_NAME, val -> val.getInt() == HELLO_WORLD_MY_NAME_IS_CINEAST_ID); + checkContains(results, ID_COL_NAME, val -> val.getString().equals(String.valueOf(HELLO_WORLD_ID))); + checkContains(results, ID_COL_NAME, val -> val.getString().equals(String.valueOf(HELLO_ID))); + checkContains(results, ID_COL_NAME, val -> val.getString().equals(String.valueOf(HELLO_WORLD_MY_NAME_IS_CINEAST_ID))); } + /** + * TODO: Fuzzy search on whole phrases is currently not supported. + * + * Something like "hello world"~1 would need to be implemented as either hello~1 AND world~1, but that is not implemented in the DBSelector / cottontail. + * The cottontail implementation in december 19 parses hello world~1 as hello .. world~1, which is not what we're looking for + * Therefore, this test serves as a note that this functionality is lacking. + */ @Test @DisplayName("Text: One el (Two words), Fuzzy") + @Disabled void testRetrievalSingleTwoWordsFuzzy() { - /* - * Fuzzy search on whole phrases is currently not supported. - * Something like "hello world"~1 would need to be implemented as either hello~1 AND world~1, but that is not implemented in the DBSelector / cottontail. - * The cottontail implementation in december 19 parses hello world~1 as hello .. world~1, which is not what we're looking for - * Therefore, this test serves as a note that this functionality is lacking. - */ - return; + Assertions.fail(); } - /** * Verify that searching for two terms retrieves both individual results */ @Test @DisplayName("Text: Two elements w/ single word") - void testRetrievalTwo() { + public void testRetrievalTwo() { selector.open(testTextTableName); List> results = selector.getFulltextRows(10, TEXT_COL_NAME, queryConfig, "single", "double"); Assertions.assertEquals(2, results.size()); - checkContains(results, ID_COL_NAME, val -> val.getInt() == SINGLE_ID); - checkContains(results, ID_COL_NAME, val -> val.getInt() == DOUBLE_ID); + checkContains(results, ID_COL_NAME, val -> val.getString().equals(String.valueOf(SINGLE_ID))); + checkContains(results, ID_COL_NAME, val -> val.getString().equals(String.valueOf(DOUBLE_ID))); } @Test @DisplayName("Text: Three elements, two are a match for the same id") - void testRetrievalThreeDouble() { + public void testRetrievalThreeDouble() { selector.open(testTextTableName); List> results = selector.getFulltextRows(10, TEXT_COL_NAME, queryConfig, "double", "single", "duplicate"); Assertions.assertEquals(3, results.size()); - checkContains(results, ID_COL_NAME, val -> val.getInt() == DOUBLE_ID); - checkContains(results, ID_COL_NAME, val -> val.getInt() == SINGLE_ID); + checkContains(results, ID_COL_NAME, val -> val.getString().equals(String.valueOf(DOUBLE_ID))); + checkContains(results, ID_COL_NAME, val -> val.getString().equals(String.valueOf(SINGLE_ID))); float score = results.get(0).get(DB_DISTANCE_VALUE_QUALIFIER).getFloat(); Assertions.assertEquals(score, results.get(1).get(DB_DISTANCE_VALUE_QUALIFIER).getFloat(), 0.01); Assertions.assertEquals(score, results.get(2).get(DB_DISTANCE_VALUE_QUALIFIER).getFloat(), 0.01); @@ -376,14 +384,14 @@ void testRetrievalThreeDouble() { @Test @DisplayName("Text: Three els, one of those with quotes") - void testRetrievalThree() { + public void testRetrievalThree() { selector.open(testTextTableName); List> results = selector.getFulltextRows(10, TEXT_COL_NAME, queryConfig, "single", "double", "\"hello world\""); Assertions.assertEquals(4, results.size()); - checkContains(results, ID_COL_NAME, val -> val.getInt() == SINGLE_ID); - checkContains(results, ID_COL_NAME, val -> val.getInt() == DOUBLE_ID); - checkContains(results, ID_COL_NAME, val -> val.getInt() == HELLO_WORLD_ID); - checkContains(results, ID_COL_NAME, val -> val.getInt() == HELLO_WORLD_MY_NAME_IS_CINEAST_ID); + checkContains(results, ID_COL_NAME, val -> val.getString().equals(String.valueOf(SINGLE_ID))); + checkContains(results, ID_COL_NAME, val -> val.getString().equals(String.valueOf(DOUBLE_ID))); + checkContains(results, ID_COL_NAME, val -> val.getString().equals(String.valueOf(HELLO_WORLD_ID))); + checkContains(results, ID_COL_NAME, val -> val.getString().equals(String.valueOf(HELLO_WORLD_MY_NAME_IS_CINEAST_ID))); } } diff --git a/cineast-core/src/test/java/org/vitrivr/cineast/core/db/IntegrationDBProvider.java b/cineast-core/src/test/java/org/vitrivr/cineast/core/db/IntegrationDBProvider.java index d816f515c..47d2393e1 100644 --- a/cineast-core/src/test/java/org/vitrivr/cineast/core/db/IntegrationDBProvider.java +++ b/cineast-core/src/test/java/org/vitrivr/cineast/core/db/IntegrationDBProvider.java @@ -1,8 +1,9 @@ package org.vitrivr.cineast.core.db; +import java.io.Closeable; import org.vitrivr.cineast.core.db.setup.EntityCreator; -public interface IntegrationDBProvider { +public interface IntegrationDBProvider extends Closeable { /** diff --git a/cineast-core/src/test/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailBooleanIntegrationTest.java b/cineast-core/src/test/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailBooleanIntegrationTest.java index 9a9aaffe6..c24d99890 100644 --- a/cineast-core/src/test/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailBooleanIntegrationTest.java +++ b/cineast-core/src/test/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailBooleanIntegrationTest.java @@ -9,7 +9,12 @@ public class CottontailBooleanIntegrationTest extends DBBooleanIntegrationTest { + private static final Supplier WRAPPER_CONFIG_PROVIDER = () -> { + DatabaseConfig config = new DatabaseConfig(); + config.setHost("localhost"); + config.setPort(1865); + return config; + }; - private final DatabaseConfig config; - + /** The {@link CottontailWrapper} used to establish a database connection. */ private final CottontailWrapper wrapper; + /** The {@link DatabaseConfig} to use to run this test.*/ + private final DatabaseConfig config = WRAPPER_CONFIG_PROVIDER.get(); + + /** + * Constructor. + */ public CottontailIntegrationDBProvider() { - config = new DatabaseConfig(); - config.setPort(1865); - wrapper = new CottontailWrapper(config, true); + this.wrapper = new CottontailWrapper(this.config.getHost(), this.config.getPort()); } - CottontailWrapper getWrapper() { - return wrapper; + return this.wrapper; } @Override public PersistencyWriter getPersistencyWriter() { - return new CottontailWriter(getWrapper()); + return new CottontailWriter(getWrapper(), this.config.getBatchsize()); } @Override @@ -40,5 +49,8 @@ public EntityCreator getEntityCreator() { return new CottontailEntityCreator(getWrapper()); } - + @Override + public void close() { + /* No op. */ + } } diff --git a/cineast-core/src/test/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailIntegrationTest.java b/cineast-core/src/test/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailIntegrationTest.java index b85ed73be..a2356b89c 100644 --- a/cineast-core/src/test/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailIntegrationTest.java +++ b/cineast-core/src/test/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailIntegrationTest.java @@ -12,15 +12,17 @@ public class CottontailIntegrationTest extends DBIntegrationTest { private final CottontailIntegrationDBProvider _provider; public CottontailIntegrationTest() { - _provider = new CottontailIntegrationDBProvider(); + try { + _provider = new CottontailIntegrationDBProvider(); + } catch (Throwable e) { + LOGGER.error("Error occurred while starting and connecting to Cottontail DB: " + e.getMessage()); + throw e; + } } @Override public void finishSetup() { - final CottontailWrapper wrapper = _provider.getWrapper(); - final String fqn = wrapper.fqnInput(this.getTestTextTableName()); - wrapper.client.optimize(new OptimizeEntity(fqn), null); - wrapper.close(); + //no-op } @Test diff --git a/cineast-core/src/test/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailMetadataTest.java b/cineast-core/src/test/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailMetadataTest.java index 196bc2656..a2983907d 100644 --- a/cineast-core/src/test/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailMetadataTest.java +++ b/cineast-core/src/test/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailMetadataTest.java @@ -12,17 +12,17 @@ public class CottontailMetadataTest extends MetadataTest { private final CottontailIntegrationDBProvider _provider; public CottontailMetadataTest() { - _provider = new CottontailIntegrationDBProvider(); + try { + _provider = new CottontailIntegrationDBProvider(); + } catch (Throwable e) { + LOGGER.error("Error occurred while starting and connecting to Cottontail DB: " + e.getMessage()); + throw e; + } } @Override - public void finishInitialSetup() { - final CottontailWrapper wrapper = _provider.getWrapper(); - final String obj = wrapper.fqnInput(this.getTestObjMetaTableName()); - final String seg = wrapper.fqnInput(this.getTestSegMetaTableName()); - wrapper.client.optimize(new OptimizeEntity(obj), null); - wrapper.client.optimize(new OptimizeEntity(seg), null); - wrapper.close(); + public void finishSetup() { + //no-op } @Test diff --git a/cineast-core/src/test/java/org/vitrivr/cineast/core/db/dao/MetadataTest.java b/cineast-core/src/test/java/org/vitrivr/cineast/core/db/dao/MetadataTest.java index 6c06d4409..d863a2d49 100644 --- a/cineast-core/src/test/java/org/vitrivr/cineast/core/db/dao/MetadataTest.java +++ b/cineast-core/src/test/java/org/vitrivr/cineast/core/db/dao/MetadataTest.java @@ -42,7 +42,6 @@ public abstract class MetadataTest { private PersistencyWriter objPersWriter; private PersistencyWriter segPersWriter; private EntityCreator ec; - private QueryConfig queryConfig; private MediaObjectMetadataWriter objWriter; private MediaSegmentMetadataWriter segWriter; private MediaObjectMetadataReader objReader; @@ -50,7 +49,7 @@ public abstract class MetadataTest { private static final int ELEMENT_COUNT = 8; private static final int METADATA_PER_ELEMENT_COUNT = 7; - private static final Logger LOGGER = LogManager.getLogger(); + protected static final Logger LOGGER = LogManager.getLogger(); private IntegrationDBProvider provider; @@ -59,43 +58,54 @@ void initTest() { if (ELEMENT_COUNT % 2 != 0) { Assertions.fail("ELEMENT_COUNT must be an even number"); } - provider = provider(); - objSelector = provider.getSelector(); - segSelector = provider.getSelector(); + this.provider = provider(); + this.testObjMetaTableName = getTestObjMetaTableName(); + this.testSegMetaTableName = getTestSegMetaTableName(); + } + + @BeforeEach + void setupTest() { + /* Open all readers. */ + this.objSelector = this.provider.getSelector(); + this.segSelector = this.provider.getSelector(); + this.objReader = new MediaObjectMetadataReader(this.objSelector, this.testObjMetaTableName); + this.segReader = new MediaSegmentMetadataReader(this.segSelector, this.testSegMetaTableName); + + /* Test database connection. */ LOGGER.info("Trying to establish connection to Database"); + if(!objSelector.ping()){ + LOGGER.error("Connection to DB could not be established, failing test using assumeTrue()"); + } assumeTrue(objSelector.ping(), "Connection to database could not be established"); - assumeTrue(segSelector.ping(), "Connection to database could not be established"); LOGGER.info("Connection to Database established"); - // write data - testObjMetaTableName = getTestObjMetaTableName(); - testSegMetaTableName = getTestSegMetaTableName(); - objPersWriter = provider.getPersistencyWriter(); - segPersWriter = provider.getPersistencyWriter(); - objWriter = new MediaObjectMetadataWriter(objPersWriter, 1000, testObjMetaTableName); - segWriter = new MediaSegmentMetadataWriter(segPersWriter, 1000, testSegMetaTableName); - ec = provider.getEntityCreator(); + + /* Open all writers. */ + this.ec = provider.getEntityCreator(); + this.objPersWriter = provider.getPersistencyWriter(); + this.segPersWriter = provider.getPersistencyWriter(); + this.objWriter = new MediaObjectMetadataWriter(objPersWriter, testObjMetaTableName); + this.segWriter = new MediaSegmentMetadataWriter(segPersWriter, testSegMetaTableName); + + /* Prepare databases. */ dropTables(); createTables(); fillMetadata(); - finishInitialSetup(); - CineastIOUtils.closeQuietly(objPersWriter, segPersWriter, objWriter, segWriter); + finishSetup(); } - @BeforeEach - void setupTest() { - provider = provider(); - objSelector = provider.getSelector(); - segSelector = provider.getSelector(); - assumeTrue(objSelector.ping(), "Connection to database could not be established"); - assumeTrue(segSelector.ping(), "Connection to database could not be established"); - testObjMetaTableName = getTestObjMetaTableName(); - testSegMetaTableName = getTestSegMetaTableName(); - objReader = new MediaObjectMetadataReader(objSelector, testObjMetaTableName); - segReader = new MediaSegmentMetadataReader(segSelector, testSegMetaTableName); - queryConfig = getQueryConfig(); + @AfterEach + void tearDownTest() { + dropTables(); + } + + @AfterAll + void finalTearDownTest() { + dropTables(); + CineastIOUtils.closeQuietly(this.objWriter, this.segWriter, this.objReader, this.segReader); + CineastIOUtils.closeQuietly(this.provider); } - protected abstract void finishInitialSetup(); + protected abstract void finishSetup(); protected QueryConfig getQueryConfig() { QueryConfig qc = new QueryConfig("test-" + RandomStringUtils.randomNumeric(4), new ArrayList<>()); @@ -105,17 +115,6 @@ protected QueryConfig getQueryConfig() { protected abstract IntegrationDBProvider provider(); - @AfterEach - void tearDownTest() { - CineastIOUtils.closeQuietly(objSelector, segSelector, objReader, segReader); - } - - @AfterAll - void finalTearDownTest() { - dropTables(); - CineastIOUtils.closeQuietly(ec); - } - protected void fillMetadata() { for (int i = 0; i < ELEMENT_COUNT; i++) { objWriter.write(objMetadataForID(i)); @@ -209,22 +208,22 @@ void existAll() { objDescriptors.addAll(objDescriptor); List segDescriptor = segMetadataForID(i); segDescriptors.addAll(segDescriptor); - Assertions.assertEquals(objDescriptor, objReader.lookupMultimediaMetadata(id)); - Assertions.assertEquals(segDescriptor, segReader.lookupMultimediaMetadata(id)); + Assertions.assertEquals(objDescriptor, this.objReader.lookupMultimediaMetadata(id)); + Assertions.assertEquals(segDescriptor, this.segReader.lookupMultimediaMetadata(id)); } - Assertions.assertEquals(objDescriptors, objReader.lookupMultimediaMetadata(ids)); - Assertions.assertEquals(segDescriptors, segReader.lookupMultimediaMetadata(ids)); + Assertions.assertEquals(objDescriptors, this.objReader.lookupMultimediaMetadata(ids)); + Assertions.assertEquals(segDescriptors, this.segReader.lookupMultimediaMetadata(ids)); } @Test @DisplayName("Find by domain") void findByDomain() { - MetadataAccessSpecification objSpec = new MetadataAccessSpecification(MetadataType.OBJECT, "one", "*"); - MetadataAccessSpecification segSpec = new MetadataAccessSpecification(MetadataType.SEGMENT, "one", "*"); - List oneDomainObject = objReader.findBySpec(objSpec); - List oneDomainSegment = segReader.findBySpec(segSpec); - List objDescriptors = new ArrayList<>(); - List segDescriptors = new ArrayList<>(); + final MetadataAccessSpecification objSpec = new MetadataAccessSpecification(MetadataType.OBJECT, "one", "*"); + final MetadataAccessSpecification segSpec = new MetadataAccessSpecification(MetadataType.SEGMENT, "one", "*"); + final List oneDomainObject = this.objReader.findBySpec(objSpec); + final List oneDomainSegment = this.segReader.findBySpec(segSpec); + final List objDescriptors = new ArrayList<>(); + final List segDescriptors = new ArrayList<>(); for (int i = 0; i < ELEMENT_COUNT / 2; i++) { objDescriptors.addAll(objMetadataForID(i)); segDescriptors.addAll(segMetadataForID(i)); diff --git a/cineast-core/src/test/java/org/vitrivr/cineast/core/db/polyphenydb/PolyphenyBooleanIntegrationTest.java b/cineast-core/src/test/java/org/vitrivr/cineast/core/db/polyphenydb/PolyphenyBooleanIntegrationTest.java new file mode 100644 index 000000000..c7b4f7a14 --- /dev/null +++ b/cineast-core/src/test/java/org/vitrivr/cineast/core/db/polyphenydb/PolyphenyBooleanIntegrationTest.java @@ -0,0 +1,57 @@ +package org.vitrivr.cineast.core.db.polyphenydb; + +import java.sql.PreparedStatement; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.DisplayName; +import org.vitrivr.cineast.core.data.providers.primitive.PrimitiveTypeProvider; +import org.vitrivr.cineast.core.db.DBBooleanIntegrationTest; +import org.vitrivr.cineast.core.db.IntegrationDBProvider; +import org.vitrivr.cineast.core.db.cottontaildb.CottontailIntegrationDBProvider; +import org.vitrivr.cineast.core.db.setup.AttributeDefinition; +import org.vitrivr.cineast.core.db.setup.AttributeDefinition.AttributeType; +import org.vitrivr.cottontail.client.language.dml.Insert; + +public class PolyphenyBooleanIntegrationTest extends DBBooleanIntegrationTest { + + private final PolyphenyIntegrationDBProvider _provider; + + public PolyphenyBooleanIntegrationTest() { + try { + _provider = new PolyphenyIntegrationDBProvider(); + } catch (Throwable e) { + LOGGER.error("Error occurred while starting and connecting to Polypheny: " + e.getMessage()); + throw e; + } + } + + /** + * Create table for boolean retrieval + */ + @Override + protected void createTables() { + final HashMap hints = new HashMap<>(); + hints.put("pk", "true"); + ec.createEntity(testTableName, new AttributeDefinition(ID_COL_NAME, AttributeType.INT, hints), new AttributeDefinition(DATA_COL_NAME_1, AttributeType.TEXT), new AttributeDefinition(DATA_COL_NAME_2, AttributeType.INT), new AttributeDefinition(DATA_COL_NAME_3, AttributeType.INT)); + } + + @Override + protected void finishSetup() { + } + + @Override + protected IntegrationDBProvider provider() { + return _provider; + } + + @Test + @Override + @Disabled + public void testFulltextQuery() { + /* TODO: Not supported on Polypheny DB yet. */ + } +} diff --git a/cineast-core/src/test/java/org/vitrivr/cineast/core/db/polyphenydb/PolyphenyIntegrationDBProvider.java b/cineast-core/src/test/java/org/vitrivr/cineast/core/db/polyphenydb/PolyphenyIntegrationDBProvider.java new file mode 100644 index 000000000..553273a17 --- /dev/null +++ b/cineast-core/src/test/java/org/vitrivr/cineast/core/db/polyphenydb/PolyphenyIntegrationDBProvider.java @@ -0,0 +1,64 @@ +package org.vitrivr.cineast.core.db.polyphenydb; + +import java.sql.PreparedStatement; +import java.util.function.Supplier; +import org.vitrivr.cineast.core.config.DatabaseConfig; +import org.vitrivr.cineast.core.db.DBSelector; +import org.vitrivr.cineast.core.db.IntegrationDBProvider; +import org.vitrivr.cineast.core.db.PersistencyWriter; +import org.vitrivr.cineast.core.db.cottontaildb.CottontailEntityCreator; +import org.vitrivr.cineast.core.db.cottontaildb.CottontailSelector; +import org.vitrivr.cineast.core.db.cottontaildb.CottontailWrapper; +import org.vitrivr.cineast.core.db.cottontaildb.CottontailWriter; +import org.vitrivr.cineast.core.db.polypheny.PolyphenyEntityCreator; +import org.vitrivr.cineast.core.db.polypheny.PolyphenySelector; +import org.vitrivr.cineast.core.db.polypheny.PolyphenyWrapper; +import org.vitrivr.cineast.core.db.polypheny.PolyphenyWriter; +import org.vitrivr.cineast.core.db.setup.EntityCreator; +import org.vitrivr.cottontail.client.language.dml.Insert; + +public class PolyphenyIntegrationDBProvider implements IntegrationDBProvider { + + private static final Supplier WRAPPER_CONFIG_PROVIDER = () -> { + DatabaseConfig config = new DatabaseConfig(); + config.setHost("localhost"); + config.setPort(20591); + return config; + }; + + private final PolyphenyWrapper wrapper; + + /** The {@link DatabaseConfig} to use to run this test.*/ + private final DatabaseConfig config = WRAPPER_CONFIG_PROVIDER.get(); + + /** + * Constructor. + */ + public PolyphenyIntegrationDBProvider() { + this.wrapper = new PolyphenyWrapper(this.config.getHost(), this.config.getPort()); + } + + PolyphenyWrapper getWrapper() { + return this.wrapper; + } + + @Override + public PersistencyWriter getPersistencyWriter() { + return new PolyphenyWriter(getWrapper(), this.config.getBatchsize()); + } + + @Override + public DBSelector getSelector() { + return new PolyphenySelector(getWrapper()); + } + + @Override + public EntityCreator getEntityCreator() { + return new PolyphenyEntityCreator(getWrapper()); + } + + @Override + public void close() { + this.wrapper.close(); + } +} diff --git a/cineast-core/src/test/java/org/vitrivr/cineast/core/db/polyphenydb/PolyphenyIntegrationTest.java b/cineast-core/src/test/java/org/vitrivr/cineast/core/db/polyphenydb/PolyphenyIntegrationTest.java new file mode 100644 index 000000000..81b3a0a81 --- /dev/null +++ b/cineast-core/src/test/java/org/vitrivr/cineast/core/db/polyphenydb/PolyphenyIntegrationTest.java @@ -0,0 +1,113 @@ +package org.vitrivr.cineast.core.db.polyphenydb; + + +import java.sql.PreparedStatement; +import java.util.HashMap; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.vitrivr.cineast.core.db.DBIntegrationTest; +import org.vitrivr.cineast.core.db.IntegrationDBProvider; +import org.vitrivr.cineast.core.db.setup.AttributeDefinition; +import org.vitrivr.cineast.core.db.setup.AttributeDefinition.AttributeType; + +public class PolyphenyIntegrationTest extends DBIntegrationTest { + + + private final PolyphenyIntegrationDBProvider _provider; + + public PolyphenyIntegrationTest() { + try { + _provider = new PolyphenyIntegrationDBProvider(); + } catch (Throwable e) { + LOGGER.error("Error occurred while starting and connecting to Polypheny: " + e.getMessage()); + throw e; + } + } + + /** + * Create both a table for vector retrieval & text retrieval + */ + @Override + protected void createTables() { + final HashMap hints = new HashMap<>(); + hints.put("pk", "true"); + this.ec.createEntity(testTextTableName, new AttributeDefinition(ID_COL_NAME, AttributeType.STRING, hints), new AttributeDefinition(TEXT_COL_NAME, AttributeType.TEXT)); + this.ec.createEntity(testVectorTableName, new AttributeDefinition(ID_COL_NAME, AttributeType.STRING, hints), new AttributeDefinition(FEATURE_VECTOR_COL_NAME, AttributeType.VECTOR, 3)); + } + + @Override + public void finishSetup() { + //no-op + } + + @Test + protected void simpleTest() { + //no-op + } + + @Override + protected IntegrationDBProvider provider() { + return _provider; + } + + /** + * This test verifies that a simple "hello" query retrieves exact and partial matches, but no fuzziness + */ + @Test + @DisplayName("Text: One el, no quotes") + @Disabled + @Override + public void textRetrievalSingleLike() { + /* TODO: Not supported by Polypheny DB yet. */ + } + + @Test + @DisplayName("Text: two words, inverted, no quotes") + @Disabled + @Override + public void textRetrievalSingleTwoWordsLike() { + /* TODO: Not supported by Polypheny DB yet. */ + } + + @Test + @DisplayName("Text: One el (two words), quotes") + @Disabled + @Override + public void textRetrievalSingleTwoWordsQuotedLike() { + /* TODO: Not supported by Polypheny DB yet. */ + } + + @Test + @DisplayName("Text: One el, one word, Fuzzy") + @Disabled + @Override + public void testRetrievalSingleFuzzy() { + /* TODO: Not supported by Polypheny DB yet. */ + } + + + @Test + @DisplayName("Text: Two elements w/ single word") + @Disabled + @Override + public void testRetrievalTwo() { + /* TODO: Not supported by Polypheny DB yet. */ + } + + @Test + @DisplayName("Text: Three elements, two are a match for the same id") + @Disabled + @Override + public void testRetrievalThreeDouble() { + /* TODO: Not supported by Polypheny DB yet. */ + } + + @Test + @DisplayName("Text: Three els, one of those with quotes") + @Disabled + @Override + public void testRetrievalThree() { + /* TODO: Not supported by Polypheny DB yet. */ + } +} diff --git a/cineast-core/src/test/java/org/vitrivr/cineast/core/db/polyphenydb/PolyphenyMetadataTest.java b/cineast-core/src/test/java/org/vitrivr/cineast/core/db/polyphenydb/PolyphenyMetadataTest.java new file mode 100644 index 000000000..653704309 --- /dev/null +++ b/cineast-core/src/test/java/org/vitrivr/cineast/core/db/polyphenydb/PolyphenyMetadataTest.java @@ -0,0 +1,38 @@ +package org.vitrivr.cineast.core.db.polyphenydb; + +import java.sql.PreparedStatement; +import org.junit.jupiter.api.Test; +import org.vitrivr.cineast.core.db.IntegrationDBProvider; +import org.vitrivr.cineast.core.db.cottontaildb.CottontailIntegrationDBProvider; +import org.vitrivr.cineast.core.db.dao.MetadataTest; +import org.vitrivr.cottontail.client.language.dml.Insert; + +public class PolyphenyMetadataTest extends MetadataTest { + + + private final PolyphenyIntegrationDBProvider _provider; + + public PolyphenyMetadataTest() { + try { + _provider = new PolyphenyIntegrationDBProvider(); + } catch (Throwable e) { + LOGGER.error("Error occurred while starting and connecting to Polypheny: " + e.getMessage()); + throw e; + } + } + + @Override + public void finishSetup() { + //no-op + } + + @Test + protected void simpleTest() { + //no-op + } + + @Override + protected IntegrationDBProvider provider() { + return _provider; + } +} diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/CineastCli.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/CineastCli.java index c518a4279..d40b38de0 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/CineastCli.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/CineastCli.java @@ -4,13 +4,14 @@ import com.github.rvesse.airline.help.Help; import org.vitrivr.cineast.standalone.cli.db.DropTableCommand; import org.vitrivr.cineast.standalone.cli.db.LSC21TemporalUpdateCommand; +import org.vitrivr.cineast.standalone.cli.db.PolyphenyBenchmarkCommand; @Cli(name = "cineast-api", description = "The CLI provided by the Cineast API.", commands = { DropTableCommand.class, TagRetrievalCommand.class, OptimizeEntitiesCommand.class, CodebookCommand.class, DatabaseSetupCommand.class, ExtractionCommand.class, ImportCommand.class, ThreeDeeTestCommand.class, RetrieveCommand.class, Help.class, SingleObjRetrievalCommand.class, TextRetrievalCommand.class, DistinctColumnApiCommand.class, - LSC21TemporalUpdateCommand.class}, defaultCommand = Help.class) + LSC21TemporalUpdateCommand.class, PolyphenyBenchmarkCommand.class}, defaultCommand = Help.class) public class CineastCli { } diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/ExtractionCommand.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/ExtractionCommand.java index 5266b14fc..1ae9b1f83 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/ExtractionCommand.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/ExtractionCommand.java @@ -14,7 +14,7 @@ import java.util.UUID; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.vitrivr.cineast.core.config.DatabaseConfig; +import org.vitrivr.cineast.core.db.DataSource; import org.vitrivr.cineast.core.iiif.IIIFConfig; import org.vitrivr.cineast.core.iiif.discoveryapi.v1.OrderedCollectionFactory; import org.vitrivr.cineast.core.iiif.imageapi.ImageFactory; @@ -76,7 +76,7 @@ public void run() { final ExtractionContainerProvider provider = ExtractionContainerProviderFactory.tryCreatingTreeWalkPathProvider(file, context); if (dispatcher.initialize(provider, context)) { /* Only attempt to optimize Cottontail entities if we were extracting into Cottontail, otherwise an unavoidable error message would be displayed when extracting elsewhere. */ - if (!doNotFinalize && context != null && context.getDatabase().getSelector() == DatabaseConfig.Selector.COTTONTAIL && context.getDatabase().getWriter() == DatabaseConfig.Writer.COTTONTAIL) { + if (!doNotFinalize && context != null && context.getDatabase().getSelector() == DataSource.COTTONTAIL && context.getDatabase().getWriter() == DataSource.COTTONTAIL) { dispatcher.registerListener(new ExtractionCompleteListener() { @Override public void extractionComplete() { diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/ImportCommand.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/ImportCommand.java index 81f485aec..1e1775247 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/ImportCommand.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/ImportCommand.java @@ -7,7 +7,7 @@ import java.nio.file.Paths; import java.util.ArrayList; import java.util.List; -import org.vitrivr.cineast.core.config.DatabaseConfig; +import org.vitrivr.cineast.core.db.DataSource; import org.vitrivr.cineast.standalone.config.Config; import org.vitrivr.cineast.standalone.importer.handlers.AsrDataImportHandler; import org.vitrivr.cineast.standalone.importer.handlers.DataImportHandler; @@ -143,7 +143,7 @@ public void run() { } /* Only attempt to optimize Cottontail entities if we were importing into Cottontail, otherwise an unavoidable error message would be displayed when importing elsewhere. */ - if (!doNotFinalize && Config.sharedConfig().getDatabase().getSelector() == DatabaseConfig.Selector.COTTONTAIL && Config.sharedConfig().getDatabase().getWriter() == DatabaseConfig.Writer.COTTONTAIL) { + if (!doNotFinalize && Config.sharedConfig().getDatabase().getSelector() == DataSource.COTTONTAIL && Config.sharedConfig().getDatabase().getWriter() == DataSource.COTTONTAIL) { OptimizeEntitiesCommand.optimizeAllCottontailEntities(); } diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/OptimizeEntitiesCommand.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/OptimizeEntitiesCommand.java index 975f5324b..6293294d4 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/OptimizeEntitiesCommand.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/OptimizeEntitiesCommand.java @@ -1,8 +1,7 @@ package org.vitrivr.cineast.standalone.cli; import com.github.rvesse.airline.annotations.Command; -import org.vitrivr.cineast.core.config.DatabaseConfig.Selector; -import org.vitrivr.cineast.core.config.DatabaseConfig.Writer; +import org.vitrivr.cineast.core.db.DataSource; import org.vitrivr.cineast.core.db.cottontaildb.CottontailWrapper; import org.vitrivr.cineast.standalone.config.Config; import org.vitrivr.cottontail.client.language.ddl.ListEntities; @@ -17,16 +16,16 @@ public void run() { } public static void optimizeAllCottontailEntities() { - if (Config.sharedConfig().getDatabase().getSelector() != Selector.COTTONTAIL || Config.sharedConfig().getDatabase().getWriter() != Writer.COTTONTAIL) { + if (Config.sharedConfig().getDatabase().getSelector() != DataSource.COTTONTAIL || Config.sharedConfig().getDatabase().getWriter() != DataSource.COTTONTAIL) { System.err.println("Cottontail DB is not both selector & writer in the config. exiting"); return; } - try (final CottontailWrapper wrapper = new CottontailWrapper(Config.sharedConfig().getDatabase(), false)) { + try (final CottontailWrapper wrapper = new CottontailWrapper(Config.sharedConfig().getDatabase().getHost(), Config.sharedConfig().getDatabase().getPort())) { System.out.println("Optimizing all entities for schema '" + CottontailWrapper.CINEAST_SCHEMA + "' in Cottontail"); - wrapper.client.list(new ListEntities(CottontailWrapper.CINEAST_SCHEMA), null).forEachRemaining(entity -> { + wrapper.client.list(new ListEntities(CottontailWrapper.CINEAST_SCHEMA)).forEachRemaining(entity -> { System.out.println("Optimizing entity " + entity); final String name = entity.asString("dbo").replace("warren.", ""); - wrapper.client.optimize(new OptimizeEntity(name), null); + wrapper.client.optimize(new OptimizeEntity(name)); }); System.out.println("Finished optimizing all entities"); } diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/db/LSC21TemporalUpdateCommand.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/db/LSC21TemporalUpdateCommand.java index 6d06909f8..b361e0c00 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/db/LSC21TemporalUpdateCommand.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/db/LSC21TemporalUpdateCommand.java @@ -4,25 +4,24 @@ import com.github.rvesse.airline.annotations.Option; import java.time.LocalDateTime; import java.time.ZoneOffset; -import java.util.ArrayList; import java.util.Arrays; +import java.util.LinkedList; import java.util.List; import java.util.Optional; import java.util.stream.Collectors; import kotlin.Pair; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.vitrivr.cineast.core.config.DatabaseConfig.Selector; -import org.vitrivr.cineast.core.config.DatabaseConfig.Writer; import org.vitrivr.cineast.core.data.entities.MediaSegmentDescriptor; +import org.vitrivr.cineast.core.db.DataSource; import org.vitrivr.cineast.core.db.cottontaildb.CottontailWrapper; import org.vitrivr.cineast.core.util.CineastConstants; import org.vitrivr.cineast.standalone.config.Config; -import org.vitrivr.cineast.standalone.importer.lsc2020.LSCUtilities; -import org.vitrivr.cottontail.client.TupleIterator; -import org.vitrivr.cottontail.client.TupleIterator.Tuple; +import org.vitrivr.cottontail.client.iterators.Tuple; +import org.vitrivr.cottontail.client.iterators.TupleIterator; import org.vitrivr.cottontail.client.language.dml.Update; import org.vitrivr.cottontail.client.language.dql.Query; +import org.vitrivr.cottontail.grpc.CottontailGrpc; import org.vitrivr.cottontail.grpc.CottontailGrpc.ColumnName; import org.vitrivr.cottontail.grpc.CottontailGrpc.Literal; import org.vitrivr.cottontail.grpc.CottontailGrpc.Literal.Builder; @@ -39,6 +38,9 @@ public class LSC21TemporalUpdateCommand implements Runnable { "--progress"}, title = "Progress", description = "Flag to indicate that some progress information is desired") private boolean progress = false; + @Option(name = {"-R", "--reset-abs"}, title = "Reset Absolute TImes", description = "Flag to indicate that the startAbs and endAbs fields should be reset to 0.0") + private boolean resetAbsTime = false; + private static MediaSegmentDescriptor convert(Tuple segmentTuple) { final String oid = (String) segmentTuple.get(CineastConstants.OBJECT_ID_COLUMN_QUALIFIER); final String sid = (String) segmentTuple.get(CineastConstants.SEGMENT_ID_COLUMN_QUALIFIER); @@ -57,7 +59,7 @@ private static MediaSegmentDescriptor convert(Tuple segmentTuple) { private static List convert(MediaSegmentDescriptor segment) { return Arrays.stream(MediaSegmentDescriptor.FIELDNAMES).map(name -> UpdateElement.newBuilder() .setColumn(ColumnName.newBuilder().setName(name).build()) - .setValue(forValue(segment, name)) + .setValue(CottontailGrpc.Expression.newBuilder().setLiteral(forValue(segment, name)).build()) .build()).collect(Collectors.toList()); } @@ -97,38 +99,50 @@ private static Literal forValue(MediaSegmentDescriptor segment, String name) { @Override @SuppressWarnings("unchecked") public void run() { - if (Config.sharedConfig().getDatabase().getSelector() != Selector.COTTONTAIL || - Config.sharedConfig().getDatabase().getWriter() != Writer.COTTONTAIL - ) { + if (Config.sharedConfig().getDatabase().getSelector() != DataSource.COTTONTAIL) { System.out.println("Other DB than Cottontail DB not supported (yet). Aborting"); return; } - final CottontailWrapper cottontail = new CottontailWrapper(Config.sharedConfig().getDatabase(), false); + + /* Preparation. */ + final CottontailWrapper cottontail = new CottontailWrapper(Config.sharedConfig().getDatabase().getHost(), Config.sharedConfig().getDatabase().getPort()); long txId = cottontail.client.begin(); - long uTxId = cottontail.client.begin(); - final Query query = new Query(ENTITY_NAME).select("*"); - final TupleIterator ti = cottontail.client.query(query, txId); - final List updateElements = new ArrayList<>(); + final Query query = new Query(ENTITY_NAME).select("*", null).txId(txId); + final TupleIterator ti = cottontail.client.query(query); + final LinkedList updates = new LinkedList<>(); int counter = 0; int totalCounter = 0; + + /* Prepare updates. */ while (ti.hasNext()) { final Tuple t = ti.next(); final MediaSegmentDescriptor segment = convert(t); - final Optional minuteIdOpt = LSCUtilities.filenameToMinuteId(segment.getSegmentId().substring(4)); - if (!minuteIdOpt.isPresent()) { - LOGGER.warn("Could not update " + segment.getSegmentId()); - continue; + try { + final Optional oldt = extractTimeInformation(segment.getSegmentId()); + if (!oldt.isPresent()) { + continue; + } + final LocalDateTime ldt = oldt.get(); + final long msOfTheDay = ldt.getHour() * 60 * 60 * 1000 + ldt.getMinute() * 60 * 1000 + ldt.getSecond() * 1000; + final long sAbs = ldt.toInstant(ZoneOffset.UTC).toEpochMilli() / 1000L; + final Update update = new Update(ENTITY_NAME).values( + new Pair<>(MediaSegmentDescriptor.SEGMENT_START_COL_NAME, (double) msOfTheDay), + new Pair<>(MediaSegmentDescriptor.SEGMENT_END_COL_NAME, (double) msOfTheDay + 1), + new Pair<>(MediaSegmentDescriptor.SEGMENT_STARTABS_COL_NAME, + resetAbsTime ? 0.0 : sAbs), + new Pair<>(MediaSegmentDescriptor.SEGMENT_ENDABS_COL_NAME, + resetAbsTime ? 0.0 : (sAbs + 1)) + ).where(new org.vitrivr.cottontail.client.language.extensions.Literal(CineastConstants.SEGMENT_ID_COLUMN_QUALIFIER, "=", segment.getSegmentId())).txId(txId); + updates.add(update); + } catch (Exception e) { + LOGGER.warn("Could not update " + segment.getSegmentId() + " due to exception: " + e.getMessage()); } - final LocalDateTime ldt = LSCUtilities.fromMinuteId(minuteIdOpt.get()); - final long msAbs = ldt.toInstant(ZoneOffset.UTC).toEpochMilli(); - final long msAbsNext = msAbs + 1; - final Update update = new Update(ENTITY_NAME) - .values( - new Pair<>(MediaSegmentDescriptor.SEGMENT_START_COL_NAME, (double) msAbs), - new Pair<>(MediaSegmentDescriptor.SEGMENT_END_COL_NAME, (double) msAbsNext) - ) - .where(new org.vitrivr.cottontail.client.language.extensions.Literal(CineastConstants.SEGMENT_ID_COLUMN_QUALIFIER, "=", segment.getSegmentId())); - cottontail.client.update(update, txId); + } + + /* Execute updates. */ + Update update; + while ((update = updates.poll()) != null) { + cottontail.client.update(update); totalCounter++; if (counter++ > 99) { if (progress) { @@ -137,7 +151,32 @@ public void run() { counter = 0; } } + cottontail.client.commit(txId); System.out.println("Done."); } + + static Optional extractTimeInformation(String segmentId) { + String[] parts = segmentId.split("_"); + String date = "", time = ""; + if (parts.length == 4) { + // "Simple format, e.g. is_20160810_071157_000 + date = parts[1]; + time = parts[2]; + } else if (parts.length == 5) { + // "Extended" format, e.g. is_b00000003_21i6bq_20150305_061213e + date = parts[3]; + time = parts[4].substring(0, 6); + } else { + LOGGER.warn("Something went wrong in extracting time info from " + segmentId); + return Optional.empty(); + } + int year = Integer.parseInt(date.substring(0, 4)); + int month = Integer.parseInt(date.substring(4, 6)); + int day = Integer.parseInt(date.substring(6)); + int hours = Integer.parseInt(time.substring(0, 2)); + int minutes = Integer.parseInt(time.substring(2, 4)); + int seconds = Integer.parseInt(time.substring(4)); + return Optional.of(LocalDateTime.of(year, month, day, hours, minutes, seconds)); + } } diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/db/PolyphenyBenchmarkCommand.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/db/PolyphenyBenchmarkCommand.java new file mode 100644 index 000000000..9bda1712b --- /dev/null +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/db/PolyphenyBenchmarkCommand.java @@ -0,0 +1,303 @@ +package org.vitrivr.cineast.standalone.cli.db; + +import com.github.rvesse.airline.annotations.Command; +import com.github.rvesse.airline.annotations.Option; +import com.github.rvesse.airline.annotations.restrictions.Required; +import java.io.BufferedWriter; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.Properties; +import java.util.SplittableRandom; +import org.vitrivr.cineast.core.config.ReadableQueryConfig.Distance; + +/** + * Runs a Polypheny DB benchmark based on a feature category. + * + * @author Ralph Gasser + * @version 1.0.0 + */ +@Command(name = "polypheny-benchmark", description = "Drop a specific table") +public class PolyphenyBenchmarkCommand implements Runnable { + + + @Option(name = {"--host"}, description = "The host IP or name where Polypheny DB runs.") + private String host = "localhost"; + + @Option(name = {"--schema"}, description = "The name of the schema to run Polypheny DB tests against.") + private String schema = "cineast"; + + @Option(name = {"--limit"}, description = "The number of features to retrieve (the k in kNN).") + private int limit = 500; + + @Option(name = {"--repeat"}, description = "The number of repetitions.") + private int repeat = 5; + + @Option(name = {"--table"}, description = "The feature table to benchmark.") + @Required + private String table; + + @Option(name = {"--out"}, description = "Path to the output directory.") + @Required + private String out; + + /** + * The JDBC {@link Connection} used to communicate with Polypheny DB. + */ + private Connection connection; + + /** + * The dimensionality + */ + private int dimensionality = 0; + + /** + * {@link SplittableRandom} used to generate random query vectors. + */ + private final SplittableRandom random = new SplittableRandom(); + + private boolean prepare() { + /* Try to instantiate Polypheny driver. */ + try { + if (this.connection == null) { + Class.forName("org.polypheny.jdbc.Driver"); /* Make sure, driver was loaded. */ + final Properties properties = new Properties(); + properties.put("username", "pa"); /* TODO: Could be configurable :-) */ + this.connection = DriverManager.getConnection(String.format("jdbc:polypheny:http://%s/", this.host), properties); + } + } catch (ClassNotFoundException | SQLException e) { + System.err.println("Preparation failed because Cineast failed to initialize JDBC connection to Polypheny DB due to an error: " + e.getMessage()); + return false; + } + + /* Try to create output directory (if it doesn't exist). */ + try { + final Path path = Paths.get(this.out); + if (!Files.exists(path)) { + try (final BufferedWriter writer = Files.newBufferedWriter(path, StandardOpenOption.CREATE, StandardOpenOption.APPEND)) { + writer.write("r,schema,table,d,limit,traditional_ms,join_ms"); + writer.newLine(); + } + } + } catch (IOException e) { + System.err.println("Preparation failed because Cineast failed to create output directory: " + e.getMessage()); + return false; + } + + /* Check for schema. */ + try (final PreparedStatement stmt = this.connection.prepareStatement(String.format("SELECT * FROM %s.%s LIMIT 1", this.schema, this.table))) { + try (final ResultSet rs = stmt.executeQuery()) { + if (!rs.next()) { + System.err.println("Preparation failed because Cineast failed to fetch an example row form cineast." + table + "; table seems to be empty."); + return false; + } + final Object array = rs.getArray("feature").getArray(); + if (array instanceof float[]) { + this.dimensionality = ((float[]) array).length; + } else { + System.err.println("Preparation failed because feature column of cineast." + table + " does not seem to be a float array."); + return false; + } + } + } catch (SQLException e) { + System.err.println("Preparation failed because Cineast failed to fetch an example row form cineast." + table + " due to an error: " + e.getMessage()); + return false; + } + return true; + } + + + @Override + public void run() { + + try { + /* Run preparation. */ + if (!this.prepare()) { + return; + } + + /* Warmup. */ + this.executeTraditional(this.randomVector()); + this.executeJoin(this.randomVector()); + + /* Executes workloads. */ + final Path out = Paths.get(this.out); + + try (final BufferedWriter writer = Files.newBufferedWriter(out, StandardOpenOption.CREATE, StandardOpenOption.APPEND)) { + + float duration_traditional_s = 0.0f; + float duration_join_s = 0.0f; + + for (int r = 0; r < this.repeat; r++) { + final float[] query = this.randomVector(); + final long duration_traditional = this.executeTraditional(query); + final long duration_join = this.executeJoin(query); + writer.write(String.format("%d,%s,%s,%d,%d,%d,%d", r, this.schema, this.table, this.dimensionality, this.limit, duration_traditional, duration_join)); + writer.newLine(); + duration_traditional_s += (duration_traditional / 1000.0f); + duration_join_s += (duration_join / 1000.0f); + } + System.out.println("Traditional workload on 'cineast." + this.table + "' (d=" + this.dimensionality + ") took " + (duration_traditional_s / this.repeat) + "s on average (" + this.repeat + " repetitions)."); + System.out.println("JOIN workload on 'cineast." + this.table + "' (d=" + this.dimensionality + ") took " + (duration_join_s / this.repeat) + "s on average (" + this.repeat + " repetitions)."); + + } catch (IOException e) { + System.err.println("Failed to open output file: " + e.getMessage()); + return; + } + + } finally { + try { + if (this.connection != null) { + this.connection.close(); + } + } catch (SQLException e) { + System.err.println("Failed to close JDBC connection: " + e.getMessage()); + } + } + } + + /** + * Executes a traditional Cineast workload, which first fetches the required features using NNS and then fetches the multimedia object and segments in a second query. + */ + private long executeTraditional(float[] query) { + final LinkedList segmentids = new LinkedList<>(); + final LinkedList objectids = new LinkedList<>(); + final HashSet names = new HashSet<>(); + final long start = System.currentTimeMillis(); + + /* 1: Perform NNS. */ + final String sql = String.format("SELECT id, distance(feature, %s, 'L2') as dist FROM %s.%s ORDER BY dist ASC LIMIT %d", toVectorString(query), this.schema, this.table, this.limit); + try (final PreparedStatement statement = this.connection.prepareStatement(sql)) { + /* Execute query and return results. */ + try (final ResultSet rs = statement.executeQuery()) { + while (rs.next()) { + segmentids.add(rs.getString("id")); + } + } + } catch (SQLException e) { + System.err.println("Error occurred during query execution in: " + e.getMessage()); + return -1L; + } + + /* 2: Fetch all segments. */ + try (final PreparedStatement statement = this.prepareInStatement("cineast_segment", "segmentid", segmentids)) { + /* Execute query and return results. */ + try (final ResultSet rs = statement.executeQuery()) { + while (rs.next()) { + objectids.add(rs.getString("objectid")); + } + } + } catch (SQLException e) { + System.err.println("Error occurred during query execution in: " + e.getMessage()); + return -1L; + } + + /* 3: Fetch all objects. */ + try (final PreparedStatement statement = this.prepareInStatement("cineast_multimediaobject", "objectid", objectids)) { + /* Execute query and return results. */ + try (final ResultSet rs = statement.executeQuery()) { + while (rs.next()) { + names.add(rs.getString("name")); + } + } + } catch (SQLException e) { + System.err.println("Error occurred during query execution in: " + e.getMessage()); + return -1L; + } + + return System.currentTimeMillis() - start; + } + + /** + * Executes a JOIN-based Cineast workload, which first fetches the required features using NNS and then fetches the multimedia object and segments in a second query. + */ + private long executeJoin(float[] query) { + final HashSet names = new HashSet<>(); + final long start = System.currentTimeMillis(); + + /* 1: Perform NNS. */ + final String sql = String.format("SELECT * FROM (SELECT id, distance(feature, %s, 'L2') as dist FROM %s.%s ORDER BY dist ASC LIMIT %d) AS feature" + + " INNER JOIN %s.cineast_segment AS segment ON (feature.id = segment.segmentid)" + + " INNER JOIN %s.cineast_multimediaobject AS object ON (segment.objectid = object.objectid)", toVectorString(query), this.schema, this.table, this.limit, this.schema, this.schema); + try (final PreparedStatement statement = this.connection.prepareStatement(sql)) { + /* Execute query and return results. */ + try (final ResultSet rs = statement.executeQuery()) { + while (rs.next()) { + names.add(rs.getString("name")); + } + } + } catch (SQLException e) { + System.err.println("Error occurred during query execution in: " + e.getMessage()); + return -1L; + } + + return System.currentTimeMillis() - start; + } + + /** + * Generates and returns a new random query. + * + * @return Float array representing a new random query. + */ + private float[] randomVector() { + /* Prepare query vector. */ + final float[] vector = new float[this.dimensionality]; + for (int i = 0; i < vector.length; i++) { + vector[i] = (float) this.random.nextDouble(); + } + return vector; + } + + /** + * Converts a flat vector to a string representation usable by Polypheny DB. + * + * @param vector {@link Distance} The float vector to convert. + * @return The resulting name. + */ + private static String toVectorString(float[] vector) { + final StringBuilder arrayString = new StringBuilder("ARRAY["); + int i = 0; + for (float v : vector) { + if (i++ > 0) { + arrayString.append(","); + } + arrayString.append(v); + } + arrayString.append("]"); + return arrayString.toString(); + } + + /** + * Prepares a prepared statement {@link PreparedStatement} for a query with a single IN predicate. + * + * @param fieldName Name of the field that should be queried. + * @param values Values to use in query. + * @return {@link PreparedStatement} + * @throws SQLException + */ + private PreparedStatement prepareInStatement(String table, String fieldName, Iterable values) throws SQLException { + /* Prepare query (apparently, JDBC doesn't support value binding for IN predicates).*/ + final StringBuilder stringStatement = new StringBuilder(String.format("SELECT * FROM %s.%s WHERE %s IN (", this.schema, table, fieldName)); + int index = 0; + for (String v : values) { + if (index++ > 0) { + stringStatement.append(","); + } + stringStatement.append("'"); + stringStatement.append(v); + stringStatement.append("'"); + } + stringStatement.append(")"); + + return this.connection.prepareStatement(stringStatement.toString()); + } +} diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/config/IngestConfig.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/config/IngestConfig.java index 0b267e364..69e986979 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/config/IngestConfig.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/config/IngestConfig.java @@ -9,13 +9,16 @@ import java.util.List; import java.util.Objects; import java.util.Optional; +import java.util.function.Supplier; import java.util.stream.Collectors; import org.vitrivr.cineast.core.config.CacheConfig; import org.vitrivr.cineast.core.config.DatabaseConfig; import org.vitrivr.cineast.core.config.IdConfig; import org.vitrivr.cineast.core.config.SegmenterConfig; import org.vitrivr.cineast.core.data.MediaType; +import org.vitrivr.cineast.core.db.DBSelector; import org.vitrivr.cineast.core.db.DBSelectorSupplier; +import org.vitrivr.cineast.core.db.PersistencyWriter; import org.vitrivr.cineast.core.db.PersistencyWriterSupplier; import org.vitrivr.cineast.core.extraction.ExtractionContextProvider; import org.vitrivr.cineast.core.extraction.idgenerator.ObjectIdGenerator; @@ -316,9 +319,9 @@ public IdConfig.ExistenceCheck existenceCheck() { } /** - * Returns the PersistencyWriterSupplier that can be used during the extraction run to obtain PersistencyWriter instance. + * Returns the {@link PersistencyWriterSupplier} that can be used during the extraction run to obtain {@link PersistencyWriter} instance. * - * @return PersistencyWriterSupplier instance used obtain a PersistencyWriter. + * @return {@link PersistencyWriterSupplier instance used obtain a {@link PersistencyWriter}. */ @Override public PersistencyWriterSupplier persistencyWriter() { @@ -335,7 +338,6 @@ public Integer batchSize() { return this.database.getBatchsize(); } - /** * Returns the DBSelectorSupplier that can be used during the extraction run to obtain a DBSelector instance. * diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/handlers/DataImportHandler.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/handlers/DataImportHandler.java index df3a1f6ca..1e68e876a 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/handlers/DataImportHandler.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/handlers/DataImportHandler.java @@ -9,7 +9,7 @@ import java.util.concurrent.TimeUnit; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.vitrivr.cineast.core.config.DatabaseConfig; +import org.vitrivr.cineast.core.db.DataSource; import org.vitrivr.cineast.core.db.cottontaildb.CottontailWrapper; import org.vitrivr.cineast.core.db.setup.EntityCreator; import org.vitrivr.cineast.core.db.setup.EntityDefinition; @@ -36,11 +36,11 @@ protected static void cleanOnDemand(String entityName, String taskName) { /* Beware, this drops the table */ CreateEntity createEntity = null; CottontailWrapper cottontail = null; - if (Config.sharedConfig().getDatabase().getSelector() != DatabaseConfig.Selector.COTTONTAIL || Config.sharedConfig().getDatabase().getWriter() != DatabaseConfig.Writer.COTTONTAIL) { + if (Config.sharedConfig().getDatabase().getSelector() != DataSource.COTTONTAIL || Config.sharedConfig().getDatabase().getWriter() != DataSource.COTTONTAIL) { LOGGER.warn("Other database than Cottontail DB in use. Using inconvenient database restore"); } else { LOGGER.info("Storing entity ({}) details for re-setup", entityName); - cottontail = new CottontailWrapper(Config.sharedConfig().getDatabase(), false); + cottontail = new CottontailWrapper(Config.sharedConfig().getDatabase().getHost(), Config.sharedConfig().getDatabase().getPort()); //entityDefinition = cottontail.entityDetailsBlocking(CottontailMessageBuilder.entity(entityName)); } LOGGER.info("{} - Dropping table for entity {}...", taskName, entityName); @@ -51,7 +51,7 @@ protected static void cleanOnDemand(String entityName, String taskName) { DatabaseSetupCommand setupCmd = new DatabaseSetupCommand(); setupCmd.doSetup(); } else { - cottontail.client.create(createEntity, null); + cottontail.client.create(createEntity); LOGGER.info("Re-created entity: {}", createEntity.getBuilder().getDefinition().getEntity().getName()); } } @@ -86,15 +86,10 @@ protected class DataImportRunner implements Runnable { */ private final Importer importer; /** - * A -possibly- human readable name for the import task + * A -possibly- human-readable name for the import task */ private final String taskName; - /** - * Whether or not the table of the entity to import should be dropped beforehand. Basically, if this is true: its a TRUNCATE_EXISTING write, otherwise it's an APPEND write - */ - private final boolean clean; - /** * Creates a new {@link DataImportRunner} to run the import of the specified {@link Importer}. If specified, drops the entity's table beforehand. * @@ -107,7 +102,6 @@ public DataImportRunner(Importer importer, String entityName, String taskName this.entityName = entityName; this.importer = importer; this.taskName = taskName; - this.clean = clean; if (clean) { cleanOnDemand(this.entityName, this.taskName); } @@ -149,7 +143,7 @@ public void run() { long start = System.currentTimeMillis(); final Copier copier = new Copier(this.entityName, this.importer); LOGGER.info("Starting import on entity: {} with importer {}, task {}...", this.entityName, this.importer.getClass().getSimpleName(), taskName); - copier.copyBatched(DataImportHandler.this.batchsize); + copier.copyBatched(DataImportHandler.this.batchSize); copier.close(); LOGGER.info("Completed import of entity: {}, task {}", this.entityName, taskName); long stop = System.currentTimeMillis(); @@ -171,8 +165,11 @@ public void run() { /** * Size of data batches (i.e. number if tuples) that are sent to the persistence layer. */ - protected final int batchsize; + protected final int batchSize; + /** + * Number of threads to use for data import. + */ protected int numberOfThreads; /** @@ -184,11 +181,11 @@ public void run() { * Constructor; creates a new DataImportHandler with specified number of threads and batchsize. * * @param threads Number of threads to use for data import. - * @param batchsize Size of data batches that are sent to the persistence layer. + * @param batchSize Size of data batches that are sent to the persistence layer. */ - public DataImportHandler(int threads, int batchsize) { + public DataImportHandler(int threads, int batchSize) { this.service = Executors.newFixedThreadPool(threads); - this.batchsize = batchsize; + this.batchSize = batchSize; this.numberOfThreads = threads; } diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/lsc2020/LSCUtilities.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/lsc2020/LSCUtilities.java index 4af312dab..7f37d68fa 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/lsc2020/LSCUtilities.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/lsc2020/LSCUtilities.java @@ -158,6 +158,9 @@ public static void writeLines(Path directory, String file, List lines) t * In the LSC2020 dataset, image paths (from file lsc2020-visual-concepts.csv) do contain more than the actual (image) file, hence some conversion is required *
* Prepends is_, removes anything before a slash ("/"), if present, and after a dot (".") (i.e., file extension), if present + * + * @param path + * @return */ public static String pathToSegmentId(String path) { final int beginIdx = path.contains("/") ? path.lastIndexOf("/") + 1 : 0; @@ -226,7 +229,6 @@ public static LocalDateTime convertUtc(String lscUtcFormat) { * * @param lscFormat The LSC UTC Timestamp in the format {@code yyyy-MM-dd_hh:mm} * @param zone A zoneid in format {@code Area/Region}, see {@linkplain ZoneId} - * @return */ public static ZonedDateTime convertLocal(String lscFormat, String zone) { final long epochSec = convert(lscFormat, false, zone); @@ -337,11 +339,12 @@ public static LocalDateTime fromMinuteId(String minuteId) { // YYYYMMDD_HHMM // 0123456789012 try { + final int endStrIdx = Math.min(minuteId.length(), 13); final int year = Integer.parseInt(minuteId.substring(0, 4)); final int month = Integer.parseInt(minuteId.substring(4, 6)); final int day = Integer.parseInt(minuteId.substring(6, 8)); final int hour = Integer.parseInt(minuteId.substring(9, 11)); - final int minute = Integer.parseInt(minuteId.substring(11)); + final int minute = Integer.parseInt(minuteId.substring(11, endStrIdx)); return LocalDateTime.of(year, month, day, hour, minute); } catch (NumberFormatException e) { throw new IllegalArgumentException("Cannot parse due to invalid number format in minuteId"); @@ -427,8 +430,6 @@ private List readFile(Path root) throws IOException, CsvException { /** * Immutable - * - * @return */ @Deprecated public List getHeaderlessMetaContents() { @@ -445,8 +446,6 @@ public Map getFilenameToMinuteIdLookUp() { /** * Immutable - * - * @return */ @Deprecated public Map getMinuteIdPathMap() { diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/vbs2019/v3c1analysis/ColorlabelImportHandler.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/vbs2019/v3c1analysis/ColorlabelImportHandler.java index ae66312c7..48e8238f7 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/vbs2019/v3c1analysis/ColorlabelImportHandler.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/importer/vbs2019/v3c1analysis/ColorlabelImportHandler.java @@ -20,7 +20,7 @@ public ColorlabelImportHandler(int threads, int batchsize) { @Override public void doImport(Path root) { try { - LOGGER.info("Starting data import for colorlabel files in: {} with {} threads and {} batchsize", root.toString(), this.numberOfThreads, this.batchsize); + LOGGER.info("Starting data import for colorlabel files in: {} with {} threads", root.toString(), this.numberOfThreads); Files.walk(root.resolve("colorlabels/"), 2).filter(p -> p.toString().toLowerCase().endsWith(".txt")).forEach(p -> { try { String color = p.getFileName().toString().split("\\.")[0]; diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/run/GenericExtractionItemHandler.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/run/GenericExtractionItemHandler.java index 8e774d596..01e678618 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/run/GenericExtractionItemHandler.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/run/GenericExtractionItemHandler.java @@ -27,8 +27,8 @@ import org.vitrivr.cineast.core.data.m3d.Mesh; import org.vitrivr.cineast.core.data.segments.Model3DSegment; import org.vitrivr.cineast.core.data.segments.SegmentContainer; -import org.vitrivr.cineast.core.db.DBSelectorSupplier; -import org.vitrivr.cineast.core.db.PersistencyWriterSupplier; +import org.vitrivr.cineast.core.db.DBSelector; +import org.vitrivr.cineast.core.db.PersistencyWriter; import org.vitrivr.cineast.core.db.dao.reader.MediaObjectReader; import org.vitrivr.cineast.core.db.dao.reader.MediaSegmentReader; import org.vitrivr.cineast.core.db.dao.writer.MediaObjectMetadataWriter; @@ -107,12 +107,12 @@ public GenericExtractionItemHandler(ExtractionContainerProvider pathProvider, In this.pathProvider = pathProvider; this.mediaType = mediaType; - final PersistencyWriterSupplier writerSupplier = context.persistencyWriter(); + final Supplier> writerSupplier = context.persistencyWriter(); this.objectWriter = new MediaObjectWriter(writerSupplier.get()); - this.mediaSegmentWriter = new MediaSegmentWriter(writerSupplier.get(), context.batchSize()); - this.metadataWriter = new MediaObjectMetadataWriter(writerSupplier.get(), context.batchSize()); + this.mediaSegmentWriter = new MediaSegmentWriter(writerSupplier.get()); + this.metadataWriter = new MediaObjectMetadataWriter(writerSupplier.get()); - final DBSelectorSupplier readerSupplier = context.persistencyReader(); + final Supplier readerSupplier = context.persistencyReader(); this.objectReader = new MediaObjectReader(readerSupplier.get()); this.segmentReader = new MediaSegmentReader(readerSupplier.get()); @@ -154,7 +154,7 @@ public void run() { for (MetadataExtractor extractor : this.metadataExtractors) { LOGGER.debug("Initializing metadata extractor {}", extractor.getClass().getSimpleName()); if (extractor instanceof MetadataFeatureModule) { - ((MetadataFeatureModule) extractor).init(this.context.persistencyWriter(), this.context.batchSize()); + ((MetadataFeatureModule) extractor).init(this.context.persistencyWriter()); } else { extractor.init(); } diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/runtime/ExtractionPipeline.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/runtime/ExtractionPipeline.java index 2be8bf4fe..7a67c3322 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/runtime/ExtractionPipeline.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/runtime/ExtractionPipeline.java @@ -178,7 +178,7 @@ private void startup() { for (Extractor extractor : this.context.extractors()) { try { - extractor.init(this.context.persistencyWriter(), this.context.batchSize()); + extractor.init(this.context.persistencyWriter()); this.extractors.add(extractor); } catch (Throwable t) { LOGGER.warn("Failed to initialize extractor {} due to an exception: {}", extractor.getClass().getSimpleName(), t.getMessage()); @@ -187,7 +187,7 @@ private void startup() { for (Extractor exporter : this.context.exporters()) { try { - exporter.init(this.context.persistencyWriter(), this.context.batchSize()); + exporter.init(this.context.persistencyWriter()); this.extractors.add(exporter); } catch (Throwable t) { LOGGER.warn("Failed to exporter extractor {} due to an exception: {}", exporter.getClass().getSimpleName(), t.getMessage()); diff --git a/cineast.json b/cineast.json index 6a3ef90fd..168097d8f 100644 --- a/cineast.json +++ b/cineast.json @@ -1,8 +1,6 @@ { "database": { "host": "localhost", - "selector": "COTTONTAIL", - "writer": "COTTONTAIL", "port": 1865, "plaintext": true }, @@ -52,6 +50,9 @@ "audiofingerprint": [ {"feature": "HPCP12Shingle", "weight": 1.0} ], + "ocr": [ + {"feature": "OCRSearch", "weight": 1.0} + ], "visualtextcoembedding": [ {"feature": "VisualTextCoEmbedding", "weight": 1.0} ], diff --git a/gradle.properties b/gradle.properties index 5a51dfff7..3335bdedf 100644 --- a/gradle.properties +++ b/gradle.properties @@ -2,13 +2,13 @@ version_adampro=1.0.0 version_airline=2.8.1 version_boofcv=0.27 -version_cineast_proto=0.1.0 +version_cineast_proto=0.1.1 version_commonscodec=1.15 version_commonstext=1.9 version_commonsio=2.8.0 version_commonslang3=3.12.0 version_commonsmath3=3.6.1 -version_cottontail_proto=0.12.5 +version_cottontaildb_proto=0.13.0 version_ffmpeg=4.1-1.4.4 version_gson=2.8.6 version_guava=30.1.1-jre @@ -29,7 +29,7 @@ version_junit_platform=1.7.1 version_metadataextractor=2.15.0 version_opencsv=5.4 version_prometheus=0.10.0 -version_polypheny=1.4 +version_polypheny=1.5.2 version_tensorflow=0.3.1 version_thumbnailator=0.4.14 version_twelvemonkeys=3.6.4