diff --git a/build.gradle b/build.gradle index 2aae184b7..44ee1019f 100644 --- a/build.gradle +++ b/build.gradle @@ -11,7 +11,7 @@ allprojects { group = 'org.vitrivr' /* Our current version, on dev branch this should always be release+1-SNAPSHOT */ - version = '3.10.1' + version = '3.10.2' apply plugin: 'java-library' apply plugin: 'maven-publish' diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailEntityCreator.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailEntityCreator.java index 675d66a55..79b24a0cc 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailEntityCreator.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailEntityCreator.java @@ -5,13 +5,12 @@ import static org.vitrivr.cineast.core.db.setup.AttributeDefinition.AttributeType.TEXT; import static org.vitrivr.cineast.core.db.setup.AttributeDefinition.AttributeType.VECTOR; import static org.vitrivr.cineast.core.util.CineastConstants.GENERIC_ID_COLUMN_QUALIFIER; -import static org.vitrivr.cottontail.grpc.CottontailGrpc.IndexType.HASH; import static org.vitrivr.cottontail.grpc.CottontailGrpc.IndexType.HASH_UQ; +import io.grpc.Status; import io.grpc.StatusRuntimeException; import java.util.Arrays; import java.util.HashMap; -import java.util.Map; import java.util.Objects; import java.util.Optional; import org.vitrivr.cineast.core.data.entities.MediaObjectDescriptor; @@ -88,11 +87,11 @@ public boolean createTagEntity() { this.cottontail.client.create(create); /* tag ids should be unique */ - this.createIndex(entityName, TagReader.TAG_ID_COLUMNNAME, HASH_UQ, txId); + this.createIndexWithoutSchemaName(TagReader.TAG_ENTITY_NAME, TagReader.TAG_ID_COLUMNNAME, HASH_UQ, txId); /* tag names do not necessarily have to be unique */ - this.createIndex(entityName, TagReader.TAG_NAME_COLUMNNAME, IndexType.HASH, txId); + this.createIndexWithoutSchemaName(TagReader.TAG_ENTITY_NAME, TagReader.TAG_NAME_COLUMNNAME, IndexType.HASH, txId); /* could be used for autocomplete */ - this.createIndex(entityName, TagReader.TAG_NAME_COLUMNNAME, IndexType.LUCENE, txId); + this.createIndexWithoutSchemaName(TagReader.TAG_ENTITY_NAME, TagReader.TAG_NAME_COLUMNNAME, IndexType.LUCENE, txId); this.cottontail.client.commit(txId); return true; @@ -117,7 +116,7 @@ public boolean createMultiMediaObjectsEntity() { this.cottontail.client.create(entity); /* Create index. */ - this.createIndex(entityName, MediaObjectDescriptor.FIELDNAMES[0], HASH_UQ, txId); + this.createIndexWithoutSchemaName(MediaObjectDescriptor.ENTITY, MediaObjectDescriptor.FIELDNAMES[0], HASH_UQ, txId); this.cottontail.client.commit(txId); return true; } catch (StatusRuntimeException e) { @@ -144,8 +143,8 @@ public boolean createSegmentEntity() { this.cottontail.client.create(entity); /* Create indexes. */ - this.createIndex(entityName, MediaSegmentDescriptor.FIELDNAMES[0], HASH_UQ, txId); - this.createIndex(entityName, MediaSegmentDescriptor.FIELDNAMES[1], IndexType.HASH, txId); + this.createIndexWithoutSchemaName(MediaSegmentDescriptor.ENTITY, MediaSegmentDescriptor.FIELDNAMES[0], HASH_UQ, txId); + this.createIndexWithoutSchemaName(MediaSegmentDescriptor.ENTITY, MediaSegmentDescriptor.FIELDNAMES[1], IndexType.HASH, txId); this.cottontail.client.commit(txId); return true; } catch (StatusRuntimeException e) { @@ -169,7 +168,7 @@ public boolean createMetadataEntity(String tableName) { this.cottontail.client.create(entity); /* Create Index. */ - this.createIndex(entityName, MediaObjectMetadataDescriptor.FIELDNAMES[0], IndexType.HASH, txId); + this.createIndexWithoutSchemaName(tableName, MediaObjectMetadataDescriptor.FIELDNAMES[0], IndexType.HASH, txId); this.cottontail.client.commit(txId); return true; } catch (StatusRuntimeException e) { @@ -193,7 +192,7 @@ public boolean createSegmentMetadataEntity(String tableName) { this.cottontail.client.create(entity); /* Create Index. */ - this.createIndex(entityName, MediaSegmentMetadataDescriptor.FIELDNAMES[0], IndexType.HASH, txId); + this.createIndexWithoutSchemaName(tableName, MediaSegmentMetadataDescriptor.FIELDNAMES[0], IndexType.HASH, txId); this.cottontail.client.commit(txId); return true; } catch (StatusRuntimeException e) { @@ -208,9 +207,7 @@ public boolean createFeatureEntity(String featureEntityName, boolean unique, int final AttributeDefinition[] attributes = Arrays.stream(featureNames) .map(s -> new AttributeDefinition(s, VECTOR, length)) .toArray(AttributeDefinition[]::new); - var feature = this.createFeatureEntity(featureEntityName, unique, attributes); - this.createIndex(featureEntityName, GENERIC_ID_COLUMN_QUALIFIER, unique ? IndexType.HASH_UQ : IndexType.HASH, null); - return feature; + return this.createFeatureEntity(featureEntityName, unique, attributes); } @Override @@ -220,7 +217,13 @@ public boolean createFeatureEntity(String featureEntityName, boolean unique, Att extended[0] = new AttributeDefinition(GENERIC_ID_COLUMN_QUALIFIER, AttributeDefinition.AttributeType.STRING, hints); System.arraycopy(attributes, 0, extended, 1, attributes.length); var feature = this.createEntity(featureEntityName, extended); - this.createIndex(featureEntityName, GENERIC_ID_COLUMN_QUALIFIER, unique ? IndexType.HASH_UQ : IndexType.HASH, null); + long txId = this.cottontail.client.begin(); + boolean success = this.createIndexWithoutSchemaName(featureEntityName, GENERIC_ID_COLUMN_QUALIFIER, unique ? IndexType.HASH_UQ : IndexType.HASH, txId); + if (success) { + this.cottontail.client.commit(txId); + } else { + this.cottontail.client.rollback(txId); + } return feature; } @@ -230,7 +233,13 @@ public boolean createIdEntity(String entityName, AttributeDefinition... attribut extended[0] = new AttributeDefinition(GENERIC_ID_COLUMN_QUALIFIER, AttributeDefinition.AttributeType.STRING); System.arraycopy(attributes, 0, extended, 1, attributes.length); var ent = this.createEntity(entityName, extended); - this.createIndex(entityName, GENERIC_ID_COLUMN_QUALIFIER, IndexType.HASH, null); + long txId = this.cottontail.client.begin(); + boolean success =this.createIndexWithoutSchemaName(entityName, GENERIC_ID_COLUMN_QUALIFIER, IndexType.HASH, txId); + if (success) { + this.cottontail.client.commit(txId); + } else { + this.cottontail.client.rollback(txId); + } return ent; } @@ -260,13 +269,13 @@ public boolean createEntity(org.vitrivr.cineast.core.db.setup.EntityDefinition d /* Create Index. */ for (AttributeDefinition attribute : def.getAttributes()) { if (attribute.getType() == TEXT) { - this.createIndex(CottontailWrapper.CINEAST_SCHEMA + "." + def.getEntityName(), attribute.getName(), IndexType.LUCENE, txId); + this.createIndexWithoutSchemaName(def.getEntityName(), attribute.getName(), IndexType.LUCENE, txId); } // TODO (LS, 18.11.2020) Shouldn't we also have abstract indices in the db abstraction layer? final Optional hint = attribute.getHint(INDEX_HINT); if (hint.isPresent()) { IndexType idx = IndexType.valueOf(hint.get()); - this.createIndex(CottontailWrapper.CINEAST_SCHEMA + "." + def.getEntityName(), attribute.getName(), idx, txId); + this.createIndexWithoutSchemaName(def.getEntityName(), attribute.getName(), idx, txId); } } this.cottontail.client.commit(txId); @@ -281,8 +290,7 @@ public boolean createEntity(org.vitrivr.cineast.core.db.setup.EntityDefinition d public boolean createHashNonUniqueIndex(String entityName, String column) { final long txId = this.cottontail.client.begin(); try { - final String fqn = CottontailWrapper.CINEAST_SCHEMA + "." + entityName; - this.createIndex(fqn, column, IndexType.HASH, txId); + this.createIndexWithoutSchemaName(entityName, column, IndexType.HASH, txId); this.cottontail.client.commit(txId); return true; } catch (StatusRuntimeException e) { @@ -350,13 +358,20 @@ public static Type mapAttributeType(AttributeDefinition.AttributeType type) { } - private void createIndex(String entityName, String attribute, IndexType type, Long txId) { - final String indexName = entityName + ".idx_" + attribute + "_" + type.name().toLowerCase(); - final CreateIndex index = new CreateIndex(indexName, type).column(entityName + "." + attribute); - if (txId != null) { - index.txId(txId); + private boolean createIndexWithoutSchemaName(String entityName, String attribute, IndexType type, long txId) { + var fqn = CottontailWrapper.CINEAST_SCHEMA + "." + entityName; + final String indexName = fqn + ".idx_" + attribute + "_" + type.name().toLowerCase(); + final CreateIndex index = new CreateIndex(indexName, type).column(entityName + "." + attribute).txId(txId); + try{ + this.cottontail.client.create(index); + }catch(StatusRuntimeException e){ + if(e.getStatus().getCode()== Status.ALREADY_EXISTS.getCode()){ + LOGGER.warn("Index {} was not created because it already exists", indexName); + return false; + } + throw e; } - this.cottontail.client.create(index); + return true; } } diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/config/RetrievalRuntimeConfig.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/config/RetrievalRuntimeConfig.java index a2856a0a1..8b68b95d0 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/config/RetrievalRuntimeConfig.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/config/RetrievalRuntimeConfig.java @@ -8,6 +8,8 @@ import java.util.List; import java.util.Optional; import java.util.Set; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.vitrivr.cineast.core.features.AverageColor; import org.vitrivr.cineast.core.features.AverageColorARP44; import org.vitrivr.cineast.core.features.AverageColorCLD; @@ -41,6 +43,8 @@ public final class RetrievalRuntimeConfig { + private static final Logger LOGGER = LogManager.getLogger(); + private static final HashMap> DEFAULT_RETRIEVER_CATEGORIES = new HashMap<>(); private int threadPoolSize = 4; @@ -157,6 +161,11 @@ public TObjectDoubleHashMap getRetrieversByCategory(String category) Retriever rev; + if(config.getRetrieverClass()==null){ + LOGGER.error("Could not find class {} in category {}, skipping retriever instantiation", config.getRetrieverClassName(), category); + continue; + } + if (config.getProperties() == null) { rev = ReflectionHelper.instantiate(config.getRetrieverClass()); } else { diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/config/RetrieverConfig.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/config/RetrieverConfig.java index 14903641d..46381f50b 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/config/RetrieverConfig.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/config/RetrieverConfig.java @@ -9,11 +9,13 @@ public class RetrieverConfig { private final Class retrieverClass; + private final String retrieverClassName; private final double weight; private final Map properties; RetrieverConfig(Class retrieverClass, double weight, Map properties) { this.retrieverClass = retrieverClass; + this.retrieverClassName = retrieverClass == null ? null : retrieverClass.getSimpleName(); this.weight = weight; this.properties = properties; } @@ -24,6 +26,7 @@ public RetrieverConfig( @JsonProperty(value = "weight", required = false, defaultValue = "1.0") Double weight, @JsonProperty(value = "properties", required = false) Map properties ) throws InstantiationException, ClassNotFoundException { + this.retrieverClassName = retrieverClassName; this.retrieverClass = ReflectionHelper.getClassFromName(retrieverClassName, Retriever.class, ReflectionHelper.FEATURE_MODULE_PACKAGE); this.weight = weight; this.properties = properties; @@ -49,4 +52,7 @@ public Map getProperties() { return this.properties; } + public String getRetrieverClassName() { + return retrieverClassName; + } }