Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix index creation #283

Merged
merged 2 commits into from
Mar 21, 2022
Merged

Fix index creation #283

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
6cd6445
fixing indices not being created
silvanheller Mar 21, 2022
877ca4c
bumping version
silvanheller Mar 21, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ allprojects {
group = 'org.vitrivr'

/* Our current version, on dev branch this should always be release+1-SNAPSHOT */
version = '3.10.1'
version = '3.10.2'

apply plugin: 'java-library'
apply plugin: 'maven-publish'
Expand Down
65 changes: 40 additions & 25 deletions ...-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailEntityCreator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,12 @@
import static org.vitrivr.cineast.core.db.setup.AttributeDefinition.AttributeType.TEXT;
import static org.vitrivr.cineast.core.db.setup.AttributeDefinition.AttributeType.VECTOR;
import static org.vitrivr.cineast.core.util.CineastConstants.GENERIC_ID_COLUMN_QUALIFIER;
import static org.vitrivr.cottontail.grpc.CottontailGrpc.IndexType.HASH;
import static org.vitrivr.cottontail.grpc.CottontailGrpc.IndexType.HASH_UQ;

import io.grpc.Status;
import io.grpc.StatusRuntimeException;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import org.vitrivr.cineast.core.data.entities.MediaObjectDescriptor;
Expand Down Expand Up @@ -88,11 +87,11 @@ public boolean createTagEntity() {
this.cottontail.client.create(create);

/* tag ids should be unique */
this.createIndex(entityName, TagReader.TAG_ID_COLUMNNAME, HASH_UQ, txId);
this.createIndexWithoutSchemaName(TagReader.TAG_ENTITY_NAME, TagReader.TAG_ID_COLUMNNAME, HASH_UQ, txId);
/* tag names do not necessarily have to be unique */
this.createIndex(entityName, TagReader.TAG_NAME_COLUMNNAME, IndexType.HASH, txId);
this.createIndexWithoutSchemaName(TagReader.TAG_ENTITY_NAME, TagReader.TAG_NAME_COLUMNNAME, IndexType.HASH, txId);
/* could be used for autocomplete */
this.createIndex(entityName, TagReader.TAG_NAME_COLUMNNAME, IndexType.LUCENE, txId);
this.createIndexWithoutSchemaName(TagReader.TAG_ENTITY_NAME, TagReader.TAG_NAME_COLUMNNAME, IndexType.LUCENE, txId);

this.cottontail.client.commit(txId);
return true;
Expand All @@ -117,7 +116,7 @@ public boolean createMultiMediaObjectsEntity() {
this.cottontail.client.create(entity);

/* Create index. */
this.createIndex(entityName, MediaObjectDescriptor.FIELDNAMES[0], HASH_UQ, txId);
this.createIndexWithoutSchemaName(MediaObjectDescriptor.ENTITY, MediaObjectDescriptor.FIELDNAMES[0], HASH_UQ, txId);
this.cottontail.client.commit(txId);
return true;
} catch (StatusRuntimeException e) {
Expand All @@ -144,8 +143,8 @@ public boolean createSegmentEntity() {
this.cottontail.client.create(entity);

/* Create indexes. */
this.createIndex(entityName, MediaSegmentDescriptor.FIELDNAMES[0], HASH_UQ, txId);
this.createIndex(entityName, MediaSegmentDescriptor.FIELDNAMES[1], IndexType.HASH, txId);
this.createIndexWithoutSchemaName(MediaSegmentDescriptor.ENTITY, MediaSegmentDescriptor.FIELDNAMES[0], HASH_UQ, txId);
this.createIndexWithoutSchemaName(MediaSegmentDescriptor.ENTITY, MediaSegmentDescriptor.FIELDNAMES[1], IndexType.HASH, txId);
this.cottontail.client.commit(txId);
return true;
} catch (StatusRuntimeException e) {
Expand All @@ -169,7 +168,7 @@ public boolean createMetadataEntity(String tableName) {
this.cottontail.client.create(entity);

/* Create Index. */
this.createIndex(entityName, MediaObjectMetadataDescriptor.FIELDNAMES[0], IndexType.HASH, txId);
this.createIndexWithoutSchemaName(tableName, MediaObjectMetadataDescriptor.FIELDNAMES[0], IndexType.HASH, txId);
this.cottontail.client.commit(txId);
return true;
} catch (StatusRuntimeException e) {
Expand All @@ -193,7 +192,7 @@ public boolean createSegmentMetadataEntity(String tableName) {
this.cottontail.client.create(entity);

/* Create Index. */
this.createIndex(entityName, MediaSegmentMetadataDescriptor.FIELDNAMES[0], IndexType.HASH, txId);
this.createIndexWithoutSchemaName(tableName, MediaSegmentMetadataDescriptor.FIELDNAMES[0], IndexType.HASH, txId);
this.cottontail.client.commit(txId);
return true;
} catch (StatusRuntimeException e) {
Expand All @@ -208,9 +207,7 @@ public boolean createFeatureEntity(String featureEntityName, boolean unique, int
final AttributeDefinition[] attributes = Arrays.stream(featureNames)
.map(s -> new AttributeDefinition(s, VECTOR, length))
.toArray(AttributeDefinition[]::new);
var feature = this.createFeatureEntity(featureEntityName, unique, attributes);
this.createIndex(featureEntityName, GENERIC_ID_COLUMN_QUALIFIER, unique ? IndexType.HASH_UQ : IndexType.HASH, null);
return feature;
return this.createFeatureEntity(featureEntityName, unique, attributes);
}

@Override
Expand All @@ -220,7 +217,13 @@ public boolean createFeatureEntity(String featureEntityName, boolean unique, Att
extended[0] = new AttributeDefinition(GENERIC_ID_COLUMN_QUALIFIER, AttributeDefinition.AttributeType.STRING, hints);
System.arraycopy(attributes, 0, extended, 1, attributes.length);
var feature = this.createEntity(featureEntityName, extended);
this.createIndex(featureEntityName, GENERIC_ID_COLUMN_QUALIFIER, unique ? IndexType.HASH_UQ : IndexType.HASH, null);
long txId = this.cottontail.client.begin();
boolean success = this.createIndexWithoutSchemaName(featureEntityName, GENERIC_ID_COLUMN_QUALIFIER, unique ? IndexType.HASH_UQ : IndexType.HASH, txId);
if (success) {
this.cottontail.client.commit(txId);
} else {
this.cottontail.client.rollback(txId);
}
return feature;
}

Expand All @@ -230,7 +233,13 @@ public boolean createIdEntity(String entityName, AttributeDefinition... attribut
extended[0] = new AttributeDefinition(GENERIC_ID_COLUMN_QUALIFIER, AttributeDefinition.AttributeType.STRING);
System.arraycopy(attributes, 0, extended, 1, attributes.length);
var ent = this.createEntity(entityName, extended);
this.createIndex(entityName, GENERIC_ID_COLUMN_QUALIFIER, IndexType.HASH, null);
long txId = this.cottontail.client.begin();
boolean success =this.createIndexWithoutSchemaName(entityName, GENERIC_ID_COLUMN_QUALIFIER, IndexType.HASH, txId);
if (success) {
this.cottontail.client.commit(txId);
} else {
this.cottontail.client.rollback(txId);
}
return ent;
}

Expand Down Expand Up @@ -260,13 +269,13 @@ public boolean createEntity(org.vitrivr.cineast.core.db.setup.EntityDefinition d
/* Create Index. */
for (AttributeDefinition attribute : def.getAttributes()) {
if (attribute.getType() == TEXT) {
this.createIndex(CottontailWrapper.CINEAST_SCHEMA + "." + def.getEntityName(), attribute.getName(), IndexType.LUCENE, txId);
this.createIndexWithoutSchemaName(def.getEntityName(), attribute.getName(), IndexType.LUCENE, txId);
}
// TODO (LS, 18.11.2020) Shouldn't we also have abstract indices in the db abstraction layer?
final Optional<String> hint = attribute.getHint(INDEX_HINT);
if (hint.isPresent()) {
IndexType idx = IndexType.valueOf(hint.get());
this.createIndex(CottontailWrapper.CINEAST_SCHEMA + "." + def.getEntityName(), attribute.getName(), idx, txId);
this.createIndexWithoutSchemaName(def.getEntityName(), attribute.getName(), idx, txId);
}
}
this.cottontail.client.commit(txId);
Expand All @@ -281,8 +290,7 @@ public boolean createEntity(org.vitrivr.cineast.core.db.setup.EntityDefinition d
public boolean createHashNonUniqueIndex(String entityName, String column) {
final long txId = this.cottontail.client.begin();
try {
final String fqn = CottontailWrapper.CINEAST_SCHEMA + "." + entityName;
this.createIndex(fqn, column, IndexType.HASH, txId);
this.createIndexWithoutSchemaName(entityName, column, IndexType.HASH, txId);
this.cottontail.client.commit(txId);
return true;
} catch (StatusRuntimeException e) {
Expand Down Expand Up @@ -350,13 +358,20 @@ public static Type mapAttributeType(AttributeDefinition.AttributeType type) {
}


private void createIndex(String entityName, String attribute, IndexType type, Long txId) {
final String indexName = entityName + ".idx_" + attribute + "_" + type.name().toLowerCase();
final CreateIndex index = new CreateIndex(indexName, type).column(entityName + "." + attribute);
if (txId != null) {
index.txId(txId);
private boolean createIndexWithoutSchemaName(String entityName, String attribute, IndexType type, long txId) {
var fqn = CottontailWrapper.CINEAST_SCHEMA + "." + entityName;
final String indexName = fqn + ".idx_" + attribute + "_" + type.name().toLowerCase();
final CreateIndex index = new CreateIndex(indexName, type).column(entityName + "." + attribute).txId(txId);
try{
this.cottontail.client.create(index);
}catch(StatusRuntimeException e){
if(e.getStatus().getCode()== Status.ALREADY_EXISTS.getCode()){
LOGGER.warn("Index {} was not created because it already exists", indexName);
return false;
}
throw e;
}
this.cottontail.client.create(index);
return true;
}

}
9 changes: 9 additions & 0 deletions ...t-runtime/src/main/java/org/vitrivr/cineast/standalone/config/RetrievalRuntimeConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@
import java.util.List;
import java.util.Optional;
import java.util.Set;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.vitrivr.cineast.core.features.AverageColor;
import org.vitrivr.cineast.core.features.AverageColorARP44;
import org.vitrivr.cineast.core.features.AverageColorCLD;
Expand Down Expand Up @@ -41,6 +43,8 @@

public final class RetrievalRuntimeConfig {

private static final Logger LOGGER = LogManager.getLogger();

private static final HashMap<String, List<RetrieverConfig>> DEFAULT_RETRIEVER_CATEGORIES = new HashMap<>();

private int threadPoolSize = 4;
Expand Down Expand Up @@ -157,6 +161,11 @@ public TObjectDoubleHashMap<Retriever> getRetrieversByCategory(String category)

Retriever rev;

if(config.getRetrieverClass()==null){
LOGGER.error("Could not find class {} in category {}, skipping retriever instantiation", config.getRetrieverClassName(), category);
continue;
}

if (config.getProperties() == null) {
rev = ReflectionHelper.instantiate(config.getRetrieverClass());
} else {
Expand Down
6 changes: 6 additions & 0 deletions cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/config/RetrieverConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,13 @@
public class RetrieverConfig {

private final Class<? extends Retriever> retrieverClass;
private final String retrieverClassName;
private final double weight;
private final Map<String, String> properties;

RetrieverConfig(Class<? extends Retriever> retrieverClass, double weight, Map<String, String> properties) {
this.retrieverClass = retrieverClass;
this.retrieverClassName = retrieverClass == null ? null : retrieverClass.getSimpleName();
this.weight = weight;
this.properties = properties;
}
Expand All @@ -24,6 +26,7 @@ public RetrieverConfig(
@JsonProperty(value = "weight", required = false, defaultValue = "1.0") Double weight,
@JsonProperty(value = "properties", required = false) Map<String, String> properties
) throws InstantiationException, ClassNotFoundException {
this.retrieverClassName = retrieverClassName;
this.retrieverClass = ReflectionHelper.getClassFromName(retrieverClassName, Retriever.class, ReflectionHelper.FEATURE_MODULE_PACKAGE);
this.weight = weight;
this.properties = properties;
Expand All @@ -49,4 +52,7 @@ public Map<String, String> getProperties() {
return this.properties;
}

public String getRetrieverClassName() {
return retrieverClassName;
}
}