diff --git a/build.gradle b/build.gradle index 31d3fbfdf..3df8e5c0e 100644 --- a/build.gradle +++ b/build.gradle @@ -11,7 +11,7 @@ allprojects { group = 'org.vitrivr' /* Our current version, on dev branch this should always be release+1-SNAPSHOT */ - version = '3.11.0' + version = '3.11.1' apply plugin: 'java-library' apply plugin: 'maven-publish' diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/APIEndpoint.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/APIEndpoint.java index ffe73f9cf..83b6fc963 100644 --- a/cineast-api/src/main/java/org/vitrivr/cineast/api/APIEndpoint.java +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/APIEndpoint.java @@ -25,9 +25,11 @@ import org.vitrivr.cineast.api.rest.handlers.actions.feature.FindSegmentFeaturesGetHandler; import org.vitrivr.cineast.api.rest.handlers.actions.feature.FindSegmentTextGetHandler; import org.vitrivr.cineast.api.rest.handlers.actions.feature.FindTagsForElementGetHandler; +import org.vitrivr.cineast.api.rest.handlers.actions.bool.CountRowsGetHandler; import org.vitrivr.cineast.api.rest.handlers.actions.mediaobject.FindObjectAllGetHandler; import org.vitrivr.cineast.api.rest.handlers.actions.mediaobject.FindObjectByIdPostHandler; import org.vitrivr.cineast.api.rest.handlers.actions.mediaobject.FindObjectGetHandler; +import org.vitrivr.cineast.api.rest.handlers.actions.mediaobject.FindObjectPaginationGetHandler; import org.vitrivr.cineast.api.rest.handlers.actions.metadata.FindObjectMetadataByDomainGetHandler; import org.vitrivr.cineast.api.rest.handlers.actions.metadata.FindObjectMetadataByDomainPostHandler; import org.vitrivr.cineast.api.rest.handlers.actions.metadata.FindObjectMetadataByKeyGetHandler; @@ -401,6 +403,7 @@ private void registerRestOperations() { new FindObjectAllGetHandler(), new FindObjectByIdPostHandler(), new FindObjectGetHandler(), + new FindObjectPaginationGetHandler(), /* Segments */ new FindSegmentByIdPostHandler(), new FindSegmentsByIdGetHandler(), @@ -427,6 +430,7 @@ private void registerRestOperations() { /* Boolean */ new FindDistinctElementsByColumnPostHandler(), new SelectFromTablePostHandler(), + new CountRowsGetHandler(), /* Status */ new StatusInvocationHandler() )); diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/messages/general/IntegerMessage.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/messages/general/IntegerMessage.java new file mode 100644 index 000000000..8861c219d --- /dev/null +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/messages/general/IntegerMessage.java @@ -0,0 +1,18 @@ +package org.vitrivr.cineast.api.messages.general; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +public class IntegerMessage { + + private final int value; + + @JsonCreator + public IntegerMessage(@JsonProperty("value") int value) { + this.value = value; + } + + public int getValue() { + return value; + } +} diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/bool/CountRowsGetHandler.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/bool/CountRowsGetHandler.java new file mode 100644 index 000000000..046911300 --- /dev/null +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/bool/CountRowsGetHandler.java @@ -0,0 +1,52 @@ +package org.vitrivr.cineast.api.rest.handlers.actions.bool; + +import static org.vitrivr.cineast.api.util.APIConstants.TABLE_NAME; + +import io.javalin.http.Context; +import io.javalin.plugin.openapi.dsl.OpenApiBuilder; +import io.javalin.plugin.openapi.dsl.OpenApiDocumentation; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.vitrivr.cineast.api.messages.general.IntegerMessage; +import org.vitrivr.cineast.api.rest.handlers.interfaces.GetRestHandler; +import org.vitrivr.cineast.standalone.config.Config; + +public class CountRowsGetHandler implements GetRestHandler { + + public static final String ROUTE = "count/table/{" + TABLE_NAME + "}"; + + private static final Logger LOGGER = LogManager.getLogger(CountRowsGetHandler.class); + + @Override + public IntegerMessage doGet(Context ctx) { + try (final var selector = Config.sharedConfig().getDatabase().getSelectorSupplier().get()) { + var tableName = ctx.pathParam(TABLE_NAME); + selector.open(tableName); + var count = selector.rowCount(); + LOGGER.trace("counted {} objects in table {}", count, tableName); + return new IntegerMessage(count); + } + } + + @Override + public Class outClass() { + return IntegerMessage.class; + } + + @Override + public String route() { + return ROUTE; + } + + @Override + public OpenApiDocumentation docs() { + return OpenApiBuilder.document() + .operation(op -> { + op.summary("Count objects"); + op.description("Equivalent to calling SELECT count(*) FROM table. Used to determined #pages for pagination in a frontend or statistical purposes"); + op.operationId("countRows"); + op.addTagsItem("Misc"); + }) + .json("200", outClass()); + } +} diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/feature/FindSegmentFeaturesGetHandler.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/feature/FindSegmentFeaturesGetHandler.java index 5777e1159..554c7a50d 100644 --- a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/feature/FindSegmentFeaturesGetHandler.java +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/feature/FindSegmentFeaturesGetHandler.java @@ -11,8 +11,7 @@ import org.vitrivr.cineast.api.rest.handlers.interfaces.GetRestHandler; import org.vitrivr.cineast.api.util.QueryUtil; -public class FindSegmentFeaturesGetHandler implements - GetRestHandler { +public class FindSegmentFeaturesGetHandler implements GetRestHandler { public static final String ROUTE = "find/feature/all/by/id/{" + ID_QUALIFIER + "}"; diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/mediaobject/FindObjectAllGetHandler.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/mediaobject/FindObjectAllGetHandler.java index 4d956bbb9..4455353f1 100644 --- a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/mediaobject/FindObjectAllGetHandler.java +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/mediaobject/FindObjectAllGetHandler.java @@ -3,28 +3,20 @@ import io.javalin.http.Context; import io.javalin.plugin.openapi.dsl.OpenApiBuilder; import io.javalin.plugin.openapi.dsl.OpenApiDocumentation; -import java.util.List; import org.vitrivr.cineast.api.messages.result.MediaObjectQueryResult; import org.vitrivr.cineast.api.rest.handlers.interfaces.GetRestHandler; -import org.vitrivr.cineast.core.data.entities.MediaObjectDescriptor; import org.vitrivr.cineast.core.db.dao.reader.MediaObjectReader; import org.vitrivr.cineast.standalone.config.Config; public class FindObjectAllGetHandler implements GetRestHandler { - - public static final String TYPE_NAME = "type"; - public static final String ROUTE = "find/objects/all/"; // The more honest route -// public static final String ROUTE = "find/objects/all/:"+TYPE_NAME; @Override public MediaObjectQueryResult doGet(Context ctx) { - // TODO :type is not being used - final MediaObjectReader ol = new MediaObjectReader(Config.sharedConfig().getDatabase().getSelectorSupplier().get()); - final List multimediaobjectIds = ol.getAllObjects(); - ol.close(); - return new MediaObjectQueryResult("", multimediaobjectIds); + try (final MediaObjectReader ol = new MediaObjectReader(Config.sharedConfig().getDatabase().getSelectorSupplier().get())) { + return new MediaObjectQueryResult("", ol.getAllObjects()); + } } @Override diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/mediaobject/FindObjectByIdPostHandler.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/mediaobject/FindObjectByIdPostHandler.java index 332dcaefc..4651e2deb 100644 --- a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/mediaobject/FindObjectByIdPostHandler.java +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/mediaobject/FindObjectByIdPostHandler.java @@ -19,14 +19,12 @@ public class FindObjectByIdPostHandler implements ParsingPostRestHandler parameters = ctx.pathParamMap(); if (context == null || context.getIds().length == 0) { return new MediaObjectQueryResult("", new ArrayList<>(0)); } - final MediaObjectReader ol = new MediaObjectReader(Config.sharedConfig().getDatabase().getSelectorSupplier().get()); - final Map objects = ol.lookUpObjects(Arrays.asList(context.getIds())); - ol.close(); - return new MediaObjectQueryResult("", new ArrayList<>(objects.values())); + try (final MediaObjectReader ol = new MediaObjectReader(Config.sharedConfig().getDatabase().getSelectorSupplier().get())) { + return new MediaObjectQueryResult("", new ArrayList<>(ol.lookUpObjects(Arrays.asList(context.getIds())).values())); + } } @Override diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/mediaobject/FindObjectGetHandler.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/mediaobject/FindObjectGetHandler.java index d04340d23..3f6abb258 100644 --- a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/mediaobject/FindObjectGetHandler.java +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/mediaobject/FindObjectGetHandler.java @@ -1,5 +1,8 @@ package org.vitrivr.cineast.api.rest.handlers.actions.mediaobject; +import static org.vitrivr.cineast.api.util.APIConstants.ATTRIBUTE_NAME; +import static org.vitrivr.cineast.api.util.APIConstants.VALUE_NAME; + import com.google.common.collect.Lists; import io.javalin.http.Context; import io.javalin.plugin.openapi.dsl.OpenApiBuilder; @@ -15,14 +18,10 @@ public class FindObjectGetHandler implements GetRestHandler { - public static final String ATTRIBUTE_NAME = "attribute"; - public static final String VALUE_NAME = "value"; - public static final String ROUTE = "find/object/by/{" + ATTRIBUTE_NAME + "}/{" + VALUE_NAME + "}"; private static final Logger LOGGER = LogManager.getLogger(FindObjectGetHandler.class); - @Override public MediaObjectQueryResult doGet(Context ctx) { final Map parameters = ctx.pathParamMap(); @@ -30,29 +29,27 @@ public MediaObjectQueryResult doGet(Context ctx) { final String attribute = parameters.get(ATTRIBUTE_NAME); final String value = parameters.get(VALUE_NAME); - final MediaObjectReader ol = new MediaObjectReader(Config.sharedConfig().getDatabase().getSelectorSupplier().get()); - MediaObjectDescriptor object = null; - - switch (attribute.toLowerCase()) { - case "id": { - object = ol.lookUpObjectById(value); - break; - } - case "name": { - object = ol.lookUpObjectByName(value); - break; - } - case "path": { - object = ol.lookUpObjectByPath(value); - break; - } - default: { - LOGGER.error("Unknown attribute '{}' in FindObjectByActionHandler", attribute); + try (final MediaObjectReader ol = new MediaObjectReader(Config.sharedConfig().getDatabase().getSelectorSupplier().get())) { + MediaObjectDescriptor object = null; + switch (attribute.toLowerCase()) { + case "id": { + object = ol.lookUpObjectById(value); + break; + } + case "name": { + object = ol.lookUpObjectByName(value); + break; + } + case "path": { + object = ol.lookUpObjectByPath(value); + break; + } + default: { + LOGGER.error("Unknown attribute '{}' in FindObjectByActionHandler", attribute); + } } + return new MediaObjectQueryResult("", Lists.newArrayList(object)); } - - ol.close(); - return new MediaObjectQueryResult("", Lists.newArrayList(object)); } @Override @@ -75,6 +72,7 @@ public OpenApiDocumentation docs() { op.addTagsItem("Object"); }) .pathParam(ATTRIBUTE_NAME, String.class, p -> p.description("The attribute type of the value. One of: id, name, path")) + .pathParam(VALUE_NAME, String.class, p -> p.description("The actual value that you want to filter for")) .json("200", outClass()); } } diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/mediaobject/FindObjectPaginationGetHandler.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/mediaobject/FindObjectPaginationGetHandler.java new file mode 100644 index 000000000..5177f3745 --- /dev/null +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/mediaobject/FindObjectPaginationGetHandler.java @@ -0,0 +1,66 @@ +package org.vitrivr.cineast.api.rest.handlers.actions.mediaobject; + +import static org.vitrivr.cineast.api.util.APIConstants.LIMIT_NAME; +import static org.vitrivr.cineast.api.util.APIConstants.SKIP_NAME; + +import io.javalin.http.Context; +import io.javalin.plugin.openapi.dsl.OpenApiBuilder; +import io.javalin.plugin.openapi.dsl.OpenApiDocumentation; +import java.util.ArrayList; +import java.util.Map; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.vitrivr.cineast.api.messages.result.MediaObjectQueryResult; +import org.vitrivr.cineast.api.rest.handlers.interfaces.GetRestHandler; +import org.vitrivr.cineast.core.db.dao.reader.MediaObjectReader; +import org.vitrivr.cineast.standalone.config.Config; + +public class FindObjectPaginationGetHandler implements GetRestHandler { + + public static final String ROUTE = "find/object/all/{" + SKIP_NAME + "}/{" + LIMIT_NAME + "}"; + + private static final Logger LOGGER = LogManager.getLogger(FindObjectPaginationGetHandler.class); + + @Override + public MediaObjectQueryResult doGet(Context ctx) { + final Map parameters = ctx.pathParamMap(); + + try (final MediaObjectReader ol = new MediaObjectReader(Config.sharedConfig().getDatabase().getSelectorSupplier().get())) { + final var skipParam = parameters.get(SKIP_NAME); + final var skip = skipParam == null ? 0 : Integer.parseInt(skipParam); + final var limitParam = parameters.get(LIMIT_NAME); + final var limit = limitParam == null ? Integer.MAX_VALUE : Integer.parseInt(limitParam); + + var result = ol.getAllObjects(skip, limit); + LOGGER.trace("returning {} elements for skip {} and limit {}", result.size(), skip, limit); + return new MediaObjectQueryResult("", result); + } catch (Exception e) { + LOGGER.error("Error during request", e); + return new MediaObjectQueryResult("", new ArrayList<>()); + } + } + + @Override + public Class outClass() { + return MediaObjectQueryResult.class; + } + + @Override + public String route() { + return ROUTE; + } + + @Override + public OpenApiDocumentation docs() { + return OpenApiBuilder.document() + .operation(op -> { + op.summary("Get a fixed amount of objects from the sorted list"); + op.description("Equivalent to calling SELECT * FROM multimediaobject ORDER BY objectid ASC LIMIT limit SKIP skip. Mostly used for pagination when wanting to retrieve all objects"); + op.operationId("findObjectsPagination"); + op.addTagsItem("Object"); + }) + .pathParam(LIMIT_NAME, Integer.class, p -> p.description("How many object at most should be fetched")) + .pathParam(SKIP_NAME, Integer.class, p -> p.description("How many objects should be skipped")) + .json("200", outClass()); + } +} diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/metadata/FindObjectMetadataByDomainGetHandler.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/metadata/FindObjectMetadataByDomainGetHandler.java index 57f16827a..692ee6ae0 100644 --- a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/metadata/FindObjectMetadataByDomainGetHandler.java +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/metadata/FindObjectMetadataByDomainGetHandler.java @@ -1,7 +1,7 @@ package org.vitrivr.cineast.api.rest.handlers.actions.metadata; -import static org.vitrivr.cineast.api.rest.handlers.actions.metadata.FindObjectMetadataFullyQualifiedGetHandler.DOMAIN_NAME; import static org.vitrivr.cineast.api.rest.handlers.actions.metadata.FindObjectMetadataFullyQualifiedGetHandler.OBJECT_ID_NAME; +import static org.vitrivr.cineast.api.util.APIConstants.DOMAIN_NAME; import io.javalin.http.Context; import io.javalin.plugin.openapi.dsl.OpenApiBuilder; @@ -21,8 +21,7 @@ public class FindObjectMetadataByDomainGetHandler implements GetRestHandler { - public static final String ROUTE = "find/metadata/in/{" + DOMAIN_NAME + "}/by/id/{" + APIConstants.ID_QUALIFIER - + "}"; + public static final String ROUTE = "find/metadata/in/{" + DOMAIN_NAME + "}/by/id/{" + APIConstants.ID_QUALIFIER + "}"; @Override public MediaObjectMetadataQueryResult doGet(Context ctx) { @@ -30,8 +29,7 @@ public MediaObjectMetadataQueryResult doGet(Context ctx) { final String objectId = parameters.get(OBJECT_ID_NAME); final String domain = parameters.get(DOMAIN_NAME); final MetadataRetrievalService service = new MetadataRetrievalService(); - return new MediaObjectMetadataQueryResult("", - service.findByDomain(objectId, domain)); + return new MediaObjectMetadataQueryResult("", service.findByDomain(objectId, domain)); } @Override diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/metadata/FindObjectMetadataByDomainPostHandler.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/metadata/FindObjectMetadataByDomainPostHandler.java index 0cf4c9188..b8f238d8e 100644 --- a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/metadata/FindObjectMetadataByDomainPostHandler.java +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/metadata/FindObjectMetadataByDomainPostHandler.java @@ -1,6 +1,6 @@ package org.vitrivr.cineast.api.rest.handlers.actions.metadata; -import static org.vitrivr.cineast.api.rest.handlers.actions.metadata.FindObjectMetadataFullyQualifiedGetHandler.DOMAIN_NAME; +import static org.vitrivr.cineast.api.util.APIConstants.DOMAIN_NAME; import io.javalin.http.Context; import io.javalin.plugin.openapi.dsl.OpenApiBuilder; @@ -33,8 +33,7 @@ public MediaObjectMetadataQueryResult performPost(IdList ids, Context ctx) { } final String domain = parameters.get(DOMAIN_NAME); final MetadataRetrievalService service = new MetadataRetrievalService(); - return new MediaObjectMetadataQueryResult("", - service.findByDomain(ids.getIdList(), domain)); + return new MediaObjectMetadataQueryResult("", service.findByDomain(ids.getIdList(), domain)); } @Override diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/metadata/FindObjectMetadataByKeyGetHandler.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/metadata/FindObjectMetadataByKeyGetHandler.java index b2d3cd4db..057ea76aa 100644 --- a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/metadata/FindObjectMetadataByKeyGetHandler.java +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/metadata/FindObjectMetadataByKeyGetHandler.java @@ -1,6 +1,6 @@ package org.vitrivr.cineast.api.rest.handlers.actions.metadata; -import static org.vitrivr.cineast.api.rest.handlers.actions.metadata.FindObjectMetadataFullyQualifiedGetHandler.KEY_NAME; +import static org.vitrivr.cineast.api.util.APIConstants.KEY_NAME; import static org.vitrivr.cineast.api.rest.handlers.actions.metadata.FindObjectMetadataFullyQualifiedGetHandler.OBJECT_ID_NAME; import io.javalin.http.Context; diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/metadata/FindObjectMetadataByKeyPostHandler.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/metadata/FindObjectMetadataByKeyPostHandler.java index cdba7633f..89f9055f0 100644 --- a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/metadata/FindObjectMetadataByKeyPostHandler.java +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/metadata/FindObjectMetadataByKeyPostHandler.java @@ -1,6 +1,6 @@ package org.vitrivr.cineast.api.rest.handlers.actions.metadata; -import static org.vitrivr.cineast.api.rest.handlers.actions.metadata.FindObjectMetadataFullyQualifiedGetHandler.KEY_NAME; +import static org.vitrivr.cineast.api.util.APIConstants.KEY_NAME; import io.javalin.http.Context; import io.javalin.plugin.openapi.dsl.OpenApiBuilder; diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/metadata/FindObjectMetadataFullyQualifiedGetHandler.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/metadata/FindObjectMetadataFullyQualifiedGetHandler.java index 9fc17ac39..d1526cc25 100644 --- a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/metadata/FindObjectMetadataFullyQualifiedGetHandler.java +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/metadata/FindObjectMetadataFullyQualifiedGetHandler.java @@ -8,6 +8,7 @@ import org.vitrivr.cineast.api.rest.OpenApiCompatHelper; import org.vitrivr.cineast.api.rest.handlers.interfaces.GetRestHandler; import org.vitrivr.cineast.api.rest.services.MetadataRetrievalService; +import org.vitrivr.cineast.api.util.APIConstants; /** * This class handles GET requests with an object id, domain and key and returns all matching metadata descriptors. @@ -20,20 +21,17 @@ public class FindObjectMetadataFullyQualifiedGetHandler implements GetRestHandler { public static final String OBJECT_ID_NAME = "id"; - public static final String DOMAIN_NAME = "domain"; - public static final String KEY_NAME = "key"; - public static final String ROUTE = "find/metadata/of/{" + OBJECT_ID_NAME + "}/in/{" + DOMAIN_NAME + "}/with/{" + KEY_NAME + "}"; + public static final String ROUTE = "find/metadata/of/{" + OBJECT_ID_NAME + "}/in/{" + APIConstants.DOMAIN_NAME + "}/with/{" + APIConstants.KEY_NAME + "}"; @Override public MediaObjectMetadataQueryResult doGet(Context ctx) { final Map parameters = ctx.pathParamMap(); final String objectId = parameters.get(OBJECT_ID_NAME); - final String domain = parameters.get(DOMAIN_NAME); - final String key = parameters.get(KEY_NAME); + final String domain = parameters.get(APIConstants.DOMAIN_NAME); + final String key = parameters.get(APIConstants.KEY_NAME); final MetadataRetrievalService service = new MetadataRetrievalService(); - return new MediaObjectMetadataQueryResult("", service.find(objectId, domain, key) - ); + return new MediaObjectMetadataQueryResult("", service.find(objectId, domain, key)); } public OpenApiDocumentation docs() { @@ -47,10 +45,10 @@ public OpenApiDocumentation docs() { .pathParam(OBJECT_ID_NAME, String.class, param -> { param.description("The object id"); }) - .pathParam(DOMAIN_NAME, String.class, param -> { + .pathParam(APIConstants.DOMAIN_NAME, String.class, param -> { param.description("The domain name"); }) - .pathParam(KEY_NAME, String.class, param -> param.description("Metadata key")) + .pathParam(APIConstants.KEY_NAME, String.class, param -> param.description("Metadata key")) .json("200", outClass()); } diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/metadata/FindObjectMetadataPostHandler.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/metadata/FindObjectMetadataPostHandler.java index 910f9a5f2..7c3b6745f 100644 --- a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/metadata/FindObjectMetadataPostHandler.java +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/metadata/FindObjectMetadataPostHandler.java @@ -24,8 +24,7 @@ public MediaObjectMetadataQueryResult performPost(OptionallyFilteredIdList ids, return new MediaObjectMetadataQueryResult("", new ArrayList<>(0)); } final MetadataRetrievalService service = new MetadataRetrievalService(); - List descriptors = service - .lookupMultimediaMetadata(ids.getIdList()); + List descriptors = service.lookupMultimediaMetadata(ids.getIdList()); if (ids.hasFilters()) { final List filters = ids.getFilterList(); for (AbstractMetadataFilterDescriptor filter : filters) { diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/segment/FindSegmentByIdPostHandler.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/segment/FindSegmentByIdPostHandler.java index 3a131e312..4ed549965 100644 --- a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/segment/FindSegmentByIdPostHandler.java +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/segment/FindSegmentByIdPostHandler.java @@ -23,10 +23,10 @@ public MediaSegmentQueryResult performPost(IdList ids, Context ctx) { if (ids == null || ids.getIds().length == 0) { return new MediaSegmentQueryResult("", new ArrayList<>(0)); } - final MediaSegmentReader sl = new MediaSegmentReader(Config.sharedConfig().getDatabase().getSelectorSupplier().get()); - final Map segments = sl.lookUpSegments(Arrays.asList(ids.getIds())); - sl.close(); - return new MediaSegmentQueryResult("", new ArrayList<>(segments.values())); + try (var sl = new MediaSegmentReader(Config.sharedConfig().getDatabase().getSelectorSupplier().get());) { + final Map segments = sl.lookUpSegments(Arrays.asList(ids.getIds())); + return new MediaSegmentQueryResult("", new ArrayList<>(segments.values())); + } } @Override diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/segment/FindSegmentsByIdGetHandler.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/segment/FindSegmentsByIdGetHandler.java index d7930c7ba..490e0afe1 100644 --- a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/segment/FindSegmentsByIdGetHandler.java +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/segment/FindSegmentsByIdGetHandler.java @@ -22,14 +22,14 @@ public class FindSegmentsByIdGetHandler implements GetRestHandler parameters = ctx.pathParamMap(); final String segmentId = parameters.get(ID_NAME); - final MediaSegmentReader sl = new MediaSegmentReader(Config.sharedConfig().getDatabase().getSelectorSupplier().get()); - final List list = sl.lookUpSegment(segmentId).map(s -> { - final List segments = new ArrayList<>(1); - segments.add(s); - return segments; - }).orElse(new ArrayList<>(0)); - sl.close(); - return new MediaSegmentQueryResult("", list); + try (var sl = new MediaSegmentReader(Config.sharedConfig().getDatabase().getSelectorSupplier().get())) { + final List list = sl.lookUpSegment(segmentId).map(s -> { + final List segments = new ArrayList<>(1); + segments.add(s); + return segments; + }).orElse(new ArrayList<>(0)); + return new MediaSegmentQueryResult("", list); + } } @Override diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/segment/FindSegmentsByObjectIdGetHandler.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/segment/FindSegmentsByObjectIdGetHandler.java index 0528c4827..898231bd4 100644 --- a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/segment/FindSegmentsByObjectIdGetHandler.java +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/segment/FindSegmentsByObjectIdGetHandler.java @@ -21,10 +21,10 @@ public class FindSegmentsByObjectIdGetHandler implements GetRestHandler parameters = ctx.pathParamMap(); final String objectId = parameters.get(ID_NAME); - final MediaSegmentReader sl = new MediaSegmentReader(Config.sharedConfig().getDatabase().getSelectorSupplier().get()); - final List list = sl.lookUpSegmentsOfObject(objectId); - sl.close(); - return new MediaSegmentQueryResult("", list); + try (var sl = new MediaSegmentReader(Config.sharedConfig().getDatabase().getSelectorSupplier().get())) { + final List list = sl.lookUpSegmentsOfObject(objectId); + return new MediaSegmentQueryResult("", list); + } } @Override diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/tag/FindTagsGetHandler.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/tag/FindTagsGetHandler.java index f2314e1b0..8534a52d1 100644 --- a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/tag/FindTagsGetHandler.java +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/tag/FindTagsGetHandler.java @@ -1,5 +1,8 @@ package org.vitrivr.cineast.api.rest.handlers.actions.tag; +import static org.vitrivr.cineast.api.util.APIConstants.ATTRIBUTE_NAME; +import static org.vitrivr.cineast.api.util.APIConstants.VALUE_NAME; + import io.javalin.http.Context; import io.javalin.plugin.openapi.dsl.OpenApiBuilder; import io.javalin.plugin.openapi.dsl.OpenApiDocumentation; @@ -21,9 +24,6 @@ public class FindTagsGetHandler implements GetRestHandler { public static final String NAME_NAME = "name"; public static final String MATCHING_NAME = "matchingname"; - public static final String ATTRIBUTE_NAME = "attribute"; - public static final String VALUE_NAME = "value"; - public static final String ROUTE = "find/tags/by/{" + ATTRIBUTE_NAME + "}/{" + VALUE_NAME + "}"; private static final Logger LOGGER = LogManager.getLogger(FindTagsGetHandler.class); diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/resolvers/FileSystemThumbnailResolver.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/resolvers/FileSystemThumbnailResolver.java index 43c17075c..97cd27c0d 100644 --- a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/resolvers/FileSystemThumbnailResolver.java +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/resolvers/FileSystemThumbnailResolver.java @@ -2,11 +2,14 @@ import java.io.File; import java.io.FileNotFoundException; +import org.apache.logging.log4j.LogManager; public class FileSystemThumbnailResolver implements ThumbnailResolver { private final File baseFolder; + private static final org.apache.logging.log4j.Logger LOGGER = LogManager.getLogger(); + public FileSystemThumbnailResolver(File baseFolder) { this.baseFolder = baseFolder; } @@ -15,23 +18,27 @@ public FileSystemThumbnailResolver(File baseFolder) { public ResolutionResult resolve(String segmentId) { if (segmentId == null) { + LOGGER.error("no segment id provided"); return null; } String[] split = segmentId.split("_"); if (split.length < 3) { + LOGGER.error("invalid segment id {}", segmentId); return null; } File[] candidates = new File[]{ - new File(baseFolder, split[0] + "_" + split[1] + "/" + split[2] + ".jpg"), - new File(baseFolder, split[0] + "_" + split[1] + "/" + split[2] + ".png"), - new File(baseFolder, split[1] + "/" + split[2] + ".jpg"), - new File(baseFolder, split[1] + "/" + split[2] + ".png"), - new File(baseFolder, split[1] + "/" + split[1] + "_" + split[2] + ".jpg"), - new File(baseFolder, split[1] + "/" + split[1] + "_" + split[2] + ".png"), - new File(baseFolder, split[1] + "/shot" + split[1] + "_" + split[2] + ".jpg"), - new File(baseFolder, split[1] + "/shot" + split[1] + "_" + split[2] + ".png"), + new File(baseFolder, split[0] + "_" + split[1] + "/" + split[2] + ".jpg"), + new File(baseFolder, split[0] + "_" + split[1] + "/" + segmentId + ".jpg"), + new File(baseFolder, split[0] + "_" + split[1] + "/" + split[2] + ".png"), + new File(baseFolder, split[0] + "_" + split[1] + "/" + segmentId + ".png"), + new File(baseFolder, split[1] + "/" + split[2] + ".jpg"), + new File(baseFolder, split[1] + "/" + split[2] + ".png"), + new File(baseFolder, split[1] + "/" + split[1] + "_" + split[2] + ".jpg"), + new File(baseFolder, split[1] + "/" + split[1] + "_" + split[2] + ".png"), + new File(baseFolder, split[1] + "/shot" + split[1] + "_" + split[2] + ".jpg"), + new File(baseFolder, split[1] + "/shot" + split[1] + "_" + split[2] + ".png"), }; for (File candidate : candidates) { @@ -39,12 +46,12 @@ public ResolutionResult resolve(String segmentId) { try { return new ResolutionResult(candidate); } catch (FileNotFoundException e) { - e.printStackTrace(); + LOGGER.error(e); return null; } } } - + LOGGER.error("no thumbnail found for segment id {}", segmentId); return null; } } diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/util/APIConstants.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/util/APIConstants.java index d0b490991..92f7e317a 100644 --- a/cineast-api/src/main/java/org/vitrivr/cineast/api/util/APIConstants.java +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/util/APIConstants.java @@ -11,6 +11,14 @@ public class APIConstants { public static final String CATEGORY_NAME = "category"; public static final String ENTITY_NAME = "entity"; public static final List ACCESS_ALL_METADATA; + public static final String DOMAIN_NAME = "domain"; + public static final String KEY_NAME = "key"; + public static final String VALUE_NAME = "value"; + public static final String ATTRIBUTE_NAME = "attribute"; + public static final String LIMIT_NAME = "limit"; + public static final String SKIP_NAME = "skip"; + public static final String TABLE_NAME = "table"; + static { List _spec = new ArrayList<>(); diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/util/QueryUtil.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/util/QueryUtil.java index 42df3f4c5..d71b206a9 100644 --- a/cineast-api/src/main/java/org/vitrivr/cineast/api/util/QueryUtil.java +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/util/QueryUtil.java @@ -116,20 +116,20 @@ public static List findSegmentsSimilarTemporal(ContinuousRetriev .collect(Collectors.toList()); // TODO: New MediaSegmentReader for every request like FindSegmentByIdPostHandler or one persistent on per endpoint like AbstractQueryMessageHandler? - final var segmentReader = new MediaSegmentReader(Config.sharedConfig().getDatabase().getSelectorSupplier().get()); - - var segmentIds = stagedResults.stream().flatMap( - resultsMap -> resultsMap.values().stream().flatMap( - pairs -> pairs.stream().map(pair -> pair.key) - ) - ).distinct().collect(Collectors.toList()); - - var segmentDescriptors = segmentReader.lookUpSegments(segmentIds, config.getQueryId()); - var stagedQueryResults = stagedResults.stream().map( - resultsMap -> resultsMap.values().stream().flatMap(Collection::stream).collect(Collectors.toList()) - ).collect(Collectors.toList()); - - return TemporalScoring.score(segmentDescriptors, stagedQueryResults, query.getTimeDistances(), query.getMaxLength()); + try (var segmentReader = new MediaSegmentReader(Config.sharedConfig().getDatabase().getSelectorSupplier().get())) { + var segmentIds = stagedResults.stream().flatMap( + resultsMap -> resultsMap.values().stream().flatMap( + pairs -> pairs.stream().map(pair -> pair.key) + ) + ).distinct().collect(Collectors.toList()); + + var segmentDescriptors = segmentReader.lookUpSegments(segmentIds, config.getQueryId()); + var stagedQueryResults = stagedResults.stream().map( + resultsMap -> resultsMap.values().stream().flatMap(Collection::stream).collect(Collectors.toList()) + ).collect(Collectors.toList()); + + return TemporalScoring.score(segmentDescriptors, stagedQueryResults, query.getTimeDistances(), query.getMaxLength()); + } } /** diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/entities/MediaObjectMetadataDescriptor.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/entities/MediaObjectMetadataDescriptor.java index 2ea2b8c54..f9f9631cb 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/entities/MediaObjectMetadataDescriptor.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/entities/MediaObjectMetadataDescriptor.java @@ -48,7 +48,7 @@ public class MediaObjectMetadataDescriptor implements ExistenceCheck { /** * ID of the MultimediaObject this MediaObjectMetadataDescriptor belongs to. */ - private final String objectId; + private final String objectid; /** * String value that identifies the metadata domain (e.g. EXIF, IPTC, DC...) @@ -95,7 +95,7 @@ public MediaObjectMetadataDescriptor( @JsonProperty(KEY_COL_NAME) String key, @JsonProperty(VAL_COL_NAME) @Nullable Object value, @JsonProperty(value = "exists", defaultValue = "false") boolean exists) { - this.objectId = objectId; + this.objectid = objectId; this.key = key; this.domain = domain; this.exists = exists; @@ -135,7 +135,7 @@ static boolean isSupportedValue(Object value) { */ public MediaObjectMetadataDescriptor(Map data) throws DatabaseLookupException { if (data.get(FIELDNAMES[0]) != null && data.get(FIELDNAMES[0]).getType() == ProviderDataType.STRING) { - this.objectId = data.get(FIELDNAMES[0]).getString(); + this.objectid = data.get(FIELDNAMES[0]).getString(); } else { throw new DatabaseLookupException("Could not read column '" + FIELDNAMES[0] + "' for MediaObjectDescriptor."); } @@ -156,9 +156,9 @@ public MediaObjectMetadataDescriptor(Map data) th this.exists = true; } - @JsonProperty - public String getObjectId() { - return objectId; + @JsonProperty(OBJECT_ID_COLUMN_QUALIFIER) + public String getObjectid() { + return objectid; } @JsonProperty @@ -212,12 +212,12 @@ public boolean equals(Object o) { return false; } MediaObjectMetadataDescriptor that = (MediaObjectMetadataDescriptor) o; - return exists == that.exists && Objects.equals(objectId, that.objectId) && Objects.equals(domain, that.domain) && Objects.equals(key, that.key) && Objects.equals(value, that.value); + return exists == that.exists && Objects.equals(objectid, that.objectid) && Objects.equals(domain, that.domain) && Objects.equals(key, that.key) && Objects.equals(value, that.value); } @Override public int hashCode() { - return Objects.hash(objectId, domain, key, value, exists); + return Objects.hash(objectid, domain, key, value, exists); } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/DBSelector.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/DBSelector.java index df5c00881..553018b0b 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/DBSelector.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/DBSelector.java @@ -14,11 +14,13 @@ import java.util.Map; import java.util.Set; import java.util.stream.Collectors; +import org.apache.commons.lang3.compare.ObjectToStringComparator; import org.apache.commons.lang3.tuple.Triple; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.vitrivr.cineast.core.config.ReadableQueryConfig; import org.vitrivr.cineast.core.data.distance.DistanceElement; +import org.vitrivr.cineast.core.data.entities.MediaObjectDescriptor; import org.vitrivr.cineast.core.data.providers.primitive.FloatArrayTypeProvider; import org.vitrivr.cineast.core.data.providers.primitive.PrimitiveTypeProvider; import org.vitrivr.cineast.core.data.providers.primitive.ProviderDataType; @@ -51,17 +53,12 @@ default List getNearestNeighboursGeneric(int k, f * @param type of the {@link DistanceElement} * @return a list of elements with their distance */ - default List getNearestNeighboursGeneric(int k, - PrimitiveTypeProvider queryProvider, String column, Class distanceElementClass, - ReadableQueryConfig config) { - if (queryProvider.getType().equals(ProviderDataType.FLOAT_ARRAY) || queryProvider.getType() - .equals(ProviderDataType.INT_ARRAY)) { + default List getNearestNeighboursGeneric(int k, PrimitiveTypeProvider queryProvider, String column, Class distanceElementClass, ReadableQueryConfig config) { + if (queryProvider.getType().equals(ProviderDataType.FLOAT_ARRAY) || queryProvider.getType().equals(ProviderDataType.INT_ARRAY)) { //Default-implementation for backwards compatibility. - return getNearestNeighboursGeneric(k, PrimitiveTypeProvider.getSafeFloatArray(queryProvider), column, - distanceElementClass, config); + return getNearestNeighboursGeneric(k, PrimitiveTypeProvider.getSafeFloatArray(queryProvider), column, distanceElementClass, config); } - LogManager.getLogger().error("{} does not support other queries than float-arrays.", - this.getClass().getSimpleName()); + LogManager.getLogger().error("{} does not support other queries than float-arrays.", this.getClass().getSimpleName()); throw new UnsupportedOperationException(); } @@ -76,14 +73,12 @@ default List getNearestNeighboursGeneric(int k, * @param The type T of the resulting type of the {@link DistanceElement}. * @return List of results. */ - List getBatchedNearestNeighbours(int k, List vectors, - String column, Class distanceElementClass, List configs); + List getBatchedNearestNeighbours(int k, List vectors, String column, Class distanceElementClass, List configs); /** * In contrast to {@link #getNearestNeighboursGeneric(int, float[], String, Class, ReadableQueryConfig)}, this method returns all elements of a row */ - List> getNearestNeighbourRows(int k, float[] vector, - String column, ReadableQueryConfig config); + List> getNearestNeighbourRows(int k, float[] vector, String column, ReadableQueryConfig config); /** * SELECT 'vectorname' from entity where 'fieldname' = 'value' @@ -93,10 +88,8 @@ List> getNearestNeighbourRows(int k, float[] /** * for legacy support, takes the float[] method by default */ - default List getFeatureVectorsGeneric(String fieldName, PrimitiveTypeProvider value, - String vectorName, ReadableQueryConfig qc) { - return getFeatureVectors(fieldName, value, vectorName, qc).stream().map(FloatArrayTypeProvider::new) - .collect(Collectors.toList()); + default List getFeatureVectorsGeneric(String fieldName, PrimitiveTypeProvider value, String vectorName, ReadableQueryConfig qc) { + return getFeatureVectors(fieldName, value, vectorName, qc).stream().map(FloatArrayTypeProvider::new).collect(Collectors.toList()); } default List> getRows(String fieldName, PrimitiveTypeProvider value) { @@ -141,14 +134,12 @@ default List> getRows(String fieldName, List< * @param terms The query terms. Individual terms will be connected by a logical OR. * @return List of rows that math the fulltext search. */ - List> getFulltextRows(int rows, String fieldname, ReadableQueryConfig queryConfig, - String... terms); + List> getFulltextRows(int rows, String fieldname, ReadableQueryConfig queryConfig, String... terms); /** * {@link #getRows(String, RelationalOperator, Iterable)} */ - default List> getRows(String fieldName, RelationalOperator operator, - PrimitiveTypeProvider value) { + default List> getRows(String fieldName, RelationalOperator operator, PrimitiveTypeProvider value) { return getRows(fieldName, operator, Collections.singleton(value)); } @@ -162,8 +153,7 @@ default List> getRows(String fieldName, Relat * @param values The values the field should be compared to. * @return List of rows (one row is represented by one Map of the field ames and the data contained in the field). */ - List> getRows(String fieldName, RelationalOperator operator, - Iterable values); + List> getRows(String fieldName, RelationalOperator operator, Iterable values); /** * Performs a boolean lookup based on multiple conditions, linked with AND. Each element of the list specifies one of the conditions - left middle right, i.e. id IN (1, 5, 7) @@ -179,8 +169,7 @@ default List> getRowsAND(List ids = rows.stream().map(x -> x.get(identifier).getString()) - .collect(Collectors.toSet()); + Set ids = rows.stream().map(x -> x.get(identifier).getString()).collect(Collectors.toSet()); if (relevant.size() == 0) { rows.forEach(map -> relevant.put(map.get(identifier).getString(), map)); @@ -275,7 +264,9 @@ default List> getMetadataByIdAndSpec(List getAll(String column); + default List getAll(String column) { + return getAll().stream().map(el -> el.get(column)).collect(Collectors.toList()); + } /** * SELECT columns from the table. Be careful with large entities @@ -298,11 +289,26 @@ default List> getAll(List columns, in return collect; } + /** + * SELECT * FROM entity ORDER BY order ASC LIMIT limit SKIP skip + *
+ * skip is also sometimes called offset. This is horribly inefficient in the default implementation, as it serializes to string and then sorts. + */ + default List> getAll(String order, int skip, int limit) { + return getAll().stream().sorted((o1, o2) -> ObjectToStringComparator.INSTANCE.compare(o1.get(order), o2.get(order))).skip(skip).limit(limit).collect(Collectors.toList()); + } + /** * Get all rows from all tables */ List> getAll(); + /** + * SELECT count(*) FROM table + */ + default int rowCount(){ + return getAll().size(); + } boolean existsEntity(String name); @@ -310,4 +316,5 @@ default List> getAll(List columns, in * Healthcheck. Returns false if something is wrong */ boolean ping(); + } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/DBSelectorSupplier.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/DBSelectorSupplier.java index b83ef1c59..2538d4e98 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/DBSelectorSupplier.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/DBSelectorSupplier.java @@ -8,6 +8,6 @@ * Important: This class is required because of signature clashes in {@link org.vitrivr.cineast.core.features.abstracts.AbstractFeatureModule} due * to type erasure! */ -public interface DBSelectorSupplier extends Supplier{ +public interface DBSelectorSupplier extends Supplier { } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/ImporterSelector.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/ImporterSelector.java index 2138c6f2b..13f34bf59 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/ImporterSelector.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/ImporterSelector.java @@ -55,33 +55,23 @@ public void close() { } @Override - public List getNearestNeighboursGeneric(int k, - PrimitiveTypeProvider queryProvider, String column, Class distanceElementClass, - ReadableQueryConfig config) { + public List getNearestNeighboursGeneric(int k, PrimitiveTypeProvider queryProvider, String column, Class distanceElementClass, ReadableQueryConfig config) { List> results; - if (queryProvider.getType().equals(ProviderDataType.FLOAT_ARRAY) || queryProvider.getType() - .equals(ProviderDataType.INT_ARRAY)) { + if (queryProvider.getType().equals(ProviderDataType.FLOAT_ARRAY) || queryProvider.getType().equals(ProviderDataType.INT_ARRAY)) { results = getNearestNeighbourRows(k, queryProvider.getFloatArray(), column, config); } else { results = getNearestNeighbourRows(k, queryProvider, column, config); } - return results.stream() - .map(m -> DistanceElement.create( - distanceElementClass, m.get(GENERIC_ID_COLUMN_QUALIFIER).getString(), m.get(DB_DISTANCE_VALUE_QUALIFIER).getDouble())) - .limit(k) - .collect(Collectors.toList()); + return results.stream().map(m -> DistanceElement.create(distanceElementClass, m.get(GENERIC_ID_COLUMN_QUALIFIER).getString(), m.get(DB_DISTANCE_VALUE_QUALIFIER).getDouble())).limit(k).collect(Collectors.toList()); } /** * Full table scan. Don't do it for performance-intensive stuff. */ @SuppressWarnings("unchecked") - private List> getNearestNeighbourRows(int k, - PrimitiveTypeProvider queryProvider, String column, ReadableQueryConfig config) { - if (queryProvider.getType().equals(ProviderDataType.FLOAT_ARRAY) || queryProvider.getType() - .equals(ProviderDataType.INT_ARRAY)) { - return getNearestNeighbourRows(k, PrimitiveTypeProvider.getSafeFloatArray(queryProvider), - column, config); + private List> getNearestNeighbourRows(int k, PrimitiveTypeProvider queryProvider, String column, ReadableQueryConfig config) { + if (queryProvider.getType().equals(ProviderDataType.FLOAT_ARRAY) || queryProvider.getType().equals(ProviderDataType.INT_ARRAY)) { + return getNearestNeighbourRows(k, PrimitiveTypeProvider.getSafeFloatArray(queryProvider), column, config); } LOGGER.debug("Switching to non-float based lookup, reading from file"); Importer importer = newImporter(this.file); @@ -90,8 +80,7 @@ private List> getNearestNeighbourRows(int k, FixedSizePriorityQueue> knn; if (queryProvider.getType().equals(ProviderDataType.BITSET)) { distance = new BitSetHammingDistance(); - knn = FixedSizePriorityQueue - .create(k, new BitSetComparator(column, distance, queryProvider.getBitSet())); + knn = FixedSizePriorityQueue.create(k, new BitSetComparator(column, distance, queryProvider.getBitSet())); } else { throw new RuntimeException(queryProvider.getType().toString()); } @@ -103,8 +92,7 @@ private List> getNearestNeighbourRows(int k, } double d; if (queryProvider.getType().equals(ProviderDataType.BITSET)) { - d = distance - .applyAsDouble(queryProvider.getBitSet(), map.get(column).getBitSet()); + d = distance.applyAsDouble(queryProvider.getBitSet(), map.get(column).getBitSet()); map.put("distance", new FloatTypeProvider((float) d)); knn.add(map); } else { @@ -127,8 +115,7 @@ private List> getNearestNeighbourRows(int k, @Override - public List> getNearestNeighbourRows(int k, float[] vector, - String column, ReadableQueryConfig config) { + public List> getNearestNeighbourRows(int k, float[] vector, String column, ReadableQueryConfig config) { config = QueryConfig.clone(config); @@ -136,8 +123,7 @@ public List> getNearestNeighbourRows(int k, f FloatArrayDistance distance = FloatArrayDistance.fromQueryConfig(config); - FixedSizePriorityQueue> knn = FixedSizePriorityQueue - .create(k, new PrimitiveTypeMapDistanceComparator(column, vector, distance)); + FixedSizePriorityQueue> knn = FixedSizePriorityQueue.create(k, new PrimitiveTypeMapDistanceComparator(column, vector, distance)); HashSet relevant = null; if (config.hasRelevantSegmentIds()) { @@ -154,8 +140,7 @@ public List> getNearestNeighbourRows(int k, f if (relevant != null && !relevant.contains(map.get(GENERIC_ID_COLUMN_QUALIFIER))) { continue; } - double d = distance - .applyAsDouble(vector, PrimitiveTypeProvider.getSafeFloatArray(map.get(column))); + double d = distance.applyAsDouble(vector, PrimitiveTypeProvider.getSafeFloatArray(map.get(column))); map.put("distance", new FloatTypeProvider((float) d)); knn.add(map); } @@ -225,8 +210,7 @@ public List> getRows(String fieldName, Primit } @Override - public List> getRows(String fieldName, - Iterable values) { + public List> getRows(String fieldName, Iterable values) { if (values == null) { return new ArrayList<>(0); } @@ -242,21 +226,6 @@ public List> getRows(String fieldName, return this.getRows(fieldName, valueArr); } - @Override - public List getAll(String column) { - List _return = new ArrayList<>(); - - Importer importer = newImporter(this.file); - Map map; - while ((map = importer.readNextAsMap()) != null) { - PrimitiveTypeProvider p = map.get(column); - if (p != null) { - _return.add(p); - } - } - return _return; - } - @Override public List> getAll() { @@ -281,21 +250,17 @@ public boolean existsEntity(String name) { protected abstract String getFileExtension(); @Override - public List getBatchedNearestNeighbours(int k, - List vectors, String column, Class distanceElementClass, - List configs) { + public List getBatchedNearestNeighbours(int k, List vectors, String column, Class distanceElementClass, List configs) { // TODO Auto-generated method stub return null; } @Override - public List> getRows(String fieldName, - RelationalOperator operator, Iterable values) { + public List> getRows(String fieldName, RelationalOperator operator, Iterable values) { throw new IllegalStateException("Not implemented."); } - public List> getFulltextRows(int rows, String fieldname, ReadableQueryConfig queryConfig, - String... terms) { + public List> getFulltextRows(int rows, String fieldname, ReadableQueryConfig queryConfig, String... terms) { throw new IllegalStateException("Not implemented."); } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/NoDBSelector.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/NoDBSelector.java index 7cc568a70..9dad2b9c9 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/NoDBSelector.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/NoDBSelector.java @@ -46,11 +46,6 @@ public List> getRows(String fieldName, Iterab return new ArrayList<>(0); } - @Override - public List getAll(String column) { - return new ArrayList<>(0); - } - @Override public List> getAll() { return new ArrayList<>(0); diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailSelector.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailSelector.java index bb7c37c9b..b2b04cdb6 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailSelector.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailSelector.java @@ -363,6 +363,22 @@ public List> getAll() { } } + @Override + public List> getAll(String order, int skip, int limit) { + final Query query = new Query(this.fqn).select("*", null) + .queryId(generateQueryID("get-all-order-skip-limit-"+this.fqn)) + .order(order, Direction.ASC) + .skip(skip) + .limit(limit); + return processResults(this.cottontail.client.query(query)); + } + + @Override + public int rowCount() { + final Query query = new Query(this.fqn).count().queryId("count-star-"+this.fqn); + return Math.toIntExact(this.cottontail.client.query(query).next().asLong(0)); + } + @Override public boolean existsEntity(String name) { final AboutEntity about = new AboutEntity(this.cottontail.fqnInput(name)); diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailWriter.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailWriter.java index e0d7f0643..efa92ecaa 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailWriter.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailWriter.java @@ -2,6 +2,8 @@ import io.grpc.StatusRuntimeException; import java.util.List; +import java.util.stream.Collectors; +import org.apache.commons.lang3.StringUtils; import org.vitrivr.cineast.core.data.ReadableFloatVector; import org.vitrivr.cineast.core.db.AbstractPersistencyWriter; import org.vitrivr.cineast.core.db.PersistentTuple; @@ -59,6 +61,9 @@ public boolean exists(String key, String value) { @Override public boolean persist(List tuples) { + if (this.fqn == null) { + LOGGER.warn("fqn was null, not inserting {} tuples {}", tuples.size(), StringUtils.join(tuples, ", ")); + } long start = System.currentTimeMillis(); int size = tuples.size(); long txId = 0L; @@ -84,7 +89,7 @@ public boolean persist(List tuples) { LOGGER.trace("Inserting msg of size {} into {}", insert.size(), this.fqn); this.cottontail.client.insert(insert); insert = new BatchInsert().into(this.fqn).columns(this.names); - if(useTransactions){ + if (useTransactions) { insert.txId(txId); } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/reader/AbstractEntityReader.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/reader/AbstractEntityReader.java index dd4690480..cf70686d8 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/reader/AbstractEntityReader.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/reader/AbstractEntityReader.java @@ -1,6 +1,8 @@ package org.vitrivr.cineast.core.db.dao.reader; import java.io.Closeable; +import java.util.List; +import org.vitrivr.cineast.core.data.entities.MediaObjectDescriptor; import org.vitrivr.cineast.core.db.DBSelector; @@ -27,4 +29,8 @@ public AbstractEntityReader(DBSelector selector) { public void close() { this.selector.close(); } + + public int rowCount(){ + return this.selector.rowCount(); + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/reader/MediaObjectReader.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/reader/MediaObjectReader.java index 02d3d2481..28981e03f 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/reader/MediaObjectReader.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/reader/MediaObjectReader.java @@ -129,4 +129,20 @@ public List getAllObjects() { return _return; } + /** + * SELECT * from mediaobjects ORDER BY id ASC LIMIT limit SKIP skip + * + * @param skip how many objects should be skipped + * @param limit how many objects should be fetched + * @return descriptors + */ + public List getAllObjects(int skip, int limit) { + List> all = selector.getAll(MediaObjectDescriptor.FIELDNAMES[0], skip, limit); + List _return = new ArrayList<>(all.size()); + for (Map map : all) { + _return.add(mapToDescriptor(map)); + } + return _return; + } + } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/reader/MediaSegmentReader.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/reader/MediaSegmentReader.java index 2c80994da..9f35c053f 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/reader/MediaSegmentReader.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/reader/MediaSegmentReader.java @@ -32,8 +32,7 @@ public MediaSegmentReader(DBSelector dbSelector) { this.selector.open(MediaSegmentDescriptor.ENTITY); } - private static Optional propertiesToDescriptor( - Map properties) { + private static Optional propertiesToDescriptor(Map properties) { if (properties.containsKey(FIELDNAMES[0]) && properties.containsKey(FIELDNAMES[1]) @@ -60,8 +59,7 @@ private static Optional propertiesToDescriptor( } public Optional lookUpSegment(String segmentId) { - Stream descriptors = - this.lookUpSegmentsByField(FIELDNAMES[0], segmentId); + Stream descriptors = this.lookUpSegmentsByField(FIELDNAMES[0], segmentId); return descriptors.findFirst(); } @@ -78,15 +76,12 @@ public Map lookUpSegments(Iterable segme } public List lookUpSegmentsOfObject(String objectId) { - Stream descriptors = - this.lookUpSegmentsByField(FIELDNAMES[1], objectId); + Stream descriptors = this.lookUpSegmentsByField(FIELDNAMES[1], objectId); return descriptors.collect(Collectors.toList()); } - public ListMultimap lookUpSegmentsOfObjects( - Iterable objectIds) { - Stream descriptors = - this.lookUpSegmentsByField(FIELDNAMES[1], objectIds); + public ListMultimap lookUpSegmentsOfObjects(Iterable objectIds) { + Stream descriptors = this.lookUpSegmentsByField(FIELDNAMES[1], objectIds); return Multimaps.index(descriptors.iterator(), MediaSegmentDescriptor::getObjectId); } @@ -100,18 +95,15 @@ public List lookUpSegmentsByNumberRange(String objectId, return all.stream().filter(it -> it.getSequenceNumber() >= lower && it.getSequenceNumber() <= upper).collect(Collectors.toList()); } - private Stream lookUpSegmentsByField( - String fieldName, String fieldValue) { + private Stream lookUpSegmentsByField(String fieldName, String fieldValue) { return lookUpSegmentsByField(fieldName, Collections.singletonList(fieldValue)); } - private Stream lookUpSegmentsByField( - String fieldName, Iterable fieldValues) { + private Stream lookUpSegmentsByField(String fieldName, Iterable fieldValues) { return lookUpSegmentsByField(fieldName, fieldValues, null); } - private Stream lookUpSegmentsByField( - String fieldName, Iterable fieldValues, String queryID) { + private Stream lookUpSegmentsByField(String fieldName, Iterable fieldValues, String queryID) { String dbQueryID = DBQueryIDGenerator.generateQueryID("seg-lookup", queryID); Set uniqueFieldValues = new HashSet<>(); fieldValues.forEach(el -> uniqueFieldValues.add(new StringTypeProvider(el))); diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/writer/MediaObjectMetadataWriter.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/writer/MediaObjectMetadataWriter.java index 53a507336..ef2041adc 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/writer/MediaObjectMetadataWriter.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/writer/MediaObjectMetadataWriter.java @@ -29,7 +29,7 @@ protected PersistentTuple generateTuple(MediaObjectMetadataDescriptor entity) { if (entity.getValueProvider() instanceof NothingProvider) { return null; } - return this.writer.generateTuple(entity.getObjectId(), entity.getDomain(), entity.getKey(), + return this.writer.generateTuple(entity.getObjectid(), entity.getDomain(), entity.getKey(), entity.getValue()); } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/writer/SimpleFulltextFeatureDescriptorWriter.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/writer/SimpleFulltextFeatureDescriptorWriter.java index 9e9127af3..42a4b1d4a 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/writer/SimpleFulltextFeatureDescriptorWriter.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/writer/SimpleFulltextFeatureDescriptorWriter.java @@ -10,6 +10,9 @@ public class SimpleFulltextFeatureDescriptorWriter extends AbstractBatchedEntity public SimpleFulltextFeatureDescriptorWriter(PersistencyWriter writer, String entityname) { super(writer); + if (entityname == null) { + throw new IllegalArgumentException("An entity name cannot be null"); + } this.entityname = entityname; } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/TagsFtSearch.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/TagsFtSearch.java deleted file mode 100644 index 084153b94..000000000 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/TagsFtSearch.java +++ /dev/null @@ -1,15 +0,0 @@ -package org.vitrivr.cineast.core.features; - -import org.vitrivr.cineast.core.features.abstracts.AbstractTextRetriever; - -public class TagsFtSearch extends AbstractTextRetriever { - - public static final String TAGS_FT_TABLE_NAME = "features_tagsft"; - - /** - * Default constructor for {@link TagsFtSearch}. - */ - public TagsFtSearch() { - super(TagsFtSearch.TAGS_FT_TABLE_NAME); - } -} \ No newline at end of file diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/AbstractTextRetriever.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/AbstractTextRetriever.java index cb06aa87b..30d33f1b9 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/AbstractTextRetriever.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/AbstractTextRetriever.java @@ -26,7 +26,6 @@ import org.vitrivr.cineast.core.data.segments.SegmentContainer; import org.vitrivr.cineast.core.db.DBSelector; import org.vitrivr.cineast.core.db.DBSelectorSupplier; -import org.vitrivr.cineast.core.db.PersistencyWriter; import org.vitrivr.cineast.core.db.PersistencyWriterSupplier; import org.vitrivr.cineast.core.db.dao.writer.SimpleFulltextFeatureDescriptorWriter; import org.vitrivr.cineast.core.db.setup.AttributeDefinition; @@ -79,6 +78,7 @@ public void init(DBSelectorSupplier selectorSupply) { @Override public void init(PersistencyWriterSupplier phandlerSupply) { this.writer = new SimpleFulltextFeatureDescriptorWriter(phandlerSupply.get(), this.tableName); + writer.init(); } @Override diff --git a/cineast-core/src/test/java/org/vitrivr/cineast/core/db/DBIntegrationTest.java b/cineast-core/src/test/java/org/vitrivr/cineast/core/db/DBIntegrationTest.java index 080835836..cc1bfd330 100644 --- a/cineast-core/src/test/java/org/vitrivr/cineast/core/db/DBIntegrationTest.java +++ b/cineast-core/src/test/java/org/vitrivr/cineast/core/db/DBIntegrationTest.java @@ -11,6 +11,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Function; import java.util.stream.Collectors; +import java.util.stream.IntStream; import org.apache.commons.lang3.RandomStringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -50,10 +51,10 @@ public abstract class DBIntegrationTest { protected EntityCreator ec; protected QueryConfig queryConfig; protected static final String ID_COL_NAME = "id"; - protected static final int VECTOR_ELEMENT_COUNT = 11; - protected static final int MAX_VECTOR_ID = 10; + protected static final int VECTOR_ELEMENT_COUNT = 110; + protected static final int MAX_VECTOR_ID = VECTOR_ELEMENT_COUNT - 1; protected static final int TEXT_ELEMENT_COUNT = 8; - protected static final int MAX_TEXT_ID = 7; + protected static final int MAX_TEXT_ID = TEXT_ELEMENT_COUNT - 1; /** * This is not called "feature" by design as it avoid the storage-layers doing optimization by col name */ @@ -143,10 +144,10 @@ protected void fillVectorData() { vector[0] = i; vector[1] = 1; vector[2] = 0; - vectors.add(writer.generateTuple(String.valueOf(i), vector)); + vectors.add(writer.generateTuple(String.format("%05d", i), vector)); } /* We write a second vector with the same id in the db */ - vectors.add(writer.generateTuple(String.valueOf(0), new float[]{0, 0, 0})); + vectors.add(writer.generateTuple(String.format("%05d", 0), new float[]{0, 0, 0})); writer.persist(vectors); } @@ -195,14 +196,23 @@ void entitiesExist() { } @Test - @DisplayName("Verify element count") - void count() { + @DisplayName("Verify element count using getAll()") + void countGetAll() { selector.open(testVectorTableName); Assertions.assertEquals(VECTOR_ELEMENT_COUNT, selector.getAll().size()); selector.open(testTextTableName); Assertions.assertEquals(TEXT_ELEMENT_COUNT, selector.getAll().size()); } + @Test + @DisplayName("Verify element count using rowCount()") + void countRowCount() { + selector.open(testVectorTableName); + Assertions.assertEquals(VECTOR_ELEMENT_COUNT, selector.rowCount()); + selector.open(testTextTableName); + Assertions.assertEquals(TEXT_ELEMENT_COUNT, selector.rowCount()); + } + @Test @DisplayName("Verify elements exist by id") void entriesExistById() { @@ -237,6 +247,26 @@ void knnSearch() { Assertions.assertTrue(result.get(1).getSegmentId().equals("0") || result.get(2).getSegmentId().equals("0")); } + @Test + @DisplayName("Get all skip limit") + void getAllSkip() { + selector.open(testVectorTableName); + int limit = 10; + IntStream.range(0, 3).forEach(i -> { + List> all = selector.getAll(ID_COL_NAME, i, limit); + Assertions.assertEquals(all.size(), limit); + all.forEach(el -> { + Assertions.assertTrue(Integer.parseInt(el.get(ID_COL_NAME).getString()) >= i - 1, "id " + el.get(ID_COL_NAME).getString() + " was smaller than " + i); + // we use <= for 0 because there are two elements with id 0 in the table + if (i == 0 || i == 1) { + Assertions.assertTrue(Integer.parseInt(el.get(ID_COL_NAME).getString()) <= i + limit, "id " + el.get(ID_COL_NAME).getString() + " was larger than " + (i + limit)); + } else { + Assertions.assertTrue(Integer.parseInt(el.get(ID_COL_NAME).getString()) < i + limit, "id " + el.get(ID_COL_NAME).getString() + " was larger than " + (i + limit)); + } + }); + }); + } + /** * TODO: Currently not supported in Cottontail DB v0.12.0. Re-activate, once support is back. */ @@ -340,7 +370,7 @@ public void testRetrievalSingleFuzzy() { /** * TODO: Fuzzy search on whole phrases is currently not supported. - * + *

* Something like "hello world"~1 would need to be implemented as either hello~1 AND world~1, but that is not implemented in the DBSelector / cottontail. * The cottontail implementation in december 19 parses hello world~1 as hello .. world~1, which is not what we're looking for * Therefore, this test serves as a note that this functionality is lacking. diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/CliUtils.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/CliUtils.java index afa020d11..1758a3dbd 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/CliUtils.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/CliUtils.java @@ -29,31 +29,33 @@ public class CliUtils { public static void printInfoForObject(String objectId, DBSelector selector) { System.out.println("= Retrieving object information for " + objectId + " ="); - MediaObjectReader objectReader = new MediaObjectReader(selector); - System.out.println(objectReader.lookUpObjectById(objectId)); - + try (var objectReader = new MediaObjectReader(selector)) { + System.out.println(objectReader.lookUpObjectById(objectId)); + } System.out.println("= Retrieving object metadata for ="); - MediaObjectMetadataReader reader = new MediaObjectMetadataReader(selector); - List metadataDescriptors = reader.lookupMultimediaMetadata(objectId); - metadataDescriptors.forEach(System.out::println); + try (var reader = new MediaObjectMetadataReader(selector)) { + List metadataDescriptors = reader.lookupMultimediaMetadata(objectId); + metadataDescriptors.forEach(System.out::println); + } } public static void printInfoForSegment(String segmentId, DBSelector selector, String _filterCategory, boolean printObjInfo) { System.out.println("= Retrieving segment information for " + segmentId + "="); - MediaSegmentReader segmentReader = new MediaSegmentReader(selector); - Optional segmentDescriptor = segmentReader.lookUpSegment(segmentId); - segmentDescriptor.ifPresent(System.out::println); - - segmentDescriptor.ifPresent(descriptor -> { - if (printObjInfo) { - printInfoForObject(descriptor.getObjectId(), selector); - } - }); + try (var segmentReader = new MediaSegmentReader(selector)) { + Optional segmentDescriptor = segmentReader.lookUpSegment(segmentId); + segmentDescriptor.ifPresent(System.out::println); + segmentDescriptor.ifPresent(descriptor -> { + if (printObjInfo) { + printInfoForObject(descriptor.getObjectId(), selector); + } + }); + } System.out.println("= Retrieving segment metadata ="); - MediaSegmentMetadataReader reader = new MediaSegmentMetadataReader(selector); - reader.lookupMultimediaMetadata(segmentId).forEach(System.out::println); + try (var reader = new MediaSegmentMetadataReader(selector)) { + reader.lookupMultimediaMetadata(segmentId).forEach(System.out::println); + } System.out.println("Retrieving all columns for segment " + segmentId); RetrievalRuntimeConfig retrievalRuntimeConfig = Config.sharedConfig().getRetriever(); diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/TextRetrievalCommand.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/TextRetrievalCommand.java index a9c954726..4004d6fe0 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/TextRetrievalCommand.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/TextRetrievalCommand.java @@ -6,10 +6,8 @@ import java.util.List; import org.vitrivr.cineast.core.data.query.containers.TextQueryTermContainer; import org.vitrivr.cineast.core.features.AudioTranscriptionSearch; -import org.vitrivr.cineast.core.features.DescriptionTextSearch; import org.vitrivr.cineast.core.features.OCRSearch; import org.vitrivr.cineast.core.features.SubtitleFulltextSearch; -import org.vitrivr.cineast.core.features.TagsFtSearch; import org.vitrivr.cineast.core.features.retriever.Retriever; import org.vitrivr.cineast.standalone.config.Config; import org.vitrivr.cineast.standalone.util.ContinuousRetrievalLogic; diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/config/Config.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/config/Config.java index 485801759..1afb60ddd 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/config/Config.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/config/Config.java @@ -54,8 +54,9 @@ public static Config loadConfig(String name) { LOGGER.warn("Could not read config file '{}'.", name); return null; } else { - LOGGER.info("Config file loaded!"); + LOGGER.trace("Config file loaded!"); initSharedConfig(config); + LOGGER.trace("Config initialized!"); return config; } } diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/config/MetadataConfig.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/config/MetadataConfig.java index 370e23592..a4f691fad 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/config/MetadataConfig.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/config/MetadataConfig.java @@ -43,7 +43,6 @@ public void setProperties(HashMap properties) { @JsonIgnore public MetadataExtractor getMetadataExtractor() { - MetadataExtractor extractor = ReflectionHelper.newMetadataExtractor(this.name); - return extractor; + return ReflectionHelper.newMetadataExtractor(this.name); } } diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/monitoring/PrometheusServer.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/monitoring/PrometheusServer.java index 7e2dad6d4..bd14827fe 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/monitoring/PrometheusServer.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/monitoring/PrometheusServer.java @@ -32,7 +32,7 @@ public static synchronized void initialize() { return; } if (!Config.sharedConfig().getMonitoring().enablePrometheus) { - LOGGER.info("Prometheus monitoring not enabled"); + LOGGER.trace("Prometheus monitoring not enabled"); lock.release(); return; } diff --git a/cineast.json b/cineast.json index 577af8e98..ccbcab07b 100644 --- a/cineast.json +++ b/cineast.json @@ -15,25 +15,18 @@ "globalcolor": [ {"feature": "AverageColor", "weight": 2.3}, {"feature": "MedianColor", "weight": 1.2}, - {"feature": "AverageFuzzyHist", "weight": 0.7}, - {"feature": "HueHistogram", "weight": 0.7}, - {"feature": "MedianFuzzyHist", "weight": 1.3 } + {"feature": "AverageFuzzyHist", "weight": 0.7} ], "localcolor": [ {"feature": "AverageColorARP44", "weight": 0.5}, - {"feature": "AverageColorARP44Normalized", "weight": 0.5}, - {"feature": "SubDivMedianFuzzyColor", "weight": 0.85}, {"feature": "AverageColorGrid8", "weight": 1.8}, - {"feature": "AverageColorGrid8Normalized", "weight": 1.8}, {"feature": "CLD", "weight": 1.3}, - {"feature": "CLDNormalized", "weight": 1.3}, {"feature": "MedianColorGrid8", "weight": 1.7}, {"feature": "AverageColorRaster", "weight": 1.0} ], "edge": [ {"feature": "EHD", "weight": 0.7}, - {"feature": "DominantEdgeGrid16", "weight": 1.4}, - {"feature": "DominantEdgeGrid8", "weight": 1.4} + {"feature": "DominantEdgeGrid16", "weight": 1.4} ], "localfeatures": [ {"feature": "HOGMirflickr25K512", "weight": 1.0} @@ -106,9 +99,9 @@ "serveContent": true, "serveUI": false, "uiLocation": "../vitrivr-ng/dist", - "thumbnailLocation": "/Volumes/V3C1/V3C1/thumbnails", - "objectLocation": "/Volumes/V3C1/V3C1/videos", - "objectsFilesAreIDed": true + "thumbnailLocation": "./thumbnails", + "objectLocation": "./data/", + "objectsFilesAreIDed": false }, "monitoring":{ diff --git a/docs/openapi.json b/docs/openapi.json index 0c6cf3ac3..6e09facea 100644 --- a/docs/openapi.json +++ b/docs/openapi.json @@ -370,6 +370,7 @@ }, { "name" : "value", "in" : "path", + "description" : "The actual value that you want to filter for", "required" : true, "schema" : { "type" : "string" @@ -389,6 +390,45 @@ } } }, + "/api/v1/find/object/all/{skip}/{limit}" : { + "get" : { + "tags" : [ "Object" ], + "summary" : "Get a fixed amount of objects from the sorted list", + "description" : "Equivalent to calling SELECT * FROM multimediaobject ORDER BY objectid ASC LIMIT limit SKIP skip. Mostly used for pagination when wanting to retrieve all objects", + "operationId" : "findObjectsPagination", + "parameters" : [ { + "name" : "skip", + "in" : "path", + "description" : "How many objects should be skipped", + "required" : true, + "schema" : { + "type" : "integer", + "format" : "int32" + } + }, { + "name" : "limit", + "in" : "path", + "description" : "How many object at most should be fetched", + "required" : true, + "schema" : { + "type" : "integer", + "format" : "int32" + } + } ], + "responses" : { + "200" : { + "description" : "OK", + "content" : { + "application/json" : { + "schema" : { + "$ref" : "#/components/schemas/MediaObjectQueryResult" + } + } + } + } + } + } + }, "/api/v1/find/segments/by/id" : { "post" : { "tags" : [ "Segment" ], @@ -1030,6 +1070,34 @@ } } }, + "/api/v1/count/table/{table}" : { + "get" : { + "tags" : [ "Misc" ], + "summary" : "Count objects", + "description" : "Equivalent to calling SELECT count(*) FROM table. Used to determined #pages for pagination in a frontend or statistical purposes", + "operationId" : "countRows", + "parameters" : [ { + "name" : "table", + "in" : "path", + "required" : true, + "schema" : { + "type" : "string" + } + } ], + "responses" : { + "200" : { + "description" : "OK", + "content" : { + "application/json" : { + "schema" : { + "$ref" : "#/components/schemas/IntegerMessage" + } + } + } + } + } + } + }, "/api/v1/status" : { "get" : { "tags" : [ "Status" ], @@ -1094,8 +1162,7 @@ "type" : "object", "properties" : { "objectid" : { - "type" : "string", - "writeOnly" : true + "type" : "string" }, "domain" : { "type" : "string" @@ -1109,9 +1176,6 @@ "exists" : { "type" : "boolean", "writeOnly" : true - }, - "objectId" : { - "type" : "string" } } }, @@ -1387,6 +1451,9 @@ "type" : "string" } }, + "correspondenceFunctionIfEmpty" : { + "$ref" : "#/components/schemas/QueryConfig" + }, "distanceIfEmpty" : { "$ref" : "#/components/schemas/QueryConfig" }, @@ -1399,9 +1466,6 @@ "correspondenceFunction" : { "$ref" : "#/components/schemas/CorrespondenceFunction" }, - "correspondenceFunctionIfEmpty" : { - "$ref" : "#/components/schemas/QueryConfig" - }, "rawResultsPerModule" : { "type" : "integer", "format" : "int32" @@ -1561,6 +1625,14 @@ "$ref" : "#/components/schemas/MetadataAccessSpecification" } }, + "messageType" : { + "type" : "string", + "enum" : [ "PING", "Q_SIM", "Q_MLT", "Q_NESEG", "Q_SEG", "M_LOOKUP", "Q_TEMPORAL", "SESSION_START", "QR_START", "QR_END", "QR_ERROR", "QR_OBJECT", "QR_METADATA_O", "QR_METADATA_S", "QR_SEGMENT", "QR_SIMILARITY", "QR_TEMPORAL" ] + }, + "maxLength" : { + "type" : "number", + "format" : "float" + }, "timeDistances" : { "type" : "array", "items" : { @@ -1570,14 +1642,6 @@ }, "temporalQueryConfig" : { "$ref" : "#/components/schemas/TemporalQueryConfig" - }, - "messageType" : { - "type" : "string", - "enum" : [ "PING", "Q_SIM", "Q_MLT", "Q_NESEG", "Q_SEG", "M_LOOKUP", "Q_TEMPORAL", "SESSION_START", "QR_START", "QR_END", "QR_ERROR", "QR_OBJECT", "QR_METADATA_O", "QR_METADATA_S", "QR_SEGMENT", "QR_SIMILARITY", "QR_TEMPORAL" ] - }, - "maxLength" : { - "type" : "number", - "format" : "float" } } }, @@ -1639,6 +1703,9 @@ "type" : "string" } }, + "correspondenceFunctionIfEmpty" : { + "$ref" : "#/components/schemas/QueryConfig" + }, "distanceIfEmpty" : { "$ref" : "#/components/schemas/QueryConfig" }, @@ -1651,9 +1718,6 @@ "correspondenceFunction" : { "$ref" : "#/components/schemas/CorrespondenceFunction" }, - "correspondenceFunctionIfEmpty" : { - "$ref" : "#/components/schemas/QueryConfig" - }, "rawResultsPerModule" : { "type" : "integer", "format" : "int32" @@ -1964,6 +2028,15 @@ } } }, + "IntegerMessage" : { + "type" : "object", + "properties" : { + "value" : { + "type" : "integer", + "format" : "int32" + } + } + }, "Ping" : { "type" : "object", "properties" : { diff --git a/example_json_job.json b/example_json_job.json deleted file mode 100644 index 10f0a1b49..000000000 --- a/example_json_job.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "input":{ - "path": "data/", - "depth": 3, - "skip": 0, - "id": { - "name": "SequentialObjectIdGenerator", - "properties": {} - } - }, - "extractors":[ - {"name": "AverageColor"}, - {"name": "AverageColorARP44"}, - {"name": "AverageColorCLD"}, - {"name": "AverageColorGrid8"}, - {"name": "AverageColorRaster"}, - {"name": "AverageFuzzyHist"}, - {"name": "AverageColorGrid8Reduced15"}, - {"name": "CLD"}, - {"name": "EdgeARP88"}, - {"name": "EdgeGrid16"}, - {"name": "EHD"}, - {"name": "HPCP12Shingle"}, - {"name": "MedianColor"}, - {"name": "MedianColorGrid8"}, - {"name": "HOGMirflickr25K512"}, - {"name": "SURFMirflickr25K512"}, - {"name": "VisualTextCoEmbedding"} - ], - "exporters":[ - { - "name": "ShotThumbnailsExporter", - "properties": { - "destination":"thumbnails/" - } - } - ], - "database":{ - "host": "output/", - "writer": "JSON", - "selector": "NONE" - } -} diff --git a/extraction_config.json b/extraction_config.json index ce6666813..008d50d7f 100644 --- a/extraction_config.json +++ b/extraction_config.json @@ -1,17 +1,41 @@ { - "input": { - "path": "videos", - "depth": 1, + "input":{ + "path": "data/", + "depth": 3, + "skip": 0, "id": { "name": "SequentialObjectIdGenerator", "properties": {} } }, - "extractors": [ + "extractors":[ {"name": "AverageColor"}, + {"name": "AverageColorARP44"}, + {"name": "AverageColorGrid8"}, + {"name": "AverageColorRaster"}, + {"name": "AverageFuzzyHist"}, + {"name": "CLD"}, + {"name": "DominantEdgeGrid16"}, + {"name": "EHD"}, + {"name": "HPCP12Shingle"}, {"name": "MedianColor"}, + {"name": "MedianColorGrid8"}, {"name": "OCRSearch"}, - {"name": "SubtitleFulltextSearch"} + {"name": "HOGMirflickr25K512"}, + {"name": "SURFMirflickr25K512"}, + {"name": "VisualTextCoEmbedding"} + ], + "metadata": [ + {"name": "TechnicalVideoMetadataExtractor"}, + {"name": "EXIFMetadataExtractor"} + ], + "exporters":[ + { + "name": "ShotThumbnailsExporter", + "properties": { + "destination":"thumbnails/" + } + } ], "database": { "writer": "COTTONTAIL",