Skip to content

Commit

Permalink
feat: support order by random #658
Browse files Browse the repository at this point in the history
  • Loading branch information
@fengelniederhammer
fengelniederhammer committed Mar 5, 2024
1 parent 24f535c commit 598b05f
Show file tree
Hide file tree
Showing 8 changed files with 233 additions and 45 deletions.
5 changes: 5 additions & 0 deletions lapis2-docs/src/content/docs/references/additional-request-properties.mdx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,11 @@ POST /sample/aggregated
}
```

:::note
You can also specify `random` as the field name to get the results in a random order.
It will act as a tiebreaker if it is provided along with other `orderBy` fields.
:::

:::caution
LAPIS will throw an error if you try to order by a field that is not in the response.
:::
Expand Down
38 changes: 23 additions & 15 deletions lapis2/src/main/kotlin/org/genspectrum/lapis/openApi/OpenApiDocs.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package org.genspectrum.lapis.openApi

import io.swagger.v3.oas.models.Components
import io.swagger.v3.oas.models.OpenAPI
import io.swagger.v3.oas.models.media.ArraySchema
import io.swagger.v3.oas.models.media.BooleanSchema
import io.swagger.v3.oas.models.media.IntegerSchema
import io.swagger.v3.oas.models.media.Schema
Expand Down Expand Up @@ -43,6 +44,7 @@ import org.genspectrum.lapis.request.NucleotideInsertion
import org.genspectrum.lapis.request.NucleotideMutation
import org.genspectrum.lapis.request.OrderByField
import org.genspectrum.lapis.response.COUNT_PROPERTY
import org.genspectrum.lapis.silo.ORDER_BY_RANDOM_FIELD_NAME

fun buildOpenApiSchema(
sequenceFilterFields: SequenceFilterFields,
Expand Down Expand Up @@ -90,7 +92,7 @@ fun buildOpenApiSchema(
getSequenceFiltersWithFormat(
databaseConfig,
sequenceFilterFields,
fieldsEnum(databaseConfig.schema.metadata),
detailsOrderByFieldsEnum(databaseConfig),
),
DETAILS_FIELDS_DESCRIPTION,
databaseConfig.schema.metadata,
Expand All @@ -112,7 +114,7 @@ fun buildOpenApiSchema(
getSequenceFilters(
databaseConfig,
sequenceFilterFields,
aminoAcidSequenceFieldsEnum(referenceGenomeSchema, databaseConfig),
aminoAcidSequenceOrderByFieldsEnum(referenceGenomeSchema, databaseConfig),
),
),
)
Expand All @@ -122,7 +124,7 @@ fun buildOpenApiSchema(
getSequenceFilters(
databaseConfig,
sequenceFilterFields,
nucleotideSequenceFieldsEnum(referenceGenomeSchema, databaseConfig),
nucleotideSequenceOrderByFieldsEnum(referenceGenomeSchema, databaseConfig),
),
),
)
Expand Down Expand Up @@ -201,7 +203,7 @@ fun buildOpenApiSchema(
AGGREGATED_ORDER_BY_FIELDS_SCHEMA,
arraySchema(aggregatedOrderByFieldsEnum(databaseConfig)),
)
.addSchemas(DETAILS_ORDER_BY_FIELDS_SCHEMA, fieldsArray(databaseConfig.schema.metadata))
.addSchemas(DETAILS_ORDER_BY_FIELDS_SCHEMA, arraySchema(detailsOrderByFieldsEnum(databaseConfig)))
.addSchemas(
MUTATIONS_ORDER_BY_FIELDS_SCHEMA,
arraySchema(mutationsOrderByFieldsEnum()),
Expand All @@ -212,11 +214,11 @@ fun buildOpenApiSchema(
)
.addSchemas(
AMINO_ACID_SEQUENCES_ORDER_BY_FIELDS_SCHEMA,
arraySchema(aminoAcidSequenceFieldsEnum(referenceGenomeSchema, databaseConfig)),
arraySchema(aminoAcidSequenceOrderByFieldsEnum(referenceGenomeSchema, databaseConfig)),
)
.addSchemas(
NUCLEOTIDE_SEQUENCES_ORDER_BY_FIELDS_SCHEMA,
arraySchema(nucleotideSequenceFieldsEnum(referenceGenomeSchema, databaseConfig)),
arraySchema(nucleotideSequenceOrderByFieldsEnum(referenceGenomeSchema, databaseConfig)),
)
.addSchemas(
SEGMENT_SCHEMA,
Expand Down Expand Up @@ -500,27 +502,34 @@ private fun fieldsArray(
) = arraySchema(fieldsEnum(databaseConfig, additionalFields))

private fun aggregatedOrderByFieldsEnum(databaseConfig: DatabaseConfig) =
fieldsEnum(databaseConfig.schema.metadata, listOf("count"))
orderByFieldsEnum(databaseConfig.schema.metadata, listOf("count"))

private fun mutationsOrderByFieldsEnum() = fieldsEnum(emptyList(), listOf("mutation", "count", "proportion"))
private fun mutationsOrderByFieldsEnum() = orderByFieldsEnum(emptyList(), listOf("mutation", "count", "proportion"))

private fun insertionsOrderByFieldsEnum() = fieldsEnum(emptyList(), listOf("insertion", "count"))
private fun insertionsOrderByFieldsEnum() = orderByFieldsEnum(emptyList(), listOf("insertion", "count"))

private fun aminoAcidSequenceFieldsEnum(
private fun aminoAcidSequenceOrderByFieldsEnum(
referenceGenomeSchema: ReferenceGenomeSchema,
databaseConfig: DatabaseConfig,
) = fieldsEnum(emptyList(), referenceGenomeSchema.genes.map { it.name } + databaseConfig.schema.primaryKey)
) = orderByFieldsEnum(emptyList(), referenceGenomeSchema.genes.map { it.name } + databaseConfig.schema.primaryKey)

private fun nucleotideSequenceFieldsEnum(
private fun nucleotideSequenceOrderByFieldsEnum(
referenceGenomeSchema: ReferenceGenomeSchema,
databaseConfig: DatabaseConfig,
) = fieldsEnum(
) = orderByFieldsEnum(
emptyList(),
referenceGenomeSchema.nucleotideSequences.map {
it.name
} + databaseConfig.schema.primaryKey,
)

private fun detailsOrderByFieldsEnum(databaseConfig: DatabaseConfig) = orderByFieldsEnum(databaseConfig.schema.metadata)

private fun orderByFieldsEnum(
databaseConfig: List<DatabaseMetadata> = emptyList(),
additionalFields: List<String> = emptyList(),
) = fieldsEnum(databaseConfig, additionalFields + ORDER_BY_RANDOM_FIELD_NAME)

private fun fieldsEnum(
databaseConfig: List<DatabaseMetadata> = emptyList(),
additionalFields: List<String> = emptyList(),
Expand All @@ -529,6 +538,5 @@ private fun fieldsEnum(
._enum(databaseConfig.map { it.name } + additionalFields)

private fun arraySchema(schema: Schema<Any>) =
Schema<Any>()
.type("array")
ArraySchema()
.items(schema)
7 changes: 6 additions & 1 deletion lapis2/src/main/kotlin/org/genspectrum/lapis/scheduler/DataVersionCacheInvalidator.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,12 @@ class DataVersionCacheInvalidator(
fun invalidateSiloCache() {
log.debug { "checking for data version change" }

val info = cachedSiloClient.callInfo()
val info = try {
cachedSiloClient.callInfo()
} catch (e: Exception) {
log.debug { "Failed to call info: $e" }
return
}
if (info.dataVersion != currentlyCachedDataVersion) {
log.info {
"Invalidating cache, old data version: $currentlyCachedDataVersion, " +
Expand Down
2 changes: 1 addition & 1 deletion lapis2/src/main/kotlin/org/genspectrum/lapis/silo/SiloClient.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ class CachedSiloClient(
) {
private val httpClient = HttpClient.newHttpClient()

@Cacheable(SILO_QUERY_CACHE_NAME, condition = "#query.action.cacheable")
@Cacheable(SILO_QUERY_CACHE_NAME, condition = "#query.action.cacheable && !#query.action.randomize")
fun <ResponseType> sendQuery(query: SiloQuery<ResponseType>): WithDataVersion<ResponseType> {
val queryJson = objectMapper.writeValueAsString(query)

Expand Down
78 changes: 65 additions & 13 deletions lapis2/src/main/kotlin/org/genspectrum/lapis/silo/SiloQuery.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,11 @@ interface CommonActionFields {
val orderByFields: List<OrderByField>
val limit: Int?
val offset: Int?
val randomize: Boolean?
}

const val ORDER_BY_RANDOM_FIELD_NAME = "random"

sealed class SiloAction<ResponseType>(
@JsonIgnore val typeReference: TypeReference<SiloQueryResponse<ResponseType>>,
@JsonIgnore val cacheable: Boolean,
Expand All @@ -42,7 +45,14 @@ sealed class SiloAction<ResponseType>(
orderByFields: List<OrderByField> = emptyList(),
limit: Int? = null,
offset: Int? = null,
): SiloAction<List<AggregationData>> = AggregatedAction(groupByFields, orderByFields, limit, offset)
): SiloAction<List<AggregationData>> =
AggregatedAction(
groupByFields = groupByFields,
orderByFields = getNonRandomizedOrderByFields(orderByFields),
limit = limit,
offset = offset,
randomize = getRandomize(orderByFields),
)

fun mutations(
minProportion: Double? = null,
Expand All @@ -51,10 +61,11 @@ sealed class SiloAction<ResponseType>(
offset: Int? = null,
): SiloAction<List<MutationData>> =
MutationsAction(
minProportion,
orderByFields,
limit,
offset,
minProportion = minProportion,
orderByFields = getNonRandomizedOrderByFields(orderByFields),
limit = limit,
offset = offset,
randomize = getRandomize(orderByFields),
)

fun aminoAcidMutations(
Expand All @@ -64,44 +75,79 @@ sealed class SiloAction<ResponseType>(
offset: Int? = null,
): SiloAction<List<MutationData>> =
AminoAcidMutationsAction(
minProportion,
orderByFields,
limit,
offset,
minProportion = minProportion,
orderByFields = getNonRandomizedOrderByFields(orderByFields),
limit = limit,
offset = offset,
randomize = getRandomize(orderByFields),
)

fun details(
fields: List<String> = emptyList(),
orderByFields: List<OrderByField> = emptyList(),
limit: Int? = null,
offset: Int? = null,
): SiloAction<List<DetailsData>> = DetailsAction(fields, orderByFields, limit, offset)
): SiloAction<List<DetailsData>> =
DetailsAction(
fields = fields,
orderByFields = getNonRandomizedOrderByFields(orderByFields),
limit = limit,
offset = offset,
randomize = getRandomize(orderByFields),
)

fun nucleotideInsertions(
orderByFields: List<OrderByField> = emptyList(),
limit: Int? = null,
offset: Int? = null,
): SiloAction<List<InsertionData>> = NucleotideInsertionsAction(orderByFields, limit, offset)
): SiloAction<List<InsertionData>> =
NucleotideInsertionsAction(
orderByFields = getNonRandomizedOrderByFields(orderByFields),
limit = limit,
offset = offset,
randomize = getRandomize(orderByFields),
)

fun aminoAcidInsertions(
orderByFields: List<OrderByField> = emptyList(),
limit: Int? = null,
offset: Int? = null,
): SiloAction<List<InsertionData>> = AminoAcidInsertionsAction(orderByFields, limit, offset)
): SiloAction<List<InsertionData>> =
AminoAcidInsertionsAction(
orderByFields = getNonRandomizedOrderByFields(orderByFields),
limit = limit,
offset = offset,
randomize = getRandomize(orderByFields),
)

fun genomicSequence(
type: SequenceType,
sequenceName: String,
orderByFields: List<OrderByField> = emptyList(),
limit: Int? = null,
offset: Int? = null,
): SiloAction<List<SequenceData>> = SequenceAction(orderByFields, limit, offset, type, sequenceName)
): SiloAction<List<SequenceData>> =
SequenceAction(
type = type,
sequenceName = sequenceName,
orderByFields = getNonRandomizedOrderByFields(orderByFields),
limit = limit,
offset = offset,
randomize = getRandomize(orderByFields),
)

private fun getRandomize(orderByFields: List<OrderByField>) =
orderByFields.any { it.field == ORDER_BY_RANDOM_FIELD_NAME }

private fun getNonRandomizedOrderByFields(orderByFields: List<OrderByField>) =
orderByFields.filter { it.field != ORDER_BY_RANDOM_FIELD_NAME }
}

@JsonInclude(JsonInclude.Include.NON_EMPTY)
private data class AggregatedAction(
val groupByFields: List<String>,
override val orderByFields: List<OrderByField> = emptyList(),
override val randomize: Boolean? = null,
override val limit: Int? = null,
override val offset: Int? = null,
val type: String = "Aggregated",
Expand All @@ -111,6 +157,7 @@ sealed class SiloAction<ResponseType>(
private data class MutationsAction(
val minProportion: Double?,
override val orderByFields: List<OrderByField> = emptyList(),
override val randomize: Boolean? = null,
override val limit: Int? = null,
override val offset: Int? = null,
val type: String = "Mutations",
Expand All @@ -120,6 +167,7 @@ sealed class SiloAction<ResponseType>(
private data class AminoAcidMutationsAction(
val minProportion: Double?,
override val orderByFields: List<OrderByField> = emptyList(),
override val randomize: Boolean? = null,
override val limit: Int? = null,
override val offset: Int? = null,
val type: String = "AminoAcidMutations",
Expand All @@ -129,6 +177,7 @@ sealed class SiloAction<ResponseType>(
private data class DetailsAction(
val fields: List<String> = emptyList(),
override val orderByFields: List<OrderByField> = emptyList(),
override val randomize: Boolean? = null,
override val limit: Int? = null,
override val offset: Int? = null,
val type: String = "Details",
Expand All @@ -137,6 +186,7 @@ sealed class SiloAction<ResponseType>(
@JsonInclude(JsonInclude.Include.NON_EMPTY)
private data class NucleotideInsertionsAction(
override val orderByFields: List<OrderByField> = emptyList(),
override val randomize: Boolean? = null,
override val limit: Int? = null,
override val offset: Int? = null,
val type: String = "Insertions",
Expand All @@ -145,6 +195,7 @@ sealed class SiloAction<ResponseType>(
@JsonInclude(JsonInclude.Include.NON_EMPTY)
private data class AminoAcidInsertionsAction(
override val orderByFields: List<OrderByField> = emptyList(),
override val randomize: Boolean? = null,
override val limit: Int? = null,
override val offset: Int? = null,
val type: String = "AminoAcidInsertions",
Expand All @@ -153,6 +204,7 @@ sealed class SiloAction<ResponseType>(
@JsonInclude(JsonInclude.Include.NON_EMPTY)
private data class SequenceAction(
override val orderByFields: List<OrderByField> = emptyList(),
override val randomize: Boolean? = null,
override val limit: Int? = null,
override val offset: Int? = null,
val type: SequenceType,
Expand Down
32 changes: 32 additions & 0 deletions lapis2/src/test/kotlin/org/genspectrum/lapis/silo/SiloClientTest.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@ import com.fasterxml.jackson.databind.node.DoubleNode
import com.fasterxml.jackson.databind.node.IntNode
import com.fasterxml.jackson.databind.node.TextNode
import org.genspectrum.lapis.logging.RequestIdContext
import org.genspectrum.lapis.request.Order
import org.genspectrum.lapis.request.OrderByField
import org.genspectrum.lapis.response.AggregationData
import org.genspectrum.lapis.response.DetailsData
import org.genspectrum.lapis.response.MutationData
Expand Down Expand Up @@ -418,6 +420,36 @@ class SiloClientTest(
assertThat(dataVersion.dataVersion, `is`(dataVersionValue))
}

@Test
fun `GIVEN a cacheable action with randomize=true THEN is not cached`() {
val errorMessage = "This error should appear"
expectQueryRequestAndRespondWith(
response()
.withStatusCode(200)
.withBody("""{"queryResult": []}"""),
Times.once(),
)
expectQueryRequestAndRespondWith(
response()
.withStatusCode(500)
.withBody(errorMessage),
Times.exactly(1),
)

val orderByRandom = OrderByField(
ORDER_BY_RANDOM_FIELD_NAME,
Order.ASCENDING,
)
val query = SiloQuery(SiloAction.mutations(orderByFields = listOf(orderByRandom)), True)
assertThat(query.action.cacheable, `is`(true))

val result = underTest.sendQuery(query)
assertThat(result, hasSize(0))

val exception = assertThrows<SiloException> { underTest.sendQuery(query) }
assertThat(exception.message, containsString(errorMessage))
}

companion object {
@JvmStatic
val queriesThatShouldNotBeCached = listOf(
Expand Down
Loading

0 comments on commit 598b05f

Please sign in to comment.