diff --git a/lapis2-docs/src/content/docs/references/additional-request-properties.mdx b/lapis2-docs/src/content/docs/references/additional-request-properties.mdx index 118e90b31..4cc658cc4 100644 --- a/lapis2-docs/src/content/docs/references/additional-request-properties.mdx +++ b/lapis2-docs/src/content/docs/references/additional-request-properties.mdx @@ -92,7 +92,16 @@ GET /sample/aggregated?downloadAsFile=true ## Compression LAPIS supports gzip and Zstd compression. -You can request compressed data via the `Accept-Encoding` header. +You can request compressed data by setting the `compression` property in the request. +Refer to the Swagger UI for allowed values. + +```http +GET /sample/aggregated?compression=gzip +``` + +:::note + +Alternatively, you can set the `Accept-Encoding` header. ```http GET /sample/aggregated @@ -106,12 +115,11 @@ Accept-Encoding: zstd LAPIS will set the `Content-Encoding` header in the response to indicate the compression used. -:::note -Alternatively, you can use the `compression` property in the request. -Refer to the Swagger UI for allowed values. - -```http -GET /sample/aggregated?compression=gzip -``` +:::caution +If you want to download compressed data via the `downloadAsFile` property, +then you need to specify the compression type via the `compression` property. +LAPIS will ignore the `Accept-Encoding` header, if `downloadAsFile == true`, +since browsers always accept GZIP encoding. +Then, you would not be able to download uncompressed data in a browser. ::: diff --git a/lapis2/src/main/kotlin/org/genspectrum/lapis/controller/CompressionFilter.kt b/lapis2/src/main/kotlin/org/genspectrum/lapis/controller/CompressionFilter.kt index 264758d93..db7f7eb19 100644 --- a/lapis2/src/main/kotlin/org/genspectrum/lapis/controller/CompressionFilter.kt +++ b/lapis2/src/main/kotlin/org/genspectrum/lapis/controller/CompressionFilter.kt @@ -36,11 +36,13 @@ enum class Compression( val value: String, val contentType: MediaType, val compressionOutputStreamFactory: (OutputStream) -> OutputStream, + val fileEnding: String, ) { GZIP( value = "gzip", contentType = MediaType.parseMediaType("application/gzip"), compressionOutputStreamFactory = ::LazyGzipOutputStream, + fileEnding = ".gz", ), ZSTD( value = "zstd", @@ -48,6 +50,7 @@ enum class Compression( compressionOutputStreamFactory = { ZstdOutputStream(it).apply { commitUnderlyingResponseToPreventContentLengthFromBeingSet() } }, + fileEnding = ".zst", ), ; @@ -58,6 +61,8 @@ enum class Compression( } val headersList = acceptEncodingHeaders.toList() + .flatMap { it.split(',') } + .map { it.trim() } return when { headersList.contains(GZIP.value) -> GZIP @@ -140,7 +145,7 @@ class CompressionFilter(val objectMapper: ObjectMapper, val requestCompression: val maybeCompressingResponse = createMaybeCompressingResponse( response, - reReadableRequest.getHeaders(ACCEPT_ENCODING), + reReadableRequest, compressionPropertyInRequest, ) @@ -162,7 +167,7 @@ class CompressionFilter(val objectMapper: ObjectMapper, val requestCompression: private fun createMaybeCompressingResponse( response: HttpServletResponse, - acceptEncodingHeaders: Enumeration?, + reReadableRequest: CachedBodyHttpServletRequest, compressionPropertyInRequest: Compression?, ): HttpServletResponse { if (compressionPropertyInRequest != null) { @@ -175,6 +180,15 @@ class CompressionFilter(val objectMapper: ObjectMapper, val requestCompression: ) } + val downloadAsFile = reReadableRequest.getBooleanField(DOWNLOAD_AS_FILE_PROPERTY) ?: false + if (downloadAsFile) { + return response + } + if (!reReadableRequest.getProxyAwarePath().startsWith("/sample")) { + return response + } + + val acceptEncodingHeaders = reReadableRequest.getHeaders(ACCEPT_ENCODING) val compression = Compression.fromHeaders(acceptEncodingHeaders) ?: return response log.info { "Compressing using $compression from $ACCEPT_ENCODING header" } diff --git a/lapis2/src/main/kotlin/org/genspectrum/lapis/controller/DownloadAsFileFilter.kt b/lapis2/src/main/kotlin/org/genspectrum/lapis/controller/DownloadAsFileFilter.kt index 00e36ed12..70ef7e0c1 100644 --- a/lapis2/src/main/kotlin/org/genspectrum/lapis/controller/DownloadAsFileFilter.kt +++ b/lapis2/src/main/kotlin/org/genspectrum/lapis/controller/DownloadAsFileFilter.kt @@ -9,14 +9,16 @@ import org.genspectrum.lapis.controller.LapisMediaType.TEXT_TSV import org.genspectrum.lapis.util.CachedBodyHttpServletRequest import org.springframework.core.annotation.Order import org.springframework.http.HttpHeaders.ACCEPT -import org.springframework.http.HttpHeaders.ACCEPT_ENCODING import org.springframework.http.HttpHeaders.CONTENT_DISPOSITION import org.springframework.stereotype.Component import org.springframework.web.filter.OncePerRequestFilter @Component @Order(DOWNLOAD_AS_FILE_FILTER_ORDER) -class DownloadAsFileFilter(private val objectMapper: ObjectMapper) : OncePerRequestFilter() { +class DownloadAsFileFilter( + private val objectMapper: ObjectMapper, + private val requestCompression: RequestCompression, +) : OncePerRequestFilter() { override fun doFilterInternal( request: HttpServletRequest, response: HttpServletResponse, @@ -37,10 +39,9 @@ class DownloadAsFileFilter(private val objectMapper: ObjectMapper) : OncePerRequ SampleRoute.entries.find { request.getProxyAwarePath().startsWith("/sample${it.pathSegment}") } val dataName = matchingRoute?.pathSegment?.trim('/') ?: "data" - val compressionEnding = when (Compression.fromHeaders(request.getHeaders(ACCEPT_ENCODING))) { - Compression.GZIP -> ".gzip" - Compression.ZSTD -> ".zstd" - null -> "" + val compressionEnding = when (val compressionSource = requestCompression.compressionSource) { + is CompressionSource.RequestProperty -> compressionSource.compression.fileEnding + else -> "" } val fileEnding = when (request.getHeader(ACCEPT)) { diff --git a/lapis2/src/main/kotlin/org/genspectrum/lapis/controller/ErrorController.kt b/lapis2/src/main/kotlin/org/genspectrum/lapis/controller/ErrorController.kt deleted file mode 100644 index 39c675c6d..000000000 --- a/lapis2/src/main/kotlin/org/genspectrum/lapis/controller/ErrorController.kt +++ /dev/null @@ -1,69 +0,0 @@ -package org.genspectrum.lapis.controller - -import jakarta.servlet.http.HttpServletRequest -import jakarta.servlet.http.HttpServletResponse -import mu.KotlinLogging -import org.genspectrum.lapis.config.DatabaseConfig -import org.springframework.boot.autoconfigure.web.ErrorProperties -import org.springframework.boot.autoconfigure.web.servlet.error.BasicErrorController -import org.springframework.boot.web.servlet.error.ErrorAttributes -import org.springframework.http.MediaType -import org.springframework.stereotype.Component -import org.springframework.web.bind.annotation.RequestMapping -import org.springframework.web.servlet.ModelAndView -import org.springframework.web.servlet.View -import org.springframework.web.servlet.support.ServletUriComponentsBuilder - -private val log = KotlinLogging.logger { } - -@Component -class ErrorController(private val databaseConfig: DatabaseConfig, errorAttributes: ErrorAttributes) : - BasicErrorController(errorAttributes, ErrorProperties()) { - @RequestMapping(produces = [MediaType.TEXT_HTML_VALUE]) - override fun errorHtml( - request: HttpServletRequest, - response: HttpServletResponse, - ): ModelAndView { - val modelAndView = super.errorHtml(request, response) - - response.addHeader("Content-Type", MediaType.TEXT_HTML_VALUE) - - val urlPrefix = removeErrorSegmentFromUrl(ServletUriComponentsBuilder.fromCurrentRequest().toUriString()) - val url = "$urlPrefix/swagger-ui/index.html" - - log.debug { "Generated url $url to Swagger UI in 'not found page'" } - - modelAndView.view = NotFoundView(databaseConfig.schema.instanceName, url) - return modelAndView - } - - fun removeErrorSegmentFromUrl(url: String): String { - val lastSlashIndex = url.trimEnd('/').lastIndexOf("error") - return url.substring(0, lastSlashIndex).trim('/') - } -} - -data class NotFoundView(private val instanceName: String, private val url: String?) : View { - override fun render( - model: MutableMap?, - request: HttpServletRequest, - response: HttpServletResponse, - ) { - val html: String = """ - - - - - Error 404 - - -

LAPIS - $instanceName

-

Page not found!

- Visit our swagger-ui - - - """.trimIndent() - - response.outputStream.write(html.toByteArray()) - } -} diff --git a/lapis2/src/main/kotlin/org/genspectrum/lapis/controller/ExceptionHandler.kt b/lapis2/src/main/kotlin/org/genspectrum/lapis/controller/ExceptionHandler.kt index 31915ea81..5632bf5b7 100644 --- a/lapis2/src/main/kotlin/org/genspectrum/lapis/controller/ExceptionHandler.kt +++ b/lapis2/src/main/kotlin/org/genspectrum/lapis/controller/ExceptionHandler.kt @@ -1,10 +1,12 @@ package org.genspectrum.lapis.controller import mu.KotlinLogging +import org.genspectrum.lapis.config.DatabaseConfig import org.genspectrum.lapis.model.SiloNotImplementedError import org.genspectrum.lapis.silo.SiloException import org.genspectrum.lapis.silo.SiloUnavailableException -import org.springframework.core.annotation.Order +import org.springframework.http.HttpHeaders +import org.springframework.http.HttpHeaders.ACCEPT import org.springframework.http.HttpHeaders.RETRY_AFTER import org.springframework.http.HttpStatus import org.springframework.http.HttpStatusCode @@ -12,38 +14,21 @@ import org.springframework.http.MediaType import org.springframework.http.ProblemDetail import org.springframework.http.ResponseEntity import org.springframework.http.ResponseEntity.BodyBuilder +import org.springframework.stereotype.Component import org.springframework.web.bind.annotation.ControllerAdvice import org.springframework.web.bind.annotation.ExceptionHandler import org.springframework.web.bind.annotation.ResponseStatus +import org.springframework.web.context.request.WebRequest import org.springframework.web.servlet.mvc.method.annotation.ResponseEntityExceptionHandler import org.springframework.web.servlet.resource.NoResourceFoundException +import org.springframework.web.servlet.support.ServletUriComponentsBuilder private val log = KotlinLogging.logger {} private typealias ErrorResponse = ResponseEntity -/** - * Taken from https://github.com/spring-projects/spring-framework/issues/31569#issuecomment-1825444419 - * Due to https://github.com/spring-projects/spring-framework/commit/c00508d6cf2408d06a0447ed193ad96466d0d7b4 - * - * This forwards "404" errors to the ErrorController to allow it to return a view. - * Thus, browsers get their own error page. - * - * Spring reworked handling of "404 not found" errors. This was introduced with the upgrade to Spring boot 3.2.0. - * This can be removed/reworked once Spring decides on how to return a view from an ExceptionHandler - * or allows them to respect the Accept header. - */ @ControllerAdvice -@Order(-1) -internal class ExceptionToErrorControllerBypass { - @ExceptionHandler(NoResourceFoundException::class) - fun handleResourceNotFound(e: Exception): Nothing { - throw e - } -} - -@ControllerAdvice -class ExceptionHandler : ResponseEntityExceptionHandler() { +class ExceptionHandler(private val notFoundView: NotFoundView) : ResponseEntityExceptionHandler() { @ExceptionHandler(Throwable::class) @ResponseStatus(HttpStatus.INTERNAL_SERVER_ERROR) fun handleUnexpectedException(e: Throwable): ErrorResponse { @@ -89,6 +74,23 @@ class ExceptionHandler : ResponseEntityExceptionHandler() { return responseEntity(HttpStatus.SERVICE_UNAVAILABLE, e.message) { header(RETRY_AFTER, e.retryAfter) } } + override fun handleNoResourceFoundException( + ex: NoResourceFoundException, + headers: HttpHeaders, + status: HttpStatusCode, + request: WebRequest, + ): ResponseEntity? { + val acceptMediaTypes = MediaType.parseMediaTypes(request.getHeaderValues(ACCEPT)?.toList()) + if (MediaType.TEXT_HTML.isPresentIn(acceptMediaTypes)) { + return ResponseEntity + .status(HttpStatus.NOT_FOUND) + .contentType(MediaType.TEXT_HTML) + .body(notFoundView.render()) + } + + return super.handleNoResourceFoundException(ex, headers, status, request) + } + private fun responseEntity( httpStatus: HttpStatus, detail: String?, @@ -123,6 +125,32 @@ class ExceptionHandler : ResponseEntityExceptionHandler() { } } +@Component +class NotFoundView(private val databaseConfig: DatabaseConfig) { + fun render(): String { + val uri = ServletUriComponentsBuilder.fromCurrentRequest() + .replacePath("/swagger-ui/index.html") + .replaceQuery(null) + .fragment(null) + .toUriString() + + return """ + + + + + Error 404 + + +

LAPIS - ${databaseConfig.schema.instanceName}

+

Page not found!

+ Visit our swagger-ui + + + """.trimIndent() + } +} + /** This is not yet actually thrown, but makes "403 Forbidden" appear in OpenAPI docs. */ class AddForbiddenToOpenApiDocsHelper(message: String) : Exception(message) diff --git a/lapis2/src/test/kotlin/org/genspectrum/lapis/controller/LapisControllerCommonFieldsTest.kt b/lapis2/src/test/kotlin/org/genspectrum/lapis/controller/LapisControllerCommonFieldsTest.kt index 0b13e2100..75f8c6da5 100644 --- a/lapis2/src/test/kotlin/org/genspectrum/lapis/controller/LapisControllerCommonFieldsTest.kt +++ b/lapis2/src/test/kotlin/org/genspectrum/lapis/controller/LapisControllerCommonFieldsTest.kt @@ -6,14 +6,7 @@ import io.mockk.every import org.genspectrum.lapis.FIELD_WITH_ONLY_LOWERCASE_LETTERS import org.genspectrum.lapis.FIELD_WITH_UPPERCASE_LETTER import org.genspectrum.lapis.controller.SampleRoute.AGGREGATED -import org.genspectrum.lapis.controller.SampleRoute.ALIGNED_AMINO_ACID_SEQUENCES -import org.genspectrum.lapis.controller.SampleRoute.ALIGNED_NUCLEOTIDE_SEQUENCES -import org.genspectrum.lapis.controller.SampleRoute.AMINO_ACID_INSERTIONS -import org.genspectrum.lapis.controller.SampleRoute.AMINO_ACID_MUTATIONS import org.genspectrum.lapis.controller.SampleRoute.DETAILS -import org.genspectrum.lapis.controller.SampleRoute.NUCLEOTIDE_INSERTIONS -import org.genspectrum.lapis.controller.SampleRoute.NUCLEOTIDE_MUTATIONS -import org.genspectrum.lapis.controller.SampleRoute.UNALIGNED_NUCLEOTIDE_SEQUENCES import org.genspectrum.lapis.model.SiloQueryModel import org.genspectrum.lapis.request.AminoAcidInsertion import org.genspectrum.lapis.request.LapisInfo @@ -32,13 +25,8 @@ import org.junit.jupiter.params.provider.MethodSource import org.springframework.beans.factory.annotation.Autowired import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc import org.springframework.boot.test.context.SpringBootTest -import org.springframework.http.HttpHeaders.ACCEPT -import org.springframework.http.HttpHeaders.ACCEPT_ENCODING import org.springframework.http.MediaType.APPLICATION_JSON import org.springframework.test.web.servlet.MockMvc -import org.springframework.test.web.servlet.ResultActions -import org.springframework.test.web.servlet.request.MockHttpServletRequestBuilder -import org.springframework.test.web.servlet.result.MockMvcResultMatchers.header import org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath import org.springframework.test.web.servlet.result.MockMvcResultMatchers.status @@ -398,67 +386,6 @@ class LapisControllerCommonFieldsTest( .andExpect(jsonPath("\$.detail").value(Matchers.containsString("Failed to convert 'aminoAcidInsertions'"))) } - @ParameterizedTest(name = "GET data from {0} as file") - @MethodSource("getDownloadAsFileScenarios") - fun `GET data as file`(scenario: DownloadAsFileScenario) { - scenario.mockData.mockWithData(siloQueryModelMock) - - var queryString = "$DOWNLOAD_AS_FILE_PROPERTY=true" - if (scenario.requestedDataFormat != null) { - queryString += "&$FORMAT_PROPERTY=${scenario.requestedDataFormat}" - } - - mockMvc.perform(getSample("${scenario.endpoint}?$queryString")) - .andExpect(status().isOk) - .andExpectAttachmentWithContent( - expectedFilename = scenario.expectedFilename, - assertFileContentMatches = scenario.mockData.assertDataMatches, - ) - } - - @ParameterizedTest(name = "POST data from {0} as file") - @MethodSource("getDownloadAsFileScenarios") - fun `POST data as file`(scenario: DownloadAsFileScenario) { - scenario.mockData.mockWithData(siloQueryModelMock) - - val maybeDataFormat = when { - scenario.requestedDataFormat != null -> """, "$FORMAT_PROPERTY": "${scenario.requestedDataFormat}" """ - else -> "" - } - val request = """{ "$DOWNLOAD_AS_FILE_PROPERTY": true $maybeDataFormat }""" - - mockMvc.perform(postSample(scenario.endpoint).content(request).contentType(APPLICATION_JSON)) - .andExpect(status().isOk) - .andExpectAttachmentWithContent( - expectedFilename = scenario.expectedFilename, - assertFileContentMatches = scenario.mockData.assertDataMatches, - ) - } - - @ParameterizedTest(name = "{0} should return compressed file") - @MethodSource("getCompressedFileScenarios") - fun `WHEN I request compressed files THEN the filenames have a corresponding suffix`( - scenario: DownloadCompressedFileScenario, - ) { - scenario.mockData.mockToReturnEmptyData(siloQueryModelMock) - - mockMvc.perform(scenario.request) - .andExpect(status().isOk) - .andExpect(header().string("Content-Disposition", attachmentWithFilename(scenario.expectedFilename))) - .andExpect(header().string("Content-Encoding", scenario.compressionFormat)) - } - - private fun ResultActions.andExpectAttachmentWithContent( - expectedFilename: String, - assertFileContentMatches: (String) -> Unit, - ) { - this.andExpect(header().string("Content-Disposition", attachmentWithFilename(expectedFilename))) - .andReturn() - .response - .contentAsString - .apply(assertFileContentMatches) - } - @ParameterizedTest(name = "GET {0} with non existing field should throw") @MethodSource("getEndpointsWithFields") fun `GET with non existing field should throw`(endpoint: String) { @@ -471,8 +398,6 @@ class LapisControllerCommonFieldsTest( ) } - private fun attachmentWithFilename(filename: String) = "attachment; filename=$filename" - private companion object { val endpointsOfController = SampleRoute.entries .map { it.pathSegment } @@ -498,164 +423,5 @@ class LapisControllerCommonFieldsTest( @JvmStatic fun getEndpointsWithFields() = listOf(AGGREGATED, DETAILS).map { it.pathSegment }.map { Arguments.of(it) } - - @JvmStatic - val downloadAsFileScenarios = SampleRoute.entries.flatMap { DownloadAsFileScenario.forEndpoint(it) } - - private val dataFormatsSequence = generateSequence { - listOf( - MockDataCollection.DataFormat.PLAIN_JSON, - MockDataCollection.DataFormat.CSV, - MockDataCollection.DataFormat.TSV, - ) - }.flatten() - - @JvmStatic - val compressedFileScenarios = dataFormatsSequence.zip(SampleRoute.entries.asSequence()) - .flatMap { (dataFormat, route) -> DownloadCompressedFileScenario.scenariosFor(dataFormat, route) } - .toList() - } -} - -fun SampleRoute.getExpectedFilename() = - when (this) { - AGGREGATED -> "aggregated" - DETAILS -> "details" - NUCLEOTIDE_MUTATIONS -> "nucleotideMutations" - AMINO_ACID_MUTATIONS -> "aminoAcidMutations" - NUCLEOTIDE_INSERTIONS -> "nucleotideInsertions" - AMINO_ACID_INSERTIONS -> "aminoAcidInsertions" - ALIGNED_NUCLEOTIDE_SEQUENCES -> "alignedNucleotideSequences" - ALIGNED_AMINO_ACID_SEQUENCES -> "alignedAminoAcidSequences" - UNALIGNED_NUCLEOTIDE_SEQUENCES -> "unalignedNucleotideSequences" - } - -data class DownloadAsFileScenario( - val endpoint: String, - val mockData: MockData, - val requestedDataFormat: String?, - val expectedFilename: String, -) { - override fun toString() = - when (requestedDataFormat) { - null -> endpoint - else -> "$endpoint as $requestedDataFormat" - } - - companion object { - fun forEndpoint(route: SampleRoute): List { - val expectedFilename = route.getExpectedFilename() - - if (route.servesFasta) { - return listOf( - DownloadAsFileScenario( - endpoint = "${route.pathSegment}/segmentName", - mockData = MockDataForEndpoints.fastaMockData, - requestedDataFormat = null, - expectedFilename = "$expectedFilename.fasta", - ), - ) - } - - return forDataFormats(route.pathSegment, expectedFilename) - } - - private fun forDataFormats( - endpoint: String, - expectedFilename: String, - ) = listOf( - DownloadAsFileScenario( - mockData = MockDataForEndpoints.getMockData(endpoint) - .expecting(MockDataCollection.DataFormat.PLAIN_JSON), - expectedFilename = "$expectedFilename.json", - endpoint = endpoint, - requestedDataFormat = "json", - ), - DownloadAsFileScenario( - mockData = MockDataForEndpoints.getMockData(endpoint).expecting(MockDataCollection.DataFormat.CSV), - expectedFilename = "$expectedFilename.csv", - endpoint = endpoint, - requestedDataFormat = "csv", - ), - DownloadAsFileScenario( - mockData = MockDataForEndpoints.getMockData(endpoint).expecting(MockDataCollection.DataFormat.TSV), - expectedFilename = "$expectedFilename.tsv", - endpoint = endpoint, - requestedDataFormat = "tsv", - ), - ) - } -} - -data class DownloadCompressedFileScenario( - val description: String, - val mockData: MockData, - val request: MockHttpServletRequestBuilder, - val expectedFilename: String, - val compressionFormat: String, -) { - override fun toString() = description - - companion object { - fun scenariosFor( - dataFormat: MockDataCollection.DataFormat, - route: SampleRoute, - ) = scenariosFor( - dataFormat = dataFormat, - route = route, - compressionFormat = "gzip", - ) + - scenariosFor( - dataFormat = dataFormat, - route = route, - compressionFormat = "zstd", - ) - - private fun scenariosFor( - dataFormat: MockDataCollection.DataFormat, - route: SampleRoute, - compressionFormat: String, - ): List { - val (mockData, dataFileFormat) = if (route.servesFasta) { - MockDataForEndpoints.fastaMockData to "fasta" - } else { - MockDataForEndpoints.getMockData(route.pathSegment).expecting(dataFormat) to dataFormat.fileFormat - } - - val endpoint = when (route) { - ALIGNED_NUCLEOTIDE_SEQUENCES -> "${route.pathSegment}/main" - ALIGNED_AMINO_ACID_SEQUENCES -> "${route.pathSegment}/main" - UNALIGNED_NUCLEOTIDE_SEQUENCES -> "${route.pathSegment}/gene1" - else -> route.pathSegment - } - val acceptHeader = when (route.servesFasta) { - true -> "*/*" - false -> dataFormat.acceptHeader - } - - val expectedFilename = "${route.getExpectedFilename()}.$dataFileFormat.$compressionFormat" - - return listOf( - DownloadCompressedFileScenario( - description = "GET $endpoint as $compressionFormat ${dataFormat.fileFormat}", - mockData = mockData, - request = getSample("$endpoint?$DOWNLOAD_AS_FILE_PROPERTY=true") - .header(ACCEPT_ENCODING, compressionFormat) - .header(ACCEPT, acceptHeader), - expectedFilename = expectedFilename, - compressionFormat = compressionFormat, - ), - DownloadCompressedFileScenario( - description = "POST $endpoint as $compressionFormat ${dataFormat.fileFormat}", - mockData = mockData, - request = postSample(endpoint).content("""{ "$DOWNLOAD_AS_FILE_PROPERTY": true }""") - .contentType(APPLICATION_JSON) - .header(ACCEPT_ENCODING, compressionFormat) - .header(ACCEPT, acceptHeader), - expectedFilename = expectedFilename, - compressionFormat = compressionFormat, - ), - ) - } } } diff --git a/lapis2/src/test/kotlin/org/genspectrum/lapis/controller/LapisControllerCompressionTest.kt b/lapis2/src/test/kotlin/org/genspectrum/lapis/controller/LapisControllerCompressionTest.kt index f4be08301..2868e6cb4 100644 --- a/lapis2/src/test/kotlin/org/genspectrum/lapis/controller/LapisControllerCompressionTest.kt +++ b/lapis2/src/test/kotlin/org/genspectrum/lapis/controller/LapisControllerCompressionTest.kt @@ -7,6 +7,10 @@ import io.mockk.every import org.genspectrum.lapis.controller.LapisMediaType.TEXT_CSV import org.genspectrum.lapis.controller.LapisMediaType.TEXT_TSV import org.genspectrum.lapis.controller.LapisMediaType.TEXT_X_FASTA +import org.genspectrum.lapis.controller.MockDataCollection.DataFormat.CSV +import org.genspectrum.lapis.controller.MockDataCollection.DataFormat.NESTED_JSON +import org.genspectrum.lapis.controller.MockDataCollection.DataFormat.PLAIN_JSON +import org.genspectrum.lapis.controller.MockDataCollection.DataFormat.TSV import org.genspectrum.lapis.controller.SampleRoute.AGGREGATED import org.genspectrum.lapis.controller.SampleRoute.ALIGNED_AMINO_ACID_SEQUENCES import org.genspectrum.lapis.controller.SampleRoute.ALIGNED_NUCLEOTIDE_SEQUENCES @@ -17,6 +21,7 @@ import org.hamcrest.MatcherAssert.assertThat import org.hamcrest.Matchers.containsString import org.hamcrest.Matchers.`is` import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test import org.junit.jupiter.params.ParameterizedTest import org.junit.jupiter.params.provider.Arguments import org.junit.jupiter.params.provider.MethodSource @@ -39,6 +44,9 @@ import java.util.zip.GZIPInputStream private const val INVALID_COMPRESSION_FORMAT = "invalidCompressionFormat" +const val COMPRESSION_FORMAT_GZIP = "gzip" +const val COMPRESSION_FORMAT_ZSTD = "zstd" + @SpringBootTest @AutoConfigureMockMvc class LapisControllerCompressionTest( @@ -132,6 +140,19 @@ class LapisControllerCompressionTest( assertThat(errorDetail, `is`(errorMessage)) } + @Test + fun `GIVEN multiple values in accept encoding header THEN it should return compressed data`() { + val mockData = MockDataForEndpoints.getMockData(AGGREGATED.pathSegment).expecting(PLAIN_JSON) + mockData.mockWithData(siloQueryModelMock) + + val acceptEncodingAsBrowsersSendIt = "$COMPRESSION_FORMAT_GZIP, br, deflate" + + mockMvc.perform(getSample(AGGREGATED.pathSegment).header(ACCEPT_ENCODING, acceptEncodingAsBrowsersSendIt)) + .andExpect(status().isOk) + .andExpect(content().contentType(APPLICATION_JSON)) + .andExpect(header().string(CONTENT_ENCODING, COMPRESSION_FORMAT_GZIP)) + } + private fun decompressContent( response: MvcResult, compressionFormat: String, @@ -165,34 +186,39 @@ class LapisControllerCompressionTest( .flatMap { getRequests( endpoint = it, - dataFormat = MockDataCollection.DataFormat.CSV, - compressionFormat = "gzip", + dataFormat = CSV, + compressionFormat = COMPRESSION_FORMAT_GZIP, ) + getRequests( endpoint = it, - dataFormat = MockDataCollection.DataFormat.CSV, - compressionFormat = "zstd", + dataFormat = CSV, + compressionFormat = COMPRESSION_FORMAT_ZSTD, ) } + getRequests( AGGREGATED, - dataFormat = MockDataCollection.DataFormat.NESTED_JSON, - compressionFormat = "gzip", + dataFormat = NESTED_JSON, + compressionFormat = COMPRESSION_FORMAT_GZIP, ) + getRequests( AGGREGATED, - dataFormat = MockDataCollection.DataFormat.TSV, - compressionFormat = "zstd", + dataFormat = TSV, + compressionFormat = COMPRESSION_FORMAT_ZSTD, ) + listOf( "${UNALIGNED_NUCLEOTIDE_SEQUENCES.pathSegment}/main", "${ALIGNED_NUCLEOTIDE_SEQUENCES.pathSegment}/main", "${ALIGNED_AMINO_ACID_SEQUENCES.pathSegment}/gene1", ) - .flatMap { getFastaRequests(it, "gzip") + getFastaRequests(it, "zstd") } + .flatMap { + getFastaRequests(it, COMPRESSION_FORMAT_GZIP) + getFastaRequests( + it, + COMPRESSION_FORMAT_ZSTD, + ) + } @JvmStatic - val compressionFormats = listOf("gzip", "zstd") + val compressionFormats = listOf(COMPRESSION_FORMAT_GZIP, COMPRESSION_FORMAT_ZSTD) } } @@ -306,17 +332,17 @@ private fun getFastaRequests( ), ) -private fun getContentTypeForCompressionFormat(compressionFormat: String) = +fun getContentTypeForCompressionFormat(compressionFormat: String) = when (compressionFormat) { - "gzip" -> "application/gzip" - "zstd" -> "application/zstd" + COMPRESSION_FORMAT_GZIP -> "application/gzip" + COMPRESSION_FORMAT_ZSTD -> "application/zstd" else -> throw Exception("Test issue: unknown compression format $compressionFormat") } private fun getContentTypeForDataFormat(dataFormat: MockDataCollection.DataFormat) = when (dataFormat) { - MockDataCollection.DataFormat.PLAIN_JSON -> APPLICATION_JSON_VALUE - MockDataCollection.DataFormat.NESTED_JSON -> APPLICATION_JSON_VALUE - MockDataCollection.DataFormat.CSV -> "$TEXT_CSV;charset=UTF-8" - MockDataCollection.DataFormat.TSV -> "$TEXT_TSV;charset=UTF-8" + PLAIN_JSON -> APPLICATION_JSON_VALUE + NESTED_JSON -> APPLICATION_JSON_VALUE + CSV -> "$TEXT_CSV;charset=UTF-8" + TSV -> "$TEXT_TSV;charset=UTF-8" } diff --git a/lapis2/src/test/kotlin/org/genspectrum/lapis/controller/LapisControllerDownloadAsFileTest.kt b/lapis2/src/test/kotlin/org/genspectrum/lapis/controller/LapisControllerDownloadAsFileTest.kt new file mode 100644 index 000000000..7b7493b17 --- /dev/null +++ b/lapis2/src/test/kotlin/org/genspectrum/lapis/controller/LapisControllerDownloadAsFileTest.kt @@ -0,0 +1,303 @@ +package org.genspectrum.lapis.controller + +import com.ninjasquad.springmockk.MockkBean +import io.mockk.every +import org.genspectrum.lapis.controller.MockDataCollection.DataFormat.PLAIN_JSON +import org.genspectrum.lapis.controller.SampleRoute.AGGREGATED +import org.genspectrum.lapis.controller.SampleRoute.ALIGNED_AMINO_ACID_SEQUENCES +import org.genspectrum.lapis.controller.SampleRoute.ALIGNED_NUCLEOTIDE_SEQUENCES +import org.genspectrum.lapis.controller.SampleRoute.AMINO_ACID_INSERTIONS +import org.genspectrum.lapis.controller.SampleRoute.AMINO_ACID_MUTATIONS +import org.genspectrum.lapis.controller.SampleRoute.DETAILS +import org.genspectrum.lapis.controller.SampleRoute.NUCLEOTIDE_INSERTIONS +import org.genspectrum.lapis.controller.SampleRoute.NUCLEOTIDE_MUTATIONS +import org.genspectrum.lapis.controller.SampleRoute.UNALIGNED_NUCLEOTIDE_SEQUENCES +import org.genspectrum.lapis.model.SiloQueryModel +import org.genspectrum.lapis.request.LapisInfo +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.params.ParameterizedTest +import org.junit.jupiter.params.provider.MethodSource +import org.junit.jupiter.params.provider.ValueSource +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc +import org.springframework.boot.test.context.SpringBootTest +import org.springframework.http.HttpHeaders.ACCEPT +import org.springframework.http.HttpHeaders.ACCEPT_ENCODING +import org.springframework.http.HttpHeaders.CONTENT_DISPOSITION +import org.springframework.http.HttpHeaders.CONTENT_TYPE +import org.springframework.http.MediaType.APPLICATION_JSON +import org.springframework.test.web.servlet.MockMvc +import org.springframework.test.web.servlet.ResultActions +import org.springframework.test.web.servlet.request.MockHttpServletRequestBuilder +import org.springframework.test.web.servlet.result.MockMvcResultMatchers.header +import org.springframework.test.web.servlet.result.MockMvcResultMatchers.status + +@SpringBootTest +@AutoConfigureMockMvc +class LapisControllerDownloadAsFileTest( + @Autowired val mockMvc: MockMvc, +) { + @MockkBean + lateinit var siloQueryModelMock: SiloQueryModel + + @MockkBean + lateinit var lapisInfo: LapisInfo + + @BeforeEach + fun setup() { + every { + lapisInfo.dataVersion + } returns "1234" + } + + @ParameterizedTest(name = "GET data from {0} as file") + @MethodSource("getDownloadAsFileScenarios") + fun `GET data as file`(scenario: DownloadAsFileScenario) { + scenario.mockData.mockWithData(siloQueryModelMock) + + var queryString = "$DOWNLOAD_AS_FILE_PROPERTY=true" + if (scenario.requestedDataFormat != null) { + queryString += "&$FORMAT_PROPERTY=${scenario.requestedDataFormat}" + } + + mockMvc.perform(getSample("${scenario.endpoint}?$queryString")) + .andExpect(status().isOk) + .andExpectAttachmentWithContent( + expectedFilename = scenario.expectedFilename, + assertFileContentMatches = scenario.mockData.assertDataMatches, + ) + } + + @ParameterizedTest(name = "POST data from {0} as file") + @MethodSource("getDownloadAsFileScenarios") + fun `POST data as file`(scenario: DownloadAsFileScenario) { + scenario.mockData.mockWithData(siloQueryModelMock) + + val maybeDataFormat = when { + scenario.requestedDataFormat != null -> """, "$FORMAT_PROPERTY": "${scenario.requestedDataFormat}" """ + else -> "" + } + val request = """{ "$DOWNLOAD_AS_FILE_PROPERTY": true $maybeDataFormat }""" + + mockMvc.perform(postSample(scenario.endpoint).content(request).contentType(APPLICATION_JSON)) + .andExpect(status().isOk) + .andExpectAttachmentWithContent( + expectedFilename = scenario.expectedFilename, + assertFileContentMatches = scenario.mockData.assertDataMatches, + ) + } + + @ParameterizedTest(name = "{0} should return compressed file") + @MethodSource("getCompressedFileScenarios") + fun `WHEN I request compressed files THEN the filenames have a corresponding suffix`( + scenario: DownloadCompressedFileScenario, + ) { + scenario.mockData.mockToReturnEmptyData(siloQueryModelMock) + + mockMvc.perform(scenario.request) + .andExpect(status().isOk) + .andExpect(header().string(CONTENT_DISPOSITION, attachmentWithFilename(scenario.expectedFilename))) + .andExpect(header().string(CONTENT_TYPE, scenario.expectedContentType)) + } + + @ParameterizedTest + @ValueSource(strings = [COMPRESSION_FORMAT_GZIP, COMPRESSION_FORMAT_ZSTD, "$COMPRESSION_FORMAT_GZIP, br, deflate"]) + fun `WHEN I request data as file and accept encoding THEN it should return uncompressed file`( + acceptEncodingHeader: String, + ) { + val mockData = MockDataForEndpoints.getMockData(AGGREGATED.pathSegment).expecting(PLAIN_JSON) + mockData.mockWithData(siloQueryModelMock) + + mockMvc.perform( + getSample("${AGGREGATED.pathSegment}?$DOWNLOAD_AS_FILE_PROPERTY=true") + .header(ACCEPT_ENCODING, acceptEncodingHeader), + ) + .andExpect(status().isOk) + .andExpectAttachmentWithContent( + expectedFilename = "aggregated.json", + assertFileContentMatches = mockData.assertDataMatches, + ) + } + + private fun ResultActions.andExpectAttachmentWithContent( + expectedFilename: String, + assertFileContentMatches: (String) -> Unit, + ) { + this.andExpect(header().string("Content-Disposition", attachmentWithFilename(expectedFilename))) + .andReturn() + .response + .contentAsString + .apply(assertFileContentMatches) + } + + private fun attachmentWithFilename(filename: String) = "attachment; filename=$filename" + + private companion object { + @JvmStatic + val downloadAsFileScenarios = SampleRoute.entries.flatMap { DownloadAsFileScenario.forEndpoint(it) } + + private val dataFormatsSequence = generateSequence { + listOf( + PLAIN_JSON, + MockDataCollection.DataFormat.CSV, + MockDataCollection.DataFormat.TSV, + ) + }.flatten() + + @JvmStatic + val compressedFileScenarios = dataFormatsSequence.zip(SampleRoute.entries.asSequence()) + .flatMap { (dataFormat, route) -> DownloadCompressedFileScenario.scenariosFor(dataFormat, route) } + .toList() + } +} + +fun SampleRoute.getExpectedFilename() = + when (this) { + AGGREGATED -> "aggregated" + DETAILS -> "details" + NUCLEOTIDE_MUTATIONS -> "nucleotideMutations" + AMINO_ACID_MUTATIONS -> "aminoAcidMutations" + NUCLEOTIDE_INSERTIONS -> "nucleotideInsertions" + AMINO_ACID_INSERTIONS -> "aminoAcidInsertions" + ALIGNED_NUCLEOTIDE_SEQUENCES -> "alignedNucleotideSequences" + ALIGNED_AMINO_ACID_SEQUENCES -> "alignedAminoAcidSequences" + UNALIGNED_NUCLEOTIDE_SEQUENCES -> "unalignedNucleotideSequences" + } + +data class DownloadAsFileScenario( + val endpoint: String, + val mockData: MockData, + val requestedDataFormat: String?, + val expectedFilename: String, +) { + override fun toString() = + when (requestedDataFormat) { + null -> endpoint + else -> "$endpoint as $requestedDataFormat" + } + + companion object { + fun forEndpoint(route: SampleRoute): List { + val expectedFilename = route.getExpectedFilename() + + if (route.servesFasta) { + return listOf( + DownloadAsFileScenario( + endpoint = "${route.pathSegment}/segmentName", + mockData = MockDataForEndpoints.fastaMockData, + requestedDataFormat = null, + expectedFilename = "$expectedFilename.fasta", + ), + ) + } + + return forDataFormats(route.pathSegment, expectedFilename) + } + + private fun forDataFormats( + endpoint: String, + expectedFilename: String, + ) = listOf( + DownloadAsFileScenario( + mockData = MockDataForEndpoints.getMockData(endpoint) + .expecting(PLAIN_JSON), + expectedFilename = "$expectedFilename.json", + endpoint = endpoint, + requestedDataFormat = "json", + ), + DownloadAsFileScenario( + mockData = MockDataForEndpoints.getMockData(endpoint).expecting(MockDataCollection.DataFormat.CSV), + expectedFilename = "$expectedFilename.csv", + endpoint = endpoint, + requestedDataFormat = "csv", + ), + DownloadAsFileScenario( + mockData = MockDataForEndpoints.getMockData(endpoint).expecting(MockDataCollection.DataFormat.TSV), + expectedFilename = "$expectedFilename.tsv", + endpoint = endpoint, + requestedDataFormat = "tsv", + ), + ) + } +} + +data class DownloadCompressedFileScenario( + val description: String, + val mockData: MockData, + val request: MockHttpServletRequestBuilder, + val expectedFilename: String, + val expectedContentType: String, +) { + override fun toString() = description + + companion object { + fun scenariosFor( + dataFormat: MockDataCollection.DataFormat, + route: SampleRoute, + ) = scenariosFor( + dataFormat = dataFormat, + route = route, + compressionFormat = COMPRESSION_FORMAT_GZIP, + ) + + scenariosFor( + dataFormat = dataFormat, + route = route, + compressionFormat = COMPRESSION_FORMAT_ZSTD, + ) + + private fun scenariosFor( + dataFormat: MockDataCollection.DataFormat, + route: SampleRoute, + compressionFormat: String, + ): List { + val (mockData, dataFileFormat) = if (route.servesFasta) { + MockDataForEndpoints.fastaMockData to "fasta" + } else { + MockDataForEndpoints.getMockData(route.pathSegment).expecting(dataFormat) to dataFormat.fileFormat + } + + val endpoint = when (route) { + ALIGNED_NUCLEOTIDE_SEQUENCES -> "${route.pathSegment}/main" + ALIGNED_AMINO_ACID_SEQUENCES -> "${route.pathSegment}/main" + UNALIGNED_NUCLEOTIDE_SEQUENCES -> "${route.pathSegment}/gene1" + else -> route.pathSegment + } + val acceptHeader = when (route.servesFasta) { + true -> "*/*" + false -> dataFormat.acceptHeader + } + + val fileEnding = when (compressionFormat) { + COMPRESSION_FORMAT_ZSTD -> "zst" + COMPRESSION_FORMAT_GZIP -> "gz" + else -> throw Exception("Test issue: unknown compression format $compressionFormat") + } + val expectedFilename = "${route.getExpectedFilename()}.$dataFileFormat.$fileEnding" + val expectedContentType = getContentTypeForCompressionFormat(compressionFormat) + + return listOf( + DownloadCompressedFileScenario( + description = "GET $endpoint as $compressionFormat ${dataFormat.fileFormat}", + mockData = mockData, + request = getSample( + "$endpoint?$DOWNLOAD_AS_FILE_PROPERTY=true&$COMPRESSION_PROPERTY=$compressionFormat", + ) + .header(ACCEPT, acceptHeader), + expectedFilename = expectedFilename, + expectedContentType = expectedContentType, + ), + DownloadCompressedFileScenario( + description = "POST $endpoint as $compressionFormat ${dataFormat.fileFormat}", + mockData = mockData, + request = postSample(endpoint).content( + """{ "$DOWNLOAD_AS_FILE_PROPERTY": true, "$COMPRESSION_PROPERTY": "$compressionFormat" }""", + ) + .contentType(APPLICATION_JSON) + .header(ACCEPT, acceptHeader), + expectedFilename = expectedFilename, + expectedContentType = expectedContentType, + ), + ) + } + } +} diff --git a/siloLapisTests/test/aggregatedQueries/noMutationAt230.json b/siloLapisTests/test/aggregatedQueries/noMutationAt230.json index 4dff9a317..7d6d4a130 100644 --- a/siloLapisTests/test/aggregatedQueries/noMutationAt230.json +++ b/siloLapisTests/test/aggregatedQueries/noMutationAt230.json @@ -5,7 +5,7 @@ }, "expected": [ { - "count": 3 + "count": 96 } ] } diff --git a/siloLapisTests/test/unknownUrl.spec.ts b/siloLapisTests/test/unknownUrl.spec.ts index 318ce440b..154e76da6 100644 --- a/siloLapisTests/test/unknownUrl.spec.ts +++ b/siloLapisTests/test/unknownUrl.spec.ts @@ -6,8 +6,9 @@ describe('Error handling: UnknownUrl', () => { const result = await fetch(basePath + '/unknownUrl'); expect(result.status).equals(404); - expect(result.headers.get('Content-Type')).equals('application/json'); - expect((await result.json())?.error).equals('Not Found'); + expect(result.headers.get('Content-Type')).equals('application/problem+json'); + const body = await result.json(); + expect(body, JSON.stringify(body)).to.have.nested.property('title', 'Not Found'); }); it('should return a 404 HTML response when a browser asks for HTML', async () => { @@ -16,7 +17,7 @@ describe('Error handling: UnknownUrl', () => { expect(result.status).equals(404); expect(result.headers.get('Content-Type')).equals('text/html'); - let responseBody = await result.text(); + const responseBody = await result.text(); expect(responseBody).contains('Page not found'); expect(responseBody).contains(''); });