From 8bd41d99d00d52879177e7408e2080cdc99db31c Mon Sep 17 00:00:00 2001 From: Alex Arvanitidis Date: Mon, 25 Nov 2024 14:03:10 +0200 Subject: [PATCH 1/8] feat(JAQPOT-414): archive model --- .../kotlin/org/jaqpot/api/entity/Model.kt | 5 + ...ic.kt => ModelUpdateAuthorizationLogic.kt} | 4 +- .../jaqpot/api/service/model/DoaService.kt | 2 +- .../api/service/model/FeatureService.kt | 2 +- .../jaqpot/api/service/model/ModelService.kt | 66 ++++- .../service/prediction/PredictionService.kt | 4 +- .../kotlin/org/jaqpot/api/storage/Storage.kt | 5 + .../org/jaqpot/api/storage/StorageService.kt | 246 ++++++++++-------- .../org/jaqpot/api/storage/s3/S3Storage.kt | 9 + .../db/migration/V42__add_model_archived.sql | 5 + src/main/resources/openapi.yaml | 85 +++++- 11 files changed, 319 insertions(+), 114 deletions(-) rename src/main/kotlin/org/jaqpot/api/service/authorization/{PartialFeatureUpdateAuthorizationLogic.kt => ModelUpdateAuthorizationLogic.kt} (90%) create mode 100644 src/main/resources/db/migration/V42__add_model_archived.sql diff --git a/src/main/kotlin/org/jaqpot/api/entity/Model.kt b/src/main/kotlin/org/jaqpot/api/entity/Model.kt index 37921643..16b61bbf 100644 --- a/src/main/kotlin/org/jaqpot/api/entity/Model.kt +++ b/src/main/kotlin/org/jaqpot/api/entity/Model.kt @@ -5,6 +5,7 @@ import jakarta.validation.constraints.Size import org.hibernate.annotations.JdbcTypeCode import org.hibernate.annotations.SQLRestriction import org.hibernate.type.SqlTypes +import java.time.OffsetDateTime @Entity class Model( @@ -67,6 +68,10 @@ class Model( val selectedFeatures: List?, + var archived: Boolean? = null, + + var archivedAt: OffsetDateTime? = null, + @Size(min = 3, max = 1000) @Column(columnDefinition = "TEXT") var tags: String?, diff --git a/src/main/kotlin/org/jaqpot/api/service/authorization/PartialFeatureUpdateAuthorizationLogic.kt b/src/main/kotlin/org/jaqpot/api/service/authorization/ModelUpdateAuthorizationLogic.kt similarity index 90% rename from src/main/kotlin/org/jaqpot/api/service/authorization/PartialFeatureUpdateAuthorizationLogic.kt rename to src/main/kotlin/org/jaqpot/api/service/authorization/ModelUpdateAuthorizationLogic.kt index 0a0154ae..98e65e7d 100644 --- a/src/main/kotlin/org/jaqpot/api/service/authorization/PartialFeatureUpdateAuthorizationLogic.kt +++ b/src/main/kotlin/org/jaqpot/api/service/authorization/ModelUpdateAuthorizationLogic.kt @@ -7,8 +7,8 @@ import org.springframework.security.access.expression.method.MethodSecurityExpre import org.springframework.stereotype.Component import org.springframework.web.server.ResponseStatusException -@Component("partialFeatureUpdateAuthorizationLogic") -class PartialFeatureUpdateAuthorizationLogic( +@Component("modelUpdateAuthorizationLogic") +class ModelUpdateAuthorizationLogic( private val modelRepository: ModelRepository, private val authenticationFacade: AuthenticationFacade ) { diff --git a/src/main/kotlin/org/jaqpot/api/service/model/DoaService.kt b/src/main/kotlin/org/jaqpot/api/service/model/DoaService.kt index e77b7834..f68f7307 100644 --- a/src/main/kotlin/org/jaqpot/api/service/model/DoaService.kt +++ b/src/main/kotlin/org/jaqpot/api/service/model/DoaService.kt @@ -19,7 +19,7 @@ class DoaService(private val doaRepository: DoaRepository, private val storageSe return } logger.info { "Storing raw doa to storage for doa with id ${doa.id} and model ${doa.model.id}" } - if (storageService.storeDoa(doa)) { + if (storageService.storeRawDoa(doa)) { logger.info { "Successfully moved raw doa to storage for doa ${doa.id} and model ${doa.model.id}" } doaRepository.setRawDoaToNull(doa.id) } diff --git a/src/main/kotlin/org/jaqpot/api/service/model/FeatureService.kt b/src/main/kotlin/org/jaqpot/api/service/model/FeatureService.kt index 6ba8a63b..30c1faad 100644 --- a/src/main/kotlin/org/jaqpot/api/service/model/FeatureService.kt +++ b/src/main/kotlin/org/jaqpot/api/service/model/FeatureService.kt @@ -17,7 +17,7 @@ import org.springframework.web.server.ResponseStatusException @Service class FeatureService(private val modelRepository: ModelRepository, private val featureRepository: FeatureRepository) : FeatureApiDelegate { - @PreAuthorize("@partialFeatureUpdateAuthorizationLogic.decide(#root, #modelId)") + @PreAuthorize("@modelUpdateAuthorizationLogic.decide(#root, #modelId)") override fun partiallyUpdateModelFeature( modelId: Long, featureId: Long, diff --git a/src/main/kotlin/org/jaqpot/api/service/model/ModelService.kt b/src/main/kotlin/org/jaqpot/api/service/model/ModelService.kt index e92e8e82..54c4a6b7 100644 --- a/src/main/kotlin/org/jaqpot/api/service/model/ModelService.kt +++ b/src/main/kotlin/org/jaqpot/api/service/model/ModelService.kt @@ -34,6 +34,7 @@ import org.springframework.stereotype.Service import org.springframework.web.server.ResponseStatusException import org.springframework.web.servlet.support.ServletUriComponentsBuilder import java.net.URI +import java.time.OffsetDateTime private val logger = KotlinLogging.logger {} const val JAQPOT_METADATA_KEY = "jaqpotMetadata" @@ -363,6 +364,48 @@ class ModelService( return ResponseEntity.ok(model.toDto(modelCreator, userCanEdit, isAdmin)) } + @PreAuthorize("@modelUpdateAuthorizationLogic.decide(#root, #modelId)") + override fun archiveModel(modelId: Long): ResponseEntity { + val existingModel = modelRepository.findById(modelId).orElseThrow { + throw ResponseStatusException(HttpStatus.NOT_FOUND, "Model with id $modelId not found") + } + + if (existingModel.archived == true) { + throw ResponseStatusException(HttpStatus.BAD_REQUEST, "Model with id $modelId is already archived") + } + + existingModel.archived = true + existingModel.archivedAt = OffsetDateTime.now() + + modelRepository.save(existingModel) + + return ResponseEntity( + ArchiveModel200ResponseDto(id = modelId, archivedAt = existingModel.archivedAt), + HttpStatus.OK + ) + } + + @PreAuthorize("@modelUpdateAuthorizationLogic.decide(#root, #modelId)") + override fun unarchiveModel(modelId: Long): ResponseEntity { + val existingModel = modelRepository.findById(modelId).orElseThrow { + throw ResponseStatusException(HttpStatus.NOT_FOUND, "Model with id $modelId not found") + } + + if (existingModel.archived != true) { + throw ResponseStatusException(HttpStatus.BAD_REQUEST, "Model with id $modelId is not archived") + } + + existingModel.archived = false + existingModel.archivedAt = null + + modelRepository.save(existingModel) + + return ResponseEntity( + UnarchiveModel200ResponseDto(id = modelId), + HttpStatus.OK + ) + } + @Cacheable(CacheKeys.SEARCH_MODELS) override fun searchModels( query: String, @@ -383,9 +426,28 @@ class ModelService( @CacheEvict("searchModels", allEntries = true) @PreAuthorize("hasAuthority('admin')") override fun deleteModelById(id: Long): ResponseEntity { - modelRepository.delete(modelRepository.findById(id).orElseThrow { + val model = modelRepository.findById(id).orElseThrow { throw ResponseStatusException(HttpStatus.NOT_FOUND, "Model with id $id not found") - }) + } + + model.doas.forEach { + logger.info { "Deleting DOA with id ${it.id} for model with id $id" } + val deletedRawDoa = storageService.deleteRawDoa(it) + logger.info { "Deleted raw DOA for model with id $id: $deletedRawDoa" } + } + + logger.info { "Deleting raw preprocessor for model with id $id" } + val deletedRawPreprocessor = storageService.deleteRawPreprocessor(model) + logger.info { "Deleted raw preprocessor for model with id $id: $deletedRawPreprocessor" } + + logger.info { "Deleting raw model for model with id $id" } + val deletedRawModel = storageService.deleteRawModel(model) + logger.info { "Deleted raw model for model with id $id: $deletedRawModel" } + + logger.info { "Deleting model with id $id" } + modelRepository.delete(model) + logger.info { "Deleted model with id $id" } + return ResponseEntity.noContent().build() } } diff --git a/src/main/kotlin/org/jaqpot/api/service/prediction/PredictionService.kt b/src/main/kotlin/org/jaqpot/api/service/prediction/PredictionService.kt index 8aa003d8..da1e0101 100644 --- a/src/main/kotlin/org/jaqpot/api/service/prediction/PredictionService.kt +++ b/src/main/kotlin/org/jaqpot/api/service/prediction/PredictionService.kt @@ -60,7 +60,7 @@ class PredictionService( dataset.result = results dataset.executionFinishedAt = OffsetDateTime.now() datasetRepository.save(dataset) - if (storageService.storeDataset(dataset)) { + if (storageService.storeRawDataset(dataset)) { datasetRepository.setDatasetInputAndResultToNull(dataset.id) } } @@ -70,7 +70,7 @@ class PredictionService( dataset.failureReason = err.toString() datasetRepository.save(dataset) - if (storageService.storeDataset(dataset)) { + if (storageService.storeRawDataset(dataset)) { datasetRepository.setDatasetInputAndResultToNull(dataset.id) } } diff --git a/src/main/kotlin/org/jaqpot/api/storage/Storage.kt b/src/main/kotlin/org/jaqpot/api/storage/Storage.kt index 5cba4e51..4de4d789 100644 --- a/src/main/kotlin/org/jaqpot/api/storage/Storage.kt +++ b/src/main/kotlin/org/jaqpot/api/storage/Storage.kt @@ -16,4 +16,9 @@ interface Storage { obj: ByteArray, metadata: Map = mapOf(), ) + + fun deleteObject( + bucketName: String, + keyName: String + ) } diff --git a/src/main/kotlin/org/jaqpot/api/storage/StorageService.kt b/src/main/kotlin/org/jaqpot/api/storage/StorageService.kt index 1463f855..df85c786 100644 --- a/src/main/kotlin/org/jaqpot/api/storage/StorageService.kt +++ b/src/main/kotlin/org/jaqpot/api/storage/StorageService.kt @@ -24,7 +24,8 @@ class StorageService( private const val METADATA_VERSION = "1" } - fun storeDataset(dataset: Dataset): Boolean { + // Dataset + fun storeRawDataset(dataset: Dataset): Boolean { try { val metadata = mapOf( "version" to METADATA_VERSION, @@ -58,7 +59,95 @@ class StorageService( } } - fun storeDoa(doa: Doa): Boolean { + fun readRawDatasetInputs(datasets: List): Map> { + var rawDatasetsFromStorage = mutableMapOf() + try { + rawDatasetsFromStorage = + this.storage.getObjects(awsS3Config.datasetsBucketName, datasets.map { "${it.id.toString()}/input" }) + .mapKeys { it.key.substringBefore("/") }.toMutableMap() + } catch (e: Exception) { + logger.warn(e) { "Failed to read datasets" } + } + + return if (rawDatasetsFromStorage.isNotEmpty()) { + rawDatasetsFromStorage.mapValues { (_, value) -> + val type = object : TypeToken>() {}.type + val input: List = Gson().fromJson(value.decodeToString(), type) + input + } + } else { + return datasets.associateBy({ it.id.toString() }, { it.input!! }) + } + } + + + fun readRawDatasetResults(datasets: List): Map?> { + var rawDatasetsFromStorage = mutableMapOf() + try { + rawDatasetsFromStorage = + this.storage.getObjects(awsS3Config.datasetsBucketName, datasets.map { "${it.id.toString()}/result" }) + .mapKeys { it.key.substringBefore("/") }.toMutableMap() + } catch (e: Exception) { + logger.warn(e) { "Failed to read datasets" } + } + + return if (rawDatasetsFromStorage.isNotEmpty()) { + rawDatasetsFromStorage.mapValues { (_, value) -> + val type = object : TypeToken>() {}.type + val result: List = Gson().fromJson(value.decodeToString(), type) + result + } + } else { + return datasets.associateBy({ it.id.toString() }, { it.result }) + } + } + + fun readRawDatasetInput(dataset: Dataset): List { + var rawDatasetFromStorage = Optional.empty() + try { + rawDatasetFromStorage = + this.storage.getObject(awsS3Config.datasetsBucketName, "${dataset.id.toString()}/input") + } catch (e: Exception) { + logger.warn(e) { "Failed to read dataset with id ${dataset.id}" } + } + + if (rawDatasetFromStorage.isPresent) { + val rawData = rawDatasetFromStorage.get() + val type = object : TypeToken>() {}.type + val input: List = Gson().fromJson(rawData.decodeToString(), type) + return input + } else if (dataset.input != null) { + logger.warn { "Failed to find input with id ${dataset.id} in storage, falling back to input from database" } + return dataset.input + } + + throw JaqpotRuntimeException("Failed to find raw dataset input with id ${dataset.id}") + } + + fun readRawDatasetResult(dataset: Dataset): List? { + var rawDatasetFromStorage = Optional.empty() + try { + rawDatasetFromStorage = + this.storage.getObject(awsS3Config.datasetsBucketName, "${dataset.id.toString()}/result") + } catch (e: Exception) { + logger.warn(e) { "Failed to read dataset with id ${dataset.id}" } + } + + if (rawDatasetFromStorage.isPresent) { + val rawData = rawDatasetFromStorage.get() + val type = object : TypeToken>() {}.type + val result: List = Gson().fromJson(rawData.decodeToString(), type) + return result + } else if (dataset.result != null) { + logger.warn { "Failed to find dataset result with id ${dataset.id} in storage, falling back to result from database" } + return dataset.result!! + } + + return null + } + + // Doa + fun storeRawDoa(doa: Doa): Boolean { try { val metadata = mapOf( "version" to METADATA_VERSION, @@ -80,8 +169,47 @@ class StorageService( } } + fun deleteRawDoa(doa: Doa): Boolean { + try { + this.storage.deleteObject(awsS3Config.doasBucketName, getDoaStorageKey(doa)) + return true + } catch (e: Exception) { + logger.error(e) { "Failed to delete doa with id ${doa.id}" } + return false + } + } + + fun readRawDoa(doa: Doa): ByteArray { + var rawDoaFromStorage = Optional.empty() + try { + rawDoaFromStorage = this.storage.getObject(awsS3Config.doasBucketName, getDoaStorageKey(doa)) + } catch (e: Exception) { + logger.warn(e) { "Failed to read doa with id ${doa.id}" } + } + + if (rawDoaFromStorage.isPresent) { + return rawDoaFromStorage.get() + } else if (doa.rawDoa != null) { + logger.warn { "Failed to find raw doa with id ${doa.id} in storage, falling back to raw doa from database" } + return doa.rawDoa!! + } + + throw JaqpotRuntimeException("Failed to find raw doa with id ${doa.id}") + } + private fun getDoaStorageKey(doa: Doa) = "${doa.model.id}/${doa.method.name}" + // Model + fun deleteRawModel(model: Model): Boolean { + try { + this.storage.deleteObject(awsS3Config.modelsBucketName, getModelStorageKey(model)) + return true + } catch (e: Exception) { + logger.error(e) { "Failed to delete model with id ${model.id}" } + return false + } + } + fun storeRawModel(model: Model): Boolean { try { val metadata = mapOf( @@ -122,6 +250,7 @@ class StorageService( private fun getModelStorageKey(model: Model) = model.id.toString() + // Preprocessor fun storeRawPreprocessor(model: Model): Boolean { try { val metadata = mapOf( @@ -142,6 +271,16 @@ class StorageService( } } + fun deleteRawPreprocessor(model: Model): Boolean { + try { + this.storage.deleteObject(awsS3Config.preprocessorsBucketName, getModelStorageKey(model)) + return true + } catch (e: Exception) { + logger.error(e) { "Failed to delete preprocessor for model with id ${model.id}" } + return false + } + } + fun readRawPreprocessor(model: Model): ByteArray? { var rawPreprocessorFromStorage = Optional.empty() try { @@ -161,108 +300,5 @@ class StorageService( return null } - fun readRawDoa(doa: Doa): ByteArray { - var rawDoaFromStorage = Optional.empty() - try { - rawDoaFromStorage = this.storage.getObject(awsS3Config.doasBucketName, getDoaStorageKey(doa)) - } catch (e: Exception) { - logger.warn(e) { "Failed to read doa with id ${doa.id}" } - } - - if (rawDoaFromStorage.isPresent) { - return rawDoaFromStorage.get() - } else if (doa.rawDoa != null) { - logger.warn { "Failed to find raw doa with id ${doa.id} in storage, falling back to raw doa from database" } - return doa.rawDoa!! - } - - throw JaqpotRuntimeException("Failed to find raw doa with id ${doa.id}") - } - - fun readRawDatasetInputs(datasets: List): Map> { - var rawDatasetsFromStorage = mutableMapOf() - try { - rawDatasetsFromStorage = - this.storage.getObjects(awsS3Config.datasetsBucketName, datasets.map { "${it.id.toString()}/input" }) - .mapKeys { it.key.substringBefore("/") }.toMutableMap() - } catch (e: Exception) { - logger.warn(e) { "Failed to read datasets" } - } - - return if (rawDatasetsFromStorage.isNotEmpty()) { - rawDatasetsFromStorage.mapValues { (_, value) -> - val type = object : TypeToken>() {}.type - val input: List = Gson().fromJson(value.decodeToString(), type) - input - } - } else { - return datasets.associateBy({ it.id.toString() }, { it.input!! }) - } - } - - - fun readRawDatasetResults(datasets: List): Map?> { - var rawDatasetsFromStorage = mutableMapOf() - try { - rawDatasetsFromStorage = - this.storage.getObjects(awsS3Config.datasetsBucketName, datasets.map { "${it.id.toString()}/result" }) - .mapKeys { it.key.substringBefore("/") }.toMutableMap() - } catch (e: Exception) { - logger.warn(e) { "Failed to read datasets" } - } - - return if (rawDatasetsFromStorage.isNotEmpty()) { - rawDatasetsFromStorage.mapValues { (_, value) -> - val type = object : TypeToken>() {}.type - val result: List = Gson().fromJson(value.decodeToString(), type) - result - } - } else { - return datasets.associateBy({ it.id.toString() }, { it.result }) - } - } - - fun readRawDatasetInput(dataset: Dataset): List { - var rawDatasetFromStorage = Optional.empty() - try { - rawDatasetFromStorage = - this.storage.getObject(awsS3Config.datasetsBucketName, "${dataset.id.toString()}/input") - } catch (e: Exception) { - logger.warn(e) { "Failed to read dataset with id ${dataset.id}" } - } - if (rawDatasetFromStorage.isPresent) { - val rawData = rawDatasetFromStorage.get() - val type = object : TypeToken>() {}.type - val input: List = Gson().fromJson(rawData.decodeToString(), type) - return input - } else if (dataset.input != null) { - logger.warn { "Failed to find input with id ${dataset.id} in storage, falling back to input from database" } - return dataset.input - } - - throw JaqpotRuntimeException("Failed to find raw dataset input with id ${dataset.id}") - } - - fun readRawDatasetResult(dataset: Dataset): List? { - var rawDatasetFromStorage = Optional.empty() - try { - rawDatasetFromStorage = - this.storage.getObject(awsS3Config.datasetsBucketName, "${dataset.id.toString()}/result") - } catch (e: Exception) { - logger.warn(e) { "Failed to read dataset with id ${dataset.id}" } - } - - if (rawDatasetFromStorage.isPresent) { - val rawData = rawDatasetFromStorage.get() - val type = object : TypeToken>() {}.type - val result: List = Gson().fromJson(rawData.decodeToString(), type) - return result - } else if (dataset.result != null) { - logger.warn { "Failed to find dataset result with id ${dataset.id} in storage, falling back to result from database" } - return dataset.result!! - } - - return null - } } diff --git a/src/main/kotlin/org/jaqpot/api/storage/s3/S3Storage.kt b/src/main/kotlin/org/jaqpot/api/storage/s3/S3Storage.kt index c65cf11b..29d1cc34 100644 --- a/src/main/kotlin/org/jaqpot/api/storage/s3/S3Storage.kt +++ b/src/main/kotlin/org/jaqpot/api/storage/s3/S3Storage.kt @@ -77,4 +77,13 @@ class S3Storage( s3Client.putObject(request, RequestBody.fromBytes(obj)) } + override fun deleteObject(bucketName: String, keyName: String) { + logger.info { "Deleting object from S3 bucket $bucketName with key $keyName on region ${awsConfig.region}" } + + s3Client.deleteObject { + it.bucket(bucketName) + it.key(keyName) + } + } + } diff --git a/src/main/resources/db/migration/V42__add_model_archived.sql b/src/main/resources/db/migration/V42__add_model_archived.sql new file mode 100644 index 00000000..e53f2d73 --- /dev/null +++ b/src/main/resources/db/migration/V42__add_model_archived.sql @@ -0,0 +1,5 @@ +ALTER TABLE model + ADD archived BOOLEAN; + +ALTER TABLE model + ADD archived_at TIMESTAMP WITH TIME ZONE; diff --git a/src/main/resources/openapi.yaml b/src/main/resources/openapi.yaml index 0af33f6c..3aaa8c1b 100644 --- a/src/main/resources/openapi.yaml +++ b/src/main/resources/openapi.yaml @@ -208,6 +208,82 @@ paths: type: integer '400': description: Invalid input + /v1/models/{modelId}/archive: + post: + summary: Archive a model + description: Archives a model. Models that remain archived for more than 30 days will be permanently deleted. + security: + - bearerAuth: [ ] + tags: + - model + operationId: archiveModel + parameters: + - name: modelId + in: path + required: true + schema: + type: integer + format: int64 + example: 0 + description: The ID of the model to archive + responses: + '200': + description: Model successfully archived + content: + application/json: + schema: + type: object + properties: + id: + type: integer + format: int64 + example: 0 + archivedAt: + type: string + format: date-time + description: Timestamp when the model was archived + '404': + description: Model not found + '409': + description: Model is already archived + '403': + description: Insufficient permissions to archive the model + /v1/models/{modelId}/unarchive: + post: + summary: Unarchive a model + description: Unarchives a previously archived model. This will cancel any scheduled deletion. + security: + - bearerAuth: [ ] + tags: + - model + operationId: unarchiveModel + parameters: + - name: modelId + in: path + required: true + schema: + type: integer + format: int64 + example: 0 + description: The ID of the model to unarchive + responses: + '200': + description: Model successfully unarchived + content: + application/json: + schema: + type: object + properties: + id: + type: integer + format: int64 + example: 0 + '404': + description: Model not found + '409': + description: Model is not currently archived + '403': + description: Insufficient permissions to unarchive the model /v1/user/shared-models: get: x-spring-paginated: true @@ -1286,6 +1362,13 @@ components: $ref: '#/components/schemas/ModelVisibility' task: $ref: '#/components/schemas/ModelTask' + archived: + type: boolean + archivedAt: + type: string + format: date-time + description: The date and time when the model was last archived. + example: '2023-01-01T12:00:00Z' torchConfig: type: object maxProperties: 20 @@ -1353,7 +1436,7 @@ components: updatedAt: type: string format: date-time - description: The date and time when the feature was last updated. + description: The date and time when the model was last updated. example: '2023-01-01T12:00:00Z' ModelSummary: type: object From 731588e1265a2582edd5193ae0d13c7eed037b2c Mon Sep 17 00:00:00 2001 From: Alex Arvanitidis Date: Mon, 25 Nov 2024 16:59:36 +0200 Subject: [PATCH 2/8] feat: add model + change queries --- .../kotlin/org/jaqpot/api/entity/Model.kt | 3 +- .../jaqpot/api/repository/ModelRepository.kt | 12 ++-- .../jaqpot/api/service/model/ModelService.kt | 22 +++++++- .../db/migration/V42__add_model_archived.sql | 2 +- src/main/resources/openapi.yaml | 55 +++++++++++++++++++ 5 files changed, 86 insertions(+), 8 deletions(-) diff --git a/src/main/kotlin/org/jaqpot/api/entity/Model.kt b/src/main/kotlin/org/jaqpot/api/entity/Model.kt index 16b61bbf..74a1339f 100644 --- a/src/main/kotlin/org/jaqpot/api/entity/Model.kt +++ b/src/main/kotlin/org/jaqpot/api/entity/Model.kt @@ -68,7 +68,8 @@ class Model( val selectedFeatures: List?, - var archived: Boolean? = null, + @Column(nullable = false) + var archived: Boolean = false, var archivedAt: OffsetDateTime? = null, diff --git a/src/main/kotlin/org/jaqpot/api/repository/ModelRepository.kt b/src/main/kotlin/org/jaqpot/api/repository/ModelRepository.kt index 42f10840..99a4ca88 100644 --- a/src/main/kotlin/org/jaqpot/api/repository/ModelRepository.kt +++ b/src/main/kotlin/org/jaqpot/api/repository/ModelRepository.kt @@ -14,13 +14,16 @@ import java.util.* interface ModelRepository : PagingAndSortingRepository, CrudRepository { - fun findAllByCreatorId(creatorId: String, pageable: Pageable): Page - fun findOneByLegacyId(legacyId: String): Optional + fun findAllByCreatorIdAndArchivedIsFalse(creatorId: String, pageable: Pageable): Page + fun findAllByCreatorIdAndArchivedIsTrue(creatorId: String, pageable: Pageable): Page + + fun findOneByLegacyIdAndArchivedIsFalse(legacyId: String): Optional @Query( """ SELECT m FROM Model m - WHERE m.visibility = 'ORG_SHARED' + WHERE m.visibility = 'ORG_SHARED' + AND m.archived = false AND EXISTS ( SELECT 1 FROM m.sharedWithOrganizations o JOIN o.organization org @@ -35,6 +38,7 @@ interface ModelRepository : PagingAndSortingRepository, CrudReposit """ SELECT m FROM Model m WHERE m.visibility = 'ORG_SHARED' + AND m.archived = false AND EXISTS ( SELECT 1 FROM m.sharedWithOrganizations o JOIN o.organization org @@ -49,7 +53,7 @@ interface ModelRepository : PagingAndSortingRepository, CrudReposit value = """ SELECT *, ts_rank_cd(textsearchable_index_col, to_tsquery(:query)) AS rank FROM model, to_tsquery(:query) query - WHERE model.visibility = 'PUBLIC' AND textsearchable_index_col @@ query + WHERE model.visibility = 'PUBLIC' AND model.archived = false AND textsearchable_index_col @@ query ORDER BY rank DESC """, // countQuery = """ diff --git a/src/main/kotlin/org/jaqpot/api/service/model/ModelService.kt b/src/main/kotlin/org/jaqpot/api/service/model/ModelService.kt index 54c4a6b7..26412689 100644 --- a/src/main/kotlin/org/jaqpot/api/service/model/ModelService.kt +++ b/src/main/kotlin/org/jaqpot/api/service/model/ModelService.kt @@ -90,7 +90,7 @@ class ModelService( override fun getModels(page: Int, size: Int, sort: List?): ResponseEntity { val creatorId = authenticationFacade.userId val pageable = PageRequest.of(page, size, Sort.by(parseSortParameters(sort))) - val modelsPage = modelRepository.findAllByCreatorId(creatorId, pageable) + val modelsPage = modelRepository.findAllByCreatorIdAndArchivedIsFalse(creatorId, pageable) val modelIdToUserMap = modelsPage.content.associateBy( { it.id!! }, { userService.getUserById(it.creatorId).orElse(UserDto(it.creatorId)) } @@ -185,7 +185,7 @@ class ModelService( @PostAuthorize("@getModelAuthorizationLogic.decide(#root)") override fun getLegacyModelById(id: String): ResponseEntity { - val model = modelRepository.findOneByLegacyId(id) + val model = modelRepository.findOneByLegacyIdAndArchivedIsFalse(id) return model.map { val userCanEdit = authenticationFacade.isAdmin || isCreator(authenticationFacade, it) @@ -423,6 +423,24 @@ class ModelService( return ResponseEntity.ok(modelsPage.toGetModels200ResponseDto(modelIdToUserMap)) } + override fun getArchivedModels( + page: Int, + size: Int, + sort: List? + ): ResponseEntity { + val userId = authenticationFacade.userId + val pageable = PageRequest.of(page, size, Sort.by(parseSortParameters(sort))) + + val archivedModelsPage = modelRepository.findAllByCreatorIdAndArchivedIsTrue(userId, pageable) + + val modelIdToUserMap = archivedModelsPage.content.associateBy( + { it.id!! }, + { userService.getUserById(it.creatorId).orElse(UserDto(it.creatorId)) } + ) + + return ResponseEntity.ok().body(archivedModelsPage.toGetModels200ResponseDto(modelIdToUserMap)) + } + @CacheEvict("searchModels", allEntries = true) @PreAuthorize("hasAuthority('admin')") override fun deleteModelById(id: Long): ResponseEntity { diff --git a/src/main/resources/db/migration/V42__add_model_archived.sql b/src/main/resources/db/migration/V42__add_model_archived.sql index e53f2d73..8f0f20aa 100644 --- a/src/main/resources/db/migration/V42__add_model_archived.sql +++ b/src/main/resources/db/migration/V42__add_model_archived.sql @@ -1,5 +1,5 @@ ALTER TABLE model - ADD archived BOOLEAN; + ADD archived BOOLEAN NOT NULL DEFAULT FALSE; ALTER TABLE model ADD archived_at TIMESTAMP WITH TIME ZONE; diff --git a/src/main/resources/openapi.yaml b/src/main/resources/openapi.yaml index 3aaa8c1b..12d0294e 100644 --- a/src/main/resources/openapi.yaml +++ b/src/main/resources/openapi.yaml @@ -342,6 +342,61 @@ paths: type: integer '400': description: Invalid input + /v1/user/archived-models: + get: + x-spring-paginated: true + summary: Get paginated archived models + description: Retrieve a paginated list of models that have been archived by the user + security: + - bearerAuth: [ ] + tags: + - model + operationId: getArchivedModels + parameters: + - name: page + in: query + required: false + schema: + type: integer + default: 0 + - name: size + in: query + required: false + schema: + type: integer + default: 10 + - name: sort + in: query + required: false + schema: + type: array + items: + type: string + example: [ "archivedAt|desc", "name|asc" ] + responses: + '200': + description: Paginated list of archived models + content: + application/json: + schema: + type: object + properties: + content: + type: array + items: + $ref: '#/components/schemas/ModelSummary' + totalElements: + type: integer + totalPages: + type: integer + pageSize: + type: integer + pageNumber: + type: integer + '400': + description: Invalid input + '403': + description: Insufficient permissions to access archived models '/v1/models/{id}': get: summary: Get a Model From 5a9b7ef457c30483d8a3c66c5f2864d07e391925 Mon Sep 17 00:00:00 2001 From: alarv Date: Tue, 26 Nov 2024 13:33:33 +0200 Subject: [PATCH 3/8] feat: return archived properties on dto --- src/main/kotlin/org/jaqpot/api/mapper/ModelMapper.kt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/kotlin/org/jaqpot/api/mapper/ModelMapper.kt b/src/main/kotlin/org/jaqpot/api/mapper/ModelMapper.kt index 82a4d457..72e2a187 100644 --- a/src/main/kotlin/org/jaqpot/api/mapper/ModelMapper.kt +++ b/src/main/kotlin/org/jaqpot/api/mapper/ModelMapper.kt @@ -34,6 +34,8 @@ fun Model.toDto(userDto: UserDto? = null, userCanEdit: Boolean? = null, isAdmin: test = this.testScores?.map { it.toDto() }, crossValidation = this.crossValidationScores?.map { it.toDto() }, ), + archived = this.archived, + archivedAt = this.archivedAt, createdAt = this.createdAt, updatedAt = this.updatedAt, ) From ea6ee318753971cd4ccd560fe2b17539f56149ef Mon Sep 17 00:00:00 2001 From: alarv Date: Tue, 26 Nov 2024 13:33:48 +0200 Subject: [PATCH 4/8] fix: do not allow predictions for archived models --- .../kotlin/org/jaqpot/api/service/model/ModelService.kt | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/main/kotlin/org/jaqpot/api/service/model/ModelService.kt b/src/main/kotlin/org/jaqpot/api/service/model/ModelService.kt index 26412689..80ed08ef 100644 --- a/src/main/kotlin/org/jaqpot/api/service/model/ModelService.kt +++ b/src/main/kotlin/org/jaqpot/api/service/model/ModelService.kt @@ -214,6 +214,10 @@ class ModelService( // TODO once there are no models with rawPreprocessor in the database, remove this storeRawPreprocessorToStorage(model) + if (model.archived) { + throw ResponseStatusException(HttpStatus.BAD_REQUEST, "Model with id $modelId is archived") + } + val userId = authenticationFacade.userId val toEntity = datasetDto.toEntity( model, @@ -370,7 +374,7 @@ class ModelService( throw ResponseStatusException(HttpStatus.NOT_FOUND, "Model with id $modelId not found") } - if (existingModel.archived == true) { + if (existingModel.archived) { throw ResponseStatusException(HttpStatus.BAD_REQUEST, "Model with id $modelId is already archived") } @@ -391,7 +395,7 @@ class ModelService( throw ResponseStatusException(HttpStatus.NOT_FOUND, "Model with id $modelId not found") } - if (existingModel.archived != true) { + if (!existingModel.archived) { throw ResponseStatusException(HttpStatus.BAD_REQUEST, "Model with id $modelId is not archived") } From e41e7d9573ca9a206da22e6962c6a448196c51e9 Mon Sep 17 00:00:00 2001 From: alarv Date: Tue, 26 Nov 2024 14:30:42 +0200 Subject: [PATCH 5/8] feat: add archived models purge functionality --- .../jaqpot/api/repository/ModelRepository.kt | 3 ++ .../ModelUpdateAuthorizationLogic.kt | 2 - .../jaqpot/api/service/model/ModelService.kt | 50 +++++++++++++++---- 3 files changed, 43 insertions(+), 12 deletions(-) diff --git a/src/main/kotlin/org/jaqpot/api/repository/ModelRepository.kt b/src/main/kotlin/org/jaqpot/api/repository/ModelRepository.kt index 99a4ca88..641bb4c2 100644 --- a/src/main/kotlin/org/jaqpot/api/repository/ModelRepository.kt +++ b/src/main/kotlin/org/jaqpot/api/repository/ModelRepository.kt @@ -9,6 +9,7 @@ import org.springframework.data.jpa.repository.Query import org.springframework.data.repository.CrudRepository import org.springframework.data.repository.PagingAndSortingRepository import org.springframework.data.repository.query.Param +import java.time.OffsetDateTime import java.util.* @@ -74,4 +75,6 @@ interface ModelRepository : PagingAndSortingRepository, CrudReposit @Transactional @Query("UPDATE Model m SET m.rawPreprocessor = NULL WHERE m.id = :id") fun setRawPreprocessorToNull(@Param("id") id: Long?) + + fun findAllByArchivedIsTrueAndArchivedAtBefore(date: OffsetDateTime): List } diff --git a/src/main/kotlin/org/jaqpot/api/service/authorization/ModelUpdateAuthorizationLogic.kt b/src/main/kotlin/org/jaqpot/api/service/authorization/ModelUpdateAuthorizationLogic.kt index 98e65e7d..5fffc389 100644 --- a/src/main/kotlin/org/jaqpot/api/service/authorization/ModelUpdateAuthorizationLogic.kt +++ b/src/main/kotlin/org/jaqpot/api/service/authorization/ModelUpdateAuthorizationLogic.kt @@ -21,8 +21,6 @@ class ModelUpdateAuthorizationLogic( throw ResponseStatusException(HttpStatus.NOT_FOUND, "Model with id $modelId not found") } - - return authenticationFacade.userId == model.creatorId } diff --git a/src/main/kotlin/org/jaqpot/api/service/model/ModelService.kt b/src/main/kotlin/org/jaqpot/api/service/model/ModelService.kt index 80ed08ef..1c3be8f0 100644 --- a/src/main/kotlin/org/jaqpot/api/service/model/ModelService.kt +++ b/src/main/kotlin/org/jaqpot/api/service/model/ModelService.kt @@ -28,6 +28,7 @@ import org.springframework.data.domain.PageRequest import org.springframework.data.domain.Sort import org.springframework.http.HttpStatus import org.springframework.http.ResponseEntity +import org.springframework.scheduling.annotation.Scheduled import org.springframework.security.access.prepost.PostAuthorize import org.springframework.security.access.prepost.PreAuthorize import org.springframework.stereotype.Service @@ -36,7 +37,7 @@ import org.springframework.web.servlet.support.ServletUriComponentsBuilder import java.net.URI import java.time.OffsetDateTime -private val logger = KotlinLogging.logger {} + const val JAQPOT_METADATA_KEY = "jaqpotMetadata" const val JAQPOT_ROW_ID_KEY = "jaqpotRowId" const val JAQPOT_ROW_LABEL_KEY = "jaqpotRowLabel" @@ -73,6 +74,8 @@ class ModelService( ModelTypeDto.R_TREE_CLASS, ModelTypeDto.R_TREE_REGR ) + const val ARCHIVED_MODEL_EXPIRATION_DAYS = 30L + private val logger = KotlinLogging.logger {} } @PreAuthorize("hasAnyAuthority('admin', 'upci')") @@ -368,6 +371,7 @@ class ModelService( return ResponseEntity.ok(model.toDto(modelCreator, userCanEdit, isAdmin)) } + @WithRateLimitProtectionByUser(limit = 10, intervalInSeconds = 60) @PreAuthorize("@modelUpdateAuthorizationLogic.decide(#root, #modelId)") override fun archiveModel(modelId: Long): ResponseEntity { val existingModel = modelRepository.findById(modelId).orElseThrow { @@ -389,6 +393,7 @@ class ModelService( ) } + @WithRateLimitProtectionByUser(limit = 10, intervalInSeconds = 60) @PreAuthorize("@modelUpdateAuthorizationLogic.decide(#root, #modelId)") override fun unarchiveModel(modelId: Long): ResponseEntity { val existingModel = modelRepository.findById(modelId).orElseThrow { @@ -452,25 +457,50 @@ class ModelService( throw ResponseStatusException(HttpStatus.NOT_FOUND, "Model with id $id not found") } + deleteModel(model) + + return ResponseEntity.noContent().build() + } + + private fun deleteModel(model: Model) { + logger.info { "Deleting model with id ${model.id}" } + model.doas.forEach { - logger.info { "Deleting DOA with id ${it.id} for model with id $id" } + logger.info { "Deleting DOA with id ${it.id} for model with id ${model.id}" } val deletedRawDoa = storageService.deleteRawDoa(it) - logger.info { "Deleted raw DOA for model with id $id: $deletedRawDoa" } + logger.info { "Deleted raw DOA for model with id ${model.id}: $deletedRawDoa" } } - logger.info { "Deleting raw preprocessor for model with id $id" } + logger.info { "Deleting raw preprocessor for model with id ${model.id}" } val deletedRawPreprocessor = storageService.deleteRawPreprocessor(model) - logger.info { "Deleted raw preprocessor for model with id $id: $deletedRawPreprocessor" } + logger.info { "Deleted raw preprocessor for model with id ${model.id}: $deletedRawPreprocessor" } - logger.info { "Deleting raw model for model with id $id" } + logger.info { "Deleting raw model for model with id ${model.id}" } val deletedRawModel = storageService.deleteRawModel(model) - logger.info { "Deleted raw model for model with id $id: $deletedRawModel" } + logger.info { "Deleted raw model for model with id ${model.id}: $deletedRawModel" } - logger.info { "Deleting model with id $id" } modelRepository.delete(model) - logger.info { "Deleted model with id $id" } - return ResponseEntity.noContent().build() + logger.info { "Deleted model with id ${model.id}" } + } + + @Scheduled(cron = "0 0 3 * * *" /* every day at 3:00 AM */) + fun purgeExpiredArchivedModels() { + logger.info { "Purging expired archived models" } + + val expiredArchivedModels = modelRepository.findAllByArchivedIsTrueAndArchivedAtBefore( + OffsetDateTime.now().minusDays(ARCHIVED_MODEL_EXPIRATION_DAYS) + ) + + expiredArchivedModels.forEach { + try { + this.deleteModel(it) + } catch (e: Exception) { + logger.error(e) { "Failed to delete model with id ${it.id}" } + } + } + + logger.info { "Purged ${expiredArchivedModels.size} expired archived models" } } } From 3a86365907a89f0f4d2bb26dbd595b96f5b834a8 Mon Sep 17 00:00:00 2001 From: alarv Date: Tue, 26 Nov 2024 14:44:23 +0200 Subject: [PATCH 6/8] fix: allow access to legacy archived models --- src/main/kotlin/org/jaqpot/api/repository/ModelRepository.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/kotlin/org/jaqpot/api/repository/ModelRepository.kt b/src/main/kotlin/org/jaqpot/api/repository/ModelRepository.kt index 641bb4c2..6747ea20 100644 --- a/src/main/kotlin/org/jaqpot/api/repository/ModelRepository.kt +++ b/src/main/kotlin/org/jaqpot/api/repository/ModelRepository.kt @@ -18,7 +18,7 @@ interface ModelRepository : PagingAndSortingRepository, CrudReposit fun findAllByCreatorIdAndArchivedIsFalse(creatorId: String, pageable: Pageable): Page fun findAllByCreatorIdAndArchivedIsTrue(creatorId: String, pageable: Pageable): Page - fun findOneByLegacyIdAndArchivedIsFalse(legacyId: String): Optional + fun findOneByLegacyId(legacyId: String): Optional @Query( """ From ce4618daa73ae1f4ffae257c857a7b4506e29cde Mon Sep 17 00:00:00 2001 From: alarv Date: Tue, 26 Nov 2024 14:44:42 +0200 Subject: [PATCH 7/8] fix: disable admin deletion of models --- .../jaqpot/api/service/model/ModelService.kt | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/main/kotlin/org/jaqpot/api/service/model/ModelService.kt b/src/main/kotlin/org/jaqpot/api/service/model/ModelService.kt index 1c3be8f0..7395957b 100644 --- a/src/main/kotlin/org/jaqpot/api/service/model/ModelService.kt +++ b/src/main/kotlin/org/jaqpot/api/service/model/ModelService.kt @@ -188,7 +188,7 @@ class ModelService( @PostAuthorize("@getModelAuthorizationLogic.decide(#root)") override fun getLegacyModelById(id: String): ResponseEntity { - val model = modelRepository.findOneByLegacyIdAndArchivedIsFalse(id) + val model = modelRepository.findOneByLegacyId(id) return model.map { val userCanEdit = authenticationFacade.isAdmin || isCreator(authenticationFacade, it) @@ -453,13 +453,14 @@ class ModelService( @CacheEvict("searchModels", allEntries = true) @PreAuthorize("hasAuthority('admin')") override fun deleteModelById(id: Long): ResponseEntity { - val model = modelRepository.findById(id).orElseThrow { - throw ResponseStatusException(HttpStatus.NOT_FOUND, "Model with id $id not found") - } - - deleteModel(model) - - return ResponseEntity.noContent().build() + throw ResponseStatusException(HttpStatus.BAD_REQUEST, "This endpoint is not supported") +// val model = modelRepository.findById(id).orElseThrow { +// throw ResponseStatusException(HttpStatus.NOT_FOUND, "Model with id $id not found") +// } +// +// deleteModel(model) +// +// return ResponseEntity.noContent().build() } private fun deleteModel(model: Model) { From 64ca1acc68dcff19f8ce59ebdb9c2f5bc2a0bf38 Mon Sep 17 00:00:00 2001 From: alarv Date: Tue, 26 Nov 2024 14:55:54 +0200 Subject: [PATCH 8/8] fix: add session to deletion query --- .../jaqpot/api/service/dataset/DatasetService.kt | 2 ++ .../org/jaqpot/api/service/model/ModelService.kt | 16 +++++++++++----- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/src/main/kotlin/org/jaqpot/api/service/dataset/DatasetService.kt b/src/main/kotlin/org/jaqpot/api/service/dataset/DatasetService.kt index ad71961a..4f5d7d43 100644 --- a/src/main/kotlin/org/jaqpot/api/service/dataset/DatasetService.kt +++ b/src/main/kotlin/org/jaqpot/api/service/dataset/DatasetService.kt @@ -1,6 +1,7 @@ package org.jaqpot.api.service.dataset import io.github.oshai.kotlinlogging.KotlinLogging +import jakarta.transaction.Transactional import org.jaqpot.api.DatasetApiDelegate import org.jaqpot.api.mapper.toDto import org.jaqpot.api.mapper.toGetDatasets200ResponseDto @@ -52,6 +53,7 @@ class DatasetService( return ResponseEntity.ok().body(datasets.toGetDatasets200ResponseDto(inputsMap, resultsMap)) } + @Transactional @Scheduled(cron = "0 0 3 * * *" /* every day at 3:00 AM */) fun purgeExpiredDatasets() { logger.info { "Purging expired datasets" } diff --git a/src/main/kotlin/org/jaqpot/api/service/model/ModelService.kt b/src/main/kotlin/org/jaqpot/api/service/model/ModelService.kt index 7395957b..ccd507e0 100644 --- a/src/main/kotlin/org/jaqpot/api/service/model/ModelService.kt +++ b/src/main/kotlin/org/jaqpot/api/service/model/ModelService.kt @@ -466,10 +466,12 @@ class ModelService( private fun deleteModel(model: Model) { logger.info { "Deleting model with id ${model.id}" } - model.doas.forEach { - logger.info { "Deleting DOA with id ${it.id} for model with id ${model.id}" } - val deletedRawDoa = storageService.deleteRawDoa(it) - logger.info { "Deleted raw DOA for model with id ${model.id}: $deletedRawDoa" } + if (model.doas.isNotEmpty()) { + model.doas.forEach { + logger.info { "Deleting DOA with id ${it.id} for model with id ${model.id}" } + val deletedRawDoa = storageService.deleteRawDoa(it) + logger.info { "Deleted raw DOA for model with id ${model.id}: $deletedRawDoa" } + } } logger.info { "Deleting raw preprocessor for model with id ${model.id}" } @@ -485,6 +487,7 @@ class ModelService( logger.info { "Deleted model with id ${model.id}" } } + @Transactional @Scheduled(cron = "0 0 3 * * *" /* every day at 3:00 AM */) fun purgeExpiredArchivedModels() { logger.info { "Purging expired archived models" } @@ -493,15 +496,18 @@ class ModelService( OffsetDateTime.now().minusDays(ARCHIVED_MODEL_EXPIRATION_DAYS) ) + var deletionCount = 0 + expiredArchivedModels.forEach { try { this.deleteModel(it) + deletionCount++ } catch (e: Exception) { logger.error(e) { "Failed to delete model with id ${it.id}" } } } - logger.info { "Purged ${expiredArchivedModels.size} expired archived models" } + logger.info { "Purged $deletionCount expired archived models" } } }