From b440965749991e5ec181e7dbec092e34d9318f21 Mon Sep 17 00:00:00 2001 From: Nariman Abdullin Date: Fri, 15 Jul 2022 15:40:15 +0300 Subject: [PATCH] TestSuitesSource storage ### What's done: - added storage for TestSuitesSource snapshots - refactored discovering tests and test suites - added GIT cloning with new run --- db/v-2/tables/db.changelog-tables.xml | 1 + db/v-2/tables/git.xml | 8 + db/v-2/tables/test_suite.xml | 22 ++ db/v-2/tables/test_suites_source.xml | 31 ++ .../controllers/TestSuitesController.kt | 9 - .../save/backend/repository/GitRepository.kt | 6 + .../backend/repository/TestSuiteRepository.kt | 32 ++ .../repository/TestSuitesSourceRepository.kt | 25 ++ .../save/backend/service/GitService.kt | 55 +++ .../backend/service/OrganizationService.kt | 11 + .../save/backend/service/TestSuitesService.kt | 79 ++--- .../service/TestSuitesSourceService.kt | 173 +++++++++ .../save/backend/storage/DebugInfoStorage.kt | 25 +- .../save/backend/storage/FileStorage.kt | 13 +- .../TestSuitesSourceSnapshotStorage.kt | 42 +++ .../save/backend/utils/ReactorUtils.kt | 16 +- save-cloud-common/build.gradle.kts | 1 + .../com/saveourtool/save/entities/GitDto.kt | 12 +- .../saveourtool/save/entities/Organization.kt | 8 + .../com/saveourtool/save/entities/Project.kt | 8 + .../save/testsuite/TestSuiteDto.kt | 22 +- .../save/testsuite/TestSuiteType.kt | 10 +- .../save/testsuite/TestSuitesSourceDto.kt | 13 + .../testsuite/TestSuitesSourceSnapshotKey.kt | 21 ++ .../save/testsuite/TestSuitesSourceType.kt | 10 + .../com/saveourtool/save/entities/Git.kt | 13 + .../saveourtool/save/entities/TestSuite.kt | 30 +- .../save/entities/TestSuitesSource.kt | 39 +++ .../saveourtool/save/utils/ArchiveUtils.kt | 60 ++++ .../com/saveourtool/save/utils/FileUtils.kt | 25 ++ .../preprocessor/config/TestSuitesRepo.kt | 14 +- .../controllers/DownloadProjectController.kt | 331 +++++------------- .../TestSuitesPreprocessorController.kt | 133 +++++++ .../service/GitPreprocessorService.kt | 88 +++++ .../service/PreprocessorToBackendBridge.kt | 145 ++++++++ .../service/TestDiscoveringService.kt | 96 ++++- .../save/preprocessor/utils/GitUtil.kt | 94 ++++- .../save/preprocessor/utils/WebClientUtil.kt | 58 +++ .../src/main/resources/TestSuitesRepos | 9 +- .../src/main/resources/TestSuitesRepos.json | 9 + .../controllers/DownloadProjectTest.kt | 2 +- 41 files changed, 1420 insertions(+), 379 deletions(-) create mode 100644 db/v-2/tables/test_suites_source.xml create mode 100644 save-backend/src/main/kotlin/com/saveourtool/save/backend/repository/TestSuitesSourceRepository.kt create mode 100644 save-backend/src/main/kotlin/com/saveourtool/save/backend/service/TestSuitesSourceService.kt create mode 100644 save-backend/src/main/kotlin/com/saveourtool/save/backend/storage/TestSuitesSourceSnapshotStorage.kt create mode 100644 save-cloud-common/src/commonMain/kotlin/com/saveourtool/save/testsuite/TestSuitesSourceDto.kt create mode 100644 save-cloud-common/src/commonMain/kotlin/com/saveourtool/save/testsuite/TestSuitesSourceSnapshotKey.kt create mode 100644 save-cloud-common/src/commonMain/kotlin/com/saveourtool/save/testsuite/TestSuitesSourceType.kt create mode 100644 save-cloud-common/src/jvmMain/kotlin/com/saveourtool/save/entities/TestSuitesSource.kt create mode 100644 save-cloud-common/src/jvmMain/kotlin/com/saveourtool/save/utils/ArchiveUtils.kt create mode 100644 save-preprocessor/src/main/kotlin/com/saveourtool/save/preprocessor/controllers/TestSuitesPreprocessorController.kt create mode 100644 save-preprocessor/src/main/kotlin/com/saveourtool/save/preprocessor/service/GitPreprocessorService.kt create mode 100644 save-preprocessor/src/main/kotlin/com/saveourtool/save/preprocessor/service/PreprocessorToBackendBridge.kt create mode 100644 save-preprocessor/src/main/kotlin/com/saveourtool/save/preprocessor/utils/WebClientUtil.kt create mode 100644 save-preprocessor/src/main/resources/TestSuitesRepos.json diff --git a/db/v-2/tables/db.changelog-tables.xml b/db/v-2/tables/db.changelog-tables.xml index 5b35b77064..736307872e 100644 --- a/db/v-2/tables/db.changelog-tables.xml +++ b/db/v-2/tables/db.changelog-tables.xml @@ -12,6 +12,7 @@ + diff --git a/db/v-2/tables/git.xml b/db/v-2/tables/git.xml index 9e39793caf..c046ed2838 100644 --- a/db/v-2/tables/git.xml +++ b/db/v-2/tables/git.xml @@ -9,4 +9,12 @@ + + + + + + + + \ No newline at end of file diff --git a/db/v-2/tables/test_suite.xml b/db/v-2/tables/test_suite.xml index e70e679221..6ee88445f4 100644 --- a/db/v-2/tables/test_suite.xml +++ b/db/v-2/tables/test_suite.xml @@ -15,4 +15,26 @@ + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/db/v-2/tables/test_suites_source.xml b/db/v-2/tables/test_suites_source.xml new file mode 100644 index 0000000000..1f64bb9a56 --- /dev/null +++ b/db/v-2/tables/test_suites_source.xml @@ -0,0 +1,31 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/save-backend/src/main/kotlin/com/saveourtool/save/backend/controllers/TestSuitesController.kt b/save-backend/src/main/kotlin/com/saveourtool/save/backend/controllers/TestSuitesController.kt index fdce4c68a6..b8ab6ba619 100644 --- a/save-backend/src/main/kotlin/com/saveourtool/save/backend/controllers/TestSuitesController.kt +++ b/save-backend/src/main/kotlin/com/saveourtool/save/backend/controllers/TestSuitesController.kt @@ -95,15 +95,6 @@ class TestSuitesController( ) } - /** - * @param testSuiteDtos suites, which need to be marked as obsolete - * @return response entity - */ - @PostMapping("/internal/markObsoleteTestSuites") - @Transactional - fun markObsoleteTestSuites(@RequestBody testSuiteDtos: List) = - ResponseEntity.status(HttpStatus.OK).body(testSuitesService.markObsoleteTestSuites(testSuiteDtos)) - /** * @param testSuiteDtos suites, which need to be deleted * @return response entity diff --git a/save-backend/src/main/kotlin/com/saveourtool/save/backend/repository/GitRepository.kt b/save-backend/src/main/kotlin/com/saveourtool/save/backend/repository/GitRepository.kt index 4c83f9df19..1ff3a315f4 100644 --- a/save-backend/src/main/kotlin/com/saveourtool/save/backend/repository/GitRepository.kt +++ b/save-backend/src/main/kotlin/com/saveourtool/save/backend/repository/GitRepository.kt @@ -20,4 +20,10 @@ interface GitRepository : BaseEntityRepository { * @return git by project */ fun findByProjectId(projectId: Long): Git? + + /** + * @param url + * @return all [Git] entities by [Git.url] + */ + fun findAllByUrl(url: String): List } diff --git a/save-backend/src/main/kotlin/com/saveourtool/save/backend/repository/TestSuiteRepository.kt b/save-backend/src/main/kotlin/com/saveourtool/save/backend/repository/TestSuiteRepository.kt index 81d534ae70..6c437117b1 100644 --- a/save-backend/src/main/kotlin/com/saveourtool/save/backend/repository/TestSuiteRepository.kt +++ b/save-backend/src/main/kotlin/com/saveourtool/save/backend/repository/TestSuiteRepository.kt @@ -1,6 +1,7 @@ package com.saveourtool.save.backend.repository import com.saveourtool.save.entities.TestSuite +import com.saveourtool.save.entities.TestSuitesSource import com.saveourtool.save.testsuite.TestSuiteType import org.springframework.data.repository.query.QueryByExampleExecutor import org.springframework.stereotype.Repository @@ -42,4 +43,35 @@ interface TestSuiteRepository : BaseEntityRepository, QueryByExampleE testRootPath: String, testSuiteRepoUrl: String?, ): TestSuite + + /** + * @param name name of the test suite + * @param source source of the test suite + * @param version version of snapshot of source + * @return matched test suite + */ + fun findByNameAndSourceAndVersion( + name: String, + source: TestSuitesSource, + version: String + ): TestSuite? + + /** + * @param source source of the test suite + * @param version version of snapshot of source + * @return matched test suites + */ + fun findAllBySourceAndVersion( + source: TestSuitesSource, + version: String + ): List + + + /** + * @param source source of the test suite + * @return matched test suites + */ + fun findAllBySource( + source: TestSuitesSource, + ): List } diff --git a/save-backend/src/main/kotlin/com/saveourtool/save/backend/repository/TestSuitesSourceRepository.kt b/save-backend/src/main/kotlin/com/saveourtool/save/backend/repository/TestSuitesSourceRepository.kt new file mode 100644 index 0000000000..1795950785 --- /dev/null +++ b/save-backend/src/main/kotlin/com/saveourtool/save/backend/repository/TestSuitesSourceRepository.kt @@ -0,0 +1,25 @@ +package com.saveourtool.save.backend.repository + +import com.saveourtool.save.entities.TestSuitesSource +import com.saveourtool.save.testsuite.TestSuitesSourceType +import org.springframework.stereotype.Repository + +/** + * Repository of [TestSuitesSource] + */ +@Repository +interface TestSuitesSourceRepository : BaseEntityRepository { + /** + * @param organizationId + * @param name + * @return found entity or null + */ + fun findByOrganizationIdAndName(organizationId: Long, name: String): TestSuitesSource? + + /** + * @param type + * @param additionalInfo + * @return found entity or null + */ + fun findByTypeAndAdditionalInfo(type: TestSuitesSourceType, additionalInfo: String): TestSuitesSource? +} diff --git a/save-backend/src/main/kotlin/com/saveourtool/save/backend/service/GitService.kt b/save-backend/src/main/kotlin/com/saveourtool/save/backend/service/GitService.kt index 0019a28e4f..5213001ab8 100644 --- a/save-backend/src/main/kotlin/com/saveourtool/save/backend/service/GitService.kt +++ b/save-backend/src/main/kotlin/com/saveourtool/save/backend/service/GitService.kt @@ -1,15 +1,26 @@ package com.saveourtool.save.backend.service import com.saveourtool.save.backend.repository.GitRepository +import com.saveourtool.save.backend.utils.justOrNotFound +import com.saveourtool.save.backend.utils.switchToNotFoundIfEmpty import com.saveourtool.save.entities.Git import com.saveourtool.save.entities.GitDto import com.saveourtool.save.entities.Project +import org.springframework.data.repository.findByIdOrNull +import org.springframework.http.HttpStatus import org.springframework.stereotype.Service +import org.springframework.web.bind.annotation.GetMapping +import org.springframework.web.bind.annotation.RequestParam +import org.springframework.web.bind.annotation.RestController +import org.springframework.web.server.ResponseStatusException +import reactor.core.publisher.Flux +import reactor.core.publisher.Mono /** * Service of git */ @Service +@RestController class GitService(private val gitRepository: GitRepository) { /** * @param project @@ -37,4 +48,48 @@ class GitService(private val gitRepository: GitRepository) { * @return git by project id if exists */ fun findByProjectId(projectId: Long) = gitRepository.findByProjectId(projectId) + + @GetMapping("/internal/git") + fun getById(@RequestParam id: Long): Mono = Mono.justOrEmpty(gitRepository.findByIdOrNull(id)) + .map { it.toDto() } + .switchToNotFoundIfEmpty { + "Git entity not found by id $id" + } + + fun findByUrlSubDirectoryAndBranchOrCopy( + url: String, + subDirectory: String, + branch: String, + ): Mono { + val gits = gitRepository.findAllByUrl(url) + return if (gits.isEmpty()) { + Mono.error( + ResponseStatusException( + HttpStatus.NOT_FOUND, + "There is no registered Git configuration for url: $url" + ) + ) + } else { + Mono.fromCallable { + gits.find { it.subDirectory == subDirectory && it.branch == branch } + ?: gits.first() + .createCopy( + subDirectory = subDirectory, + branch = branch, + ) + } + } + } + + private fun Git.createCopy(subDirectory: String, branch: String): Git { + return saveGit( + toDto().copy( + branch = branch, + subDirectory = subDirectory + ), + project.requiredId() + ) + } + + fun findAllByUrl(url: String): List = gitRepository.findAllByUrl(url) } diff --git a/save-backend/src/main/kotlin/com/saveourtool/save/backend/service/OrganizationService.kt b/save-backend/src/main/kotlin/com/saveourtool/save/backend/service/OrganizationService.kt index 8d8945b5a5..3b8f008586 100644 --- a/save-backend/src/main/kotlin/com/saveourtool/save/backend/service/OrganizationService.kt +++ b/save-backend/src/main/kotlin/com/saveourtool/save/backend/service/OrganizationService.kt @@ -1,10 +1,15 @@ package com.saveourtool.save.backend.service import com.saveourtool.save.backend.repository.OrganizationRepository +import com.saveourtool.save.backend.utils.switchToNotFoundIfEmpty import com.saveourtool.save.domain.OrganizationSaveStatus import com.saveourtool.save.entities.Organization import com.saveourtool.save.entities.OrganizationStatus import org.springframework.stereotype.Service +import org.springframework.web.bind.annotation.GetMapping +import org.springframework.web.bind.annotation.PathVariable +import org.springframework.web.bind.annotation.RestController +import reactor.core.publisher.Mono /** * Service for organization @@ -12,6 +17,7 @@ import org.springframework.stereotype.Service * @property organizationRepository */ @Service +@RestController class OrganizationService( private val organizationRepository: OrganizationRepository, ) { @@ -58,6 +64,11 @@ class OrganizationService( */ fun findByName(name: String) = organizationRepository.findByName(name) + @GetMapping("/internal/organization/{name}") + fun getByName(@PathVariable name: String): Mono = Mono.justOrEmpty(findByName(name)).switchToNotFoundIfEmpty { + "Organization not found by name $name" + } + /** * @param organization * @return organization diff --git a/save-backend/src/main/kotlin/com/saveourtool/save/backend/service/TestSuitesService.kt b/save-backend/src/main/kotlin/com/saveourtool/save/backend/service/TestSuitesService.kt index 2878cd6a71..49f68a4d76 100644 --- a/save-backend/src/main/kotlin/com/saveourtool/save/backend/service/TestSuitesService.kt +++ b/save-backend/src/main/kotlin/com/saveourtool/save/backend/service/TestSuitesService.kt @@ -1,11 +1,10 @@ package com.saveourtool.save.backend.service -import com.saveourtool.save.backend.repository.ProjectRepository import com.saveourtool.save.backend.repository.TestExecutionRepository import com.saveourtool.save.backend.repository.TestRepository import com.saveourtool.save.backend.repository.TestSuiteRepository -import com.saveourtool.save.entities.Project import com.saveourtool.save.entities.TestSuite +import com.saveourtool.save.entities.TestSuitesSource import com.saveourtool.save.testsuite.TestSuiteDto import com.saveourtool.save.testsuite.TestSuiteType import com.saveourtool.save.utils.debug @@ -13,8 +12,10 @@ import com.saveourtool.save.utils.info import org.apache.commons.io.FilenameUtils import org.slf4j.LoggerFactory import org.springframework.data.domain.Example +import org.springframework.http.HttpStatus import org.springframework.stereotype.Service -import org.springframework.transaction.annotation.Transactional +import org.springframework.web.server.ResponseStatusException +import reactor.core.publisher.Mono import java.time.LocalDateTime /** @@ -25,7 +26,7 @@ class TestSuitesService( private val testSuiteRepository: TestSuiteRepository, private val testRepository: TestRepository, private val testExecutionRepository: TestExecutionRepository, - private val projectRepository: ProjectRepository, + private val testSuitesSourceService: TestSuitesSourceService, ) { /** * Save new test suites to DB @@ -35,6 +36,7 @@ class TestSuitesService( */ @Suppress("TOO_MANY_LINES_IN_LAMBDA", "UnsafeCallOnNullableType") fun saveTestSuite(testSuitesDto: List): List { + // FIXME: need to check logic about [dateAdded] val testSuites = testSuitesDto .distinctBy { // Same suites may be declared in different directories, we unify them here. @@ -43,13 +45,11 @@ class TestSuitesService( } .map { TestSuite( - type = it.type, name = it.name, description = it.description, - project = it.project, + source = testSuitesSourceService.getByDto(it.source), + version = it.version, dateAdded = null, - testRootPath = FilenameUtils.separatorsToUnix(it.testRootPath), - testSuiteRepoUrl = it.testSuiteRepoUrl, language = it.language ) } @@ -73,6 +73,16 @@ class TestSuitesService( return testSuites.toList() } + /** + * @param dto entity as DTO + * @return entity is found by provided values + */ + fun getByDto(dto: TestSuiteDto): TestSuite = testSuiteRepository.findByNameAndSourceAndVersion( + dto.name, + testSuitesSourceService.getByDto(dto.source), + dto.version + ) ?: throw ResponseStatusException(HttpStatus.NOT_FOUND, "TestSuite (name=${dto.name} in ${dto.source.name} with version ${dto.version}) not found") + /** * @return all standard test suites */ @@ -86,41 +96,16 @@ class TestSuitesService( fun findStandardTestSuitesByName(name: String) = testSuiteRepository.findAllByNameAndType(name, TestSuiteType.STANDARD) - /** - * @param project a project associated with test suites - * @return a list of test suites - */ - @Transactional - fun findTestSuitesByProject(project: Project) = - testSuiteRepository.findByProjectId( - requireNotNull(project.id) { "Cannot find test suites for project with missing id (name=${project.name}, organization=${project.organization.name})" } - ) - /** * @param id * @return test suite with [id] */ fun findTestSuiteById(id: Long) = testSuiteRepository.findById(id) - /** - * Mark provided testSuites as obsolete - * - * @param testSuiteDtos - */ - @Suppress("UnsafeCallOnNullableType") - fun markObsoleteTestSuites(testSuiteDtos: List) { - testSuiteDtos.forEach { testSuiteDto -> - val testSuite = testSuiteRepository.findByNameAndTypeAndTestRootPathAndTestSuiteRepoUrl( - testSuiteDto.name, - testSuiteDto.type!!, - testSuiteDto.testRootPath, - testSuiteDto.testSuiteRepoUrl, - ) - log.info { "Mark test suite ${testSuite.name} with id ${testSuite.id} as obsolete" } - testSuite.type = TestSuiteType.OBSOLETE_STANDARD - testSuiteRepository.save(testSuite) - } - } + fun findBySourceAndVersion( + source: TestSuitesSource, + version: String + ) = Mono.fromCallable { testSuiteRepository.findAllBySourceAndVersion(source, version) } /** * Delete testSuites and related tests & test executions from DB @@ -131,21 +116,17 @@ class TestSuitesService( fun deleteTestSuiteDto(testSuiteDtos: List) { testSuiteDtos.forEach { testSuiteDto -> // Get test suite id by testSuiteDto - val testSuiteId = testSuiteRepository.findByNameAndTypeAndTestRootPathAndTestSuiteRepoUrl( - testSuiteDto.name, - testSuiteDto.type!!, - testSuiteDto.testRootPath, - testSuiteDto.testSuiteRepoUrl, - ).id!! + val testSuiteId = getByDto(testSuiteDto).requiredId() // Get test ids related to the current testSuiteId - val testIds = testRepository.findAllByTestSuiteId(testSuiteId).map { it.id } + val testIds = testRepository.findAllByTestSuiteId(testSuiteId).map { it.requiredId() } testIds.forEach { testId -> // Executions could be absent - testExecutionRepository.findByTestId(testId!!).ifPresent { testExecution -> + testExecutionRepository.findByTestId(testId).ifPresent { testExecution -> // Delete test executions - log.debug { "Delete test execution with id ${testExecution.id}" } - testExecutionRepository.deleteById(testExecution.id!!) + val testExecutionId = testExecution.requiredId() + log.debug { "Delete test execution with id $testExecutionId" } + testExecutionRepository.deleteById(testExecutionId) } // Delete tests log.debug { "Delete test with id $testId" } @@ -156,6 +137,10 @@ class TestSuitesService( } } + fun getMaxVersion(source: TestSuitesSource): String? = testSuiteRepository.findAllBySource(source) + .maxByOrNull { it.dateAdded!! } + ?.version + companion object { private val log = LoggerFactory.getLogger(TestSuitesService::class.java) } diff --git a/save-backend/src/main/kotlin/com/saveourtool/save/backend/service/TestSuitesSourceService.kt b/save-backend/src/main/kotlin/com/saveourtool/save/backend/service/TestSuitesSourceService.kt new file mode 100644 index 0000000000..0c2703961e --- /dev/null +++ b/save-backend/src/main/kotlin/com/saveourtool/save/backend/service/TestSuitesSourceService.kt @@ -0,0 +1,173 @@ +package com.saveourtool.save.backend.service + +import com.saveourtool.save.backend.repository.TestSuitesSourceRepository +import com.saveourtool.save.backend.storage.TestSuitesSourceSnapshotStorage +import com.saveourtool.save.backend.utils.switchToNotFoundIfEmpty +import com.saveourtool.save.entities.Organization +import com.saveourtool.save.entities.TestSuitesSource +import com.saveourtool.save.testsuite.TestSuitesSourceDto +import com.saveourtool.save.testsuite.TestSuitesSourceSnapshotKey +import com.saveourtool.save.testsuite.TestSuitesSourceType +import com.saveourtool.save.utils.getLogger +import com.saveourtool.save.utils.info +import org.slf4j.Logger +import org.springframework.http.HttpStatus +import org.springframework.http.MediaType +import org.springframework.stereotype.Service +import org.springframework.web.bind.annotation.* +import org.springframework.web.server.ResponseStatusException +import reactor.core.publisher.Flux +import reactor.core.publisher.Mono +import java.nio.ByteBuffer +import reactor.kotlin.core.util.function.component1 +import reactor.kotlin.core.util.function.component2 + +/** + * Service for [com.saveourtool.save.entities.TestSuitesSource] + */ +@Service +@RestController +class TestSuitesSourceService( + private val testSuitesSourceRepository: TestSuitesSourceRepository, + private val organizationService: OrganizationService, + private val testSuitesSourceSnapshotStorage: TestSuitesSourceSnapshotStorage, + private val testSuitesService: TestSuitesService, + private val gitService: GitService, +) { + /** + * @param organization [TestSuitesSource.organization] + * @param name [TestSuitesSource.name] + * @return entity of [TestSuitesSource] or null + */ + fun findByName(organization: Organization, name: String) = + testSuitesSourceRepository.findByOrganizationIdAndName(organization.requiredId(), name) + + /** + * @param organizationName [Organization.name] from [TestSuitesSource.organization] + * @param name [TestSuitesSource.name] + * @return entity of [TestSuitesSource] or error + */ + + fun getByName(organizationName: String, name: String): Mono = organizationService.getByName(organizationName) + .flatMap { organization -> + Mono.justOrEmpty(findByName(organization, name)).switchToNotFoundIfEmpty { + "TestSuitesSource not found by name $name in $organizationName" + } + } + + @GetMapping("/internal/test-suites-source/{organizationName}/{name}") + fun findAsDtoByName(@PathVariable organizationName: String, @PathVariable name: String): Mono = + organizationService.getByName(organizationName).flatMap { organization -> + Mono.justOrEmpty(findByName(organization, name)).switchToNotFoundIfEmpty { + "TestSuitesSource not found by name $name for organization ${organization.name}" + } + }.map { it.toDto() } + + + /** + * @param organization [TestSuitesSource.organization] + * @param name [TestSuitesSource.name] + * @return entity of [TestSuitesSource] + * @throws ResponseStatusException entity not found + */ + fun getByName(organization: Organization, name: String) = findByName(organization, name) + ?: throw ResponseStatusException(HttpStatus.NOT_FOUND, "TestSuitesSource (name=$name in organization=${organization.name}) not found") + + + /** + * @param dto entity as dto [TestSuitesSourceDto] + * @return entity of [TestSuitesSource] + * @throws ResponseStatusException entity not found + */ + fun getByDto(dto: TestSuitesSourceDto) = getByName(dto.organization, dto.name) + + /** + * @param dto entity to save as DTO [TestSuitesSourceDto] + * @return saved entity as [TestSuitesSource] + */ + fun createNew(dto: TestSuitesSourceDto) = testSuitesSourceRepository.save( + TestSuitesSource( + organization = dto.organization, + type = dto.type, + name = dto.name, + description = dto.description, + additionalInfo = dto.additionalInfo, + ) + ) + + @PostMapping("/internal/test-suites-source/{organizationName}/{name}/{version}/upload-snapshot", consumes = [MediaType.MULTIPART_FORM_DATA_VALUE]) + fun uploadSnapshot( + @PathVariable organizationName: String, + @PathVariable name: String, + @PathVariable version: String, + @RequestPart content: Flux + ): Mono { + return findAsDtoByName(organizationName, name) + .map { TestSuitesSourceSnapshotKey(it, version) } + .flatMap { key -> + testSuitesSourceSnapshotStorage.upload(key, content).map { writtenBytes -> + log.info { "Saved ($writtenBytes bytes) snapshot of ${key.testSuitesSourceName} in ${key.organizationName} with version $version" } + } + } + } + + @GetMapping("/internal/test-suites-source/{organizationName}/{name}/{version}/contains") + fun containsSnapshot( + @PathVariable organizationName: String, + @PathVariable name: String, + @PathVariable version: String, + ): Mono { + return findAsDtoByName(organizationName, name) + .map { TestSuitesSourceSnapshotKey(it, version) } + .flatMap { key -> + testSuitesSourceSnapshotStorage.doesExist(key) + } + } + + @GetMapping("/internal/test-suites-source/{organizationName}/{name}/latest") + fun getLatestVersion( + @PathVariable organizationName: String, + @PathVariable name: String, + ): Mono { + return getByName(organizationName, name) + .flatMap { Mono.justOrEmpty(testSuitesService.getMaxVersion(it)) } + } + + @GetMapping("/internal/test-suites-source/{organizationName}/{name}/{version}/get-test-suites") + fun getTestSuites( + @PathVariable organizationName: String, + @PathVariable name: String, + @PathVariable version: String, + ) = getByName(organizationName, name) + .flatMap { source -> + testSuitesService.findBySourceAndVersion(source, version) + } + + @GetMapping("/internal/test-suites-source/{organizationName}/get-or-create") + fun getOrCreate( + @PathVariable organizationName: String, + @RequestParam gitUrl: String, + @RequestParam subDirectory: String, + @RequestParam branch: String, + ): Mono = organizationService.getByName(organizationName) + .zipWhen { + gitService.findByUrlSubDirectoryAndBranchOrCopy(gitUrl, subDirectory, branch) + } + .map { (organization, git) -> + val additionalInfo = git.requiredId().toString() + testSuitesSourceRepository.findByTypeAndAdditionalInfo(TestSuitesSourceType.GIT, additionalInfo) + ?: createNew( + TestSuitesSourceDto( + organization, + git.defaultTestSuitesSourceName(), + "auto created test suites source by git coordinates", + TestSuitesSourceType.GIT, + additionalInfo + ) + ) + } + + companion object { + private val log: Logger = getLogger() + } +} diff --git a/save-backend/src/main/kotlin/com/saveourtool/save/backend/storage/DebugInfoStorage.kt b/save-backend/src/main/kotlin/com/saveourtool/save/backend/storage/DebugInfoStorage.kt index d0d61874be..dd05e652d8 100644 --- a/save-backend/src/main/kotlin/com/saveourtool/save/backend/storage/DebugInfoStorage.kt +++ b/save-backend/src/main/kotlin/com/saveourtool/save/backend/storage/DebugInfoStorage.kt @@ -9,6 +9,8 @@ import com.saveourtool.save.utils.debug import com.saveourtool.save.utils.getLogger import com.fasterxml.jackson.databind.ObjectMapper +import com.saveourtool.save.utils.countPartsTill +import com.saveourtool.save.utils.pathNamesTill import org.slf4j.Logger import org.springframework.stereotype.Service import reactor.core.publisher.Mono @@ -33,7 +35,8 @@ class DebugInfoStorage( * @param pathToContent * @return true if path endsWith [SUFFIX_FILE_NAME] */ - override fun isKey(rootDir: Path, pathToContent: Path): Boolean = pathToContent.name.endsWith(SUFFIX_FILE_NAME) + override fun isKey(rootDir: Path, pathToContent: Path): Boolean = + pathToContent.name.endsWith(SUFFIX_FILE_NAME) && pathToContent.countPartsTill(rootDir) == PATH_PARTS_COUNT /** * @param rootDir @@ -42,18 +45,13 @@ class DebugInfoStorage( */ @Suppress("MAGIC_NUMBER", "MagicNumber") override fun buildKey(rootDir: Path, pathToContent: Path): Pair { - val folderNames = generateSequence(pathToContent, Path::getParent) - .takeWhile { it != rootDir } - .map { it.name } - .toList() - require(folderNames.size == 5) { - "Invalid path to debugInfo: $pathToContent" - } - val testName = folderNames[0].dropLast(SUFFIX_FILE_NAME.length) - val testLocation = folderNames[1] - val testSuiteName = folderNames[2] - val pluginName = folderNames[3] - val executionId = folderNames[4].toLong() + val pathNames = pathToContent.pathNamesTill(rootDir) + + val testName = pathNames[0].dropLast(SUFFIX_FILE_NAME.length) + val testLocation = pathNames[1] + val testSuiteName = pathNames[2] + val pluginName = pathNames[3] + val executionId = pathNames[4].toLong() return Pair( executionId, TestResultLocation(testSuiteName, pluginName, testLocation, testName) @@ -98,5 +96,6 @@ class DebugInfoStorage( companion object { private val log: Logger = getLogger() private const val SUFFIX_FILE_NAME = "-debug.json" + private const val PATH_PARTS_COUNT = 5 } } diff --git a/save-backend/src/main/kotlin/com/saveourtool/save/backend/storage/FileStorage.kt b/save-backend/src/main/kotlin/com/saveourtool/save/backend/storage/FileStorage.kt index a44cb02f9d..24c8c5de03 100644 --- a/save-backend/src/main/kotlin/com/saveourtool/save/backend/storage/FileStorage.kt +++ b/save-backend/src/main/kotlin/com/saveourtool/save/backend/storage/FileStorage.kt @@ -6,6 +6,7 @@ import com.saveourtool.save.domain.FileKey import com.saveourtool.save.domain.ProjectCoordinates import com.saveourtool.save.domain.ShortFileInfo import com.saveourtool.save.storage.AbstractFileBasedStorage +import com.saveourtool.save.utils.countPartsTill import org.springframework.http.codec.multipart.FilePart import org.springframework.stereotype.Service import reactor.core.publisher.Flux @@ -48,13 +49,7 @@ class FileStorage( * @param pathToContent * @return true if there is 4 parts between pathToContent and rootDir */ - @Suppress("MAGIC_NUMBER", "MagicNumber") - override fun isKey(rootDir: Path, pathToContent: Path): Boolean { - val partsCount = generateSequence(pathToContent, Path::getParent) - .takeWhile { it != rootDir } - .count() - return partsCount == 4 // organization + project + uploadedMills + fileName - } + override fun isKey(rootDir: Path, pathToContent: Path): Boolean = pathToContent.countPartsTill(rootDir) == PATH_PARTS_COUNT /** * @param projectCoordinates @@ -144,4 +139,8 @@ class FileStorage( ) } } + + companion object { + private const val PATH_PARTS_COUNT = 4 // organization + project + uploadedMills + fileName + } } diff --git a/save-backend/src/main/kotlin/com/saveourtool/save/backend/storage/TestSuitesSourceSnapshotStorage.kt b/save-backend/src/main/kotlin/com/saveourtool/save/backend/storage/TestSuitesSourceSnapshotStorage.kt new file mode 100644 index 0000000000..d320d5bcec --- /dev/null +++ b/save-backend/src/main/kotlin/com/saveourtool/save/backend/storage/TestSuitesSourceSnapshotStorage.kt @@ -0,0 +1,42 @@ +package com.saveourtool.save.backend.storage + +import com.saveourtool.save.backend.configs.ConfigProperties +import com.saveourtool.save.storage.AbstractFileBasedStorage +import com.saveourtool.save.testsuite.TestSuitesSourceSnapshotKey +import com.saveourtool.save.utils.TAR_EXTENSION +import com.saveourtool.save.utils.countPartsTill +import org.springframework.stereotype.Component +import java.nio.file.Path +import kotlin.io.path.div +import kotlin.io.path.name + +/** + * Storage for snapshots of [com.saveourtool.save.entities.TestSuitesSource] + */ +@Component +class TestSuitesSourceSnapshotStorage( + configProperties: ConfigProperties, +) : AbstractFileBasedStorage(Path.of(configProperties.fileStorage.location) / "testSuites") { + /** + * @param rootDir + * @param pathToContent + * @return true if there is 4 parts between pathToContent and rootDir and ends with [TAR_EXTENSION] + */ + override fun isKey(rootDir: Path, pathToContent: Path): Boolean = + pathToContent.endsWith(TAR_EXTENSION) && pathToContent.countPartsTill(rootDir) == PATH_PARTS_COUNT + + override fun buildKey(rootDir: Path, pathToContent: Path): TestSuitesSourceSnapshotKey = + TestSuitesSourceSnapshotKey( + pathToContent.parent.parent.name, + pathToContent.parent.name, + pathToContent.name.dropLast(TAR_EXTENSION.length) + ) + + override fun buildPathToContent(rootDir: Path, key: TestSuitesSourceSnapshotKey): Path = with(key) { + return rootDir / organizationName / testSuitesSourceName / "$version$TAR_EXTENSION" + } + + companion object { + private const val PATH_PARTS_COUNT = 3 // organizationName + testSuitesSourceName + version.tar + } +} diff --git a/save-backend/src/main/kotlin/com/saveourtool/save/backend/utils/ReactorUtils.kt b/save-backend/src/main/kotlin/com/saveourtool/save/backend/utils/ReactorUtils.kt index 5c9daa6185..325df440c5 100644 --- a/save-backend/src/main/kotlin/com/saveourtool/save/backend/utils/ReactorUtils.kt +++ b/save-backend/src/main/kotlin/com/saveourtool/save/backend/utils/ReactorUtils.kt @@ -12,6 +12,7 @@ import reactor.core.publisher.Flux import reactor.core.publisher.Mono import reactor.kotlin.core.publisher.switchIfEmpty import reactor.kotlin.core.publisher.toFlux +import reactor.util.function.Tuple2 import java.io.InputStream import java.io.SequenceInputStream import java.nio.ByteBuffer @@ -73,7 +74,14 @@ inline fun Flux.readAsJson(objectMapper: ObjectMapper): * @param message * @return [Mono] containing [data] or [Mono.error] with 404 status otherwise */ -fun justOrNotFound(data: Optional, message: String? = null) = Mono.justOrEmpty(data) - .switchIfEmpty { - Mono.error(ResponseStatusException(HttpStatus.NOT_FOUND, message)) - } +fun justOrNotFound(data: Optional, message: String? = null) = Mono.justOrEmpty(data).switchToNotFoundIfEmpty { + message +} + +/** + * @param messageCreator + * @return original [Mono] or [Mono.error] with 404 status otherwise + */ +fun Mono.switchToNotFoundIfEmpty(messageCreator: (() -> String?) = { null }) = this.switchIfEmpty { + Mono.error(ResponseStatusException(HttpStatus.NOT_FOUND, messageCreator())) +} diff --git a/save-cloud-common/build.gradle.kts b/save-cloud-common/build.gradle.kts index bbcb686053..ea1b5a70f0 100644 --- a/save-cloud-common/build.gradle.kts +++ b/save-cloud-common/build.gradle.kts @@ -48,6 +48,7 @@ kotlin { implementation(libs.hibernate.jpa21.api) api(libs.slf4j.api) implementation(libs.reactor.kotlin.extensions) + implementation(libs.commons.compress) } } val jvmTest by getting { diff --git a/save-cloud-common/src/commonMain/kotlin/com/saveourtool/save/entities/GitDto.kt b/save-cloud-common/src/commonMain/kotlin/com/saveourtool/save/entities/GitDto.kt index f603ff09cf..69dd327f6f 100644 --- a/save-cloud-common/src/commonMain/kotlin/com/saveourtool/save/entities/GitDto.kt +++ b/save-cloud-common/src/commonMain/kotlin/com/saveourtool/save/entities/GitDto.kt @@ -16,4 +16,14 @@ data class GitDto( val password: String? = null, val branch: String? = null, val hash: String? = null, -) + val subDirectory: String +) { + /** + * @return default name fot [com.saveourtool.save.entities.TestSuitesSource] + */ + fun defaultTestSuitesSourceName() = buildString { + append("Git:$url") + append("/$branch") + append("/$subDirectory") + } +} diff --git a/save-cloud-common/src/commonMain/kotlin/com/saveourtool/save/entities/Organization.kt b/save-cloud-common/src/commonMain/kotlin/com/saveourtool/save/entities/Organization.kt index 5ad7f7e505..058704ea10 100644 --- a/save-cloud-common/src/commonMain/kotlin/com/saveourtool/save/entities/Organization.kt +++ b/save-cloud-common/src/commonMain/kotlin/com/saveourtool/save/entities/Organization.kt @@ -33,6 +33,14 @@ data class Organization( @GeneratedValue var id: Long? = null + /** + * @return [id] as not null with validating + * @throws IllegalArgumentException when [id] is not set that means entity is not saved yet + */ + fun requiredId(): Long = requireNotNull(id) { + "Entity is not saved yet: $this" + } + companion object { /** * Create a stub for testing. diff --git a/save-cloud-common/src/commonMain/kotlin/com/saveourtool/save/entities/Project.kt b/save-cloud-common/src/commonMain/kotlin/com/saveourtool/save/entities/Project.kt index 6afd68ac59..d9bf9148a0 100644 --- a/save-cloud-common/src/commonMain/kotlin/com/saveourtool/save/entities/Project.kt +++ b/save-cloud-common/src/commonMain/kotlin/com/saveourtool/save/entities/Project.kt @@ -51,6 +51,14 @@ data class Project( @GeneratedValue var id: Long? = null + /** + * @return [id] as not null with validating + * @throws IllegalArgumentException when [id] is not set that means entity is not saved yet + */ + fun requiredId(): Long = requireNotNull(id) { + "Entity is not saved yet: $this" + } + companion object { /** * Create a stub for testing. Since all fields are mutable, only required ones can be set after calling this method. diff --git a/save-cloud-common/src/commonMain/kotlin/com/saveourtool/save/testsuite/TestSuiteDto.kt b/save-cloud-common/src/commonMain/kotlin/com/saveourtool/save/testsuite/TestSuiteDto.kt index 76e168978e..3884f9045a 100644 --- a/save-cloud-common/src/commonMain/kotlin/com/saveourtool/save/testsuite/TestSuiteDto.kt +++ b/save-cloud-common/src/commonMain/kotlin/com/saveourtool/save/testsuite/TestSuiteDto.kt @@ -1,27 +1,21 @@ package com.saveourtool.save.testsuite -import com.saveourtool.save.entities.Project - import kotlinx.serialization.Serializable /** - * @property type [TestSuite.type] - * @property name [TestSuite.name] - * @property project [TestSuite.project] - * @property testRootPath [TestSuite.testRootPath] - * @property testSuiteRepoUrl url of the repo with test suites - * @property description [TestSuite.description] - * @property language [TestSuite.language] - * @property tags [TestSuite.tags] + * @property name [com.saveourtool.save.entities.TestSuite.name] + * @property description [com.saveourtool.save.entities.TestSuite.description] + * @property source [com.saveourtool.save.entities.TestSuite.source] + * @property version [com.saveourtool.save.entities.TestSuite.testRootPath] + * @property language [com.saveourtool.save.entities.TestSuite.language] + * @property tags [com.saveourtool.save.entities.TestSuite.tags] */ @Serializable data class TestSuiteDto( - val type: TestSuiteType?, val name: String, val description: String?, - val project: Project? = null, - val testRootPath: String, - val testSuiteRepoUrl: String? = null, + val source: TestSuitesSourceDto, + val version: String, val language: String? = null, val tags: List? = null, ) diff --git a/save-cloud-common/src/commonMain/kotlin/com/saveourtool/save/testsuite/TestSuiteType.kt b/save-cloud-common/src/commonMain/kotlin/com/saveourtool/save/testsuite/TestSuiteType.kt index 216220a441..e3a9b27025 100644 --- a/save-cloud-common/src/commonMain/kotlin/com/saveourtool/save/testsuite/TestSuiteType.kt +++ b/save-cloud-common/src/commonMain/kotlin/com/saveourtool/save/testsuite/TestSuiteType.kt @@ -7,11 +7,6 @@ import kotlinx.serialization.Serializable */ @Serializable enum class TestSuiteType { - /** - * Type Obsolete Standard for old test suites - */ - OBSOLETE_STANDARD, - /** * Type Project */ @@ -21,5 +16,10 @@ enum class TestSuiteType { * Type Standard */ STANDARD, + + /** + * Type Obsolete for old test suites + */ + OBSOLETE, ; } diff --git a/save-cloud-common/src/commonMain/kotlin/com/saveourtool/save/testsuite/TestSuitesSourceDto.kt b/save-cloud-common/src/commonMain/kotlin/com/saveourtool/save/testsuite/TestSuitesSourceDto.kt new file mode 100644 index 0000000000..631bc875ec --- /dev/null +++ b/save-cloud-common/src/commonMain/kotlin/com/saveourtool/save/testsuite/TestSuitesSourceDto.kt @@ -0,0 +1,13 @@ +package com.saveourtool.save.testsuite + +import com.saveourtool.save.entities.Organization +import kotlinx.serialization.Serializable + +@Serializable +data class TestSuitesSourceDto( + val organization: Organization, + val name: String, + val description: String?, + val type: TestSuitesSourceType, + val additionalInfo: String, +) diff --git a/save-cloud-common/src/commonMain/kotlin/com/saveourtool/save/testsuite/TestSuitesSourceSnapshotKey.kt b/save-cloud-common/src/commonMain/kotlin/com/saveourtool/save/testsuite/TestSuitesSourceSnapshotKey.kt new file mode 100644 index 0000000000..8cd1100d02 --- /dev/null +++ b/save-cloud-common/src/commonMain/kotlin/com/saveourtool/save/testsuite/TestSuitesSourceSnapshotKey.kt @@ -0,0 +1,21 @@ +package com.saveourtool.save.testsuite + +import kotlinx.serialization.Serializable + +/** + * @param organizationName + * @param testSuitesSourceName + * @param version + */ +@Serializable +data class TestSuitesSourceSnapshotKey( + val organizationName: String, + val testSuitesSourceName: String, + val version: String, +) { + constructor(testSuitesSourceDto: TestSuitesSourceDto, version: String) : this( + testSuitesSourceDto.organization.name, + testSuitesSourceDto.name, + version + ) +} diff --git a/save-cloud-common/src/commonMain/kotlin/com/saveourtool/save/testsuite/TestSuitesSourceType.kt b/save-cloud-common/src/commonMain/kotlin/com/saveourtool/save/testsuite/TestSuitesSourceType.kt new file mode 100644 index 0000000000..10555654e5 --- /dev/null +++ b/save-cloud-common/src/commonMain/kotlin/com/saveourtool/save/testsuite/TestSuitesSourceType.kt @@ -0,0 +1,10 @@ +package com.saveourtool.save.testsuite + +/** + * Type of [com.saveourtool.save.entities.TestSuitesSource] describes location + */ +enum class TestSuitesSourceType { + GIT, + STANDARD, + ; +} \ No newline at end of file diff --git a/save-cloud-common/src/jvmMain/kotlin/com/saveourtool/save/entities/Git.kt b/save-cloud-common/src/jvmMain/kotlin/com/saveourtool/save/entities/Git.kt index 1278d62ee5..d146d9aaed 100644 --- a/save-cloud-common/src/jvmMain/kotlin/com/saveourtool/save/entities/Git.kt +++ b/save-cloud-common/src/jvmMain/kotlin/com/saveourtool/save/entities/Git.kt @@ -12,6 +12,7 @@ import javax.persistence.OneToOne * @property username username to credential * @property password password to credential * @property branch branch to clone + * @property subDirectory * @property project */ @Entity @@ -20,6 +21,7 @@ class Git( var username: String? = null, var password: String? = null, var branch: String? = null, + var subDirectory: String, @OneToOne @JoinColumn(name = "project_id") @@ -33,5 +35,16 @@ class Git( username = username, password = password, branch = branch, + subDirectory = subDirectory ) + + /** + * @return default name fot [com.saveourtool.save.entities.TestSuitesSource] + */ + fun defaultTestSuitesSourceName() = buildString { + append(url) + append("/tree") + append(branch ?: "master") + append("/$subDirectory") + } } diff --git a/save-cloud-common/src/jvmMain/kotlin/com/saveourtool/save/entities/TestSuite.kt b/save-cloud-common/src/jvmMain/kotlin/com/saveourtool/save/entities/TestSuite.kt index fa59159527..bc2150dd85 100644 --- a/save-cloud-common/src/jvmMain/kotlin/com/saveourtool/save/entities/TestSuite.kt +++ b/save-cloud-common/src/jvmMain/kotlin/com/saveourtool/save/entities/TestSuite.kt @@ -1,45 +1,35 @@ package com.saveourtool.save.entities import com.saveourtool.save.testsuite.TestSuiteDto -import com.saveourtool.save.testsuite.TestSuiteType import java.time.LocalDateTime import javax.persistence.Entity -import javax.persistence.EnumType -import javax.persistence.Enumerated import javax.persistence.JoinColumn import javax.persistence.ManyToOne /** - * @property type type of the test suite, one of [TestSuiteType] * @property name name of the test suite - * @property project project, which this test suite belongs to - * @property dateAdded date and time, when this test suite was added to the project - * @property testRootPath location of save.properties file for this test suite, relative to project's root directory - * @property testSuiteRepoUrl url of the repo with test suites * @property description description of the test suite + * @property source source, which this test suite is created from + * @property version version of source, which this test suite is created from + * @property dateAdded date and time, when this test suite was added to the project * @property language * @property tags */ @Suppress("LongParameterList") @Entity class TestSuite( - @Enumerated(EnumType.STRING) - var type: TestSuiteType? = null, - var name: String = "Undefined", var description: String? = "Undefined", @ManyToOne - @JoinColumn(name = "project_id") - var project: Project? = null, + @JoinColumn(name = "source_id") + var source: TestSuitesSource, - var dateAdded: LocalDateTime? = null, - - var testRootPath: String, + var version: String, - var testSuiteRepoUrl: String? = null, + var dateAdded: LocalDateTime? = null, var language: String? = null, @@ -55,12 +45,10 @@ class TestSuite( */ fun toDto() = TestSuiteDto( - this.type, this.name, this.description, - this.project, - this.testRootPath, - this.testSuiteRepoUrl, + this.source.toDto(), + this.version, this.language, this.tagsAsList(), ) diff --git a/save-cloud-common/src/jvmMain/kotlin/com/saveourtool/save/entities/TestSuitesSource.kt b/save-cloud-common/src/jvmMain/kotlin/com/saveourtool/save/entities/TestSuitesSource.kt new file mode 100644 index 0000000000..f37646bf39 --- /dev/null +++ b/save-cloud-common/src/jvmMain/kotlin/com/saveourtool/save/entities/TestSuitesSource.kt @@ -0,0 +1,39 @@ +package com.saveourtool.save.entities + +import com.saveourtool.save.testsuite.TestSuitesSourceDto +import com.saveourtool.save.testsuite.TestSuitesSourceType +import javax.persistence.FetchType +import javax.persistence.JoinColumn +import javax.persistence.ManyToOne + +/** + * @param organization which this test suites source belongs to + * @param name unique name of [TestSuitesSource] + * @param description free text + * @param type + * @param additionalInfo additional info of source, it depends on [type] + */ +@Entity +class TestSuitesSource( + @ManyToOne(fetch = FetchType.LAZY) + @JoinColumn(name = "organization_id") + var organization: Organization, + + var name: String, + var description: String?, + + var type: TestSuitesSourceType, + var additionalInfo: String, +) : BaseEntity() { + + /** + * @return entity as dto [TestSuitesSourceDto] + */ + fun toDto(): TestSuitesSourceDto = TestSuitesSourceDto( + organization = organization, + name = name, + description = description, + type = type, + additionalInfo = additionalInfo, + ) +} diff --git a/save-cloud-common/src/jvmMain/kotlin/com/saveourtool/save/utils/ArchiveUtils.kt b/save-cloud-common/src/jvmMain/kotlin/com/saveourtool/save/utils/ArchiveUtils.kt new file mode 100644 index 0000000000..009affbd75 --- /dev/null +++ b/save-cloud-common/src/jvmMain/kotlin/com/saveourtool/save/utils/ArchiveUtils.kt @@ -0,0 +1,60 @@ +/** + * This file contains util methods to work with archives + */ + +package com.saveourtool.save.utils + +import org.apache.commons.compress.archivers.tar.TarArchiveInputStream +import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream +import org.apache.commons.compress.utils.IOUtils +import java.nio.file.Files +import java.nio.file.Path +import kotlin.io.path.inputStream +import kotlin.io.path.outputStream + +const val TAR_EXTENSION = ".tar" + +/** + * Extract path as TAR archive to provided directory + * + * @param targetPath + */ +@Suppress("NestedBlockDepth") +fun Path.extractTarTo(targetPath: Path) { + this.inputStream().use { buffIn -> + TarArchiveInputStream(buffIn).use { archiveIn -> + generateSequence { archiveIn.nextTarEntry }.forEach { archiveEntry -> + val extractedPath = targetPath.resolve(archiveEntry.name) + if (archiveEntry.isDirectory) { + Files.createDirectories(extractedPath) + } else { + extractedPath.outputStream().buffered().use { + IOUtils.copy(archiveIn, it) + } + } + } + } + } +} + +/** + * Compress path as TAR archive to provided file + * + * @param targetPath + */ +@Suppress("NestedBlockDepth") +fun Path.compressAsTarTo(targetPath: Path) { + targetPath.outputStream().buffered().use { buffOut -> + TarArchiveOutputStream(buffOut).use { archiveOut -> + Files.walk(this).forEach { path -> + val archiveEntry = archiveOut.createArchiveEntry(path, path.relativize(this).toString()) + archiveOut.putArchiveEntry(archiveEntry) + if (Files.isRegularFile(path)) { + path.inputStream().use { IOUtils.copy(it, archiveOut) } + } + archiveOut.closeArchiveEntry() + } + archiveOut.finish() + } + } +} \ No newline at end of file diff --git a/save-cloud-common/src/jvmMain/kotlin/com/saveourtool/save/utils/FileUtils.kt b/save-cloud-common/src/jvmMain/kotlin/com/saveourtool/save/utils/FileUtils.kt index bf73e1245f..6cb23b7b71 100644 --- a/save-cloud-common/src/jvmMain/kotlin/com/saveourtool/save/utils/FileUtils.kt +++ b/save-cloud-common/src/jvmMain/kotlin/com/saveourtool/save/utils/FileUtils.kt @@ -8,10 +8,12 @@ import org.springframework.core.io.buffer.DefaultDataBufferFactory import reactor.core.publisher.Flux import java.io.File import java.io.FileNotFoundException +import java.nio.ByteBuffer import java.nio.file.Files import java.nio.file.Path import java.nio.file.StandardCopyOption import kotlin.io.path.exists +import kotlin.io.path.name private const val DEFAULT_BUFFER_SIZE = 4096 @@ -25,6 +27,11 @@ fun Path.toDataBufferFlux(): Flux = if (exists()) { Flux.empty() } +/** + * @return content of file as [Flux] of [ByteBuffer] + */ +fun Path.toByteBufferFlux(): Flux = this.toDataBufferFlux().map { it.asByteBuffer() } + /** * Move [source] into [destinationDir], while also copying original file attributes * @@ -40,3 +47,21 @@ fun moveFileWithAttributes(source: File, destinationDir: File) { Files.copy(source.toPath(), destinationDir.resolve(source.name).toPath(), StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.COPY_ATTRIBUTES) Files.delete(source.toPath()) } + +/** + * @param stop + * @return list of name of paths (folders + current file) till [stop] + */ + +fun Path.pathNamesTill(stop: Path): List = generateSequence(this, Path::getParent) + .takeWhile { it != stop } + .map { it.name } + .toList() + +/** + * @param stop + * @return count of parts (folders + current file) till [stop] + */ +fun Path.countPartsTill(stop: Path): Int = generateSequence(this, Path::getParent) + .takeWhile { it != stop } + .count() diff --git a/save-preprocessor/src/main/kotlin/com/saveourtool/save/preprocessor/config/TestSuitesRepo.kt b/save-preprocessor/src/main/kotlin/com/saveourtool/save/preprocessor/config/TestSuitesRepo.kt index fc2845a7ff..f0eafa50e3 100644 --- a/save-preprocessor/src/main/kotlin/com/saveourtool/save/preprocessor/config/TestSuitesRepo.kt +++ b/save-preprocessor/src/main/kotlin/com/saveourtool/save/preprocessor/config/TestSuitesRepo.kt @@ -3,12 +3,14 @@ package com.saveourtool.save.preprocessor.config /** * Class for repositories with standard test suites * - * @property gitUrl git url of repo - * @property gitBranchOrCommit git branch or commit in repo - * @property testSuitePaths list of test suite root paths + * @property organizationName name of organization for standard test suites + * @property url git url of repo + * @property branch git branch or commit in repo + * @property sources map of source name to test suite root path */ data class TestSuitesRepo( - val gitUrl: String, - val gitBranchOrCommit: String, - val testSuitePaths: List, + val organizationName: String, + val url: String, + val branch: String, + val sources: Map, ) diff --git a/save-preprocessor/src/main/kotlin/com/saveourtool/save/preprocessor/controllers/DownloadProjectController.kt b/save-preprocessor/src/main/kotlin/com/saveourtool/save/preprocessor/controllers/DownloadProjectController.kt index bda75563cb..5af17c5c35 100644 --- a/save-preprocessor/src/main/kotlin/com/saveourtool/save/preprocessor/controllers/DownloadProjectController.kt +++ b/save-preprocessor/src/main/kotlin/com/saveourtool/save/preprocessor/controllers/DownloadProjectController.kt @@ -2,12 +2,6 @@ package com.saveourtool.save.preprocessor.controllers import com.saveourtool.save.core.config.TestConfig import com.saveourtool.save.domain.FileInfo -import com.saveourtool.save.entities.Execution -import com.saveourtool.save.entities.ExecutionRequest -import com.saveourtool.save.entities.ExecutionRequestForStandardSuites -import com.saveourtool.save.entities.GitDto -import com.saveourtool.save.entities.Project -import com.saveourtool.save.entities.TestSuite import com.saveourtool.save.execution.ExecutionInitializationDto import com.saveourtool.save.execution.ExecutionStatus import com.saveourtool.save.execution.ExecutionUpdateDto @@ -24,6 +18,11 @@ import com.saveourtool.save.utils.debug import com.saveourtool.save.utils.info import com.fasterxml.jackson.databind.ObjectMapper +import com.saveourtool.save.entities.* +import com.saveourtool.save.latestVersion +import com.saveourtool.save.preprocessor.service.PreprocessorToBackendBridge +import com.saveourtool.save.testsuite.TestSuitesSourceDto +import com.saveourtool.save.testsuite.TestSuitesSourceType import org.eclipse.jgit.api.errors.GitAPIException import org.eclipse.jgit.api.errors.InvalidRemoteException import org.eclipse.jgit.api.errors.TransportException @@ -49,6 +48,7 @@ import org.springframework.web.server.ResponseStatusException import reactor.core.publisher.Flux import reactor.core.publisher.Mono import reactor.core.scheduler.Schedulers +import reactor.kotlin.core.publisher.switchIfEmpty import reactor.kotlin.core.publisher.toFlux import reactor.kotlin.core.util.function.component1 import reactor.kotlin.core.util.function.component2 @@ -73,8 +73,10 @@ typealias Status = Mono> class DownloadProjectController( private val configProperties: ConfigProperties, private val testDiscoveringService: TestDiscoveringService, - objectMapper: ObjectMapper, + private val objectMapper: ObjectMapper, kotlinSerializationWebClientCustomizer: WebClientCustomizer, + private val testSuitesPreprocessorController: TestSuitesPreprocessorController, + private val preprocessorToBackendBridge: PreprocessorToBackendBridge, ) { private val log = LoggerFactory.getLogger(DownloadProjectController::class.java) private val webClientBackend = WebClient.builder() @@ -90,6 +92,14 @@ class DownloadProjectController( .build() private val scheduler = Schedulers.boundedElastic() + private val standardTestSuitesRepo: TestSuitesRepo by lazy { + ClassPathResource(configProperties.reposFileName) + .file + .let { + objectMapper.readValue(it, TestSuitesRepo::class.java)!! + } + } + /** * @param executionRequest Dto of repo information to clone and project info * @return response entity with text @@ -100,19 +110,23 @@ class DownloadProjectController( @RequestBody executionRequest: ExecutionRequest, ): Mono = Mono.just(ResponseEntity(executionResponseBody(executionRequest.executionId), HttpStatus.ACCEPTED)) .doOnSuccess { - downLoadRepository(executionRequest) - .flatMap { (location, version) -> - val testRootPath = executionRequest.testRootPath - val testRootAbsolutePath = getResourceLocationForGit(location, testRootPath) - initializeTestSuitesAndTests(executionRequest.project, testRootPath, testRootAbsolutePath, executionRequest.gitDto.url) - .flatMap { testsSuites -> - updateExecution( - executionRequest.project, - location, - version, - testsSuites.map { it.requiredId() } - ) - } + preprocessorToBackendBridge.getTestSuitesSource( + executionRequest.project.organization.name, + executionRequest.gitDto.url, + executionRequest.testRootPath, + executionRequest.gitDto.branch ?: executionRequest.gitDto.detectDefaultBranchName() + ) + .zipWhen { testSuitesSource -> + testSuitesPreprocessorController.detectLatestVersion(testSuitesSource.toDto()) + } + .flatMap { (testSuitesSource, latestVersion) -> + testSuitesPreprocessorController.getOrFetchTestSuites(testSuitesSource.toDto(), latestVersion) + } + .flatMap { testSuites -> + updateExecution( + project = executionRequest.project, + testSuiteIds = testSuites.map { it.requiredId() } + ) } .flatMap { it.executeTests() } .subscribeOn(scheduler) @@ -128,17 +142,28 @@ class DownloadProjectController( @RequestBody executionRequestForStandardSuites: ExecutionRequestForStandardSuites, ) = Mono.just(ResponseEntity(executionResponseBody(executionRequestForStandardSuites.executionId), HttpStatus.ACCEPTED)) .doOnSuccess { - val version = requireNotNull(executionRequestForStandardSuites.version) { - "Version is not provided for execution.id = ${executionRequestForStandardSuites.executionId}" - } val execCmd = executionRequestForStandardSuites.execCmd val batchSizeForAnalyzer = executionRequestForStandardSuites.batchSizeForAnalyzer - getStandardTestSuiteIds(executionRequestForStandardSuites.testSuites) + Flux.fromIterable(standardTestSuitesRepo.sources.keys) + .flatMap { testSuitesSourceName -> + preprocessorToBackendBridge.getTestSuitesLatestVersion( + standardTestSuitesRepo.organizationName, + testSuitesSourceName + ).flatMap { version -> + preprocessorToBackendBridge.getTestSuites( + standardTestSuitesRepo.organizationName, + testSuitesSourceName, + version + ) + }.flatMapMany { Flux.fromIterable(it) } + }.filter { it.name in executionRequestForStandardSuites.testSuites } + .map { it.requiredId() } + .collectList() .flatMap { testSuiteIds -> updateExecution( executionRequestForStandardSuites.project, "N/A", - version, + "N/A", testSuiteIds, execCmd, batchSizeForAnalyzer, @@ -182,102 +207,33 @@ class DownloadProjectController( @PostMapping("/uploadStandardTestSuite") fun uploadStandardTestSuite() = Mono.just(ResponseEntity("Upload standard test suites pending...\n", HttpStatus.ACCEPTED)) .doOnSuccess { - val (user, token) = readGitCredentialsForStandardMode(configProperties.reposTokenFileName) - Flux.fromIterable(readStandardTestSuitesFile(configProperties.reposFileName)).flatMap { testSuiteRepoInfo -> - val testSuiteUrl = testSuiteRepoInfo.gitUrl - log.info("Starting clone repository url=$testSuiteUrl for standard test suites") - val tmpDir = generateDirectory(listOf(testSuiteUrl), configProperties.repository, deleteExisting = false) - Mono.fromCallable { - val gitDto = if (user != null && token != null) { - GitDto(url = testSuiteUrl, username = user, password = token) - } else { - GitDto(testSuiteUrl) - } - pullOrCloneProjectWithSpecificBranch(gitDto, tmpDir, testSuiteRepoInfo.gitBranchOrCommit) - ?.use { /* noop here, just need to close Git object */ } + Flux.fromIterable(standardTestSuitesRepo.sources.entries).flatMap { (sourceName, testRootPath) -> + preprocessorToBackendBridge.getTestSuitesSource( + organizationName = standardTestSuitesRepo.organizationName, + name = sourceName, + ).switchIfEmpty { + preprocessorToBackendBridge.getOrganization(standardTestSuitesRepo.organizationName) + .flatMap { organization -> + preprocessorToBackendBridge.createTestSuitesSource( + TestSuitesSourceDto( + organization = organization, + name = sourceName, + description = "Standard tests from $testRootPath", + type = TestSuitesSourceType.STANDARD, + additionalInfo = sourceName + ) + ) + }.map { it.toDto() } + }.map { testSuitesSourceDto -> + testSuitesPreprocessorController.pullLatestTestSuites(testSuitesSourceDto) + }.doOnError { + log.error("Error to update test suite with url=${standardTestSuitesRepo.url}, path=${testRootPath}") } - .flatMapMany { Flux.fromIterable(testSuiteRepoInfo.testSuitePaths) } - .flatMap { testRootPath -> - log.info("Starting to discover root test config in test root path: $testRootPath") - val testRootAbsolutePath = tmpDir.resolve(testRootPath).absoluteFile - initializeTestSuitesAndTests(null, testRootPath, testRootAbsolutePath, testSuiteUrl) - } - .doOnError { - log.error("Error to update test suite with url=$testSuiteUrl, path=${testSuiteRepoInfo.testSuitePaths}") - } } - .flatMapIterable { it } - .map { it.toDto() } - .collectList() - .flatMap { - markObsoleteOldStandardTestSuites(it) - } .subscribeOn(scheduler) .subscribe() } - private fun markObsoleteOldStandardTestSuites(newTestSuites: List) = webClientBackend.get() - .uri("/allStandardTestSuites") - .retrieve() - .bodyToMono>() - .map { existingSuites -> - if (newTestSuites.isEmpty()) { - log.warn("No new test suites have been provided, will mark all standard test suites as obsolete") - } - existingSuites.filter { it !in newTestSuites } - } - .flatMap { obsoleteSuites -> - webClientBackend.makeRequest( - BodyInserters.fromValue(obsoleteSuites), - "/markObsoleteTestSuites" - ) { - it.toBodilessEntity() - } - } - - private fun downloadRepositoryLocation(gitDto: GitDto): Pair { - val tmpDir = generateDirectory(listOf(gitDto.url), configProperties.repository, deleteExisting = false) - return tmpDir to tmpDir.relativeTo(File(configProperties.repository)).normalize().path - } - - @Suppress( - "TYPE_ALIAS", - "TOO_LONG_FUNCTION", - "TOO_MANY_LINES_IN_LAMBDA", - "UnsafeCallOnNullableType" - ) - private fun downLoadRepository(executionRequest: ExecutionRequest): Mono> { - val gitDto = executionRequest.gitDto - val (tmpDir, location) = downloadRepositoryLocation(gitDto) - return Mono.fromCallable { - pullOrCloneProjectWithSpecificBranch(gitDto, tmpDir, branchOrCommit = gitDto.branch ?: gitDto.hash)?.use { git -> - val version = git.log().call().first() - .name - log.info("Cloned repository ${gitDto.url}, head is at $version") - return@fromCallable location to version - } - } - .onErrorResume { exception -> - tmpDir.deleteRecursively() - val failReason = when (exception) { - is InvalidRemoteException, - is TransportException, - is GitAPIException -> "Error with git API while cloning ${gitDto.url} repository" - else -> "Cloning ${gitDto.url} repository failed. Reason: ${exception.message}" - } - log.error(failReason, exception) - updateExecutionStatus(executionRequest.executionId!!, ExecutionStatus.ERROR, failReason).flatMap { - Mono.error(exception) - } - } - } - - private fun getStandardTestSuiteIds(testSuiteNames: List): Mono> = webClientBackend.post() - .uri("/test-suites/standard/ids-by-name") - .bodyValue(testSuiteNames) - .retrieve() - .bodyToMono() - /** * Execute tests by execution id: * - Post request to backend to find all tests by test suite id which are set in execution and create TestExecutions for them @@ -302,10 +258,6 @@ class DownloadProjectController( updateExecutionStatus(id!!, ExecutionStatus.ERROR, failReason) } - private fun getResourceLocationForGit(location: String, testRootPath: String) = File(configProperties.repository) - .resolve(location) - .resolve(testRootPath) - private fun getExecution(executionId: Long) = webClientBackend.get() .uri("${configProperties.backend}/execution?id=$executionId") .retrieve() @@ -314,8 +266,8 @@ class DownloadProjectController( @Suppress("TOO_MANY_PARAMETERS", "LongParameterList") private fun updateExecution( project: Project, - projectRootRelativePath: String, - executionVersion: String, + projectRootRelativePath: String = "N/A", + executionVersion: String = "N/A", testSuiteIds: List, execCmd: String? = null, batchSizeForAnalyzer: String? = null, @@ -345,56 +297,6 @@ class DownloadProjectController( .retrieve() .toBodilessEntity() - private fun initializeTestSuitesAndTests(project: Project?, - testRootPath: String, - testRootAbsolutePath: File, - gitUrl: String, - ): Mono> { - log.info { "Starting to save new test suites for root test config in $testRootPath" } - return Mono.fromCallable { - testDiscoveringService.getRootTestConfig(testRootAbsolutePath.path) - } - .zipWhen { rootTestConfig -> - log.info { "Starting to discover test suites for root test config ${rootTestConfig.location}" } - discoverAndSaveTestSuites(project, rootTestConfig, testRootPath, gitUrl) - } - .flatMap { (rootTestConfig, testSuites) -> - log.info { "Test suites size = ${testSuites.size}" } - log.info { "Starting to save new tests for config test root $testRootPath" } - initializeTests(testSuites, rootTestConfig) - .collectList() - .map { testSuites } - } - } - - private fun discoverAndSaveTestSuites(project: Project?, - rootTestConfig: TestConfig, - testRootPath: String, - gitUrl: String, - ): Mono> { - val testSuites: List = testDiscoveringService.getAllTestSuites(project, rootTestConfig, testRootPath, gitUrl) - return webClientBackend.makeRequest(BodyInserters.fromValue(testSuites), "/saveTestSuites") { - it.bodyToMono() - } - } - - /** - * Discover tests and send them to backend - */ - private fun initializeTests(testSuites: List, - rootTestConfig: TestConfig - ): Flux = testDiscoveringService.getAllTests(rootTestConfig, testSuites) - .toFlux() - .buffer(TESTS_BUFFER_SIZE) - .doOnNext { - log.debug { "Processing chunk of tests [${it.first()} ... ${it.last()}]" } - } - .flatMap { testDtos -> - webClientBackend.makeRequest(BodyInserters.fromValue(testDtos), "/initializeTests") { - it.toBodilessEntity() - } - } - /** * POST request to orchestrator to initiate its work */ @@ -432,32 +334,6 @@ class DownloadProjectController( return toBody(responseSpec) } - @Suppress("TYPE_ALIAS") - private fun Flux>.download(destination: File): Mono> = flatMap { (filePart, fileInfo) -> - val file = File(destination, filePart.filename()).apply { - createNewFile() - } - // todo: don't use `filename()` - log.info("Downloading ${filePart.filename()} into ${file.absolutePath}") - filePart.content().map { dtBuffer -> - FileOutputStream(file, true).use { os -> - dtBuffer.asInputStream().use { - it.copyTo(os) - } - } - file - } - // return a single Mono per file, discarding how many parts `content()` has - .last() - .doOnSuccess { - log.debug("File ${fileInfo.name} should have executable=${fileInfo.isExecutable}") - if (!it.setExecutable(fileInfo.isExecutable)) { - log.warn("Failed to mark file ${fileInfo.name} as executable") - } - } - } - .collectList() - private fun updateExecutionStatus(executionId: Long, executionStatus: ExecutionStatus, failReason: String? = null) = webClientBackend.makeRequest( BodyInserters.fromValue(ExecutionUpdateDto(executionId, executionStatus, failReason)), @@ -467,34 +343,30 @@ class DownloadProjectController( log.info("Making request to set execution status for id=$executionId to $executionStatus") } - companion object { - // default Webflux in-memory buffer is 256 KiB - private const val TESTS_BUFFER_SIZE = 128 + private fun TestSuitesRepo.getGitDtoForStandardMode(subDirectory: String): GitDto { + val (user, token) = readGitCredentialsForStandardMode() + return if (user != null && token != null) { + GitDto(url = url, username = user, password = token, subDirectory = subDirectory) + } else { + GitDto(url = url, subDirectory = subDirectory) + } } -} -/** - * @param name file name to read - * @return map repository to paths to test configs - */ -@Suppress("MagicNumber", "TOO_MANY_LINES_IN_LAMBDA") -fun readStandardTestSuitesFile(name: String) = - ClassPathResource(name) - .file - .readText() - .lines() - .filter { it.isNotBlank() } - .map { line -> - val splitRow = line.split("\\s".toRegex()) - require(splitRow.size == 3) { - "Follow the format for each line: (Gir url) (branch or commit hash) (testRootPath1;testRootPath2;...)" - } - TestSuitesRepo( - gitUrl = splitRow.first(), - gitBranchOrCommit = splitRow[1], - testSuitePaths = splitRow[2].split(";") - ) - } + private fun readGitCredentialsForStandardMode(): Pair { + val credentialsFile = ClassPathResource(configProperties.reposTokenFileName) + val fileData = if (credentialsFile.exists()) { + credentialsFile.file.readLines().single { it.isNotBlank() } + } else { + return null to null + } + + val splitRow = fileData.split("\\s".toRegex()) + require(splitRow.size == 2) { + "Credentials file should contain git username and git token, separated by whitespace, but provided $splitRow" + } + return splitRow.first() to splitRow[1] + } +} /** * @param executionId @@ -503,17 +375,4 @@ fun readStandardTestSuitesFile(name: String) = @Suppress("UnsafeCallOnNullableType") fun executionResponseBody(executionId: Long?): String = "Clone pending, execution id is ${executionId!!}" -private fun readGitCredentialsForStandardMode(name: String): Pair { - val credentialsFile = ClassPathResource(name) - val fileData = if (credentialsFile.exists()) { - credentialsFile.file.readLines().single { it.isNotBlank() } - } else { - return null to null - } - val splitRow = fileData.split("\\s".toRegex()) - require(splitRow.size == 2) { - "Credentials file should contain git username and git token, separated by whitespace, but provided $splitRow" - } - return splitRow.first() to splitRow[1] -} diff --git a/save-preprocessor/src/main/kotlin/com/saveourtool/save/preprocessor/controllers/TestSuitesPreprocessorController.kt b/save-preprocessor/src/main/kotlin/com/saveourtool/save/preprocessor/controllers/TestSuitesPreprocessorController.kt new file mode 100644 index 0000000000..bc0da1edee --- /dev/null +++ b/save-preprocessor/src/main/kotlin/com/saveourtool/save/preprocessor/controllers/TestSuitesPreprocessorController.kt @@ -0,0 +1,133 @@ +package com.saveourtool.save.preprocessor.controllers + +import com.saveourtool.save.entities.GitDto +import com.saveourtool.save.entities.TestSuite +import com.saveourtool.save.preprocessor.service.GitPreprocessorService +import com.saveourtool.save.preprocessor.service.PreprocessorToBackendBridge +import com.saveourtool.save.preprocessor.service.TestDiscoveringService +import com.saveourtool.save.preprocessor.utils.detectLatestSha1 +import com.saveourtool.save.testsuite.TestSuitesSourceDto +import com.saveourtool.save.testsuite.TestSuitesSourceType +import com.saveourtool.save.utils.getLogger +import com.saveourtool.save.utils.info +import org.slf4j.Logger +import org.springframework.web.bind.annotation.GetMapping +import org.springframework.web.bind.annotation.PostMapping +import org.springframework.web.bind.annotation.RequestBody +import org.springframework.web.bind.annotation.RequestMapping +import org.springframework.web.bind.annotation.RequestParam +import org.springframework.web.bind.annotation.RestController +import reactor.core.publisher.Mono + +/** + * Preprocessor's controller for [com.saveourtool.save.entities.TestSuitesSource] + */ +@RestController +@RequestMapping("/test-suites-source") +class TestSuitesPreprocessorController( + private val gitPreprocessorService: GitPreprocessorService, + private val testDiscoveringService: TestDiscoveringService, + private val preprocessorToBackendBridge: PreprocessorToBackendBridge, +) { + /** + * @param testSuitesSourceDto source of test suites + * @return list of pulled [TestSuite]s + */ + @PostMapping("/pull-latest") + fun pullLatestTestSuites( + @RequestBody testSuitesSourceDto: TestSuitesSourceDto, + ): Mono> = detectLatestVersion(testSuitesSourceDto) + .flatMap { getOrFetchTestSuites(testSuitesSourceDto, it) } + + /** + * @param testSuitesSourceDto source of test suites + * @param version which needs to be provided + * @return list of requested [TestSuite]s + */ + @PostMapping("/get-or-fetch") + fun getOrFetchTestSuites( + @RequestBody testSuitesSourceDto: TestSuitesSourceDto, + @RequestParam version: String, + ): Mono> = + preprocessorToBackendBridge.doesTestSuitesSourceContainVersion(testSuitesSourceDto, version) + .flatMap { contains -> + if (contains) { + getTestSuites(testSuitesSourceDto, version) + } else { + fetchTestSuites(testSuitesSourceDto, version) + } + } + + /** + * @param testSuitesSourceDto source of test suites + * @param version which needs to be provided + * @return list of [TestSuite]s associated with provided [version] + */ + @GetMapping("/get-test-suites") + fun getTestSuites( + testSuitesSourceDto: TestSuitesSourceDto, + version: String, + ): Mono> = preprocessorToBackendBridge.getTestSuites(testSuitesSourceDto, version) + + /** + * Detect latest version of TestSuitesSource + * + * @param testSuitesSourceDto source of test suites + * @return latest available version on source + */ + @GetMapping("/detect-latest-version") + fun detectLatestVersion( + @RequestBody testSuitesSourceDto: TestSuitesSourceDto, + ): Mono = testSuitesSourceDto.processAsGit { + preprocessorToBackendBridge.getGitInfo(testSuitesSourceDto.additionalInfo.toLong()) + .map { gitDto -> + gitDto.detectLatestSha1() + } + } + + /** + * Fetch new tests suites from provided source + * + * @param testSuitesSourceDto source from which test suites need to be loaded + * @param version version which needs to be loaded + * @return new version of saved snapshot + */ + @PostMapping("/fetch-version") + fun fetchTestSuites( + @RequestBody testSuitesSourceDto: TestSuitesSourceDto, + @RequestParam version: String, + ): Mono> = testSuitesSourceDto.processAsGit { + preprocessorToBackendBridge.getGitInfo(testSuitesSourceDto.additionalInfo.toLong()) + .flatMap { gitDto -> + fetchTestSuitesFromGit(testSuitesSourceDto, version, gitDto) + } + } + + private fun fetchTestSuitesFromGit( + testSuitesSourceDto: TestSuitesSourceDto, + sha1: String, + gitDto: GitDto, + ): Mono> { + return gitPreprocessorService.cloneAndProcessDirectory(gitDto, sha1) { repositoryDirectory -> + testDiscoveringService.detectAndSaveAllTestSuitesAndTests( + repositoryPath = repositoryDirectory, + testSuitesSourceDto = testSuitesSourceDto, + version = sha1 + ).flatMap { testSuites -> + log.info { "Loaded: $testSuites" } + val content = gitPreprocessorService.archiveToTar(repositoryDirectory) + preprocessorToBackendBridge.saveTestsSuiteSourceSnapshot(testSuitesSourceDto, sha1, content) + .map { testSuites } + } + } + } + + private fun TestSuitesSourceDto.processAsGit(action: (TestSuitesSourceDto) -> T): T = when (type) { + TestSuitesSourceType.GIT -> action(this) + else -> throw NotImplementedError("Not supported test suites source type $type") + } + + companion object { + private val log: Logger = getLogger() + } +} \ No newline at end of file diff --git a/save-preprocessor/src/main/kotlin/com/saveourtool/save/preprocessor/service/GitPreprocessorService.kt b/save-preprocessor/src/main/kotlin/com/saveourtool/save/preprocessor/service/GitPreprocessorService.kt new file mode 100644 index 0000000000..16fd83d8ae --- /dev/null +++ b/save-preprocessor/src/main/kotlin/com/saveourtool/save/preprocessor/service/GitPreprocessorService.kt @@ -0,0 +1,88 @@ +package com.saveourtool.save.preprocessor.service + +import com.saveourtool.save.entities.GitDto +import com.saveourtool.save.preprocessor.config.ConfigProperties +import com.saveourtool.save.preprocessor.utils.cloneToDirectory +import com.saveourtool.save.utils.TAR_EXTENSION +import com.saveourtool.save.utils.compressAsTarTo +import com.saveourtool.save.utils.toByteBufferFlux +import org.springframework.stereotype.Service +import org.springframework.util.FileSystemUtils +import reactor.core.publisher.Flux +import reactor.core.publisher.Mono +import java.io.IOException +import java.nio.ByteBuffer +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.Paths +import kotlin.io.path.deleteExisting + +/** + * Additional service for Git based [com.saveourtool.save.entities.TestSuitesSource]s + */ +@Service +class GitPreprocessorService( + private val configProperties: ConfigProperties, +) { + private fun createTempDirectoryForRepository() = Files.createTempDirectory( + Paths.get(configProperties.repository), GitPreprocessorService::class.simpleName + ) + + private fun createTempTarFile() = Files.createTempFile( + Paths.get(configProperties.repository), + GitPreprocessorService::class.simpleName, + TAR_EXTENSION + ) + + /** + * @param gitDto + * @param sha1 + * @param repositoryProcessor + * @return result of [repositoryProcessor] + */ + fun cloneAndProcessDirectory( + gitDto: GitDto, + sha1: String, + repositoryProcessor: (Path) -> Mono, + ): Mono { + val cloneAction: () -> Path = { + val tmpDir = createTempDirectoryForRepository() + try { + cloneToDirectory(gitDto, sha1, tmpDir) + } catch (ex: IllegalStateException) { + // clean up in case of exception + FileSystemUtils.deleteRecursively(tmpDir) + throw ex + } + tmpDir + } + return Mono.using( + cloneAction, + { repositoryProcessor(it) }, + FileSystemUtils::deleteRecursively + ) + } + + /** + * @param pathToRepository + */ + fun archiveToTar( + pathToRepository: Path + ): Flux { + val archiveAction: () -> Path = { + val tmpFile = createTempTarFile() + try { + pathToRepository.compressAsTarTo(tmpFile) + } catch (ex: IOException) { + tmpFile.deleteExisting() + throw ex + } + tmpFile + } + return Flux.using( + archiveAction, + { it.toByteBufferFlux() }, + Files::delete + ) + } +} diff --git a/save-preprocessor/src/main/kotlin/com/saveourtool/save/preprocessor/service/PreprocessorToBackendBridge.kt b/save-preprocessor/src/main/kotlin/com/saveourtool/save/preprocessor/service/PreprocessorToBackendBridge.kt new file mode 100644 index 0000000000..3d0a350fba --- /dev/null +++ b/save-preprocessor/src/main/kotlin/com/saveourtool/save/preprocessor/service/PreprocessorToBackendBridge.kt @@ -0,0 +1,145 @@ +package com.saveourtool.save.preprocessor.service + +import com.saveourtool.save.entities.GitDto +import com.saveourtool.save.entities.Organization +import com.saveourtool.save.entities.TestSuite +import com.saveourtool.save.entities.TestSuitesSource +import com.saveourtool.save.preprocessor.EmptyResponse +import com.saveourtool.save.preprocessor.config.ConfigProperties +import com.saveourtool.save.test.TestDto +import com.saveourtool.save.testsuite.TestSuiteDto +import com.saveourtool.save.testsuite.TestSuitesSourceDto +import com.saveourtool.save.utils.debug +import org.slf4j.LoggerFactory +import org.springframework.boot.web.reactive.function.client.WebClientCustomizer +import org.springframework.http.MediaType +import org.springframework.stereotype.Service +import org.springframework.web.reactive.function.client.WebClient +import org.springframework.web.reactive.function.client.bodyToMono +import reactor.core.publisher.Flux +import reactor.core.publisher.Mono +import java.nio.ByteBuffer + +/** + * A bridge from preprocesor to backend (rest api wrapper) + */ +@Service +class PreprocessorToBackendBridge( + configProperties: ConfigProperties, + kotlinSerializationWebClientCustomizer: WebClientCustomizer, +) { + private val webClientBackend = WebClient.builder() + .baseUrl(configProperties.backend) + .apply(kotlinSerializationWebClientCustomizer::customize) + .build() + + fun saveTestsSuiteSourceSnapshot( + testSuitesSource: TestSuitesSourceDto, + version: String, + content: Flux + ): Mono = webClientBackend.post() + .uri("/test-suites-source/{organizationName}/{testSuitesSourceName}/{version}/upload", + testSuitesSource.organization.name, testSuitesSource.name, version) + .contentType(MediaType.MULTIPART_FORM_DATA) + .bodyValue(content) + .retrieve() + .bodyToMono() + + fun saveTestSuites(testSuiteDtos: List): Mono> = webClientBackend.post() + .uri("/test-suites/save") + .bodyValue(testSuiteDtos) + .retrieve() + .bodyToMono() + + fun saveTests(tests: Flux): Flux = tests + .buffer(TESTS_BUFFER_SIZE) + .doOnNext { + log.debug { "Processing chuck of tests [${it.first()} ... ${it.last()}]" } + } + .flatMap { chunk -> + webClientBackend.post() + .uri("/tests/save") + .bodyValue(chunk) + .retrieve() + .toBodilessEntity() + } + + fun getGitInfo(id: Long): Mono = webClientBackend.get() + .uri("/git?id={id}", id) + .retrieve() + .bodyToMono() + + fun doesTestSuitesSourceContainVersion(testSuitesSource: TestSuitesSourceDto, version: String): Mono = + webClientBackend.get() + .uri("/test-suites-source/{organizationName}/{testSuitesSourceName}/{version}/contains", + testSuitesSource.organization.name, testSuitesSource.name, version) + .retrieve() + .bodyToMono() + + fun getTestSuites(organizationName: String, testSuitesSourceName: String, version: String) = webClientBackend.get() + .uri( + "/test-suites-source/{organizationName}/{testSuitesSourceName}/{version}/get-test-suites", + organizationName, testSuitesSourceName, version + ) + .retrieve() + .bodyToMono>() + + fun getTestSuites(testSuitesSource: TestSuitesSourceDto, version: String) = getTestSuites(testSuitesSource.organization.name, testSuitesSource.name, version) + + fun getTestSuitesSource( + organizationName: String, + name: String + ): Mono = webClientBackend.post() + .uri( + "/test-suites-source/{organizationName}/{name}", + organizationName, + name + ) + .retrieve() + .bodyToMono() + + fun getTestSuitesSource( + organizationName: String, + gitUrl: String, + subDirectory: String, + branch: String + ): Mono = webClientBackend.post() + .uri( + "/test-suites-source/{organizationName}/get-or-create?gitUrl={}&subDirectory={}&branch={}", + organizationName, + gitUrl, + subDirectory, + branch + ) + .retrieve() + .bodyToMono() + + fun getTestSuitesLatestVersion( + organizationName: String, + name: String, + ): Mono = webClientBackend.post() + .uri( + "/test-suites-source/{organizationName}/{name}/latest", + organizationName, + name, + ) + .retrieve() + .bodyToMono() + + fun getOrganization( + organizationName: String, + ): Mono = webClientBackend.get() + .uri( + "/organization/{organizationName}", + organizationName + ) + .retrieve() + .bodyToMono() + + companion object { + private val log = LoggerFactory.getLogger(PreprocessorToBackendBridge::class.java) + + // default Webflux in-memory buffer is 256 KiB + private const val TESTS_BUFFER_SIZE = 128 + } +} \ No newline at end of file diff --git a/save-preprocessor/src/main/kotlin/com/saveourtool/save/preprocessor/service/TestDiscoveringService.kt b/save-preprocessor/src/main/kotlin/com/saveourtool/save/preprocessor/service/TestDiscoveringService.kt index 2201063e54..c13b9d18a2 100644 --- a/save-preprocessor/src/main/kotlin/com/saveourtool/save/preprocessor/service/TestDiscoveringService.kt +++ b/save-preprocessor/src/main/kotlin/com/saveourtool/save/preprocessor/service/TestDiscoveringService.kt @@ -8,25 +8,65 @@ import com.saveourtool.save.core.utils.processInPlace import com.saveourtool.save.entities.Project import com.saveourtool.save.entities.TestSuite import com.saveourtool.save.plugins.fix.FixPlugin +import com.saveourtool.save.preprocessor.EmptyResponse import com.saveourtool.save.preprocessor.utils.toHash import com.saveourtool.save.test.TestDto import com.saveourtool.save.test.TestFilesContent import com.saveourtool.save.test.TestFilesRequest import com.saveourtool.save.testsuite.TestSuiteDto import com.saveourtool.save.testsuite.TestSuiteType - +import com.saveourtool.save.testsuite.TestSuitesSourceDto +import com.saveourtool.save.utils.info import okio.FileSystem import okio.Path import okio.Path.Companion.toPath - import org.slf4j.LoggerFactory import org.springframework.stereotype.Service +import reactor.core.publisher.Flux +import reactor.core.publisher.Mono +import reactor.kotlin.core.publisher.toFlux +import reactor.kotlin.core.util.function.component1 +import reactor.kotlin.core.util.function.component2 +import kotlin.io.path.absolutePathString /** * A service that call SAVE core to discover test suites and tests */ @Service -class TestDiscoveringService { +class TestDiscoveringService( + private val preprocessorToBackendBridge: PreprocessorToBackendBridge, +) { + + /** + * @param repositoryPath + * @param testSuitesSourceDto + * @param version + * @return list of [TestSuite] initialized from provided directory + */ + fun detectAndSaveAllTestSuitesAndTests( + repositoryPath: java.nio.file.Path, + testSuitesSourceDto: TestSuitesSourceDto, + version: String, + ): Mono> { + log.info { "Starting to save new test suites for root test config in $repositoryPath" } + return Mono.fromCallable { getRootTestConfig(repositoryPath.absolutePathString()) } + .zipWhen { rootTestConfig -> + log.info { "Starting to discover test suites for root test config ${rootTestConfig.location}" } + discoverAndSaveAllTestSuites( + rootTestConfig, + testSuitesSourceDto, + version + ) + } + .flatMap { (rootTestConfig, testSuites) -> + log.info { "Test suites size = ${testSuites.size}" } + log.info { "Starting to save new tests for config test root $repositoryPath" } + discoverAndSaveAllTests(rootTestConfig, testSuites) + .collectList() + .map { testSuites } + } + } + /** * Returns a root config of hierarchy; this config will already have all descendants merged with their parents. * @@ -53,10 +93,9 @@ class TestDiscoveringService { */ @Suppress("UnsafeCallOnNullableType") fun getAllTestSuites( - project: Project?, rootTestConfig: TestConfig, - testRootPath: String, - testSuiteRepoUrl: String, + source: TestSuitesSourceDto, + version: String, ) = rootTestConfig .getAllTestConfigs() .asSequence() @@ -66,12 +105,10 @@ class TestDiscoveringService { .map { config -> // we operate here with suite names from only those TestConfigs, that have General section with suiteName key TestSuiteDto( - project?.let { TestSuiteType.PROJECT } ?: TestSuiteType.STANDARD, config.suiteName!!, config.description, - project, - testRootPath, - testSuiteRepoUrl, + source, + version, config.language, config.tags ) @@ -79,6 +116,22 @@ class TestDiscoveringService { .distinct() .toList() + /** + * Discover all test suites in the project + * + * @param rootTestConfig root config of SAVE configs hierarchy + * @param source source with test suites + * @param version + * @return a list of saved [TestSuite]s + * @throws IllegalArgumentException when provided path doesn't point to a valid config file + */ + @Suppress("UnsafeCallOnNullableType") + fun discoverAndSaveAllTestSuites( + rootTestConfig: TestConfig, + source: TestSuitesSourceDto, + version: String, + ) = getAllTestSuites(rootTestConfig, source, version).save() + private fun Path.getRelativePath(rootTestConfig: TestConfig) = this.toFile() .relativeTo(rootTestConfig.directory.toFile()) .path @@ -127,6 +180,19 @@ class TestDiscoveringService { log.debug("Discovered the following test: $it") } + /** + * Discover all tests in the project + * + * @param testSuites testSuites in this project + * @param rootTestConfig root config of SAVE configs hierarchy + * @return a list of [TestDto]s + * @throws PluginException if configs use unknown plugin + */ + @Suppress("UnsafeCallOnNullableType") + fun discoverAndSaveAllTests(rootTestConfig: TestConfig, testSuites: List) = getAllTests(rootTestConfig, testSuites) + .toFlux() + .save() + private fun getTestLinesByPath(pathPrefix: String, testPath: String?) = testPath?.let { (pathPrefix.toPath() / it).toFile().readLines() } @@ -143,6 +209,16 @@ class TestDiscoveringService { private fun TestConfig.getGeneralConfigOrNull() = pluginConfigs.filterIsInstance().singleOrNull() + /** + * Save test suites via backend + */ + private fun List.save(): Mono> = preprocessorToBackendBridge.saveTestSuites(this) + + /** + * Save tests via backend + */ + private fun Flux.save(): Flux = preprocessorToBackendBridge.saveTests(this) + companion object { private val log = LoggerFactory.getLogger(TestDiscoveringService::class.java) } diff --git a/save-preprocessor/src/main/kotlin/com/saveourtool/save/preprocessor/utils/GitUtil.kt b/save-preprocessor/src/main/kotlin/com/saveourtool/save/preprocessor/utils/GitUtil.kt index 55024b2bb5..b1c603d309 100644 --- a/save-preprocessor/src/main/kotlin/com/saveourtool/save/preprocessor/utils/GitUtil.kt +++ b/save-preprocessor/src/main/kotlin/com/saveourtool/save/preprocessor/utils/GitUtil.kt @@ -5,10 +5,8 @@ package com.saveourtool.save.preprocessor.utils import com.saveourtool.save.entities.GitDto -import org.eclipse.jgit.api.CreateBranchCommand -import org.eclipse.jgit.api.Git -import org.eclipse.jgit.api.MergeCommand -import org.eclipse.jgit.api.ResetCommand +import com.saveourtool.save.utils.debug +import org.eclipse.jgit.api.* import org.eclipse.jgit.api.errors.GitAPIException import org.eclipse.jgit.api.errors.InvalidConfigurationException import org.eclipse.jgit.api.errors.RefAlreadyExistsException @@ -21,6 +19,7 @@ import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider import org.slf4j.LoggerFactory import java.io.File import java.lang.IllegalArgumentException +import java.nio.file.Path private val log = LoggerFactory.getLogger(object {}.javaClass.enclosingClass::class.java) @@ -167,6 +166,38 @@ private fun getDefaultBranchName(repoUrl: String): String? { return defaultBranch.replace("refs/heads/", "${Constants.DEFAULT_REMOTE_NAME}/") } +/** + * @param httpUrl + * @param password + * @param username + * @return default branch name + * @throws IllegalStateException when failed to detect default branch name + */ +fun detectDefaultBranchName(httpUrl: String, username: String?, password: String?): String { + return Git.lsRemoteRepository() + .setCredentialsProvider(credentialsProvider(username, password)) + // ls without clone + // FIXME: need to extract to a common place + .setRemote(httpUrl) + .withRethrow { + it.callAsMap()[Constants.HEAD] + } + ?.takeIf { it.isSymbolic } + ?.target + ?.name + ?.also { defaultBranch -> + log.debug { "Getting default branch name $defaultBranch for httpUrl $httpUrl" } + } + ?.replace(Constants.R_HEADS, "${Constants.DEFAULT_REMOTE_NAME}/") + ?: throw IllegalStateException("Couldn't detect default branch name for $httpUrl") +} + +/** + * @return default branch name + * @throws IllegalStateException when failed to detect default branch name + */ +fun GitDto.detectDefaultBranchName() = detectDefaultBranchName(url, username, password) + private fun checkout(git: Git, branchOrCommit: String, setCreateBranchFlag: Boolean) { git.checkout() // We need to call this method anyway, in aim not to have `detached head` state, @@ -177,3 +208,58 @@ private fun checkout(git: Git, branchOrCommit: String, setCreateBranchFlag: Bool .setStartPoint(branchOrCommit) .call() } + + +/** + * @return latest commit + */ +fun GitDto.detectLatestSha1(): String = Git.lsRemoteRepository() + .setCredentialsProvider(credentialsProvider()) + .setRemote(url) + .withRethrow { + it.callAsMap()["${Constants.R_HEADS}$branch"] + } + ?.objectId + ?.name + ?: throw IllegalStateException("Couldn't detect hash of ${Constants.HEAD} for $url/$branch") + +/** + * @param gitDto + * @param sha1 + * @param pathToDirectory + * @throws IllegalStateException + */ +fun cloneToDirectory(gitDto: GitDto, sha1: String, pathToDirectory: Path) { + Git.cloneRepository() + .setCredentialsProvider(gitDto.credentialsProvider()) + .setURI(gitDto.url) + .setDirectory(pathToDirectory.toFile()) + .setRemote(Constants.DEFAULT_REMOTE_NAME) + .setBranch(sha1) + .setCloneAllBranches(false) + .withRethrow { it.call() } + .use { + // need to close Git after all + } +} + +private fun GitDto.credentialsProvider(): CredentialsProvider = credentialsProvider(username, password) + +private fun credentialsProvider(username: String?, password: String?): CredentialsProvider = + if (username != null && password != null) { + listOf().single() + UsernamePasswordCredentialsProvider(username, password) + } else if (username == null) { + // https://stackoverflow.com/questions/28073266/how-to-use-jgit-to-push-changes-to-remote-with-oauth-access-token + UsernamePasswordCredentialsProvider(password, "") + } else { + CredentialsProvider.getDefault() + } + +private fun > T.withRethrow(call: (T) -> R): R { + try { + return call(this) + } catch (ex: GitAPIException) { + throw IllegalStateException("Error in JGit API", ex) + } +} diff --git a/save-preprocessor/src/main/kotlin/com/saveourtool/save/preprocessor/utils/WebClientUtil.kt b/save-preprocessor/src/main/kotlin/com/saveourtool/save/preprocessor/utils/WebClientUtil.kt new file mode 100644 index 0000000000..2e83f11133 --- /dev/null +++ b/save-preprocessor/src/main/kotlin/com/saveourtool/save/preprocessor/utils/WebClientUtil.kt @@ -0,0 +1,58 @@ +/** + * WebClient utilities that are used in preprocessor for download content from backend + */ + +package com.saveourtool.save.preprocessor.utils + +import org.slf4j.LoggerFactory +import org.springframework.http.HttpStatus +import org.springframework.http.ReactiveHttpOutputMessage +import org.springframework.web.reactive.function.BodyInserter +import org.springframework.web.reactive.function.client.WebClient +import org.springframework.web.server.ResponseStatusException +import reactor.core.publisher.Mono + +private val log = LoggerFactory.getLogger(WebClient::class.java) + +/** + * @param body + * @param uri + * @param toBody + * @return result of [toBody] using response on post request after checking status + */ +fun WebClient.makePost( + body: BodyInserter, + uri: String, + toBody: (WebClient.ResponseSpec) -> Mono +): Mono = this.makePost(body, uri).let(toBody) + +/** + * @param body + * @param uri + * @return response on post request after checking status + * @throws ResponseStatusException + */ +fun WebClient.makePost( + body: BodyInserter, + uri: String, +): WebClient.ResponseSpec = this + .post() + .uri(uri) + .body(body) + .retrieve() + .onStatus({status -> status != HttpStatus.OK }) { clientResponse -> + log.error("Error when making request to $uri: ${clientResponse.statusCode()}") + throw ResponseStatusException( + HttpStatus.INTERNAL_SERVER_ERROR, + "Upstream request error" + ) + } + +fun WebClient.ResponseSpec.validateStatus(): WebClient.ResponseSpec = this + .onStatus({status -> status != HttpStatus.OK }) { clientResponse -> + log.error("Error when making request: ${clientResponse.statusCode()}") + throw ResponseStatusException( + HttpStatus.INTERNAL_SERVER_ERROR, + "Upstream request error" + ) + } diff --git a/save-preprocessor/src/main/resources/TestSuitesRepos b/save-preprocessor/src/main/resources/TestSuitesRepos index 8a18e5231f..a9317d90b4 100644 --- a/save-preprocessor/src/main/resources/TestSuitesRepos +++ b/save-preprocessor/src/main/resources/TestSuitesRepos @@ -1 +1,8 @@ -https://github.com/saveourtool/save-cli origin/main examples/kotlin-diktat;examples/discovery-test +{ + "url": "https://github.com/saveourtool/save-cli", + "branch": "origin/main", + "sources": { + "Standard test suites for kotlin diktat": "examples/kotlin-diktat", + "Test suites to test discovery": "examples/discovery-test" + } +} diff --git a/save-preprocessor/src/main/resources/TestSuitesRepos.json b/save-preprocessor/src/main/resources/TestSuitesRepos.json new file mode 100644 index 0000000000..bd8ee06446 --- /dev/null +++ b/save-preprocessor/src/main/resources/TestSuitesRepos.json @@ -0,0 +1,9 @@ +{ + "organizationName": "CQFN.org", + "url": "https://github.com/saveourtool/save-cli", + "branch": "origin/main", + "sources": { + "Standard test suites for kotlin diktat": "examples/kotlin-diktat", + "Test suites to test discovery": "examples/discovery-test" + } +} diff --git a/save-preprocessor/src/test/kotlin/com/saveourtool/save/preprocessor/controllers/DownloadProjectTest.kt b/save-preprocessor/src/test/kotlin/com/saveourtool/save/preprocessor/controllers/DownloadProjectTest.kt index 18139e1c0c..db5aa76f6a 100644 --- a/save-preprocessor/src/test/kotlin/com/saveourtool/save/preprocessor/controllers/DownloadProjectTest.kt +++ b/save-preprocessor/src/test/kotlin/com/saveourtool/save/preprocessor/controllers/DownloadProjectTest.kt @@ -257,7 +257,7 @@ class DownloadProjectTest( fun testStandardTestSuites() { val requestSize = readStandardTestSuitesFile(configProperties.reposFileName) .toList() - .flatMap { it.testSuitePaths } + .flatMap { it.sources } .size repeat(requestSize) { val project = Project.stub(42)