Execution run by contest id #6

saveourtool · Aug 11, 2022 · 8247cd0 · 8247cd0
1 parent 1095445
commit 8247cd0
Show file tree

Hide file tree

Showing 7 changed files with 167 additions and 440 deletions.
diff --git a/save-backend/src/main/kotlin/com/saveourtool/save/backend/SaveApplication.kt b/save-backend/src/main/kotlin/com/saveourtool/save/backend/SaveApplication.kt
@@ -14,7 +14,6 @@ typealias StringResponse = ResponseEntity<String>
 typealias EmptyResponse = ResponseEntity<Void>
 typealias ByteBufferFluxResponse = ResponseEntity<Flux<ByteBuffer>>
 typealias ResourceResponse = ResponseEntity<Resource>
-typealias IdResponse = ResponseEntity<Long>
 
 /**
  * An entrypoint for spring for save-backend

diff --git a/...ackend/src/main/kotlin/com/saveourtool/save/backend/controllers/RunExecutionController.kt b/...ackend/src/main/kotlin/com/saveourtool/save/backend/controllers/RunExecutionController.kt
@@ -1,19 +1,21 @@
 package com.saveourtool.save.backend.controllers
 
 import com.fasterxml.jackson.databind.ObjectMapper
-import com.saveourtool.save.backend.IdResponse
+import com.saveourtool.save.backend.StringResponse
 import com.saveourtool.save.backend.configs.ConfigProperties
 import com.saveourtool.save.backend.service.*
 import com.saveourtool.save.backend.storage.ExecutionInfoStorage
 import com.saveourtool.save.backend.utils.blockingToMono
 import com.saveourtool.save.backend.utils.username
+import com.saveourtool.save.domain.ProjectCoordinates
 import com.saveourtool.save.entities.Execution
 import com.saveourtool.save.entities.ExecutionRunRequest
 import com.saveourtool.save.execution.ExecutionStatus
 import com.saveourtool.save.execution.ExecutionUpdateDto
 import com.saveourtool.save.permission.Permission
 import com.saveourtool.save.utils.debug
 import com.saveourtool.save.utils.getLogger
+import com.saveourtool.save.utils.switchIfEmptyToNotFound
 import com.saveourtool.save.utils.switchIfEmptyToResponseException
 import com.saveourtool.save.v1
 import io.micrometer.core.instrument.MeterRegistry
@@ -27,6 +29,7 @@ import org.springframework.security.core.Authentication
 import org.springframework.web.bind.annotation.PostMapping
 import org.springframework.web.bind.annotation.RequestBody
 import org.springframework.web.bind.annotation.RequestMapping
+import org.springframework.web.bind.annotation.RequestParam
 import org.springframework.web.bind.annotation.RestController
 import org.springframework.web.reactive.function.BodyInserters
 import org.springframework.web.reactive.function.client.WebClient
@@ -58,17 +61,11 @@ class RunExecutionController(
     fun trigger(
         @RequestBody request: ExecutionRunRequest,
         authentication: Authentication,
-    ): Mono<IdResponse> = with(request.projectCoordinates) {
-        // Project cannot be taken from executionRequest directly for permission evaluation:
-        // it can be fudged by user, who submits it. We should get project from DB based on name/owner combination.
-        projectService.findWithPermissionByNameAndOrganization(authentication, projectName, organizationName, Permission.WRITE)
-    }
-        .switchIfEmptyToResponseException(HttpStatus.FORBIDDEN) {
-            "User ${authentication.username()} doesn't have access to ${request.projectCoordinates}"
-        }
-        .map { project ->
+    ): Mono<StringResponse> = Mono.just(request.projectCoordinates)
+        .validateAccess(authentication) { it }
+        .map {
             executionService.createNew(
-                project = project,
+                projectCoordinates = request.projectCoordinates,
                 testSuiteIds = request.testSuiteIds,
                 files = request.files,
                 username = authentication.username(),
@@ -79,44 +76,87 @@ class RunExecutionController(
         }
         .subscribeOn(Schedulers.boundedElastic())
         .flatMap { execution ->
-            val executionId = execution.requiredId()
-            Mono.just(ResponseEntity.accepted().body(executionId))
+            Mono.just(execution.toAcceptedResponse())
                 .doOnSuccess {
-                    blockingToMono {
-                        val tests = request.testSuiteIds.flatMap { testService.findTestsByTestSuiteId(it) }
-                        log.debug { "Received the following test ids for saving test execution under executionId=$executionId: ${tests.map { it.requiredId() }}" }
-                        meterRegistry.timer("save.backend.saveTestExecution").record {
-                            testExecutionService.saveTestExecutions(execution, tests)
-                        }
-                    }
-                        .flatMap {
-                            initializeAgents(execution)
-                                .map {
-                                    log.debug { "Initialized agents for execution $executionId" }
-                                }
-                        }
-                        .onErrorResume { ex ->
-                            val failReason = "Error during preprocessing. Reason: ${ex.message}"
-                            log.error(
-                                "$failReason, will mark execution.id=$executionId as failed; error: ",
-                                ex
-                            )
-                            val executionUpdateDto = ExecutionUpdateDto(
-                                executionId,
-                                ExecutionStatus.ERROR,
-                                failReason
-                            )
-                            blockingToMono {
-                                executionService.updateExecutionStatus(execution, executionUpdateDto.status)
-                            }.flatMap {
-                                executionInfoStorage.upsertIfRequired(executionUpdateDto)
-                            }
-                        }
-                        .subscribeOn(scheduler)
-                        .subscribe()
+                    asyncTrigger(execution)
+                }
+        }
+
+    @PostMapping("/reTrigger")
+    fun reTrigger(
+        @RequestParam executionId: Long,
+        authentication: Authentication,
+    ): Mono<StringResponse> = blockingToMono { executionService.findExecution(executionId) }
+        .switchIfEmptyToNotFound { "Not found execution id = $executionId" }
+        .validateAccess(authentication) { execution ->
+            ProjectCoordinates(
+                execution.project.organization.name,
+                execution.project.name
+            )
+        }
+        .map { executionService.createNewCopy(it, authentication.username()) }
+        .flatMap { execution ->
+            Mono.just(execution.toAcceptedResponse())
+                .doOnSuccess {
+                    asyncTrigger(execution)
                 }
         }
 
+    private fun <T> Mono<T>.validateAccess(
+        authentication: Authentication,
+        projectCoordinatesGetter: (T) -> ProjectCoordinates,
+    ): Mono<T> =
+            flatMap { value ->
+                val projectCoordinates = projectCoordinatesGetter(value)
+                with(projectCoordinates) {
+                    projectService.findWithPermissionByNameAndOrganization(
+                        authentication,
+                        projectName,
+                        organizationName,
+                        Permission.WRITE
+                    )
+                }.switchIfEmptyToResponseException(HttpStatus.FORBIDDEN) {
+                    "User ${authentication.username()} doesn't have access to $projectCoordinates"
+                }.map { value }
+            }
+
+    private fun asyncTrigger(execution: Execution) {
+        val executionId = execution.requiredId()
+        blockingToMono {
+            val tests = execution.parseAndGetTestSuiteIds()
+                .orEmpty()
+                .flatMap { testService.findTestsByTestSuiteId(it) }
+            log.debug { "Received the following test ids for saving test execution under executionId=$executionId: ${tests.map { it.requiredId() }}" }
+            meterRegistry.timer("save.backend.saveTestExecution").record {
+                testExecutionService.saveTestExecutions(execution, tests)
+            }
+        }
+            .flatMap {
+                initializeAgents(execution)
+                    .map {
+                        log.debug { "Initialized agents for execution $executionId" }
+                    }
+            }
+            .onErrorResume { ex ->
+                val failReason = "Error during preprocessing. Reason: ${ex.message}"
+                log.error(
+                    "$failReason, will mark execution.id=$executionId as failed; error: ",
+                    ex
+                )
+                val executionUpdateDto = ExecutionUpdateDto(
+                    executionId,
+                    ExecutionStatus.ERROR,
+                    failReason
+                )
+                blockingToMono {
+                    executionService.updateExecutionStatus(execution, executionUpdateDto.status)
+                }.flatMap {
+                    executionInfoStorage.upsertIfRequired(executionUpdateDto)
+                }
+            }
+            .subscribeOn(scheduler)
+            .subscribe()
+    }
 
     /**
      * POST request to orchestrator to initiate its work
@@ -135,6 +175,9 @@ class RunExecutionController(
             .toEntity()
     }
 
+    private fun Execution.toAcceptedResponse(): StringResponse =
+            ResponseEntity.accepted().body("Clone pending, execution id is ${requiredId()}")
+
     companion object {
         private val log: Logger = getLogger<RunExecutionController>()
     }

diff --git a/...ckend/src/main/kotlin/com/saveourtool/save/backend/controllers/internal/TestController.kt b/...ckend/src/main/kotlin/com/saveourtool/save/backend/controllers/internal/TestController.kt
@@ -1,9 +1,7 @@
 package com.saveourtool.save.backend.controllers.internal
 
-import com.saveourtool.save.backend.service.TestExecutionService
 import com.saveourtool.save.backend.service.TestService
 import com.saveourtool.save.backend.storage.TestSuitesSourceSnapshotStorage
-import com.saveourtool.save.entities.Test
 import com.saveourtool.save.test.TestDto
 import com.saveourtool.save.test.TestFilesContent
 import com.saveourtool.save.test.TestFilesRequest
@@ -28,7 +26,6 @@ import reactor.core.publisher.Mono
 @RequestMapping("/internal")
 class TestController(
     private val testService: TestService,
-    private val testExecutionService: TestExecutionService,
     private val meterRegistry: MeterRegistry,
     private val testSuitesSourceSnapshotStorage: TestSuitesSourceSnapshotStorage,
 ) {
@@ -44,26 +41,6 @@ class TestController(
         }
     }
 
-    /**
-     * @param executionId ID of the [Execution][com.saveourtool.save.entities.Execution],
-     * all tests are initialized for this execution will be executed (creating an instance of [TestExecution][com.saveourtool.save.entities.TestExecution])
-     */
-    @PostMapping("/executeTestsByExecutionId")
-    fun executeTestsByExecutionId(@RequestParam executionId: Long) {
-        val testIds = testService.findTestsByExecutionId(executionId).map { it.requiredId() }
-        log.debug { "Received the following test ids for saving test execution under executionId=$executionId: $testIds" }
-        meterRegistry.timer("save.backend.saveTestExecution").record {
-            testExecutionService.saveTestExecutionsByTestIds(executionId, testIds)
-        }
-    }
-
-    /**
-     * @param testSuiteId ID of the [TestSuite], for which all corresponding tests will be returned
-     * @return list of tests
-     */
-    @GetMapping("/getTestsByTestSuiteId")
-    fun getTestsByTestSuiteId(@RequestParam testSuiteId: Long): List<Test> = testService.findTestsByTestSuiteId(testSuiteId)
-
     /**
      * @param agentId
      * @return test batches

diff --git a/save-backend/src/main/kotlin/com/saveourtool/save/backend/scheduling/UpdateJob.kt b/save-backend/src/main/kotlin/com/saveourtool/save/backend/scheduling/UpdateJob.kt
@@ -1,28 +1,37 @@
 package com.saveourtool.save.backend.scheduling
 
 import com.saveourtool.save.backend.configs.ConfigProperties
+import com.saveourtool.save.backend.service.TestSuitesSourceService
 import org.quartz.Job
 import org.quartz.JobExecutionContext
 import org.quartz.JobKey
 import org.slf4j.LoggerFactory
 import org.springframework.web.reactive.function.client.WebClient
+import reactor.kotlin.core.publisher.toFlux
 import java.time.Duration
 
 /**
  * a [Job] that commands preprocessor to update standard test suites
  */
 class UpdateJob(
-    configProperties: ConfigProperties
+    private val testSuitesSourceService: TestSuitesSourceService,
+    configProperties: ConfigProperties,
 ) : Job {
     private val preprocessorWebClient = WebClient.create(configProperties.preprocessorUrl)
 
     @Suppress("MagicNumber")
     override fun execute(context: JobExecutionContext?) {
         logger.info("Running job $jobKey")
-        preprocessorWebClient.post()
-            .uri("/uploadStandardTestSuite")
-            .retrieve()
-            .toBodilessEntity()
+        testSuitesSourceService.getStandardTestSuitesSources()
+            .toFlux()
+            .flatMap { testSuitesSource ->
+                preprocessorWebClient.post()
+                    .uri("/test-suites-sources/fetch")
+                    .bodyValue(testSuitesSource)
+                    .retrieve()
+                    .toBodilessEntity()
+            }
+            .collectList()
             .block(Duration.ofSeconds(10))
     }
 

diff --git a/save-backend/src/main/kotlin/com/saveourtool/save/backend/service/ExecutionService.kt b/save-backend/src/main/kotlin/com/saveourtool/save/backend/service/ExecutionService.kt
@@ -28,6 +28,7 @@ import java.util.Optional
 @Service
 class ExecutionService(
     private val executionRepository: ExecutionRepository,
+    private val projectService: ProjectService,
     private val userRepository: UserRepository,
     private val testRepository: TestRepository,
     private val testExecutionRepository: TestExecutionRepository,
@@ -219,15 +220,66 @@ class ExecutionService(
         })
     }
 
+    @Suppress("LongParameterList")
     @Transactional
     fun createNew(
-        project: Project,
+        projectCoordinates: ProjectCoordinates,
         testSuiteIds: List<Long>,
-        files: List<FileInfo>,
+        files: List<FileKey>,
         username: String,
         sdk: Sdk,
         execCmd: String?,
         batchSizeForAnalyzer: String?,
+    ): Execution {
+        val project = with(projectCoordinates) {
+            projectService.findByNameAndOrganizationName(projectName, organizationName).orNotFound {
+                "Not found project $projectName in $organizationName"
+            }
+        }
+        return doCreateNew(
+            project = project,
+            formattedTestSuiteIds = Execution.formatTestSuiteIds(testSuiteIds),
+            version = testSuitesService.getSingleVersionByIds(testSuiteIds),
+            allTests = testSuiteIds.flatMap { testRepository.findAllByTestSuiteId(it) }
+                .count()
+                .toLong(),
+            additionalFiles = files.format(),
+            username = username,
+            sdk = sdk.toString(),
+            execCmd = execCmd,
+            batchSizeForAnalyzer = batchSizeForAnalyzer,
+        )
+    }
+
+    @Transactional
+    fun createNewCopy(
+        execution: Execution,
+        username: String,
+    ): Execution {
+        return doCreateNew(
+            project = execution.project,
+            formattedTestSuiteIds = execution.testSuiteIds,
+            version = execution.version,
+            allTests = execution.allTests,
+            additionalFiles = execution.additionalFiles,
+            username = username,
+            sdk = execution.sdk,
+            execCmd = execution.execCmd,
+            batchSizeForAnalyzer = execution.batchSizeForAnalyzer,
+        )
+    }
+
+    @Suppress("LongParameterList")
+    fun doCreateNew(
+        project: Project,
+        formattedTestSuiteIds: String?,
+        version: String?,
+        allTests: Long,
+        additionalFiles: String,
+        username: String,
+        sdk: String,
+        execCmd: String?,
+        batchSizeForAnalyzer: String?,
     ): Execution {
         val user = userRepository.findByName(username).orNotFound {
             "Not found user $username"
@@ -237,14 +289,12 @@ class ExecutionService(
             startTime = LocalDateTime.now(),
             endTime = null,
             status = ExecutionStatus.PENDING,
-            testSuiteIds = Execution.formatTestSuiteIds(testSuiteIds),
+            testSuiteIds = formattedTestSuiteIds,
             batchSize = configProperties.initialBatchSize,
             // FIXME: remove this type
             type = ExecutionType.GIT,
-            version = testSuitesService.getSingleVersionByIds(testSuiteIds),
-            allTests = testSuiteIds.flatMap { testRepository.findAllByTestSuiteId(it) }
-                .count()
-                .toLong(),
+            version = version,
+            allTests = allTests,
             runningTests = 0,
             passedTests = 0,
             failedTests = 0,
@@ -253,8 +303,8 @@ class ExecutionService(
             matchedChecks = 0,
             expectedChecks = 0,
             unexpectedChecks = 0,
-            sdk = sdk.toString(),
-            additionalFiles = files.map { it.toStorageKey() }.format(),
+            sdk = sdk,
+            additionalFiles = additionalFiles,
             user = user,
             execCmd = execCmd,
             batchSizeForAnalyzer = batchSizeForAnalyzer,