refactor(controller): make runtime dockerizing be a job (#2651)

star-whale · Aug 24, 2023 · 5d53026 · 5d53026
1 parent e777c1e
commit 5d53026
Show file tree

Hide file tree

Showing 33 changed files with 763 additions and 853 deletions.
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -156,6 +156,7 @@ jobs:
         run: |
           make build-server
           make build-dataset-builder
+          make build-runtime-dockerizing-builder
 
   helm-charts-release:
     runs-on: ubuntu-latest

diff --git a/docker/Dockerfile.runtime_dockerizing b/docker/Dockerfile.runtime_dockerizing
@@ -0,0 +1,24 @@
+ARG BASE_IMAGE=starwhaleai/base:latest
+FROM gcr.io/kaniko-project/executor:v1.14.0 AS kaniko
+FROM ${BASE_IMAGE}
+
+ARG PYTHON_VERSION
+ARG SW_VERSION
+
+ENV SW_RUNTIME_PYTHON_VERSION=${PYTHON_VERSION:-3.10}
+ENV SW_VERSION=${SW_VERSION:-0.5.8}
+COPY --from=kaniko /kaniko/executor /kaniko/executor
+COPY --from=kaniko /kaniko/docker-credential-gcr /kaniko/docker-credential-gcr
+COPY --from=kaniko /kaniko/docker-credential-ecr-login /kaniko/docker-credential-ecr-login
+COPY --from=kaniko /kaniko/docker-credential-acr-env /kaniko/docker-credential-acr-env
+COPY --from=kaniko /kaniko/.docker /kaniko/.docker
+COPY --from=kaniko /kaniko/ssl/certs/ /kaniko/ssl/certs/
+COPY --from=kaniko /etc/nsswitch.conf /etc/nsswitch.conf
+RUN /opt/starwhale.bin/base-entrypoint set_environment || true
+
+ENV SW_USER_RUNTIME_RESTORED=1
+ENV PATH $PATH:/usr/local/bin:/kaniko
+ENV SSL_CERT_DIR=/kaniko/ssl/certs
+ENV DOCKER_CONFIG /kaniko/.docker/
+ENV DOCKER_CREDENTIAL_GCR_CONFIG /kaniko/.config/gcloud/docker_credential_gcr_config.json
+ENTRYPOINT ["/kaniko/executor"]
diff --git a/docker/Makefile b/docker/Makefile
@@ -81,6 +81,9 @@ build-server:
 build-dataset-builder:
 	$(call build-image,$(GHCR_IO_REPO)/base:latest,Dockerfile.dataset_build,dataset_builder:${RELEASE_VERSION} dataset_builder:latest)
 
+build-runtime-dockerizing-builder:
+	$(call build-image,$(GHCR_IO_REPO)/base:latest,Dockerfile.Dockerfile.runtime_dockerizing,runtime-dockerizing:${RELEASE_VERSION} runtime-dockerizing:latest)
+
 build-server-all: build-console build-jar build-server
 
 build-server-for-local-usage:

diff --git a/scripts/publish/image_sync.sh b/scripts/publish/image_sync.sh
@@ -42,7 +42,7 @@ function get_tags() {
 }
 
 # start to sync images
-declare -a starwhale_images=("server" "base" "cuda" "dataset_builder")
+declare -a starwhale_images=("server" "base" "cuda" "dataset_builder" "runtime-dockerizing")
 for image in "${starwhale_images[@]}"; do
     tags=$(get_tags "$image")
     # if the image has already synced, regctl would skip it.

diff --git a/server/controller/src/main/java/ai/starwhale/mlops/api/RuntimeApi.java b/server/controller/src/main/java/ai/starwhale/mlops/api/RuntimeApi.java
@@ -27,7 +27,7 @@
 import ai.starwhale.mlops.api.protocol.runtime.RuntimeVersionVo;
 import ai.starwhale.mlops.api.protocol.runtime.RuntimeViewVo;
 import ai.starwhale.mlops.api.protocol.runtime.RuntimeVo;
-import ai.starwhale.mlops.domain.job.spec.RunConfig;
+import ai.starwhale.mlops.domain.job.spec.RunEnvs;
 import com.github.pagehelper.PageInfo;
 import io.swagger.v3.oas.annotations.Operation;
 import io.swagger.v3.oas.annotations.Parameter;
@@ -341,7 +341,24 @@ ResponseEntity<ResponseMessage<BuildImageResult>> buildRuntimeImage(
             @Parameter(in = ParameterIn.PATH, required = true, schema = @Schema())
             @PathVariable("versionUrl") String versionUrl,
             @Parameter(description = "user defined running configurations such environment variables")
-            @RequestBody(required = false) RunConfig runConfig);
+            @RequestBody(required = false) RunEnvs runEnvs);
+
+    @Operation(summary = "update image for runtime", description = "update image for runtime")
+    @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "ok")})
+    @RequestMapping(
+            value = "/project/{projectUrl}/runtime/{runtimeUrl}/version/{versionUrl}/image",
+            produces = MediaType.APPLICATION_JSON_VALUE,
+            method = RequestMethod.PUT)
+    @PreAuthorize("hasAnyRole('OWNER', 'MAINTAINER')")
+    ResponseEntity<ResponseMessage<?>> updateRuntime(
+            @Parameter(in = ParameterIn.PATH, required = true, description = "Project url", schema = @Schema())
+            @PathVariable("projectUrl") String projectUrl,
+            @Parameter(in = ParameterIn.PATH, required = true, schema = @Schema())
+            @PathVariable("runtimeUrl") String runtimeUrl,
+            @Parameter(in = ParameterIn.PATH, required = true, schema = @Schema())
+            @PathVariable("versionUrl") String versionUrl,
+            @Parameter(description = "the image used for this runtime")
+            String runtimeImage);
 
 
 }
diff --git a/server/controller/src/main/java/ai/starwhale/mlops/api/RuntimeController.java b/server/controller/src/main/java/ai/starwhale/mlops/api/RuntimeController.java
@@ -27,7 +27,7 @@
 import ai.starwhale.mlops.api.protocol.runtime.RuntimeViewVo;
 import ai.starwhale.mlops.api.protocol.runtime.RuntimeVo;
 import ai.starwhale.mlops.common.PageParams;
-import ai.starwhale.mlops.domain.job.spec.RunConfig;
+import ai.starwhale.mlops.domain.job.spec.RunEnvs;
 import ai.starwhale.mlops.domain.runtime.RuntimeService;
 import ai.starwhale.mlops.domain.runtime.bo.RuntimeQuery;
 import ai.starwhale.mlops.domain.runtime.bo.RuntimeVersion;
@@ -305,9 +305,16 @@ public ResponseEntity<ResponseMessage<BuildImageResult>> buildRuntimeImage(
             String projectUrl,
             String runtimeUrl,
             String versionUrl,
-            RunConfig runConfig
+            RunEnvs runEnvs
     ) {
-        BuildImageResult res = runtimeService.buildImage(projectUrl, runtimeUrl, versionUrl, runConfig);
+        BuildImageResult res = runtimeService.dockerize(projectUrl, runtimeUrl, versionUrl, runEnvs);
         return ResponseEntity.ok(Code.success.asResponse(res));
     }
+
+    @Override
+    public ResponseEntity<ResponseMessage<?>> updateRuntime(String projectUrl, String runtimeUrl, String versionUrl,
+            String runtimeImage) {
+        runtimeService.updateImage(projectUrl, runtimeUrl, versionUrl, runtimeImage);
+        return ResponseEntity.ok(Code.success.asResponse(null));
+    }
 }
diff --git a/...er/controller/src/main/java/ai/starwhale/mlops/api/protocol/runtime/BuildImageResult.java b/...er/controller/src/main/java/ai/starwhale/mlops/api/protocol/runtime/BuildImageResult.java
@@ -17,11 +17,13 @@
 package ai.starwhale.mlops.api.protocol.runtime;
 
 import io.swagger.v3.oas.annotations.media.Schema;
+import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
 import org.simpleframework.xml.core.Validate;
 
 @Data
+@AllArgsConstructor
 @Builder
 @Schema(description = "Build runtime image result", title = "Runtime")
 @Validate

diff --git a/...roller/src/main/java/ai/starwhale/mlops/configuration/security/RuntimeTokenValidator.java b/...roller/src/main/java/ai/starwhale/mlops/configuration/security/RuntimeTokenValidator.java
diff --git a/server/controller/src/main/java/ai/starwhale/mlops/domain/job/JobCreator.java b/server/controller/src/main/java/ai/starwhale/mlops/domain/job/JobCreator.java
@@ -30,7 +30,8 @@
 import ai.starwhale.mlops.domain.job.status.JobUpdateHelper;
 import ai.starwhale.mlops.domain.model.ModelService;
 import ai.starwhale.mlops.domain.project.bo.Project;
-import ai.starwhale.mlops.domain.runtime.RuntimeService;
+import ai.starwhale.mlops.domain.runtime.RuntimeDao;
+import ai.starwhale.mlops.domain.runtime.bo.Runtime;
 import ai.starwhale.mlops.domain.runtime.bo.RuntimeVersion;
 import ai.starwhale.mlops.domain.storage.StoragePathCoordinator;
 import ai.starwhale.mlops.domain.system.SystemSettingService;
@@ -64,34 +65,37 @@ public class JobCreator {
     private final JobDao jobDao;
     private final ModelService modelService;
     private final DatasetDao datasetDao;
-    private final RuntimeService runtimeService;
+    private final RuntimeDao runtimeDao;
     private final JobUpdateHelper jobUpdateHelper;
 
     private final SystemSettingService systemSettingService;
     private final JobSpecParser jobSpecParser;
 
-    public JobCreator(JobBoConverter jobBoConverter,
+    public JobCreator(
+            JobBoConverter jobBoConverter,
             JobSpliterator jobSpliterator, JobLoader jobLoader,
             StoragePathCoordinator storagePathCoordinator,
             JobDao jobDao, ModelService modelService,
-            DatasetDao datasetDao, RuntimeService runtimeService, JobUpdateHelper jobUpdateHelper,
-            SystemSettingService systemSettingService, JobSpecParser jobSpecParser) {
+            DatasetDao datasetDao, RuntimeDao runtimeDao, JobUpdateHelper jobUpdateHelper,
+            SystemSettingService systemSettingService, JobSpecParser jobSpecParser
+    ) {
         this.jobBoConverter = jobBoConverter;
         this.jobSpliterator = jobSpliterator;
         this.jobLoader = jobLoader;
         this.storagePathCoordinator = storagePathCoordinator;
         this.jobDao = jobDao;
         this.modelService = modelService;
         this.datasetDao = datasetDao;
-        this.runtimeService = runtimeService;
+        this.runtimeDao = runtimeDao;
         this.jobUpdateHelper = jobUpdateHelper;
         this.systemSettingService = systemSettingService;
         this.jobSpecParser = jobSpecParser;
     }
 
 
     @Transactional
-    public Job createJob(Project project,
+    public Job createJob(
+            Project project,
             String modelVersionUrl,
             String datasetVersionUrls,
             String runtimeVersionUrl,
@@ -104,25 +108,31 @@ public Job createJob(Project project,
             boolean devMode,
             String devPassword,
             Long ttlInSec,
-            User creator) {
+            User creator
+    ) {
         String jobUuid = IdUtil.simpleUUID();
         var modelVersion = StringUtils.hasText(modelVersionUrl) ? modelService.findModelVersion(modelVersionUrl) : null;
         var model = null == modelVersion ? null : modelService.findModel(modelVersion.getModelId());
 
         RuntimeVersion runtimeVersion;
         if (StringUtils.hasText(runtimeVersionUrl)) {
-            runtimeVersion = runtimeService.findRuntimeVersion(runtimeVersionUrl);
+            runtimeVersion = RuntimeVersion.fromEntity(runtimeDao.getRuntimeVersion(runtimeVersionUrl));
         } else if (null != modelVersion) {
             log.debug("try to find built-in runtime for model:{}", modelVersion.getId());
             runtimeVersionUrl = modelVersion.getBuiltInRuntime();
             if (!StringUtils.hasText(runtimeVersionUrl)) {
                 throw new SwValidationException(ValidSubject.RUNTIME, "no runtime or built-in runtime");
             }
-            runtimeVersion = runtimeService.findBuiltInRuntimeVersion(model.getProjectId(), runtimeVersionUrl);
+            var runtime = runtimeDao.getRuntimeByName(Constants.SW_BUILT_IN_RUNTIME, model.getProjectId());
+            runtimeVersion = RuntimeVersion.fromEntity(runtimeDao.getRuntimeVersion(
+                    runtime.getId(),
+                    runtimeVersionUrl
+            ));
         } else {
             runtimeVersion = null;
         }
-        var runtime = null == runtimeVersion ? null : runtimeService.findRuntime(runtimeVersion.getRuntimeId());
+        var runtime = null == runtimeVersion ? null :
+                Runtime.fromEntity(runtimeDao.getRuntime(runtimeVersion.getRuntimeId()));
 
         var datasetVersionIdMaps = StringUtils.hasText(datasetVersionUrls)
                 ? Arrays.stream(datasetVersionUrls.split("[,;]"))
@@ -134,7 +144,8 @@ public Job createJob(Project project,
                 || (StringUtils.hasText(stepSpecOverWrites) && StringUtils.hasText(handler))) {
             throw new StarwhaleApiException(
                     new SwValidationException(ValidSubject.JOB, "handler or stepSpec must be provided only one"),
-                    HttpStatus.BAD_REQUEST);
+                    HttpStatus.BAD_REQUEST
+            );
         }
 
         List<StepSpec> steps;
@@ -164,7 +175,8 @@ public Job createJob(Project project,
             if (!pool.allowUser(creator.getId())) {
                 throw new StarwhaleApiException(
                         new SwValidationException(ValidSubject.JOB, "creator is not allowed to use this resource pool"),
-                        HttpStatus.BAD_REQUEST);
+                        HttpStatus.BAD_REQUEST
+                );
             }
         }
 

diff --git a/...hale/mlops/domain/job/spec/RunConfig.java → ...rwhale/mlops/domain/job/spec/RunEnvs.java b/...hale/mlops/domain/job/spec/RunConfig.java → ...rwhale/mlops/domain/job/spec/RunEnvs.java
@@ -24,7 +24,7 @@
 @Data
 @AllArgsConstructor
 @NoArgsConstructor
-public class RunConfig {
+public class RunEnvs {
 
     Map<String, String> envVars;