diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000000..33731529b0 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +* text=auto +*.bat eol=crlf \ No newline at end of file diff --git a/.github/codecov.yml b/.github/codecov.yml new file mode 100644 index 0000000000..d2504c7543 --- /dev/null +++ b/.github/codecov.yml @@ -0,0 +1,7 @@ +codecov: + max_report_age: off +coverage: + status: + project: + default: + threshold: 2% \ No newline at end of file diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml new file mode 100644 index 0000000000..6a06a28111 --- /dev/null +++ b/.github/workflows/build_and_test.yml @@ -0,0 +1,55 @@ +name: Build and test + +on: + pull_request + +jobs: + build_and_test_with_code_coverage: + name: Build, test and upload code coverage + runs-on: ubuntu-20.04 + + steps: + # actions/checkout v1.* is needed for correct codecov upload, see https://github.com/actions/checkout/issues/237 for details + - uses: actions/checkout@v1 + # ensure that gradle wrapper files in repository are valid by checking checksums + - uses: gradle/wrapper-validation-action@v1 + - name: Set up JDK 11 + uses: actions/setup-java@v1 + with: + java-version: 1.11 + # todo: consider using https://github.com/burrunan/gradle-cache-action instead + - name: Cache gradle caches + uses: actions/cache@v2 + with: + path: | + ~/.gradle/caches/ + ~/.gradle/wrapper/ + key: ${{ runner.os }}-gradle-build-${{ hashFiles('**/*.gradle*') }} + restore-keys: ${{ runner.os }}-gradle-build + # https://gvisor.dev/docs/user_guide/install/ + - name: Install gvisor runsc runtime + run: | + ARCH=$(uname -m) + URL=https://storage.googleapis.com/gvisor/releases/release/latest/${ARCH} + wget ${URL}/runsc ${URL}/runsc.sha512 \ + ${URL}/containerd-shim-runsc-v1 ${URL}/containerd-shim-runsc-v1.sha512 + sha512sum -c runsc.sha512 \ + -c containerd-shim-runsc-v1.sha512 + rm -f *.sha512 + chmod a+rx runsc containerd-shim-runsc-v1 + sudo mv runsc containerd-shim-runsc-v1 /usr/local/bin + sudo /usr/local/bin/runsc install + sudo systemctl reload docker + - name: Gradle build + run: | + ./gradlew build + - name: Upload test reports + uses: actions/upload-artifact@v2 + with: + name: gradle-test-report + path: '**/build/reports/' + - name: Code coverage report + uses: codecov/codecov-action@v1 + with: + flags: unittests + fail_ci_if_error: true # optional (default = false) \ No newline at end of file diff --git a/.github/workflows/metrics_for_master.yml b/.github/workflows/metrics_for_master.yml new file mode 100644 index 0000000000..3025f2e3cd --- /dev/null +++ b/.github/workflows/metrics_for_master.yml @@ -0,0 +1,38 @@ +name: Update metrics for master branch + +on: + push: + branches: + - 'master' + +env: + GRADLE_OPTS: -Dorg.gradle.daemon=false + +jobs: + master_flow: + name: Master branch update + runs-on: ubuntu-20.04 + + steps: + - uses: actions/checkout@v2.3.3 + - name: Set up JDK 11 + uses: actions/setup-java@v1 + with: + java-version: 1.11 + # todo: consider using https://github.com/burrunan/gradle-cache-action instead + - name: Cache gradle caches + uses: actions/cache@v2 + with: + path: | + ~/.gradle/caches/ + ~/.gradle/wrapper/ + key: ${{ runner.os }}-gradle-build-${{ hashFiles('**/*.gradle*') }} + restore-keys: ${{ runner.os }}-gradle-build + - name: Run tests + # we need to run `install` goal here so that gradle will be able to resolve dependencies and run tests on diktat-gradle-plugin + run: ./gradlew build + - name: Generate code coverage report + uses: codecov/codecov-action@v1 + with: + flags: unittests + fail_ci_if_error: true # optional (default = false) diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000..1baf345e0a --- /dev/null +++ b/.gitignore @@ -0,0 +1,8 @@ +target +.gradle +build +!ktlint/src/main/resources/config/.idea +/.idea +*.iml +out +.DS_Store \ No newline at end of file diff --git a/README.md b/README.md index 905def9f53..55a6f69d13 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,7 @@ +![Build and test](https://github.com/cqfn/save/workflows/Build%20and%20test/badge.svg) +[![License](https://img.shields.io/github/license/cqfn/save)](https://github.com/cqfn/save/blob/master/LICENSE) +[![codecov](https://codecov.io/gh/cqfn/save/branch/master/graph/badge.svg)](https://codecov.io/gh/cqfn/save) + ## Purpose of Static Analysis Verification and Evaluation (SAVE) project Usage of [static analyzers](https://en.wikipedia.org/wiki/Static_program_analysis) - is a very important part of development each and every software product. All human beings can make a mistake in their code even when a software developer is writing all kinds of tests and has a very good test-coverage. diff --git a/buildSrc/build.gradle.kts b/buildSrc/build.gradle.kts new file mode 100644 index 0000000000..c39a297b0f --- /dev/null +++ b/buildSrc/build.gradle.kts @@ -0,0 +1,7 @@ +plugins { + `kotlin-dsl` +} + +repositories { + jcenter() +} \ No newline at end of file diff --git a/buildSrc/src/main/kotlin/Versions.kt b/buildSrc/src/main/kotlin/Versions.kt new file mode 100644 index 0000000000..24a41555a3 --- /dev/null +++ b/buildSrc/src/main/kotlin/Versions.kt @@ -0,0 +1,10 @@ +object Versions { + val kotlin = "1.4.30" + val springBoot = "2.4.2" + val springSecurity = "5.4.2" + val hibernate = "5.4.2.Final" + val liquibase = "4.2.2" + val slf4j = "1.7.30" + val logback = "1.2.3" + val dockerJavaApi = "3.2.7" +} \ No newline at end of file diff --git a/buildSrc/src/main/kotlin/org/cqfn/save/buildutils/JacocoConfiguration.kt b/buildSrc/src/main/kotlin/org/cqfn/save/buildutils/JacocoConfiguration.kt new file mode 100644 index 0000000000..82502bc689 --- /dev/null +++ b/buildSrc/src/main/kotlin/org/cqfn/save/buildutils/JacocoConfiguration.kt @@ -0,0 +1,29 @@ +package org.cqfn.save.buildutils + +import org.gradle.api.Project +import org.gradle.api.tasks.testing.Test +import org.gradle.kotlin.dsl.apply +import org.gradle.kotlin.dsl.configure +import org.gradle.kotlin.dsl.named +import org.gradle.testing.jacoco.plugins.JacocoPlugin +import org.gradle.testing.jacoco.plugins.JacocoPluginExtension +import org.gradle.testing.jacoco.tasks.JacocoReport + +fun Project.configureJacoco() { + apply() + + configure { + toolVersion = "0.8.6" + } + + tasks.named("test") { + finalizedBy("jacocoTestReport") + } + tasks.named("jacocoTestReport") { + dependsOn(tasks.named("test")) + reports { + xml.isEnabled = true + html.isEnabled = true + } + } +} diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 0000000000..e708b1c023 Binary files /dev/null and b/gradle/wrapper/gradle-wrapper.jar differ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 0000000000..da9702f9e7 --- /dev/null +++ b/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,5 @@ +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-6.8-bin.zip +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists diff --git a/gradlew b/gradlew new file mode 100755 index 0000000000..4f906e0c81 --- /dev/null +++ b/gradlew @@ -0,0 +1,185 @@ +#!/usr/bin/env sh + +# +# Copyright 2015 the original author or authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +############################################################################## +## +## Gradle start up script for UN*X +## +############################################################################## + +# Attempt to set APP_HOME +# Resolve links: $0 may be a link +PRG="$0" +# Need this for relative symlinks. +while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG=`dirname "$PRG"`"/$link" + fi +done +SAVED="`pwd`" +cd "`dirname \"$PRG\"`/" >/dev/null +APP_HOME="`pwd -P`" +cd "$SAVED" >/dev/null + +APP_NAME="Gradle" +APP_BASE_NAME=`basename "$0"` + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD="maximum" + +warn () { + echo "$*" +} + +die () { + echo + echo "$*" + echo + exit 1 +} + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "`uname`" in + CYGWIN* ) + cygwin=true + ;; + Darwin* ) + darwin=true + ;; + MINGW* ) + msys=true + ;; + NONSTOP* ) + nonstop=true + ;; +esac + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD="java" + which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." +fi + +# Increase the maximum file descriptors if we can. +if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then + MAX_FD_LIMIT=`ulimit -H -n` + if [ $? -eq 0 ] ; then + if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then + MAX_FD="$MAX_FD_LIMIT" + fi + ulimit -n $MAX_FD + if [ $? -ne 0 ] ; then + warn "Could not set maximum file descriptor limit: $MAX_FD" + fi + else + warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" + fi +fi + +# For Darwin, add options to specify how the application appears in the dock +if $darwin; then + GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" +fi + +# For Cygwin or MSYS, switch paths to Windows format before running java +if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then + APP_HOME=`cygpath --path --mixed "$APP_HOME"` + CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` + + JAVACMD=`cygpath --unix "$JAVACMD"` + + # We build the pattern for arguments to be converted via cygpath + ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` + SEP="" + for dir in $ROOTDIRSRAW ; do + ROOTDIRS="$ROOTDIRS$SEP$dir" + SEP="|" + done + OURCYGPATTERN="(^($ROOTDIRS))" + # Add a user-defined pattern to the cygpath arguments + if [ "$GRADLE_CYGPATTERN" != "" ] ; then + OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" + fi + # Now convert the arguments - kludge to limit ourselves to /bin/sh + i=0 + for arg in "$@" ; do + CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` + CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option + + if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition + eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` + else + eval `echo args$i`="\"$arg\"" + fi + i=`expr $i + 1` + done + case $i in + 0) set -- ;; + 1) set -- "$args0" ;; + 2) set -- "$args0" "$args1" ;; + 3) set -- "$args0" "$args1" "$args2" ;; + 4) set -- "$args0" "$args1" "$args2" "$args3" ;; + 5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; + 6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; + 7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; + 8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; + 9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; + esac +fi + +# Escape application args +save () { + for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done + echo " " +} +APP_ARGS=`save "$@"` + +# Collect all arguments for the java command, following the shell quoting and substitution rules +eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" + +exec "$JAVACMD" "$@" diff --git a/gradlew.bat b/gradlew.bat new file mode 100644 index 0000000000..107acd32c4 --- /dev/null +++ b/gradlew.bat @@ -0,0 +1,89 @@ +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem + +@if "%DEBUG%" == "" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%" == "" set DIRNAME=. +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if "%ERRORLEVEL%" == "0" goto execute + +echo. +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto execute + +echo. +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* + +:end +@rem End local scope for the variables with windows NT shell +if "%ERRORLEVEL%"=="0" goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 +exit /b 1 + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/save-backend/build.gradle.kts b/save-backend/build.gradle.kts index 7e54d1dab3..9e1d112d4f 100644 --- a/save-backend/build.gradle.kts +++ b/save-backend/build.gradle.kts @@ -1,33 +1,30 @@ +import org.cqfn.save.buildutils.configureJacoco import org.jetbrains.kotlin.gradle.tasks.KotlinCompile plugins { kotlin("jvm") } -val kotlinVersion = "1.4.21" -val springBootVersion = "2.4.2" -val springSecurityVersion = "5.4.2" -val hibernateVersion = "5.4.2.Final" -val liquibaseVersion = "4.2.2" -val slf4jVersion = "1.7.30" -val logbackVersion = "1.2.3" -val compileKotlin: KotlinCompile by tasks - tasks.withType { kotlinOptions { jvmTarget = "1.8" } } -kotlin { - dependencies { - implementation("org.springframework.boot:spring-boot-starter-webflux:$springBootVersion") - implementation("org.springframework.boot:spring-boot-starter-actuator:$springBootVersion") - implementation("org.springframework.security:spring-security-core:$springSecurityVersion") - implementation("org.liquibase:liquibase-core:$liquibaseVersion") - implementation("org.hibernate:hibernate-core:$hibernateVersion") - implementation("org.slf4j:slf4j-api:$slf4jVersion") - implementation("ch.qos.logback:logback-core:$logbackVersion") - testImplementation("org.springframework.boot:spring-boot-starter-test:$springBootVersion") - } -} \ No newline at end of file +tasks.withType { + useJUnitPlatform() +} + +dependencies { + implementation(project(":save-common")) + implementation("org.springframework.boot:spring-boot-starter-webflux:${Versions.springBoot}") + implementation("org.springframework.boot:spring-boot-starter-actuator:${Versions.springBoot}") + implementation("org.springframework.security:spring-security-core:${Versions.springSecurity}") + implementation("org.liquibase:liquibase-core:${Versions.liquibase}") + implementation("org.hibernate:hibernate-core:${Versions.hibernate}") + implementation("org.slf4j:slf4j-api:${Versions.slf4j}") + implementation("ch.qos.logback:logback-core:${Versions.logback}") + testImplementation("org.springframework.boot:spring-boot-starter-test:${Versions.springBoot}") +} + +configureJacoco() diff --git a/save-backend/src/main/resources/logback.xml b/save-backend/src/main/resources/logback.xml index 22110fb2cf..87314de407 100644 --- a/save-backend/src/main/resources/logback.xml +++ b/save-backend/src/main/resources/logback.xml @@ -19,7 +19,7 @@ - + \ No newline at end of file diff --git a/save-common/build.gradle.kts b/save-common/build.gradle.kts index b8d0d2fdf6..33a85fe5a1 100644 --- a/save-common/build.gradle.kts +++ b/save-common/build.gradle.kts @@ -4,4 +4,5 @@ plugins { kotlin { jvm() -} \ No newline at end of file + js(IR).browser() +} diff --git a/save-common/src/commonMain/kotlin/org.cqfn.save.domain/RunConfiguration.kt b/save-common/src/commonMain/kotlin/org.cqfn.save.domain/RunConfiguration.kt new file mode 100644 index 0000000000..129917f12e --- /dev/null +++ b/save-common/src/commonMain/kotlin/org.cqfn.save.domain/RunConfiguration.kt @@ -0,0 +1,9 @@ +package org.cqfn.save.domain + +/** + * Describes the configuration to run tests on SAVE backend. + * + * @property startCommand shell command to run the file + * @property fileName name of the supplied file + */ +data class RunConfiguration(val startCommand: String, val fileName: String) diff --git a/save-orchestrator/build.gradle.kts b/save-orchestrator/build.gradle.kts new file mode 100644 index 0000000000..45c14ff6c4 --- /dev/null +++ b/save-orchestrator/build.gradle.kts @@ -0,0 +1,33 @@ +import org.cqfn.save.buildutils.configureJacoco +import org.jetbrains.kotlin.gradle.tasks.KotlinCompile + +plugins { + kotlin("jvm") +} + +tasks.withType { + kotlinOptions { + jvmTarget = "1.8" + } +} + +tasks.withType { + useJUnitPlatform() +} + +dependencies { + implementation(project(":save-common")) + implementation("org.springframework.boot:spring-boot-starter-webflux:${Versions.springBoot}") + implementation("org.springframework.boot:spring-boot-starter-actuator:${Versions.springBoot}") + implementation("org.springframework.security:spring-security-core:${Versions.springSecurity}") + implementation("org.liquibase:liquibase-core:${Versions.liquibase}") + implementation("org.hibernate:hibernate-core:${Versions.hibernate}") + implementation("org.slf4j:slf4j-api:${Versions.slf4j}") + implementation("ch.qos.logback:logback-core:${Versions.logback}") + implementation("com.github.docker-java:docker-java-core:${Versions.dockerJavaApi}") + implementation("com.github.docker-java:docker-java-transport-httpclient5:${Versions.dockerJavaApi}") + implementation("org.apache.commons:commons-compress:1.20") + testImplementation("org.springframework.boot:spring-boot-starter-test:${Versions.springBoot}") +} + +configureJacoco() diff --git a/save-orchestrator/src/main/kotlin/org/cqfn/save/orchestrator/SaveOrchestrator.kt b/save-orchestrator/src/main/kotlin/org/cqfn/save/orchestrator/SaveOrchestrator.kt new file mode 100644 index 0000000000..503b8df470 --- /dev/null +++ b/save-orchestrator/src/main/kotlin/org/cqfn/save/orchestrator/SaveOrchestrator.kt @@ -0,0 +1,13 @@ +package org.cqfn.save.backend + +import org.springframework.boot.SpringApplication +import org.springframework.boot.autoconfigure.SpringBootApplication +import org.springframework.web.reactive.config.EnableWebFlux + +@SpringBootApplication +@EnableWebFlux +open class SaveOrchestrator + +fun main(args: Array) { + SpringApplication.run(SaveOrchestrator::class.java, *args) +} diff --git a/save-orchestrator/src/main/kotlin/org/cqfn/save/orchestrator/docker/ContainerManager.kt b/save-orchestrator/src/main/kotlin/org/cqfn/save/orchestrator/docker/ContainerManager.kt new file mode 100644 index 0000000000..f5b14cc259 --- /dev/null +++ b/save-orchestrator/src/main/kotlin/org/cqfn/save/orchestrator/docker/ContainerManager.kt @@ -0,0 +1,137 @@ +package org.cqfn.save.backend.docker + +import com.github.dockerjava.api.DockerClient +import com.github.dockerjava.api.command.BuildImageResultCallback +import com.github.dockerjava.core.DefaultDockerClientConfig +import com.github.dockerjava.core.DockerClientConfig +import com.github.dockerjava.api.model.HostConfig +import com.github.dockerjava.core.DockerClientImpl +import com.github.dockerjava.httpclient5.ApacheDockerHttpClient +import com.github.dockerjava.transport.DockerHttpClient +import org.apache.commons.compress.archivers.tar.TarArchiveEntry +import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream +import org.cqfn.save.domain.RunConfiguration +import org.slf4j.LoggerFactory + +import java.io.BufferedOutputStream +import java.io.ByteArrayOutputStream +import java.io.File +import java.nio.file.Files +import java.util.zip.GZIPOutputStream + +import kotlin.io.path.ExperimentalPathApi +import kotlin.io.path.createTempDirectory +import kotlin.io.path.createTempFile + +class ContainerManager(private val dockerHost: String = "unix:///var/run/docker.sock") { + private val dockerClientConfig: DockerClientConfig = DefaultDockerClientConfig + .createDefaultConfigBuilder() + .withDockerHost(dockerHost) + .withDockerTlsVerify(false) + .build() + private val dockerHttpClient: DockerHttpClient = ApacheDockerHttpClient.Builder() + .dockerHost(dockerClientConfig.dockerHost) + .build() + internal val dockerClient: DockerClient = DockerClientImpl.getInstance(dockerClientConfig, dockerHttpClient) + + /** + * Creates a docker container with [file], prepared to execute it + * + * @param runConfiguration a [RunConfiguration] for the supplied binary + * @param file an executable file + * @param resources additional files to be copied in the container too + * @throws DockerException if docker daemon has returned an error + * @throws RuntimeException if an exception not specific to docker has occurred + * @return id of created container or null if it wasn't created + */ + internal fun createWithFile(runConfiguration: RunConfiguration, + containerName: String, + file: File, + resources: Collection = emptySet()): String { + // ensure the image is present in the system + dockerClient.pullImageCmd(DOCKER_REPO) + .withTag("latest") + .start() + .awaitCompletion() + + val createContainerCmdResponse = dockerClient.createContainerCmd("ubuntu:latest") + .withCmd(runConfiguration.startCommand) + .withName(containerName) + .withHostConfig(HostConfig.newHostConfig() + .withRuntime("runsc") + ) + .exec() + + createTgzStream(file, *resources.toTypedArray()).use { out -> + dockerClient.copyArchiveToContainerCmd(createContainerCmdResponse.id) + .withTarInputStream(out.toByteArray().inputStream()) + .withRemotePath("/run") + .exec() + } + return createContainerCmdResponse.id + } + + /** + * Creates a docker image with proided [resources] + * + * @param baseImage base docker image rom which this image will be built + * @param resources files to be included into the image + * @param resourcesPath absolute path to resources inside the image's FS + * @throws DockerException + * @return id of the created docker image + */ + @OptIn(ExperimentalPathApi::class) + internal fun buildImageWithResources(baseImage: String = "ubuntu:latest", + baseDir: File, + resourcesPath: String): String { + val tmpDir = createTempDirectory().toFile() + baseDir.copyRecursively(File(tmpDir, "resources")) + val dockerFileAsText = + """ + FROM $baseImage + COPY resources $resourcesPath + RUN /bin/bash + """.trimIndent() // RUN command shouldn't matter because it will be replaced on container creation + val dockerFile = createTempFile(tmpDir.toPath()).toFile() + dockerFile.writeText(dockerFileAsText) + val buildImageResultCallback: BuildImageResultCallback = try { + dockerClient.buildImageCmd(dockerFile) + .withBaseDirectory(tmpDir) + .start() + } finally { + dockerFile.delete() + tmpDir.deleteRecursively() + } + return buildImageResultCallback.awaitImageId() + } + + /** + * Add [files] to .tar.gz archive and return the underlying [ByteArrayOutputStream] + * + * @param files files to be added to archive + * @return resulting [ByteArrayOutputStream] + */ + private fun createTgzStream(vararg files: File): ByteArrayOutputStream { + val out = ByteArrayOutputStream() + BufferedOutputStream(out).use { buffOut -> + GZIPOutputStream(buffOut).use { gzOut -> + TarArchiveOutputStream(gzOut).use { tgzOut -> + files.forEach { + tgzOut.putArchiveEntry(TarArchiveEntry(it)) + Files.copy(it.toPath(), tgzOut) + tgzOut.closeArchiveEntry() + } + tgzOut.finish() + } + gzOut.finish() + } + buffOut.flush() + } + return out + } + + companion object { + private const val DOCKER_REPO = "docker.io/library/ubuntu" + private val log = LoggerFactory.getLogger(ContainerManager::class.java) + } +} diff --git a/save-orchestrator/src/main/resources/application.properties b/save-orchestrator/src/main/resources/application.properties new file mode 100644 index 0000000000..61cb11ac09 --- /dev/null +++ b/save-orchestrator/src/main/resources/application.properties @@ -0,0 +1 @@ +server.port = 5100 \ No newline at end of file diff --git a/save-orchestrator/src/test/kotlin/org/cqfn/save/orchestrator/docker/ContainerManagerTest.kt b/save-orchestrator/src/test/kotlin/org/cqfn/save/orchestrator/docker/ContainerManagerTest.kt new file mode 100644 index 0000000000..90633b8b41 --- /dev/null +++ b/save-orchestrator/src/test/kotlin/org/cqfn/save/orchestrator/docker/ContainerManagerTest.kt @@ -0,0 +1,65 @@ +package org.cqfn.save.backend.docker + +import org.cqfn.save.domain.RunConfiguration +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.condition.DisabledIfSystemProperty +import kotlin.io.path.ExperimentalPathApi +import kotlin.io.path.createTempDirectory +import kotlin.io.path.createTempFile + +@OptIn(ExperimentalPathApi::class) +class ContainerManagerTest { + private lateinit var containerManager: ContainerManager + private lateinit var testContainerId: String + + @BeforeEach + fun setUp() { + containerManager = if (System.getProperty("os.name").startsWith("Windows")) { + // for docker inside WSL2 use it's eth0 network IP + ContainerManager("tcp://172.20.51.70:2375") + } else { + // for Linux "it just works"(c) with unix socket + ContainerManager() + } + } + + @Test + fun `should create a container and copy files into it`() { + val testFile = createTempFile().toFile() + testFile.writeText("wow such testing") + val resourceFile = createTempFile().toFile() + resourceFile.writeText("Lorem ipsum dolor sit amet") + testContainerId = containerManager.createWithFile( + RunConfiguration("./script.sh", testFile.name), + "testContainer", + testFile, + listOf(resourceFile) + ) + val inspectContainerResponse = containerManager.dockerClient + .inspectContainerCmd(testContainerId) + .exec() + Assertions.assertEquals("./script.sh", inspectContainerResponse.path) + Assertions.assertEquals(0, inspectContainerResponse.args.size) + Assertions.assertEquals("/testContainer", inspectContainerResponse.name) + } + + @Test + @DisabledIfSystemProperty(named = "os.name", matches = "Windows.*", disabledReason = "Cannot properly use Dockerfiles on Windows") + fun `should build an image with provided resources`() { + val resourcesDir = createTempDirectory() + repeat(5) { createTempFile(resourcesDir) } + val imageId = containerManager.buildImageWithResources(baseDir = resourcesDir.toFile(), resourcesPath = "/app/resources") + val inspectImageResponse = containerManager.dockerClient.inspectImageCmd(imageId).exec() + Assertions.assertTrue(inspectImageResponse.size!! > 0) + } + + @AfterEach + fun tearDown() { + if (::testContainerId.isInitialized) { + containerManager.dockerClient.removeContainerCmd(testContainerId).exec() + } + } +} diff --git a/settings.gradle.kts b/settings.gradle.kts index ebfb35b956..c34679643d 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -1,6 +1,7 @@ rootProject.name = "save" include("save-backend") +include("save-orchestrator") include("save-core") include("save-frontend") include("save-plugins")