Merge pull request #198 from CodeKaio/github_module_import

✨ : github module import

pom.xml

@@ -40,6 +40,7 @@
 	</ciManagement>
 
 	<properties>
+		<antlr4.version>4.7.2</antlr4.version>
 		<java.version>11</java.version>
 		<jersey.version>2.27</jersey.version>
 		<junit-jupiter.version>5.5.2</junit-jupiter.version>
@@ -215,7 +216,7 @@
 		<dependency>
 			<groupId>org.antlr</groupId>
 			<artifactId>antlr4-runtime</artifactId>
-			<version>4.7.2</version>
+			<version>${antlr4.version}</version>
 		</dependency>
 
 		<!-- selenium tests -->
@@ -313,7 +314,7 @@
 			<plugin>
 				<groupId>org.antlr</groupId>
 				<artifactId>antlr4-maven-plugin</artifactId>
-				<version>4.7.2</version>
+				<version>${antlr4.version}</version>
 				<executions>
 					<execution>
 						<goals>
@@ -326,6 +327,25 @@
 					</execution>
 				</executions>
 			</plugin>
+
+			<plugin>
+				<groupId>org.pitest</groupId>
+				<artifactId>pitest-maven</artifactId>
+				<version>1.4.10</version>
+				<dependencies>
+					<dependency>
+						<groupId>org.pitest</groupId>
+						<artifactId>pitest-junit5-plugin</artifactId>
+						<version>0.10</version>
+					</dependency>
+				</dependencies>
+				<configuration>
+					<excludedTestClasses>*IT</excludedTestClasses>
+					<avoidCallsTo>
+						<avoidCallsTo>kotlin.jvm.internal</avoidCallsTo>
+					</avoidCallsTo>
+				</configuration>
+			</plugin>
 		</plugins>
 	</build>
 

src/main/java/io/codeka/gaia/Gaia.java

@@ -2,10 +2,8 @@
 
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
-import org.springframework.data.mongodb.repository.config.EnableMongoRepositories;
 
 @SpringBootApplication
-@EnableMongoRepositories
 public class Gaia {
 
 	public static void main(String[] args) {

src/main/java/io/codeka/gaia/config/MongoConfig.java

@@ -3,8 +3,10 @@
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.context.annotation.Configuration;
 import org.springframework.data.mongodb.core.convert.MappingMongoConverter;
+import org.springframework.data.mongodb.repository.config.EnableMongoRepositories;
 
 @Configuration
+@EnableMongoRepositories(basePackages = "io.codeka.gaia")
 public class MongoConfig {
 
     @Autowired

src/main/java/io/codeka/gaia/config/security/SecurityConfig.java

@@ -35,7 +35,7 @@ protected void configure(HttpSecurity http) throws Exception {
         http
                 .csrf().disable()
                 .authorizeRequests()
-                    .antMatchers("/api/**").permitAll()
+                    .antMatchers("/api/state/**").permitAll()
                     .antMatchers("/webjars/**").permitAll()
                     .antMatchers("/css/**", "/js/**", "/favicon.ico", "/images/**").permitAll()
                     .antMatchers("/**").authenticated()

src/main/java/io/codeka/gaia/hcl/HclParser.kt → src/main/java/io/codeka/gaia/hcl/HclParserImpl.kt

@@ -6,10 +6,18 @@ import io.codeka.gaia.modules.bo.Output
 import io.codeka.gaia.modules.bo.Variable
 import org.antlr.v4.runtime.CharStreams
 import org.antlr.v4.runtime.CommonTokenStream
+import org.springframework.stereotype.Service
 
-class HclParser {
+interface HclParser {
+    fun parseContent(content: String): HclVisitor
+    fun parseVariables(content: String): List<Variable>
+    fun parseOutputs(content: String): List<Output>
+}
 
-    private fun parseContent(content: String): HclVisitor {
+@Service
+class HclParserImpl : HclParser {
+
+    override fun parseContent(content: String): HclVisitor {
         // loading test file
         val charStream = CharStreams.fromString(content)
 
@@ -28,12 +36,12 @@ class HclParser {
         return hclVisitor
     }
 
-    fun parseVariables(content:String): List<Variable> {
+    override fun parseVariables(content:String): List<Variable> {
         val hclVisitor = parseContent(content)
         return hclVisitor.variables
     }
 
-    fun parseOutputs(content:String): List<Output> {
+    override fun parseOutputs(content:String): List<Output> {
         val hclVisitor = parseContent(content)
         return hclVisitor.outputs
     }

src/main/java/io/codeka/gaia/modules/controller/ModuleRestController.java

@@ -4,11 +4,13 @@
 import io.codeka.gaia.modules.repository.TerraformModuleRepository;
 import io.codeka.gaia.teams.User;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.http.HttpStatus;
 import org.springframework.security.access.annotation.Secured;
 import org.springframework.web.bind.annotation.*;
 
 import javax.validation.Valid;
 import java.util.List;
+import java.util.UUID;
 
 /**
  * Rest controller for the module API
@@ -45,6 +47,14 @@ public TerraformModule findModule(@PathVariable String id, User user){
         return module;
     }
 
+    @PostMapping
+    @ResponseStatus(HttpStatus.CREATED)
+    public TerraformModule createModule(@RequestBody TerraformModule module, User user){
+        module.setId(UUID.randomUUID().toString());
+        module.setCreatedBy(user);
+        return moduleRepository.save(module);
+    }
+
     @PutMapping("/{id}")
     public TerraformModule saveModule(@PathVariable String id, @RequestBody @Valid TerraformModule module, User user){
         var existingModule = moduleRepository.findById(id).orElseThrow(ModuleNotFoundException::new);

src/main/java/io/codeka/gaia/modules/controller/ModulesMVCController.java

@@ -35,6 +35,11 @@ public String modulesList(){
         return "modules";
     }
 
+    @GetMapping("/modules/import")
+    public String importModule() {
+        return "modules_import";
+    }
+
     @GetMapping("/modules/{id}")
     public String module(@PathVariable String id, Model model, User user){
         var module = terraformModuleRepository.findById(id).orElseThrow(ModuleNotFoundException::new);

src/main/java/io/codeka/gaia/registries/RegistryApi.kt

@@ -0,0 +1,15 @@
+package io.codeka.gaia.registries
+
+import io.codeka.gaia.registries.github.GithubRepository
+import io.codeka.gaia.teams.User
+import org.springframework.web.client.RestTemplate
+
+interface RegistryApi<T> {
+
+    fun getRepositories(user: User) : List<String>
+
+    fun getRepository(user: User, owner: String, repo: String): T
+
+    fun getFileContent(user: User, projectId: String, filename: String): String
+
+}

src/main/java/io/codeka/gaia/registries/RegistryRawContent.kt

@@ -35,14 +35,13 @@ abstract class RegistryRawContent(private val registryType: RegistryType, privat
         val requestEntity = HttpEntity<Any>(headers)
 
         val response = restTemplate.exchange(
-                this.registryType.readmeUrl,
+                this.registryType.readmeUrl.replace("{id}", module.registryDetails.projectId),
                 HttpMethod.GET,
                 requestEntity,
-                RegistryFile::class.java,
-                module.registryDetails.projectId)
+                RegistryFile::class.java)
 
         if(response.statusCode == HttpStatus.OK) {
-            return Optional.of(String(Base64.getDecoder().decode(response.body?.content)))
+            return Optional.of(String(Base64.getDecoder().decode(response.body?.content?.replace("\n",""))))
         }
         return Optional.empty()
     }

src/main/java/io/codeka/gaia/registries/controller/GithubRegistryController.kt

@@ -0,0 +1,54 @@
+package io.codeka.gaia.registries.controller
+
+import io.codeka.gaia.hcl.HclParser
+import io.codeka.gaia.modules.bo.TerraformModule
+import io.codeka.gaia.modules.repository.TerraformCLIRepository
+import io.codeka.gaia.modules.repository.TerraformModuleRepository
+import io.codeka.gaia.registries.RegistryApi
+import io.codeka.gaia.registries.RegistryDetails
+import io.codeka.gaia.registries.RegistryType
+import io.codeka.gaia.registries.github.GithubRepository
+import io.codeka.gaia.teams.User
+import org.springframework.http.HttpStatus
+import org.springframework.security.access.annotation.Secured
+import org.springframework.web.bind.annotation.*
+import java.util.*
+
+@RestController
+@RequestMapping("/api/registries/github")
+@Secured
+class GithubRegistryController(
+        private val githubRegistryApi: RegistryApi<GithubRepository>,
+        private val hclParser: HclParser,
+        private val cliRepository: TerraformCLIRepository,
+        private val moduleRepository: TerraformModuleRepository) {
+
+    @GetMapping("/repositories")
+    fun getRepositories(user: User): List<String> {
+        return this.githubRegistryApi.getRepositories(user)
+    }
+
+    @GetMapping("/repositories/{owner}/{repo}/import")
+    @ResponseStatus(HttpStatus.CREATED)
+    fun importRepository(@PathVariable owner: String, @PathVariable repo: String, user: User): TerraformModule {
+        val module = TerraformModule()
+        module.id = UUID.randomUUID().toString()
+
+        val githubRepository = githubRegistryApi.getRepository(user, owner, repo)
+
+        module.gitRepositoryUrl = githubRepository.htmlUrl
+        module.gitBranch = "master"
+        module.name = githubRepository.fullName
+        module.cliVersion = cliRepository.listCLIVersion().first()
+
+        module.registryDetails = RegistryDetails(RegistryType.GITHUB, githubRepository.fullName)
+        module.createdBy = user
+
+        // get variables
+        val variablesFile = githubRegistryApi.getFileContent(user, "$owner/$repo", "variables.tf")
+        module.variables = hclParser.parseVariables(variablesFile)
+
+        // saving module !
+        return moduleRepository.save(module)
+    }
+}

src/main/java/io/codeka/gaia/registries/github/GitHubRawContent.kt

@@ -2,11 +2,13 @@ package io.codeka.gaia.registries.github
 
 import io.codeka.gaia.registries.RegistryType
 import io.codeka.gaia.registries.RegistryRawContent
+import org.springframework.stereotype.Component
 import org.springframework.web.client.RestTemplate
 import java.util.regex.Pattern
 
+@Component
 class GitHubRawContent(restTemplate: RestTemplate) : RegistryRawContent(RegistryType.GITHUB, restTemplate) {
 
-    override val pattern: Pattern = Pattern.compile("^http[s]?://[www.]?github.com(.*).git$")
+    override val pattern: Pattern = Pattern.compile("^https?:\\/\\/(w{3}\\.)?github\\.com(.*)(.git)?\$")
 
 }

src/main/java/io/codeka/gaia/registries/github/GithubRegistryApi.kt

@@ -0,0 +1,79 @@
+package io.codeka.gaia.registries.github
+
+import com.fasterxml.jackson.annotation.JsonAlias
+import com.fasterxml.jackson.annotation.JsonProperty
+import com.fasterxml.jackson.annotation.JsonValue
+import com.fasterxml.jackson.databind.annotation.JsonNaming
+import io.codeka.gaia.registries.RegistryApi
+import io.codeka.gaia.registries.RegistryFile
+import io.codeka.gaia.registries.RegistryType
+import io.codeka.gaia.teams.User
+import org.springframework.http.HttpEntity
+import org.springframework.http.HttpHeaders
+import org.springframework.http.HttpMethod
+import org.springframework.http.HttpStatus
+import org.springframework.stereotype.Service
+import org.springframework.web.client.RestTemplate
+import java.util.*
+
+@Service
+class GithubRegistryApi(val restTemplate: RestTemplate): RegistryApi<GithubRepository> {
+
+    fun <T> callWithAuth(url: String, token: String, responseType: Class<T>): T{
+        val headers = HttpHeaders()
+        if(token != null) {
+            headers.add("Authorization", "Bearer $token")
+        }
+
+        val requestEntity = HttpEntity<Any>(headers)
+
+        val response = restTemplate.exchange(
+                url,
+                HttpMethod.GET,
+                requestEntity,
+                responseType)
+        if(response.statusCode == HttpStatus.OK) {
+            return response.body
+        }
+        else {
+            TODO("error code mgmt")
+        }
+    }
+
+    override fun getRepositories(user: User): List<String> {
+        // fetching repositories
+        val url = "https://api.github.com/user/repos?visibility=public"
+
+        val token = user.oAuth2User?.token!!
+
+        val repos = callWithAuth(url, token, Array<GithubRepository>::class.java)
+
+        return repos.map { it.fullName }.toList()
+    }
+
+    override fun getRepository(user: User, owner: String, repo: String): GithubRepository {
+        // fetching repositories
+        val url = "https://api.github.com/repos/$owner/$repo"
+
+        val token = user.oAuth2User?.token!!
+
+        return callWithAuth(url, token, GithubRepository::class.java)
+    }
+
+    override fun getFileContent(user: User, projectId: String, filename: String): String {
+        val url = "https://api.github.com/repos/$projectId/contents/$filename?ref=master";
+
+        val token = user.oAuth2User?.token!!;
+
+        val file = callWithAuth(url, token, RegistryFile::class.java)
+
+        // removing trailing \n
+        println(file.content.replace("\n", ""))
+        return String(Base64.getDecoder().decode(file.content.replace("\n", "")))
+    }
+
+}
+
+data class GithubRepository(
+        @JsonProperty("full_name") val fullName: String,
+        @JsonProperty("html_url") val htmlUrl: String)

src/main/resources/application.properties

@@ -1,4 +1,5 @@
-logging.level.org.springframework.web=TRACE
+logging.level.org.springframework.web=INFO
+logging.level.org.springframework.data.mongodb=DEBUG
 
 spring.data.rest.basePath=/api
 

src/main/resources/static/css/style.css

@@ -2274,7 +2274,7 @@ i.icon {
 /** button section **/
 
 .btn {
-    font-size: 14px;
+    /*font-size: 14px;*/
 }
 
 .button_sction {

src/main/resources/templates/error/403.html

@@ -2,7 +2,7 @@
 <html lang="en" xmlns:th="http://www.thymeleaf.org">
 <head th:replace="layout/header :: header(~{::title})">
 
-    <title>Gaia - Page not found</title>
+    <title>Gaia - Access Forbidden</title>
 
 </head>
 <body>

src/main/resources/templates/error/500.html

@@ -2,7 +2,7 @@
 <html lang="en" xmlns:th="http://www.thymeleaf.org">
 <head th:replace="layout/header :: header(~{::title})">
 
-    <title>Gaia - Page not found</title>
+    <title>Gaia - Internal Server Error</title>
 
 </head>
 <body>

src/main/resources/templates/modules.html

@@ -27,9 +27,16 @@
                         </div>
                     </div>
                 </div>
-                <div id="modules">
-
+                <div class="row column_title">
+                    <div class="col-md-12">
+                        <div id="controls">
+                            <div class="page_controls">
+                                <a href="/modules/import" class="btn btn-success"><i class="far fa-save"></i> Import Module</a>
+                            </div>
+                        </div>
+                    </div>
                 </div>
+                <div id="modules"></div>
             </div>
 
         </div>