Merge pull request #21 in APPBUILD/appverse-builder-api from develop …

…to master

* commit '52743592943a83083b3aa40f044a0595e2ce0f79': (35 commits)
  Update README
  update docs
  enable non expiry download tokens
  using spring on both sides to make sure it encodes/decodes correctly: fixes APB-158
  encoding name before adding to url: fixes APB-158
  will it work?!
  making sure the file is created and the content is flushed
  fixing condition
  adding before_build capability
  fixing some issues when logging. Fixing an issue with ldap authentication. Using the -q in dockgrant command when log debug is disabled
  update version
  fixing error when user does not have email setup in the LDAP
  removing explicit class frm config
  quoting the artifact regex
  fixing artifactRegex for LocalBuildExecutor
  fixing logs breaking
  adding OTA link to finished emails in case the artifact is an IPA
  using hikari builder instead of config
  fixing the place where swagger expects the token to be
  addin name placeholder
  ...

README.md

@@ -1,25 +1 @@
-# Appverse Builder
-
-## License
-
-    Copyright (c) 2012 GFT Appverse, S.L., Sociedad Unipersonal.
-
-     This Source  Code Form  is subject to the  terms of  the Appverse Public License
-     Version 2.0  ("APL v2.0").  If a copy of  the APL  was not  distributed with this
-     file, You can obtain one at <http://appverse.org/legal/appverse-license/>.
-
-     Redistribution and use in  source and binary forms, with or without modification,
-     are permitted provided that the  conditions  of the  AppVerse Public License v2.0
-     are met.
-
-     THIS SOFTWARE IS PROVIDED BY THE  COPYRIGHT HOLDERS  AND CONTRIBUTORS "AS IS" AND
-     ANY EXPRESS  OR IMPLIED WARRANTIES, INCLUDING, BUT  NOT LIMITED TO,   THE IMPLIED
-     WARRANTIES   OF  MERCHANTABILITY   AND   FITNESS   FOR A PARTICULAR  PURPOSE  ARE
-     DISCLAIMED. EXCEPT IN CASE OF WILLFUL MISCONDUCT OR GROSS NEGLIGENCE, IN NO EVENT
-     SHALL THE  COPYRIGHT OWNER  OR  CONTRIBUTORS  BE LIABLE FOR ANY DIRECT, INDIRECT,
-     INCIDENTAL,  SPECIAL,   EXEMPLARY,  OR CONSEQUENTIAL DAMAGES  (INCLUDING, BUT NOT
-     LIMITED TO,  PROCUREMENT OF SUBSTITUTE  GOODS OR SERVICES;  LOSS OF USE, DATA, OR
-     PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
-     WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT(INCLUDING NEGLIGENCE OR OTHERWISE)
-     ARISING  IN  ANY WAY OUT  OF THE USE  OF THIS  SOFTWARE,  EVEN  IF ADVISED OF THE
-     POSSIBILITY OF SUCH DAMAGE.
+# Appverse Builder API

pom.xml

@@ -12,7 +12,7 @@
 
     <groupId>org.appverse.builder</groupId>
     <artifactId>apb</artifactId>
-    <version>1.0</version>
+    <version>1.1</version>
     <packaging>war</packaging>
     <name>Apb</name>
 
@@ -381,6 +381,11 @@
             <artifactId>zip4j</artifactId>
             <version>1.3.2</version>
         </dependency>
+        <dependency>
+            <groupId>org.apache.velocity</groupId>
+            <artifactId>velocity</artifactId>
+            <version>1.6.2</version>
+        </dependency>
         <dependency>
             <groupId>com.fasterxml.jackson.dataformat</groupId>
             <artifactId>jackson-dataformat-yaml</artifactId>
@@ -406,6 +411,17 @@
             <version>1.4.1</version>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>com.googlecode.plist</groupId>
+            <artifactId>dd-plist</artifactId>
+            <version>1.16</version>
+        </dependency>
+        <dependency>
+            <groupId>com.unboundid</groupId>
+            <artifactId>unboundid-ldapsdk</artifactId>
+            <version>2.3.1</version>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
     <build>
         <defaultGoal>spring-boot:run</defaultGoal>

src/main/java/org/appverse/builder/build/BuildExecutorWorker.java

@@ -20,12 +20,9 @@
 
 import javax.inject.Inject;
 import java.io.*;
-import java.nio.file.Files;
 import java.util.List;
 import java.util.Optional;
 import java.util.concurrent.CopyOnWriteArrayList;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReentrantLock;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
@@ -36,14 +33,16 @@
 @Scope(value = ConfigurableBeanFactory.SCOPE_PROTOTYPE)
 public abstract class BuildExecutorWorker implements Runnable {
 
+    public static final String ARTIFACT_REGEX = "artifactRegex";
+    public static final String BUILD_TIMEOUT = "build.timeout";
+    public static final String ECHO_BEFORE_BUILD = "echo ==========BEFORE BUILD==========";
+    public static final String ECHO_BUILD = "echo =============BUILD==============";
     private final Logger log = LoggerFactory.getLogger(BuildExecutorWorker.class);
 
     public static InputStream raceConditionStream() {
         return new ByteArrayInputStream("Something unexpected has happened and we couldn't get the logs, please try again in a few seconds".getBytes());
     }
 
-    private Lock logLock = new ReentrantLock();
-
     @Inject
     private AppverseBuilderProperties appverseBuilderProperties;
 
@@ -96,11 +95,16 @@ public void setBuildServerController(BuildAgentQueueController buildServerContro
     }
 
     protected void log(String line, Object... params) {
+        if (log.isDebugEnabled()) {
+            log.debug("[{}] BUILD-LOG-LINE: {}", currentBuildRequest.getId(), line, params);
+        }
         if (currentLogger != null) {
-            logLock.lock();
-            currentLogger.println(DateTime.now().toString() + " " + MessageFormatter.arrayFormat(line, params).getMessage());
-            currentLogger.flush();
-            logLock.unlock();
+            try {
+                currentLogger.println(DateTime.now().toString() + " " + MessageFormatter.arrayFormat(line, params).getMessage());
+                currentLogger.flush();
+            } catch (Throwable t) {
+                log.debug("Could not write log message {} to build log", line, t);
+            }
         }
     }
 
@@ -115,10 +119,14 @@ private PrintWriter createLogWriter() throws IOException {
             public void println(String s) {
                 super.println(s);
                 super.flush();
-                connectedLoggers.forEach(writer -> {
-                    writer.println(s);
-                    writer.flush();
-                });
+                try {
+                    connectedLoggers.forEach(writer -> {
+                        writer.println(s);
+                        writer.flush();
+                    });
+                } catch (Throwable t) {
+                    log.warn("Error writing to to the log", t);
+                }
             }
 
             @Override
@@ -139,14 +147,10 @@ public InputStream getLogInputStream() {
                 if (currentBuildRequest == null) {
                     return raceConditionStream();
                 } else {
-                    logLock.lock();
                     final PipedOutputStream pipedOutputStream = new PipedOutputStream();
                     final PipedInputStream pipedInputStream = new PipedInputStream(pipedOutputStream);
                     final PrintWriter logger = new PrintWriter(pipedOutputStream);
-                    Files.lines(getBuildLogFile().toPath()).forEachOrdered(logger::println);
-                    logger.flush();
                     connectedLoggers.add(logger);
-                    logLock.unlock();
                     return pipedInputStream;
                 }
             } catch (IOException e) {
@@ -289,13 +293,15 @@ protected void redirectErrorStream(InputStream errorStream) {
         BufferedReader errorReader = new BufferedReader(new InputStreamReader(errorStream));
         new Thread(() -> {
             errorReader.lines().forEachOrdered(BuildExecutorWorker.this::logError);
+            log.debug("[{}] Finished writing error stream from execution", currentBuildRequest.getId());
         }).start();
     }
 
     protected void redirectInputStream(InputStream errorStream) {
         BufferedReader errorReader = new BufferedReader(new InputStreamReader(errorStream));
         new Thread(() -> {
             errorReader.lines().forEachOrdered(BuildExecutorWorker.this::log);
+            log.debug("[{}] Finished writing logs stream from execution", currentBuildRequest.getId());
         }).start();
     }
 
@@ -309,9 +315,26 @@ protected List<Artifact> getRequestArtifacts() {
 
     @SuppressWarnings("ResultOfMethodCallIgnored")
     protected File createBuildScript(BuildCommand buildCommand, File inputDir) throws IOException {
+
         File file = new File(inputDir, buildCommand.getScriptFileName());
-        FileUtils.writeStringToFile(file, buildCommand.getBuildScript());
+        StringBuilder builder = new StringBuilder();
+        if (buildCommand.getBeforeBuildScript() != null) {
+            builder.append(ECHO_BEFORE_BUILD).append("\n");
+
+            builder.append(buildCommand.getBeforeBuildScript()).append("\n");
+        }
+        builder.append(ECHO_BUILD).append("\n");
+        builder.append(buildCommand.getBuildScript()).append("\n");
+        FileUtils.writeStringToFile(file, builder.toString());
+
         file.setExecutable(true);
         return file;
     }
+
+    protected Long getTimeoutForRequest() {
+        return Optional
+            .ofNullable(getCurrentBuildRequest().getVariables().get(BUILD_TIMEOUT))
+            .map(Long::parseLong)
+            .orElse(getAppverseBuilderProperties().getBuild().getMaxBuildTimeout());
+    }
 }

src/main/java/org/appverse/builder/build/LocalBuildExecutorWorker.java

@@ -12,9 +12,14 @@
 
 import java.io.File;
 import java.io.IOException;
+import java.nio.file.FileVisitResult;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.SimpleFileVisitor;
+import java.nio.file.attribute.BasicFileAttributes;
 import java.util.Optional;
 import java.util.concurrent.TimeUnit;
-import java.util.stream.Stream;
+import java.util.regex.Pattern;
 
 /**
  * Created by panthro on 22/02/16.
@@ -106,8 +111,7 @@ protected void execute(BuildRequestDTO request) {
             Process buildProcess = Runtime.getRuntime().exec(buildCommand.asArray());
             redirectInputStream(buildProcess.getInputStream());
             redirectErrorStream(buildProcess.getErrorStream());
-            //TODO externalize this timeout
-            if (!buildProcess.waitFor(30, TimeUnit.MINUTES) || buildProcess.exitValue() != 0) {
+            if (!buildProcess.waitFor(getTimeoutForRequest(), TimeUnit.SECONDS) || buildProcess.exitValue() != 0) {
                 failed("Error executing the build command");
                 return;
             }
@@ -118,21 +122,35 @@ protected void execute(BuildRequestDTO request) {
         }
 
 
-        Optional.ofNullable(request.getVariables().get("artifactRegex")).ifPresent(artifactsRegex -> {
+        Optional.ofNullable(request.getVariables().get(ARTIFACT_REGEX)).ifPresent(artifactsRegex -> {
             getTemporaryArtifactsDir().ifPresent(localArtifactsDir -> {
-                Optional.ofNullable(inputDir.listFiles()).ifPresent(files -> {
-                    Stream.of(files).filter(file -> file.getName().matches(artifactsRegex)).forEach(file -> {
-                        try {
-                            FileUtils.copyFileToDirectory(file, localArtifactsDir);
-                        } catch (IOException e) {
-                            log.warn("Error copying artifact {} to localArtifactsDir {} ", file, localArtifactsDir, e);
-                            logError("Could not copy artifact {} to local artifacts dir", file.getName());
-                        }
-                    });
-                });
+                try {
+                    Files.walkFileTree(inputDir.toPath(), new LocalArtifactFileVisitor(Pattern.compile(artifactsRegex), localArtifactsDir.toPath()));
+                } catch (Throwable e) {
+                    log.warn("Error getting artifacts from localArtifactsDir {} ", localArtifactsDir, e);
+                }
             });
         });
 
     }
+
+    private class LocalArtifactFileVisitor extends SimpleFileVisitor<Path> {
+
+        private Pattern artifactRegex;
+        private Path localArtifactsDir;
+
+        public LocalArtifactFileVisitor(Pattern artifactRegex, Path localArtifactsDir) {
+            this.artifactRegex = artifactRegex;
+            this.localArtifactsDir = localArtifactsDir;
+        }
+
+        @Override
+        public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
+            if (Files.exists(file) && artifactRegex.matcher(file.toString()).find()) {
+                Files.copy(file, localArtifactsDir.resolve(file.getFileName()));
+            }
+            return super.visitFile(file, attrs);
+        }
+    }
 }
 

src/main/java/org/appverse/builder/build/comand/BuildCommand.java

@@ -14,10 +14,11 @@ public abstract class BuildCommand {
 
 
     protected String buildScript;
-    private String binary;
-    private String workDir;
-    private boolean createScript = true;
-    private String scriptFileName;
+    protected String binary;
+    protected String workDir;
+    protected boolean createScript = true;
+    protected String scriptFileName;
+    protected String beforeBuildScript;
 
     /**
      * Builds the command as String
@@ -93,4 +94,12 @@ public String getBuildScript() {
     public void setBuildScript(String buildScript) {
         this.buildScript = buildScript;
     }
+
+    public void setBeforeBuildScript(String beforeBuildScript) {
+        this.beforeBuildScript = beforeBuildScript;
+    }
+
+    public String getBeforeBuildScript() {
+        return beforeBuildScript;
+    }
 }

src/main/java/org/appverse/builder/build/comand/BuildCommandBuilder.java

@@ -8,6 +8,8 @@
  */
 public interface BuildCommandBuilder<T extends BuildCommand> {
 
+    String BEFORE_BUILD = "before_build";
+
     String SCRIPT = "script";
 
     String IMAGE_NAME = "imageName";

src/main/java/org/appverse/builder/build/comand/docker/DockerBuildCommandBuilder.java

@@ -42,6 +42,8 @@ public DockerCommand buildCommand(BuildAgentDTO buildAgent, BuildRequestDTO buil
         dockerCommand.setImageName(imageName);
         dockerCommand.setScriptFileName(RandomStringUtils.randomAlphabetic(10) + ".sh");//TODO get from the platform the actual script name and or extension at least to support .sh and .bat
 
+        dockerCommand.setBeforeBuildScript(Optional.ofNullable(buildRequest.getVariables().get(BuildCommandBuilder.BEFORE_BUILD)).orElse(null));
+
         dockerCommand.setBuildScript(Optional.ofNullable(buildRequest.getVariables().get(BuildCommandBuilder.SCRIPT))
             .orElse(null));
         return dockerCommand;

src/main/java/org/appverse/builder/build/comand/docker/DockerCommand.java

@@ -47,9 +47,9 @@ public List<String> getArgs() {
             args.add("--rm");
         }
         environmentVariables.forEach((key, value) -> {
-            if (!StringUtils.isEmpty(value) && !value.equals(buildScript)) {
+            if (!StringUtils.isEmpty(value) && !value.equals(buildScript) && !value.equals(beforeBuildScript)) {
                 args.add("-e");
-                args.add(key + "=" + value);
+                args.add(key + "='" + value + "'");
             }
         });
         args.add(getImageName());

src/main/java/org/appverse/builder/build/comand/dockgrant/DockgrantBuildCommandBuilder.java

@@ -49,6 +49,7 @@ public DockgrantCommand buildCommand(BuildAgentDTO buildAgent, BuildRequestDTO b
         dockgrantCommand.setRequestInputDir(buildAgentService.getBuildAgentRemoteRequestInputDir(buildAgent, buildRequest));
         dockgrantCommand.getEnvironmentVariables().putAll(buildRequest.getVariables());
         dockgrantCommand.setScriptFileName(RandomStringUtils.randomAlphabetic(10) + ".sh");//TODO get from the platform the actual script name and or extension at least to support .sh and .bat
+        dockgrantCommand.setQuiet(!log.isDebugEnabled());
 
 
         String imageName = Optional.ofNullable(buildRequest.getVariables().get(IMAGE_NAME))
@@ -63,6 +64,7 @@ public DockgrantCommand buildCommand(BuildAgentDTO buildAgent, BuildRequestDTO b
             dockgrantCommand.setImageName(imageName);
         }
 
+        dockgrantCommand.setBeforeBuildScript(Optional.ofNullable(buildRequest.getVariables().get(BuildCommandBuilder.BEFORE_BUILD)).orElse(null));
 
         dockgrantCommand.setBuildScript(Optional.ofNullable(buildRequest.getVariables().get(BuildCommandBuilder.SCRIPT))
             .orElse(null));

src/main/java/org/appverse/builder/build/comand/dockgrant/DockgrantCommand.java

@@ -44,9 +44,9 @@ public List<String> getArgs() {
 
 
         environmentVariables.forEach((key, value) -> {
-            if (!StringUtils.isEmpty(value) && !value.equals(buildScript)) {
+            if (!StringUtils.isEmpty(value) && !value.equals(buildScript) && !value.equals(beforeBuildScript)) {
                 args.add("-e");
-                args.add(key + "=" + value);
+                args.add(key + "='" + value + "'");
             }
         });
         args.add("--image");
@@ -58,9 +58,10 @@ public List<String> getArgs() {
         if (imageUrl != null) {
             args.add("--imageurl=" + getImageUrl());
         }
-        if (getBuildScript() != null) {
-
+        if (isQuiet()) {
+            args.add("-q");
         }
+
         if (getBuildScript() != null) {
             args.add("--script");
             if (!isCreateScript()) {