Upload stream

binders/difido-testng/src/main/java/il/co/topq/difido/config/RemoteDifidoConfig.java

@@ -21,7 +21,11 @@ public enum RemoteDifidoOptions implements ConfigOptions {
 		USE_SHARED_EXECUTION("use.shared.execution", "false"),
 		EXISTING_EXECUTION_ID("existing.execution.id", "-1"),
 		FORCE_NEW_EXECUTION("force.new.execution", "false"),
-		APPEND_TO_EXISTING_EXECUTION("append.to.existing.execution", "false");
+		APPEND_TO_EXISTING_EXECUTION("append.to.existing.execution", "false"),
+		COMPRESS_FILES_ABOVE("compress.files.above","5000"),
+		DONT_COMPRESS_EXTENSIONS("dont.compress.extensions","7z;zip;rar;gzip;gz;jpg;jpeg;png;gif;avi;xvid;mp4;mp3;tif;tiff;pdf;wmf;svg;exe;jar");
+
+
 		// @formatter:on
 
 		private String property;

binders/difido-testng/src/main/java/il/co/topq/difido/reporters/DifidoClient.java

@@ -13,6 +13,7 @@
 import org.apache.commons.httpclient.methods.PutMethod;
 import org.apache.commons.httpclient.methods.RequestEntity;
 import org.apache.commons.httpclient.methods.StringRequestEntity;
+import org.apache.commons.httpclient.methods.multipart.ByteArrayPartSource;
 import org.apache.commons.httpclient.methods.multipart.FilePart;
 import org.apache.commons.httpclient.methods.multipart.MultipartRequestEntity;
 import org.apache.commons.httpclient.methods.multipart.Part;
@@ -83,9 +84,18 @@ public void addTestDetails(int executionId, TestDetails testDetails) throws Exce
 		handleResponseCode(method, responseCode);
 	}
 
+
+	public void addFile(final int executionId, final String uid, final byte[] bytes, String fileName) throws Exception {
+		Part[] parts = new Part[] { new FilePart("file", new ByteArrayPartSource(fileName, bytes))};
+		addFile(executionId,uid,parts);
+	}
 	public void addFile(final int executionId, final String uid, final File file) throws Exception {
-		PostMethod method = new PostMethod(baseUri + "executions/" + executionId + "/details/" + uid + "/file/");
 		Part[] parts = new Part[] { new FilePart("file", file) };
+		addFile(executionId,uid,parts);
+	}
+
+	private void addFile(final int executionId, final String uid, final Part[] parts) throws Exception {
+		PostMethod method = new PostMethod(baseUri + "executions/" + executionId + "/details/" + uid + "/file/");
 		method.setRequestEntity(new MultipartRequestEntity(parts, method.getParams()));
 		final int responseCode = client.executeMethod(method);
 		handleResponseCode(method, responseCode);

binders/difido-testng/src/main/java/il/co/topq/difido/reporters/RemoteDifidoReporter.java

@@ -2,13 +2,20 @@
 
 import java.io.File;
 import java.util.ArrayList;
+import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.logging.Logger;
 
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.FilenameUtils;
 import org.testng.ISuite;
 
+import il.co.topq.difido.ZipUtils;
+import il.co.topq.difido.config.DifidoConfig;
 import il.co.topq.difido.config.RemoteDifidoConfig;
+import il.co.topq.difido.config.DifidoConfig.DifidoOptions;
 import il.co.topq.difido.config.RemoteDifidoConfig.RemoteDifidoOptions;
 import il.co.topq.difido.model.execution.Execution;
 import il.co.topq.difido.model.execution.ScenarioNode;
@@ -34,6 +41,8 @@ public class RemoteDifidoReporter extends AbstractDifidoReporter {
 	private RemoteDifidoConfig difidoConfig;
 
 	private ExecutionDetails details;
+
+	private Set<String> extentionsToSkip = null;
 
 	/**
 	 * When files are added in the setup phase, there is no test context and no
@@ -200,11 +209,54 @@ public void addFile(File file) {
 
 	private void sendFileToServer(File file) {
 		try {
-			client.addFile(executionId, getTestDetails().getUid(), file);
+
+			int thresholdInBytes = difidoConfig.getPropertyAsInt(RemoteDifidoOptions.COMPRESS_FILES_ABOVE);
+			if (thresholdInBytes > 0 && file.length() > thresholdInBytes && isCompressable(file)){
+
+				//zip the file to an in-memory byte array and upload to server 
+				//adding .gz to the original fileName;
+				byte[] zippped = ZipUtils.gzipToBytesArray(file);
+				if (null != zippped){
+					client.addFile(executionId, getTestDetails().getUid(), zippped, file.getName().concat(".gz"));
+				}
+				else {
+					log.warning("Failed to zip file on the fly, uploading original");
+					client.addFile(executionId, getTestDetails().getUid(), file);
+				}
+			}
+			else {
+				client.addFile(executionId, getTestDetails().getUid(), file);
+			}
 		} catch (Exception e) {
 			log.warning("Failed uploading file " + file.getName() + " to remote server due to " + e.getMessage());
 		}
 	}
+
+	/**
+	 * checks whether the file extension is 'blacklisted' for compression
+	 */
+	private boolean isCompressable(File f){
+		String ext = FilenameUtils.getExtension(f.getAbsolutePath()).toLowerCase();
+		return !getSkippedExtensions().contains(ext);
+	}
+
+	private Set<String> getSkippedExtensions(){
+		initSkippedExtensions();
+		return this.extentionsToSkip;
+	}
+
+	private void initSkippedExtensions(){
+		if (this.extentionsToSkip != null)
+			return;
+
+		this.extentionsToSkip = new LinkedHashSet<>();
+		List<String> extentionsList = difidoConfig.getPropertyAsList(RemoteDifidoOptions.DONT_COMPRESS_EXTENSIONS);
+		for (String extension : extentionsList){
+			int startFrom = extension.lastIndexOf(".") + 1; //strip off any irrelevant file parts a user may have entered (e.g. tar.gz, or *.exe)
+			extentionsToSkip.add(extension.toLowerCase().substring(startFrom));
+		}
+	}
+
 
 	/**
 	 * Elements that are created in setup phases, before test context is created

binders/difido-testng/src/test/java/il/co/topq/difido/DifidoReporterTests.java

@@ -11,6 +11,7 @@
 
 import javax.imageio.ImageIO;
 
+import org.apache.commons.io.FileUtils;
 import org.testng.annotations.Test;
 
 import il.co.topq.difido.model.Enums.Status;
@@ -77,6 +78,33 @@ public void testAddFile() {
 		File file = new File("pom.xml");
 		report.addFile(file, "This is the file");
 	}
+
+	@Test 
+	public void testAddBigFile(){
+		try {
+			File file = File.createTempFile("big", ".log");
+			StringBuilder sb = new StringBuilder();
+			for (int i=0;i<1000000;i++){
+				sb.append("line: #" + i).append("\n");
+			}
+			FileUtils.write(file, sb.toString());
+
+
+			report.addFile(file, "Really big file");
+
+		} catch (Exception e) {
+			// TODO: handle exception
+		}
+
+	}
+
+	@Test
+	public void testAddImageFile() throws Exception{
+		//Reporter.log("This file should be uploaded uncompressed");
+		File image = new File(DifidoReporterTests.class.getResource("/login.png").toURI());
+		report.log("File: " + image.getAbsolutePath() + " exists()=" + image.exists());
+		report.addFile(image, "This file should be uncompressed on the server");
+	}
 
 	@Test(description = "Adding screenshot to the report")
 	public void testAddScreenshot() throws IOException, AWTException {

difido-reports-common/src/main/java/il/co/topq/difido/ZipUtils.java

@@ -1,15 +1,23 @@
 package il.co.topq.difido;
 
+import java.io.ByteArrayOutputStream;
 import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
 import java.io.FileOutputStream;
+import java.io.IOException;
 import java.io.InputStream;
+import java.nio.file.Files;
 import java.util.ArrayList;
 import java.util.Enumeration;
 import java.util.List;
 import java.util.jar.JarEntry;
 import java.util.jar.JarFile;
+import java.util.zip.GZIPOutputStream;
 
-class ZipUtils {
+import org.apache.commons.io.FileUtils;
+
+public class ZipUtils {
 
 	/**
 	 * uncompress all the files in the specified zipFile to the specified
@@ -51,4 +59,110 @@ static List<File> decopmerss(String zipFile, String outputFolder, String filter)
 
 	}
 
+	/**
+	 * Gzips the originalFile and returns a zipped one.
+	 * @param originalFile
+	 * @return
+	 */
+	public static File gzip(File originalFile) {
+		if (originalFile == null || !originalFile.exists())
+			return null;
+
+
+		File zippedFile = getZippedFile(originalFile);
+		if (zippedFile == null)
+			return null;
+
+		try (FileInputStream input = new FileInputStream(originalFile); 
+			 FileOutputStream output = new FileOutputStream(zippedFile);
+			 GZIPOutputStream gzipOS = new GZIPOutputStream(output)) {
+
+			byte[] buffer = new byte[1024];
+			int len;
+			while((len= input.read(buffer)) != -1){
+				gzipOS.write(buffer, 0, len);
+			}
+
+			gzipOS.close();
+			output.close();
+			input.close();
+
+			return zippedFile;
+		}
+
+		catch(Exception e){
+			e.printStackTrace();
+		}
+
+		return originalFile;
+
+
+	}
+
+	/**
+	 * Gzipps the given file and returns it as byte[] 
+	 * @param file
+	 * @return
+	 */
+	public static byte[] gzipToBytesArray(File file){
+
+		try(FileInputStream input = new FileInputStream(file);
+				ByteArrayOutputStream output = new ByteArrayOutputStream();
+				GZIPOutputStream gzipOS = new GZIPOutputStream(output);){
+
+			byte[] buffer = new byte[1024];
+			int len;
+			while((len= input.read(buffer)) != -1){
+				gzipOS.write(buffer, 0, len);
+			}
+
+
+			return output.toByteArray();
+
+		} catch (FileNotFoundException e) {
+			e.printStackTrace();
+		} catch (IOException e) {
+			e.printStackTrace();
+		}
+
+
+		return null;
+
+	}
+
+
+	/**
+	 * Since we must preserve the fileName of the originalFile (or browser auto-unzip 
+	 * will not work  later, when the resource is requested), so if a file x.y.gz already exists in temp dir
+	 * we will have to create a nester dir and place our file there.
+	 * 
+	 * @param originalFile
+	 * @return
+	 */
+	private static File getZippedFile(File originalFile){
+		String tempDir = System.getProperty("java.io.tmpdir");
+		String fileName = originalFile.getName().concat(".gz");
+		File f = new File(String.format("%s%s",tempDir,fileName));
+		if (!f.exists())
+			return f;
+
+		File nestedDir = new File(String.format("%s%s",tempDir,System.nanoTime()));
+		try {
+			if (!nestedDir.mkdirs()){
+				return null;
+			}
+
+
+			return new File(nestedDir,fileName);
+
+
+		} catch (Exception e) {
+			e.printStackTrace();
+			return null;
+		}
+
+
+
+
+	}
 }

server/difido-server/config/difido_config.properties

@@ -18,6 +18,9 @@ custom.execution.properties=
 enable.html.reports=true
 
 
+# Will enable to store/fetch only archived (.gz) version of resources
+enable.archived.resources=false
+
 # List of plugin classes
 plugin.classes=il.co.topq.report.plugins.mail.DefaultMailPlugin
 

server/difido-server/src/main/java/il/co/topq/report/Configuration.java

@@ -29,6 +29,7 @@ public enum ConfigProps {
 		ELASTIC_HTTP_PORT("elastic.http.port","9200"),
 		ELASTIC_TRANSPORT_TCP_PORT("elastic.transport.tcp.port","9300"),
 		ENABLE_HTML_REPORTS("enable.html.reports", "true"), 
+		ENABLE_ARCHIVED_RESOURCES("enable.archived.resources","false"),
 		DAYS_TO_KEEP_HTML_REPORTS("days.to.keep.html.reports","0"),
 		EXTERNAL_LINKS("external.links",""),
 		ENABLE_MAIL("enable.mail", "false"), MAIL_USER_NAME("mail.user.name",""), 

server/difido-server/src/main/java/il/co/topq/report/front/StaticResourceConfiguration.java

@@ -3,11 +3,15 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.context.annotation.Configuration;
+import org.springframework.web.servlet.config.annotation.ResourceChainRegistration;
+import org.springframework.web.servlet.config.annotation.ResourceHandlerRegistration;
 import org.springframework.web.servlet.config.annotation.ResourceHandlerRegistry;
 import org.springframework.web.servlet.config.annotation.ViewControllerRegistry;
 import org.springframework.web.servlet.config.annotation.WebMvcConfigurerAdapter;
+import org.springframework.web.servlet.resource.PathResourceResolver;
 
 import il.co.topq.report.Configuration.ConfigProps;
+import il.co.topq.report.front.resolvers.GzipArchivedResourceResolver;
 
 @Configuration
 public class StaticResourceConfiguration extends WebMvcConfigurerAdapter {
@@ -23,7 +27,20 @@ public void addResourceHandlers(ResourceHandlerRegistry registry) {
 			docRoot += "/";
 		}
 		log.debug("docRoot folder is set to " + docRoot);
-		registry.addResourceHandler("/**").addResourceLocations("file:" + docRoot);
+
+		ResourceHandlerRegistration resourceHandlerRegistration = registry.addResourceHandler("/**")
+		.addResourceLocations("file:" + docRoot);
+
+		boolean enableArchivedResources = il.co.topq.report.Configuration.INSTANCE.readBoolean(ConfigProps.ENABLE_ARCHIVED_RESOURCES);
+		log.debug("enableArchivedResources={}",enableArchivedResources);
+		if (enableArchivedResources){
+			//many of our resources are dynamic
+			//so we probably don't want to cache them.  
+			resourceHandlerRegistration.resourceChain(false) 
+			.addResolver(new GzipArchivedResourceResolver())
+			.addResolver(new PathResourceResolver());
+		}
+
 	}
 
 	/**