diff --git a/SeleniumGridExtras/pom.xml b/SeleniumGridExtras/pom.xml index 60058dce..03fd3835 100644 --- a/SeleniumGridExtras/pom.xml +++ b/SeleniumGridExtras/pom.xml @@ -91,20 +91,15 @@ dom4j 1.6.1 - - xuggle - xuggle-xuggler - 5.4 + io.humble + humble-video-noarch + 0.2.1 - org.boofcv - xuggler - 0.16 + io.humble + humble-video-arch-x86_64-pc-linux-gnu6 + 0.2.1 guava @@ -112,11 +107,7 @@ jar 23.0 - + org.mockito mockito-core @@ -154,6 +145,11 @@ . + + humble-video + . + . + x86_64-pc-linux-gnu6 diff --git a/SeleniumGridExtras/src/main/java/com/groupon/seleniumgridextras/utilities/threads/video/VideoRecorderCallable.java b/SeleniumGridExtras/src/main/java/com/groupon/seleniumgridextras/utilities/threads/video/VideoRecorderCallable.java index 5128692e..b778952b 100644 --- a/SeleniumGridExtras/src/main/java/com/groupon/seleniumgridextras/utilities/threads/video/VideoRecorderCallable.java +++ b/SeleniumGridExtras/src/main/java/com/groupon/seleniumgridextras/utilities/threads/video/VideoRecorderCallable.java @@ -5,11 +5,10 @@ import com.groupon.seleniumgridextras.utilities.ScreenshotUtility; import com.groupon.seleniumgridextras.utilities.TimeStampUtility; import com.groupon.seleniumgridextras.videorecording.ImageProcessor; -import com.xuggle.mediatool.IMediaWriter; -import com.xuggle.mediatool.ToolFactory; -import com.xuggle.xuggler.ICodec; -import com.xuggle.xuggler.IRational; +import io.humble.ferry.*; +import io.humble.video.*; +import io.humble.video.awt.*; import org.apache.commons.io.comparator.LastModifiedFileComparator; import org.apache.log4j.Logger; @@ -19,7 +18,6 @@ import java.util.Arrays; import java.util.Date; import java.util.concurrent.Callable; -import java.util.concurrent.TimeUnit; public class VideoRecorderCallable implements Callable { @@ -38,10 +36,10 @@ public class VideoRecorderCallable implements Callable { final private static - IRational - FRAME_RATE = - IRational.make(RuntimeConfig.getConfig().getVideoRecording().getFrames(), - RuntimeConfig.getConfig().getVideoRecording().getSecondsPerFrame()); + Rational + FRAME_RATE = + Rational.make(RuntimeConfig.getConfig().getVideoRecording().getSecondsPerFrame(), + RuntimeConfig.getConfig().getVideoRecording().getFrames()); private static Dimension dimension; @@ -61,6 +59,10 @@ public VideoRecorderCallable(String sessionID, int timeout) { @Override public String call() throws Exception { + if (Boolean.getBoolean("memory.debug")) { + JNIMemoryManager.getMgr().setMemoryDebugging(true); + } + //Probably overkill to null these out, but i'm playing it safe until proven otherwise this.nodeName = "Node: " + RuntimeConfig.getOS().getHostName() + " (" + RuntimeConfig.getHostIp() @@ -78,55 +80,190 @@ public String call() throws Exception { // Note we're writing to a temp file. This is to prevent it from being // downloaded while we're mid-write. final File tempFile = new File(outputDir, sessionId + ".temp.mp4"); - final - IMediaWriter - writer = - ToolFactory.makeWriter(tempFile.getAbsolutePath()); + + /** First we create a muxer using the passed in filename and formatname if given. */ + Muxer muxer = Muxer.make(tempFile.getAbsolutePath(), null, /*formatname*/null); + + /** Now, we need to decide what type of codec to use to encode video. Muxers + * have limited sets of codecs they can use. We're going to pick the first one that + * works, or if the user supplied a codec name, we're going to force-fit that + * in instead. + */ + MuxerFormat format = muxer.getFormat(); + + final Codec codec = Codec.findEncodingCodec(Codec.ID.CODEC_ID_H264); // We tell it we're going to add one video stream, with id 0, // at position 0, and that it will have a fixed frame rate of // FRAME_RATE. - writer.addVideoStream(0, 0, ICodec.ID.CODEC_ID_H264, - FRAME_RATE, - screenBounds.width, screenBounds.height); + + /** + * Now that we know what codec, we need to create an encoder + */ + Encoder encoder = Encoder.make(codec); + + /** + * Video encoders need to know at a minimum: + * width + * height + * pixel format + * Some also need to know frame-rate (older codecs that had a fixed rate at which video files could + * be written needed this). There are many other options you can set on an encoder, but we're + * going to keep it simpler here. + */ + encoder.setWidth(screenBounds.width); + encoder.setHeight(screenBounds.height); + // We are going to use 420P as the format because that's what most video formats these days use + final PixelFormat.Type pixelformat = PixelFormat.Type.PIX_FMT_YUV420P; + encoder.setPixelFormat(pixelformat); + encoder.setTimeBase(FRAME_RATE); + + /** An annoynace of some formats is that they need global (rather than per-stream) headers, + * and in that case you have to tell the encoder. And since Encoders are decoupled from + * Muxers, there is no easy way to know this beyond + */ + if (format.getFlag(MuxerFormat.Flag.GLOBAL_HEADER)) + encoder.setFlag(Encoder.Flag.FLAG_GLOBAL_HEADER, true); + + /** Open the encoder. */ + encoder.open(null, null); + + /** Add this stream to the muxer. */ + muxer.addNewStream(encoder); + + /** And open the muxer for business. */ + muxer.open(null, null); + + int n = muxer.getNumStreams(); + MuxerStream[] muxerStreams = new MuxerStream[n]; + Coder[] coder = new Coder[n]; + for (int i = 0; i < n; i++) { + muxerStreams[i] = muxer.getStream(i); + if (muxerStreams[i] != null) { + coder[i] = muxerStreams[i].getCoder(); + } + } + + /** Next, we need to make sure we have the right MediaPicture format objects + * to encode data with. Java (and most on-screen graphics programs) use some + * variant of Red-Green-Blue image encoding (a.k.a. RGB or BGR). Most video + * codecs use some variant of YCrCb formatting. So we're going to have to + * convert. To do that, we'll introduce a MediaPictureConverter object later. object. + */ + MediaPictureConverter converter = null; + final MediaPicture picture = MediaPicture.make( + encoder.getWidth(), + encoder.getHeight(), + pixelformat); + picture.setTimeBase(FRAME_RATE); logger - .info("Starting video recording for session " + getSessionId() + " to " + outputDir - .getAbsolutePath()); + .info("Starting video recording for session " + getSessionId() + " to " + outputDir + .getAbsolutePath()); + MediaPacket packet = MediaPacket.make(); try { - int imageFrame = 1; - long startTime = System.nanoTime(); - addTitleFrame(writer); + int imageFrame = 0; + + { + BufferedImage titleFrame + = ImageProcessor.createTitleFrame( + dimension, + BufferedImage.TYPE_3BYTE_BGR, + "Session :" + this.sessionId, + "Host :" + RuntimeConfig.getOS().getHostName() + " (" + + RuntimeConfig.getHostIp() + ")", + getTimestamp().toString()); + if (converter == null) + converter = MediaPictureConverterFactory + .createConverter(titleFrame, picture); + converter.toPicture(picture, titleFrame, imageFrame++); + + do + { + encoder.encode(packet, picture); + if (packet.isComplete()) + muxer.write(packet, false); + } + while (packet.isComplete()); + try + { + Thread.sleep(2); + } + catch (InterruptedException e) + { + e.printStackTrace(); + } + } while (stopActionNotCalled() && idleTimeoutNotReached()) { // take the screen shot BufferedImage - screenshot = - ScreenshotUtility.getResizedScreenshot(dimension.width, dimension.height); + screenshot = + ScreenshotUtility.getResizedScreenshot(dimension.width, dimension.height); screenshot = ImageProcessor.addTextCaption(screenshot, - "Session: " + this.sessionId, - "Host: " + this.nodeName, - "Timestamp: " + getTimestamp().toString(), - this.lastAction + "Session: " + this.sessionId, + "Host: " + this.nodeName, + "Timestamp: " + getTimestamp().toString(), + this.lastAction ); // convert to the right image type BufferedImage bgrScreen = convertToType(screenshot, - BufferedImage.TYPE_3BYTE_BGR); + BufferedImage.TYPE_3BYTE_BGR); // encode the image - writer.encodeVideo(0, bgrScreen, - System.nanoTime() - startTime, TimeUnit.NANOSECONDS); + /** This is LIKELY not in YUV420P format, so we're going to convert it using some handy utilities. */ + if (converter == null) + converter = MediaPictureConverterFactory.createConverter(bgrScreen, picture); + converter.toPicture(picture, bgrScreen, imageFrame++); + + do { + encoder.encode(packet, picture); + if (packet.isComplete()) + muxer.write(packet, false); + } while (packet.isComplete()); // sleep for framerate milliseconds - Thread.sleep((long) (1000 / FRAME_RATE.getDouble())); + Thread.sleep((long) (1000 * FRAME_RATE.getDouble())); } } finally { - writer.close(); + /** Encoders, like decoders, sometimes cache pictures so it can do the right key-frame optimizations. + * So, they need to be flushed as well. As with the decoders, the convention is to pass in a null + * input until the output is not complete. + */ + do { + encoder.encode(packet, null); + if (packet.isComplete()) + muxer.write(packet, false); + } while (packet.isComplete()); + + /** Finally, let's clean up after ourselves. */ + muxer.close(); + + muxer.delete(); + converter.delete(); + packet.delete(); + format.delete(); + + muxer = null; + converter = null; + packet = null; + format = null; + + for (int i=0; i < muxerStreams.length; i++) { + if (muxerStreams[i] != null) { + muxerStreams[i].delete(); + muxerStreams[i] = null; + } + if (coder[i] != null) { + coder[i].delete(); + coder[i] = null; + } + } // Now, rename our temporary file to the final filename, so that the downloaders can detect it final File finalFile = new File(outputDir, sessionId + ".mp4"); @@ -143,26 +280,12 @@ public String call() throws Exception { } } - return getSessionId(); - } - - protected void addTitleFrame(IMediaWriter writer) { - writer.encodeVideo(0, - ImageProcessor - .createTitleFrame(dimension, BufferedImage.TYPE_3BYTE_BGR, - "Session :" + this.sessionId, - "Host :" + RuntimeConfig.getOS().getHostName() + " (" - + RuntimeConfig.getHostIp() + ")", - getTimestamp().toString()), - 0, - TimeUnit.NANOSECONDS); - try { - Thread.sleep(2); - } catch (InterruptedException e) { - e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates. + if (Boolean.getBoolean("memory.debug")) { + logger.info("number of alive objects:" + JNIMemoryManager.getMgr().getNumPinnedObjects()); } - } + return getSessionId(); + } public void lastAction(String action) { this.lastActionTimestamp = getTimestamp(); @@ -180,7 +303,7 @@ public void stop() { protected void setOutputDirExists(String sessionId) { if (!outputDir.exists()) { System.out.println( - "Root Video output dir does not exist, creating it here " + outputDir.getAbsolutePath()); + "Root Video output dir does not exist, creating it here " + outputDir.getAbsolutePath()); outputDir.mkdir(); } } @@ -217,17 +340,17 @@ public static boolean isResolutionDivisibleByTwo(Dimension d) { protected void dynamicallySetDimension() { try { BufferedImage - sample = - ScreenshotUtility - .getResizedScreenshot(RuntimeConfig.getConfig().getVideoRecording().getWidth(), - RuntimeConfig.getConfig().getVideoRecording().getHeight()); + sample = + ScreenshotUtility + .getResizedScreenshot(RuntimeConfig.getConfig().getVideoRecording().getWidth(), + RuntimeConfig.getConfig().getVideoRecording().getHeight()); dimension = new Dimension(sample.getWidth(), sample.getHeight()); } catch (AWTException e) { e.printStackTrace(); logger.equals(e); dimension = - new Dimension(RuntimeConfig.getConfig().getVideoRecording().getWidth(), - RuntimeConfig.getConfig().getVideoRecording().getHeight()); + new Dimension(RuntimeConfig.getConfig().getVideoRecording().getWidth(), + RuntimeConfig.getConfig().getVideoRecording().getHeight()); } } @@ -260,7 +383,9 @@ public static BufferedImage convertToType(BufferedImage sourceImage, else { image = new BufferedImage(sourceImage.getWidth(), sourceImage.getHeight(), targetType); - image.getGraphics().drawImage(sourceImage, 0, 0, null); + Graphics g = image.getGraphics(); + g.drawImage(sourceImage, 0, 0, null); + g.dispose(); } return image; diff --git a/SeleniumGridExtras/src/main/java/com/groupon/seleniumgridextras/videorecording/ImageProcessor.java b/SeleniumGridExtras/src/main/java/com/groupon/seleniumgridextras/videorecording/ImageProcessor.java index 9e7151bc..bfce4a64 100644 --- a/SeleniumGridExtras/src/main/java/com/groupon/seleniumgridextras/videorecording/ImageProcessor.java +++ b/SeleniumGridExtras/src/main/java/com/groupon/seleniumgridextras/videorecording/ImageProcessor.java @@ -82,6 +82,7 @@ public static BufferedImage createTitleFrame(Dimension dimension, int imageType, g.setFont(new Font("TimesRoman", Font.PLAIN, 14)); g.drawString("" + line2, firstLineX, secondLineY); g.drawString("" + line3, firstLineX, thirdLineY); + g.dispose(); return image; } diff --git a/SeleniumGridExtras/src/main/java/com/groupon/seleniumgridextras/videorecording/ImageToVideoConverter.java b/SeleniumGridExtras/src/main/java/com/groupon/seleniumgridextras/videorecording/ImageToVideoConverter.java deleted file mode 100644 index 02d07bd2..00000000 --- a/SeleniumGridExtras/src/main/java/com/groupon/seleniumgridextras/videorecording/ImageToVideoConverter.java +++ /dev/null @@ -1,136 +0,0 @@ -//Ended up not using this guy, maybe find use for it later - -package com.groupon.seleniumgridextras.videorecording; - -import com.groupon.seleniumgridextras.config.RuntimeConfig; -import com.groupon.seleniumgridextras.utilities.ImageUtils; -import com.xuggle.mediatool.IMediaWriter; -import com.xuggle.mediatool.ToolFactory; -import com.xuggle.xuggler.ICodec; - -import org.apache.commons.io.FilenameUtils; -import org.apache.log4j.Logger; - -import java.awt.*; -import java.awt.image.BufferedImage; -import java.io.File; -import java.io.IOException; -import java.util.Calendar; -import java.util.LinkedList; -import java.util.List; -import java.util.concurrent.Callable; -import java.util.concurrent.TimeUnit; - -public class ImageToVideoConverter implements Callable { - - private static Logger logger = Logger.getLogger(ImageToVideoConverter.class); - - protected File inputDirectory; - protected File outputVideo; - protected Dimension screenBounds; - protected int imageType; - protected boolean readyToConvert = true; - - protected IMediaWriter writer; - - protected List imageList; - - public ImageToVideoConverter(String inputDir, String sessionID, String host, String timestamp) { - - this.inputDirectory = new File(inputDir); - this.outputVideo = new File(sessionID + ".mp4"); - this.imageList = new LinkedList(); - - generateListOfImages(this.inputDirectory, this.imageList); - - try { - this.screenBounds = getImageDimensionAndType(this.imageList.get(0)); - } catch (IOException e) { - logger.warn("Cannot determine the input image dimensions for " + this.imageList.get(0) - .getAbsolutePath()); - logger.warn(e); - readyToConvert = false; - } - - this.writer = ToolFactory.makeWriter(this.outputVideo.getAbsolutePath()); - - logger.info("Ready to start converting images in " + inputDir + " into " + this.outputVideo - .getAbsolutePath()); - - } - - @Override - public Object call() throws Exception { - logger.info("Starting to generation test video " + this.outputVideo.getName()); - - this.writer.addVideoStream(0, 0, ICodec.ID.CODEC_ID_MPEG4, screenBounds.width, - screenBounds.height); - - try { - writer.encodeVideo(0, getTitleFrame(), 0, TimeUnit.MILLISECONDS); - - int index = 1; - - for (File image : this.imageList) { - BufferedImage currentFrame = ImageUtils.readImage(image); - - int frameIndex = 1000 * index; - writer.encodeVideo(0, currentFrame, frameIndex, TimeUnit.MILLISECONDS); - - index++; - } - } catch (Exception e) { - logger.warn( - "Something went wrong, and video may be corrupted. Check the movie " + this.outputVideo - .getAbsolutePath()); - logger.warn(e); - e.printStackTrace(); - return "error"; - } finally { - writer.close(); - logger.info("Video conversion done for " + this.outputVideo.getName()); - return "done"; - } - - } - - protected BufferedImage getTitleFrame() { - - String line1 = "Test Session: " + outputVideo.getName(); - String line2 = "Recorded on: "; - String - line3 = - "Encoded on: " + RuntimeConfig.getOS().getHostName() + "(" + RuntimeConfig.getHostIp() - + ") at " + new java.sql.Timestamp( - Calendar.getInstance().getTime().getTime()); - - BufferedImage - title = - ImageProcessor.createTitleFrame(screenBounds, imageType, line1, - line2, - line3); - - return title; - - } - - - protected Dimension getImageDimensionAndType(File exampleImage) throws IOException { - BufferedImage image = ImageUtils.readImage(exampleImage); - this.imageType = image.getType(); - logger.debug("Image type " + this.imageType); - logger.debug("Image dimensions " + image.getWidth() + "X" + image.getHeight()); - return new Dimension(image.getWidth(), image.getHeight()); - } - - protected void generateListOfImages(File sourceDir, List acceptedList) { - for (File file : sourceDir.listFiles()) { - if (file.isFile() && (FilenameUtils.getExtension(file.getName()).equals("png"))) { - logger.debug("file.getName() :" + file.getAbsolutePath()); - imageList.add(file); - } - } - } - - -} diff --git a/SeleniumGridExtras/src/test/java/com/groupon/seleniumgridextras/videorecording/ImageToVideoConverterTest.java b/SeleniumGridExtras/src/test/java/com/groupon/seleniumgridextras/videorecording/ImageToVideoConverterTest.java deleted file mode 100644 index 52ebd3d2..00000000 --- a/SeleniumGridExtras/src/test/java/com/groupon/seleniumgridextras/videorecording/ImageToVideoConverterTest.java +++ /dev/null @@ -1,61 +0,0 @@ -package com.groupon.seleniumgridextras.videorecording; - -import com.groupon.seleniumgridextras.config.RuntimeConfig; -import org.junit.After; -import org.junit.Test; - -import java.io.File; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -public class ImageToVideoConverterTest { - - final private File outputVideo = new File("1.mp4"); - final private - String - inputDirt = - ClassLoader.getSystemResource("fixtures/videoscreenshots").getFile(); - - @After - public void tearDown() throws Exception { - if (outputVideo.exists()) { - outputVideo.delete(); - } - } - - @Test - public void testVideoConverter() throws Exception { - if (!ImageProcessorTest.testIfDimasComputer()) { - return; - } - if (RuntimeConfig.getOS().hasGUI()) { - ImageToVideoConverter - converter = - new ImageToVideoConverter(inputDirt, "1", "localhost", "Today"); - - ExecutorService cachedPool = Executors.newCachedThreadPool(); - try { - - Future future = cachedPool.submit(converter); - - - String result = future.get(); - - assertEquals("done", result); - assertEquals(true, outputVideo.exists()); - assertTrue(outputVideo.length() > 20000); //Can't check video frame by frame but know it's - //roughly 200K when properly formatted - - } finally { - cachedPool.shutdown(); - } - - } - - } - -}