Improve JPEG writing performance when there's transparency in the source image

.gitignore

@@ -12,3 +12,5 @@ build/
 *~
 *.gpg.enc
 .gradle/
+
+.DS_Store

scrimage-benchmarks/build.gradle.kts

@@ -0,0 +1,33 @@
+plugins {
+   `java-library`
+   kotlin("jvm")
+   id("me.champeau.jmh") version "0.7.2"
+}
+
+dependencies {
+   jmh(project(":scrimage-core"))
+   jmh(project(":scrimage-format-png"))
+   jmh("org.imgscalr:imgscalr-lib:4.2")
+   jmh("net.coobird:thumbnailator:0.4.18")
+}
+
+java {
+   sourceCompatibility = JavaVersion.VERSION_1_8
+   targetCompatibility = JavaVersion.VERSION_1_8
+}
+
+jmh {
+   warmupIterations.set(1)
+   iterations.set(3)
+   fork.set(1)
+   batchSize.set(5)
+   operationsPerInvocation.set(5)
+   warmup.set("1s")
+   warmupBatchSize.set(5)
+   zip64.set(true)
+   duplicateClassesStrategy.set(DuplicatesStrategy.EXCLUDE)
+   profilers.set(listOf("gc"))
+   if (project.hasProperty("includes")) {
+      includes.set(listOf(project.property("includes") as String))
+   }
+}

scrimage-benchmarks/readme.md

@@ -0,0 +1,15 @@
+# General
+
+This a benchmark suite for some image operations
+
+
+## Usage
+
+```bash
+./gradlew scrimage-benchmarks:jmh
+```
+Runs all benchmarks and
+```bash
+./gradlew scrimage-benchmarks:jmh -Pincludes=<REGEX>
+```
+specific benchmarks.

scrimage-benchmarks/results_fyi.md

@@ -0,0 +1,53 @@
+# Benchmarking notebook
+
+This journey was inspired by my company noticing that writing some images with scrimage allocated a lot of memory.
+
+I wondered why that is.
+
+### Setup
+Running
+./gradlew scrimage-benchmarks:jmh -Pincludes=testJpegWritingOfTransparentImage
+With:
+OpenJDK 64-Bit Server VM Corretto-17.0.6.10.1 (build 17.0.6+10-LTS, mixed mode, sharing)
+
+## Results
+
+### Starting situation (At 04faed43439004d1105b1b6783b6edf32df5af1d)
+```
+Benchmark                                                                           Mode  Cnt         Score        Error   Units
+JpegWriterBenchmarks.Writing.testJpegWritingOfTransparentImage                      avgt    3        47.722 ±     16.861   ms/op
+JpegWriterBenchmarks.Writing.testJpegWritingOfTransparentImage:·gc.alloc.rate       avgt    3      1659.080 ±    594.940  MB/sec
+JpegWriterBenchmarks.Writing.testJpegWritingOfTransparentImage:·gc.alloc.rate.norm  avgt    3  83000739.473 ± 124693.723    B/op
+JpegWriterBenchmarks.Writing.testJpegWritingOfTransparentImage:·gc.count            avgt    3        56.000               counts
+JpegWriterBenchmarks.Writing.testJpegWritingOfTransparentImage:·gc.time             avgt    3      2793.000                   ms
+```
+
+### Skip creation of Array of pixels beforehand in replaceTransparencyInPlace and use an iterator instead
+This is current ImmutableImage#removeTransparencySlow.
+```
+Benchmark                                                                           Mode  Cnt         Score       Error   Units
+JpegWriterBenchmarks.Writing.testJpegWritingOfTransparentImage                      avgt    3        35.520 ±     2.477   ms/op
+JpegWriterBenchmarks.Writing.testJpegWritingOfTransparentImage:·gc.alloc.rate       avgt    3      1770.621 ±   123.737  MB/sec
+JpegWriterBenchmarks.Writing.testJpegWritingOfTransparentImage:·gc.alloc.rate.norm  avgt    3  65948072.753 ± 21026.857    B/op
+JpegWriterBenchmarks.Writing.testJpegWritingOfTransparentImage:·gc.count            avgt    3       161.000              counts
+JpegWriterBenchmarks.Writing.testJpegWritingOfTransparentImage:·gc.time             avgt    3       102.000                  ms
+```
+
+### Use Graphics2D for writing, skipping calls to ColorModel#getRGB for each pixel
+This is current ImmutableImage#removeTransparencyFast.
+```
+Benchmark                                                                           Mode  Cnt         Score        Error   Units
+JpegWriterBenchmarks.Writing.testJpegWritingOfTransparentImage                      avgt    3        12.342 ±      1.227   ms/op
+JpegWriterBenchmarks.Writing.testJpegWritingOfTransparentImage:·gc.alloc.rate       avgt    3      1052.704 ±     96.164  MB/sec
+JpegWriterBenchmarks.Writing.testJpegWritingOfTransparentImage:·gc.alloc.rate.norm  avgt    3  13623604.076 ± 110662.316    B/op
+JpegWriterBenchmarks.Writing.testJpegWritingOfTransparentImage:·gc.count            avgt    3       165.000               counts
+JpegWriterBenchmarks.Writing.testJpegWritingOfTransparentImage:·gc.time             avgt    3       114.000                   ms
+```
+This does, however, alter results.
+
+#### Observations
+I think PixelTools.replaceTransparencyWithColor does the transformation from alpha-channelled into RGB-only differently.
+For example, with test picture of /transparent_chip.png, pixel (125, 17) is transformed into (202, 90, 101) with
+Graphics2D and into (201, 90, 101) with PixelTools.replaceTransparencyWithColor.
+
+Original value is (165, 173, 0, 17).

scrimage-benchmarks/src/jmh/java/com/sksamuel/scrimage/benchmarks/JpegWriterBenchmarks.java

@@ -0,0 +1,66 @@
+package com.sksamuel.scrimage.benchmarks;
+
+import com.sksamuel.scrimage.ImmutableImage;
+import com.sksamuel.scrimage.nio.ImageWriter;
+import com.sksamuel.scrimage.nio.JpegWriter;
+import org.openjdk.jmh.annotations.*;
+import org.openjdk.jmh.infra.Blackhole;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.util.concurrent.TimeUnit;
+
+public class JpegWriterBenchmarks {
+
+   private static ImmutableImage loadImageFromResource(String resourceName) {
+      try {
+         return ImmutableImage.loader().fromResource(resourceName);
+      } catch (IOException e) {
+         throw new RuntimeException(e);
+      }
+   }
+
+   @State(value = Scope.Benchmark)
+   public static class ReadImages {
+      ImmutableImage opaqueImage = loadImageFromResource("/bench_bird.jpg");
+      ImmutableImage transparentImage = loadImageFromResource("/bench_3.png");
+
+      ByteArrayOutputStream out = new ByteArrayOutputStream(1024 * 1024);
+   }
+
+   public static class Reading {
+      @Benchmark
+      @BenchmarkMode(Mode.AverageTime)
+      @OutputTimeUnit(TimeUnit.MILLISECONDS)
+      public void testScrimageJpegWriterDefault(Blackhole blackhole) throws IOException {
+         ImmutableImage image = ImmutableImage.loader().fromResource("/bench_bird.jpg");
+         blackhole.consume(image);
+      }
+   }
+
+
+   public static class Writing {
+
+      @Benchmark
+      @BenchmarkMode(Mode.AverageTime)
+      @OutputTimeUnit(TimeUnit.MILLISECONDS)
+      public void testJpegWritingOfTransparentImage(Blackhole blackhole, ReadImages state) throws IOException {
+         ImmutableImage image = state.transparentImage;
+         image.forWriter(JpegWriter.Default).write(state.out);
+         blackhole.consume(state.out);
+         state.out.reset();
+      }
+
+      @Benchmark
+      @BenchmarkMode(Mode.AverageTime)
+      @OutputTimeUnit(TimeUnit.MILLISECONDS)
+      public void testJpegWritingOfOpaqueImage(Blackhole blackhole, ReadImages state) throws IOException {
+         ImmutableImage image = state.opaqueImage;
+         image.forWriter(JpegWriter.Default).write(state.out);
+         blackhole.consume(state.out);
+         state.out.reset();
+      }
+   }
+
+
+}

scrimage-core/src/main/java/com/sksamuel/scrimage/AwtImage.java

@@ -15,17 +15,18 @@
 import java.awt.image.BufferedImage;
 import java.awt.image.DataBuffer;
 import java.awt.image.DataBufferInt;
+import java.awt.image.Raster;
 import java.io.ByteArrayInputStream;
 import java.io.File;
 import java.io.IOException;
 import java.nio.file.Path;
 import java.nio.file.Paths;
-import java.util.Arrays;
-import java.util.Iterator;
-import java.util.Set;
+import java.util.*;
 import java.util.function.Consumer;
 import java.util.function.Predicate;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import java.util.stream.StreamSupport;
 
 /**
  * Wraps an AWT BufferedImage with some basic helper functions related to sizes, pixels etc.
@@ -103,7 +104,7 @@ public int getType() {
     * Returns the colors of this image represented as an array of RGBColor.
     */
    public RGBColor[] colors() {
-      return Arrays.stream(pixels()).map(Pixel::toColor).toArray(RGBColor[]::new);
+      return asPixelStream().map(Pixel::toColor).toArray(RGBColor[]::new);
    }
 
    /**
@@ -177,6 +178,15 @@ public Pixel next() {
       };
    }
 
+   public void forEachPixel(PixelTools.PixelConsumer consumer) {
+      for (int k = 0; k < AwtImage.this.count(); k++) {
+         Point point = PixelTools.offsetToPoint(k, width);
+         int rgb = awt.getRGB(point.x, point.y);
+         consumer.consume(point.x, point.y, rgb);
+      }
+   }
+
+
    // This tuple contains all the state that identifies this particular image.
    private ImageState imageState() {
       return new ImageState(width, height, pixels());
@@ -240,7 +250,7 @@ public int count() {
     * @param fn a function that accepts a pixel
     */
    public void forEach(Consumer<Pixel> fn) {
-      Arrays.stream(points()).forEach(p -> fn.accept(pixel(p)));
+      iterator().forEachRemaining(fn);
    }
 
    /**
@@ -287,7 +297,17 @@ public boolean contains(Color color) {
     * @return true if p holds for at least one pixel
     */
    public boolean exists(Predicate<Pixel> p) {
-      return Arrays.stream(pixels()).anyMatch(p);
+      return asPixelStream().anyMatch(p);
+   }
+
+   private Stream<Pixel> asPixelStream() {
+      long size = count();
+      return StreamSupport.stream(
+         Spliterators.spliterator(
+            iterator(),
+            size,
+            Spliterator.SIZED
+         ), false);
    }
 
    /**
@@ -413,7 +433,7 @@ public int offset(int x, int y) {
     * @return the set of distinct Colors
     */
    public Set<RGBColor> colours() {
-      return Arrays.stream(pixels()).map(Pixel::toColor).collect(Collectors.toSet());
+      return asPixelStream().map(Pixel::toColor).collect(Collectors.toSet());
    }
 
    /**
@@ -423,7 +443,8 @@ public Set<RGBColor> colours() {
     * @return the number of pixels that matched the colour of the given pixel
     */
    public long count(Color color) {
-      return count(p -> p.toColor().equals(RGBColor.fromAwt(color)));
+      RGBColor needle = RGBColor.fromAwt(color);
+      return count(p -> p.toColor().equals(needle));
    }
 
    /**
@@ -433,7 +454,7 @@ public long count(Color color) {
     * @return the number of pixels that evaluated true
     */
    public long count(Predicate<Pixel> p) {
-      return Arrays.stream(pixels()).filter(p).count();
+      return asPixelStream().filter(p).count();
    }
 
    /**
@@ -525,7 +546,7 @@ protected BufferedImage rotateByRadians(Radians angle, Color bgcolor) {
     * Returns true if the given predicate holds for all pixels in the image.
     */
    public boolean forAll(Predicate<Pixel> predicate) {
-      return Arrays.stream(pixels()).allMatch(predicate);
+      return asPixelStream().allMatch(predicate);
    }
 
    /**
@@ -566,7 +587,7 @@ public boolean hasTransparency() {
     * Returns the average colour of all pixels in this image
     */
    public RGBColor average() {
-      return Arrays.stream(pixels()).map(Pixel::toColor).reduce(com.sksamuel.scrimage.color.Color::average).get();
+      return asPixelStream().map(Pixel::toColor).reduce(com.sksamuel.scrimage.color.Color::average).get();
    }
 
    /**

scrimage-core/src/main/java/com/sksamuel/scrimage/ImmutableImage.java

@@ -42,9 +42,7 @@
 import javax.imageio.ImageIO;
 import java.awt.*;
 import java.awt.geom.AffineTransform;
-import java.awt.image.AffineTransformOp;
-import java.awt.image.BufferedImage;
-import java.awt.image.BufferedImageOp;
+import java.awt.image.*;
 import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
@@ -1098,9 +1096,25 @@ public ImmutableImage padRight(int k, Color color) {
     * Returns a new ImmutableImage with transparency replaced with the given color.
     */
    public ImmutableImage removeTransparency(Color color) {
-      ImmutableImage target = copy();
-      target.replaceTransparencyInPlace(color);
-      return target;
+      if (awt().getColorModel().hasAlpha()) {
+         BufferedImage newImage = new BufferedImage(awt().getWidth(), awt().getHeight(), BufferedImage.TYPE_INT_RGB);
+         Graphics2D g = null;
+         try {
+            g = newImage.createGraphics();
+            g.setColor(color);
+            g.fillRect(0, 0, newImage.getWidth(), newImage.getHeight());
+            g.drawImage(awt(), 0, 0, null);
+         } finally {
+            if (g != null) {
+               g.dispose();
+            }
+         }
+
+         return ImmutableImage.wrapAwt(newImage);
+      } else {
+         return this;
+      }
+
    }
 
    /**

scrimage-core/src/main/java/com/sksamuel/scrimage/MutableImage.java

@@ -5,8 +5,7 @@
 import com.sksamuel.scrimage.pixels.PixelTools;
 
 import java.awt.*;
-import java.awt.image.BufferedImage;
-import java.awt.image.RescaleOp;
+import java.awt.image.*;
 import java.util.Arrays;
 import java.util.function.Function;
 
@@ -38,10 +37,15 @@ public void mapInPlace(Function<Pixel, Color> mapper) {
       });
    }
 
+   /**
+    * This is very slow and you probably should not use it.
+    * @param color background color
+    */
+   @Deprecated
    public void replaceTransparencyInPlace(java.awt.Color color) {
-      Arrays.stream(pixels()).forEach(pixel -> {
-         Pixel withoutTrans = PixelTools.replaceTransparencyWithColor(pixel, color);
-         awt().setRGB(pixel.x, pixel.y, withoutTrans.toARGBInt());
+      forEachPixel((x, y, currentArgb) -> {
+         Pixel withoutTrans = PixelTools.replaceTransparencyWithColor(new Pixel(x, y, currentArgb), color);
+         awt().setRGB(withoutTrans.x, withoutTrans.y, withoutTrans.toARGBInt());
       });
    }