MRF: JPEG-XL (brunsli) support (OSGeo#3945)

* Decode brotli tiles

* Enable JPEG-XL (brunsli) on *nix

tested read

* Write JPEG XL

By default, JPEGs are written as JPEG-XL, except if noJXL option is set

* Use fullsize

The compiler complains if a variable is not used

* Rename option to generate JFIF JPEGs

* Example of turning on the brunsli/brotli libraries

* remove function pointer cast

* Describe compression dependent parameters in mrf info

* autoconfigure brunsli

Assume brunsli include files are preinstalled and that libraries are in the library path

gdal/GDALmake.opt.in

@@ -33,7 +33,7 @@ INSTALL_DIR	= 	$(GDAL_ROOT)/install-sh -d
 
 LIBS	=	@LIBS@ $(KAK_LIBS) $(DWG_LIBS) $(CURL_LIB) \
 		$(MRSID_LIBS) $(MRSID_LIDAR_LIBS) $(ECW_LIBS) $(INGRES_LIB) \
-		$(PCIDSK_LIB) $(RASDAMAN_LIB) $(SOSI_LIB) \
+		$(PCIDSK_LIB) $(RASDAMAN_LIB) $(SOSI_LIB) $(BRUNSLI_LIB) \
 		$(OPENCL_LIB) $(JVM_LIB) $(LIBICONV) $(FGDB_LIB) $(LIBXML2_LIB) $(MONGODB_LIB) \
 		$(MONGOCXXV3_LIBS) $(JNI_LIB) $(HDFS_LIB)
 
@@ -459,6 +459,14 @@ RASDAMAN_ENABLED  =       @RASDAMAN_ENABLED@
 RASDAMAN_INC  =   @RASDAMAN_INC@
 RASDAMAN_LIB  =   @RASDAMAN_LIB@
 
+
+#
+# BRUNSLI (JPEG XL)
+#
+BRUNSLI_ENABLED = @BRUNSLI_ENABLED@
+BRUNSLI_INCLUDE = @BRUNSLI_INCLUDE@
+BRUNSLI_LIB	= @BRUNSLI_LIB@
+
 #
 # PDF stuff
 #

gdal/configure

@@ -642,6 +642,9 @@ USE_ONLY_CRYPTODLL_ALG
 HAVE_CRYPTOPP
 HAVE_ARMADILLO
 RDB_SETTING
+BRUNSLI_LIB
+BRUNSLI_INCLUDE
+BRUNSLI_ENABLED
 RASDAMAN_LIB
 RASDAMAN_INC
 RASDAMAN_ENABLED
@@ -1216,6 +1219,7 @@ with_mdb
 with_jvm_lib
 with_jvm_lib_add_rpath
 with_rasdaman
+with_brunsli
 with_rdb
 with_armadillo
 with_cryptopp
@@ -2227,6 +2231,7 @@ Optional Packages:
   --with-jvm-lib=ARG        ARG is dlopen or points to Java libjvm path
   --with-jvm-lib-add-rpath    Add the libjvm path to the RPATH (no by default)
   --with-rasdaman=DIR        Include rasdaman support (DIR is rasdaman's install dir).
+  --with-brunsli    Include brunsli support via libbrunsli library
   --with-rdb=ARG    Include RDB support (ARG=no, yes or RDB SDK root path)
   --with-armadillo=ARG       Include Armadillo support for faster TPS transform computation (ARG=yes/no/path to armadillo install root) [default=no]
   --with-cryptopp=ARG       Include cryptopp support (ARG=yes, no or path)
@@ -41354,6 +41359,27 @@ RASDAMAN_LIB=$RASDAMAN_LIB
 
 
 
+
+# Check whether --with-brunsli was given.
+if test "${with_brunsli+set}" = set; then :
+  withval=$with_brunsli;
+fi
+
+
+BRUNSLI_ENABLED=no
+
+if test "$with_brunsli" != "" -a "$with_brunsli" != "no" ; then
+  BRUNSLI_ENABLED="yes"
+  BRUNSLI_ENABLED=$BRUNSLI_ENABLED
+
+  BRUNSLI_INCLUDE=-I$prefix/include
+
+  BRUNSLI_LIB="-lbrunslicommon -lbrunslienc -lbrunslidec -lbrotlicommon -lbrotlienc -lbrotlidec"
+
+fi
+
+
+
 # Check whether --with-rdb was given.
 if test "${with_rdb+set}" = set; then :
   withval=$with_rdb;
@@ -45054,6 +45080,9 @@ echo "  Kea support:               ${HAVE_KEA}"
 echo "  LERC support:              ${HAVE_LERC}"
 
 
+echo "  libbrunsli support:        ${BRUNSLI_ENABLED}"
+
+
 echo "  libdeflate support:        ${LIBDEFLATE_SETTING}"
 
 

gdal/configure.ac

@@ -5594,6 +5594,22 @@ AC_SUBST(RASDAMAN_ENABLED,$RASDAMAN_ENABLED)
 AC_SUBST(RASDAMAN_INC,    $RASDAMAN_INC)
 AC_SUBST(RASDAMAN_LIB,    $RASDAMAN_LIB)
 
+
+dnl ---------------------------------------------------------------------------
+dnl brunsli support
+dnl ---------------------------------------------------------------------------
+
+AC_ARG_WITH(brunsli,[  --with-brunsli    Include brunsli support via libbrunsli library],,)
+
+BRUNSLI_ENABLED=no
+
+if test "$with_brunsli" != "" -a "$with_brunsli" != "no" ; then
+  BRUNSLI_ENABLED="yes"
+  AC_SUBST(BRUNSLI_ENABLED, $BRUNSLI_ENABLED)
+  AC_SUBST(BRUNSLI_INCLUDE, -I$prefix/include)
+  AC_SUBST(BRUNSLI_LIB, "-lbrunslicommon -lbrunslienc -lbrunslidec -lbrotlicommon -lbrotlienc -lbrotlidec")
+fi 
+
 dnl ---------------------------------------------------------------------------
 dnl RDB support
 dnl ---------------------------------------------------------------------------
@@ -6093,6 +6109,7 @@ LOC_MSG([  JPEG-Lossless/CharLS:      ${HAVE_CHARLS}])
 LOC_MSG([  Kakadu support:            ${HAVE_KAKADU}])
 LOC_MSG([  Kea support:               ${HAVE_KEA}])
 LOC_MSG([  LERC support:              ${HAVE_LERC}])
+LOC_MSG([  libbrunsli support:        ${BRUNSLI_ENABLED}])
 LOC_MSG([  libdeflate support:        ${LIBDEFLATE_SETTING}])
 LOC_MSG([  LIBGEOTIFF support:        ${GEOTIFF_SETTING}])
 LOC_MSG([  LIBGIF support:            ${GIF_SETTING}])

gdal/frmts/mrf/GNUmakefile

@@ -48,6 +48,10 @@ XTRA_OPT	:=	$(XTRA_OPT) -DLERC -I../../third_party/LercLib
 SUBLIBS := lib-LERCV1
 endif
 
+ifeq ($(BRUNSLI_ENABLED),yes)
+XTRA_OPT		:=	$(XTRA_OPT) -DBRUNSLI $(BRUNSLI_INCLUDE)
+endif
+
 CPPFLAGS        :=  $(XTRA_OPT)  $(CPPFLAGS)
 
 default:  $(PREDEPEND)	$(OBJ:.o=.$(OBJ_EXT)) $(SUBLIBS)

gdal/frmts/mrf/JPEG_band.cpp

@@ -57,6 +57,7 @@
 
 #include "marfa.h"
 #include <setjmp.h>
+#include <vector>
 
 CPL_C_START
 #include <jpeglib.h>
@@ -67,6 +68,13 @@ CPL_C_END
 #include "BitMask2D.h"
 #include "Packer_RLE.h"
 
+#if defined(BRUNSLI)
+#include <brunsli/brunsli_decode.h>
+#include <brunsli/jpeg_data_writer.h>
+#include <brunsli/brunsli_encode.h>
+#include <brunsli/jpeg_data_reader.h>
+#endif
+
 NAMESPACE_MRF_START
 
 typedef BitMap2D<> BitMask;
@@ -342,8 +350,28 @@ CPLErr JPEG_Codec::CompressJPEG(buf_mgr &dst, buf_mgr &src)
     CPLFree(rowp);
     CPLFree(buffer);     // Safe to call on null
 
-    // Figure out the size
-    dst.size -= jmgr.free_in_buffer;
+    size_t fullsize = dst.size;
+    // Figure out the size of the JFIF
+    dst.size = fullsize - jmgr.free_in_buffer;
+
+#if defined(BRUNSLI)
+    if (!JFIF) {
+        brunsli::JPEGData bjpg;
+        if (!brunsli::ReadJpeg(reinterpret_cast<GByte *>(dst.buffer), dst.size, brunsli::JPEG_READ_ALL, &bjpg)) {
+            CPLError(CE_Failure, CPLE_AppDefined, "MRF: Error parsing JPEG before conversion to JPEG-XL");
+            return CE_Failure;
+        }
+        std::vector<GByte> out;
+        out.resize(fullsize);
+        if (!brunsli::BrunsliEncodeJpeg(bjpg, out.data(), &fullsize)) {
+            CPLError(CE_Failure, CPLE_AppDefined, "MRF: JPEG-XL encoding error");
+            return CE_Failure;
+        }
+        // fullsize holds the JPEG XL output size
+        memcpy(dst.buffer, out.data(), fullsize);
+        dst.size = fullsize;
+    }
+#endif
 
     return CE_None;
 }
@@ -631,9 +659,67 @@ CPLErr JPEG_Codec::DecompressJPEG(buf_mgr& dst, buf_mgr& isrc)
 char CHUNK_NAME[] = "Zen";
 size_t CHUNK_NAME_SIZE = strlen(CHUNK_NAME) + 1;
 
+#if defined(BRUNSLI)
+// Append to end of out vector
+static size_t out_callback(void* out, const GByte* data, size_t size) {
+    auto outv = static_cast<std::vector<GByte> *>(out);
+    outv->insert(outv->end(), data, data + size);
+    return size;
+}
+#endif // BRUNSLI
+
+const static GUInt32 JPEG_SIG = 0xe0ffd8ff; // JPEG 4CC code
+const static GUInt32 BRUN_SIG = 0xd242040a; // Brunsli 4CC code, native
+
+static bool isbrunsli(const buf_mgr& src) {
+    GUInt32 signature;
+    memcpy(&signature, src.buffer, sizeof(signature));
+    if (BRUN_SIG == CPL_LSBWORD32(signature))
+        return true;
+    return false;
+}
+
+bool JPEG_Codec::IsJPEG(const buf_mgr& src)
+{
+    if (isbrunsli(src))
+        return true;
+    GUInt32 signature;
+    memcpy(&signature, src.buffer, sizeof(signature));
+    if (JPEG_SIG == CPL_LSBWORD32(signature))
+        return true;
+    return false;
+}
+
 
 // Type dependent dispachers
 CPLErr JPEG_Band::Decompress(buf_mgr &dst, buf_mgr &src) {
+    if (isbrunsli(src)) { // Need conversion to JPEG first
+#if !defined(BRUNSLI)
+        CPLError(CE_Failure, CPLE_NotSupported, "MRF: JPEG-XL content, yet this GDAL was not compiled with BRUNSLI support");
+        return CE_Failure;
+#else
+        std::vector<GByte> out;
+        brunsli::JPEGData bjpg;
+        // This call reads the all the input data internally and stores it in the bjpg structure
+        auto status = brunsli::BrunsliDecodeJpeg(reinterpret_cast<uint8_t*>(src.buffer), src.size, &bjpg);
+        if (status != brunsli::BRUNSLI_OK) {
+            CPLError(CE_Failure, CPLE_AppDefined, "MRF: JPEG-XL (brunsli) tile decode failed");
+            return CE_Failure;
+        }
+
+        brunsli::JPEGOutput writer(out_callback, &out);
+
+        if (!brunsli::WriteJpeg(bjpg, writer)) {
+            CPLError(CE_Failure, CPLE_AppDefined, "MRF: JPEG-XL (brunsli) to JPEG conversion failed");
+            return CE_Failure;
+        }
+
+        buf_mgr jfif_src;
+        jfif_src.buffer = reinterpret_cast<char *>(out.data());
+        jfif_src.size = out.size();
+        return Decompress(dst, jfif_src); // Call itself with JFIF JPEG source
+#endif // BRUNSLI
+    }
 #if defined(LIBJPEG_12_H)
     if (GDT_Byte != img.dt)
         return codec.DecompressJPEG12(dst, src);
@@ -678,10 +764,13 @@ JPEG_Band::JPEG_Band( MRFDataset *pDS, const ILImage &image,
             codec.sameres = TRUE;
     }
 
-    if( GDT_Byte == image.dt )
+    if( GDT_Byte == image.dt ) {
         codec.optimize = GetOptlist().FetchBoolean("OPTIMIZE", FALSE) != FALSE;
-    else
+        codec.JFIF = GetOptlist().FetchBoolean("JFIF", FALSE) != FALSE;
+    }
+    else {
         codec.optimize = true; // Required for 12bit
+    }
 }
 #endif
 

gdal/frmts/mrf/JPNG_band.cpp

@@ -86,16 +86,15 @@ static CPLErr initBuffer(buf_mgr &b) {
 }
 
 CPLErr JPNG_Band::Decompress(buf_mgr &dst, buf_mgr &src) {
-    const static GUInt32 JPEG_SIG = 0xe0ffd8ff; // JPEG 4CC code
     const static GUInt32 PNG_SIG  = 0x474e5089;  // PNG 4CC code
 
     CPLErr retval = CE_None;
     ILImage image(img);
     GUInt32 signature;
-    memcpy(&signature, src.buffer, sizeof(GUInt32));
+    memcpy(&signature, src.buffer, sizeof(signature));
 
     // test against an LSB signature
-    if (JPEG_SIG == CPL_LSBWORD32(signature)) {
+    if (JPEG_Codec::IsJPEG(src)) {
         image.pagesize.c -= 1;
         JPEG_Codec codec(image);
 
@@ -142,6 +141,7 @@ CPLErr JPNG_Band::Compress(buf_mgr &dst, buf_mgr &src) {
         codec.rgb = rgb;
         codec.optimize = optimize;
         codec.sameres = sameres;
+        codec.JFIF = JFIF;
         retval = codec.CompressJPEG(dst, temp);
     }
     else if (!AllAlpha<0>(src, image)) {
@@ -165,7 +165,8 @@ JPNG_Band::JPNG_Band( MRFDataset *pDS, const ILImage &image,
     MRFRasterBand(pDS, image, b, level),
     rgb(FALSE),
     sameres(FALSE),
-    optimize(false)
+    optimize(false),
+    JFIF(false)
 {   // Check error conditions
     if (image.dt != GDT_Byte) {
         CPLError(CE_Failure, CPLE_NotSupported, "Data type not supported by MRF JPNG");
@@ -187,6 +188,7 @@ JPNG_Band::JPNG_Band( MRFDataset *pDS, const ILImage &image,
     }
 
     optimize = GetOptlist().FetchBoolean("OPTIMIZE", FALSE) != FALSE;
+    JFIF = GetOptlist().FetchBoolean("JFIF", FALSE) != FALSE;
 
     // PNGs and JPGs can be larger than the source, especially for
     // small page size.

gdal/frmts/mrf/makefile.vc

@@ -25,6 +25,14 @@ EXTRAFLAGS	= -DLERC -ILERCV1 -I../../third_party/LercLib $(EXTRAFLAGS)
 OBJ = $(LERC1) LERC_Band.obj $(OBJ)
 !ENDIF
 
+# Brotli is needed for brunsli
+!IFDEF BRUNSLI_DIR
+!IFDEF BROTLI_DIR
+EXTRAFLAGS = $(EXTRAFLAGS) -DBRUNSLI -I$(BRUNSLI_INC)
+EXTRA_LINK_FLAGS = $(EXTRA_LINK_FLAGS) $(BRUNSLI_LIB)
+!ENDIF
+!ENDIF
+
 !IFDEF JPEG_EXTERNAL_LIB
 EXTRAFLAGS =	-I$(JPEGDIR) $(EXTRAFLAGS)
 !ELSE

gdal/frmts/mrf/marfa.h

@@ -643,11 +643,14 @@ class PNG_Band final: public MRFRasterBand {
 
 class JPEG_Codec {
 public:
-    explicit JPEG_Codec(const ILImage &image) : img(image), sameres(FALSE), rgb(FALSE), optimize(false) {}
+    explicit JPEG_Codec(const ILImage &image) : img(image), sameres(FALSE), rgb(FALSE), optimize(false), JFIF(false) {}
 
     CPLErr CompressJPEG(buf_mgr &dst, buf_mgr &src);
     CPLErr DecompressJPEG(buf_mgr &dst, buf_mgr &src);
 
+    // Returns true for both JPEG and JPEG-XL (brunsli)
+    static bool IsJPEG(const buf_mgr& src);
+
 #if defined(JPEG12_SUPPORTED) // Internal only
 #define LIBJPEG_12_H "../jpeg/libjpeg12/jpeglib.h"
     CPLErr CompressJPEG12(buf_mgr &dst, buf_mgr &src);
@@ -657,12 +660,14 @@ class JPEG_Codec {
     const ILImage img;
 
     // JPEG specific flags
-    bool sameres;
-    bool rgb;
-    bool optimize;
+    bool sameres;  // No color space subsample
+    bool rgb;      // No conversion to YCbCr
+    bool optimize; // Optimize Huffman tables
+    bool JFIF;     // Write JFIF only
 
 private:
-    JPEG_Codec& operator= (const JPEG_Codec& src); // not implemented. but suppress MSVC warning about 'assignment operator could not be generated'
+    // not implemented. but suppress MSVC warning about 'assignment operator could not be generated'
+    JPEG_Codec& operator= (const JPEG_Codec& src); 
 };
 
 class JPEG_Band final: public MRFRasterBand {
@@ -690,7 +695,7 @@ class JPNG_Band final: public MRFRasterBand {
 
     CPLErr CompressJPNG(buf_mgr &dst, buf_mgr &src);
     CPLErr DecompressJPNG(buf_mgr &dst, buf_mgr &src);
-    bool rgb, sameres, optimize;
+    bool rgb, sameres, optimize, JFIF;
 };
 
 class Raw_Band final: public MRFRasterBand {