From 5dea6d23400ce64c05f5a045a9a185de81a5b242 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Fri, 27 Sep 2019 00:52:00 +0200 Subject: [PATCH] ISIS3: preserve label in PAM .aux.xml when copying to other formats (fixes #1854) --- autotest/gdrivers/isis.py | 149 +++++++++++++++++++++++++ gdal/apps/gdal_translate_lib.cpp | 104 ++++++++++++++++- gdal/frmts/gtiff/geotiff.cpp | 19 +++- gdal/frmts/pds/pds4dataset.cpp | 6 + gdal/gcore/gdalmultidomainmetadata.cpp | 41 ++++++- gdal/gcore/gdalpamdataset.cpp | 21 ++-- 6 files changed, 316 insertions(+), 24 deletions(-) diff --git a/autotest/gdrivers/isis.py b/autotest/gdrivers/isis.py index 19dfa431ad30..59b011df45b8 100755 --- a/autotest/gdrivers/isis.py +++ b/autotest/gdrivers/isis.py @@ -1538,3 +1538,152 @@ def test_isis3_bandbin_multiple_bands(): } ds = None gdal.Unlink('/vsimem/test.lbl') + + +############################################################################### +# Test that when converting from ISIS3 to other formats (PAM-enabled), the +# json:ISIS3 metadata domain is preserved. + +def test_isis3_preserve_label_across_format(): + + gdal.FileFromMemBuffer('/vsimem/multiband.lbl', """Object = IsisCube + Object = Core + Format = BandSequential + Group = Dimensions + Samples = 1 + Lines = 1 + Bands = 2 + End_Group + Group = Pixels + Type = UnsignedByte + ByteOrder = Lsb + Base = 0.0 + Multiplier = 1.0 + End_Group + End_Object + + Group = BandBin + BandSuffixName = ("first band", "second band") + BandSuffixUnit = (DEGREE, DEGREE) + BandBinCenter = (1.0, 2.0) + BandBinUnit = MICROMETER + Width = (0.5, 1.0) + End_Group + +End_Object +End""") + src_ds = gdal.Open('/vsimem/multiband.lbl') + + # Copy ISIS3 to GeoTIFF + gdal.GetDriverByName('GTiff').CreateCopy('/vsimem/out.tif', src_ds) + + # Check GeoTIFF + ds = gdal.Open('/vsimem/out.tif') + assert len(ds.GetMetadataDomainList()) == 3 + assert set(ds.GetMetadataDomainList()) == set(['IMAGE_STRUCTURE', 'json:ISIS3', 'DERIVED_SUBDATASETS']) + lbl = ds.GetMetadata_List('json:ISIS3')[0] + assert lbl + ds = None + assert gdal.VSIStatL('/vsimem/out.tif.aux.xml') + + # Check that the label is in PAM, and not internal to GTiff + with gdaltest.config_option('GDAL_PAM_ENABLED', 'NO'): + ds = gdal.Open('/vsimem/out.tif') + assert not ds.GetMetadata_List('json:ISIS3') + + # Copy back from GeoTIFF to ISIS3 + src_ds_gtiff = gdal.Open('/vsimem/out.tif') + gdal.GetDriverByName('ISIS3').CreateCopy('/vsimem/out.cub', src_ds_gtiff) + assert not gdal.VSIStatL('/vsimem/out.cub.aux.xml') + ds = gdal.Open('/vsimem/out.cub') + lbl = ds.GetMetadata_List('json:ISIS3')[0] + # Check label preservation + assert 'BandBin' in lbl + ds = None + + gdal.GetDriverByName('GTiff').Delete('/vsimem/out.tif') + gdal.GetDriverByName('ISIS3').Delete('/vsimem/out.cub') + + # Copy ISIS3 to PDS4 + with gdaltest.error_handler(): + gdal.GetDriverByName('PDS4').CreateCopy('/vsimem/out.xml', src_ds) + ds = gdal.Open('/vsimem/out.xml') + lbl = ds.GetMetadata_List('json:ISIS3')[0] + assert lbl + ds = None + assert gdal.VSIStatL('/vsimem/out.xml.aux.xml') + gdal.GetDriverByName('PDS4').Delete('/vsimem/out.xml') + + # Copy ISIS3 to PNG + gdal.GetDriverByName('PNG').CreateCopy('/vsimem/out.png', src_ds) + ds = gdal.Open('/vsimem/out.png') + lbl = ds.GetMetadata_List('json:ISIS3')[0] + assert lbl + ds = None + assert gdal.VSIStatL('/vsimem/out.png.aux.xml') + gdal.GetDriverByName('PNG').Delete('/vsimem/out.png') + + # Check GeoTIFF with non pure copy mode (test gdal_translate_lib) + gdal.Translate('/vsimem/out.tif', src_ds, options = '-mo FOO=BAR') + ds = gdal.Open('/vsimem/out.tif') + lbl = ds.GetMetadata_List('json:ISIS3')[0] + assert lbl + ds = None + gdal.GetDriverByName('GTiff').Delete('/vsimem/out.tif') + + # Test converting a subset of bands + gdal.Translate('/vsimem/out.tif', src_ds, options = '-b 2 -mo FOO=BAR') + ds = gdal.Open('/vsimem/out.tif') + lbl = ds.GetMetadata_List('json:ISIS3')[0] + lbl = json.loads(lbl) + + assert lbl['IsisCube']['BandBin'] == json.loads("""{ + "_type":"group", + "BandBinUnit":"MICROMETER", + "Width":{ + "unit":"um", + "value":[ + 1.000000 + ] + }, + "BandSuffixName":[ + "second band" + ], + "BandSuffixUnit":[ + "DEGREE" + ], + "BandBinCenter":[ + 2.000000 + ] + }""") + + assert 'OriginalBandBin' in lbl['IsisCube'] + assert lbl['IsisCube']['OriginalBandBin'] == json.loads("""{ + "_type":"group", + "BandSuffixName":[ + "first band", + "second band" + ], + "BandSuffixUnit":[ + "DEGREE", + "DEGREE" + ], + "BandBinCenter":[ + 1.000000, + 2.000000 + ], + "BandBinUnit":"MICROMETER", + "Width":{ + "value":[ + 0.500000, + 1.000000 + ], + "unit":"um" + } + }""") + + ds = None + gdal.GetDriverByName('GTiff').Delete('/vsimem/out.tif') + + src_ds = None + gdal.Unlink('/vsimem/multiband.lbl') diff --git a/gdal/apps/gdal_translate_lib.cpp b/gdal/apps/gdal_translate_lib.cpp index a2185ab5312e..dddd56f622c1 100644 --- a/gdal/apps/gdal_translate_lib.cpp +++ b/gdal/apps/gdal_translate_lib.cpp @@ -42,6 +42,7 @@ #include "commonutils.h" #include "cpl_conv.h" #include "cpl_error.h" +#include "cpl_json.h" #include "cpl_progress.h" #include "cpl_string.h" #include "cpl_vsi.h" @@ -506,6 +507,85 @@ static GDALDatasetH GDALTranslateFlush(GDALDatasetH hOutDS) return hOutDS; } +/************************************************************************/ +/* EditISIS3MetadataForBandChange() */ +/************************************************************************/ + +static CPLJSONObject Clone(const CPLJSONObject& obj) +{ + auto serialized = obj.Format(CPLJSONObject::Plain); + CPLJSONDocument oJSONDocument; + const GByte *pabyData = reinterpret_cast(serialized.c_str()); + oJSONDocument.LoadMemory( pabyData ); + return oJSONDocument.GetRoot(); +} + +static void ReworkArray(CPLJSONObject& container, const CPLJSONObject& obj, + int nSrcBandCount, + const GDALTranslateOptions *psOptions) +{ + auto oArray = obj.ToArray(); + if( oArray.Size() == nSrcBandCount ) + { + CPLJSONArray oNewArray; + for( int i = 0; i < psOptions->nBandCount; i++ ) + { + const int iSrcIdx = psOptions->panBandList[i]-1; + oNewArray.Add(oArray[iSrcIdx]); + } + const auto childName(obj.GetName()); + container.Delete(childName); + container.Add(childName, oNewArray); + } +} + +static CPLString EditISIS3MetadataForBandChange(const char* pszJSON, + int nSrcBandCount, + const GDALTranslateOptions *psOptions) +{ + CPLJSONDocument oJSONDocument; + const GByte *pabyData = reinterpret_cast(pszJSON); + if( !oJSONDocument.LoadMemory( pabyData ) ) + { + return CPLString(); + } + + auto oRoot = oJSONDocument.GetRoot(); + if( !oRoot.IsValid() ) + { + return CPLString(); + } + + auto oBandBin = oRoot.GetObj( "IsisCube/BandBin" ); + if( oBandBin.IsValid() && oBandBin.GetType() == CPLJSONObject::Object ) + { + // Backup original BandBin object + oRoot.GetObj("IsisCube").Add("OriginalBandBin", Clone(oBandBin)); + + // Iterate over BandBin members and reorder/resize its arrays that + // have the same number of elements than the number of bands of the + // source dataset. + for( auto& child: oBandBin.GetChildren() ) + { + if( child.GetType() == CPLJSONObject::Array ) + { + ReworkArray(oBandBin, child, nSrcBandCount, psOptions); + } + else if( child.GetType() == CPLJSONObject::Object ) + { + auto oValue = child.GetObj("value"); + auto oUnit = child.GetObj("unit"); + if( oValue.GetType() == CPLJSONObject::Array ) + { + ReworkArray(child, oValue, nSrcBandCount, psOptions); + } + } + } + } + + return oRoot.Format(CPLJSONObject::Pretty); +} + /************************************************************************/ /* GDALTranslate() */ /************************************************************************/ @@ -1349,14 +1429,28 @@ GDALDatasetH GDALTranslate( const char *pszDest, GDALDatasetH hSrcDataset, if (pszInterleave) poVDS->SetMetadataItem("INTERLEAVE", pszInterleave, "IMAGE_STRUCTURE"); - /* ISIS3 -> ISIS3 special case */ - if( EQUAL(psOptions->pszFormat, "ISIS3") ) + /* ISIS3 metadata preservation */ + char** papszMD_ISIS3 = poSrcDS->GetMetadata("json:ISIS3"); + if( papszMD_ISIS3 != nullptr) { - char** papszMD_ISIS3 = poSrcDS->GetMetadata("json:ISIS3"); - if( papszMD_ISIS3 != nullptr) + if( !bAllBandsInOrder ) + { + CPLString osJSON = EditISIS3MetadataForBandChange( + papszMD_ISIS3[0], GDALGetRasterCount( hSrcDataset ), psOptions); + if( !osJSON.empty() ) + { + char* apszMD[] = { &osJSON[0], nullptr }; + poVDS->SetMetadata( apszMD, "json:ISIS3" ); + } + } + else + { poVDS->SetMetadata( papszMD_ISIS3, "json:ISIS3" ); + } } - else if( EQUAL(psOptions->pszFormat, "PDS4") ) + + // PDS4 -> PDS4 special case + if( EQUAL(psOptions->pszFormat, "PDS4") ) { char** papszMD_PDS4 = poSrcDS->GetMetadata("xml:PDS4"); if( papszMD_PDS4 != nullptr) diff --git a/gdal/frmts/gtiff/geotiff.cpp b/gdal/frmts/gtiff/geotiff.cpp index fe79aa535869..2c5482764d01 100644 --- a/gdal/frmts/gtiff/geotiff.cpp +++ b/gdal/frmts/gtiff/geotiff.cpp @@ -17830,10 +17830,12 @@ GTiffDataset::CreateCopy( const char * pszFilename, GDALDataset *poSrcDS, /* to write metadata that we could not write as a TIFF tag. */ /* -------------------------------------------------------------------- */ if( !bHasWrittenMDInGeotiffTAG && !bStreaming ) + { GTiffDataset::WriteMetadata( poDS, l_hTIFF, true, eProfile, pszFilename, papszOptions, true /* don't write RPC and IMD file again */ ); + } if( !bStreaming ) GTiffDataset::WriteRPC( @@ -17841,6 +17843,16 @@ GTiffDataset::CreateCopy( const char * pszFilename, GDALDataset *poSrcDS, pszFilename, papszOptions, true /* write only in PAM AND if needed */ ); + // Propagate ISIS3 metadata, but only as PAM metadata. + { + char **papszISIS3MD = poSrcDS->GetMetadata("json:ISIS3"); + if( papszISIS3MD ) + { + poDS->SetMetadata( papszISIS3MD, "json:ISIS3"); + poDS->PushMetadataToPam(); + } + } + poDS->m_bWriteCOGLayout = bCopySrcOverviews; // To avoid unnecessary directory rewriting. @@ -18687,7 +18699,12 @@ char **GTiffDataset::GetMetadataDomainList() const int nbBaseDomains = CSLCount(papszBaseList); for( int domainId = 0; domainId < nbBaseDomains; ++domainId ) - papszDomainList = CSLAddString(papszDomainList,papszBaseList[domainId]); + { + if( CSLFindString(papszDomainList, papszBaseList[domainId]) < 0 ) + { + papszDomainList = CSLAddString(papszDomainList,papszBaseList[domainId]); + } + } CSLDestroy(papszBaseList); diff --git a/gdal/frmts/pds/pds4dataset.cpp b/gdal/frmts/pds/pds4dataset.cpp index 63ddee097bd7..1205e9d8da61 100644 --- a/gdal/frmts/pds/pds4dataset.cpp +++ b/gdal/frmts/pds/pds4dataset.cpp @@ -4326,6 +4326,12 @@ GDALDataset* PDS4Dataset::CreateCopy( const char *pszFilename, delete poDS; return nullptr; } + + char **papszISIS3MD = poSrcDS->GetMetadata("json:ISIS3"); + if( papszISIS3MD ) + { + poDS->SetMetadata( papszISIS3MD, "json:ISIS3"); + } } return poDS; diff --git a/gdal/gcore/gdalmultidomainmetadata.cpp b/gdal/gcore/gdalmultidomainmetadata.cpp index 668cb8f44d3a..41baf8232399 100644 --- a/gdal/gcore/gdalmultidomainmetadata.cpp +++ b/gdal/gcore/gdalmultidomainmetadata.cpp @@ -127,8 +127,12 @@ CPLErr GDALMultiDomainMetadata::SetMetadata( char **papszMetadata, // we want to mark name/value pair domains as being sorted for fast // access. - if( !STARTS_WITH_CI(pszDomain, "xml:") && !EQUAL(pszDomain, "SUBDATASETS") ) + if( !STARTS_WITH_CI(pszDomain, "xml:") && + !STARTS_WITH_CI(pszDomain, "json:") && + !EQUAL(pszDomain, "SUBDATASETS") ) + { papoMetadataLists[iDomain]->Sort(); + } return CE_None; } @@ -221,7 +225,7 @@ int GDALMultiDomainMetadata::XMLInit( CPLXMLNode *psTree, int /* bMerge */ ) /* -------------------------------------------------------------------- */ /* XML format subdocuments. */ /* -------------------------------------------------------------------- */ - if( EQUAL(pszFormat,"xml") ) + if( EQUAL(pszFormat,"xml") ) { // Find first non-attribute child of current element. CPLXMLNode *psSubDoc = psMetadata->psChild; @@ -234,6 +238,22 @@ int GDALMultiDomainMetadata::XMLInit( CPLXMLNode *psTree, int /* bMerge */ ) poMDList->AddStringDirectly( pszDoc ); } +/* -------------------------------------------------------------------- */ +/* JSon format subdocuments. */ +/* -------------------------------------------------------------------- */ + else if( EQUAL(pszFormat,"json") ) + { + // Find first text child of current element. + CPLXMLNode *psSubDoc = psMetadata->psChild; + while( psSubDoc != nullptr && psSubDoc->eType != CXT_Text ) + psSubDoc = psSubDoc->psNext; + if( psSubDoc ) + { + poMDList->Clear(); + poMDList->AddString( psSubDoc->pszValue ); + } + } + /* -------------------------------------------------------------------- */ /* Name value format. */ /* value_Text */ @@ -288,7 +308,7 @@ CPLXMLNode *GDALMultiDomainMetadata::Serialize() CPLCreateXMLNode( psMD, CXT_Attribute, "domain" ), CXT_Text, papszDomainList[iDomain] ); - bool bFormatXML = false; + bool bFormatXMLOrJSon = false; if( STARTS_WITH_CI(papszDomainList[iDomain], "xml:") && CSLCount(papszMD) == 1 ) @@ -296,7 +316,7 @@ CPLXMLNode *GDALMultiDomainMetadata::Serialize() CPLXMLNode *psValueAsXML = CPLParseXMLString( papszMD[0] ); if( psValueAsXML != nullptr ) { - bFormatXML = true; + bFormatXMLOrJSon = true; CPLCreateXMLNode( CPLCreateXMLNode( psMD, CXT_Attribute, "format" ), @@ -306,7 +326,18 @@ CPLXMLNode *GDALMultiDomainMetadata::Serialize() } } - if( !bFormatXML ) + if( STARTS_WITH_CI(papszDomainList[iDomain], "json:") + && CSLCount(papszMD) == 1 ) + { + bFormatXMLOrJSon = true; + + CPLCreateXMLNode( + CPLCreateXMLNode( psMD, CXT_Attribute, "format" ), + CXT_Text, "json" ); + CPLCreateXMLNode( psMD, CXT_Text, *papszMD ); + } + + if( !bFormatXMLOrJSon ) { CPLXMLNode* psLastChild = nullptr; // To go after domain attribute. diff --git a/gdal/gcore/gdalpamdataset.cpp b/gdal/gcore/gdalpamdataset.cpp index 36e230216401..39bc4912df84 100644 --- a/gdal/gcore/gdalpamdataset.cpp +++ b/gdal/gcore/gdalpamdataset.cpp @@ -986,21 +986,16 @@ CPLErr GDALPamDataset::CloneInfo( GDALDataset *poSrcDS, int nCloneFlags ) /* -------------------------------------------------------------------- */ if( nCloneFlags & GCIF_METADATA ) { - if( poSrcDS->GetMetadata() != nullptr ) + for( const char* pszMDD: { "", "RPC", "json:ISIS3" } ) { - if( !bOnlyIfMissing - || CSLCount(GetMetadata()) != CSLCount(poSrcDS->GetMetadata()) ) - { - SetMetadata( poSrcDS->GetMetadata() ); - } - } - if( poSrcDS->GetMetadata("RPC") != nullptr ) - { - if( !bOnlyIfMissing - || CSLCount(GetMetadata("RPC")) - != CSLCount(poSrcDS->GetMetadata("RPC")) ) + auto papszSrcMD = poSrcDS->GetMetadata(pszMDD); + if(papszSrcMD != nullptr ) { - SetMetadata( poSrcDS->GetMetadata("RPC"), "RPC" ); + if( !bOnlyIfMissing + || CSLCount(GetMetadata(pszMDD)) != CSLCount(papszSrcMD) ) + { + SetMetadata( papszSrcMD, pszMDD ); + } } } }