Add VSIError mechanism to store errors related to filesystem calls, a…

…nd use it for /vsis3/. Add new CPLE_ error numbers. (patch by Rob Emanuele, OSGeo/gdal#98)

git-svn-id: https://svn.osgeo.org/gdal/trunk/gdal@33758 f0d54148-0727-0410-94bb-9a71ac55c965

Vagrantfile

@@ -16,15 +16,17 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
   config.vm.hostname = "gdal-vagrant"
   config.vm.box_url = "http://files.vagrantup.com/precise64.box"
   config.vm.host_name = "gdal-vagrant"
-  
+
   config.vm.network :forwarded_port, guest: 80, host: 8080
 
+  config.vm.synced_folder "../autotest/", "/home/vagrant/autotest/"
+
   config.vm.provider :virtualbox do |vb|
      vb.customize ["modifyvm", :id, "--memory", vm_ram]
      vb.customize ["modifyvm", :id, "--cpus", vm_cpu]
      vb.customize ["modifyvm", :id, "--ioapic", "on"]
      vb.name = "gdal-vagrant"
-   end  
+   end
 
   ppaRepos = [
     "ppa:ubuntugis/ubuntugis-unstable", "ppa:marlam/gta"

apps/gdalinfo_bin.cpp

@@ -127,8 +127,9 @@ int main( int argc, char ** argv )
     while (__AFL_LOOP(1000)) {
         iIter ++;
 #endif
+
     GDALDatasetH hDataset
-        = GDALOpenEx( psOptionsForBinary->pszFilename, GDAL_OF_READONLY | GDAL_OF_RASTER, NULL,
+        = GDALOpenEx( psOptionsForBinary->pszFilename, GDAL_OF_READONLY | GDAL_OF_RASTER | GDAL_OF_VERBOSE_ERROR, NULL,
                       (const char* const* )psOptionsForBinary->papszOpenOptions, NULL );
 
     if( hDataset == NULL )

gcore/gdaldataset.cpp

@@ -31,6 +31,7 @@
 #include "cpl_string.h"
 #include "cpl_hash_set.h"
 #include "cpl_multiproc.h"
+#include "cpl_vsi_error.h"
 #include "gdal_priv.h"
 #include "ogr_attrind.h"
 #include "ogr_featurestyle.h"
@@ -2663,6 +2664,21 @@ GDALDatasetH CPL_STDCALL GDALOpenEx( const char* pszFilename,
     if( (nOpenFlags & GDAL_OF_KIND_MASK) == 0 )
         nOpenFlags |= GDAL_OF_KIND_MASK;
 
+    int         iDriver;
+    GDALDriverManager *poDM = GetGDALDriverManager();
+    //CPLLocaleC  oLocaleForcer;
+
+    CPLErrorReset();
+    VSIErrorReset();
+    CPLAssert( NULL != poDM );
+
+    /* Build GDALOpenInfo just now to avoid useless file stat'ing if a */
+    /* shared dataset was asked before */
+    GDALOpenInfo oOpenInfo(pszFilename,
+                           nOpenFlags,
+                           (char**) papszSiblingFiles);
+
+    // Prevent infinite recursion
     {
         int* pnRecCount = (int*)CPLGetTLS( CTLS_GDALDATASET_REC_PROTECT_MAP );
         if( pnRecCount == NULL )
@@ -2680,19 +2696,6 @@ GDALDatasetH CPL_STDCALL GDALOpenEx( const char* pszFilename,
         (*pnRecCount) ++;
     }
 
-    int         iDriver;
-    GDALDriverManager *poDM = GetGDALDriverManager();
-    //CPLLocaleC  oLocaleForcer;
-
-    CPLErrorReset();
-    CPLAssert( NULL != poDM );
-
-    /* Build GDALOpenInfo just now to avoid useless file stat'ing if a */
-    /* shared dataset was asked before */
-    GDALOpenInfo oOpenInfo(pszFilename,
-                           nOpenFlags,
-                           (char**) papszSiblingFiles);
-
     // Remove leading @ if present
     char** papszOpenOptionsCleaned = CSLDuplicate((char**)papszOpenOptions);
     for(char** papszIter = papszOpenOptionsCleaned;
@@ -2852,6 +2855,7 @@ GDALDatasetH CPL_STDCALL GDALOpenEx( const char* pszFilename,
                     poDS = poOvrDS;
                 }
             }
+            VSIErrorReset();
 
             CSLDestroy(papszOpenOptionsCleaned);
             return (GDALDatasetH) poDS;
@@ -2872,15 +2876,22 @@ GDALDatasetH CPL_STDCALL GDALOpenEx( const char* pszFilename,
 
     if( nOpenFlags & GDAL_OF_VERBOSE_ERROR )
     {
-        if( oOpenInfo.bStatOK )
-            CPLError( CE_Failure, CPLE_OpenFailed,
-                    "`%s' not recognized as a supported file format.\n",
-                    pszFilename );
-        else
-            CPLError( CE_Failure, CPLE_OpenFailed,
-                    "`%s' does not exist in the file system,\n"
-                    "and is not recognized as a supported dataset name.\n",
-                    pszFilename );
+        // Check to see if there was a filesystem error, and report it if so.
+        // If not, return a more generic error.
+        if(!VSIToCPLError(CE_Failure, CPLE_OpenFailed)) {
+            if( oOpenInfo.bStatOK )
+                CPLError( CE_Failure, CPLE_OpenFailed,
+                          "`%s' not recognized as a supported file format.\n",
+                          pszFilename );
+            else {
+                // If Stat failed and no VSI error was set,
+                // assume it is because the file did not exist on the filesystem.
+                CPLError( CE_Failure, CPLE_OpenFailed,
+                          "`%s' does not exist in the file system,\n"
+                          "and is not recognized as a supported dataset name.\n",
+                          pszFilename );
+            }
+        }
     }
 
     int* pnRecCount = (int*)CPLGetTLS( CTLS_GDALDATASET_REC_PROTECT_MAP );

gcore/gdalopeninfo.cpp

@@ -67,6 +67,7 @@ GDALOpenInfo::GDALOpenInfo( const char * pszFilenameIn, int nOpenFlagsIn,
     nHeaderBytes(0),
     pabyHeader(NULL)
 {
+
 /* -------------------------------------------------------------------- */
 /*      Ensure that C: is treated as C:\ so we can stat it on           */
 /*      Windows.  Similar to what is done in CPLStat().                 */
@@ -123,22 +124,25 @@ GDALOpenInfo::GDALOpenInfo( const char * pszFilenameIn, int nOpenFlagsIn,
 
     if( bPotentialDirectory )
     {
+        int nStatFlags = VSI_STAT_EXISTS_FLAG | VSI_STAT_NATURE_FLAG;
+        if(nOpenFlagsIn & GDAL_OF_VERBOSE_ERROR)
+            nStatFlags |= VSI_STAT_SET_ERROR_FLAG;
+
         // For those special files, opening them with VSIFOpenL() might result
         // in content, even if they should be considered as directories, so
         // use stat.
         VSIStatBufL sStat;
 
-        if( VSIStatExL( pszFilename, &sStat,
-                        VSI_STAT_EXISTS_FLAG | VSI_STAT_NATURE_FLAG ) == 0 )
-        {
+        if(VSIStatExL( pszFilename, &sStat, nStatFlags) == 0) {
             bStatOK = TRUE;
             if( VSI_ISDIR( sStat.st_mode ) )
                 bIsDirectory = TRUE;
         }
     }
 
-    if( !bIsDirectory )
-        fpL = VSIFOpenL( pszFilename, (eAccess == GA_Update) ? "r+b" : "rb" );
+    if( !bIsDirectory ) {
+        fpL = VSIFOpenExL( pszFilename, (eAccess == GA_Update) ? "r+b" : "rb", (nOpenFlagsIn & GDAL_OF_VERBOSE_ERROR) > 0);
+    }
     if( fpL != NULL )
     {
         bStatOK = TRUE;
@@ -166,7 +170,7 @@ GDALOpenInfo::GDALOpenInfo( const char * pszFilenameIn, int nOpenFlagsIn,
     else if( !bStatOK )
     {
         VSIStatBufL sStat;
-        if( VSIStatExL( pszFilename, &sStat,
+        if( !bPotentialDirectory && VSIStatExL( pszFilename, &sStat,
                         VSI_STAT_EXISTS_FLAG | VSI_STAT_NATURE_FLAG ) == 0 )
         {
             bStatOK = TRUE;

port/GNUmakefile

@@ -7,11 +7,11 @@ else
 include GDALmake.opt
 endif
 
-ifeq ($(LIBZ_SETTING),internal) 
-XTRA_OPT =      -I../frmts/zlib 
-else 
-XTRA_OPT = 
-endif 
+ifeq ($(LIBZ_SETTING),internal)
+XTRA_OPT =      -I../frmts/zlib
+else
+XTRA_OPT =
+endif
 
 CPPFLAGS	:= $(CPPFLAGS)	$(CURL_INC) $(XTRA_OPT)
 
@@ -27,7 +27,7 @@ OBJ =	cpl_conv.o cpl_error.o cpl_string.o cplgetsymbol.o cplstringlist.o \
 	cpl_base64.o cpl_vsil_curl.o cpl_vsil_curl_streaming.o \
 	cpl_vsil_cache.o cpl_xml_validate.o cpl_spawn.o \
 	cpl_google_oauth2.o cpl_progress.o cpl_virtualmem.o cpl_worker_thread_pool.o \
-	cpl_vsil_crypt.o cpl_sha256.o cpl_aws.o
+	cpl_vsil_crypt.o cpl_sha256.o cpl_aws.o cpl_vsi_error.o
 
 ifeq ($(ODBC_SETTING),yes)
 OBJ	:= 	$(OBJ) cpl_odbc.o
@@ -66,5 +66,5 @@ clean:
 install:
 	for f in *.h ; do $(INSTALL_DATA) $$f $(DESTDIR)$(INST_INCLUDE) ; done
 
-xmlreformat:	xmlreformat.o 
+xmlreformat:	xmlreformat.o
 	$(CXX) $(CXXFLAGS) xmlreformat.o $(CONFIG_LIBS) -o xmlreformat

port/cpl_aws.cpp

@@ -29,6 +29,7 @@
  ****************************************************************************/
 
 #include "cpl_aws.h"
+#include "cpl_vsi_error.h"
 #include "cpl_sha256.h"
 #include "cpl_time.h"
 #include "cpl_minixml.h"
@@ -405,14 +406,14 @@ VSIS3HandleHelper* VSIS3HandleHelper::BuildFromURI(const char* pszURI,
     CPLString osSecretAccessKey = CPLGetConfigOption("AWS_SECRET_ACCESS_KEY", "");
     if( osSecretAccessKey.size() == 0 )
     {
-        CPLError(CE_Failure, CPLE_AppDefined,
+        VSIError(VSIE_AWSInvalidCredentials,
                  "AWS_SECRET_ACCESS_KEY configuration option not defined");
         return NULL;
     }
     CPLString osAccessKeyId = CPLGetConfigOption("AWS_ACCESS_KEY_ID", "");
     if( osAccessKeyId.size() == 0 )
     {
-        CPLError(CE_Failure, CPLE_AppDefined,
+        VSIError(VSIE_AWSInvalidCredentials,
                  "AWS_ACCESS_KEY_ID configuration option not defined");
         return NULL;
     }
@@ -509,30 +510,30 @@ struct curl_slist* VSIS3HandleHelper::GetCurlHeaders(const CPLString& osVerb,
 /*                          CanRestartOnError()                         */
 /************************************************************************/
 
-bool VSIS3HandleHelper::CanRestartOnError(const char* pszErrorMsg)
+bool VSIS3HandleHelper::CanRestartOnError(const char* pszErrorMsg, bool bSetError)
 {
 #ifdef DEBUG_VERBOSE
     CPLDebug("S3", "%s", pszErrorMsg);
 #endif
 
     if( !STARTS_WITH(pszErrorMsg, "<?xml") )
     {
-        CPLError(CE_Failure, CPLE_AppDefined, "%s", pszErrorMsg);
+        if(bSetError) { VSIError(VSIE_AWSError, "Invalid AWS response: %s", pszErrorMsg); }
         return false;
     }
 
     CPLXMLNode* psTree = CPLParseXMLString(pszErrorMsg);
     if( psTree == NULL )
     {
-        CPLError(CE_Failure, CPLE_AppDefined, "%s", pszErrorMsg);
+        if(bSetError) { VSIError(VSIE_AWSError, "Malformed AWS XML repsonse: %s", pszErrorMsg); }
         return false;
     }
 
     const char* pszCode = CPLGetXMLValue(psTree, "=Error.Code", NULL);
     if( pszCode == NULL )
     {
         CPLDestroyXMLNode(psTree);
-        CPLError(CE_Failure, CPLE_AppDefined, "%s", pszErrorMsg);
+        if(bSetError) { VSIError(VSIE_AWSError, "Malformed AWS XML repsonse: %s", pszErrorMsg); }
         return false;
     }
 
@@ -542,7 +543,7 @@ bool VSIS3HandleHelper::CanRestartOnError(const char* pszErrorMsg)
         if( pszRegion == NULL )
         {
             CPLDestroyXMLNode(psTree);
-            CPLError(CE_Failure, CPLE_AppDefined, "%s", pszErrorMsg);
+            if(bSetError) { VSIError(VSIE_AWSError, "Malformed AWS XML repsonse: %s", pszErrorMsg); }
             return false;
         }
         SetAWSRegion(pszRegion);
@@ -559,7 +560,7 @@ bool VSIS3HandleHelper::CanRestartOnError(const char* pszErrorMsg)
                                     pszEndpoint[m_osBucket.size()] != '.')) )
         {
             CPLDestroyXMLNode(psTree);
-            CPLError(CE_Failure, CPLE_AppDefined, "%s", pszErrorMsg);
+            if(bSetError) { VSIError(VSIE_AWSError, "Malformed AWS XML repsonse: %s", pszErrorMsg); }
             return false;
         }
         if( !m_bUseVirtualHosting &&
@@ -575,13 +576,25 @@ bool VSIS3HandleHelper::CanRestartOnError(const char* pszErrorMsg)
         return true;
     }
 
-    if( !EQUAL(pszCode, "NoSuchKey") )
-    {
+    if(bSetError) {
+        /*
+         * Translate AWS errors into VSI errors
+         */
         const char* pszMessage = CPLGetXMLValue(psTree, "=Error.Message", NULL);
-        if( pszMessage )
-            CPLError(CE_Failure, CPLE_AppDefined, "%s", pszMessage);
-        else
-            CPLError(CE_Failure, CPLE_AppDefined, "%s", pszErrorMsg);
+
+        if( pszMessage == NULL ) {
+            VSIError(VSIE_AWSError, "%s", pszErrorMsg);
+        } else if( EQUAL(pszCode, "AccessDenied") ) {
+            VSIError(VSIE_AWSAccessDenied, "%s", pszMessage);
+        } else if( EQUAL(pszCode, "NoSuchBucket") ) {
+            VSIError(VSIE_AWSBucketNotFound, "%s", pszMessage);
+        } else if( EQUAL(pszCode, "NoSuchKey") ) {
+            VSIError(VSIE_AWSObjectNotFound, "%s", pszMessage);
+        } else if( EQUAL(pszCode, "SignatureDoesNotMatch") ) {
+            VSIError(VSIE_AWSSignatureDoesNotMatch, "%s", pszMessage);
+        } else {
+            VSIError(VSIE_AWSError, "%s", pszMessage);
+        }
     }
 
     CPLDestroyXMLNode(psTree);

port/cpl_aws.h

@@ -101,7 +101,8 @@ class VSIS3HandleHelper
         struct curl_slist* GetCurlHeaders(const CPLString& osVerb,
                                           const void *pabyDataContent = NULL,
                                           size_t nBytesContent = 0);
-        bool CanRestartOnError(const char*);
+        bool CanRestartOnError(const char* pszErrorMsg) { return CanRestartOnError(pszErrorMsg, false); }
+        bool CanRestartOnError(const char*, bool bSetError);
 
         const CPLString& GetURL() const { return m_osURL; }
         const CPLString& GetBucket() const { return m_osBucket; }

port/cpl_error.h

@@ -65,17 +65,24 @@ typedef enum
 /* between CPLErr and CPLErrorNum */
 typedef enum
 {
- CPLE_None,
- CPLE_AppDefined,
- CPLE_OutOfMemory,
- CPLE_FileIO,
- CPLE_OpenFailed,
- CPLE_IllegalArg,
- CPLE_NotSupported,
- CPLE_AssertionFailed,
- CPLE_NoWriteAccess,
- CPLE_UserInterrupt,
- CPLE_ObjectNull,
+  CPLE_None,
+  CPLE_AppDefined,
+  CPLE_OutOfMemory,
+  CPLE_FileIO,
+  CPLE_OpenFailed,
+  CPLE_IllegalArg,
+  CPLE_NotSupported,
+  CPLE_AssertionFailed,
+  CPLE_NoWriteAccess,
+  CPLE_UserInterrupt,
+  CPLE_ObjectNull,
+  CPLE_HttpResponse,
+  CPLE_HttpResponse,
+  CPLE_AWSBucketNotFound,
+  CPLE_AWSObjectNotFound,
+  CPLE_AWSAccessDenied,
+  CPLE_AWSInvalidCredentials,
+  CPLE_AWSSignaturDoesNotMatch,
 } CPLErrorNum;
 
 #else
@@ -94,6 +101,16 @@ typedef int CPLErrorNum;
 #define CPLE_UserInterrupt              9
 #define CPLE_ObjectNull                 10
 
+/*
+ * Filesystem-specific errors
+ */
+#define CPLE_HttpResponse               11
+#define CPLE_AWSBucketNotFound          12
+#define CPLE_AWSObjectNotFound          13
+#define CPLE_AWSAccessDenied            14
+#define CPLE_AWSInvalidCredentials      15
+#define CPLE_AWSSignaturDoesNotMatch    16
+
 /* 100 - 299 reserved for GDAL */
 
 #endif