wip [skip ci]

apache · Sep 24, 2019 · 1551b52 · 1551b52
1 parent dd41d21
commit 1551b52
Show file tree

Hide file tree

Showing 7 changed files with 275 additions and 121 deletions.
diff --git a/cpp/src/arrow/filesystem/s3fs.h b/cpp/src/arrow/filesystem/s3fs.h
@@ -129,7 +129,7 @@ class ARROW_EXPORT S3FileSystem : public FileSystem {
   std::unique_ptr<Impl> impl_;
 };
 
-enum class S3LogLevel { Off, Fatal, Error, Warn, Info, Debug, Trace };
+enum class S3LogLevel : int8_t { Off, Fatal, Error, Warn, Info, Debug, Trace };
 
 struct ARROW_EXPORT S3GlobalOptions {
   S3LogLevel log_level;

diff --git a/python/pyarrow/_fs.pyx b/python/pyarrow/_fs.pyx
@@ -372,6 +372,7 @@ cdef class FileSystem:
 
         stream.set_input_stream(in_handle)
         stream.is_readable = True
+        stream.is_seekable = True
 
         return self._wrap_input_stream(
             stream, path=path, compression=compression, buffer_size=buffer_size
@@ -494,4 +495,4 @@ cdef class SubTreeFileSystem(FileSystem):
 
     cdef init(self, const shared_ptr[CFileSystem]& wrapped):
         FileSystem.init(self, wrapped)
-        self.subtreefs = <CSubTreeFileSystem*> wrapped.get()
+        self.subtreefs = <CSubTreeFileSystem*> wrapped.get()
diff --git a/python/pyarrow/_s3.pyx b/python/pyarrow/_s3.pyx
@@ -27,23 +27,47 @@ from pyarrow._fs cimport FileSystem
 from pyarrow.lib cimport check_status
 
 
+cpdef enum S3LogLevel:
+    Off = <int8_t> CS3LogLevel_Off
+    Fatal = <int8_t> CS3LogLevel_Fatal
+    Error = <int8_t> CS3LogLevel_Error
+    Warn = <int8_t> CS3LogLevel_Warn
+    Info = <int8_t> CS3LogLevel_Info
+    Debug = <int8_t> CS3LogLevel_Debug
+    Trace = <int8_t> CS3LogLevel_Trace
+
+
+def initialize_s3(S3LogLevel log_level=S3LogLevel.Error):
+    cdef CS3GlobalOptions options
+    options.log_level = <CS3LogLevel> log_level
+    check_status(CInitializeS3(options))
+
+
+def finalize_s3():
+    check_status(CFinalizeS3())
+
+
 cdef class S3FileSystem(FileSystem):
 
     cdef:
         CS3FileSystem* s3fs
 
-    def __init__(self, str access_key, str secret_key, str region='us-east-1',
-                 str scheme='https', str endpoint_override=None):
+    def __init__(self, str access_key=None, str secret_key=None,
+                 str region='us-east-1', str scheme='https',
+                 str endpoint_override=None, bint background_writes=True):
         cdef:
-            CS3Options options
+            CS3Options options = CS3Options.Defaults()
             shared_ptr[CS3FileSystem] wrapped
 
-        options.access_key = tobytes(access_key)
-        options.secret_key = tobytes(secret_key)
+        if access_key is not None or secret_key is not None:
+            options.ConfigureAccessKey(tobytes(access_key),
+                                       tobytes(secret_key))
+
         options.region = tobytes(region)
         options.scheme = tobytes(scheme)
+        options.background_writes = background_writes
         if endpoint_override is not None:
-            options.endpoint_override = endpoint_override
+            options.endpoint_override = tobytes(endpoint_override)
 
         check_status(CS3FileSystem.Make(options, &wrapped))
         self.init(<shared_ptr[CFileSystem]> wrapped)

diff --git a/python/pyarrow/fs.py b/python/pyarrow/fs.py
@@ -17,9 +17,16 @@
 
 from __future__ import absolute_import
 
-from pyarrow._fs import *  # noqa
-
+from pyarrow._fs import (
+    Selector,
+    FileType,
+    FileStats,
+    FileSystem,
+    LocalFileSystem,
+    SubTreeFileSystem
+)
+from pyarrow._s3 import S3FileSystem
 try:
-    from pyarrow._s3 import *  # noqa
+    from pyarrow._s3 import S3FileSystem, initialize_s3, finalize_s3
 except ImportError:
     pass
diff --git a/python/pyarrow/includes/libarrow_s3.pxd b/python/pyarrow/includes/libarrow_s3.pxd
@@ -24,13 +24,36 @@ from pyarrow.includes.libarrow cimport CFileSystem
 
 cdef extern from "arrow/filesystem/api.h" namespace "arrow::fs" nogil:
 
-    cdef struct CS3Options "arrow::fs::S3Options":
+    enum CS3LogLevel "arrow::fs::S3LogLevel":
+        CS3LogLevel_Off "arrow::fs::S3LogLevel::Off"
+        CS3LogLevel_Fatal "arrow::fs::S3LogLevel::Fatal"
+        CS3LogLevel_Error "arrow::fs::S3LogLevel::Error"
+        CS3LogLevel_Warn "arrow::fs::S3LogLevel::Warn"
+        CS3LogLevel_Info "arrow::fs::S3LogLevel::Info"
+        CS3LogLevel_Debug "arrow::fs::S3LogLevel::Debug"
+        CS3LogLevel_Trace "arrow::fs::S3LogLevel::Trace"
+
+    cdef struct CS3GlobalOptions "arrow::fs::S3GlobalOptions":
+        CS3LogLevel log_level
+
+    cdef cppclass CS3Options "arrow::fs::S3Options":
         c_string region
         c_string endpoint_override
         c_string scheme
-        c_string access_key
-        c_string secret_key
+        c_bool background_writes
+        void ConfigureDefaultCredentials()
+        void ConfigureAccessKey(const c_string& access_key,
+                                const c_string& secret_key)
+        @staticmethod
+        CS3Options Defaults()
+        @staticmethod
+        CS3Options FromAccessKey(const c_string& access_key,
+                                 const c_string& secret_key)
 
     cdef cppclass CS3FileSystem "arrow::fs::S3FileSystem"(CFileSystem):
         @staticmethod
         CStatus Make(const CS3Options& options, shared_ptr[CS3FileSystem]* out)
+
+    cdef CStatus CInitializeS3 "arrow::fs::InitializeS3"(
+        const CS3GlobalOptions& options)
+    cdef CStatus CFinalizeS3 "arrow::fs::FinalizeS3"()