Github Action for Buildbot Builders

.github/workflows/buildbot.yml

@@ -0,0 +1,54 @@
+name: Buildbot
+
+on: [push]
+
+jobs:
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        builder:
+        - "AMD64 Debian 9 Rust 1.35"
+        - "AMD64 Debian 9 Go 1.11.11"
+        - "AMD64 Debian 9 Go 1.12.6"
+        - "AMD64 Conda C++"
+        - "AMD64 Conda Python 2.7"
+        - "AMD64 Conda Python 3.6"
+        - "AMD64 Conda Python 3.7"
+        - "AMD64 Conda R"
+        - "AMD64 Debian 9 NodeJS 11"
+        - "AMD64 Java OpenJDK 11"
+        - "AMD64 Java OpenJDK 8"
+        - "AMD64 Ubuntu 18.04 C GLib"
+        - "AMD64 Ubuntu 18.04 C++"
+        - "AMD64 Ubuntu 18.04 Python 3"
+        - "AMD64 Ubuntu 18.04 R"
+
+    steps:
+
+    - name: Checkout Ursabot
+      uses: actions/checkout@v1
+      with:
+        repository: ursa-labs/ursabot
+        ref: refs/heads/master
+        path: ursabot
+    - name: Install Python
+      uses: actions/setup-python@v1
+      with:
+        python-version: '3.7'
+    - name: Install Ursabot
+      run: pip install -e ../ursabot
+    - name: Check Ursabot Command
+      run: ursabot --help
+
+    - name: Run ${{ matrix.builder }} Builder
+      run: |
+        cd ../ursabot/projects/arrow
+        ursabot project build \
+          --repo https://github.com/$GITHUB_REPOSITORY \
+          --branch $GITHUB_REF \
+          --commit $GITHUB_SHA \
+          "${{ matrix.builder }}"
+

LICENSE.txt

@@ -1841,8 +1841,8 @@ This project includes code from the autobrew project.
 * r/tools/autobrew and dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb
   are based on code from the autobrew project.
 
-Copyright: Copyright (c) 2017 - 2019, Jeroen Ooms.
-All rights reserved.
+Copyright (c) 2019, Jeroen Ooms
+License: MIT
 Homepage: https://github.com/jeroen/autobrew
 
 --------------------------------------------------------------------------------
@@ -1874,3 +1874,31 @@ SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
 CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
 OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+----------------------------------------------------------------------
+
+cpp/src/arrow/vendored/base64.cpp has the following license
+
+ZLIB License
+
+Copyright (C) 2004-2017 René Nyffenegger
+
+This source code is provided 'as-is', without any express or implied
+warranty. In no event will the author be held liable for any damages arising
+from the use of this software.
+
+Permission is granted to anyone to use this software for any purpose, including
+commercial applications, and to alter it and redistribute it freely, subject to
+the following restrictions:
+
+1. The origin of this source code must not be misrepresented; you must not
+   claim that you wrote the original source code. If you use this source code
+   in a product, an acknowledgment in the product documentation would be
+   appreciated but is not required.
+
+2. Altered source versions must be plainly marked as such, and must not be
+   misrepresented as being the original source code.
+
+3. This notice may not be removed or altered from any source distribution.
+
+René Nyffenegger [email protected]

ci/travis_script_python.sh

@@ -226,7 +226,7 @@ if [ "$ARROW_TRAVIS_PYTHON_DOCS" == "1" ]; then
   doxygen
   popd
   cd ../docs
-  sphinx-build -q -b html -d _build/doctrees -W source _build/html
+  sphinx-build -q -b html -d _build/doctrees -W --keep-going source _build/html
 fi
 
 popd  # $ARROW_PYTHON_DIR

cpp/apidoc/Doxyfile

@@ -2074,7 +2074,9 @@ INCLUDE_FILE_PATTERNS  =
 
 PREDEFINED = __attribute__(x)= \
              __declspec(x)= \
+             PARQUET_EXPORT= \
              ARROW_EXPORT= \
+             ARROW_FLIGHT_EXPORT= \
              ARROW_EXTERN_TEMPLATE= \
              ARROW_DEPRECATED(x)=
 

cpp/build-support/run_clang_format.py

@@ -28,12 +28,10 @@
 
 # examine the output of clang-format and if changes are
 # present assemble a (unified)patch of the difference
-def _check_one_file(completed_processes, filename):
+def _check_one_file(filename, formatted):
     with open(filename, "rb") as reader:
         original = reader.read()
 
-    returncode, stdout, stderr = completed_processes[filename]
-    formatted = stdout
     if formatted != original:
         # Run the equivalent of diff -u
         diff = list(difflib.unified_diff(
@@ -106,20 +104,21 @@ def _check_one_file(completed_processes, filename):
             [arguments.clang_format_binary, filename]
             for filename in formatted_filenames
         ], stdout=PIPE, stderr=PIPE)
-        for returncode, stdout, stderr in results:
+
+        checker_args = []
+        for filename, res in zip(formatted_filenames, results):
             # if any clang-format reported a parse error, bubble it
+            returncode, stdout, stderr = res
             if returncode != 0:
+                print(stderr)
                 sys.exit(returncode)
+            checker_args.append((filename, stdout))
 
         error = False
-        checker = partial(_check_one_file, {
-            filename: result
-            for filename, result in zip(formatted_filenames, results)
-        })
         pool = mp.Pool()
         try:
             # check the output from each invocation of clang-format in parallel
-            for filename, diff in pool.imap(checker, formatted_filenames):
+            for filename, diff in pool.starmap(_check_one_file, checker_args):
                 if not arguments.quiet:
                     print("Checking {}".format(filename))
                 if diff:

cpp/src/arrow/CMakeLists.txt

@@ -117,6 +117,7 @@ set(ARROW_SRCS
     filesystem/filesystem.cc
     filesystem/localfs.cc
     filesystem/mockfs.cc
+    filesystem/path_tree.cc
     filesystem/path_util.cc
     filesystem/util_internal.cc
     io/buffered.cc
@@ -127,6 +128,7 @@ set(ARROW_SRCS
     io/interfaces.cc
     io/memory.cc
     io/readahead.cc
+    io/slow.cc
     testing/util.cc
     util/basic_decimal.cc
     util/bit_util.cc
@@ -144,6 +146,7 @@ set(ARROW_SRCS
     util/thread_pool.cc
     util/trie.cc
     util/utf8.cc
+    vendored/base64.cpp
     vendored/datetime/tz.cpp)
 
 # Add dependencies for third-party allocators.

cpp/src/arrow/array.cc

@@ -1234,6 +1234,7 @@ struct ValidateVisitor {
   }
 
   Status Visit(const StructArray& array) {
+    const auto& struct_type = checked_cast<const StructType&>(*array.type());
     if (array.num_fields() > 0) {
       // Validate fields
       int64_t array_length = array.field(0)->length();
@@ -1245,10 +1246,17 @@ struct ValidateVisitor {
                                  it->type()->ToString(), " at position [", idx, "]");
         }
 
+        auto it_type = struct_type.child(i)->type();
+        if (!it->type()->Equals(it_type)) {
+          return Status::Invalid("Child array at position [", idx,
+                                 "] does not match type field: ", it->type()->ToString(),
+                                 " vs ", it_type->ToString());
+        }
+
         const Status child_valid = it->Validate();
         if (!child_valid.ok()) {
           return Status::Invalid("Child array invalid: ", child_valid.ToString(),
-                                 " at position [", idx, "}");
+                                 " at position [", idx, "]");
         }
         ++idx;
       }

cpp/src/arrow/csv/options.h

@@ -32,82 +32,91 @@ class DataType;
 
 namespace csv {
 
+// Silly workaround for https://github.com/michaeljones/breathe/issues/453
+constexpr char kDefaultEscapeChar = '\\';
+
 struct ARROW_EXPORT ParseOptions {
   // Parsing options
 
-  // Field delimiter
+  /// Field delimiter
   char delimiter = ',';
-  // Whether quoting is used
+  /// Whether quoting is used
   bool quoting = true;
-  // Quoting character (if `quoting` is true)
+  /// Quoting character (if `quoting` is true)
   char quote_char = '"';
-  // Whether a quote inside a value is double-quoted
+  /// Whether a quote inside a value is double-quoted
   bool double_quote = true;
-  // Whether escaping is used
+  /// Whether escaping is used
   bool escaping = false;
-  // Escaping character (if `escaping` is true)
-  char escape_char = '\\';
-  // Whether values are allowed to contain CR (0x0d) and LF (0x0a) characters
+  /// Escaping character (if `escaping` is true)
+  char escape_char = kDefaultEscapeChar;
+  /// Whether values are allowed to contain CR (0x0d) and LF (0x0a) characters
   bool newlines_in_values = false;
-  // Whether empty lines are ignored.  If false, an empty line represents
-  // a single empty value (assuming a one-column CSV file).
+  /// Whether empty lines are ignored.  If false, an empty line represents
+  /// a single empty value (assuming a one-column CSV file).
   bool ignore_empty_lines = true;
 
+  /// Create parsing options with default values
   static ParseOptions Defaults();
 };
 
 struct ARROW_EXPORT ConvertOptions {
   // Conversion options
 
-  // Whether to check UTF8 validity of string columns
+  /// Whether to check UTF8 validity of string columns
   bool check_utf8 = true;
-  // Optional per-column types (disabling type inference on those columns)
+  /// Optional per-column types (disabling type inference on those columns)
   std::unordered_map<std::string, std::shared_ptr<DataType>> column_types;
-  // Recognized spellings for null values
+  /// Recognized spellings for null values
   std::vector<std::string> null_values;
-  // Recognized spellings for boolean values
+  /// Recognized spellings for boolean true values
   std::vector<std::string> true_values;
+  /// Recognized spellings for boolean false values
   std::vector<std::string> false_values;
-  // Whether string / binary columns can have null values.
-  // If true, then strings in "null_values" are considered null for string columns.
-  // If false, then all strings are valid string values.
+  /// Whether string / binary columns can have null values.
+  ///
+  /// If true, then strings in "null_values" are considered null for string columns.
+  /// If false, then all strings are valid string values.
   bool strings_can_be_null = false;
 
   // XXX Should we have a separate FilterOptions?
 
-  // If non-empty, indicates the names of columns from the CSV file that should
-  // be actually read and converted (in the vector's order).
-  // Columns not in this vector will be ignored.
+  /// If non-empty, indicates the names of columns from the CSV file that should
+  /// be actually read and converted (in the vector's order).
+  /// Columns not in this vector will be ignored.
   std::vector<std::string> include_columns;
-  // If false, columns in `include_columns` but not in the CSV file will error out.
-  // If true, columns in `include_columns` but not in the CSV file will produce
-  // a column of nulls (whose type is selected using `column_types`,
-  // or null by default)
-  // This option is ignored if `include_columns` is empty.
+  /// If false, columns in `include_columns` but not in the CSV file will error out.
+  /// If true, columns in `include_columns` but not in the CSV file will produce
+  /// a column of nulls (whose type is selected using `column_types`,
+  /// or null by default)
+  /// This option is ignored if `include_columns` is empty.
   bool include_missing_columns = false;
 
+  /// Create conversion options with default values, including conventional
+  /// values for `null_values`, `true_values` and `false_values`
   static ConvertOptions Defaults();
 };
 
 struct ARROW_EXPORT ReadOptions {
   // Reader options
 
-  // Whether to use the global CPU thread pool
+  /// Whether to use the global CPU thread pool
   bool use_threads = true;
-  // Block size we request from the IO layer; also determines the size of
-  // chunks when use_threads is true
+  /// Block size we request from the IO layer; also determines the size of
+  /// chunks when use_threads is true
   int32_t block_size = 1 << 20;  // 1 MB
 
-  // Number of header rows to skip (not including the row of column names, if any)
+  /// Number of header rows to skip (not including the row of column names, if any)
   int32_t skip_rows = 0;
-  // Column names for the target table.
-  // If empty, fall back on autogenerate_column_names.
+  /// Column names for the target table.
+  /// If empty, fall back on autogenerate_column_names.
   std::vector<std::string> column_names;
-  // Whether to autogenerate column names if `column_names` is empty.
-  // If true, column names will be of the form "f0", "f1"...
-  // If false, column names will be read from the first CSV row after `skip_rows`.
+  /// Whether to autogenerate column names if `column_names` is empty.
+  /// If true, column names will be of the form "f0", "f1"...
+  /// If false, column names will be read from the first CSV row after `skip_rows`.
   bool autogenerate_column_names = false;
 
+  /// Create read options with default values
   static ReadOptions Defaults();
 };
 

cpp/src/arrow/csv/reader.h

@@ -35,12 +35,15 @@ class InputStream;
 
 namespace csv {
 
+/// A class that reads an entire CSV file into a Arrow Table
 class ARROW_EXPORT TableReader {
  public:
   virtual ~TableReader() = default;
 
+  /// Read the entire CSV file and convert it to a Arrow Table
   virtual Status Read(std::shared_ptr<Table>* out) = 0;
 
+  /// Create a TableReader instance
   static Status Make(MemoryPool* pool, std::shared_ptr<io::InputStream> input,
                      const ReadOptions&, const ParseOptions&, const ConvertOptions&,
                      std::shared_ptr<TableReader>* out);

cpp/src/arrow/extension_type.cc

@@ -138,4 +138,7 @@ std::shared_ptr<ExtensionType> GetExtensionType(const std::string& type_name) {
   return registry->GetType(type_name);
 }
 
+extern const char kExtensionTypeKeyName[] = "ARROW:extension:name";
+extern const char kExtensionMetadataKeyName[] = "ARROW:extension:metadata";
+
 }  // namespace arrow

cpp/src/arrow/extension_type.h

@@ -142,4 +142,7 @@ Status UnregisterExtensionType(const std::string& type_name);
 ARROW_EXPORT
 std::shared_ptr<ExtensionType> GetExtensionType(const std::string& type_name);
 
+ARROW_EXPORT extern const char kExtensionTypeKeyName[];
+ARROW_EXPORT extern const char kExtensionMetadataKeyName[];
+
 }  // namespace arrow

cpp/src/arrow/filesystem/CMakeLists.txt

@@ -20,6 +20,7 @@ arrow_install_all_headers("arrow/filesystem")
 
 add_arrow_test(filesystem_test)
 add_arrow_test(localfs_test)
+add_arrow_test(path_tree_test)
 
 if(ARROW_S3)
   add_arrow_test(s3fs_test)