-
Notifications
You must be signed in to change notification settings - Fork 3.6k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
ARROW-67: C++ metadata flatbuffer serialization and data movement to …
…memory maps Several things here: * Add Google flatbuffers dependency * Flatbuffers IDL draft in collaboration with @jacques-n and @StevenMPhillips * Add Schema wrapper in Cython * arrow::Schema conversion to/from flatbuffer representation * Remove unneeded physical layout types from type.h * Refactor ListType to be a nested type with a single child * Implement shared memory round-trip for numeric row batches * mmap-based shared memory interface and MemorySource abstract API Quite a bit of judicious code cleaning and consolidation as part of this. For example, List types are now internally equivalent to a nested type with 1 named child field (versus a struct, which can have any number of child fields). Associated JIRAs: ARROW-48, ARROW-57, ARROW-58 Author: Wes McKinney <[email protected]> Closes #28 from wesm/cpp-ipc-draft and squashes the following commits: 0cef7ea [Wes McKinney] Add NullArray type now that Array is virtual, fix pyarrow build 5e841f7 [Wes McKinney] Create explicit PrimitiveArray subclasses to avoid unwanted template instantiation 6fa6319 [Wes McKinney] ARROW-28: Draft C++ shared memory IPC workflow and related refactoring / scaffolding / cleaning.
- Loading branch information
Showing
88 changed files
with
3,113 additions
and
838 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,95 @@ | ||
| # | ||
| # Licensed under the Apache License, Version 2.0 (the "License"); | ||
| # you may not use this file except in compliance with the License. | ||
| # You may obtain a copy of the License at | ||
| # | ||
| # http://www.apache.org/licenses/LICENSE-2.0 | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, software | ||
| # distributed under the License is distributed on an "AS IS" BASIS, | ||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| # See the License for the specific language governing permissions and | ||
| # limitations under the License. | ||
| # | ||
| # Tries to find Flatbuffers headers and libraries. | ||
| # | ||
| # Usage of this module as follows: | ||
| # | ||
| # find_package(Flatbuffers) | ||
| # | ||
| # Variables used by this module, they can change the default behaviour and need | ||
| # to be set before calling find_package: | ||
| # | ||
| # Flatbuffers_HOME - | ||
| # When set, this path is inspected instead of standard library locations as | ||
| # the root of the Flatbuffers installation. The environment variable | ||
| # FLATBUFFERS_HOME overrides this veriable. | ||
| # | ||
| # This module defines | ||
| # FLATBUFFERS_INCLUDE_DIR, directory containing headers | ||
| # FLATBUFFERS_LIBS, directory containing flatbuffers libraries | ||
| # FLATBUFFERS_STATIC_LIB, path to libflatbuffers.a | ||
| # FLATBUFFERS_FOUND, whether flatbuffers has been found | ||
|
|
||
| if( NOT "$ENV{FLATBUFFERS_HOME}" STREQUAL "") | ||
| file( TO_CMAKE_PATH "$ENV{FLATBUFFERS_HOME}" _native_path ) | ||
| list( APPEND _flatbuffers_roots ${_native_path} ) | ||
| elseif ( Flatbuffers_HOME ) | ||
| list( APPEND _flatbuffers_roots ${Flatbuffers_HOME} ) | ||
| endif() | ||
|
|
||
| # Try the parameterized roots, if they exist | ||
| if ( _flatbuffers_roots ) | ||
| find_path( FLATBUFFERS_INCLUDE_DIR NAMES flatbuffers/flatbuffers.h | ||
| PATHS ${_flatbuffers_roots} NO_DEFAULT_PATH | ||
| PATH_SUFFIXES "include" ) | ||
| find_library( FLATBUFFERS_LIBRARIES NAMES flatbuffers | ||
| PATHS ${_flatbuffers_roots} NO_DEFAULT_PATH | ||
| PATH_SUFFIXES "lib" ) | ||
| else () | ||
| find_path( FLATBUFFERS_INCLUDE_DIR NAMES flatbuffers/flatbuffers.h ) | ||
| find_library( FLATBUFFERS_LIBRARIES NAMES flatbuffers ) | ||
| endif () | ||
|
|
||
| find_program(FLATBUFFERS_COMPILER flatc | ||
| $ENV{FLATBUFFERS_HOME}/bin | ||
| /usr/local/bin | ||
| /usr/bin | ||
| NO_DEFAULT_PATH | ||
| ) | ||
|
|
||
| if (FLATBUFFERS_INCLUDE_DIR AND FLATBUFFERS_LIBRARIES) | ||
| set(FLATBUFFERS_FOUND TRUE) | ||
| get_filename_component( FLATBUFFERS_LIBS ${FLATBUFFERS_LIBRARIES} PATH ) | ||
| set(FLATBUFFERS_LIB_NAME libflatbuffers) | ||
| set(FLATBUFFERS_STATIC_LIB ${FLATBUFFERS_LIBS}/${FLATBUFFERS_LIB_NAME}.a) | ||
| else () | ||
| set(FLATBUFFERS_FOUND FALSE) | ||
| endif () | ||
|
|
||
| if (FLATBUFFERS_FOUND) | ||
| if (NOT Flatbuffers_FIND_QUIETLY) | ||
| message(STATUS "Found the Flatbuffers library: ${FLATBUFFERS_LIBRARIES}") | ||
| endif () | ||
| else () | ||
| if (NOT Flatbuffers_FIND_QUIETLY) | ||
| set(FLATBUFFERS_ERR_MSG "Could not find the Flatbuffers library. Looked in ") | ||
| if ( _flatbuffers_roots ) | ||
| set(FLATBUFFERS_ERR_MSG "${FLATBUFFERS_ERR_MSG} in ${_flatbuffers_roots}.") | ||
| else () | ||
| set(FLATBUFFERS_ERR_MSG "${FLATBUFFERS_ERR_MSG} system search paths.") | ||
| endif () | ||
| if (Flatbuffers_FIND_REQUIRED) | ||
| message(FATAL_ERROR "${FLATBUFFERS_ERR_MSG}") | ||
| else (Flatbuffers_FIND_REQUIRED) | ||
| message(STATUS "${FLATBUFFERS_ERR_MSG}") | ||
| endif (Flatbuffers_FIND_REQUIRED) | ||
| endif () | ||
| endif () | ||
|
|
||
| mark_as_advanced( | ||
| FLATBUFFERS_INCLUDE_DIR | ||
| FLATBUFFERS_LIBS | ||
| FLATBUFFERS_STATIC_LIB | ||
| FLATBUFFERS_COMPILER | ||
| ) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.