From 29489830249b4c039bece1b478273b12ab5b6bfe Mon Sep 17 00:00:00 2001 From: David Li Date: Tue, 24 Sep 2019 14:59:26 -0400 Subject: [PATCH] ARROW-6677: [FlightRPC][C++] Document Flight in C++ --- cpp/apidoc/Doxyfile | 1 + cpp/src/arrow/flight/server.h | 4 + cpp/src/arrow/flight/types.h | 45 ++++++----- cpp/src/arrow/status.h | 5 +- docs/source/cpp/api/flight.rst | 54 ++++++++++++- docs/source/cpp/api/support.rst | 5 +- docs/source/cpp/flight.rst | 119 ++++++++++++++++++++++++++++ docs/source/cpp/getting_started.rst | 1 + docs/source/format/Flight.rst | 46 ++++++++++- 9 files changed, 256 insertions(+), 24 deletions(-) create mode 100644 docs/source/cpp/flight.rst diff --git a/cpp/apidoc/Doxyfile b/cpp/apidoc/Doxyfile index 38ce17fb810cc..09ac5b9868517 100644 --- a/cpp/apidoc/Doxyfile +++ b/cpp/apidoc/Doxyfile @@ -2075,6 +2075,7 @@ INCLUDE_FILE_PATTERNS = PREDEFINED = __attribute__(x)= \ __declspec(x)= \ ARROW_EXPORT= \ + ARROW_FLIGHT_EXPORT= \ ARROW_EXTERN_TEMPLATE= \ ARROW_DEPRECATED(x)= diff --git a/cpp/src/arrow/flight/server.h b/cpp/src/arrow/flight/server.h index 352f4044d75fd..c2a3f4411d7ad 100644 --- a/cpp/src/arrow/flight/server.h +++ b/cpp/src/arrow/flight/server.h @@ -104,8 +104,12 @@ class ARROW_FLIGHT_EXPORT FlightServerOptions { public: explicit FlightServerOptions(const Location& location_); + /// \brief The host & port (or domain socket path) to listen on. + /// Use port 0 to bind to an available port. Location location; + /// \brief The authentication handler to use. std::unique_ptr auth_handler; + /// \brief A list of TLS certificate+key pairs to use. std::vector tls_certificates; /// \brief A Flight implementation-specific callback to customize /// transport-specific options. diff --git a/cpp/src/arrow/flight/types.h b/cpp/src/arrow/flight/types.h index 330c1231e7a0e..2aaa3d36877a1 100644 --- a/cpp/src/arrow/flight/types.h +++ b/cpp/src/arrow/flight/types.h @@ -74,7 +74,7 @@ enum class FlightStatusCode : int8_t { #pragma warning(disable : 4275) #endif -/// \brief Flight-specific information in a Status. +/// \brief Flight-specific error information in a Status. class ARROW_FLIGHT_EXPORT FlightStatusDetail : public arrow::StatusDetail { public: explicit FlightStatusDetail(FlightStatusCode code) : code_{code} {} @@ -101,7 +101,11 @@ class ARROW_FLIGHT_EXPORT FlightStatusDetail : public arrow::StatusDetail { #pragma warning(pop) #endif -/// \brief Make an appropriate Arrow status for the given Flight status. +/// \brief Make an appropriate Arrow status for the given +/// Flight-specific status. +/// +/// \param code The status code. +/// \param message The message for the error. ARROW_FLIGHT_EXPORT Status MakeFlightError(FlightStatusCode code, const std::string& message); @@ -114,16 +118,16 @@ struct ARROW_FLIGHT_EXPORT CertKeyPair { std::string pem_key; }; -/// \brief A type of action that can be performed with the DoAction RPC +/// \brief A type of action that can be performed with the DoAction RPC. struct ARROW_FLIGHT_EXPORT ActionType { - /// Name of action + /// \brief The name of the action. std::string type; - /// Opaque action description + /// \brief A human-readable description of the action. std::string description; }; -/// \brief Opaque selection critera for ListFlights RPC +/// \brief Opaque selection criteria for ListFlights RPC struct ARROW_FLIGHT_EXPORT Criteria { /// Opaque criteria expression, dependent on server implementation std::string expression; @@ -153,9 +157,6 @@ struct ARROW_FLIGHT_EXPORT BasicAuth { static Status Serialize(const BasicAuth& basic_auth, std::string* out); }; -/// \brief A message received after completing a DoPut stream -struct ARROW_FLIGHT_EXPORT PutResult {}; - /// \brief A request to retrieve or generate a dataset struct ARROW_FLIGHT_EXPORT FlightDescriptor { enum DescriptorType { @@ -406,26 +407,26 @@ class ARROW_FLIGHT_EXPORT FlightInfo { mutable bool reconstructed_schema_; }; -/// \brief An iterator to FlightInfo instances returned by ListFlights +/// \brief An iterator to FlightInfo instances returned by ListFlights. class ARROW_FLIGHT_EXPORT FlightListing { public: virtual ~FlightListing() = default; - /// \brief Retrieve the next FlightInfo from the iterator. Returns nullptr - /// when there are none left - /// \param[out] info a single FlightInfo + /// \brief Retrieve the next FlightInfo from the iterator. + /// \param[out] info A single FlightInfo. Set to \a nullptr if there + /// are none left. /// \return Status virtual Status Next(std::unique_ptr* info) = 0; }; -/// \brief An iterator to Result instances returned by DoAction +/// \brief An iterator to Result instances returned by DoAction. class ARROW_FLIGHT_EXPORT ResultStream { public: virtual ~ResultStream() = default; - /// \brief Retrieve the next Result from the iterator. Returns nullptr - /// when there are none left - /// \param[out] info a single Result + /// \brief Retrieve the next Result from the iterator. + /// \param[out] info A single result. Set to \a nullptr if there + /// are none left. /// \return Status virtual Status Next(std::unique_ptr* info) = 0; }; @@ -454,8 +455,10 @@ class ARROW_FLIGHT_EXPORT MetadataRecordBatchReader { virtual Status ReadAll(std::shared_ptr* table); }; -// \brief Create a FlightListing from a vector of FlightInfo objects. This can -// be iterated once, then it is consumed +/// \brief A FlightListing implementation based on a vector of +/// FlightInfo objects. +/// +/// This can be iterated once, then it is consumed. class ARROW_FLIGHT_EXPORT SimpleFlightListing : public FlightListing { public: explicit SimpleFlightListing(const std::vector& flights); @@ -468,6 +471,10 @@ class ARROW_FLIGHT_EXPORT SimpleFlightListing : public FlightListing { std::vector flights_; }; +/// \brief A ResultStream implementation based on a vector of +/// Result objects. +/// +/// This can be iterated once, then it is consumed. class ARROW_FLIGHT_EXPORT SimpleResultStream : public ResultStream { public: explicit SimpleResultStream(std::vector&& results); diff --git a/cpp/src/arrow/status.h b/cpp/src/arrow/status.h index 6a3269f6b2f29..f07d410eefaad 100644 --- a/cpp/src/arrow/status.h +++ b/cpp/src/arrow/status.h @@ -105,9 +105,10 @@ class ARROW_MUST_USE_RESULT ARROW_EXPORT Status; class ARROW_EXPORT StatusDetail { public: virtual ~StatusDetail() = default; - // Return a unique id for the type of the StatusDetail - // (effectively a poor man's substitude for RTTI). + /// \brief Return a unique id for the type of the StatusDetail + /// (effectively a poor man's substitude for RTTI). virtual const char* type_id() const = 0; + /// \brief Produce a human-readable description of this status. virtual std::string ToString() const = 0; }; diff --git a/docs/source/cpp/api/flight.rst b/docs/source/cpp/api/flight.rst index d226a1e5673df..7801d88f04f13 100644 --- a/docs/source/cpp/api/flight.rst +++ b/docs/source/cpp/api/flight.rst @@ -15,6 +15,9 @@ .. specific language governing permissions and limitations .. under the License. +.. default-domain:: cpp +.. highlight:: cpp + ================ Arrow Flight RPC ================ @@ -64,7 +67,7 @@ Common Types :project: arrow_cpp :members: -.. doxygenstruct:: arrow::flight::PutResult +.. doxygenclass:: arrow::flight::MetadataRecordBatchReader :project: arrow_cpp :members: @@ -87,6 +90,10 @@ Clients :project: arrow_cpp :members: +.. doxygenclass:: arrow::flight::FlightClientOptions + :project: arrow_cpp + :members: + .. doxygenclass:: arrow::flight::FlightCallOptions :project: arrow_cpp :members: @@ -98,6 +105,14 @@ Clients .. doxygentypedef:: arrow::flight::TimeoutDuration :project: arrow_cpp +.. doxygenclass:: arrow::flight::FlightStreamReader + :project: arrow_cpp + :members: + +.. doxygenclass:: arrow::flight::FlightStreamWriter + :project: arrow_cpp + :members: + Servers ======= @@ -105,6 +120,14 @@ Servers :project: arrow_cpp :members: +.. doxygenclass:: arrow::flight::FlightServerOptions + :project: arrow_cpp + :members: + +.. doxygenstruct:: arrow::flight::CertKeyPair + :project: arrow_cpp + :members: + .. doxygenclass:: arrow::flight::FlightDataStream :project: arrow_cpp :members: @@ -113,6 +136,10 @@ Servers :project: arrow_cpp :members: +.. doxygenclass:: arrow::flight::FlightMetadataWriter + :project: arrow_cpp + :members: + .. doxygenclass:: arrow::flight::RecordBatchStream :project: arrow_cpp :members: @@ -124,3 +151,28 @@ Servers .. doxygenclass:: arrow::flight::ServerCallContext :project: arrow_cpp :members: + +.. doxygenclass:: arrow::flight::SimpleFlightListing + :project: arrow_cpp + :members: + +.. doxygenclass:: arrow::flight::SimpleResultStream + :project: arrow_cpp + :members: + +Error Handling +============== + +Error handling uses the normal :class:`arrow::Status` class, combined +with a custom :class:`arrow::StatusDetail` object for Flight-specific +error codes. + +.. doxygenenum:: arrow::flight::FlightStatusCode + :project: arrow_cpp + +.. doxygenclass:: arrow::flight::FlightStatusDetail + :project: arrow_cpp + :members: + +.. doxygenfunction:: arrow::flight::MakeFlightError + :project: arrow_cpp diff --git a/docs/source/cpp/api/support.rst b/docs/source/cpp/api/support.rst index b165a9973b4c1..4749229b4583c 100644 --- a/docs/source/cpp/api/support.rst +++ b/docs/source/cpp/api/support.rst @@ -25,5 +25,8 @@ Error return and reporting :project: arrow_cpp :members: -.. doxygendefine:: ARROW_RETURN_NOT_OK +.. doxygenclass:: arrow::StatusDetail + :project: arrow_cpp + :members: +.. doxygendefine:: ARROW_RETURN_NOT_OK diff --git a/docs/source/cpp/flight.rst b/docs/source/cpp/flight.rst new file mode 100644 index 0000000000000..c1d2e43b9f454 --- /dev/null +++ b/docs/source/cpp/flight.rst @@ -0,0 +1,119 @@ +.. Licensed to the Apache Software Foundation (ASF) under one +.. or more contributor license agreements. See the NOTICE file +.. distributed with this work for additional information +.. regarding copyright ownership. The ASF licenses this file +.. to you under the Apache License, Version 2.0 (the +.. "License"); you may not use this file except in compliance +.. with the License. You may obtain a copy of the License at + +.. http://www.apache.org/licenses/LICENSE-2.0 + +.. Unless required by applicable law or agreed to in writing, +.. software distributed under the License is distributed on an +.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +.. KIND, either express or implied. See the License for the +.. specific language governing permissions and limitations +.. under the License. + +.. default-domain:: cpp +.. highlight:: cpp + +================ +Arrow Flight RPC +================ + +Arrow Flight is an RPC framework for efficient transfer of Flight data +over the network. See :doc:`../format/Flight` for full details on +the protocol, or :doc:`./api/flight` for API docs. + +Writing a Flight Service +======================== + +Servers are subclasses of :class:`arrow::flight::FlightServerBase`. To +implement individual RPCs, override the RPC methods on this class. + +.. code-block:: cpp + + class MyFlightServer : public FlightServerBase { + Status ListFlights(const ServerCallContext& context, const Criteria* criteria, + std::unique_ptr* listings) override { + std::vector flights = ...; + *listings = std::unique_ptr(new SimpleFlightListing(flights)); + return Status::OK(); + } + }; + +Each RPC method always takes a +:class:`arrow::flight::ServerCallContext` for common parameters and +returns a :class:`arrow::Status` to indicate success or +failure. Flight-specific error codes can be returned via +:func:`arrow::flight::MakeFlightError`. + +RPC methods that return a value in addition to a status will use an +out parameter, as shown above. Often, there are helper classes +providing basic implementations of these out parameters. For instance, +above, :class:`arrow::flight::SimpleFlightListing` uses a vector of +:class:`arrow::flight::FlightInfo` objects as the result of a +``ListFlights`` RPC. + +To start a server, create a :class:`arrow::flight::Location` to +specify where to listen, and call +:func:`arrow::flight::FlightServerBase::Init`. This will start the +server, but won't block the rest of the program. Use +:func:`arrow::flight::FlightServerBase::SetShutdownOnSignals` to +enable stopping the server if an interrupt signal is received, then +call :func:`arrow::flight::FlightServerBase::Serve` to block until the +server stops. + +.. code-block:: cpp + + std::unique_ptr server; + // Initialize server + arrow::flight::Location location; + // Listen to all interfaces on a free port + ARROW_CHECK_OK(arrow::flight::Location::ForGrpcTcp("0.0.0.0", 0, &location)); + arrow::flight::FlightServerOptions options(location); + + // Start the server + ARROW_CHECK_OK(server->Init(options)); + // Exit with a clean error code (0) on SIGTERM + ARROW_CHECK_OK(server->SetShutdownOnSignals({SIGTERM})); + + std::cout << "Server listening on localhost:" << server->port() << std::endl; + ARROW_CHECK_OK(server->Serve()); + + +Enabling TLS and Authentication +------------------------------- + +TLS can be enabled by providing a certificate and key pair to +:func:`FlightServerBase::Init +`. Additionally, use +:func:`Location::ForGrpcTls ` to +construct the :class:`arrow::flight::Location` to listen on. + +Similarly, authentication can be enabled by providing an +implementation of :class:`ServerAuthHandler +`. Authentication consists of two +parts: on initial client connection, the server and client +authentication implementations can perform any negotiation needed; +then, on each RPC thereafter, the client provides a token. The server +authentication handler validates the token and provides the identity +of the client. This identity can be obtained from the +:class:`arrow::flight::ServerCallContext`. + +Using the Flight Client +======================= + +To connect to a Flight service, create an instance of +:class:`arrow::flight::FlightClient` by calling :func:`Connect +`. This takes a Location and +returns the client through an out parameter. To authenticate, call +:func:`Authenticate ` with +the desired client authentication implementation. + +Each RPC method returns :class:`arrow::Status` to indicate the +success/failure of the request. Any other return values are specified +through out parameters. They also take an optional :class:`options +` parameter that allows specifying a +timeout for the call. diff --git a/docs/source/cpp/getting_started.rst b/docs/source/cpp/getting_started.rst index 0f43a61c5c8f9..4c918b3d96c30 100644 --- a/docs/source/cpp/getting_started.rst +++ b/docs/source/cpp/getting_started.rst @@ -30,3 +30,4 @@ Getting Started datatypes tables io + flight diff --git a/docs/source/format/Flight.rst b/docs/source/format/Flight.rst index b3476eadf335c..7ffa243012826 100644 --- a/docs/source/format/Flight.rst +++ b/docs/source/format/Flight.rst @@ -78,7 +78,7 @@ See `Protocol Buffer Definitions`_ for full details on the methods and messages involved. Authentication -~~~~~~~~~~~~~~ +-------------- Flight supports application-implemented authentication methods. Authentication, if enabled, has two phases: at connection @@ -92,6 +92,50 @@ call, or they may establish trust during the handshake and not validate a token for each call. (Note that the latter is not secure if you choose to deploy a layer 7 load balancer, as is common with gRPC.) +Error Handling +-------------- + +Arrow Flight defines its own set of error codes. The implementation +differs between languages (e.g. in C++, Unimplemented is a general +Arrow error status while it's a Flight-specific exception in Java), +but the following set is exposed: + ++----------------+-------------------------------------------+ +|Error Code |Description | ++================+===========================================+ +|UNKNOWN |An unknown error. The default if no other | +| |error applies. | ++----------------+-------------------------------------------+ +|INTERNAL |An error internal to the service | +| |implementation occurred. | ++----------------+-------------------------------------------+ +|INVALID_ARGUMENT|The client passed an invalid argument to | +| |the RPC. | ++----------------+-------------------------------------------+ +|TIMED_OUT |The operation exceeded a timeout or | +| |deadline. | ++----------------+-------------------------------------------+ +|NOT_FOUND |The requested resource (action, data | +| |stream) was not found. | ++----------------+-------------------------------------------+ +|ALREADY_EXISTS |The resource already exists. | ++----------------+-------------------------------------------+ +|CANCELLED |The operation was cancelled (either by the | +| |client or the server). | ++----------------+-------------------------------------------+ +|UNAUTHENTICATED |The client is not authenticated. | ++----------------+-------------------------------------------+ +|UNAUTHORIZED |The client is authenticated, but does not | +| |have permissions for the requested | +| |operation. | ++----------------+-------------------------------------------+ +|UNIMPLEMENTED |The RPC is not implemented. | ++----------------+-------------------------------------------+ +|UNAVAILABLE |The server is not available. May be emitted| +| |by the client for connectivity reasons. | ++----------------+-------------------------------------------+ + + External Resources ------------------