From cc69e1b4df09dd9f17e7aca088894f12319595c6 Mon Sep 17 00:00:00 2001 From: Deepak Majeti Date: Tue, 27 Aug 2024 10:39:24 -0400 Subject: [PATCH 1/3] Remove SeekableInputStream.h include from FormatData.h --- velox/dwio/common/FormatData.h | 1 - 1 file changed, 1 deletion(-) diff --git a/velox/dwio/common/FormatData.h b/velox/dwio/common/FormatData.h index d0d37006a2a8..700903bc2e64 100644 --- a/velox/dwio/common/FormatData.h +++ b/velox/dwio/common/FormatData.h @@ -18,7 +18,6 @@ #include "velox/common/memory/Memory.h" #include "velox/dwio/common/ScanSpec.h" -#include "velox/dwio/common/SeekableInputStream.h" #include "velox/dwio/common/Statistics.h" #include "velox/dwio/common/TypeWithId.h" #include "velox/type/Filter.h" From b6097ab32c71a10ee273b7be3b60994dea9f1ecc Mon Sep 17 00:00:00 2001 From: Deepak Majeti Date: Tue, 27 Aug 2024 11:22:36 -0400 Subject: [PATCH 2/3] forward declare PositionProvider --- velox/dwio/common/FormatData.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/velox/dwio/common/FormatData.h b/velox/dwio/common/FormatData.h index 700903bc2e64..435c24cc6719 100644 --- a/velox/dwio/common/FormatData.h +++ b/velox/dwio/common/FormatData.h @@ -23,6 +23,8 @@ #include "velox/type/Filter.h" namespace facebook::velox::dwio::common { +/// Defined in SeekableInputStream.h +class PositionProvider; /// Interface base class for format-specific state in common between different /// file format readers. @@ -39,7 +41,7 @@ class FormatData { /// data. If there are no nulls, 'nulls' is set to nullptr, else to /// a suitable sized and padded Buffer. 'incomingNulls' may be given /// if there are enclosing level nulls that should be merged into - /// the read reasult. If provided, this has 'numValues' bits and + /// the read result. If provided, this has 'numValues' bits and /// each zero marks an incoming null for which no bit is read from /// the nulls stream of 'this'. For Parquet, 'nulls' is always set /// to nullptr because nulls are represented by the data pages From 1062c20b79dd97ce86cfd3af6a03ba017bb4f840 Mon Sep 17 00:00:00 2001 From: Deepak Majeti Date: Tue, 27 Aug 2024 12:15:51 -0400 Subject: [PATCH 3/3] add PositionProvide --- velox/dwio/common/Closeable.h | 10 ++----- velox/dwio/common/FormatData.h | 3 +- velox/dwio/common/PositionProvider.h | 37 +++++++++++++++++++++++++ velox/dwio/common/SeekableInputStream.h | 17 +----------- 4 files changed, 41 insertions(+), 26 deletions(-) create mode 100644 velox/dwio/common/PositionProvider.h diff --git a/velox/dwio/common/Closeable.h b/velox/dwio/common/Closeable.h index 4572e08cf98a..bdb825437889 100644 --- a/velox/dwio/common/Closeable.h +++ b/velox/dwio/common/Closeable.h @@ -18,10 +18,7 @@ #include "velox/dwio/common/exception/Exception.h" -namespace facebook { -namespace velox { -namespace dwio { -namespace common { +namespace facebook::velox::dwio::common { // Base class for closeable object which need to be explicitly closed before // being destructed @@ -67,7 +64,4 @@ class Closeable { bool closed_; }; -} // namespace common -} // namespace dwio -} // namespace velox -} // namespace facebook +} // namespace facebook::velox::dwio::common diff --git a/velox/dwio/common/FormatData.h b/velox/dwio/common/FormatData.h index 435c24cc6719..bff3c420bc50 100644 --- a/velox/dwio/common/FormatData.h +++ b/velox/dwio/common/FormatData.h @@ -17,14 +17,13 @@ #pragma once #include "velox/common/memory/Memory.h" +#include "velox/dwio/common/PositionProvider.h" #include "velox/dwio/common/ScanSpec.h" #include "velox/dwio/common/Statistics.h" #include "velox/dwio/common/TypeWithId.h" #include "velox/type/Filter.h" namespace facebook::velox::dwio::common { -/// Defined in SeekableInputStream.h -class PositionProvider; /// Interface base class for format-specific state in common between different /// file format readers. diff --git a/velox/dwio/common/PositionProvider.h b/velox/dwio/common/PositionProvider.h new file mode 100644 index 000000000000..7be3bc7a1602 --- /dev/null +++ b/velox/dwio/common/PositionProvider.h @@ -0,0 +1,37 @@ +/* + * Copyright (c) Facebook, Inc. and its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +namespace facebook::velox::dwio::common { + +class PositionProvider { + public: + explicit PositionProvider(const std::vector& positions) + : position_{positions.begin()}, end_{positions.end()} {} + + uint64_t next(); + + bool hasNext() const; + + private: + std::vector::const_iterator position_; + std::vector::const_iterator end_; +}; + +} // namespace facebook::velox::dwio::common diff --git a/velox/dwio/common/SeekableInputStream.h b/velox/dwio/common/SeekableInputStream.h index 021d78493ab6..ab402753b8c5 100644 --- a/velox/dwio/common/SeekableInputStream.h +++ b/velox/dwio/common/SeekableInputStream.h @@ -16,30 +16,15 @@ #pragma once -#include - #include "velox/dwio/common/DataBuffer.h" #include "velox/dwio/common/InputStream.h" +#include "velox/dwio/common/PositionProvider.h" #include "velox/dwio/common/wrap/zero-copy-stream-wrapper.h" namespace facebook::velox::dwio::common { void printBuffer(std::ostream& out, const char* buffer, uint64_t length); -class PositionProvider { - public: - explicit PositionProvider(const std::vector& positions) - : position_{positions.begin()}, end_{positions.end()} {} - - uint64_t next(); - - bool hasNext() const; - - private: - std::vector::const_iterator position_; - std::vector::const_iterator end_; -}; - /** * A subclass of Google's ZeroCopyInputStream that supports seek. * By extending Google's class, we get the ability to pass it directly