PARQUET-446: Hide Thrift compiled headers and Boost from public API, …

…#include scrubbing

This is the completion of work I started in PARQUET-442. This also resolves PARQUET-277 as no boost headers are included in the public API anymore.

I've done some scrubbing of #includes using Google's Clang-based include-what-you-use tool. PARQUET-522 can also be resolved when this is merged.

Author: Wes McKinney <[email protected]>

Closes apache#49 from wesm/PARQUET-446 and squashes the following commits:

e805a0c [Wes McKinney] Use int64_t for scanner batch sizes
503b1c1 [Wes McKinney] Fix mixed-up include guard names
4c02d2b [Wes McKinney] Refactor monolithic encodings/encodings.h
9e28fc3 [Wes McKinney] Finished IWYU path. Some imported impala code left unchanged for now
6d4af8e [Wes McKinney] Some initial IWYU
5be40d6 [Wes McKinney] Remove outdated TODO
2e39062 [Wes McKinney] Remove any boost #include dependencies
07059ca [Wes McKinney] Remove serialized-page.* files, move serialized-page-test to parquet/file
9458b36 [Wes McKinney] Add more headers to parquet.h public API
b4b0412 [Wes McKinney] Remove Thrift compiled headers from public API and general use outside of deserialized-related internal headers and code paths. Add unit test to enforce this

Change-Id: Ib12b600cf6318e9da514b39fd8db2225bd6c9e09

cpp/src/parquet/CMakeLists.txt

@@ -22,4 +22,5 @@ install(FILES
   types.h
   DESTINATION include/parquet)
 
+ADD_PARQUET_TEST(public-api-test)
 ADD_PARQUET_TEST(reader-test)

cpp/src/parquet/column/CMakeLists.txt

@@ -20,10 +20,8 @@ install(FILES
   page.h
   levels.h
   reader.h
-  serialized-page.h
   scanner.h
   DESTINATION include/parquet/column)
 
 ADD_PARQUET_TEST(column-reader-test)
 ADD_PARQUET_TEST(levels-test)
-ADD_PARQUET_TEST(serialized-page-test)

cpp/src/parquet/column/column-reader-test.cc

@@ -15,10 +15,10 @@
 // specific language governing permissions and limitations
 // under the License.
 
+#include <algorithm>
 #include <cstdint>
 #include <cstdlib>
-#include <iostream>
-#include <sstream>
+#include <memory>
 #include <string>
 #include <vector>
 
@@ -28,15 +28,13 @@
 #include "parquet/column/page.h"
 #include "parquet/column/reader.h"
 #include "parquet/column/test-util.h"
-
-#include "parquet/util/output.h"
+#include "parquet/schema/descriptor.h"
+#include "parquet/schema/types.h"
 #include "parquet/util/test-common.h"
 
 using std::string;
 using std::vector;
 using std::shared_ptr;
-using parquet::FieldRepetitionType;
-using parquet::SchemaElement;
 
 namespace parquet_cpp {
 

cpp/src/parquet/column/levels-test.cc

@@ -15,98 +15,94 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include <cstdlib>
-#include <iostream>
-#include <sstream>
+#include <cstdint>
 #include <string>
+#include <vector>
 
 #include <gtest/gtest.h>
 
-#include "parquet/thrift/parquet_types.h"
 #include "parquet/column/levels.h"
+#include "parquet/types.h"
 
 using std::string;
 
 namespace parquet_cpp {
 
-class TestLevels : public ::testing::Test {
- public:
-  int GenerateLevels(int min_repeat_factor, int max_repeat_factor,
-      int max_level, std::vector<int16_t>& input_levels) {
-    int total_count = 0;
-    // for each repetition count upto max_repeat_factor
-    for (int repeat = min_repeat_factor; repeat <= max_repeat_factor; repeat++) {
-      // repeat count increase by a factor of 2 for every iteration
-      int repeat_count = (1 << repeat);
-      // generate levels for repetition count upto the maximum level
-      int value = 0;
-      int bwidth = 0;
-      while (value <= max_level) {
-        for (int i = 0; i < repeat_count; i++) {
-          input_levels[total_count++] = value;
-        }
-        value = (2 << bwidth) - 1;
-        bwidth++;
+int GenerateLevels(int min_repeat_factor, int max_repeat_factor,
+    int max_level, std::vector<int16_t>& input_levels) {
+  int total_count = 0;
+  // for each repetition count upto max_repeat_factor
+  for (int repeat = min_repeat_factor; repeat <= max_repeat_factor; repeat++) {
+    // repeat count increase by a factor of 2 for every iteration
+    int repeat_count = (1 << repeat);
+    // generate levels for repetition count upto the maximum level
+    int value = 0;
+    int bwidth = 0;
+    while (value <= max_level) {
+      for (int i = 0; i < repeat_count; i++) {
+        input_levels[total_count++] = value;
       }
+      value = (2 << bwidth) - 1;
+      bwidth++;
     }
-    return total_count;
   }
+  return total_count;
+}
 
-  void VerifyLevelsEncoding(parquet::Encoding::type encoding, int max_level,
-      std::vector<int16_t>& input_levels) {
-    LevelEncoder encoder;
-    LevelDecoder decoder;
-    int levels_count = 0;
-    std::vector<int16_t> output_levels;
-    std::vector<uint8_t> bytes;
-    int num_levels = input_levels.size();
-    output_levels.resize(num_levels);
-    bytes.resize(2 * num_levels);
-    ASSERT_EQ(num_levels, output_levels.size());
-    ASSERT_EQ(2 * num_levels, bytes.size());
-    // start encoding and decoding
-    if (encoding == parquet::Encoding::RLE) {
-      // leave space to write the rle length value
-      encoder.Init(encoding, max_level, num_levels,
-          bytes.data() + sizeof(uint32_t), bytes.size());
-
-      levels_count = encoder.Encode(num_levels, input_levels.data());
-      (reinterpret_cast<uint32_t*>(bytes.data()))[0] = encoder.len();
-
-    } else {
-      encoder.Init(encoding, max_level, num_levels,
-          bytes.data(), bytes.size());
-      levels_count = encoder.Encode(num_levels, input_levels.data());
-    }
+void VerifyLevelsEncoding(Encoding::type encoding, int max_level,
+    std::vector<int16_t>& input_levels) {
+  LevelEncoder encoder;
+  LevelDecoder decoder;
+  int levels_count = 0;
+  std::vector<int16_t> output_levels;
+  std::vector<uint8_t> bytes;
+  int num_levels = input_levels.size();
+  output_levels.resize(num_levels);
+  bytes.resize(2 * num_levels);
+  ASSERT_EQ(num_levels, output_levels.size());
+  ASSERT_EQ(2 * num_levels, bytes.size());
+  // start encoding and decoding
+  if (encoding == Encoding::RLE) {
+    // leave space to write the rle length value
+    encoder.Init(encoding, max_level, num_levels,
+        bytes.data() + sizeof(uint32_t), bytes.size());
+
+    levels_count = encoder.Encode(num_levels, input_levels.data());
+    (reinterpret_cast<uint32_t*>(bytes.data()))[0] = encoder.len();
+
+  } else {
+    encoder.Init(encoding, max_level, num_levels,
+        bytes.data(), bytes.size());
+    levels_count = encoder.Encode(num_levels, input_levels.data());
+  }
 
-    ASSERT_EQ(num_levels, levels_count);
+  ASSERT_EQ(num_levels, levels_count);
 
-    decoder.Init(encoding, max_level, num_levels, bytes.data());
-    levels_count = decoder.Decode(num_levels, output_levels.data());
+  decoder.Init(encoding, max_level, num_levels, bytes.data());
+  levels_count = decoder.Decode(num_levels, output_levels.data());
 
-    ASSERT_EQ(num_levels, levels_count);
+  ASSERT_EQ(num_levels, levels_count);
 
-    for (int i = 0; i < num_levels; i++) {
-      EXPECT_EQ(input_levels[i], output_levels[i]);
-    }
+  for (int i = 0; i < num_levels; i++) {
+    EXPECT_EQ(input_levels[i], output_levels[i]);
   }
-};
+}
+
+TEST(TestLevels, TestEncodeDecodeLevels) {
+  // test levels with maximum bit-width from 1 to 8
+  // increase the repetition count for each iteration by a factor of 2
 
-// test levels with maximum bit-width from 1 to 8
-// increase the repetition count for each iteration by a factor of 2
-TEST_F(TestLevels, TestEncodeDecodeLevels) {
   int min_repeat_factor = 0;
   int max_repeat_factor = 7; // 128
   int max_bit_width = 8;
   std::vector<int16_t> input_levels;
-  parquet::Encoding::type encodings[2] = {parquet::Encoding::RLE,
-      parquet::Encoding::BIT_PACKED};
+  Encoding::type encodings[2] = {Encoding::RLE, Encoding::BIT_PACKED};
 
   // for each encoding
   for (int encode = 0; encode < 2; encode++) {
-    parquet::Encoding::type encoding = encodings[encode];
+    Encoding::type encoding = encodings[encode];
     // BIT_PACKED requires a sequence of atleast 8
-    if (encoding == parquet::Encoding::BIT_PACKED) min_repeat_factor = 3;
+    if (encoding == Encoding::BIT_PACKED) min_repeat_factor = 3;
 
     // for each maximum bit-width
     for (int bit_width = 1; bit_width <= max_bit_width; bit_width++) {

cpp/src/parquet/column/levels.h

@@ -18,9 +18,10 @@
 #ifndef PARQUET_COLUMN_LEVELS_H
 #define PARQUET_COLUMN_LEVELS_H
 
+#include <memory>
+
 #include "parquet/exception.h"
-#include "parquet/thrift/parquet_types.h"
-#include "parquet/encodings/encodings.h"
+#include "parquet/types.h"
 #include "parquet/util/rle-encoding.h"
 
 namespace parquet_cpp {
@@ -30,16 +31,16 @@ class LevelEncoder {
   LevelEncoder() {}
 
   // Initialize the LevelEncoder.
-  void Init(parquet::Encoding::type encoding, int16_t max_level,
+  void Init(Encoding::type encoding, int16_t max_level,
       int num_buffered_values, uint8_t* data, int data_size) {
     bit_width_ = BitUtil::Log2(max_level + 1);
     encoding_ = encoding;
     switch (encoding) {
-      case parquet::Encoding::RLE: {
+      case Encoding::RLE: {
         rle_encoder_.reset(new RleEncoder(data, data_size, bit_width_));
         break;
       }
-      case parquet::Encoding::BIT_PACKED: {
+      case Encoding::BIT_PACKED: {
         int num_bytes = BitUtil::Ceil(num_buffered_values * bit_width_, 8);
         bit_packed_encoder_.reset(new BitWriter(data, num_bytes));
         break;
@@ -56,7 +57,7 @@ class LevelEncoder {
       throw ParquetException("Level encoders are not initialized.");
     }
 
-    if (encoding_ == parquet::Encoding::RLE) {
+    if (encoding_ == Encoding::RLE) {
       for (size_t i = 0; i < batch_size; ++i) {
         if (!rle_encoder_->Put(*(levels + i))) {
           break;
@@ -78,14 +79,16 @@ class LevelEncoder {
   }
 
   int32_t len() {
-    assert(encoding_ == parquet::Encoding::RLE);
+    if (encoding_ != Encoding::RLE) {
+      throw ParquetException("Only implemented for RLE encoding");
+    }
     return rle_length_;
   }
 
  private:
   int bit_width_;
   int rle_length_;
-  parquet::Encoding::type encoding_;
+  Encoding::type encoding_;
   std::unique_ptr<RleEncoder> rle_encoder_;
   std::unique_ptr<BitWriter> bit_packed_encoder_;
 };
@@ -96,20 +99,20 @@ class LevelDecoder {
   LevelDecoder() {}
 
   // Initialize the LevelDecoder and return the number of bytes consumed
-  size_t Init(parquet::Encoding::type encoding, int16_t max_level,
+  size_t Init(Encoding::type encoding, int16_t max_level,
       int num_buffered_values, const uint8_t* data) {
     uint32_t num_bytes = 0;
     uint32_t total_bytes = 0;
     bit_width_ = BitUtil::Log2(max_level + 1);
     encoding_ = encoding;
     switch (encoding) {
-      case parquet::Encoding::RLE: {
+      case Encoding::RLE: {
         num_bytes = *reinterpret_cast<const uint32_t*>(data);
         const uint8_t* decoder_data = data + sizeof(uint32_t);
         rle_decoder_.reset(new RleDecoder(decoder_data, num_bytes, bit_width_));
         return sizeof(uint32_t) + num_bytes;
       }
-      case parquet::Encoding::BIT_PACKED: {
+      case Encoding::BIT_PACKED: {
         num_bytes = BitUtil::Ceil(num_buffered_values * bit_width_, 8);
         bit_packed_decoder_.reset(new BitReader(data, num_bytes));
         return num_bytes;
@@ -127,7 +130,7 @@ class LevelDecoder {
       throw ParquetException("Level decoders are not initialized.");
     }
 
-    if (encoding_ == parquet::Encoding::RLE) {
+    if (encoding_ == Encoding::RLE) {
       for (size_t i = 0; i < batch_size; ++i) {
         if (!rle_decoder_->Get(levels + i)) {
           break;
@@ -147,7 +150,7 @@ class LevelDecoder {
 
  private:
   int bit_width_;
-  parquet::Encoding::type encoding_;
+  Encoding::type encoding_;
   std::unique_ptr<RleDecoder> rle_decoder_;
   std::unique_ptr<BitReader> bit_packed_decoder_;
 };