Fix the comparator of nonkeyColumn and Modify the extended DataType

Huawei-Spark · Jan 21, 2015 · 47d8337 · 47d8337
1 parent 9474e0f
commit 47d8337
Show file tree

Hide file tree

Showing 4 changed files with 9 additions and 6 deletions.
diff --git a/sql/hbase/src/main/scala/org/apache/spark/sql/hbase/HBaseRelation.scala b/sql/hbase/src/main/scala/org/apache/spark/sql/hbase/HBaseRelation.scala
@@ -104,7 +104,8 @@ private[hbase] case class HBaseRelation(
     .asInstanceOf[Seq[NonKeyColumn]].sortWith(
       (a: NonKeyColumn, b: NonKeyColumn) => {
         val empty = new HBaseRawType(0)
-        KeyValue.COMPARATOR.compareRows(new KeyValue(empty, a.familyRaw, a.qualifierRaw, empty),
+        KeyValue.COMPARATOR.compare(
+          new KeyValue(empty, a.familyRaw, a.qualifierRaw, empty),
           new KeyValue(empty, b.familyRaw, b.qualifierRaw, empty)) < 0
       }
     )

diff --git a/sql/hbase/src/main/scala/org/apache/spark/sql/hbase/types/HBaseBytesType.scala b/sql/hbase/src/main/scala/org/apache/spark/sql/hbase/types/HBaseBytesType.scala
@@ -24,8 +24,10 @@ import scala.reflect.runtime.universe.typeTag
 /**
  * Almost identical to BinaryType except for a different ordering to be consistent
  * with that of HBase's internal ordering
+ * This is a data type for Low-Level HBase entities.
+ * It should not be used in High-Level processing
  */
-case object HBaseBytesType extends NativeType with PrimitiveType {
+private[hbase] case object HBaseBytesType extends NativeType with PrimitiveType {
   override def defaultSize: Int = 4096
   private[sql] type JvmType = Array[Byte]
   // TODO: can not use ScalaReflectionLock now for its accessibility

diff --git a/sql/hbase/src/main/scala/org/apache/spark/sql/hbase/types/RangeType.scala b/sql/hbase/src/main/scala/org/apache/spark/sql/hbase/types/RangeType.scala
@@ -56,7 +56,7 @@ class PartitionRange[T](start: Option[T], startInclusive: Boolean,
                         val id: Int, dt: NativeType, var pred: Expression)
   extends Range[T](start, startInclusive, end, endInclusive, dt)
 
-class RangeType[T] extends PartialOrderingDataType {
+private[hbase] class RangeType[T] extends PartialOrderingDataType {
   override def defaultSize: Int = 4096
   private[sql] type JvmType = Range[T]
   // TODO: can not use ScalaReflectionLock now for its accessibility

diff --git a/sql/hbase/src/test/scala/org/apache/spark/sql/hbase/CatalogTestSuite.scala b/sql/hbase/src/test/scala/org/apache/spark/sql/hbase/CatalogTestSuite.scala
@@ -79,13 +79,13 @@ class CatalogTestSuite extends HBaseIntegrationTestBase {
     // check the data type
     assert(result.keyColumns(0).dataType === StringType)
     assert(result.keyColumns(1).dataType === IntegerType)
-    assert(result.nonKeyColumns(0).dataType === FloatType)
-    assert(result.nonKeyColumns(1).dataType === BooleanType)
+    assert(result.nonKeyColumns(1).dataType === FloatType)
+    assert(result.nonKeyColumns(0).dataType === BooleanType)
 
     val relation = catalog.lookupRelation(Seq(tableName))
     val subquery = relation.asInstanceOf[Subquery]
     val hbRelation = subquery.child.asInstanceOf[LogicalRelation].relation.asInstanceOf[HBaseRelation]
-    assert(hbRelation.nonKeyColumns.map(_.family) == List("family2", "family1"))
+    assert(hbRelation.nonKeyColumns.map(_.family) == List("family1", "family2"))
     val keyColumns = Seq(KeyColumn("column1", StringType, 0), KeyColumn("column2", IntegerType, 1))
     assert(hbRelation.keyColumns.equals(keyColumns))
     assert(relation.childrenResolved)