From e2d6b78a60811562546e806d757b4e610c2fa971 Mon Sep 17 00:00:00 2001 From: Ma Xiaoyu Date: Thu, 19 Apr 2018 14:06:04 +0800 Subject: [PATCH 1/5] fix map table --- core/src/main/scala/org/apache/spark/sql/TiContext.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/sql/TiContext.scala b/core/src/main/scala/org/apache/spark/sql/TiContext.scala index 4cad1abb1a..1f5ed3a43f 100644 --- a/core/src/main/scala/org/apache/spark/sql/TiContext.scala +++ b/core/src/main/scala/org/apache/spark/sql/TiContext.scala @@ -129,13 +129,13 @@ class TiContext(val session: SparkSession) extends Serializable with Logging { } } - def tidbTable(dbName: String, tableName: String): DataFrame = { + def tidbTable(dbName: String, tableName: String): Unit = { val tiRelation = new TiDBRelation( tiSession, new TiTableReference(dbName, tableName), meta )(sqlContext) - sqlContext.baseRelationToDataFrame(tiRelation) + sqlContext.baseRelationToDataFrame(tiRelation).createTempView(tableName) } def tidbMapDatabase(dbName: String, dbNameAsPrefix: Boolean): Unit = { From 2ba0883b855de58be60463469648926f449bb686 Mon Sep 17 00:00:00 2001 From: Ma Xiaoyu Date: Thu, 19 Apr 2018 14:45:10 +0800 Subject: [PATCH 2/5] fix map single table --- core/src/main/scala/org/apache/spark/sql/TiContext.scala | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/src/main/scala/org/apache/spark/sql/TiContext.scala b/core/src/main/scala/org/apache/spark/sql/TiContext.scala index 541cc6bee1..848b5f19bd 100644 --- a/core/src/main/scala/org/apache/spark/sql/TiContext.scala +++ b/core/src/main/scala/org/apache/spark/sql/TiContext.scala @@ -129,6 +129,8 @@ class TiContext(val session: SparkSession) extends Serializable with Logging { } } + // tidbTable does not do any check any meta information + // it just register table for later use def tidbTable(dbName: String, tableName: String): Unit = { val tiRelation = new TiDBRelation( tiSession, From 81a5f22355d66809ed687248538e6b7d6db51532 Mon Sep 17 00:00:00 2001 From: Ma Xiaoyu Date: Thu, 19 Apr 2018 15:07:49 +0800 Subject: [PATCH 3/5] fix README.md --- README.md | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index b9d35ab967..1efa5f39e1 100755 --- a/README.md +++ b/README.md @@ -31,12 +31,33 @@ Uses as below import org.apache.spark.sql.TiContext val ti = new TiContext(spark) -// Mapping all TiDB tables from database tpch as Spark SQL tables +// Map all TiDB tables from database tpch as Spark SQL tables ti.tidbMapDatabase("tpch") spark.sql("select count(*) from lineitem").show ``` +## Metadata loading +If you are using spark-shell, you need to manually load schema information as decribed above. + +If you have too many table, you might choose to disable histogram prepare and loading will be faster. + +``` +ti.tidbMapDatabase("tpch", autoLoadStatistics = true) +``` + +If you have two tables with same name in different databases, you might choose to append database name as prefix for table name: + +``` +ti.tidbMapDatabase("tpch", dbNameAsPrefix = true) +``` + +If you have too many tables and use only some of them, to speed up meta loading process, you might manually load only tables you use: + +``` +ti.tidbTable("tpch", "lineitem") +``` + ## Current Version ``` ti.version From ee52ce9b7ff3dce260d0ded111e57869da8a382d Mon Sep 17 00:00:00 2001 From: Soup Date: Thu, 19 Apr 2018 15:10:07 +0800 Subject: [PATCH 4/5] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 1efa5f39e1..0baa7a509f 100755 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ spark.sql("select count(*) from lineitem").show ## Metadata loading If you are using spark-shell, you need to manually load schema information as decribed above. -If you have too many table, you might choose to disable histogram prepare and loading will be faster. +If you have too many tables, you might choose to disable histogram preparison and loading will be faster. ``` ti.tidbMapDatabase("tpch", autoLoadStatistics = true) From 3d65817f2c981988f81f1ce298be43d13ee8db72 Mon Sep 17 00:00:00 2001 From: Soup Date: Wed, 25 Apr 2018 16:40:40 +0800 Subject: [PATCH 5/5] Update TiContext.scala --- core/src/main/scala/org/apache/spark/sql/TiContext.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/sql/TiContext.scala b/core/src/main/scala/org/apache/spark/sql/TiContext.scala index 848b5f19bd..ad3112cb5c 100644 --- a/core/src/main/scala/org/apache/spark/sql/TiContext.scala +++ b/core/src/main/scala/org/apache/spark/sql/TiContext.scala @@ -129,9 +129,9 @@ class TiContext(val session: SparkSession) extends Serializable with Logging { } } - // tidbTable does not do any check any meta information + // tidbMapTable does not do any check any meta information // it just register table for later use - def tidbTable(dbName: String, tableName: String): Unit = { + def tidbMapTable(dbName: String, tableName: String): Unit = { val tiRelation = new TiDBRelation( tiSession, new TiTableReference(dbName, tableName),