diff --git a/sql-delta-import/src/main/scala/com/razorpay/spark/jdbc/JDBCImport.scala b/sql-delta-import/src/main/scala/com/razorpay/spark/jdbc/JDBCImport.scala index e5dd167d4..4e3839b1b 100644 --- a/sql-delta-import/src/main/scala/com/razorpay/spark/jdbc/JDBCImport.scala +++ b/sql-delta-import/src/main/scala/com/razorpay/spark/jdbc/JDBCImport.scala @@ -66,7 +66,6 @@ case class ImportConfig( inputTableEscaped = schema.get + "."+ inputTableEscaped } - val boundsSql: String = boundaryQuery.getOrElse( s"(select min($splitColumn) as min, max($splitColumn) as max from $inputTableEscaped) as bounds" ) @@ -91,7 +90,6 @@ class JDBCImport( import spark.implicits._ val logger: Logger = LoggerFactory.getLogger(this.getClass) - def createDbIfNotExists(outputDbName: String): Unit = { val s3Bucket = Credentials.getSecretValue("SQOOP_S3_BUCKET") val baseS3Path = s"s3a://$s3Bucket/sqoop" @@ -160,13 +158,11 @@ class JDBCImport( if (importConfig.splitBy.nonEmpty) { val defaultString = "0" val dbType = Credentials.getSecretValue(s"${databricksScope}_DB_TYPE") - val schema = importConfig.schema - var dbTable = importConfig.jdbcQuery + val dbTable = importConfig.jdbcQuery logger.error(s"JDBC 1: jdbcUrl $buildJdbcUrl and dbTable $dbTable") - val (lower, upper) = spark.read .jdbc(buildJdbcUrl, importConfig.boundsSql, jdbcParams) .selectExpr("cast(min as string) min", "cast(max as string) max") diff --git a/sql-delta-import/src/main/scala/com/razorpay/spark/jdbc/common/Constants.scala b/sql-delta-import/src/main/scala/com/razorpay/spark/jdbc/common/Constants.scala index d42693582..f82024df1 100644 --- a/sql-delta-import/src/main/scala/com/razorpay/spark/jdbc/common/Constants.scala +++ b/sql-delta-import/src/main/scala/com/razorpay/spark/jdbc/common/Constants.scala @@ -1,5 +1,7 @@ package com.razorpay.spark.jdbc.common +import scala.io.Source + object Constants { final val HUDI_DB_PREFIX = "realtime_" @@ -12,7 +14,10 @@ object Constants { // seconds final val QUERY_TIMEOUT = 10800 - final val CREDSTASH_TABLE_NAME = "credstash-prod-emr-sqoop" + val env_file_path = "/opt/env" + val env = Source.fromFile(env_file_path).getLines().mkString + + final val CREDSTASH_TABLE_NAME = s"credstash-$env-emr-sqoop" final val CREATED_DATE = "created_date" final val CREATED_AT = "created_at"