Skip to content

Commit

Permalink
add env support for sqoop jobs
Browse files Browse the repository at this point in the history
  • Loading branch information
akulgoel96 committed Dec 20, 2024
1 parent b4f1c26 commit 3e247f9
Show file tree
Hide file tree
Showing 2 changed files with 7 additions and 6 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,6 @@ case class ImportConfig(
inputTableEscaped = schema.get + "."+ inputTableEscaped
}


val boundsSql: String = boundaryQuery.getOrElse(
s"(select min($splitColumn) as min, max($splitColumn) as max from $inputTableEscaped) as bounds"
)
Expand All @@ -91,7 +90,6 @@ class JDBCImport(
import spark.implicits._
val logger: Logger = LoggerFactory.getLogger(this.getClass)


def createDbIfNotExists(outputDbName: String): Unit = {
val s3Bucket = Credentials.getSecretValue("SQOOP_S3_BUCKET")
val baseS3Path = s"s3a://$s3Bucket/sqoop"
Expand Down Expand Up @@ -160,13 +158,11 @@ class JDBCImport(
if (importConfig.splitBy.nonEmpty) {
val defaultString = "0"
val dbType = Credentials.getSecretValue(s"${databricksScope}_DB_TYPE")
val schema = importConfig.schema

var dbTable = importConfig.jdbcQuery
val dbTable = importConfig.jdbcQuery

logger.error(s"JDBC 1: jdbcUrl $buildJdbcUrl and dbTable $dbTable")


val (lower, upper) = spark.read
.jdbc(buildJdbcUrl, importConfig.boundsSql, jdbcParams)
.selectExpr("cast(min as string) min", "cast(max as string) max")
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
package com.razorpay.spark.jdbc.common

import scala.io.Source

object Constants {
final val HUDI_DB_PREFIX = "realtime_"

Expand All @@ -12,7 +14,10 @@ object Constants {
// seconds
final val QUERY_TIMEOUT = 10800

final val CREDSTASH_TABLE_NAME = "credstash-prod-emr-sqoop"
val env_file_path = "/opt/env"
val env = Source.fromFile(env_file_path).getLines().mkString

final val CREDSTASH_TABLE_NAME = s"credstash-$env-emr-sqoop"

final val CREATED_DATE = "created_date"
final val CREATED_AT = "created_at"
Expand Down

0 comments on commit 3e247f9

Please sign in to comment.