From 45ee8896adf1136afbff60c0dd096202a9628942 Mon Sep 17 00:00:00 2001 From: Jolanrensen Date: Tue, 7 Jun 2022 13:39:00 +0200 Subject: [PATCH] added hadoop file read fix --- .../org/jetbrains/kotlinx/spark/api/jupyter/SparkIntegration.kt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/jupyter/src/main/kotlin/org/jetbrains/kotlinx/spark/api/jupyter/SparkIntegration.kt b/jupyter/src/main/kotlin/org/jetbrains/kotlinx/spark/api/jupyter/SparkIntegration.kt index e817b282..eb3d73b2 100644 --- a/jupyter/src/main/kotlin/org/jetbrains/kotlinx/spark/api/jupyter/SparkIntegration.kt +++ b/jupyter/src/main/kotlin/org/jetbrains/kotlinx/spark/api/jupyter/SparkIntegration.kt @@ -42,6 +42,8 @@ internal class SparkIntegration : Integration() { .master(SparkConf().get("spark.master", "local[*]")) .appName("Jupyter") .config("spark.sql.codegen.wholeStage", false) + .config("fs.hdfs.impl", org.apache.hadoop.hdfs.DistributedFileSystem::class.java.name) + .config("fs.file.impl", org.apache.hadoop.fs.LocalFileSystem::class.java.name) .getOrCreate()""".trimIndent(), """ spark.sparkContext.setLogLevel(org.jetbrains.kotlinx.spark.api.SparkLogLevel.ERROR)""".trimIndent(),