From e9fa255271fa140015f11632cdd58489ef9c4130 Mon Sep 17 00:00:00 2001 From: James Dunkerley Date: Mon, 22 Jul 2024 12:39:32 +0100 Subject: [PATCH 01/35] Add Tableau --- build.sbt | 42 +++++++++++++++++++++++++++++++++++++----- project/Editions.scala | 3 ++- 2 files changed, 39 insertions(+), 6 deletions(-) diff --git a/build.sbt b/build.sbt index 8874af612d53..4fbbedffef22 100644 --- a/build.sbt +++ b/build.sbt @@ -163,11 +163,9 @@ GatherLicenses.distributions := Seq( makeStdLibDistribution("Database", Distribution.sbtProjects(`std-database`)), makeStdLibDistribution("Image", Distribution.sbtProjects(`std-image`)), makeStdLibDistribution("AWS", Distribution.sbtProjects(`std-aws`)), - makeStdLibDistribution( - "Snowflake", - Distribution.sbtProjects(`std-snowflake`) - ), - makeStdLibDistribution("Microsoft", Distribution.sbtProjects(`std-microsoft`)) + makeStdLibDistribution("Snowflake", Distribution.sbtProjects(`std-snowflake`)), + makeStdLibDistribution("Microsoft", Distribution.sbtProjects(`std-microsoft`)), + makeStdLibDistribution("Tableau", Distribution.sbtProjects(`std-tableau`)) ) GatherLicenses.licenseConfigurations := Set("compile") @@ -3269,6 +3267,8 @@ val `std-snowflake-polyglot-root` = stdLibComponentRoot("Snowflake") / "polyglot" / "java" val `std-microsoft-polyglot-root` = stdLibComponentRoot("Microsoft") / "polyglot" / "java" +val `std-tableau-polyglot-root` = + stdLibComponentRoot("Tableau") / "polyglot" / "java" lazy val `std-base` = project .in(file("std-bits") / "base") @@ -3606,6 +3606,35 @@ lazy val `std-microsoft` = project .dependsOn(`std-table` % "provided") .dependsOn(`std-database` % "provided") +lazy val `std-tableau` = project + .in(file("std-bits") / "tableau") + .settings( + frgaalJavaCompilerSetting, + autoScalaLibrary := false, + Compile / compile / compileInputs := (Compile / compile / compileInputs) + .dependsOn(SPIHelpers.ensureSPIConsistency) + .value, + Compile / packageBin / artifactPath := + `std-tableau-polyglot-root` / "std-tableau.jar", + libraryDependencies ++= Seq( + "org.netbeans.api" % "org-openide-util-lookup" % netbeansApiVersion % "provided", + "net.snowflake" % "snowflake-jdbc" % snowflakeJDBCVersion + ), + Compile / packageBin := Def.task { + val result = (Compile / packageBin).value + val _ = StdBits + .copyDependencies( + `std-tableau-polyglot-root`, + Seq("std-tableau.jar"), + ignoreScalaLibrary = true + ) + .value + result + }.value + ) + .dependsOn(`std-base` % "provided") + .dependsOn(`std-table` % "provided") + /* Note [Native Image Workaround for GraalVM 20.2] * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ * In GraalVM 20.2 the Native Image build of even simple Scala programs has @@ -3826,6 +3855,8 @@ pkgStdLibInternal := Def.inputTask { (`std-snowflake` / Compile / packageBin).value case "Microsoft" => (`std-microsoft` / Compile / packageBin).value + case "Tableau" => + (`std-tableau` / Compile / packageBin).value case _ if buildAllCmd => (`std-base` / Compile / packageBin).value (`enso-test-java-helpers` / Compile / packageBin).value @@ -3838,6 +3869,7 @@ pkgStdLibInternal := Def.inputTask { (`std-aws` / Compile / packageBin).value (`std-snowflake` / Compile / packageBin).value (`std-microsoft` / Compile / packageBin).value + (`std-tableau` / Compile / packageBin).value case _ => } val libs = diff --git a/project/Editions.scala b/project/Editions.scala index 859bbf3e92fa..83c6bbfde1d0 100644 --- a/project/Editions.scala +++ b/project/Editions.scala @@ -20,7 +20,8 @@ object Editions { "Standard.Searcher", "Standard.Google_Api", "Standard.Snowflake", - "Standard.Microsoft" + "Standard.Microsoft", + "Standard.Tableau" ) case class ContribLibrary(name: String, version: String) From d8136f41cdd69015ed8f21f9861fe85788cbeea2 Mon Sep 17 00:00:00 2001 From: James Dunkerley Date: Mon, 22 Jul 2024 13:05:19 +0100 Subject: [PATCH 02/35] Building up the bits and bobs. --- build.sbt | 4 ---- .../lib/Standard/Tableau/0.0.0-dev/package.yaml | 10 ++++++++++ .../lib/Standard/Tableau/0.0.0-dev/src/Hyper_API.enso | 8 ++++++++ .../lib/Standard/Tableau/0.0.0-dev/src/Main.enso | 3 +++ 4 files changed, 21 insertions(+), 4 deletions(-) create mode 100644 distribution/lib/Standard/Tableau/0.0.0-dev/package.yaml create mode 100644 distribution/lib/Standard/Tableau/0.0.0-dev/src/Hyper_API.enso create mode 100644 distribution/lib/Standard/Tableau/0.0.0-dev/src/Main.enso diff --git a/build.sbt b/build.sbt index 4fbbedffef22..734c573fa271 100644 --- a/build.sbt +++ b/build.sbt @@ -3616,10 +3616,6 @@ lazy val `std-tableau` = project .value, Compile / packageBin / artifactPath := `std-tableau-polyglot-root` / "std-tableau.jar", - libraryDependencies ++= Seq( - "org.netbeans.api" % "org-openide-util-lookup" % netbeansApiVersion % "provided", - "net.snowflake" % "snowflake-jdbc" % snowflakeJDBCVersion - ), Compile / packageBin := Def.task { val result = (Compile / packageBin).value val _ = StdBits diff --git a/distribution/lib/Standard/Tableau/0.0.0-dev/package.yaml b/distribution/lib/Standard/Tableau/0.0.0-dev/package.yaml new file mode 100644 index 000000000000..d757c865dabe --- /dev/null +++ b/distribution/lib/Standard/Tableau/0.0.0-dev/package.yaml @@ -0,0 +1,10 @@ +name: Tableau +namespace: Standard +version: 0.0.0-dev +license: APLv2 +authors: + - name: Enso Team + email: contact@enso.org +maintainers: + - name: Enso Team + email: contact@enso.org diff --git a/distribution/lib/Standard/Tableau/0.0.0-dev/src/Hyper_API.enso b/distribution/lib/Standard/Tableau/0.0.0-dev/src/Hyper_API.enso new file mode 100644 index 000000000000..8957179f04c2 --- /dev/null +++ b/distribution/lib/Standard/Tableau/0.0.0-dev/src/Hyper_API.enso @@ -0,0 +1,8 @@ +from Standard.Base import all + +type Hyper_API + ## A representation of a Tableau Hyper Extract file. + private Value file:File + + say_hello : Text + say_hello self = "Hello, World!" diff --git a/distribution/lib/Standard/Tableau/0.0.0-dev/src/Main.enso b/distribution/lib/Standard/Tableau/0.0.0-dev/src/Main.enso new file mode 100644 index 000000000000..f4eb5f999b9e --- /dev/null +++ b/distribution/lib/Standard/Tableau/0.0.0-dev/src/Main.enso @@ -0,0 +1,3 @@ +from Standard.Base import all + +export project.Hyper_API.Hyper_API From 143c2bdab2cabe6d82bac79fb06e2452b278304b Mon Sep 17 00:00:00 2001 From: James Dunkerley Date: Wed, 24 Jul 2024 10:57:16 +0100 Subject: [PATCH 03/35] Pull hyperd. --- build.sbt | 5 ++ .../Tableau/0.0.0-dev/src/Hyper_API.enso | 1 + .../distribution/DistributionManager.scala | 2 +- .../java/org/enso/tableau/HyperReader.java | 77 +++++++++++++++++++ .../java/org/enso/tableau/OSPlatform.java | 30 ++++++++ .../org/enso/tableau/TableauFormatSPI.java | 6 ++ 6 files changed, 120 insertions(+), 1 deletion(-) create mode 100644 std-bits/tableau/src/main/java/org/enso/tableau/HyperReader.java create mode 100644 std-bits/tableau/src/main/java/org/enso/tableau/OSPlatform.java create mode 100644 std-bits/tableau/src/main/java/org/enso/tableau/TableauFormatSPI.java diff --git a/build.sbt b/build.sbt index 734c573fa271..ba517e9a8ca7 100644 --- a/build.sbt +++ b/build.sbt @@ -568,6 +568,7 @@ val apacheArrowVersion = "14.0.1" val snowflakeJDBCVersion = "3.15.0" val mssqlserverJDBCVersion = "12.6.2.jre11" val jsoniterVersion = "2.28.5" +val jnaVersion = "5.14.0" // ============================================================================ // === Utility methods ===================================================== @@ -3616,6 +3617,10 @@ lazy val `std-tableau` = project .value, Compile / packageBin / artifactPath := `std-tableau-polyglot-root` / "std-tableau.jar", + libraryDependencies ++= Seq( + "org.netbeans.api" % "org-openide-util-lookup" % netbeansApiVersion % "provided", + "net.java.dev.jna" % "jna-platform" % jnaVersion, + ), Compile / packageBin := Def.task { val result = (Compile / packageBin).value val _ = StdBits diff --git a/distribution/lib/Standard/Tableau/0.0.0-dev/src/Hyper_API.enso b/distribution/lib/Standard/Tableau/0.0.0-dev/src/Hyper_API.enso index 8957179f04c2..354f305e2a3b 100644 --- a/distribution/lib/Standard/Tableau/0.0.0-dev/src/Hyper_API.enso +++ b/distribution/lib/Standard/Tableau/0.0.0-dev/src/Hyper_API.enso @@ -1,5 +1,6 @@ from Standard.Base import all + type Hyper_API ## A representation of a Tableau Hyper Extract file. private Value file:File diff --git a/lib/scala/distribution-manager/src/main/scala/org/enso/distribution/DistributionManager.scala b/lib/scala/distribution-manager/src/main/scala/org/enso/distribution/DistributionManager.scala index 640697816221..51df5f65f5d7 100644 --- a/lib/scala/distribution-manager/src/main/scala/org/enso/distribution/DistributionManager.scala +++ b/lib/scala/distribution-manager/src/main/scala/org/enso/distribution/DistributionManager.scala @@ -292,7 +292,7 @@ class DistributionManager(val env: Environment) { private val LINUX_ENSO_DIRECTORY = "enso" private val MACOS_ENSO_DIRECTORY = "org.enso" - private val WINDOWS_ENSO_DIRECTORY = "enso" + private val WINDOWS_ENSO_DIRECTORY = "enso" /** Data directory for an installed distribution. */ diff --git a/std-bits/tableau/src/main/java/org/enso/tableau/HyperReader.java b/std-bits/tableau/src/main/java/org/enso/tableau/HyperReader.java new file mode 100644 index 000000000000..7b9d6839a464 --- /dev/null +++ b/std-bits/tableau/src/main/java/org/enso/tableau/HyperReader.java @@ -0,0 +1,77 @@ +package org.enso.tableau; + +import com.tableau.hyperapi.HyperProcess; +import com.tableau.hyperapi.Telemetry; + +import java.io.FileOutputStream; +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.channels.Channels; +import java.nio.file.Files; +import java.nio.file.Path; + +public class HyperReader { + private static final Path HYPER_PATH = Path.of(getHyperPath()); + private static HyperProcess process; + + private static String getHyperPath() { + if (System.getenv("HYPER_PATH") != null) { + return System.getenv("HYPER_PATH"); + } if (System.getenv("ENSO_DATA_DIRECTORY") != null) { + return System.getenv("ENSO_DATA_DIRECTORY") + "/hyper"; + } else { + return switch (OSPlatform.CurrentPlatform) { + case WINDOWS -> System.getenv("LocalAppData") + "/enso/hyper"; + case MAC_ARM64, MAX_X64 -> System.getProperty("user.home") + "/Library/Application Support/org.enso/hyper"; + case LINUX, OTHER -> System.getProperty("user.home") + "/.local/share/enso/hyper"; + }; + } + } + + private static HyperProcess getProcess() throws IOException, URISyntaxException { + // Check if the hyper directory exists, if not create it. + if (!Files.exists(HYPER_PATH)) { + try { + Files.createDirectories(HYPER_PATH); + } catch (Exception e) { + throw new IOException("Failed to create Hyper directory: " + HYPER_PATH, e); + } + } + + // Check if any files in the hyper directory, otherwise download them. + try (var files = Files.list(HYPER_PATH)) { + if (files.findAny().isEmpty()) { + switch (OSPlatform.CurrentPlatform) { + case WINDOWS -> + downloadHyper("https://enso-data-samples.s3.us-west-1.amazonaws.com/tableau/hyperd.exe", "hyperd.exe"); + case MAC_ARM64 -> + downloadHyper("https://enso-data-samples.s3.us-west-1.amazonaws.com/tableau/macos-arm64/hyperd", "hyperd"); + case MAX_X64 -> + throw new IOException("Unsupported platform: Only ARM64 Mac is supported."); + case LINUX -> + downloadHyper("https://enso-data-samples.s3.us-west-1.amazonaws.com/tableau/linux/hyperd", "hyperd"); + case OTHER -> + throw new IOException("Unsupported platform: " + OSPlatform.CurrentPlatform); + }; + } + } catch (Exception e) { + throw new IOException("Failed to download hyperd.", e); + } + + // Start hyper process. + if (process == null) { + process = new HyperProcess(HYPER_PATH, Telemetry.DO_NOT_SEND_USAGE_DATA_TO_TABLEAU); + } + return process; + } + + private static void downloadHyper(String uri, String fileName) throws IOException, URISyntaxException { + var url = new URI(uri); + var readChannel = Channels.newChannel(url.toURL().openStream()); + try (var fos = new FileOutputStream(HYPER_PATH.resolve(fileName).toString())) { + var writeChannel = fos.getChannel(); + writeChannel.transferFrom(readChannel, 0, Long.MAX_VALUE); + } + } +} diff --git a/std-bits/tableau/src/main/java/org/enso/tableau/OSPlatform.java b/std-bits/tableau/src/main/java/org/enso/tableau/OSPlatform.java new file mode 100644 index 000000000000..75529b29741e --- /dev/null +++ b/std-bits/tableau/src/main/java/org/enso/tableau/OSPlatform.java @@ -0,0 +1,30 @@ +package org.enso.tableau; + +public enum OSPlatform { + WINDOWS, + MAC_ARM64, + MAX_X64, + LINUX, + OTHER; + + //** Returns the current platform. */ + public static final OSPlatform CurrentPlatform = getPlatform(); + + private static OSPlatform getPlatform() { + var osName = System.getProperty("os.name").toUpperCase(); + if (osName.contains("WIN")) { + return OSPlatform.WINDOWS; + } else if (osName.contains("MAC")) { + var osArch = System.getProperty("os.arch").toUpperCase(); + if (osArch.contains("ARM64") || osArch.contains("AARCH64")) { + return OSPlatform.MAC_ARM64; + } else { + return OSPlatform.MAX_X64; + } + } else if (osName.contains("LINUX")) { + return OSPlatform.LINUX; + } else { + return OSPlatform.OTHER; + } + } +} diff --git a/std-bits/tableau/src/main/java/org/enso/tableau/TableauFormatSPI.java b/std-bits/tableau/src/main/java/org/enso/tableau/TableauFormatSPI.java new file mode 100644 index 000000000000..b7165505312d --- /dev/null +++ b/std-bits/tableau/src/main/java/org/enso/tableau/TableauFormatSPI.java @@ -0,0 +1,6 @@ +package src.main.java.org.enso.tableau; + +import org.enso.base.file_format.FileFormatSPI; + +public class TableauFormatSPI { +} From 5d5bf3650ca3d150dabbc9788ca849afa73c03f4 Mon Sep 17 00:00:00 2001 From: James Dunkerley Date: Mon, 29 Jul 2024 14:59:08 +0100 Subject: [PATCH 04/35] First test of Tableau. --- build.sbt | 2 ++ .../Tableau/0.0.0-dev/src/Hyper_API.enso | 21 +++++++++--- .../Standard/Tableau/0.0.0-dev/src/Main.enso | 2 +- .../java/org/enso/tableau/HyperReader.java | 32 +++++++++++++++++-- 4 files changed, 50 insertions(+), 7 deletions(-) diff --git a/build.sbt b/build.sbt index ba517e9a8ca7..c65058451197 100644 --- a/build.sbt +++ b/build.sbt @@ -349,6 +349,7 @@ lazy val enso = (project in file(".")) `std-aws`, `std-snowflake`, `std-microsoft`, + `std-tableau`, `http-test-helper`, `enso-test-java-helpers`, `exploratory-benchmark-java-helpers`, @@ -1948,6 +1949,7 @@ lazy val runtime = (project in file("engine/runtime")) .dependsOn(`std-aws` / Compile / packageBin) .dependsOn(`std-snowflake` / Compile / packageBin) .dependsOn(`std-microsoft` / Compile / packageBin) + .dependsOn(`std-tableau` / Compile / packageBin) .value ) .dependsOn(`common-polyglot-core-utils`) diff --git a/distribution/lib/Standard/Tableau/0.0.0-dev/src/Hyper_API.enso b/distribution/lib/Standard/Tableau/0.0.0-dev/src/Hyper_API.enso index 354f305e2a3b..130bb498b1e7 100644 --- a/distribution/lib/Standard/Tableau/0.0.0-dev/src/Hyper_API.enso +++ b/distribution/lib/Standard/Tableau/0.0.0-dev/src/Hyper_API.enso @@ -1,9 +1,22 @@ from Standard.Base import all +polyglot java import org.enso.tableau.HyperReader -type Hyper_API - ## A representation of a Tableau Hyper Extract file. +type Hyper_File + ## Creates a Hyper_File + new : File -> Hyper_File + new file:File = Hyper_File.Value file + + ## Gets the location for hyperd + hyper_path : Text + hyper_path = HyperReader.HYPER_PATH + + ## PRIVATE + A representation of a Tableau Hyper Extract file. private Value file:File - say_hello : Text - say_hello self = "Hello, World!" + ## ICON metadata + Returns the list of schemas for the connection within the current database (or catalog). + schemas : Vector Text + schemas self = + HyperReader.readSchemas self.file.path diff --git a/distribution/lib/Standard/Tableau/0.0.0-dev/src/Main.enso b/distribution/lib/Standard/Tableau/0.0.0-dev/src/Main.enso index f4eb5f999b9e..e08ef514b9c0 100644 --- a/distribution/lib/Standard/Tableau/0.0.0-dev/src/Main.enso +++ b/distribution/lib/Standard/Tableau/0.0.0-dev/src/Main.enso @@ -1,3 +1,3 @@ from Standard.Base import all -export project.Hyper_API.Hyper_API +export project.Hyper_API.Hyper_File diff --git a/std-bits/tableau/src/main/java/org/enso/tableau/HyperReader.java b/std-bits/tableau/src/main/java/org/enso/tableau/HyperReader.java index 7b9d6839a464..82bbff7c2f3b 100644 --- a/std-bits/tableau/src/main/java/org/enso/tableau/HyperReader.java +++ b/std-bits/tableau/src/main/java/org/enso/tableau/HyperReader.java @@ -1,6 +1,8 @@ package org.enso.tableau; +import com.tableau.hyperapi.Connection; import com.tableau.hyperapi.HyperProcess; +import com.tableau.hyperapi.SchemaName; import com.tableau.hyperapi.Telemetry; import java.io.FileOutputStream; @@ -10,11 +12,15 @@ import java.nio.channels.Channels; import java.nio.file.Files; import java.nio.file.Path; +import java.util.logging.Level; +import java.util.logging.Logger; public class HyperReader { - private static final Path HYPER_PATH = Path.of(getHyperPath()); + public static final Path HYPER_PATH = Path.of(getHyperPath()); private static HyperProcess process; + private static final Logger LOGGER = Logger.getLogger("enso-hyper-reader"); + private static String getHyperPath() { if (System.getenv("HYPER_PATH") != null) { return System.getenv("HYPER_PATH"); @@ -29,7 +35,7 @@ private static String getHyperPath() { } } - private static HyperProcess getProcess() throws IOException, URISyntaxException { + private static HyperProcess getProcess() throws IOException { // Check if the hyper directory exists, if not create it. if (!Files.exists(HYPER_PATH)) { try { @@ -61,12 +67,14 @@ private static HyperProcess getProcess() throws IOException, URISyntaxException // Start hyper process. if (process == null) { + LOGGER.log(Level.INFO, "Starting Hyper process: " + HYPER_PATH.toString() + "."); process = new HyperProcess(HYPER_PATH, Telemetry.DO_NOT_SEND_USAGE_DATA_TO_TABLEAU); } return process; } private static void downloadHyper(String uri, String fileName) throws IOException, URISyntaxException { + LOGGER.log(Level.INFO, "Downloading Hyper from: " + uri); var url = new URI(uri); var readChannel = Channels.newChannel(url.toURL().openStream()); try (var fos = new FileOutputStream(HYPER_PATH.resolve(fileName).toString())) { @@ -74,4 +82,24 @@ private static void downloadHyper(String uri, String fileName) throws IOExceptio writeChannel.transferFrom(readChannel, 0, Long.MAX_VALUE); } } + + public static String[] readSchemas(String path) throws IOException { + try (var process = getProcess()) { + try (var connection = new Connection(process.getEndpoint(), path)) { + var catalog = connection.getCatalog(); + return catalog.getSchemaNames().stream().map(s -> s.getName().getUnescaped()).toArray(String[]::new); + } + } + } + + public static String addSchema(String path, String schema) throws IOException { + var schemaName = new SchemaName(schema); + try (var process = getProcess()) { + try (var connection = new Connection(process.getEndpoint(), path)) { + var catalog = connection.getCatalog(); + catalog.createSchema(schemaName); + return schemaName.getName().getUnescaped(); + } + } + } } From 297ca24fdab6fe7b676e4426d665ed8d450f5c21 Mon Sep 17 00:00:00 2001 From: Hubert Plociniczak Date: Mon, 29 Jul 2024 22:56:56 +0200 Subject: [PATCH 05/35] Fix integration with HyperAPI The main problem was that Classloader used during HyperAPI was incorrect We should be using the one that loaded the class, which was a rather straightforward change. Additionally we must set correct permissions on `hyperd`. sbt will attempt to download necessary dependenciess to `unmanagedClasspath` if jars are missing. --- build.sbt | 55 +++++++++++++++- project/StdBits.scala | 5 +- .../java/org/enso/tableau/HyperReader.java | 64 +++++++++++++------ .../java/org/enso/tableau/OSPlatform.java | 2 +- .../org/enso/tableau/TableauFormatSPI.java | 5 +- 5 files changed, 102 insertions(+), 29 deletions(-) diff --git a/build.sbt b/build.sbt index c65058451197..6a49cfb60988 100644 --- a/build.sbt +++ b/build.sbt @@ -1,10 +1,10 @@ import LibraryManifestGenerator.BundledLibrary -import org.enso.build.BenchTasks._ +import org.enso.build.BenchTasks.* import org.enso.build.WithDebugCommand import org.apache.commons.io.FileUtils import sbt.Keys.{libraryDependencies, scalacOptions} import sbt.addCompilerPlugin -import sbt.complete.DefaultParsers._ +import sbt.complete.DefaultParsers.* import sbt.complete.Parser import sbt.nio.file.FileTreeView import sbt.internal.util.ManagedLogger @@ -13,11 +13,14 @@ import src.main.scala.licenses.{ SBTDistributionComponent } +import java.nio.file.Files + // This import is unnecessary, but bit adds a proper code completion features // to IntelliJ. import JPMSPlugin.autoImport._ import java.io.File +import java.nio.file.Files import java.nio.file.Paths // ============================================================================ @@ -525,6 +528,7 @@ val poiOoxmlVersion = "5.2.3" val redshiftVersion = "2.1.0.15" val univocityParsersVersion = "2.9.1" val xmlbeansVersion = "5.1.1" +val tableauVersion = "0.0.19691.r2d7e5bc8" // === ZIO ==================================================================== @@ -3614,14 +3618,55 @@ lazy val `std-tableau` = project .settings( frgaalJavaCompilerSetting, autoScalaLibrary := false, + unmanagedExternalZip := { + val platform = if (Platform.isWindows) { + "windows" + } else if (Platform.isMacOS) { + "macos" + } else if (Platform.isLinux) { + "linux" + } + val arch = if (Platform.isArm64) { + "arm64" + } else { + "x86_64" + } + new URI( + s"https://downloads.tableau.com/tssoftware//tableauhyperapi-java-$platform-$arch-release-main.$tableauVersion.zip" + ).toURL() + }, + fetchZipToUnmanaged := { + val unmanagedDirectory = (Compile / unmanagedBase).value + if (IO.listFiles(unmanagedDirectory).size < 2) { // Heuristic, should have at least hyperapi jar and os-specific one. + unmanagedDirectory.mkdirs() + val unmanagedPath = unmanagedDirectory.toPath + IO.withTemporaryDirectory( + tmp => { + val files = IO.unzipURL( + unmanagedExternalZip.value, + tmp, + f => + f.endsWith(".jar") && !f.contains("gradle") && !f.contains( + "javadoc" + ) && !f.contains("jna") + ) + files.map(f => IO.move(f, unmanagedPath.resolve(f.getName).toFile)) + }, + keepDirectory = false + ) + } + }, Compile / compile / compileInputs := (Compile / compile / compileInputs) .dependsOn(SPIHelpers.ensureSPIConsistency) .value, + Compile / unmanagedClasspath := (Compile / unmanagedClasspath) + .dependsOn(fetchZipToUnmanaged) + .value, Compile / packageBin / artifactPath := `std-tableau-polyglot-root` / "std-tableau.jar", libraryDependencies ++= Seq( "org.netbeans.api" % "org-openide-util-lookup" % netbeansApiVersion % "provided", - "net.java.dev.jna" % "jna-platform" % jnaVersion, + "net.java.dev.jna" % "jna-platform" % jnaVersion ), Compile / packageBin := Def.task { val result = (Compile / packageBin).value @@ -3638,6 +3683,10 @@ lazy val `std-tableau` = project .dependsOn(`std-base` % "provided") .dependsOn(`std-table` % "provided") +lazy val fetchZipToUnmanaged = + taskKey[Unit]("Download zip file from a given url and unpack jars to ") +lazy val unmanagedExternalZip = settingKey[URL]("URL to unmanaged file") + /* Note [Native Image Workaround for GraalVM 20.2] * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ * In GraalVM 20.2 the Native Image build of even simple Scala programs has diff --git a/project/StdBits.scala b/project/StdBits.scala index 4184e3591cc8..2e34470e3b84 100644 --- a/project/StdBits.scala +++ b/project/StdBits.scala @@ -48,14 +48,14 @@ object StdBits { !graalVmOrgs.contains(orgName) }) ) + val unmanagedFiles = (Compile / unmanagedJars).value.map(_.data) val relevantFiles = libraryUpdates .select( configuration = configFilter, module = graalModuleFilter, artifact = DependencyFilter.artifactFilter() - ) - + ) ++ unmanagedFiles val dependencyStore = streams.value.cacheStoreFactory.make("std-bits-dependencies") Tracked.diffInputs(dependencyStore, FileInfo.hash)(relevantFiles.toSet) { @@ -131,4 +131,5 @@ object StdBits { log.info(s"No changes detected for '$name' package") } } + } diff --git a/std-bits/tableau/src/main/java/org/enso/tableau/HyperReader.java b/std-bits/tableau/src/main/java/org/enso/tableau/HyperReader.java index 82bbff7c2f3b..6a6f6bede785 100644 --- a/std-bits/tableau/src/main/java/org/enso/tableau/HyperReader.java +++ b/std-bits/tableau/src/main/java/org/enso/tableau/HyperReader.java @@ -4,7 +4,6 @@ import com.tableau.hyperapi.HyperProcess; import com.tableau.hyperapi.SchemaName; import com.tableau.hyperapi.Telemetry; - import java.io.FileOutputStream; import java.io.IOException; import java.net.URI; @@ -24,12 +23,14 @@ public class HyperReader { private static String getHyperPath() { if (System.getenv("HYPER_PATH") != null) { return System.getenv("HYPER_PATH"); - } if (System.getenv("ENSO_DATA_DIRECTORY") != null) { + } + if (System.getenv("ENSO_DATA_DIRECTORY") != null) { return System.getenv("ENSO_DATA_DIRECTORY") + "/hyper"; } else { return switch (OSPlatform.CurrentPlatform) { case WINDOWS -> System.getenv("LocalAppData") + "/enso/hyper"; - case MAC_ARM64, MAX_X64 -> System.getProperty("user.home") + "/Library/Application Support/org.enso/hyper"; + case MAC_ARM64, MAX_X64 -> System.getProperty("user.home") + + "/Library/Application Support/org.enso/hyper"; case LINUX, OTHER -> System.getProperty("user.home") + "/.local/share/enso/hyper"; }; } @@ -49,17 +50,24 @@ private static HyperProcess getProcess() throws IOException { try (var files = Files.list(HYPER_PATH)) { if (files.findAny().isEmpty()) { switch (OSPlatform.CurrentPlatform) { - case WINDOWS -> - downloadHyper("https://enso-data-samples.s3.us-west-1.amazonaws.com/tableau/hyperd.exe", "hyperd.exe"); - case MAC_ARM64 -> - downloadHyper("https://enso-data-samples.s3.us-west-1.amazonaws.com/tableau/macos-arm64/hyperd", "hyperd"); - case MAX_X64 -> - throw new IOException("Unsupported platform: Only ARM64 Mac is supported."); - case LINUX -> - downloadHyper("https://enso-data-samples.s3.us-west-1.amazonaws.com/tableau/linux/hyperd", "hyperd"); - case OTHER -> - throw new IOException("Unsupported platform: " + OSPlatform.CurrentPlatform); - }; + case WINDOWS -> downloadHyper( + "https://enso-data-samples.s3.us-west-1.amazonaws.com/tableau/hyperd.exe", + "hyperd.exe", + false); + case MAC_ARM64 -> downloadHyper( + "https://enso-data-samples.s3.us-west-1.amazonaws.com/tableau/macos-arm64/hyperd", + "hyperd", + true); + case MAX_X64 -> throw new IOException( + "Unsupported platform: Only ARM64 Mac is supported."); + case LINUX -> downloadHyper( + "https://enso-data-samples.s3.us-west-1.amazonaws.com/tableau/linux/hyperd", + "hyperd", + true); + case OTHER -> throw new IOException( + "Unsupported platform: " + OSPlatform.CurrentPlatform); + } + ; } } catch (Exception e) { throw new IOException("Failed to download hyperd.", e); @@ -67,27 +75,45 @@ private static HyperProcess getProcess() throws IOException { // Start hyper process. if (process == null) { - LOGGER.log(Level.INFO, "Starting Hyper process: " + HYPER_PATH.toString() + "."); - process = new HyperProcess(HYPER_PATH, Telemetry.DO_NOT_SEND_USAGE_DATA_TO_TABLEAU); + var contextClassLoader = Thread.currentThread().getContextClassLoader(); + try { + Thread.currentThread().setContextClassLoader(HyperReader.class.getClassLoader()); + LOGGER.log(Level.INFO, "Starting Hyper process: " + HYPER_PATH + "."); + try { + process = new HyperProcess(HYPER_PATH, Telemetry.DO_NOT_SEND_USAGE_DATA_TO_TABLEAU); + } catch (Throwable ioe) { + ioe.printStackTrace(); + throw ioe; + } + } finally { + Thread.currentThread().setContextClassLoader(contextClassLoader); + } } return process; } - private static void downloadHyper(String uri, String fileName) throws IOException, URISyntaxException { + private static void downloadHyper(String uri, String fileName, boolean setExecutable) + throws IOException, URISyntaxException { LOGGER.log(Level.INFO, "Downloading Hyper from: " + uri); + var hyperdFile = HYPER_PATH.resolve(fileName).toFile(); var url = new URI(uri); var readChannel = Channels.newChannel(url.toURL().openStream()); - try (var fos = new FileOutputStream(HYPER_PATH.resolve(fileName).toString())) { + try (var fos = new FileOutputStream(hyperdFile)) { var writeChannel = fos.getChannel(); writeChannel.transferFrom(readChannel, 0, Long.MAX_VALUE); } + if (setExecutable) { + hyperdFile.setExecutable(true); + } } public static String[] readSchemas(String path) throws IOException { try (var process = getProcess()) { try (var connection = new Connection(process.getEndpoint(), path)) { var catalog = connection.getCatalog(); - return catalog.getSchemaNames().stream().map(s -> s.getName().getUnescaped()).toArray(String[]::new); + return catalog.getSchemaNames().stream() + .map(s -> s.getName().getUnescaped()) + .toArray(String[]::new); } } } diff --git a/std-bits/tableau/src/main/java/org/enso/tableau/OSPlatform.java b/std-bits/tableau/src/main/java/org/enso/tableau/OSPlatform.java index 75529b29741e..c1b8593848dd 100644 --- a/std-bits/tableau/src/main/java/org/enso/tableau/OSPlatform.java +++ b/std-bits/tableau/src/main/java/org/enso/tableau/OSPlatform.java @@ -7,7 +7,7 @@ public enum OSPlatform { LINUX, OTHER; - //** Returns the current platform. */ + // ** Returns the current platform. */ public static final OSPlatform CurrentPlatform = getPlatform(); private static OSPlatform getPlatform() { diff --git a/std-bits/tableau/src/main/java/org/enso/tableau/TableauFormatSPI.java b/std-bits/tableau/src/main/java/org/enso/tableau/TableauFormatSPI.java index b7165505312d..4063b2368113 100644 --- a/std-bits/tableau/src/main/java/org/enso/tableau/TableauFormatSPI.java +++ b/std-bits/tableau/src/main/java/org/enso/tableau/TableauFormatSPI.java @@ -1,6 +1,3 @@ package src.main.java.org.enso.tableau; -import org.enso.base.file_format.FileFormatSPI; - -public class TableauFormatSPI { -} +public class TableauFormatSPI {} From d20ae0b6c7c2cc39464f600b7f9654e3a2a6b0cf Mon Sep 17 00:00:00 2001 From: Hubert Plociniczak Date: Mon, 29 Jul 2024 23:04:50 +0200 Subject: [PATCH 06/35] nit --- build.sbt | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/build.sbt b/build.sbt index 6a49cfb60988..1f63585bbbdd 100644 --- a/build.sbt +++ b/build.sbt @@ -1,10 +1,10 @@ import LibraryManifestGenerator.BundledLibrary -import org.enso.build.BenchTasks.* +import org.enso.build.BenchTasks._ import org.enso.build.WithDebugCommand import org.apache.commons.io.FileUtils import sbt.Keys.{libraryDependencies, scalacOptions} import sbt.addCompilerPlugin -import sbt.complete.DefaultParsers.* +import sbt.complete.DefaultParsers._ import sbt.complete.Parser import sbt.nio.file.FileTreeView import sbt.internal.util.ManagedLogger @@ -3684,8 +3684,11 @@ lazy val `std-tableau` = project .dependsOn(`std-table` % "provided") lazy val fetchZipToUnmanaged = - taskKey[Unit]("Download zip file from a given url and unpack jars to ") -lazy val unmanagedExternalZip = settingKey[URL]("URL to unmanaged file") + taskKey[Unit]( + "Download zip file from an `unmanagedExternalZip` url and unpack jars to unmanaged libs directory" + ) +lazy val unmanagedExternalZip = + settingKey[URL]("URL to zip file with dependencies") /* Note [Native Image Workaround for GraalVM 20.2] * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ From b2dd092bdf90f98eb4622b90444b5430cb76efe4 Mon Sep 17 00:00:00 2001 From: James Dunkerley Date: Wed, 31 Jul 2024 12:30:27 +0100 Subject: [PATCH 07/35] First functional bits. --- build.sbt | 2 - .../Tableau/0.0.0-dev/src/Hyper_API.enso | 25 +++++++- .../java/org/enso/tableau/HyperReader.java | 62 ++++++++++++++----- 3 files changed, 68 insertions(+), 21 deletions(-) diff --git a/build.sbt b/build.sbt index 1f63585bbbdd..46475613eba7 100644 --- a/build.sbt +++ b/build.sbt @@ -13,8 +13,6 @@ import src.main.scala.licenses.{ SBTDistributionComponent } -import java.nio.file.Files - // This import is unnecessary, but bit adds a proper code completion features // to IntelliJ. import JPMSPlugin.autoImport._ diff --git a/distribution/lib/Standard/Tableau/0.0.0-dev/src/Hyper_API.enso b/distribution/lib/Standard/Tableau/0.0.0-dev/src/Hyper_API.enso index 130bb498b1e7..8ccfb71168c1 100644 --- a/distribution/lib/Standard/Tableau/0.0.0-dev/src/Hyper_API.enso +++ b/distribution/lib/Standard/Tableau/0.0.0-dev/src/Hyper_API.enso @@ -1,4 +1,7 @@ from Standard.Base import all +import Standard.Base.Errors.Illegal_Argument.Illegal_Argument +from Standard.Base.Metadata.Choice import Option +from Standard.Base.Metadata.Widget import Single_Choice polyglot java import org.enso.tableau.HyperReader @@ -9,7 +12,7 @@ type Hyper_File ## Gets the location for hyperd hyper_path : Text - hyper_path = HyperReader.HYPER_PATH + hyper_path = HyperReader.HYPER_PATH.toString ## PRIVATE A representation of a Tableau Hyper Extract file. @@ -19,4 +22,22 @@ type Hyper_File Returns the list of schemas for the connection within the current database (or catalog). schemas : Vector Text schemas self = - HyperReader.readSchemas self.file.path + array = HyperReader.readSchemas self.file.path + Vector.from_polyglot_array array + + ## ICON metadata + Returns the list of tables for the connection within the current database (or catalog). + @schema (hyper -> make_schema_selector hyper True) + tables : Text -> Vector Text + tables self schema:Text='*' = + array = case schema of + "" -> Error.throw (Illegal_Argument.Error "Schema name cannot be empty.") + "*" -> HyperReader.listTablesAllSchema self.file.path + _ -> HyperReader.listTables self.file.path schema + Vector.from_polyglot_array array + +## PRIVATE +private make_schema_selector hyper_file:Hyper_File include_any:Boolean=False = + schemas = hyper_file.schemas.map t-> Option t t.pretty + any_entry = if include_any then [Option "" "'*'"] else [] + Single_Choice values=schemas+any_entry diff --git a/std-bits/tableau/src/main/java/org/enso/tableau/HyperReader.java b/std-bits/tableau/src/main/java/org/enso/tableau/HyperReader.java index 6a6f6bede785..49acad2ee223 100644 --- a/std-bits/tableau/src/main/java/org/enso/tableau/HyperReader.java +++ b/std-bits/tableau/src/main/java/org/enso/tableau/HyperReader.java @@ -1,9 +1,7 @@ package org.enso.tableau; -import com.tableau.hyperapi.Connection; -import com.tableau.hyperapi.HyperProcess; -import com.tableau.hyperapi.SchemaName; -import com.tableau.hyperapi.Telemetry; +import com.tableau.hyperapi.*; + import java.io.FileOutputStream; import java.io.IOException; import java.net.URI; @@ -11,6 +9,8 @@ import java.nio.channels.Channels; import java.nio.file.Files; import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; import java.util.logging.Level; import java.util.logging.Logger; @@ -74,7 +74,7 @@ private static HyperProcess getProcess() throws IOException { } // Start hyper process. - if (process == null) { + if (process == null || !process.isOpen()) { var contextClassLoader = Thread.currentThread().getContextClassLoader(); try { Thread.currentThread().setContextClassLoader(HyperReader.class.getClassLoader()); @@ -89,6 +89,7 @@ private static HyperProcess getProcess() throws IOException { Thread.currentThread().setContextClassLoader(contextClassLoader); } } + return process; } @@ -108,24 +109,51 @@ private static void downloadHyper(String uri, String fileName, boolean setExecut } public static String[] readSchemas(String path) throws IOException { - try (var process = getProcess()) { - try (var connection = new Connection(process.getEndpoint(), path)) { - var catalog = connection.getCatalog(); - return catalog.getSchemaNames().stream() - .map(s -> s.getName().getUnescaped()) - .toArray(String[]::new); - } + var process = getProcess(); + try (var connection = new Connection(process.getEndpoint(), path)) { + var catalog = connection.getCatalog(); + return catalog.getSchemaNames().stream() + .map(s -> s.getName().getUnescaped()) + .toArray(String[]::new); } } public static String addSchema(String path, String schema) throws IOException { var schemaName = new SchemaName(schema); - try (var process = getProcess()) { - try (var connection = new Connection(process.getEndpoint(), path)) { - var catalog = connection.getCatalog(); - catalog.createSchema(schemaName); - return schemaName.getName().getUnescaped(); + var process = getProcess(); + + try (var connection = new Connection(process.getEndpoint(), path)) { + var catalog = connection.getCatalog(); + catalog.createSchema(schemaName); + return schemaName.getName().getUnescaped(); + } + } + + public static String[] listTablesAllSchemas(String path) throws IOException { + var process = getProcess(); + try (var connection = new Connection(process.getEndpoint(), path)) { + var catalog = connection.getCatalog(); + return listTablesImpl(catalog, catalog.getSchemaNames()); + } + } + + public static String[] listTables(String path, String schemaName) throws IOException{ + var schemaNames = List.of(new SchemaName(schemaName)); + var process = getProcess(); + try (var connection = new Connection(process.getEndpoint(), path)) { + var catalog = connection.getCatalog(); + return listTablesImpl(catalog, schemaNames); + } + } + + private static String[] listTablesImpl(Catalog catalog, List schemaNames) { + var output = new ArrayList(); + for (var schemaName : schemaNames) { + var tables = catalog.getTableNames(schemaName); + for (var table : tables) { + output.add(table.getName().getUnescaped()); } } + return output.toArray(String[]::new); } } From 0c55cba08d21a1b067b93100708f8f5e68f85e01 Mon Sep 17 00:00:00 2001 From: James Dunkerley Date: Wed, 31 Jul 2024 13:07:05 +0100 Subject: [PATCH 08/35] Typo. --- distribution/lib/Standard/Tableau/0.0.0-dev/src/Hyper_API.enso | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/distribution/lib/Standard/Tableau/0.0.0-dev/src/Hyper_API.enso b/distribution/lib/Standard/Tableau/0.0.0-dev/src/Hyper_API.enso index 8ccfb71168c1..24a9bea45e81 100644 --- a/distribution/lib/Standard/Tableau/0.0.0-dev/src/Hyper_API.enso +++ b/distribution/lib/Standard/Tableau/0.0.0-dev/src/Hyper_API.enso @@ -32,7 +32,7 @@ type Hyper_File tables self schema:Text='*' = array = case schema of "" -> Error.throw (Illegal_Argument.Error "Schema name cannot be empty.") - "*" -> HyperReader.listTablesAllSchema self.file.path + "*" -> HyperReader.listTablesAllSchemas self.file.path _ -> HyperReader.listTables self.file.path schema Vector.from_polyglot_array array From 6ac10c195472ec9830f6f7819a3306917c148e46 Mon Sep 17 00:00:00 2001 From: James Dunkerley Date: Wed, 31 Jul 2024 15:06:25 +0100 Subject: [PATCH 09/35] Read structure. --- .../Tableau/0.0.0-dev/src/Hyper_API.enso | 55 +++++++++++- .../java/org/enso/tableau/HyperReader.java | 84 +++++++++++++++---- 2 files changed, 120 insertions(+), 19 deletions(-) diff --git a/distribution/lib/Standard/Tableau/0.0.0-dev/src/Hyper_API.enso b/distribution/lib/Standard/Tableau/0.0.0-dev/src/Hyper_API.enso index 24a9bea45e81..e719acdc32e3 100644 --- a/distribution/lib/Standard/Tableau/0.0.0-dev/src/Hyper_API.enso +++ b/distribution/lib/Standard/Tableau/0.0.0-dev/src/Hyper_API.enso @@ -3,7 +3,12 @@ import Standard.Base.Errors.Illegal_Argument.Illegal_Argument from Standard.Base.Metadata.Choice import Option from Standard.Base.Metadata.Widget import Single_Choice +from Standard.Table import Value_Type +import Standard.Database.Column_Constraint.Column_Constraint +import Standard.Database.Column_Description.Column_Description + polyglot java import org.enso.tableau.HyperReader +polyglot java import java.sql.Types type Hyper_File ## Creates a Hyper_File @@ -28,13 +33,59 @@ type Hyper_File ## ICON metadata Returns the list of tables for the connection within the current database (or catalog). @schema (hyper -> make_schema_selector hyper True) - tables : Text -> Vector Text + tables : Text -> Vector Hyper_Table tables self schema:Text='*' = array = case schema of "" -> Error.throw (Illegal_Argument.Error "Schema name cannot be empty.") "*" -> HyperReader.listTablesAllSchemas self.file.path _ -> HyperReader.listTables self.file.path schema - Vector.from_polyglot_array array + array.map t-> Hyper_Table.Value self t.schema t.name + +## An Enso representation of a Tableau Hyper Table. +type Hyper_Table + ## Represents a Tableau Hyper Table. + Value file:Hyper_File schema:Text table:Text + + ## PRIVATE + to_display_text : Text + to_display_text self = self.table + " (" + self.schema + ")" + + ## PRIVATE + to_js_object : JS_Object + to_js_object self = + JS_Object.from_pairs [["type", "Hyper_Table"], ["schema", self.schema], ["table", self.table], ["file", self.file.file.path]] + + ## Reads The Columns for the Table + columns : Vector Column_Description + columns self = + array = HyperReader.readColumns self.file.file.path self.schema self.table + array.map column-> + value_type = case column.typeID of + Types.BOOLEAN -> Value_Type.Boolean + Types.BIGINT -> Value_Type.Integer ..Bits_64 + Types.SMALLINT -> Value_Type.Integer ..Bits_16 + Types.INTEGER -> Value_Type.Integer ..Bits_32 + Types.NUMERIC -> + precision = if column.precision.isEmpty then Nothing else column.precision.getAsInt + scale = if column.scale.isEmpty then Nothing else column.scale.getAsInt + Value_Type.Decimal precision scale + Types.FLOAT -> Value_Type.Float ..Bits_32 + Types.DOUBLE -> Value_Type.Float ..Bits_64 + Types.VARCHAR -> + length = if column.length.isEmpty then Nothing else column.length.getAsInt + Value_Type.Char length variable_length=True + Types.CHAR -> + length = if column.length.isEmpty then Nothing else column.length.getAsInt + Value_Type.Char length variable_length=False + Types.DATE -> Value_Type.Date + Types.TIME -> Value_Type.Time + Types.TIMESTAMP -> Value_Type.Date_Time with_timezone=False + Types.TIMESTAMP_WITH_TIMEZONE -> Value_Type.Date_Time with_timezone=True + HyperReader.JSON -> Value_Type.Unsupported_Data_Type "JSON" JS_Object + HyperReader.INTERVAL -> Value_Type.Unsupported_Data_Type "INTERVAL" Duration + _ -> Value_Type.Unsupported_Data_Type "Unknown" Any + constraints = if column.nullable then [] else [Column_Constraint.Not_Null] + Column_Description.Value column.name value_type constraints ## PRIVATE private make_schema_selector hyper_file:Hyper_File include_any:Boolean=False = diff --git a/std-bits/tableau/src/main/java/org/enso/tableau/HyperReader.java b/std-bits/tableau/src/main/java/org/enso/tableau/HyperReader.java index 49acad2ee223..288d33f8dc1d 100644 --- a/std-bits/tableau/src/main/java/org/enso/tableau/HyperReader.java +++ b/std-bits/tableau/src/main/java/org/enso/tableau/HyperReader.java @@ -1,6 +1,14 @@ package org.enso.tableau; -import com.tableau.hyperapi.*; +import com.tableau.hyperapi.Catalog; +import com.tableau.hyperapi.Connection; +import com.tableau.hyperapi.HyperProcess; +import com.tableau.hyperapi.Nullability; +import com.tableau.hyperapi.SchemaName; +import com.tableau.hyperapi.TableName; +import com.tableau.hyperapi.TableDefinition.Column; +import com.tableau.hyperapi.Telemetry; +import com.tableau.hyperapi.TypeTag; import java.io.FileOutputStream; import java.io.IOException; @@ -9,8 +17,10 @@ import java.nio.channels.Channels; import java.nio.file.Files; import java.nio.file.Path; +import java.sql.Types; import java.util.ArrayList; import java.util.List; +import java.util.OptionalInt; import java.util.logging.Level; import java.util.logging.Logger; @@ -118,18 +128,10 @@ public static String[] readSchemas(String path) throws IOException { } } - public static String addSchema(String path, String schema) throws IOException { - var schemaName = new SchemaName(schema); - var process = getProcess(); + //** Record type for representing a Hyper table. */ + public record Table(String schema, String name) {} - try (var connection = new Connection(process.getEndpoint(), path)) { - var catalog = connection.getCatalog(); - catalog.createSchema(schemaName); - return schemaName.getName().getUnescaped(); - } - } - - public static String[] listTablesAllSchemas(String path) throws IOException { + public static Table[] listTablesAllSchemas(String path) throws IOException { var process = getProcess(); try (var connection = new Connection(process.getEndpoint(), path)) { var catalog = connection.getCatalog(); @@ -137,7 +139,7 @@ public static String[] listTablesAllSchemas(String path) throws IOException { } } - public static String[] listTables(String path, String schemaName) throws IOException{ + public static Table[] listTables(String path, String schemaName) throws IOException{ var schemaNames = List.of(new SchemaName(schemaName)); var process = getProcess(); try (var connection = new Connection(process.getEndpoint(), path)) { @@ -146,14 +148,62 @@ public static String[] listTables(String path, String schemaName) throws IOExcep } } - private static String[] listTablesImpl(Catalog catalog, List schemaNames) { - var output = new ArrayList(); + private static Table[] listTablesImpl(Catalog catalog, List schemaNames) { + var output = new ArrayList(); for (var schemaName : schemaNames) { var tables = catalog.getTableNames(schemaName); for (var table : tables) { - output.add(table.getName().getUnescaped()); + output.add(new Table(schemaName.getName().getUnescaped(), table.getName().getUnescaped())); } } - return output.toArray(String[]::new); + return output.toArray(Table[]::new); + } + + public record TableColumn(String name, int typeID, boolean nullable, OptionalInt length, OptionalInt precision, OptionalInt scale) { + public static TableColumn FromHyperColumn(Column hyperColumn) { + return new TableColumn( + hyperColumn.getName().getUnescaped(), + mapTypeTag(hyperColumn.getType().getTag()), + hyperColumn.getNullability().equals(Nullability.NULLABLE), + hyperColumn.getType().getMaxLength(), + hyperColumn.getType().getPrecision(), + hyperColumn.getType().getScale()); + } + + public final static int JSON = 10001; + public final static int INTERVAL = 10002; + + private static int mapTypeTag(TypeTag tag) { + return switch (tag) { + case BOOL -> Types.BOOLEAN; + case BIG_INT -> Types.BIGINT; + case SMALL_INT -> Types.SMALLINT; + case INT -> Types.INTEGER; + case NUMERIC -> Types.NUMERIC; + case FLOAT -> Types.FLOAT; + case DOUBLE -> Types.DOUBLE; + case TEXT, VARCHAR -> Types.VARCHAR; + case CHAR -> Types.CHAR; + case DATE -> Types.DATE; + case TIME -> Types.TIME; + case TIMESTAMP -> Types.TIMESTAMP; + case TIMESTAMP_TZ -> Types.TIMESTAMP_WITH_TIMEZONE; + case JSON -> JSON; + case INTERVAL -> INTERVAL; + default -> Types.OTHER; + }; + } + } + + public static TableColumn[] readStructure(String path, String schemaName, String tableName) throws IOException { + var tableNameObject = new TableName(new SchemaName(schemaName), tableName); + var process = getProcess(); + try (var connection = new Connection(process.getEndpoint(), path)) { + var catalog = connection.getCatalog(); + var definition = catalog.getTableDefinition(tableNameObject); + return definition.getColumns().stream() + .map(TableColumn::FromHyperColumn) + .toArray(TableColumn[]::new); + } } } From 68c6d1cd61e459f91f33b02af24967e21f145367 Mon Sep 17 00:00:00 2001 From: James Dunkerley Date: Wed, 31 Jul 2024 18:20:51 +0100 Subject: [PATCH 10/35] Read structure fixed. --- .../Base/0.0.0-dev/src/Data/Json.enso | 2 +- .../Tableau/0.0.0-dev/src/Hyper_API.enso | 25 +++++++++++++------ .../java/org/enso/tableau/HyperReader.java | 6 ++--- 3 files changed, 22 insertions(+), 11 deletions(-) diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Json.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Json.enso index 0e241f074496..8082f5451a60 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Json.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Json.enso @@ -162,7 +162,7 @@ type JS_Object JS_Object.Value object_node (make_field_names object_node) ## PRIVATE - Creates a Jackon_Object from a list of key-value pairs. + Creates a Jackson_Object from a list of key-value pairs. Keys must be `Text` values. Values will be recursively converted to JSON serializable as needed. from_pairs : Vector -> JS_Object diff --git a/distribution/lib/Standard/Tableau/0.0.0-dev/src/Hyper_API.enso b/distribution/lib/Standard/Tableau/0.0.0-dev/src/Hyper_API.enso index e719acdc32e3..9c54d6e1e118 100644 --- a/distribution/lib/Standard/Tableau/0.0.0-dev/src/Hyper_API.enso +++ b/distribution/lib/Standard/Tableau/0.0.0-dev/src/Hyper_API.enso @@ -4,8 +4,6 @@ from Standard.Base.Metadata.Choice import Option from Standard.Base.Metadata.Widget import Single_Choice from Standard.Table import Value_Type -import Standard.Database.Column_Constraint.Column_Constraint -import Standard.Database.Column_Description.Column_Description polyglot java import org.enso.tableau.HyperReader polyglot java import java.sql.Types @@ -56,9 +54,9 @@ type Hyper_Table JS_Object.from_pairs [["type", "Hyper_Table"], ["schema", self.schema], ["table", self.table], ["file", self.file.file.path]] ## Reads The Columns for the Table - columns : Vector Column_Description + columns : Vector Hyper_Column columns self = - array = HyperReader.readColumns self.file.file.path self.schema self.table + array = HyperReader.readStructure self.file.file.path self.schema self.table array.map column-> value_type = case column.typeID of Types.BOOLEAN -> Value_Type.Boolean @@ -84,11 +82,24 @@ type Hyper_Table HyperReader.JSON -> Value_Type.Unsupported_Data_Type "JSON" JS_Object HyperReader.INTERVAL -> Value_Type.Unsupported_Data_Type "INTERVAL" Duration _ -> Value_Type.Unsupported_Data_Type "Unknown" Any - constraints = if column.nullable then [] else [Column_Constraint.Not_Null] - Column_Description.Value column.name value_type constraints + Hyper_Column.Value column.name value_type column.nullable + +## An Enso representation of a Column in a Tableau Hyper Table. +type Hyper_Column + Value name:Text value_type:Value_Type nullable:Boolean + + ## PRIVATE + to_display_text : Text + to_display_text self = self.name + " (" + self.value_type.to_display_text + ")" + + ## PRIVATE + to_js_object : JS_Object + to_js_object self = + JS_Object.from_pairs [["type", "Hyper_Column"], ["name", self.name], ["value_type", self.value_type], ["nullable", self.nullable]] + ## PRIVATE -private make_schema_selector hyper_file:Hyper_File include_any:Boolean=False = +make_schema_selector hyper_file:Hyper_File include_any:Boolean=False = schemas = hyper_file.schemas.map t-> Option t t.pretty any_entry = if include_any then [Option "" "'*'"] else [] Single_Choice values=schemas+any_entry diff --git a/std-bits/tableau/src/main/java/org/enso/tableau/HyperReader.java b/std-bits/tableau/src/main/java/org/enso/tableau/HyperReader.java index 288d33f8dc1d..732d77b11da4 100644 --- a/std-bits/tableau/src/main/java/org/enso/tableau/HyperReader.java +++ b/std-bits/tableau/src/main/java/org/enso/tableau/HyperReader.java @@ -159,6 +159,9 @@ private static Table[] listTablesImpl(Catalog catalog, List schemaNa return output.toArray(Table[]::new); } + public final static int JSON = 10001; + public final static int INTERVAL = 10002; + public record TableColumn(String name, int typeID, boolean nullable, OptionalInt length, OptionalInt precision, OptionalInt scale) { public static TableColumn FromHyperColumn(Column hyperColumn) { return new TableColumn( @@ -170,9 +173,6 @@ public static TableColumn FromHyperColumn(Column hyperColumn) { hyperColumn.getType().getScale()); } - public final static int JSON = 10001; - public final static int INTERVAL = 10002; - private static int mapTypeTag(TypeTag tag) { return switch (tag) { case BOOL -> Types.BOOLEAN; From 379f9a65410569652390cb5103015d3c6c30dd62 Mon Sep 17 00:00:00 2001 From: James Dunkerley Date: Thu, 1 Aug 2024 12:38:47 +0100 Subject: [PATCH 11/35] Infrastructure for reading a Tableau Table. --- .../Tableau/0.0.0-dev/src/Hyper_API.enso | 16 +- .../java/org/enso/tableau/HyperReader.java | 92 +++++++--- .../org/enso/tableau/TableColumnBuilder.java | 169 ++++++++++++++++++ 3 files changed, 254 insertions(+), 23 deletions(-) create mode 100644 std-bits/tableau/src/main/java/org/enso/tableau/TableColumnBuilder.java diff --git a/distribution/lib/Standard/Tableau/0.0.0-dev/src/Hyper_API.enso b/distribution/lib/Standard/Tableau/0.0.0-dev/src/Hyper_API.enso index 9c54d6e1e118..c2c788a63f08 100644 --- a/distribution/lib/Standard/Tableau/0.0.0-dev/src/Hyper_API.enso +++ b/distribution/lib/Standard/Tableau/0.0.0-dev/src/Hyper_API.enso @@ -3,10 +3,12 @@ import Standard.Base.Errors.Illegal_Argument.Illegal_Argument from Standard.Base.Metadata.Choice import Option from Standard.Base.Metadata.Widget import Single_Choice -from Standard.Table import Value_Type +from Standard.Table import Column, Table, Value_Type +import Standard.Table.Rows_To_Read.Rows_To_Read +import Standard.Table.Internal.Java_Problems -polyglot java import org.enso.tableau.HyperReader polyglot java import java.sql.Types +polyglot java import org.enso.tableau.HyperReader type Hyper_File ## Creates a Hyper_File @@ -84,6 +86,15 @@ type Hyper_Table _ -> Value_Type.Unsupported_Data_Type "Unknown" Any Hyper_Column.Value column.name value_type column.nullable + ## Reads the Table into Enso Table + read : Rows_To_Read -> Table + read self (max_rows : Rows_To_Read = ..All_Rows) = + Java_Problems.with_problem_aggregator Problem_Behavior.Report_Warning java_problem_aggregator-> + row_count = if max_rows == Rows_To_Read.All_Rows then Nothing else max_rows.rows + java_columns = HyperReader.readTable self.file.file.path self.schema self.table row_count java_problem_aggregator + enso_columns = java_columns.map c-> Column.from_storage c.getName c.getStorage + Table.new enso_columns + ## An Enso representation of a Column in a Tableau Hyper Table. type Hyper_Column Value name:Text value_type:Value_Type nullable:Boolean @@ -97,7 +108,6 @@ type Hyper_Column to_js_object self = JS_Object.from_pairs [["type", "Hyper_Column"], ["name", self.name], ["value_type", self.value_type], ["nullable", self.nullable]] - ## PRIVATE make_schema_selector hyper_file:Hyper_File include_any:Boolean=False = schemas = hyper_file.schemas.map t-> Option t t.pretty diff --git a/std-bits/tableau/src/main/java/org/enso/tableau/HyperReader.java b/std-bits/tableau/src/main/java/org/enso/tableau/HyperReader.java index 732d77b11da4..1fff0b3a5c65 100644 --- a/std-bits/tableau/src/main/java/org/enso/tableau/HyperReader.java +++ b/std-bits/tableau/src/main/java/org/enso/tableau/HyperReader.java @@ -5,11 +5,10 @@ import com.tableau.hyperapi.HyperProcess; import com.tableau.hyperapi.Nullability; import com.tableau.hyperapi.SchemaName; +import com.tableau.hyperapi.TableDefinition; import com.tableau.hyperapi.TableName; -import com.tableau.hyperapi.TableDefinition.Column; import com.tableau.hyperapi.Telemetry; import com.tableau.hyperapi.TypeTag; - import java.io.FileOutputStream; import java.io.IOException; import java.net.URI; @@ -19,10 +18,14 @@ import java.nio.file.Path; import java.sql.Types; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.OptionalInt; import java.util.logging.Level; import java.util.logging.Logger; +import java.util.stream.IntStream; +import org.enso.table.data.table.Column; +import org.enso.table.problems.ProblemAggregator; public class HyperReader { public static final Path HYPER_PATH = Path.of(getHyperPath()); @@ -128,10 +131,10 @@ public static String[] readSchemas(String path) throws IOException { } } - //** Record type for representing a Hyper table. */ - public record Table(String schema, String name) {} + // ** Record type for representing a Hyper table. */ + public record HyperTable(String schema, String name) {} - public static Table[] listTablesAllSchemas(String path) throws IOException { + public static HyperTable[] listTablesAllSchemas(String path) throws IOException { var process = getProcess(); try (var connection = new Connection(process.getEndpoint(), path)) { var catalog = connection.getCatalog(); @@ -139,7 +142,7 @@ public static Table[] listTablesAllSchemas(String path) throws IOException { } } - public static Table[] listTables(String path, String schemaName) throws IOException{ + public static HyperTable[] listTables(String path, String schemaName) throws IOException { var schemaNames = List.of(new SchemaName(schemaName)); var process = getProcess(); try (var connection = new Connection(process.getEndpoint(), path)) { @@ -148,23 +151,32 @@ public static Table[] listTables(String path, String schemaName) throws IOExcept } } - private static Table[] listTablesImpl(Catalog catalog, List schemaNames) { - var output = new ArrayList
(); + private static HyperTable[] listTablesImpl(Catalog catalog, List schemaNames) { + var output = new ArrayList(); for (var schemaName : schemaNames) { var tables = catalog.getTableNames(schemaName); for (var table : tables) { - output.add(new Table(schemaName.getName().getUnescaped(), table.getName().getUnescaped())); + output.add( + new HyperTable(schemaName.getName().getUnescaped(), table.getName().getUnescaped())); } } - return output.toArray(Table[]::new); + return output.toArray(HyperTable[]::new); } - public final static int JSON = 10001; - public final static int INTERVAL = 10002; + public static final int JSON = 10001; + public static final int INTERVAL = 10002; - public record TableColumn(String name, int typeID, boolean nullable, OptionalInt length, OptionalInt precision, OptionalInt scale) { - public static TableColumn FromHyperColumn(Column hyperColumn) { + public record TableColumn( + int index, + String name, + int typeID, + boolean nullable, + OptionalInt length, + OptionalInt precision, + OptionalInt scale) { + public static TableColumn FromHyperColumn(int index, TableDefinition.Column hyperColumn) { return new TableColumn( + index, hyperColumn.getName().getUnescaped(), mapTypeTag(hyperColumn.getType().getTag()), hyperColumn.getNullability().equals(Nullability.NULLABLE), @@ -195,15 +207,55 @@ private static int mapTypeTag(TypeTag tag) { } } - public static TableColumn[] readStructure(String path, String schemaName, String tableName) throws IOException { + public static TableColumn[] readStructure(String path, String schemaName, String tableName) + throws IOException { var tableNameObject = new TableName(new SchemaName(schemaName), tableName); var process = getProcess(); try (var connection = new Connection(process.getEndpoint(), path)) { - var catalog = connection.getCatalog(); - var definition = catalog.getTableDefinition(tableNameObject); - return definition.getColumns().stream() - .map(TableColumn::FromHyperColumn) - .toArray(TableColumn[]::new); + return readStructureInternal(connection, tableNameObject); + } + } + + private static TableColumn[] readStructureInternal( + Connection connection, TableName tableNameObject) { + var catalog = connection.getCatalog(); + var definition = catalog.getTableDefinition(tableNameObject); + var columns = definition.getColumns(); + return IntStream.range(0, columns.size()) + .mapToObj(i -> TableColumn.FromHyperColumn(i, columns.get(i))) + .toArray(TableColumn[]::new); + } + + public static Column[] readTable( + String path, + String schemaName, + String tableName, + Integer rowLimit, + ProblemAggregator problemAggregator) + throws IOException { + var tableNameObject = new TableName(new SchemaName(schemaName), tableName); + var query = "SELECT * FROM " + tableNameObject + (rowLimit == null ? "" : " LIMIT " + rowLimit); + var process = getProcess(); + try (var connection = new Connection(process.getEndpoint(), path)) { + var columns = readStructureInternal(connection, tableNameObject); + + var builders = + Arrays.stream(columns) + .map( + c -> + TableColumnBuilder.create( + c, rowLimit == null ? 1000 : rowLimit, problemAggregator)) + .toList(); + + var result = connection.executeQuery(query); + while (result.nextRow()) { + builders.forEach(b -> b.append(result)); + } + + var storages = builders.stream().map(TableColumnBuilder::seal).toList(); + return IntStream.range(0, columns.length) + .mapToObj(i -> new Column(columns[i].name(), storages.get(i))) + .toArray(Column[]::new); } } } diff --git a/std-bits/tableau/src/main/java/org/enso/tableau/TableColumnBuilder.java b/std-bits/tableau/src/main/java/org/enso/tableau/TableColumnBuilder.java new file mode 100644 index 000000000000..19936ded33c6 --- /dev/null +++ b/std-bits/tableau/src/main/java/org/enso/tableau/TableColumnBuilder.java @@ -0,0 +1,169 @@ +package org.enso.tableau; + +import com.tableau.hyperapi.Result; +import java.sql.Types; +import java.time.Duration; +import java.time.Period; +import java.time.ZoneId; +import java.util.function.Consumer; +import org.enso.table.data.column.builder.*; +import org.enso.table.data.column.builder.StringBuilder; +import org.enso.table.data.column.storage.Storage; +import org.enso.table.data.column.storage.type.IntegerType; +import org.enso.table.data.column.storage.type.TextType; +import org.enso.table.problems.ProblemAggregator; + +public record TableColumnBuilder(Builder builder, Consumer appendMethod) { + private static Consumer nullAppender(Builder builder, Consumer inner) { + return r -> { + if (r.isNull(0)) { + builder.appendNulls(1); + } else { + inner.accept(r); + } + }; + } + + private static Object readInterval(Result r, int index) { + var interval = r.getInterval(index); + if (interval.getMonths() == 0 && interval.getYears() == 0) { + // Treat as a Duration + long seconds = + ((interval.getDays() * 24L + interval.getHours()) * 60 + interval.getMinutes()) * 60 + + interval.getSeconds(); + return Duration.ofNanos(seconds * 1_000_000_000L + interval.getMicroseconds() * 1_000L); + } else if (interval.getHours() == 0 + && interval.getMinutes() == 0 + && interval.getSeconds() == 0 + && interval.getMicroseconds() == 0) { + // Treat as a Period + return Period.of(interval.getYears(), interval.getMonths(), interval.getDays()); + } else { + // Can't do better than toString + return interval.toString(); + } + } + + public static TableColumnBuilder create( + HyperReader.TableColumn column, int initialRowCount, ProblemAggregator problemAggregator) { + switch (column.typeID()) { + case Types.BOOLEAN: + var boolBuilder = new BoolBuilder(initialRowCount); + return new TableColumnBuilder( + boolBuilder, + nullAppender(boolBuilder, r -> boolBuilder.appendBoolean(r.getBool(column.index())))); + case Types.BIGINT: + var longBuilder = + NumericBuilder.createLongBuilder( + initialRowCount, IntegerType.INT_64, problemAggregator); + return new TableColumnBuilder( + longBuilder, + nullAppender(longBuilder, r -> longBuilder.appendLong(r.getLong(column.index())))); + case Types.INTEGER: + var intBuilder = + NumericBuilder.createLongBuilder( + initialRowCount, IntegerType.INT_32, problemAggregator); + return new TableColumnBuilder( + intBuilder, + nullAppender(intBuilder, r -> intBuilder.appendLong(r.getInt(column.index())))); + case Types.SMALLINT: + var shortBuilder = + NumericBuilder.createLongBuilder( + initialRowCount, IntegerType.INT_16, problemAggregator); + return new TableColumnBuilder( + shortBuilder, + nullAppender(shortBuilder, r -> shortBuilder.appendLong(r.getShort(column.index())))); + case Types.NUMERIC: + if (column.scale().isEmpty()) { + throw new IllegalArgumentException("NUMERIC column must have a scale."); + } + if (column.scale().getAsInt() == 0) { + var bigIntBuilder = new BigIntegerBuilder(initialRowCount, problemAggregator); + return new TableColumnBuilder( + bigIntBuilder, + nullAppender( + bigIntBuilder, + r -> bigIntBuilder.append(r.getBigDecimal(column.index()).toBigInteger()))); + } else { + var bigDecimalBuilder = new BigDecimalBuilder(initialRowCount); + return new TableColumnBuilder( + bigDecimalBuilder, + nullAppender( + bigDecimalBuilder, + r -> bigDecimalBuilder.append(r.getBigDecimal(column.index())))); + } + case Types.FLOAT: + var floatBuilder = NumericBuilder.createDoubleBuilder(initialRowCount, problemAggregator); + return new TableColumnBuilder( + floatBuilder, + nullAppender(floatBuilder, r -> floatBuilder.appendDouble(r.getFloat(column.index())))); + case Types.DOUBLE: + var doubleBuilder = NumericBuilder.createDoubleBuilder(initialRowCount, problemAggregator); + return new TableColumnBuilder( + doubleBuilder, + nullAppender( + doubleBuilder, r -> doubleBuilder.appendDouble(r.getDouble(column.index())))); + case Types.VARCHAR, Types.CHAR: + var textType = + column.length().isEmpty() + ? new TextType(-1, false) + : new TextType(column.length().getAsInt(), column.typeID() == Types.CHAR); + var textBuilder = new StringBuilder(initialRowCount, textType); + return new TableColumnBuilder( + textBuilder, + nullAppender(textBuilder, r -> textBuilder.append(r.getString(column.index())))); + case Types.DATE: + var dateBuilder = new DateBuilder(initialRowCount); + return new TableColumnBuilder( + dateBuilder, + nullAppender(dateBuilder, r -> dateBuilder.appendDate(r.getLocalDate(column.index())))); + case Types.TIME: + var timeBuilder = new TimeOfDayBuilder(initialRowCount); + return new TableColumnBuilder( + timeBuilder, + nullAppender(timeBuilder, r -> timeBuilder.append(r.getLocalTime(column.index())))); + case Types.TIMESTAMP: + var dateTimeBuilder = new DateTimeBuilder(initialRowCount); + return new TableColumnBuilder( + dateTimeBuilder, + nullAppender( + dateTimeBuilder, + r -> + dateTimeBuilder.append( + r.getLocalDateTime(column.index()).atZone(ZoneId.systemDefault())))); + case Types.TIMESTAMP_WITH_TIMEZONE: + var dateTimeTzBuilder = new DateTimeBuilder(initialRowCount); + return new TableColumnBuilder( + dateTimeTzBuilder, + nullAppender( + dateTimeTzBuilder, + r -> dateTimeTzBuilder.append(r.getZonedDateTime(column.index())))); + case HyperReader.JSON: + var jsonBuilder = new ObjectBuilder(initialRowCount); + return new TableColumnBuilder( + jsonBuilder, + nullAppender(jsonBuilder, r -> jsonBuilder.append(r.getString(column.index())))); + case HyperReader.INTERVAL: + var intervalBuilder = new InferredBuilder(initialRowCount, problemAggregator); + return new TableColumnBuilder( + intervalBuilder, + nullAppender( + intervalBuilder, r -> intervalBuilder.append(readInterval(r, column.index())))); + case Types.OTHER: + var mixedBuilder = new ObjectBuilder(initialRowCount); + return new TableColumnBuilder( + mixedBuilder, + nullAppender(mixedBuilder, r -> mixedBuilder.append(r.getObject(column.index())))); + } + + throw new IllegalArgumentException("Unsupported column type: " + column.typeID()); + } + + public void append(Result result) { + appendMethod.accept(result); + } + + public Storage seal() { + return builder.seal(); + } +} From aa2576f416b78f7a09be83408c0bacd50aa7373f Mon Sep 17 00:00:00 2001 From: James Dunkerley Date: Thu, 1 Aug 2024 12:48:37 +0100 Subject: [PATCH 12/35] Move inner classes out and split out a bit. --- .../Tableau/0.0.0-dev/src/Hyper_API.enso | 5 +- .../java/org/enso/tableau/HyperReader.java | 65 ++----------------- .../java/org/enso/tableau/HyperTable.java | 5 ++ .../org/enso/tableau/HyperTableColumn.java | 54 +++++++++++++++ .../org/enso/tableau/TableColumnBuilder.java | 25 +++++-- 5 files changed, 89 insertions(+), 65 deletions(-) create mode 100644 std-bits/tableau/src/main/java/org/enso/tableau/HyperTable.java create mode 100644 std-bits/tableau/src/main/java/org/enso/tableau/HyperTableColumn.java diff --git a/distribution/lib/Standard/Tableau/0.0.0-dev/src/Hyper_API.enso b/distribution/lib/Standard/Tableau/0.0.0-dev/src/Hyper_API.enso index c2c788a63f08..60eefc8f10dc 100644 --- a/distribution/lib/Standard/Tableau/0.0.0-dev/src/Hyper_API.enso +++ b/distribution/lib/Standard/Tableau/0.0.0-dev/src/Hyper_API.enso @@ -9,6 +9,7 @@ import Standard.Table.Internal.Java_Problems polyglot java import java.sql.Types polyglot java import org.enso.tableau.HyperReader +polyglot java import org.enso.tableau.HyperTableColumn type Hyper_File ## Creates a Hyper_File @@ -81,8 +82,8 @@ type Hyper_Table Types.TIME -> Value_Type.Time Types.TIMESTAMP -> Value_Type.Date_Time with_timezone=False Types.TIMESTAMP_WITH_TIMEZONE -> Value_Type.Date_Time with_timezone=True - HyperReader.JSON -> Value_Type.Unsupported_Data_Type "JSON" JS_Object - HyperReader.INTERVAL -> Value_Type.Unsupported_Data_Type "INTERVAL" Duration + HyperTableColumn.JSON -> Value_Type.Unsupported_Data_Type "JSON" JS_Object + HyperTableColumn.INTERVAL -> Value_Type.Unsupported_Data_Type "INTERVAL" Duration _ -> Value_Type.Unsupported_Data_Type "Unknown" Any Hyper_Column.Value column.name value_type column.nullable diff --git a/std-bits/tableau/src/main/java/org/enso/tableau/HyperReader.java b/std-bits/tableau/src/main/java/org/enso/tableau/HyperReader.java index 1fff0b3a5c65..f12b3e14f673 100644 --- a/std-bits/tableau/src/main/java/org/enso/tableau/HyperReader.java +++ b/std-bits/tableau/src/main/java/org/enso/tableau/HyperReader.java @@ -3,12 +3,10 @@ import com.tableau.hyperapi.Catalog; import com.tableau.hyperapi.Connection; import com.tableau.hyperapi.HyperProcess; -import com.tableau.hyperapi.Nullability; import com.tableau.hyperapi.SchemaName; -import com.tableau.hyperapi.TableDefinition; import com.tableau.hyperapi.TableName; import com.tableau.hyperapi.Telemetry; -import com.tableau.hyperapi.TypeTag; + import java.io.FileOutputStream; import java.io.IOException; import java.net.URI; @@ -16,17 +14,16 @@ import java.nio.channels.Channels; import java.nio.file.Files; import java.nio.file.Path; -import java.sql.Types; import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import java.util.OptionalInt; import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.IntStream; import org.enso.table.data.table.Column; import org.enso.table.problems.ProblemAggregator; +//** Class responsible for reading from Tableau Hyper files. */ public class HyperReader { public static final Path HYPER_PATH = Path.of(getHyperPath()); private static HyperProcess process; @@ -80,7 +77,6 @@ private static HyperProcess getProcess() throws IOException { case OTHER -> throw new IOException( "Unsupported platform: " + OSPlatform.CurrentPlatform); } - ; } } catch (Exception e) { throw new IOException("Failed to download hyperd.", e); @@ -95,7 +91,7 @@ private static HyperProcess getProcess() throws IOException { try { process = new HyperProcess(HYPER_PATH, Telemetry.DO_NOT_SEND_USAGE_DATA_TO_TABLEAU); } catch (Throwable ioe) { - ioe.printStackTrace(); + LOGGER.log(Level.SEVERE, "Failed to start Hyper process.", ioe); throw ioe; } } finally { @@ -131,9 +127,6 @@ public static String[] readSchemas(String path) throws IOException { } } - // ** Record type for representing a Hyper table. */ - public record HyperTable(String schema, String name) {} - public static HyperTable[] listTablesAllSchemas(String path) throws IOException { var process = getProcess(); try (var connection = new Connection(process.getEndpoint(), path)) { @@ -163,51 +156,7 @@ private static HyperTable[] listTablesImpl(Catalog catalog, List sch return output.toArray(HyperTable[]::new); } - public static final int JSON = 10001; - public static final int INTERVAL = 10002; - - public record TableColumn( - int index, - String name, - int typeID, - boolean nullable, - OptionalInt length, - OptionalInt precision, - OptionalInt scale) { - public static TableColumn FromHyperColumn(int index, TableDefinition.Column hyperColumn) { - return new TableColumn( - index, - hyperColumn.getName().getUnescaped(), - mapTypeTag(hyperColumn.getType().getTag()), - hyperColumn.getNullability().equals(Nullability.NULLABLE), - hyperColumn.getType().getMaxLength(), - hyperColumn.getType().getPrecision(), - hyperColumn.getType().getScale()); - } - - private static int mapTypeTag(TypeTag tag) { - return switch (tag) { - case BOOL -> Types.BOOLEAN; - case BIG_INT -> Types.BIGINT; - case SMALL_INT -> Types.SMALLINT; - case INT -> Types.INTEGER; - case NUMERIC -> Types.NUMERIC; - case FLOAT -> Types.FLOAT; - case DOUBLE -> Types.DOUBLE; - case TEXT, VARCHAR -> Types.VARCHAR; - case CHAR -> Types.CHAR; - case DATE -> Types.DATE; - case TIME -> Types.TIME; - case TIMESTAMP -> Types.TIMESTAMP; - case TIMESTAMP_TZ -> Types.TIMESTAMP_WITH_TIMEZONE; - case JSON -> JSON; - case INTERVAL -> INTERVAL; - default -> Types.OTHER; - }; - } - } - - public static TableColumn[] readStructure(String path, String schemaName, String tableName) + public static HyperTableColumn[] readStructure(String path, String schemaName, String tableName) throws IOException { var tableNameObject = new TableName(new SchemaName(schemaName), tableName); var process = getProcess(); @@ -216,14 +165,14 @@ public static TableColumn[] readStructure(String path, String schemaName, String } } - private static TableColumn[] readStructureInternal( + private static HyperTableColumn[] readStructureInternal( Connection connection, TableName tableNameObject) { var catalog = connection.getCatalog(); var definition = catalog.getTableDefinition(tableNameObject); var columns = definition.getColumns(); return IntStream.range(0, columns.size()) - .mapToObj(i -> TableColumn.FromHyperColumn(i, columns.get(i))) - .toArray(TableColumn[]::new); + .mapToObj(i -> HyperTableColumn.FromHyperColumn(i, columns.get(i))) + .toArray(HyperTableColumn[]::new); } public static Column[] readTable( diff --git a/std-bits/tableau/src/main/java/org/enso/tableau/HyperTable.java b/std-bits/tableau/src/main/java/org/enso/tableau/HyperTable.java new file mode 100644 index 000000000000..9a821841b8cb --- /dev/null +++ b/std-bits/tableau/src/main/java/org/enso/tableau/HyperTable.java @@ -0,0 +1,5 @@ +package org.enso.tableau; + +// ** Record type for representing a Hyper table. */ +public record HyperTable(String schema, String name) { +} diff --git a/std-bits/tableau/src/main/java/org/enso/tableau/HyperTableColumn.java b/std-bits/tableau/src/main/java/org/enso/tableau/HyperTableColumn.java new file mode 100644 index 000000000000..b7d47c741a70 --- /dev/null +++ b/std-bits/tableau/src/main/java/org/enso/tableau/HyperTableColumn.java @@ -0,0 +1,54 @@ +package org.enso.tableau; + +import com.tableau.hyperapi.Nullability; +import com.tableau.hyperapi.TableDefinition; +import com.tableau.hyperapi.TypeTag; + +import java.sql.Types; +import java.util.OptionalInt; + +public record HyperTableColumn( + int index, + String name, + int typeID, + boolean nullable, + OptionalInt length, + OptionalInt precision, + OptionalInt scale) { + //** Type ID for JSON data. */ + public static final int JSON = 10001; + //** Type ID for INTERVAL data. */ + public static final int INTERVAL = 10002; + + static HyperTableColumn FromHyperColumn(int index, TableDefinition.Column hyperColumn) { + return new HyperTableColumn( + index, + hyperColumn.getName().getUnescaped(), + mapTypeTag(hyperColumn.getType().getTag()), + hyperColumn.getNullability().equals(Nullability.NULLABLE), + hyperColumn.getType().getMaxLength(), + hyperColumn.getType().getPrecision(), + hyperColumn.getType().getScale()); + } + + private static int mapTypeTag(TypeTag tag) { + return switch (tag) { + case BOOL -> Types.BOOLEAN; + case BIG_INT -> Types.BIGINT; + case SMALL_INT -> Types.SMALLINT; + case INT -> Types.INTEGER; + case NUMERIC -> Types.NUMERIC; + case FLOAT -> Types.FLOAT; + case DOUBLE -> Types.DOUBLE; + case TEXT, VARCHAR -> Types.VARCHAR; + case CHAR -> Types.CHAR; + case DATE -> Types.DATE; + case TIME -> Types.TIME; + case TIMESTAMP -> Types.TIMESTAMP; + case TIMESTAMP_TZ -> Types.TIMESTAMP_WITH_TIMEZONE; + case JSON -> JSON; + case INTERVAL -> INTERVAL; + default -> Types.OTHER; + }; + } +} diff --git a/std-bits/tableau/src/main/java/org/enso/tableau/TableColumnBuilder.java b/std-bits/tableau/src/main/java/org/enso/tableau/TableColumnBuilder.java index 19936ded33c6..066e6766abb4 100644 --- a/std-bits/tableau/src/main/java/org/enso/tableau/TableColumnBuilder.java +++ b/std-bits/tableau/src/main/java/org/enso/tableau/TableColumnBuilder.java @@ -1,19 +1,33 @@ package org.enso.tableau; import com.tableau.hyperapi.Result; + import java.sql.Types; import java.time.Duration; import java.time.Period; import java.time.ZoneId; import java.util.function.Consumer; -import org.enso.table.data.column.builder.*; + +import org.enso.table.data.column.builder.BigDecimalBuilder; +import org.enso.table.data.column.builder.BigIntegerBuilder; +import org.enso.table.data.column.builder.BoolBuilder; +import org.enso.table.data.column.builder.Builder; +import org.enso.table.data.column.builder.DateBuilder; +import org.enso.table.data.column.builder.DateTimeBuilder; +import org.enso.table.data.column.builder.InferredBuilder; +import org.enso.table.data.column.builder.NumericBuilder; +import org.enso.table.data.column.builder.ObjectBuilder; import org.enso.table.data.column.builder.StringBuilder; +import org.enso.table.data.column.builder.TimeOfDayBuilder; + import org.enso.table.data.column.storage.Storage; import org.enso.table.data.column.storage.type.IntegerType; import org.enso.table.data.column.storage.type.TextType; + import org.enso.table.problems.ProblemAggregator; -public record TableColumnBuilder(Builder builder, Consumer appendMethod) { +//** A builder for a single column of a table. */ +record TableColumnBuilder(Builder builder, Consumer appendMethod) { private static Consumer nullAppender(Builder builder, Consumer inner) { return r -> { if (r.isNull(0)) { @@ -24,6 +38,7 @@ private static Consumer nullAppender(Builder builder, Consumer i }; } + //** Convert a Tableau Interval into either a Duration or a Period (with fallback to String if needed). */ private static Object readInterval(Result r, int index) { var interval = r.getInterval(index); if (interval.getMonths() == 0 && interval.getYears() == 0) { @@ -45,7 +60,7 @@ private static Object readInterval(Result r, int index) { } public static TableColumnBuilder create( - HyperReader.TableColumn column, int initialRowCount, ProblemAggregator problemAggregator) { + HyperTableColumn column, int initialRowCount, ProblemAggregator problemAggregator) { switch (column.typeID()) { case Types.BOOLEAN: var boolBuilder = new BoolBuilder(initialRowCount); @@ -138,12 +153,12 @@ public static TableColumnBuilder create( nullAppender( dateTimeTzBuilder, r -> dateTimeTzBuilder.append(r.getZonedDateTime(column.index())))); - case HyperReader.JSON: + case HyperTableColumn.JSON: var jsonBuilder = new ObjectBuilder(initialRowCount); return new TableColumnBuilder( jsonBuilder, nullAppender(jsonBuilder, r -> jsonBuilder.append(r.getString(column.index())))); - case HyperReader.INTERVAL: + case HyperTableColumn.INTERVAL: var intervalBuilder = new InferredBuilder(initialRowCount, problemAggregator); return new TableColumnBuilder( intervalBuilder, From 3286ba4563e5d066d10d165c4d23a6a75522f7ab Mon Sep 17 00:00:00 2001 From: James Dunkerley Date: Thu, 1 Aug 2024 13:23:27 +0100 Subject: [PATCH 13/35] Fix Nothing handling. --- .../java/org/enso/tableau/HyperReader.java | 3 +- .../java/org/enso/tableau/HyperTable.java | 3 +- .../org/enso/tableau/HyperTableColumn.java | 5 +- .../org/enso/tableau/TableColumnBuilder.java | 69 +++++++++++++------ 4 files changed, 53 insertions(+), 27 deletions(-) diff --git a/std-bits/tableau/src/main/java/org/enso/tableau/HyperReader.java b/std-bits/tableau/src/main/java/org/enso/tableau/HyperReader.java index f12b3e14f673..809600840db4 100644 --- a/std-bits/tableau/src/main/java/org/enso/tableau/HyperReader.java +++ b/std-bits/tableau/src/main/java/org/enso/tableau/HyperReader.java @@ -6,7 +6,6 @@ import com.tableau.hyperapi.SchemaName; import com.tableau.hyperapi.TableName; import com.tableau.hyperapi.Telemetry; - import java.io.FileOutputStream; import java.io.IOException; import java.net.URI; @@ -23,7 +22,7 @@ import org.enso.table.data.table.Column; import org.enso.table.problems.ProblemAggregator; -//** Class responsible for reading from Tableau Hyper files. */ +// ** Class responsible for reading from Tableau Hyper files. */ public class HyperReader { public static final Path HYPER_PATH = Path.of(getHyperPath()); private static HyperProcess process; diff --git a/std-bits/tableau/src/main/java/org/enso/tableau/HyperTable.java b/std-bits/tableau/src/main/java/org/enso/tableau/HyperTable.java index 9a821841b8cb..fcb1c40a018d 100644 --- a/std-bits/tableau/src/main/java/org/enso/tableau/HyperTable.java +++ b/std-bits/tableau/src/main/java/org/enso/tableau/HyperTable.java @@ -1,5 +1,4 @@ package org.enso.tableau; // ** Record type for representing a Hyper table. */ -public record HyperTable(String schema, String name) { -} +public record HyperTable(String schema, String name) {} diff --git a/std-bits/tableau/src/main/java/org/enso/tableau/HyperTableColumn.java b/std-bits/tableau/src/main/java/org/enso/tableau/HyperTableColumn.java index b7d47c741a70..7ecdae8c93d9 100644 --- a/std-bits/tableau/src/main/java/org/enso/tableau/HyperTableColumn.java +++ b/std-bits/tableau/src/main/java/org/enso/tableau/HyperTableColumn.java @@ -3,7 +3,6 @@ import com.tableau.hyperapi.Nullability; import com.tableau.hyperapi.TableDefinition; import com.tableau.hyperapi.TypeTag; - import java.sql.Types; import java.util.OptionalInt; @@ -15,9 +14,9 @@ public record HyperTableColumn( OptionalInt length, OptionalInt precision, OptionalInt scale) { - //** Type ID for JSON data. */ + // ** Type ID for JSON data. */ public static final int JSON = 10001; - //** Type ID for INTERVAL data. */ + // ** Type ID for INTERVAL data. */ public static final int INTERVAL = 10002; static HyperTableColumn FromHyperColumn(int index, TableDefinition.Column hyperColumn) { diff --git a/std-bits/tableau/src/main/java/org/enso/tableau/TableColumnBuilder.java b/std-bits/tableau/src/main/java/org/enso/tableau/TableColumnBuilder.java index 066e6766abb4..ed341b347249 100644 --- a/std-bits/tableau/src/main/java/org/enso/tableau/TableColumnBuilder.java +++ b/std-bits/tableau/src/main/java/org/enso/tableau/TableColumnBuilder.java @@ -1,13 +1,11 @@ package org.enso.tableau; import com.tableau.hyperapi.Result; - import java.sql.Types; import java.time.Duration; import java.time.Period; import java.time.ZoneId; import java.util.function.Consumer; - import org.enso.table.data.column.builder.BigDecimalBuilder; import org.enso.table.data.column.builder.BigIntegerBuilder; import org.enso.table.data.column.builder.BoolBuilder; @@ -19,18 +17,16 @@ import org.enso.table.data.column.builder.ObjectBuilder; import org.enso.table.data.column.builder.StringBuilder; import org.enso.table.data.column.builder.TimeOfDayBuilder; - import org.enso.table.data.column.storage.Storage; import org.enso.table.data.column.storage.type.IntegerType; import org.enso.table.data.column.storage.type.TextType; - import org.enso.table.problems.ProblemAggregator; -//** A builder for a single column of a table. */ +// ** A builder for a single column of a table. */ record TableColumnBuilder(Builder builder, Consumer appendMethod) { - private static Consumer nullAppender(Builder builder, Consumer inner) { + private static Consumer nullAppender(Builder builder, int index, Consumer inner) { return r -> { - if (r.isNull(0)) { + if (r.isNull(index)) { builder.appendNulls(1); } else { inner.accept(r); @@ -38,7 +34,8 @@ private static Consumer nullAppender(Builder builder, Consumer i }; } - //** Convert a Tableau Interval into either a Duration or a Period (with fallback to String if needed). */ + // ** Convert a Tableau Interval into either a Duration or a Period (with fallback to String if + // needed). */ private static Object readInterval(Result r, int index) { var interval = r.getInterval(index); if (interval.getMonths() == 0 && interval.getYears() == 0) { @@ -66,28 +63,38 @@ public static TableColumnBuilder create( var boolBuilder = new BoolBuilder(initialRowCount); return new TableColumnBuilder( boolBuilder, - nullAppender(boolBuilder, r -> boolBuilder.appendBoolean(r.getBool(column.index())))); + nullAppender( + boolBuilder, + column.index(), + r -> boolBuilder.appendBoolean(r.getBool(column.index())))); case Types.BIGINT: var longBuilder = NumericBuilder.createLongBuilder( initialRowCount, IntegerType.INT_64, problemAggregator); return new TableColumnBuilder( longBuilder, - nullAppender(longBuilder, r -> longBuilder.appendLong(r.getLong(column.index())))); + nullAppender( + longBuilder, + column.index(), + r -> longBuilder.appendLong(r.getLong(column.index())))); case Types.INTEGER: var intBuilder = NumericBuilder.createLongBuilder( initialRowCount, IntegerType.INT_32, problemAggregator); return new TableColumnBuilder( intBuilder, - nullAppender(intBuilder, r -> intBuilder.appendLong(r.getInt(column.index())))); + nullAppender( + intBuilder, column.index(), r -> intBuilder.appendLong(r.getInt(column.index())))); case Types.SMALLINT: var shortBuilder = NumericBuilder.createLongBuilder( initialRowCount, IntegerType.INT_16, problemAggregator); return new TableColumnBuilder( shortBuilder, - nullAppender(shortBuilder, r -> shortBuilder.appendLong(r.getShort(column.index())))); + nullAppender( + shortBuilder, + column.index(), + r -> shortBuilder.appendLong(r.getShort(column.index())))); case Types.NUMERIC: if (column.scale().isEmpty()) { throw new IllegalArgumentException("NUMERIC column must have a scale."); @@ -98,6 +105,7 @@ public static TableColumnBuilder create( bigIntBuilder, nullAppender( bigIntBuilder, + column.index(), r -> bigIntBuilder.append(r.getBigDecimal(column.index()).toBigInteger()))); } else { var bigDecimalBuilder = new BigDecimalBuilder(initialRowCount); @@ -105,19 +113,25 @@ public static TableColumnBuilder create( bigDecimalBuilder, nullAppender( bigDecimalBuilder, + column.index(), r -> bigDecimalBuilder.append(r.getBigDecimal(column.index())))); } case Types.FLOAT: var floatBuilder = NumericBuilder.createDoubleBuilder(initialRowCount, problemAggregator); return new TableColumnBuilder( floatBuilder, - nullAppender(floatBuilder, r -> floatBuilder.appendDouble(r.getFloat(column.index())))); + nullAppender( + floatBuilder, + column.index(), + r -> floatBuilder.appendDouble(r.getFloat(column.index())))); case Types.DOUBLE: var doubleBuilder = NumericBuilder.createDoubleBuilder(initialRowCount, problemAggregator); return new TableColumnBuilder( doubleBuilder, nullAppender( - doubleBuilder, r -> doubleBuilder.appendDouble(r.getDouble(column.index())))); + doubleBuilder, + column.index(), + r -> doubleBuilder.appendDouble(r.getDouble(column.index())))); case Types.VARCHAR, Types.CHAR: var textType = column.length().isEmpty() @@ -126,23 +140,31 @@ public static TableColumnBuilder create( var textBuilder = new StringBuilder(initialRowCount, textType); return new TableColumnBuilder( textBuilder, - nullAppender(textBuilder, r -> textBuilder.append(r.getString(column.index())))); + nullAppender( + textBuilder, column.index(), r -> textBuilder.append(r.getString(column.index())))); case Types.DATE: var dateBuilder = new DateBuilder(initialRowCount); return new TableColumnBuilder( dateBuilder, - nullAppender(dateBuilder, r -> dateBuilder.appendDate(r.getLocalDate(column.index())))); + nullAppender( + dateBuilder, + column.index(), + r -> dateBuilder.appendDate(r.getLocalDate(column.index())))); case Types.TIME: var timeBuilder = new TimeOfDayBuilder(initialRowCount); return new TableColumnBuilder( timeBuilder, - nullAppender(timeBuilder, r -> timeBuilder.append(r.getLocalTime(column.index())))); + nullAppender( + timeBuilder, + column.index(), + r -> timeBuilder.append(r.getLocalTime(column.index())))); case Types.TIMESTAMP: var dateTimeBuilder = new DateTimeBuilder(initialRowCount); return new TableColumnBuilder( dateTimeBuilder, nullAppender( dateTimeBuilder, + column.index(), r -> dateTimeBuilder.append( r.getLocalDateTime(column.index()).atZone(ZoneId.systemDefault())))); @@ -152,23 +174,30 @@ public static TableColumnBuilder create( dateTimeTzBuilder, nullAppender( dateTimeTzBuilder, + column.index(), r -> dateTimeTzBuilder.append(r.getZonedDateTime(column.index())))); case HyperTableColumn.JSON: var jsonBuilder = new ObjectBuilder(initialRowCount); return new TableColumnBuilder( jsonBuilder, - nullAppender(jsonBuilder, r -> jsonBuilder.append(r.getString(column.index())))); + nullAppender( + jsonBuilder, column.index(), r -> jsonBuilder.append(r.getString(column.index())))); case HyperTableColumn.INTERVAL: var intervalBuilder = new InferredBuilder(initialRowCount, problemAggregator); return new TableColumnBuilder( intervalBuilder, nullAppender( - intervalBuilder, r -> intervalBuilder.append(readInterval(r, column.index())))); + intervalBuilder, + column.index(), + r -> intervalBuilder.append(readInterval(r, column.index())))); case Types.OTHER: var mixedBuilder = new ObjectBuilder(initialRowCount); return new TableColumnBuilder( mixedBuilder, - nullAppender(mixedBuilder, r -> mixedBuilder.append(r.getObject(column.index())))); + nullAppender( + mixedBuilder, + column.index(), + r -> mixedBuilder.append(r.getObject(column.index())))); } throw new IllegalArgumentException("Unsupported column type: " + column.typeID()); From 7803fc8bc25365f5d699a832c5ef0dd051f0d0e9 Mon Sep 17 00:00:00 2001 From: James Dunkerley Date: Thu, 1 Aug 2024 14:07:14 +0100 Subject: [PATCH 14/35] Split Enso files. Add ICON and GROUP to Hyper_File. --- .../Tableau/0.0.0-dev/src/Hyper_Column.enso | 16 ++++++ .../Tableau/0.0.0-dev/src/Hyper_File.enso | 45 +++++++++++++++ .../src/{Hyper_API.enso => Hyper_Table.enso} | 56 +------------------ .../Standard/Tableau/0.0.0-dev/src/Main.enso | 4 +- 4 files changed, 67 insertions(+), 54 deletions(-) create mode 100644 distribution/lib/Standard/Tableau/0.0.0-dev/src/Hyper_Column.enso create mode 100644 distribution/lib/Standard/Tableau/0.0.0-dev/src/Hyper_File.enso rename distribution/lib/Standard/Tableau/0.0.0-dev/src/{Hyper_API.enso => Hyper_Table.enso} (61%) diff --git a/distribution/lib/Standard/Tableau/0.0.0-dev/src/Hyper_Column.enso b/distribution/lib/Standard/Tableau/0.0.0-dev/src/Hyper_Column.enso new file mode 100644 index 000000000000..7a9cc0608ae9 --- /dev/null +++ b/distribution/lib/Standard/Tableau/0.0.0-dev/src/Hyper_Column.enso @@ -0,0 +1,16 @@ +from Standard.Base import all + +from Standard.Table import Value_Type + +## An Enso representation of a Column in a Tableau Hyper Table. +type Hyper_Column + Value name:Text value_type:Value_Type nullable:Boolean + + ## PRIVATE + to_display_text : Text + to_display_text self = self.name + " (" + self.value_type.to_display_text + ")" + + ## PRIVATE + to_js_object : JS_Object + to_js_object self = + JS_Object.from_pairs [["type", "Hyper_Column"], ["name", self.name], ["value_type", self.value_type], ["nullable", self.nullable]] diff --git a/distribution/lib/Standard/Tableau/0.0.0-dev/src/Hyper_File.enso b/distribution/lib/Standard/Tableau/0.0.0-dev/src/Hyper_File.enso new file mode 100644 index 000000000000..10ee6ca7e311 --- /dev/null +++ b/distribution/lib/Standard/Tableau/0.0.0-dev/src/Hyper_File.enso @@ -0,0 +1,45 @@ +from Standard.Base import all +import Standard.Base.Errors.Illegal_Argument.Illegal_Argument +from Standard.Base.Metadata.Choice import Option +from Standard.Base.Metadata.Widget import Single_Choice + +polyglot java import org.enso.tableau.HyperReader + +## Represents a Tableau Hyper Extract file. +type Hyper_File + ## ICON data_input + Creates a Hyper_File + + Arguments: + - file: The file to read. + new : File -> Hyper_File + new file:File = Hyper_File.Value file + + ## PRIVATE + A representation of a Tableau Hyper Extract file. + private Value file:File + + ## ICON metadata + Returns the list of schemas for the connection within the current database (or catalog). + schemas : Vector Text + schemas self = + array = HyperReader.readSchemas self.file.path + Vector.from_polyglot_array array + + ## GROUP Standard.Base.Metadata + ICON metadata + Returns the list of tables for the connection within the current database (or catalog). + @schema (hyper -> make_schema_selector hyper True) + tables : Text -> Vector Hyper_Table + tables self schema:Text='*' = + array = case schema of + "" -> Error.throw (Illegal_Argument.Error "Schema name cannot be empty.") + "*" -> HyperReader.listTablesAllSchemas self.file.path + _ -> HyperReader.listTables self.file.path schema + array.map t-> Hyper_Table.Value self t.schema t.name + +## PRIVATE +make_schema_selector hyper_file:Hyper_File include_any:Boolean=False = + schemas = hyper_file.schemas.map t-> Option t t.pretty + any_entry = if include_any then [Option "" "'*'"] else [] + Single_Choice values=schemas+any_entry diff --git a/distribution/lib/Standard/Tableau/0.0.0-dev/src/Hyper_API.enso b/distribution/lib/Standard/Tableau/0.0.0-dev/src/Hyper_Table.enso similarity index 61% rename from distribution/lib/Standard/Tableau/0.0.0-dev/src/Hyper_API.enso rename to distribution/lib/Standard/Tableau/0.0.0-dev/src/Hyper_Table.enso index 60eefc8f10dc..231f069959a9 100644 --- a/distribution/lib/Standard/Tableau/0.0.0-dev/src/Hyper_API.enso +++ b/distribution/lib/Standard/Tableau/0.0.0-dev/src/Hyper_Table.enso @@ -1,47 +1,16 @@ from Standard.Base import all -import Standard.Base.Errors.Illegal_Argument.Illegal_Argument -from Standard.Base.Metadata.Choice import Option -from Standard.Base.Metadata.Widget import Single_Choice from Standard.Table import Column, Table, Value_Type import Standard.Table.Rows_To_Read.Rows_To_Read import Standard.Table.Internal.Java_Problems +import project.Hyper_Column.Hyper_Column +import project.Hyper_File.Hyper_File + polyglot java import java.sql.Types polyglot java import org.enso.tableau.HyperReader polyglot java import org.enso.tableau.HyperTableColumn -type Hyper_File - ## Creates a Hyper_File - new : File -> Hyper_File - new file:File = Hyper_File.Value file - - ## Gets the location for hyperd - hyper_path : Text - hyper_path = HyperReader.HYPER_PATH.toString - - ## PRIVATE - A representation of a Tableau Hyper Extract file. - private Value file:File - - ## ICON metadata - Returns the list of schemas for the connection within the current database (or catalog). - schemas : Vector Text - schemas self = - array = HyperReader.readSchemas self.file.path - Vector.from_polyglot_array array - - ## ICON metadata - Returns the list of tables for the connection within the current database (or catalog). - @schema (hyper -> make_schema_selector hyper True) - tables : Text -> Vector Hyper_Table - tables self schema:Text='*' = - array = case schema of - "" -> Error.throw (Illegal_Argument.Error "Schema name cannot be empty.") - "*" -> HyperReader.listTablesAllSchemas self.file.path - _ -> HyperReader.listTables self.file.path schema - array.map t-> Hyper_Table.Value self t.schema t.name - ## An Enso representation of a Tableau Hyper Table. type Hyper_Table ## Represents a Tableau Hyper Table. @@ -95,22 +64,3 @@ type Hyper_Table java_columns = HyperReader.readTable self.file.file.path self.schema self.table row_count java_problem_aggregator enso_columns = java_columns.map c-> Column.from_storage c.getName c.getStorage Table.new enso_columns - -## An Enso representation of a Column in a Tableau Hyper Table. -type Hyper_Column - Value name:Text value_type:Value_Type nullable:Boolean - - ## PRIVATE - to_display_text : Text - to_display_text self = self.name + " (" + self.value_type.to_display_text + ")" - - ## PRIVATE - to_js_object : JS_Object - to_js_object self = - JS_Object.from_pairs [["type", "Hyper_Column"], ["name", self.name], ["value_type", self.value_type], ["nullable", self.nullable]] - -## PRIVATE -make_schema_selector hyper_file:Hyper_File include_any:Boolean=False = - schemas = hyper_file.schemas.map t-> Option t t.pretty - any_entry = if include_any then [Option "" "'*'"] else [] - Single_Choice values=schemas+any_entry diff --git a/distribution/lib/Standard/Tableau/0.0.0-dev/src/Main.enso b/distribution/lib/Standard/Tableau/0.0.0-dev/src/Main.enso index e08ef514b9c0..afcc4b2ce5b3 100644 --- a/distribution/lib/Standard/Tableau/0.0.0-dev/src/Main.enso +++ b/distribution/lib/Standard/Tableau/0.0.0-dev/src/Main.enso @@ -1,3 +1,5 @@ from Standard.Base import all -export project.Hyper_API.Hyper_File +export project.Hyper_Column.Hyper_Column +export project.Hyper_File.Hyper_File +export project.Hyper_Table.Hyper_Table From 11339486d943072870cbcaf09ebb19f52e4b6101 Mon Sep 17 00:00:00 2001 From: James Dunkerley Date: Thu, 1 Aug 2024 15:03:40 +0100 Subject: [PATCH 15/35] Fix dropdowns. --- .../Tableau/0.0.0-dev/src/Hyper_File.enso | 60 +++++++++++++++++-- .../Tableau/0.0.0-dev/src/Hyper_Table.enso | 17 ++++-- 2 files changed, 68 insertions(+), 9 deletions(-) diff --git a/distribution/lib/Standard/Tableau/0.0.0-dev/src/Hyper_File.enso b/distribution/lib/Standard/Tableau/0.0.0-dev/src/Hyper_File.enso index 10ee6ca7e311..2c426cd21027 100644 --- a/distribution/lib/Standard/Tableau/0.0.0-dev/src/Hyper_File.enso +++ b/distribution/lib/Standard/Tableau/0.0.0-dev/src/Hyper_File.enso @@ -3,6 +3,11 @@ import Standard.Base.Errors.Illegal_Argument.Illegal_Argument from Standard.Base.Metadata.Choice import Option from Standard.Base.Metadata.Widget import Single_Choice +from Standard.Table import Table +import Standard.Table.Rows_To_Read.Rows_To_Read + +import project.Hyper_Table.Hyper_Table + polyglot java import org.enso.tableau.HyperReader ## Represents a Tableau Hyper Extract file. @@ -12,12 +17,13 @@ type Hyper_File Arguments: - file: The file to read. - new : File -> Hyper_File - new file:File = Hyper_File.Value file + - schema: The schema to read or `*` for all schemas. + new : File -> Text -> Hyper_File + new file:File schema:Text='*' = Hyper_File.Value file schema ## PRIVATE A representation of a Tableau Hyper Extract file. - private Value file:File + private Value file:File internal_schema:Text ## ICON metadata Returns the list of schemas for the connection within the current database (or catalog). @@ -26,20 +32,64 @@ type Hyper_File array = HyperReader.readSchemas self.file.path Vector.from_polyglot_array array + ## ICON metadata + Returns the name of the current schema. + `*` represents all schemas. + schema : Text + schema self = self.internal_schema + + ## ICON data_input + Returns a new Hyper_File with the specified schema set as default. + + Arguments: + - schema: The name of the schema to connect to. + @schema (hyper -> make_schema_selector hyper True) + set_schema : Text -> Hyper_File + set_schema self schema = + if schema == self.schema then self else + Hyper_File.Value self.file schema + ## GROUP Standard.Base.Metadata ICON metadata Returns the list of tables for the connection within the current database (or catalog). @schema (hyper -> make_schema_selector hyper True) tables : Text -> Vector Hyper_Table - tables self schema:Text='*' = + tables self schema:Text=self.schema = if schema == "" then self.tables self.schema else array = case schema of - "" -> Error.throw (Illegal_Argument.Error "Schema name cannot be empty.") "*" -> HyperReader.listTablesAllSchemas self.file.path _ -> HyperReader.listTables self.file.path schema array.map t-> Hyper_Table.Value self t.schema t.name + ## ALIAS sheet, get + GROUP Standard.Base.Input + ICON data_input + Read a table from the Hyper_File into a Table. + + Arguments: + - table: table name to read from. + - schema: the schema to read from. + - limit: the maximum number of rows to read. + @table make_table_selector + @schema (hyper -> make_schema_selector hyper True) + @limit Rows_To_Read.default_widget + read : Text -> Text -> Rows_To_Read -> Table + read self (table : Text) (schema : Text = self.schema) (limit : Rows_To_Read = ..All_Rows) = case schema of + "" -> self.read table self.schema limit + "*" -> + table_to_read = self.tables.find t-> t.table == table + if table_to_read.is_nothing then Error.throw (Illegal_Argument.Error "Table not found.") else + table_to_read.read limit + _ -> Hyper_Table.Value self table schema . read limit + ## PRIVATE make_schema_selector hyper_file:Hyper_File include_any:Boolean=False = schemas = hyper_file.schemas.map t-> Option t t.pretty any_entry = if include_any then [Option "" "'*'"] else [] Single_Choice values=schemas+any_entry + +## PRIVATE +make_table_selector hyper_file:Hyper_File cache=Nothing = + schema = cache.if_not_nothing <| cache "schema" + used_schema = if schema == "" || schema == Nothing then hyper_file.schema else schema + tables = hyper_file.tables used_schema . map t-> Option t.table t.table.pretty + Single_Choice values=tables diff --git a/distribution/lib/Standard/Tableau/0.0.0-dev/src/Hyper_Table.enso b/distribution/lib/Standard/Tableau/0.0.0-dev/src/Hyper_Table.enso index 231f069959a9..d585f0fdebc8 100644 --- a/distribution/lib/Standard/Tableau/0.0.0-dev/src/Hyper_Table.enso +++ b/distribution/lib/Standard/Tableau/0.0.0-dev/src/Hyper_Table.enso @@ -13,8 +13,9 @@ polyglot java import org.enso.tableau.HyperTableColumn ## An Enso representation of a Tableau Hyper Table. type Hyper_Table - ## Represents a Tableau Hyper Table. - Value file:Hyper_File schema:Text table:Text + ## PRIVATE + Represents a Tableau Hyper Table. + private Value file:Hyper_File schema:Text table:Text ## PRIVATE to_display_text : Text @@ -25,7 +26,9 @@ type Hyper_Table to_js_object self = JS_Object.from_pairs [["type", "Hyper_Table"], ["schema", self.schema], ["table", self.table], ["file", self.file.file.path]] - ## Reads The Columns for the Table + ## GROUP Standard.Base.Metadata + ICON metadata + Reads The Columns for the Table columns : Vector Hyper_Column columns self = array = HyperReader.readStructure self.file.file.path self.schema self.table @@ -56,7 +59,13 @@ type Hyper_Table _ -> Value_Type.Unsupported_Data_Type "Unknown" Any Hyper_Column.Value column.name value_type column.nullable - ## Reads the Table into Enso Table + ## GROUP Standard.Base.Input + ICON data_input + Reads the Table into Enso Table + + Arguments: + - max_rows: specifies the maximum number of rows to read. + @max_rows Rows_To_Read.default_widget read : Rows_To_Read -> Table read self (max_rows : Rows_To_Read = ..All_Rows) = Java_Problems.with_problem_aggregator Problem_Behavior.Report_Warning java_problem_aggregator-> From eab5a770e189bd8485d1bf5ba18f4f91385c017f Mon Sep 17 00:00:00 2001 From: James Dunkerley Date: Thu, 1 Aug 2024 15:18:59 +0100 Subject: [PATCH 16/35] Add Changelog. --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e1756056d317..37f31f7a5fd0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,9 +6,11 @@ comparisons.][10614] - [Relative paths are now resolved relative to the project location, also in the Cloud.][10660] +- [Support for reading from Tableau Hyper files.][10733] [10614]: https://github.com/enso-org/enso/pull/10614 [10660]: https://github.com/enso-org/enso/pull/10660 +[10733]: https://github.com/enso-org/enso/pull/10733 # Enso 2023.3 From 2adffe1fa72b22a28824f12aba8040660a191798 Mon Sep 17 00:00:00 2001 From: James Dunkerley Date: Thu, 1 Aug 2024 16:49:47 +0100 Subject: [PATCH 17/35] Add File_Format for Tableau. --- .../Tableau/0.0.0-dev/src/Tableau_Format.enso | 57 +++++++++++++++++++ .../org/enso/tableau/TableauFormatSPI.java | 22 ++++++- 2 files changed, 77 insertions(+), 2 deletions(-) create mode 100644 distribution/lib/Standard/Tableau/0.0.0-dev/src/Tableau_Format.enso diff --git a/distribution/lib/Standard/Tableau/0.0.0-dev/src/Tableau_Format.enso b/distribution/lib/Standard/Tableau/0.0.0-dev/src/Tableau_Format.enso new file mode 100644 index 000000000000..6344b272039e --- /dev/null +++ b/distribution/lib/Standard/Tableau/0.0.0-dev/src/Tableau_Format.enso @@ -0,0 +1,57 @@ +from Standard.Base import all +import Standard.Base.Errors.Illegal_Argument.Illegal_Argument +import Standard.Base.System.File.Generic.Writable_File.Writable_File +import Standard.Base.System.File_Format_Metadata.File_Format_Metadata +import Standard.Base.System.Input_Stream.Input_Stream +from Standard.Base.Metadata.Choice import Option + +import project.Hyper_File.Hyper_File + +## Read the file to a `Hyper_File` object. +type Tableau_Format + ## Read the file to a `Hyper_File` object. + + Arguments: + - schema: The schema to read or `*` for all schemas. + Hyper_File (schema:Text='*') + + ## PRIVATE + Resolve an unresolved constructor to the actual type. + resolve : Function -> Tableau_Format | Nothing + resolve constructor = + Panic.catch Any (constructor:Tableau_Format) _->Nothing + + ## PRIVATE + ADVANCED + If the File_Format supports reading from the file, return a configured instance. + for_read : File_Format_Metadata -> Tableau_Format | Nothing + for_read file:File_Format_Metadata = + case file.guess_extension of + ".hyper" -> Tableau_Format.Hyper_File + _ -> Nothing + + ## PRIVATE + If this File_Format should be used for writing to that file, return a configured instance. + Not currently supported. + for_file_write : Writable_File -> Tableau_Format | Nothing + for_file_write file:Writable_File = + _ = [file] + Nothing + + ## PRIVATE + get_dropdown_options : Vector Option + get_dropdown_options = [Option "Tableau Hyper" "..Hyper_File"] + + ## PRIVATE + Implements the `File.read` for this `File_Format` + read : File -> Problem_Behavior -> Any + read self file on_problems:Problem_Behavior = + _ = [on_problems] + Hyper_File.new file self.schema + + ## PRIVATE + Implements decoding the format from a stream. + read_stream : Input_Stream -> File_Format_Metadata -> Any + read_stream self stream:Input_Stream (metadata : File_Format_Metadata) = + _ = [stream, metadata] + Error.throw (Illegal_Argument.Error "Cannot connect to a Hyper file backed by a stream. Save it to a local file first.") diff --git a/std-bits/tableau/src/main/java/org/enso/tableau/TableauFormatSPI.java b/std-bits/tableau/src/main/java/org/enso/tableau/TableauFormatSPI.java index 4063b2368113..5975328ad84a 100644 --- a/std-bits/tableau/src/main/java/org/enso/tableau/TableauFormatSPI.java +++ b/std-bits/tableau/src/main/java/org/enso/tableau/TableauFormatSPI.java @@ -1,3 +1,21 @@ -package src.main.java.org.enso.tableau; +package org.enso.tableau; -public class TableauFormatSPI {} +import org.enso.base.file_format.FileFormatSPI; + +@org.openide.util.lookup.ServiceProvider(service = FileFormatSPI.class) +public class TableauFormatSPI extends FileFormatSPI { + @Override + protected String getModuleName() { + return "Standard.Tableau.Tableau_Format"; + } + + @Override + protected String getTypeName() { + return "Tableau_Format"; + } + + @Override + protected String getDataLinkFormatName() { + return "tableau"; + } +} From 8139009e701db73154bdbbd88dc774b398aee723 Mon Sep 17 00:00:00 2001 From: James Dunkerley Date: Thu, 1 Aug 2024 17:34:32 +0100 Subject: [PATCH 18/35] Legal review --- .../Tableau/0.0.0-dev/THIRD-PARTY/NOTICE | 12 ++ .../0.0.0-dev/THIRD-PARTY/licenses/APACHE2.0 | 201 ++++++++++++++++++ .../net.java.dev.jna.jna-5.14.0/NOTICES | 29 +++ .../NOTICES | 91 ++++++++ .../copyright-keep | 15 ++ .../copyright-ignore | 2 + .../copyright-keep | 47 ++++ .../copyright-keep-context | 0 tools/legal-review/Tableau/report-state | 3 + .../Tableau/reviewed-licenses/Apache-2.0 | 1 + 10 files changed, 401 insertions(+) create mode 100644 distribution/lib/Standard/Tableau/0.0.0-dev/THIRD-PARTY/NOTICE create mode 100644 distribution/lib/Standard/Tableau/0.0.0-dev/THIRD-PARTY/licenses/APACHE2.0 create mode 100644 distribution/lib/Standard/Tableau/0.0.0-dev/THIRD-PARTY/net.java.dev.jna.jna-5.14.0/NOTICES create mode 100644 distribution/lib/Standard/Tableau/0.0.0-dev/THIRD-PARTY/net.java.dev.jna.jna-platform-5.14.0/NOTICES create mode 100644 tools/legal-review/Tableau/net.java.dev.jna.jna-5.14.0/copyright-keep create mode 100644 tools/legal-review/Tableau/net.java.dev.jna.jna-platform-5.14.0/copyright-ignore create mode 100644 tools/legal-review/Tableau/net.java.dev.jna.jna-platform-5.14.0/copyright-keep create mode 100644 tools/legal-review/Tableau/net.java.dev.jna.jna-platform-5.14.0/copyright-keep-context create mode 100644 tools/legal-review/Tableau/report-state create mode 100644 tools/legal-review/Tableau/reviewed-licenses/Apache-2.0 diff --git a/distribution/lib/Standard/Tableau/0.0.0-dev/THIRD-PARTY/NOTICE b/distribution/lib/Standard/Tableau/0.0.0-dev/THIRD-PARTY/NOTICE new file mode 100644 index 000000000000..9a106bafbf2a --- /dev/null +++ b/distribution/lib/Standard/Tableau/0.0.0-dev/THIRD-PARTY/NOTICE @@ -0,0 +1,12 @@ +Enso +Copyright 2020 - 2024 New Byte Order sp. z o. o. + +'jna', licensed under the Apache-2.0, is distributed with the Tableau. +The license file can be found at `licenses/APACHE2.0`. +Copyright notices related to this dependency can be found in the directory `net.java.dev.jna.jna-5.14.0`. + + +'jna-platform', licensed under the Apache-2.0, is distributed with the Tableau. +The license file can be found at `licenses/APACHE2.0`. +Copyright notices related to this dependency can be found in the directory `net.java.dev.jna.jna-platform-5.14.0`. + diff --git a/distribution/lib/Standard/Tableau/0.0.0-dev/THIRD-PARTY/licenses/APACHE2.0 b/distribution/lib/Standard/Tableau/0.0.0-dev/THIRD-PARTY/licenses/APACHE2.0 new file mode 100644 index 000000000000..261eeb9e9f8b --- /dev/null +++ b/distribution/lib/Standard/Tableau/0.0.0-dev/THIRD-PARTY/licenses/APACHE2.0 @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/distribution/lib/Standard/Tableau/0.0.0-dev/THIRD-PARTY/net.java.dev.jna.jna-5.14.0/NOTICES b/distribution/lib/Standard/Tableau/0.0.0-dev/THIRD-PARTY/net.java.dev.jna.jna-5.14.0/NOTICES new file mode 100644 index 000000000000..ff7264fb2cba --- /dev/null +++ b/distribution/lib/Standard/Tableau/0.0.0-dev/THIRD-PARTY/net.java.dev.jna.jna-5.14.0/NOTICES @@ -0,0 +1,29 @@ +Copyright (c) 2007 Timothy Wall, All Rights Reserved + +Copyright (c) 2007 Wayne Meissner, All Rights Reserved + +Copyright (c) 2007-2008 Timothy Wall, All Rights Reserved + +Copyright (c) 2007-2012 Timothy Wall, All Rights Reserved + +Copyright (c) 2007-2013 Timothy Wall, All Rights Reserved + +Copyright (c) 2007-2015 Timothy Wall, All Rights Reserved + +Copyright (c) 2009 Timothy Wall, All Rights Reserved + +Copyright (c) 2011 Timothy Wall, All Rights Reserved + +Copyright (c) 2012 Timothy Wall, All Rights Reserved + +Copyright (c) 2017 Matthias Bläsing, All Rights Reserved + +Copyright (c) 2018 Matthias Bläsing + +Copyright (c) 2019 Matthias Bläsing, All Rights Reserved + +Copyright (c) 2021, Matthias Bläsing, All Rights Reserved + +Copyright (c) 2022 Carlos Ballesteros, All Rights Reserved + +Copyright 2007 Timothy Wall diff --git a/distribution/lib/Standard/Tableau/0.0.0-dev/THIRD-PARTY/net.java.dev.jna.jna-platform-5.14.0/NOTICES b/distribution/lib/Standard/Tableau/0.0.0-dev/THIRD-PARTY/net.java.dev.jna.jna-platform-5.14.0/NOTICES new file mode 100644 index 000000000000..c6f66e8f63e8 --- /dev/null +++ b/distribution/lib/Standard/Tableau/0.0.0-dev/THIRD-PARTY/net.java.dev.jna.jna-platform-5.14.0/NOTICES @@ -0,0 +1,91 @@ +Copyright (c) 2007 Olivier Chafik + +Copyright (c) 2007 Olivier Chafik, All Rights Reserved + +Copyright (c) 2007 Timothy Wall, All Rights Reserved + +Copyright (c) 2007, 2013 Timothy Wall, Markus Karg, All Rights Reserved + +Copyright (c) 2007-2008 Timothy Wall, All Rights Reserved + +Copyright (c) 2007-2013 Timothy Wall, All Rights Reserved + +Copyright (c) 2008 Timothy Wall, All Rights Reserved + +Copyright (c) 2010 Daniel Doubrovkine, All Rights Reserved + +Copyright (c) 2010 EugineLev, All Rights Reserved + +Copyright (c) 2010 Timothy Wall, All Rights Reserved + +Copyright (c) 2010, 2013 Daniel Doubrovkine, Markus Karg, All Rights Reserved + +Copyright (c) 2010,2011 Daniel Doubrovkine, All Rights Reserved + +Copyright (c) 2011 Daniel Doubrovkine, All Rights Reserved + +Copyright (c) 2011 Denis Tulskiy + +Copyright (c) 2011 Timothy Wall, All Rights Reserved + +Copyright (c) 2012 Tobias Wolf, All Rights Reserved + +Copyright (c) 2013 Ralf Hamberger, Markus Karg, All Rights Reserved + +Copyright (c) 2013 Tobias Wolf, All Rights Reserved + +Copyright (c) 2014 Dr David H. Akehurst (itemis), All Rights Reserved + +Copyright (c) 2014 Reinhard Pointner, All Rights Reserved + +Copyright (c) 2015 Adam Marcionek, All Rights Reserved + +Copyright (c) 2015 Andreas "PAX" L\u00FCck, All Rights Reserved + +Copyright (c) 2015 Daniel Widdis + +Copyright (c) 2015 Goldstein Lyor, 2021 Daniel Widdis, All Rights Reserved + +Copyright (c) 2015 Goldstein Lyor, All Rights Reserved + +Copyright (c) 2015 Michael Freeman, All Rights Reserved + +Copyright (c) 2016 Adam Marcionek, All Rights Reserved + +Copyright (c) 2016 Minoru Sakamoto, All Rights Reserved + +Copyright (c) 2017 Daniel Widdis, All Rights Reserved + +Copyright (c) 2017 Matthias Bläsing, All Rights Reserved + +Copyright (c) 2018 Daniel Widdis, All Rights Reserved + +Copyright (c) 2018 Matthias Bläsing, All Rights Reserved + +Copyright (c) 2018 Roshan Muralidharan, All Rights Reserved + +Copyright (c) 2018 Václav Haisman, All Rights Reserved + +Copyright (c) 2018, 2021 Daniel Widdis, All Rights Reserved + +Copyright (c) 2018,2020,2021 Daniel Widdis, All Rights Reserved + +Copyright (c) 2019 Daniel Widdis + +Copyright (c) 2019 Daniel Widdis, All Rights Reserved + +Copyright (c) 2019 Keve Müller + +Copyright (c) 2019, 2021 Daniel Widdis + +Copyright (c) 2020 Daniel Widdis, All Rights Reserved + +Copyright (c) 2020 Torbjörn Svensson, All Rights Reserved + +Copyright (c) 2022 Daniel Widdis, All Rights Reserved + +Copyright (c) 2023 Reinhard Pointner, All Rights Reserved + +Copyright 2010 Digital Rapids Corp. + +Copyright 2014 Martin Steiger diff --git a/tools/legal-review/Tableau/net.java.dev.jna.jna-5.14.0/copyright-keep b/tools/legal-review/Tableau/net.java.dev.jna.jna-5.14.0/copyright-keep new file mode 100644 index 000000000000..489c03ab287c --- /dev/null +++ b/tools/legal-review/Tableau/net.java.dev.jna.jna-5.14.0/copyright-keep @@ -0,0 +1,15 @@ +Copyright (c) 2007 Timothy Wall, All Rights Reserved +Copyright (c) 2007 Wayne Meissner, All Rights Reserved +Copyright (c) 2007-2008 Timothy Wall, All Rights Reserved +Copyright (c) 2007-2012 Timothy Wall, All Rights Reserved +Copyright (c) 2007-2013 Timothy Wall, All Rights Reserved +Copyright (c) 2007-2015 Timothy Wall, All Rights Reserved +Copyright (c) 2009 Timothy Wall, All Rights Reserved +Copyright (c) 2011 Timothy Wall, All Rights Reserved +Copyright (c) 2012 Timothy Wall, All Rights Reserved +Copyright (c) 2017 Matthias Bläsing, All Rights Reserved +Copyright (c) 2018 Matthias Bläsing +Copyright (c) 2019 Matthias Bläsing, All Rights Reserved +Copyright (c) 2021, Matthias Bläsing, All Rights Reserved +Copyright (c) 2022 Carlos Ballesteros, All Rights Reserved +Copyright 2007 Timothy Wall diff --git a/tools/legal-review/Tableau/net.java.dev.jna.jna-platform-5.14.0/copyright-ignore b/tools/legal-review/Tableau/net.java.dev.jna.jna-platform-5.14.0/copyright-ignore new file mode 100644 index 000000000000..d8210bff2846 --- /dev/null +++ b/tools/legal-review/Tableau/net.java.dev.jna.jna-platform-5.14.0/copyright-ignore @@ -0,0 +1,2 @@ +case 61: return XA_COPYRIGHT; +Atom XA_COPYRIGHT = new Atom(61); diff --git a/tools/legal-review/Tableau/net.java.dev.jna.jna-platform-5.14.0/copyright-keep b/tools/legal-review/Tableau/net.java.dev.jna.jna-platform-5.14.0/copyright-keep new file mode 100644 index 000000000000..54b68a0dc69f --- /dev/null +++ b/tools/legal-review/Tableau/net.java.dev.jna.jna-platform-5.14.0/copyright-keep @@ -0,0 +1,47 @@ +Copyright (c) 2007 Olivier Chafik +Copyright (c) 2007 Olivier Chafik, All Rights Reserved +Copyright (c) 2007 Timothy Wall, All Rights Reserved +Copyright (c) 2007, 2013 Timothy Wall, Markus Karg, All Rights Reserved +Copyright (c) 2007-2008 Timothy Wall, All Rights Reserved +Copyright (c) 2007-2013 Timothy Wall, All Rights Reserved +Copyright (c) 2008 Timothy Wall, All Rights Reserved +Copyright (c) 2010 Daniel Doubrovkine, All Rights Reserved +Copyright (c) 2010 EugineLev, All Rights Reserved +Copyright (c) 2010 Timothy Wall, All Rights Reserved +Copyright (c) 2010, 2013 Daniel Doubrovkine, Markus Karg, All Rights Reserved +Copyright (c) 2010,2011 Daniel Doubrovkine, All Rights Reserved +Copyright (c) 2011 Daniel Doubrovkine, All Rights Reserved +Copyright (c) 2011 Denis Tulskiy +Copyright (c) 2011 Timothy Wall, All Rights Reserved +Copyright (c) 2012 Tobias Wolf, All Rights Reserved +Copyright (c) 2013 Ralf Hamberger, Markus Karg, All Rights Reserved +Copyright (c) 2013 Tobias Wolf, All Rights Reserved +Copyright (c) 2014 Dr David H. Akehurst (itemis), All Rights Reserved +Copyright (c) 2014 Reinhard Pointner, All Rights Reserved +Copyright (c) 2015 Adam Marcionek, All Rights Reserved +Copyright (c) 2015 Andreas "PAX" L\u00FCck, All Rights Reserved +Copyright (c) 2015 Daniel Widdis +Copyright (c) 2015 Goldstein Lyor, 2021 Daniel Widdis, All Rights Reserved +Copyright (c) 2015 Goldstein Lyor, All Rights Reserved +Copyright (c) 2015 Michael Freeman, All Rights Reserved +Copyright (c) 2016 Adam Marcionek, All Rights Reserved +Copyright (c) 2016 Minoru Sakamoto, All Rights Reserved +Copyright (c) 2017 Daniel Widdis, All Rights Reserved +Copyright (c) 2017 Matthias Bläsing, All Rights Reserved +Copyright (c) 2018 Daniel Widdis, All Rights Reserved +Copyright (c) 2018 Matthias Bläsing, All Rights Reserved +Copyright (c) 2018 Roshan Muralidharan, All Rights Reserved +Copyright (c) 2018 Václav Haisman, All Rights Reserved +Copyright (c) 2018, 2021 Daniel Widdis, All Rights Reserved +Copyright (c) 2018,2020,2021 Daniel Widdis, All Rights Reserved +Copyright (c) 2019 Daniel Widdis +Copyright (c) 2019 Daniel Widdis, All Rights Reserved +Copyright (c) 2019 Keve Müller +Copyright (c) 2019, 2021 Daniel Widdis +Copyright (c) 2020 Daniel Widdis, All Rights Reserved +Copyright (c) 2020 Torbjörn Svensson, All Rights Reserved +Copyright (c) 2022 Daniel Widdis, All Rights Reserved +Copyright (c) 2023 Reinhard Pointner, All Rights Reserved +Copyright 2010 Digital Rapids Corp. +Copyright 2014 Martin Steiger +Conversion code in this class Copyright 2002-2004 Apache Software Foundation. diff --git a/tools/legal-review/Tableau/net.java.dev.jna.jna-platform-5.14.0/copyright-keep-context b/tools/legal-review/Tableau/net.java.dev.jna.jna-platform-5.14.0/copyright-keep-context new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tools/legal-review/Tableau/report-state b/tools/legal-review/Tableau/report-state new file mode 100644 index 000000000000..b3168619a821 --- /dev/null +++ b/tools/legal-review/Tableau/report-state @@ -0,0 +1,3 @@ +B499CA6488883A07DDC6EE0844F6FD12155E8E36FA8721714C6572E1388728F5 +D11AE1204B12BC2A9DE252969FBC7861FD9468C10E68A2C11DDDD3A958D1E425 +1 diff --git a/tools/legal-review/Tableau/reviewed-licenses/Apache-2.0 b/tools/legal-review/Tableau/reviewed-licenses/Apache-2.0 new file mode 100644 index 000000000000..1a331ec60e02 --- /dev/null +++ b/tools/legal-review/Tableau/reviewed-licenses/Apache-2.0 @@ -0,0 +1 @@ +tools/legal-review/license-texts/APACHE2.0 \ No newline at end of file From cd4c0a398d56b1b1fd68656db1072f74fdd72fa3 Mon Sep 17 00:00:00 2001 From: James Dunkerley Date: Thu, 1 Aug 2024 17:46:43 +0100 Subject: [PATCH 19/35] Scala fmt --- build.sbt | 10 ++++++++-- .../org/enso/distribution/DistributionManager.scala | 2 +- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/build.sbt b/build.sbt index 46475613eba7..96344285d1d1 100644 --- a/build.sbt +++ b/build.sbt @@ -164,8 +164,14 @@ GatherLicenses.distributions := Seq( makeStdLibDistribution("Database", Distribution.sbtProjects(`std-database`)), makeStdLibDistribution("Image", Distribution.sbtProjects(`std-image`)), makeStdLibDistribution("AWS", Distribution.sbtProjects(`std-aws`)), - makeStdLibDistribution("Snowflake", Distribution.sbtProjects(`std-snowflake`)), - makeStdLibDistribution("Microsoft", Distribution.sbtProjects(`std-microsoft`)), + makeStdLibDistribution( + "Snowflake", + Distribution.sbtProjects(`std-snowflake`) + ), + makeStdLibDistribution( + "Microsoft", + Distribution.sbtProjects(`std-microsoft`) + ), makeStdLibDistribution("Tableau", Distribution.sbtProjects(`std-tableau`)) ) diff --git a/lib/scala/distribution-manager/src/main/scala/org/enso/distribution/DistributionManager.scala b/lib/scala/distribution-manager/src/main/scala/org/enso/distribution/DistributionManager.scala index 51df5f65f5d7..640697816221 100644 --- a/lib/scala/distribution-manager/src/main/scala/org/enso/distribution/DistributionManager.scala +++ b/lib/scala/distribution-manager/src/main/scala/org/enso/distribution/DistributionManager.scala @@ -292,7 +292,7 @@ class DistributionManager(val env: Environment) { private val LINUX_ENSO_DIRECTORY = "enso" private val MACOS_ENSO_DIRECTORY = "org.enso" - private val WINDOWS_ENSO_DIRECTORY = "enso" + private val WINDOWS_ENSO_DIRECTORY = "enso" /** Data directory for an installed distribution. */ From a1f52e7f27a74905c6f079bfe1db70b899ed2a69 Mon Sep 17 00:00:00 2001 From: Hubert Plociniczak Date: Fri, 2 Aug 2024 01:05:46 +0200 Subject: [PATCH 20/35] Dependency resolution is a prerequisite for compilation Should make it more resilient to incremental compilation steps that appear on CI. Also reproducible locally when `lib`'s contents is removed manually. --- build.sbt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/build.sbt b/build.sbt index 96344285d1d1..034cfc2923f2 100644 --- a/build.sbt +++ b/build.sbt @@ -3642,6 +3642,9 @@ lazy val `std-tableau` = project fetchZipToUnmanaged := { val unmanagedDirectory = (Compile / unmanagedBase).value if (IO.listFiles(unmanagedDirectory).size < 2) { // Heuristic, should have at least hyperapi jar and os-specific one. + System.out.println( + "std-tableau's unmanaged dependencies are not up-to-date. fetching..." + ) unmanagedDirectory.mkdirs() val unmanagedPath = unmanagedDirectory.toPath IO.withTemporaryDirectory( @@ -3663,6 +3666,9 @@ lazy val `std-tableau` = project Compile / compile / compileInputs := (Compile / compile / compileInputs) .dependsOn(SPIHelpers.ensureSPIConsistency) .value, + Compile / compile := (Compile / compile) + .dependsOn(fetchZipToUnmanaged) + .value, Compile / unmanagedClasspath := (Compile / unmanagedClasspath) .dependsOn(fetchZipToUnmanaged) .value, From a0c5357f65ccc543eed6c740fd85854aa8f0932b Mon Sep 17 00:00:00 2001 From: James Dunkerley Date: Fri, 2 Aug 2024 09:45:28 +0100 Subject: [PATCH 21/35] Add Hyper Copyright --- tools/legal-review/Tableau/files-add/COPYRIGHT-HYPERAPI | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 tools/legal-review/Tableau/files-add/COPYRIGHT-HYPERAPI diff --git a/tools/legal-review/Tableau/files-add/COPYRIGHT-HYPERAPI b/tools/legal-review/Tableau/files-add/COPYRIGHT-HYPERAPI new file mode 100644 index 000000000000..a2f288c50a6e --- /dev/null +++ b/tools/legal-review/Tableau/files-add/COPYRIGHT-HYPERAPI @@ -0,0 +1,3 @@ +Copyright (c) 2016 - 2023, Salesforce, Inc. and its licensors. All rights reserved. + +Protected by U.S. Patents and Trademarks as noted at http://www.tableau.com/ip; Patents pending. From 30f3c478008b62ef2fa7049f29be7abbc9a33c4f Mon Sep 17 00:00:00 2001 From: James Dunkerley Date: Fri, 2 Aug 2024 09:54:42 +0100 Subject: [PATCH 22/35] Useless human... --- .../Standard/Tableau/0.0.0-dev/THIRD-PARTY/COPYRIGHT-HYPERAPI | 3 +++ .../THIRD-PARTY/net.java.dev.jna.jna-platform-5.14.0/NOTICES | 2 ++ tools/legal-review/Tableau/report-state | 4 ++-- 3 files changed, 7 insertions(+), 2 deletions(-) create mode 100644 distribution/lib/Standard/Tableau/0.0.0-dev/THIRD-PARTY/COPYRIGHT-HYPERAPI diff --git a/distribution/lib/Standard/Tableau/0.0.0-dev/THIRD-PARTY/COPYRIGHT-HYPERAPI b/distribution/lib/Standard/Tableau/0.0.0-dev/THIRD-PARTY/COPYRIGHT-HYPERAPI new file mode 100644 index 000000000000..a2f288c50a6e --- /dev/null +++ b/distribution/lib/Standard/Tableau/0.0.0-dev/THIRD-PARTY/COPYRIGHT-HYPERAPI @@ -0,0 +1,3 @@ +Copyright (c) 2016 - 2023, Salesforce, Inc. and its licensors. All rights reserved. + +Protected by U.S. Patents and Trademarks as noted at http://www.tableau.com/ip; Patents pending. diff --git a/distribution/lib/Standard/Tableau/0.0.0-dev/THIRD-PARTY/net.java.dev.jna.jna-platform-5.14.0/NOTICES b/distribution/lib/Standard/Tableau/0.0.0-dev/THIRD-PARTY/net.java.dev.jna.jna-platform-5.14.0/NOTICES index c6f66e8f63e8..352d5f887d59 100644 --- a/distribution/lib/Standard/Tableau/0.0.0-dev/THIRD-PARTY/net.java.dev.jna.jna-platform-5.14.0/NOTICES +++ b/distribution/lib/Standard/Tableau/0.0.0-dev/THIRD-PARTY/net.java.dev.jna.jna-platform-5.14.0/NOTICES @@ -1,3 +1,5 @@ +Conversion code in this class Copyright 2002-2004 Apache Software Foundation. + Copyright (c) 2007 Olivier Chafik Copyright (c) 2007 Olivier Chafik, All Rights Reserved diff --git a/tools/legal-review/Tableau/report-state b/tools/legal-review/Tableau/report-state index b3168619a821..2ccf9d57fa4a 100644 --- a/tools/legal-review/Tableau/report-state +++ b/tools/legal-review/Tableau/report-state @@ -1,3 +1,3 @@ B499CA6488883A07DDC6EE0844F6FD12155E8E36FA8721714C6572E1388728F5 -D11AE1204B12BC2A9DE252969FBC7861FD9468C10E68A2C11DDDD3A958D1E425 -1 +F43F163A7B66354A2F2EC3F4B9121806D0A2B596D44578AFF2EEF363E8CE2AE4 +0 From 6594c50f3d7413ebd204f1e3d56210c036583a62 Mon Sep 17 00:00:00 2001 From: Hubert Plociniczak Date: Fri, 2 Aug 2024 11:06:05 +0200 Subject: [PATCH 23/35] add more info to sbt --- build.sbt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/build.sbt b/build.sbt index 034cfc2923f2..7ff514937c52 100644 --- a/build.sbt +++ b/build.sbt @@ -3661,6 +3661,10 @@ lazy val `std-tableau` = project }, keepDirectory = false ) + } else { + System.out.println( + "std-tableau's unmanaged dependencies are up-to-date" + ) } }, Compile / compile / compileInputs := (Compile / compile / compileInputs) From 35828c3de856d9fa28fbaf6d614d9e8a8d88a55c Mon Sep 17 00:00:00 2001 From: James Dunkerley Date: Fri, 2 Aug 2024 10:49:29 +0100 Subject: [PATCH 24/35] Block comments. Rename method. --- .../Standard/Tableau/0.0.0-dev/src/Hyper_Table.enso | 12 ++++++++++-- .../java/org/enso/base/arrays/LongArrayList.java | 8 ++++---- .../org/enso/base/enso_cloud/EnsoSecretHelper.java | 2 +- .../org/enso/table/data/table/join/JoinResult.java | 2 +- .../src/main/java/org/enso/tableau/HyperReader.java | 4 ++-- .../src/main/java/org/enso/tableau/HyperTable.java | 2 +- .../main/java/org/enso/tableau/HyperTableColumn.java | 6 +++--- .../src/main/java/org/enso/tableau/OSPlatform.java | 2 +- .../java/org/enso/tableau/TableColumnBuilder.java | 5 ++--- 9 files changed, 25 insertions(+), 18 deletions(-) diff --git a/distribution/lib/Standard/Tableau/0.0.0-dev/src/Hyper_Table.enso b/distribution/lib/Standard/Tableau/0.0.0-dev/src/Hyper_Table.enso index d585f0fdebc8..b8621112a7a5 100644 --- a/distribution/lib/Standard/Tableau/0.0.0-dev/src/Hyper_Table.enso +++ b/distribution/lib/Standard/Tableau/0.0.0-dev/src/Hyper_Table.enso @@ -29,8 +29,8 @@ type Hyper_Table ## GROUP Standard.Base.Metadata ICON metadata Reads The Columns for the Table - columns : Vector Hyper_Column - columns self = + column_metadata : Vector Hyper_Column + column_metadata self = array = HyperReader.readStructure self.file.file.path self.schema self.table array.map column-> value_type = case column.typeID of @@ -59,6 +59,14 @@ type Hyper_Table _ -> Value_Type.Unsupported_Data_Type "Unknown" Any Hyper_Column.Value column.name value_type column.nullable + ## GROUP Standard.Base.Metadata + ICON metadata + Reads The Column Info for the Table + column_info : Table + column_info self = + metadata = self.column_metadata + + ## GROUP Standard.Base.Input ICON data_input Reads the Table into Enso Table diff --git a/std-bits/base/src/main/java/org/enso/base/arrays/LongArrayList.java b/std-bits/base/src/main/java/org/enso/base/arrays/LongArrayList.java index cb83d14f4041..173fdee2d290 100644 --- a/std-bits/base/src/main/java/org/enso/base/arrays/LongArrayList.java +++ b/std-bits/base/src/main/java/org/enso/base/arrays/LongArrayList.java @@ -11,12 +11,12 @@ public LongArrayList() { backingStorage = new long[32]; } - // ** Gets the number of elements in the list. */ + /** Gets the number of elements in the list. */ public int getSize() { return lastIndex + 1; } - // ** Gets an element from the list. */ + /** Gets an element from the list. */ public long get(int index) { if (index > lastIndex) { throw new IndexOutOfBoundsException(index); @@ -24,12 +24,12 @@ public long get(int index) { return backingStorage[index]; } - // ** Gets an element from the list. */ + /** Gets an element from the list. */ public long getOrLast(int index) { return backingStorage[Math.min(index, lastIndex)]; } - // ** Adds an element to the list. */ + /** Adds an element to the list. */ public void add(long x) { int index; diff --git a/std-bits/base/src/main/java/org/enso/base/enso_cloud/EnsoSecretHelper.java b/std-bits/base/src/main/java/org/enso/base/enso_cloud/EnsoSecretHelper.java index f82bf378c8d2..2260ba3ca904 100644 --- a/std-bits/base/src/main/java/org/enso/base/enso_cloud/EnsoSecretHelper.java +++ b/std-bits/base/src/main/java/org/enso/base/enso_cloud/EnsoSecretHelper.java @@ -53,7 +53,7 @@ private static URI resolveURI(URIWithSecrets uri) { } } - // ** Makes a request with secrets in the query string or headers. **// + /** Makes a request with secrets in the query string or headers. **/ public static EnsoHttpResponse makeRequest( HttpClient client, Builder builder, diff --git a/std-bits/table/src/main/java/org/enso/table/data/table/join/JoinResult.java b/std-bits/table/src/main/java/org/enso/table/data/table/join/JoinResult.java index 6eb719e403e7..8ecc76f55e61 100644 --- a/std-bits/table/src/main/java/org/enso/table/data/table/join/JoinResult.java +++ b/std-bits/table/src/main/java/org/enso/table/data/table/join/JoinResult.java @@ -14,7 +14,7 @@ public JoinResult(int[] leftIndices, int[] rightIndices, int length) { this.rightIndices = rightIndices; } - // ** Represents a pair of indices of matched rows. -1 means an unmatched row.*/ + /** Represents a pair of indices of matched rows. -1 means an unmatched row.*/ public record RowPair(int leftIndex, int rightIndex) {} public OrderMask getLeftOrderMask() { diff --git a/std-bits/tableau/src/main/java/org/enso/tableau/HyperReader.java b/std-bits/tableau/src/main/java/org/enso/tableau/HyperReader.java index 809600840db4..f737b499a1ed 100644 --- a/std-bits/tableau/src/main/java/org/enso/tableau/HyperReader.java +++ b/std-bits/tableau/src/main/java/org/enso/tableau/HyperReader.java @@ -22,7 +22,7 @@ import org.enso.table.data.table.Column; import org.enso.table.problems.ProblemAggregator; -// ** Class responsible for reading from Tableau Hyper files. */ +/** Class responsible for reading from Tableau Hyper files. */ public class HyperReader { public static final Path HYPER_PATH = Path.of(getHyperPath()); private static HyperProcess process; @@ -170,7 +170,7 @@ private static HyperTableColumn[] readStructureInternal( var definition = catalog.getTableDefinition(tableNameObject); var columns = definition.getColumns(); return IntStream.range(0, columns.size()) - .mapToObj(i -> HyperTableColumn.FromHyperColumn(i, columns.get(i))) + .mapToObj(i -> HyperTableColumn.fromHyperColumn(i, columns.get(i))) .toArray(HyperTableColumn[]::new); } diff --git a/std-bits/tableau/src/main/java/org/enso/tableau/HyperTable.java b/std-bits/tableau/src/main/java/org/enso/tableau/HyperTable.java index fcb1c40a018d..1da0defd8c63 100644 --- a/std-bits/tableau/src/main/java/org/enso/tableau/HyperTable.java +++ b/std-bits/tableau/src/main/java/org/enso/tableau/HyperTable.java @@ -1,4 +1,4 @@ package org.enso.tableau; -// ** Record type for representing a Hyper table. */ +/** Record type for representing a Hyper table. */ public record HyperTable(String schema, String name) {} diff --git a/std-bits/tableau/src/main/java/org/enso/tableau/HyperTableColumn.java b/std-bits/tableau/src/main/java/org/enso/tableau/HyperTableColumn.java index 7ecdae8c93d9..e059d7ae2f1c 100644 --- a/std-bits/tableau/src/main/java/org/enso/tableau/HyperTableColumn.java +++ b/std-bits/tableau/src/main/java/org/enso/tableau/HyperTableColumn.java @@ -14,12 +14,12 @@ public record HyperTableColumn( OptionalInt length, OptionalInt precision, OptionalInt scale) { - // ** Type ID for JSON data. */ + /** Type ID for JSON data. */ public static final int JSON = 10001; - // ** Type ID for INTERVAL data. */ + /** Type ID for INTERVAL data. */ public static final int INTERVAL = 10002; - static HyperTableColumn FromHyperColumn(int index, TableDefinition.Column hyperColumn) { + static HyperTableColumn fromHyperColumn(int index, TableDefinition.Column hyperColumn) { return new HyperTableColumn( index, hyperColumn.getName().getUnescaped(), diff --git a/std-bits/tableau/src/main/java/org/enso/tableau/OSPlatform.java b/std-bits/tableau/src/main/java/org/enso/tableau/OSPlatform.java index c1b8593848dd..568ed62b3744 100644 --- a/std-bits/tableau/src/main/java/org/enso/tableau/OSPlatform.java +++ b/std-bits/tableau/src/main/java/org/enso/tableau/OSPlatform.java @@ -7,7 +7,7 @@ public enum OSPlatform { LINUX, OTHER; - // ** Returns the current platform. */ + /** Returns the current platform. */ public static final OSPlatform CurrentPlatform = getPlatform(); private static OSPlatform getPlatform() { diff --git a/std-bits/tableau/src/main/java/org/enso/tableau/TableColumnBuilder.java b/std-bits/tableau/src/main/java/org/enso/tableau/TableColumnBuilder.java index ed341b347249..04b5b3e5c351 100644 --- a/std-bits/tableau/src/main/java/org/enso/tableau/TableColumnBuilder.java +++ b/std-bits/tableau/src/main/java/org/enso/tableau/TableColumnBuilder.java @@ -22,7 +22,7 @@ import org.enso.table.data.column.storage.type.TextType; import org.enso.table.problems.ProblemAggregator; -// ** A builder for a single column of a table. */ +/** A builder for a single column of a table. */ record TableColumnBuilder(Builder builder, Consumer appendMethod) { private static Consumer nullAppender(Builder builder, int index, Consumer inner) { return r -> { @@ -34,8 +34,7 @@ private static Consumer nullAppender(Builder builder, int index, Consume }; } - // ** Convert a Tableau Interval into either a Duration or a Period (with fallback to String if - // needed). */ + /** Convert a Tableau Interval into either a Duration or a Period (with fallback to String if needed). */ private static Object readInterval(Result r, int index) { var interval = r.getInterval(index); if (interval.getMonths() == 0 && interval.getYears() == 0) { From 3708118d817f0d8001e503ce27a4f1dc44d4af8a Mon Sep 17 00:00:00 2001 From: James Dunkerley Date: Fri, 2 Aug 2024 10:54:11 +0100 Subject: [PATCH 25/35] Java Format. --- .../main/java/org/enso/base/enso_cloud/EnsoSecretHelper.java | 2 +- .../main/java/org/enso/table/data/table/join/JoinResult.java | 2 +- .../src/main/java/org/enso/tableau/HyperTableColumn.java | 1 + .../src/main/java/org/enso/tableau/TableColumnBuilder.java | 5 ++++- 4 files changed, 7 insertions(+), 3 deletions(-) diff --git a/std-bits/base/src/main/java/org/enso/base/enso_cloud/EnsoSecretHelper.java b/std-bits/base/src/main/java/org/enso/base/enso_cloud/EnsoSecretHelper.java index 2260ba3ca904..b4134ed1b9bb 100644 --- a/std-bits/base/src/main/java/org/enso/base/enso_cloud/EnsoSecretHelper.java +++ b/std-bits/base/src/main/java/org/enso/base/enso_cloud/EnsoSecretHelper.java @@ -53,7 +53,7 @@ private static URI resolveURI(URIWithSecrets uri) { } } - /** Makes a request with secrets in the query string or headers. **/ + /** Makes a request with secrets in the query string or headers. * */ public static EnsoHttpResponse makeRequest( HttpClient client, Builder builder, diff --git a/std-bits/table/src/main/java/org/enso/table/data/table/join/JoinResult.java b/std-bits/table/src/main/java/org/enso/table/data/table/join/JoinResult.java index 8ecc76f55e61..dd905f69558f 100644 --- a/std-bits/table/src/main/java/org/enso/table/data/table/join/JoinResult.java +++ b/std-bits/table/src/main/java/org/enso/table/data/table/join/JoinResult.java @@ -14,7 +14,7 @@ public JoinResult(int[] leftIndices, int[] rightIndices, int length) { this.rightIndices = rightIndices; } - /** Represents a pair of indices of matched rows. -1 means an unmatched row.*/ + /** Represents a pair of indices of matched rows. -1 means an unmatched row. */ public record RowPair(int leftIndex, int rightIndex) {} public OrderMask getLeftOrderMask() { diff --git a/std-bits/tableau/src/main/java/org/enso/tableau/HyperTableColumn.java b/std-bits/tableau/src/main/java/org/enso/tableau/HyperTableColumn.java index e059d7ae2f1c..38f35edd15a0 100644 --- a/std-bits/tableau/src/main/java/org/enso/tableau/HyperTableColumn.java +++ b/std-bits/tableau/src/main/java/org/enso/tableau/HyperTableColumn.java @@ -16,6 +16,7 @@ public record HyperTableColumn( OptionalInt scale) { /** Type ID for JSON data. */ public static final int JSON = 10001; + /** Type ID for INTERVAL data. */ public static final int INTERVAL = 10002; diff --git a/std-bits/tableau/src/main/java/org/enso/tableau/TableColumnBuilder.java b/std-bits/tableau/src/main/java/org/enso/tableau/TableColumnBuilder.java index 04b5b3e5c351..faff8caaba54 100644 --- a/std-bits/tableau/src/main/java/org/enso/tableau/TableColumnBuilder.java +++ b/std-bits/tableau/src/main/java/org/enso/tableau/TableColumnBuilder.java @@ -34,7 +34,10 @@ private static Consumer nullAppender(Builder builder, int index, Consume }; } - /** Convert a Tableau Interval into either a Duration or a Period (with fallback to String if needed). */ + /** + * Convert a Tableau Interval into either a Duration or a Period (with fallback to String if + * needed). + */ private static Object readInterval(Result r, int index) { var interval = r.getInterval(index); if (interval.getMonths() == 0 && interval.getYears() == 0) { From 4c4991a42958d6623a44f478b4ef45585e3769f1 Mon Sep 17 00:00:00 2001 From: James Dunkerley Date: Fri, 2 Aug 2024 12:10:50 +0100 Subject: [PATCH 26/35] Add the start of unit tests for Tableau. --- .../Tableau/0.0.0-dev/src/Hyper_Table.enso | 21 +++++++--- test/Tableau_Tests/package.yaml | 7 ++++ test/Tableau_Tests/src/Main.enso | 15 ++++++++ test/Tableau_Tests/src/Read_Spec.enso | 13 +++++++ test/Tableau_Tests/src/Structure_Spec.enso | 38 +++++++++++++++++++ 5 files changed, 88 insertions(+), 6 deletions(-) create mode 100644 test/Tableau_Tests/package.yaml create mode 100644 test/Tableau_Tests/src/Main.enso create mode 100644 test/Tableau_Tests/src/Read_Spec.enso create mode 100644 test/Tableau_Tests/src/Structure_Spec.enso diff --git a/distribution/lib/Standard/Tableau/0.0.0-dev/src/Hyper_Table.enso b/distribution/lib/Standard/Tableau/0.0.0-dev/src/Hyper_Table.enso index b8621112a7a5..fcad90bcd466 100644 --- a/distribution/lib/Standard/Tableau/0.0.0-dev/src/Hyper_Table.enso +++ b/distribution/lib/Standard/Tableau/0.0.0-dev/src/Hyper_Table.enso @@ -15,7 +15,15 @@ polyglot java import org.enso.tableau.HyperTableColumn type Hyper_Table ## PRIVATE Represents a Tableau Hyper Table. - private Value file:Hyper_File schema:Text table:Text + private Value file:Hyper_File internal_schema:Text internal_table:Text + + ## The schema of the table. + schema : Text + schema self = self.internal_schema + + ## The name of the table. + table : Text + table self = self.internal_table ## PRIVATE to_display_text : Text @@ -26,8 +34,7 @@ type Hyper_Table to_js_object self = JS_Object.from_pairs [["type", "Hyper_Table"], ["schema", self.schema], ["table", self.table], ["file", self.file.file.path]] - ## GROUP Standard.Base.Metadata - ICON metadata + ## ICON metadata Reads The Columns for the Table column_metadata : Vector Hyper_Column column_metadata self = @@ -61,11 +68,13 @@ type Hyper_Table ## GROUP Standard.Base.Metadata ICON metadata - Reads The Column Info for the Table + Returns a Table describing this table's contents. + + The table lists all columns and value types of each column. column_info : Table column_info self = - metadata = self.column_metadata - + cols = self.column_metadata + Table.new [["Column", cols.map .name], ["Value Type", cols.map .value_type], ["Nullable", cols.map .nullable]] ## GROUP Standard.Base.Input ICON data_input diff --git a/test/Tableau_Tests/package.yaml b/test/Tableau_Tests/package.yaml new file mode 100644 index 000000000000..78aec8212618 --- /dev/null +++ b/test/Tableau_Tests/package.yaml @@ -0,0 +1,7 @@ +name: Tableau_Tests +namespace: enso_dev +version: 0.0.1 +license: MIT +author: enso-dev@enso.org +maintainer: enso-dev@enso.org +prefer-local-libraries: true diff --git a/test/Tableau_Tests/src/Main.enso b/test/Tableau_Tests/src/Main.enso new file mode 100644 index 000000000000..da72d33f5538 --- /dev/null +++ b/test/Tableau_Tests/src/Main.enso @@ -0,0 +1,15 @@ +from Standard.Base import all + +from Standard.Test import Test + +import project.Read_Spec +import project.Structure_Spec + +add_specs suite_builder = + Structure_Spec.add_specs suite_builder + Read_Spec.add_specs suite_builder + +main filter=Nothing = + suite = Test.build suite_builder-> + add_specs suite_builder + suite.run_with_filter filter diff --git a/test/Tableau_Tests/src/Read_Spec.enso b/test/Tableau_Tests/src/Read_Spec.enso new file mode 100644 index 000000000000..24794bee6120 --- /dev/null +++ b/test/Tableau_Tests/src/Read_Spec.enso @@ -0,0 +1,13 @@ +from Standard.Base import all + +from Standard.Tableau import Hyper_File + +from Standard.Test import all + +main filter=Nothing = + suite = Test.build suite_builder-> + add_specs suite_builder + suite.run_with_filter filter + +add_specs suite_builder = suite_builder.group "Read Tables" group_builder-> + _ = group_builder diff --git a/test/Tableau_Tests/src/Structure_Spec.enso b/test/Tableau_Tests/src/Structure_Spec.enso new file mode 100644 index 000000000000..1a4267fd5688 --- /dev/null +++ b/test/Tableau_Tests/src/Structure_Spec.enso @@ -0,0 +1,38 @@ +from Standard.Base import all + +from Standard.Tableau import Hyper_File + +from Standard.Test import all + +main filter=Nothing = + suite = Test.build suite_builder-> + add_specs suite_builder + suite.run_with_filter filter + +add_specs suite_builder = suite_builder.group "Structure" group_builder-> + names_file = enso_project.data / "names.hyper" + + group_builder.specify "should default to all schema" <| + Hyper_File.new names_file . schema . should_equal "*" + + group_builder.specify "should be able to list schema" <| + schemas = Hyper_File.new names_file . schemas + schemas.should_equal ["public", "Extract"] + + group_builder.specify "should default to set schema" <| + Hyper_File.new names_file schema="Extract" . schema . should_equal "Extract" + Hyper_File.new names_file . set_schema "Extract" . schema . should_equal "Extract" + + group_builder.specify "should be able to list table" <| + tables = Hyper_File.new names_file . tables + tables.length.should_equal 1 + tables.first.schema.should_equal "Extract" + tables.first.table.should_equal "Extract" + + tables_2 = Hyper_File.new names_file schema="Extract" . tables + tables_2.length.should_equal 1 + tables_2.first.schema.should_equal "Extract" + tables_2.first.table.should_equal "Extract" + + tables_3 = Hyper_File.new names_file schema="public" . tables + tables_3.length.should_equal 0 From 00546cc4854c8b3bd25134f9e4d1b1a250596619 Mon Sep 17 00:00:00 2001 From: James Dunkerley Date: Fri, 2 Aug 2024 13:15:31 +0100 Subject: [PATCH 27/35] Handle process failure more gracefully. Handle missing file. --- .../Tableau/0.0.0-dev/src/Hyper_File.enso | 19 +++++--- .../java/org/enso/tableau/HyperReader.java | 45 +++++++++++-------- test/Tableau_Tests/src/Structure_Spec.enso | 26 +++++++++++ 3 files changed, 64 insertions(+), 26 deletions(-) diff --git a/distribution/lib/Standard/Tableau/0.0.0-dev/src/Hyper_File.enso b/distribution/lib/Standard/Tableau/0.0.0-dev/src/Hyper_File.enso index 2c426cd21027..f7a0c1441e53 100644 --- a/distribution/lib/Standard/Tableau/0.0.0-dev/src/Hyper_File.enso +++ b/distribution/lib/Standard/Tableau/0.0.0-dev/src/Hyper_File.enso @@ -1,4 +1,5 @@ from Standard.Base import all +import Standard.Base.Errors.File_Error.File_Error import Standard.Base.Errors.Illegal_Argument.Illegal_Argument from Standard.Base.Metadata.Choice import Option from Standard.Base.Metadata.Widget import Single_Choice @@ -19,7 +20,9 @@ type Hyper_File - file: The file to read. - schema: The schema to read or `*` for all schemas. new : File -> Text -> Hyper_File - new file:File schema:Text='*' = Hyper_File.Value file schema + new file:File schema:Text='*' = + if schema == "" then Error.throw (Illegal_Argument.Error "Schema cannot be empty.") else + Hyper_File.Value file schema ## PRIVATE A representation of a Tableau Hyper Extract file. @@ -28,7 +31,7 @@ type Hyper_File ## ICON metadata Returns the list of schemas for the connection within the current database (or catalog). schemas : Vector Text - schemas self = + schemas self = File_Error.handle_java_exceptions self.file <| array = HyperReader.readSchemas self.file.path Vector.from_polyglot_array array @@ -47,7 +50,8 @@ type Hyper_File set_schema : Text -> Hyper_File set_schema self schema = if schema == self.schema then self else - Hyper_File.Value self.file schema + if schema == "" then Error.throw (Illegal_Argument.Error "Schema cannot be empty.") else + Hyper_File.Value self.file schema ## GROUP Standard.Base.Metadata ICON metadata @@ -55,10 +59,11 @@ type Hyper_File @schema (hyper -> make_schema_selector hyper True) tables : Text -> Vector Hyper_Table tables self schema:Text=self.schema = if schema == "" then self.tables self.schema else - array = case schema of - "*" -> HyperReader.listTablesAllSchemas self.file.path - _ -> HyperReader.listTables self.file.path schema - array.map t-> Hyper_Table.Value self t.schema t.name + File_Error.handle_java_exceptions self.file <| + array = case schema of + "*" -> HyperReader.listTablesAllSchemas self.file.path + _ -> HyperReader.listTables self.file.path schema + array.map t-> Hyper_Table.Value self t.schema t.name ## ALIAS sheet, get GROUP Standard.Base.Input diff --git a/std-bits/tableau/src/main/java/org/enso/tableau/HyperReader.java b/std-bits/tableau/src/main/java/org/enso/tableau/HyperReader.java index f737b499a1ed..6634df68ef62 100644 --- a/std-bits/tableau/src/main/java/org/enso/tableau/HyperReader.java +++ b/std-bits/tableau/src/main/java/org/enso/tableau/HyperReader.java @@ -1,11 +1,8 @@ package org.enso.tableau; -import com.tableau.hyperapi.Catalog; -import com.tableau.hyperapi.Connection; -import com.tableau.hyperapi.HyperProcess; -import com.tableau.hyperapi.SchemaName; -import com.tableau.hyperapi.TableName; -import com.tableau.hyperapi.Telemetry; +import com.tableau.hyperapi.*; + +import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; import java.net.URI; @@ -67,8 +64,10 @@ private static HyperProcess getProcess() throws IOException { "https://enso-data-samples.s3.us-west-1.amazonaws.com/tableau/macos-arm64/hyperd", "hyperd", true); - case MAX_X64 -> throw new IOException( - "Unsupported platform: Only ARM64 Mac is supported."); + case MAX_X64 -> downloadHyper( + "https://enso-data-samples.s3.us-west-1.amazonaws.com/tableau/macos-x64/hyperd", + "hyperd", + true); case LINUX -> downloadHyper( "https://enso-data-samples.s3.us-west-1.amazonaws.com/tableau/linux/hyperd", "hyperd", @@ -91,7 +90,7 @@ private static HyperProcess getProcess() throws IOException { process = new HyperProcess(HYPER_PATH, Telemetry.DO_NOT_SEND_USAGE_DATA_TO_TABLEAU); } catch (Throwable ioe) { LOGGER.log(Level.SEVERE, "Failed to start Hyper process.", ioe); - throw ioe; + throw new IOException("Failed to start Hyper process.", ioe); } } finally { Thread.currentThread().setContextClassLoader(contextClassLoader); @@ -116,9 +115,21 @@ private static void downloadHyper(String uri, String fileName, boolean setExecut } } - public static String[] readSchemas(String path) throws IOException { + private static Connection getConnection(String path) throws IOException { var process = getProcess(); - try (var connection = new Connection(process.getEndpoint(), path)) { + try { + return new Connection(process.getEndpoint(), path, CreateMode.NONE); + } catch (HyperException e) { + if (e.getMessage().contains("The database does not exist")) { + throw new FileNotFoundException("Database not found: " + path); + } else { + throw new IOException("Failed to open database: " + path, e); + } + } + } + + public static String[] readSchemas(String path) throws IOException { + try (var connection = getConnection(path)) { var catalog = connection.getCatalog(); return catalog.getSchemaNames().stream() .map(s -> s.getName().getUnescaped()) @@ -127,8 +138,7 @@ public static String[] readSchemas(String path) throws IOException { } public static HyperTable[] listTablesAllSchemas(String path) throws IOException { - var process = getProcess(); - try (var connection = new Connection(process.getEndpoint(), path)) { + try (var connection = getConnection(path)) { var catalog = connection.getCatalog(); return listTablesImpl(catalog, catalog.getSchemaNames()); } @@ -136,8 +146,7 @@ public static HyperTable[] listTablesAllSchemas(String path) throws IOException public static HyperTable[] listTables(String path, String schemaName) throws IOException { var schemaNames = List.of(new SchemaName(schemaName)); - var process = getProcess(); - try (var connection = new Connection(process.getEndpoint(), path)) { + try (var connection = getConnection(path)) { var catalog = connection.getCatalog(); return listTablesImpl(catalog, schemaNames); } @@ -158,8 +167,7 @@ private static HyperTable[] listTablesImpl(Catalog catalog, List sch public static HyperTableColumn[] readStructure(String path, String schemaName, String tableName) throws IOException { var tableNameObject = new TableName(new SchemaName(schemaName), tableName); - var process = getProcess(); - try (var connection = new Connection(process.getEndpoint(), path)) { + try (var connection = getConnection(path)) { return readStructureInternal(connection, tableNameObject); } } @@ -183,8 +191,7 @@ public static Column[] readTable( throws IOException { var tableNameObject = new TableName(new SchemaName(schemaName), tableName); var query = "SELECT * FROM " + tableNameObject + (rowLimit == null ? "" : " LIMIT " + rowLimit); - var process = getProcess(); - try (var connection = new Connection(process.getEndpoint(), path)) { + try (var connection = getConnection(path)) { var columns = readStructureInternal(connection, tableNameObject); var builders = diff --git a/test/Tableau_Tests/src/Structure_Spec.enso b/test/Tableau_Tests/src/Structure_Spec.enso index 1a4267fd5688..031410382d06 100644 --- a/test/Tableau_Tests/src/Structure_Spec.enso +++ b/test/Tableau_Tests/src/Structure_Spec.enso @@ -1,4 +1,6 @@ from Standard.Base import all +import Standard.Base.Errors.File_Error.File_Error +import Standard.Base.Errors.Illegal_Argument.Illegal_Argument from Standard.Tableau import Hyper_File @@ -10,6 +12,8 @@ main filter=Nothing = suite.run_with_filter filter add_specs suite_builder = suite_builder.group "Structure" group_builder-> + no_file = enso_project.data / "no_file.hyper" + empty_file = enso_project.data / "empty.hyper" names_file = enso_project.data / "names.hyper" group_builder.specify "should default to all schema" <| @@ -19,9 +23,23 @@ add_specs suite_builder = suite_builder.group "Structure" group_builder-> schemas = Hyper_File.new names_file . schemas schemas.should_equal ["public", "Extract"] + schemas_2 = Hyper_File.new empty_file . schemas + schemas_2.should_equal ["public"] + + group_builder.specify "should handle a missing file when listing schema" <| + r1 = Hyper_File.new no_file . schemas + r1.should_fail_with File_Error + r1.catch.should_be_a File_Error.Not_Found + group_builder.specify "should default to set schema" <| Hyper_File.new names_file schema="Extract" . schema . should_equal "Extract" Hyper_File.new names_file . set_schema "Extract" . schema . should_equal "Extract" + Hyper_File.new names_file schema="*" . schema . should_equal "*" + Hyper_File.new names_file schema="*" . set_schema "Extract" . schema . should_equal "Extract" + + group_builder.specify "should reject invalid schema to set schema" <| + Hyper_File.new names_file schema="" . should_fail_with Illegal_Argument + Hyper_File.new names_file . set_schema "" . should_fail_with Illegal_Argument group_builder.specify "should be able to list table" <| tables = Hyper_File.new names_file . tables @@ -36,3 +54,11 @@ add_specs suite_builder = suite_builder.group "Structure" group_builder-> tables_3 = Hyper_File.new names_file schema="public" . tables tables_3.length.should_equal 0 + + tables_4 = Hyper_File.new names_file schema="DoesNotExist" . tables + tables_4.length.should_equal 0 + + group_builder.specify "should handle a missing file when listing tables" <| + r1 = Hyper_File.new no_file . tables + r1.should_fail_with File_Error + r1.catch.should_be_a File_Error.Not_Found From f5164a68d2f0ba0d2d64b4aa7eed36c6a16d71e0 Mon Sep 17 00:00:00 2001 From: James Dunkerley Date: Fri, 2 Aug 2024 13:19:18 +0100 Subject: [PATCH 28/35] Java fmt and sample hyper files. --- .../main/java/org/enso/tableau/HyperReader.java | 1 - test/Tableau_Tests/data/empty.hyper | Bin 0 -> 65536 bytes test/Tableau_Tests/data/names.hyper | Bin 0 -> 65536 bytes 3 files changed, 1 deletion(-) create mode 100644 test/Tableau_Tests/data/empty.hyper create mode 100644 test/Tableau_Tests/data/names.hyper diff --git a/std-bits/tableau/src/main/java/org/enso/tableau/HyperReader.java b/std-bits/tableau/src/main/java/org/enso/tableau/HyperReader.java index 6634df68ef62..28a2b0516e77 100644 --- a/std-bits/tableau/src/main/java/org/enso/tableau/HyperReader.java +++ b/std-bits/tableau/src/main/java/org/enso/tableau/HyperReader.java @@ -1,7 +1,6 @@ package org.enso.tableau; import com.tableau.hyperapi.*; - import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; diff --git a/test/Tableau_Tests/data/empty.hyper b/test/Tableau_Tests/data/empty.hyper new file mode 100644 index 0000000000000000000000000000000000000000..dbbc8c7774f1bd14c793d65b722bb8d9d1f4b76f GIT binary patch literal 65536 zcmeI*O>5LZ7{Kvu>%p`3R1nsH;6Y_uSEYI>WswTLRJ5leWi~tAjpVJ9WNTZN?nS%^ zf}cSA0^%pIeh2Z;qgN3R3L;*dWRl&qDpCu@>VFTN%rldj$?w@5lI>>a)l9YeyKxOxK3-qj~!PB z)5G_q*OcnkvaT+sM<-gI;}5HL;QOxQWkc4drt8xeFHKEf(EXGQHKs_DRT{n{qInTT ztAR=rP%JeZhOAPa%PB19?7SX6&8Qy1(s_5KNR}KwmdY3IQU;?t{AmjHu4UkyY2Uv2 zIDaRvc}u%lveHv1OSF7oL^I(*wCl*#L1j7d?KHuIK&&4>GN_1lTgkS5TMi0tg_000IagfB*srAbI5I_I{1Q0*~ z0R#|0009L4eSyIz#Z_~x{rrM?=Ao%?t()?9-)B13IK5(e@)rRF5I_I{1Q0*~0R#|0 z009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{ z1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009IL nKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0;re-!u*CRTL{ literal 0 HcmV?d00001 diff --git a/test/Tableau_Tests/data/names.hyper b/test/Tableau_Tests/data/names.hyper new file mode 100644 index 0000000000000000000000000000000000000000..158342107ea7116b62b1d45da3d85828c63ab6b1 GIT binary patch literal 65536 zcmeI*Piz!b9Ki9nWoZ{m1K0z9E;d8tpH&yA5-^b!YlK?00jdYUI^BJ3CroG8nb|_Y znmtHjOicWjn0PYLgNY`Z7&Im|E;mn}g#$4rqR|jFnCJn6@6Dgx>7xB7G%fjVXlLG^ zdB68QGt=D}-n0kCOVZ5>;LeoUw1EDs~!ov+V>yV z|1B%`a{a#`dy^mtAbhII1oc*$2qhw~$HR36lYSyfOleRI(Cq^HtnjDlM%m8357-Ljle<&HTj zy^@&^DjYpy6fIr4Bb&|ch)Pw`D3u3oE3X3;Pdd_7SwY*fB%Oi^ZG~GOSLLo8lddPb z8;ew|WaC|l#veW9yJp@`wyt9r8k?A|^&oW7zHt;g6M>Qu=GuSNj8|4OFv?yq{2|l! zq{_>?()49vP>D>xMaQwN(V)q$oqM|WJg|G`p8HijSC-VSGWDU5Xnw@>`b^I|S#gw{O(W z;^Yla7(3Wm`UIVF{^^|=Yu|)ZM735jUFqm!aF^bbrtiDfVA+q)nK;&v<$C@=kW#gL zb=y(*%y`;#^CPCK4-=4oe-q)}#H_ z@If`4DIJ>7UDEw~q*!+Rz*PMNQ2(;JKmqucJ~JMd<7!Y-&YN19EF6;IE>TtDGbIXY z4BEx~2_uLiPlZQ*a3%zibciuao~$K?%1&N~t7YhBY6)|A*pX=Q84s>}s(|^sN!;Ch_KS)l`pu zFIX$yYgX%rO!YouuN4og_10#wx=CCQUq6d$E#gRKUHW>9=*oPaZWW?gZQTFt`y0iw z-pnWAhP~wzQv81HoiA!VtlSv?zIyy4`X+UEn^?U@Et|xe*5!$izeFLML}p!TN9L~U z-)#_mLBk#IUVgC_VfvNla?$v=cj&RV=-AoydhG3~6{5W}BMz&fh*fEk?$xn3h!N4A zNsIOs>T6|Mh+{hUy!bwAN=*?^rM@(G(LXx1ixSH*N@M9)`ep)d^(rg}A8+ALLpOuT!^%_fQT=W$I1Q0*~0R#|0009ILKmdXH z3Do|lxFg)Y5?wIo&W7pg>2NvI^Gsh4KlW8vb9Uc({$O@a)I#8P5(t0ZM3Yd|!1|-X zRX2w#l_;I!23;h<0s#aNKmY**5I_I{1g00ze?BjK#g`G4tzkOXkRK&k+*E598FYH@zrSj_b`^hfd1~AbE0!I0WkVTWa%sqWdkmsDt%0WO4NPcIid<$+_-784WDA zWhy-LD{hxP&NvW2009ILKmdWIB@o}~tw#5EtI_Aw`1jRSy29~P&xOlePq>`vd4xuV zS357<)0jsZ0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0 z009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKw#+z`~%2Z B@2da+ literal 0 HcmV?d00001 From 3bedc2dd8307c475a657aa95fa2a471e23c23c93 Mon Sep 17 00:00:00 2001 From: Hubert Plociniczak Date: Fri, 2 Aug 2024 15:24:41 +0200 Subject: [PATCH 29/35] sbt now has up-to-date list of unmanaged deps Previously the task was reporting an empty sequence despite fetching the required jars. --- build.sbt | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/build.sbt b/build.sbt index 7ff514937c52..4a7ffb445df1 100644 --- a/build.sbt +++ b/build.sbt @@ -3641,8 +3641,10 @@ lazy val `std-tableau` = project }, fetchZipToUnmanaged := { val unmanagedDirectory = (Compile / unmanagedBase).value + val logger = state.value.log if (IO.listFiles(unmanagedDirectory).size < 2) { // Heuristic, should have at least hyperapi jar and os-specific one. - System.out.println( + logger.log( + Level.Info, "std-tableau's unmanaged dependencies are not up-to-date. fetching..." ) unmanagedDirectory.mkdirs() @@ -3657,25 +3659,22 @@ lazy val `std-tableau` = project "javadoc" ) && !f.contains("jna") ) - files.map(f => IO.move(f, unmanagedPath.resolve(f.getName).toFile)) + files.map { f => + IO.move(f, unmanagedPath.resolve(f.getName).toFile) + Attributed.blank(unmanagedPath.resolve(f.getName).toFile) + }.toSeq }, keepDirectory = false ) } else { - System.out.println( - "std-tableau's unmanaged dependencies are up-to-date" - ) + Seq[Attributed[File]]() } }, - Compile / compile / compileInputs := (Compile / compile / compileInputs) - .dependsOn(SPIHelpers.ensureSPIConsistency) - .value, - Compile / compile := (Compile / compile) - .dependsOn(fetchZipToUnmanaged) - .value, - Compile / unmanagedClasspath := (Compile / unmanagedClasspath) - .dependsOn(fetchZipToUnmanaged) - .value, + Compile / unmanagedClasspath := Def.task { + val additionalFiles: Seq[Attributed[File]] = fetchZipToUnmanaged.value + val result = (Compile / unmanagedClasspath).value + result ++ additionalFiles + }.value, Compile / packageBin / artifactPath := `std-tableau-polyglot-root` / "std-tableau.jar", libraryDependencies ++= Seq( @@ -3698,7 +3697,7 @@ lazy val `std-tableau` = project .dependsOn(`std-table` % "provided") lazy val fetchZipToUnmanaged = - taskKey[Unit]( + taskKey[Seq[Attributed[File]]]( "Download zip file from an `unmanagedExternalZip` url and unpack jars to unmanaged libs directory" ) lazy val unmanagedExternalZip = From a7820896fe50627980235205d2ed594a3acd5432 Mon Sep 17 00:00:00 2001 From: James Dunkerley Date: Fri, 2 Aug 2024 14:33:01 +0100 Subject: [PATCH 30/35] Read test part 1. --- test/Tableau_Tests/data/names.hyper | Bin 65536 -> 65536 bytes test/Tableau_Tests/src/Read_Spec.enso | 40 +++++++++++++++++++++++++- 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/test/Tableau_Tests/data/names.hyper b/test/Tableau_Tests/data/names.hyper index 158342107ea7116b62b1d45da3d85828c63ab6b1..39f6d105fb738b362c591ec8d7dcf3e8019699e2 100644 GIT binary patch delta 1110 zcmaJ>O>ERw5FUG4G@Gthz$ue3W@|F0fZomNt}9-V`uFR z2sKidOKBn`SRCMj@^dJn96*U7C)DNu;=&b)1LD#mK)tM#9ue(0>qt>mb)?TT&u{!@ zzImTqDv?r&+#IAyMZa0;Tw&dEW;#YbcFpg+B@WUPFD|Zon>`YkTTIGaVtexYl%b5O zsr|`RhoTRS>q;?i8m$i3c)&T>xJ24NJifkAc*TUHP_-Ma+QxFB?S~*hDhdpwvXp`} zMj_rBgEW*xAS^5hnA~m@HPg5v0f;^2av!CHLIbUA z7O)b(5rF#&kpv)(xNZUvb|Us-d=Swwh>IN{urPk=!SlNM>1AJvo{$YJJKT-L(ASuhE06C!gOb7aE+L%rQ~tsN4{cBDIb#Z*J;=CqM{;x6==Q z0dz4EOF#-B2!bQ9l`lg-ZaWL|E@xQ|Z!w_j5Z-Bp*(PV2np?KfV5kNdq_#NnV?PST zD{|fwN^>%YA`v{1WD}l9Bqj)CpC<^Y#cPItmy&n_aBg*n;%*QTLewC_hzJ2%^v3MB z64~^OyLYGp3>zREFL-@~BLHX4Ul=PFj*4sg#cJ@I7>Je&t?t##wX`!cP95=hsbT8O zuPc$CkB#4ZxgjmhzyFEB#Pv2IFHX5nj!*B4zwuw^P?f93xMwF%hhy8Tt6bHAt4t0s z&df+Ck@dvkS#tG>*IV_DGO1yG-g$I46o2dQVM(C#zFYDA+(-+3v#GxIPMdbxH)v%C0kHWc*RUQRn_pPnLrY)we`>$oht JbD7tve*jwE^HBf* delta 666 zcmZo@U}{N!2e`f83x`H8!p#jqY`UnSvFalM&@G|fV0BI%$ z0d7_#J6}WXWMbeDXEfwc{r^CcApmT$!h&aY6ARTRJ7}>o@=kWsQek58nVhR7C3BGB z3j>1(qc9T-gD^7?vj8zG3xhn{UxtH|=W9F4GBL0?04@5!sHU!=sim#P!O6u93>~0v zSN!msSm+=C3qU5I*!!G`u@egoH#h3ED=%zdnLMwQW3oU%gUTK|AA3ekFXcV?EjJ`s z%4$LSkwM24Zu8As0;X69g+hf7s6iMF44b1m!+ACvu1sf~*uXYfF=?CcSdy%?orgls7iyG6MkXb)X>t diff --git a/test/Tableau_Tests/src/Read_Spec.enso b/test/Tableau_Tests/src/Read_Spec.enso index 24794bee6120..7d33097075de 100644 --- a/test/Tableau_Tests/src/Read_Spec.enso +++ b/test/Tableau_Tests/src/Read_Spec.enso @@ -1,5 +1,7 @@ from Standard.Base import all +import Standard.Base.Errors.File_Error.File_Error +from Standard.Table import all from Standard.Tableau import Hyper_File from Standard.Test import all @@ -10,4 +12,40 @@ main filter=Nothing = suite.run_with_filter filter add_specs suite_builder = suite_builder.group "Read Tables" group_builder-> - _ = group_builder + no_file = enso_project.data / "no_file.hyper" + names_file = enso_project.data / "names.hyper" + names_table = Hyper_File.new names_file . tables . first + + check_names_table table = + table.row_count.should_equal 14 + table.columns.map .name . should_equal ["first_name", "last_name", "age", "Calculation1"] + table.columns.map .value_type . should_equal [Value_Type.Char Nothing variable_length=True, Value_Type.Char Nothing variable_length=True, Value_Type.Integer ..Bits_64, Value_Type.Integer ..Bits_64] + table.at "first_name" . to_vector . should_equal ["James", "John", "Robert", "Michael", "William", "David", "Richard", "Joseph", "Charles", "Thomas", "Daniel", Nothing, "Anthony", Nothing] + table.at "last_name" . to_vector . should_equal ["Smith", "Johnson", "Williams", "Jones", "Brown", "Davis", "Miller", "Wilson", "Moore", "Taylor", "Anderson", "Garcia", Nothing, Nothing] + table.at "age" . to_vector . should_equal [39, 40, 41, 42, 43, 44, 39, 9, 10, 11, 12, 1, 2, 25] + + group_builder.specify "should be able to read the structure of a table" <| + metadata = names_table.column_metadata + metadata.length.should_equal 4 + metadata.map .name . should_equal ["first_name", "last_name", "age", "Calculation1"] + metadata.map .value_type . should_equal [Value_Type.Char Nothing variable_length=True, Value_Type.Char Nothing variable_length=True, Value_Type.Integer ..Bits_64, Value_Type.Integer ..Bits_64] + metadata.map .nullable . should_equal [True, True, True, True] + + group_builder.specify "should be able to read the column_info of a table" <| + metadata = names_table.column_info + metadata.row_count.should_equal 4 + metadata.columns.map .name . should_equal ["Column", "Value Type", "Nullable"] + metadata.at "Column" . to_vector . should_equal ["first_name", "last_name", "age", "Calculation1"] + metadata.at "Value Type" . to_vector . should_equal [Value_Type.Char Nothing variable_length=True, Value_Type.Char Nothing variable_length=True, Value_Type.Integer ..Bits_64, Value_Type.Integer ..Bits_64] + + group_builder.specify "should be able to read a table" <| + table = names_table.read + check_names_table table + + table_2 = Hyper_File.new names_file . read "Extract" + check_names_table table_2 + + group_builder.specify "should handle a missing file when reading a table" <| + r1 = Hyper_File.new no_file . read "Extract" + r1.should_fail_with File_Error + r1.catch.should_be_a File_Error.Not_Found From dfb4b7c7aa50818012c888961c23858b9b010e05 Mon Sep 17 00:00:00 2001 From: James Dunkerley Date: Fri, 2 Aug 2024 16:56:07 +0100 Subject: [PATCH 31/35] Fix failing test. --- test/Tableau_Tests/src/Structure_Spec.enso | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Tableau_Tests/src/Structure_Spec.enso b/test/Tableau_Tests/src/Structure_Spec.enso index 031410382d06..2ceaff35bd3f 100644 --- a/test/Tableau_Tests/src/Structure_Spec.enso +++ b/test/Tableau_Tests/src/Structure_Spec.enso @@ -21,7 +21,7 @@ add_specs suite_builder = suite_builder.group "Structure" group_builder-> group_builder.specify "should be able to list schema" <| schemas = Hyper_File.new names_file . schemas - schemas.should_equal ["public", "Extract"] + schemas.sort.should_equal ["Extract", "public"] schemas_2 = Hyper_File.new empty_file . schemas schemas_2.should_equal ["public"] From fa6eaf596a3a9841ce84ca8d7ffbb2880aae33a3 Mon Sep 17 00:00:00 2001 From: James Dunkerley Date: Fri, 2 Aug 2024 17:52:43 +0100 Subject: [PATCH 32/35] Fix failing test. --- .../Tableau/0.0.0-dev/src/Hyper_Errors.enso | 18 ++++++++++++ .../Tableau/0.0.0-dev/src/Hyper_File.enso | 5 ++-- .../Tableau/0.0.0-dev/src/Hyper_Table.enso | 6 ++-- .../org/enso/tableau/HyperQueryError.java | 14 ++++++++++ .../java/org/enso/tableau/HyperReader.java | 28 +++++++++++++++---- .../org/enso/tableau/HyperTableNotFound.java | 20 +++++++++++++ test/Tableau_Tests/src/Read_Spec.enso | 27 ++++++++++++++---- 7 files changed, 102 insertions(+), 16 deletions(-) create mode 100644 distribution/lib/Standard/Tableau/0.0.0-dev/src/Hyper_Errors.enso create mode 100644 std-bits/tableau/src/main/java/org/enso/tableau/HyperQueryError.java create mode 100644 std-bits/tableau/src/main/java/org/enso/tableau/HyperTableNotFound.java diff --git a/distribution/lib/Standard/Tableau/0.0.0-dev/src/Hyper_Errors.enso b/distribution/lib/Standard/Tableau/0.0.0-dev/src/Hyper_Errors.enso new file mode 100644 index 000000000000..be2ca9783c28 --- /dev/null +++ b/distribution/lib/Standard/Tableau/0.0.0-dev/src/Hyper_Errors.enso @@ -0,0 +1,18 @@ +from Standard.Base import all + +polyglot java import org.enso.tableau.HyperQueryError +polyglot java import org.enso.tableau.HyperTableNotFound + +## Error when a Table is not found in a Hyper File. +type Table_Not_Found + Error schema:Text name:Text + +## Error when a query fails. +type Query_Failed + Error message:Text query:Text + +## PRIVATE +private handle_java_exceptions ~action = + Panic.catch HyperTableNotFound handler=(c-> Error.throw (Table_Not_Found.Error c.payload.getSchema c.payload.getName)) <| + Panic.catch HyperQueryError handler=(c-> Error.throw (Query_Failed.Error c.payload.getMessage c.payload.getQuery)) <| + action diff --git a/distribution/lib/Standard/Tableau/0.0.0-dev/src/Hyper_File.enso b/distribution/lib/Standard/Tableau/0.0.0-dev/src/Hyper_File.enso index f7a0c1441e53..cdf213aefacd 100644 --- a/distribution/lib/Standard/Tableau/0.0.0-dev/src/Hyper_File.enso +++ b/distribution/lib/Standard/Tableau/0.0.0-dev/src/Hyper_File.enso @@ -8,6 +8,7 @@ from Standard.Table import Table import Standard.Table.Rows_To_Read.Rows_To_Read import project.Hyper_Table.Hyper_Table +import project.Hyper_Errors.Table_Not_Found polyglot java import org.enso.tableau.HyperReader @@ -81,8 +82,8 @@ type Hyper_File read self (table : Text) (schema : Text = self.schema) (limit : Rows_To_Read = ..All_Rows) = case schema of "" -> self.read table self.schema limit "*" -> - table_to_read = self.tables.find t-> t.table == table - if table_to_read.is_nothing then Error.throw (Illegal_Argument.Error "Table not found.") else + table_to_read = self.tables.find if_missing=Nothing t-> t.table == table + if table_to_read.is_nothing then Error.throw (Table_Not_Found.Error "*" table) else table_to_read.read limit _ -> Hyper_Table.Value self table schema . read limit diff --git a/distribution/lib/Standard/Tableau/0.0.0-dev/src/Hyper_Table.enso b/distribution/lib/Standard/Tableau/0.0.0-dev/src/Hyper_Table.enso index fcad90bcd466..cc6157150a45 100644 --- a/distribution/lib/Standard/Tableau/0.0.0-dev/src/Hyper_Table.enso +++ b/distribution/lib/Standard/Tableau/0.0.0-dev/src/Hyper_Table.enso @@ -1,4 +1,5 @@ from Standard.Base import all +import Standard.Base.Errors.File_Error.File_Error from Standard.Table import Column, Table, Value_Type import Standard.Table.Rows_To_Read.Rows_To_Read @@ -6,6 +7,7 @@ import Standard.Table.Internal.Java_Problems import project.Hyper_Column.Hyper_Column import project.Hyper_File.Hyper_File +import project.Hyper_Errors polyglot java import java.sql.Types polyglot java import org.enso.tableau.HyperReader @@ -37,7 +39,7 @@ type Hyper_Table ## ICON metadata Reads The Columns for the Table column_metadata : Vector Hyper_Column - column_metadata self = + column_metadata self = File_Error.handle_java_exceptions self.file.file <| Hyper_Errors.handle_java_exceptions <| array = HyperReader.readStructure self.file.file.path self.schema self.table array.map column-> value_type = case column.typeID of @@ -84,7 +86,7 @@ type Hyper_Table - max_rows: specifies the maximum number of rows to read. @max_rows Rows_To_Read.default_widget read : Rows_To_Read -> Table - read self (max_rows : Rows_To_Read = ..All_Rows) = + read self (max_rows : Rows_To_Read = ..All_Rows) = File_Error.handle_java_exceptions self.file.file <| Hyper_Errors.handle_java_exceptions <| Java_Problems.with_problem_aggregator Problem_Behavior.Report_Warning java_problem_aggregator-> row_count = if max_rows == Rows_To_Read.All_Rows then Nothing else max_rows.rows java_columns = HyperReader.readTable self.file.file.path self.schema self.table row_count java_problem_aggregator diff --git a/std-bits/tableau/src/main/java/org/enso/tableau/HyperQueryError.java b/std-bits/tableau/src/main/java/org/enso/tableau/HyperQueryError.java new file mode 100644 index 000000000000..afbc1a27e989 --- /dev/null +++ b/std-bits/tableau/src/main/java/org/enso/tableau/HyperQueryError.java @@ -0,0 +1,14 @@ +package org.enso.tableau; + +public class HyperQueryError extends RuntimeException { + private final String query; + + public HyperQueryError(String message, String query, Throwable cause) { + super(message, cause); + this.query = query; + } + + public String getQuery() { + return query; + } +} diff --git a/std-bits/tableau/src/main/java/org/enso/tableau/HyperReader.java b/std-bits/tableau/src/main/java/org/enso/tableau/HyperReader.java index 28a2b0516e77..6903acfc455a 100644 --- a/std-bits/tableau/src/main/java/org/enso/tableau/HyperReader.java +++ b/std-bits/tableau/src/main/java/org/enso/tableau/HyperReader.java @@ -173,12 +173,22 @@ public static HyperTableColumn[] readStructure(String path, String schemaName, S private static HyperTableColumn[] readStructureInternal( Connection connection, TableName tableNameObject) { - var catalog = connection.getCatalog(); - var definition = catalog.getTableDefinition(tableNameObject); - var columns = definition.getColumns(); - return IntStream.range(0, columns.size()) - .mapToObj(i -> HyperTableColumn.fromHyperColumn(i, columns.get(i))) - .toArray(HyperTableColumn[]::new); + try { + var catalog = connection.getCatalog(); + var definition = catalog.getTableDefinition(tableNameObject); + var columns = definition.getColumns(); + return IntStream.range(0, columns.size()) + .mapToObj(i -> HyperTableColumn.fromHyperColumn(i, columns.get(i))) + .toArray(HyperTableColumn[]::new); + } catch (HyperException e) { + if (e.getMessage().contains(" does not exist: ")) { + var schemaObject = tableNameObject.getSchemaName(); + var schemaName = schemaObject.isPresent() ? schemaObject.get().getName().getUnescaped() : ""; + throw new HyperTableNotFound(schemaName, tableNameObject.getName().getUnescaped(), e); + } else { + throw new HyperQueryError(e.getMessage(), "TABLE_INFO " + tableNameObject, e); + } + } } public static Column[] readTable( @@ -210,6 +220,12 @@ public static Column[] readTable( return IntStream.range(0, columns.length) .mapToObj(i -> new Column(columns[i].name(), storages.get(i))) .toArray(Column[]::new); + } catch (HyperException e) { + if (e.getMessage().contains(" does not exist: ")) { + throw new HyperTableNotFound(schemaName, tableName, e); + } else { + throw new HyperQueryError(e.getMessage(), query, e); + } } } } diff --git a/std-bits/tableau/src/main/java/org/enso/tableau/HyperTableNotFound.java b/std-bits/tableau/src/main/java/org/enso/tableau/HyperTableNotFound.java new file mode 100644 index 000000000000..4ce435eb5846 --- /dev/null +++ b/std-bits/tableau/src/main/java/org/enso/tableau/HyperTableNotFound.java @@ -0,0 +1,20 @@ +package org.enso.tableau; + +public class HyperTableNotFound extends RuntimeException { + private final String schema; + private final String name; + + public HyperTableNotFound(String schema, String name, Throwable cause) { + super("The table " + schema + "." + name + " does not exist.", cause); + this.schema = schema; + this.name = name; + } + + public String getSchema() { + return schema; + } + + public String getName() { + return name; + } +} diff --git a/test/Tableau_Tests/src/Read_Spec.enso b/test/Tableau_Tests/src/Read_Spec.enso index 7d33097075de..8a98139ebf8a 100644 --- a/test/Tableau_Tests/src/Read_Spec.enso +++ b/test/Tableau_Tests/src/Read_Spec.enso @@ -3,6 +3,7 @@ import Standard.Base.Errors.File_Error.File_Error from Standard.Table import all from Standard.Tableau import Hyper_File +import Standard.Tableau.Hyper_Errors.Table_Not_Found from Standard.Test import all @@ -16,13 +17,13 @@ add_specs suite_builder = suite_builder.group "Read Tables" group_builder-> names_file = enso_project.data / "names.hyper" names_table = Hyper_File.new names_file . tables . first - check_names_table table = - table.row_count.should_equal 14 + check_names_table table rows=14 = + table.row_count.should_equal rows table.columns.map .name . should_equal ["first_name", "last_name", "age", "Calculation1"] - table.columns.map .value_type . should_equal [Value_Type.Char Nothing variable_length=True, Value_Type.Char Nothing variable_length=True, Value_Type.Integer ..Bits_64, Value_Type.Integer ..Bits_64] - table.at "first_name" . to_vector . should_equal ["James", "John", "Robert", "Michael", "William", "David", "Richard", "Joseph", "Charles", "Thomas", "Daniel", Nothing, "Anthony", Nothing] - table.at "last_name" . to_vector . should_equal ["Smith", "Johnson", "Williams", "Jones", "Brown", "Davis", "Miller", "Wilson", "Moore", "Taylor", "Anderson", "Garcia", Nothing, Nothing] - table.at "age" . to_vector . should_equal [39, 40, 41, 42, 43, 44, 39, 9, 10, 11, 12, 1, 2, 25] + table.columns.map .value_type . should_equal ([Value_Type.Char Nothing variable_length=True, Value_Type.Char Nothing variable_length=True, Value_Type.Integer ..Bits_64, Value_Type.Integer ..Bits_64].take rows) + table.at "first_name" . to_vector . should_equal (["James", "John", "Robert", "Michael", "William", "David", "Richard", "Joseph", "Charles", "Thomas", "Daniel", Nothing, "Anthony", Nothing].take rows) + table.at "last_name" . to_vector . should_equal (["Smith", "Johnson", "Williams", "Jones", "Brown", "Davis", "Miller", "Wilson", "Moore", "Taylor", "Anderson", "Garcia", Nothing, Nothing].take rows) + table.at "age" . to_vector . should_equal ([39, 40, 41, 42, 43, 44, 39, 9, 10, 11, 12, 1, 2, 25].take rows) group_builder.specify "should be able to read the structure of a table" <| metadata = names_table.column_metadata @@ -45,7 +46,21 @@ add_specs suite_builder = suite_builder.group "Read Tables" group_builder-> table_2 = Hyper_File.new names_file . read "Extract" check_names_table table_2 + group_builder.specify "should be able to read a table with limited rows" <| + table = names_table.read (..First 5) + check_names_table table 5 + + table_2 = Hyper_File.new names_file . read "Extract" limit=(..First 7) + check_names_table table_2 7 + group_builder.specify "should handle a missing file when reading a table" <| r1 = Hyper_File.new no_file . read "Extract" r1.should_fail_with File_Error r1.catch.should_be_a File_Error.Not_Found + + group_builder.specify "should handle a missing table when reading a table" <| + r1 = Hyper_File.new names_file . read "NoTable" + r1.should_fail_with Table_Not_Found + + r2 = Hyper_File.new names_file . read "NoTable" schema="Extract" + r2.should_fail_with Table_Not_Found From 6bb721c28c20a6107567ce227a3f6d1efcc0d950 Mon Sep 17 00:00:00 2001 From: James Dunkerley Date: Fri, 2 Aug 2024 17:56:40 +0100 Subject: [PATCH 33/35] Java format. --- .../tableau/src/main/java/org/enso/tableau/HyperReader.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/std-bits/tableau/src/main/java/org/enso/tableau/HyperReader.java b/std-bits/tableau/src/main/java/org/enso/tableau/HyperReader.java index 6903acfc455a..f7e43d1ff6c9 100644 --- a/std-bits/tableau/src/main/java/org/enso/tableau/HyperReader.java +++ b/std-bits/tableau/src/main/java/org/enso/tableau/HyperReader.java @@ -183,7 +183,8 @@ private static HyperTableColumn[] readStructureInternal( } catch (HyperException e) { if (e.getMessage().contains(" does not exist: ")) { var schemaObject = tableNameObject.getSchemaName(); - var schemaName = schemaObject.isPresent() ? schemaObject.get().getName().getUnescaped() : ""; + var schemaName = + schemaObject.isPresent() ? schemaObject.get().getName().getUnescaped() : ""; throw new HyperTableNotFound(schemaName, tableNameObject.getName().getUnescaped(), e); } else { throw new HyperQueryError(e.getMessage(), "TABLE_INFO " + tableNameObject, e); From fe906536ac176f36e0bb97673e4db121b721045c Mon Sep 17 00:00:00 2001 From: Hubert Plociniczak Date: Fri, 2 Aug 2024 22:31:26 +0200 Subject: [PATCH 34/35] Ensure `unmanagedJars` fetches jars Since our CI builds things multiple times and it might happen `unmanagedClasspath` was not triggered before `unmanagedJars`, we could have a situation when necessary jars were not copied to the distribution. This change ensures that is no longer the case. --- build.sbt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/build.sbt b/build.sbt index 4a7ffb445df1..187cb26d9f44 100644 --- a/build.sbt +++ b/build.sbt @@ -3675,6 +3675,9 @@ lazy val `std-tableau` = project val result = (Compile / unmanagedClasspath).value result ++ additionalFiles }.value, + Compile / unmanagedJars := (Compile / unmanagedJars) + .dependsOn(fetchZipToUnmanaged) + .value, Compile / packageBin / artifactPath := `std-tableau-polyglot-root` / "std-tableau.jar", libraryDependencies ++= Seq( From 872384f86ccbf2f4cdaf6239e51f304786de1ff1 Mon Sep 17 00:00:00 2001 From: Hubert Plociniczak Date: Mon, 5 Aug 2024 10:48:24 +0200 Subject: [PATCH 35/35] Update build.sbt Co-authored-by: GregoryTravis --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index 187cb26d9f44..02b056da0215 100644 --- a/build.sbt +++ b/build.sbt @@ -3636,7 +3636,7 @@ lazy val `std-tableau` = project "x86_64" } new URI( - s"https://downloads.tableau.com/tssoftware//tableauhyperapi-java-$platform-$arch-release-main.$tableauVersion.zip" + s"https://downloads.tableau.com/tssoftware/tableauhyperapi-java-$platform-$arch-release-main.$tableauVersion.zip" ).toURL() }, fetchZipToUnmanaged := {