[Coral-Hive] Loose UDF format requirement (#446)

linkedin · Sep 5, 2023 · da49e65 · da49e65
1 parent 15e290e
commit da49e65
Show file tree

Hide file tree

Showing 3 changed files with 17 additions and 5 deletions.
diff --git a/...l-hive/src/main/java/com/linkedin/coral/hive/hive2rel/functions/HiveFunctionResolver.java b/...l-hive/src/main/java/com/linkedin/coral/hive/hive2rel/functions/HiveFunctionResolver.java
@@ -164,17 +164,16 @@ public Collection<Function> resolve(String functionName) {
    * @param table Hive metastore table handle
    * @param numOfOperands number of operands this function takes. This is needed to
    *                      create SqlOperandTypeChecker to resolve Dali function dynamically
-   * @return list of matching Functions or empty list if the function name is not in the
-   *   dali function name format of db_tableName_functionName
+   * @return list of matching Functions or empty list if the function name is not in the dali function name format
+   * of `databaseName_tableName_udfName` or `udfName` (without `databaseName_tableName_` prefix)
    * @throws UnknownSqlFunctionException if the function name is in Dali function name format but there is no mapping
    */
   public Collection<Function> tryResolveAsDaliFunction(String functionName, @Nonnull Table table, int numOfOperands) {
     Preconditions.checkNotNull(table);
     String functionPrefix = String.format("%s_%s_", table.getDbName(), table.getTableName());
     if (!functionName.toLowerCase().startsWith(functionPrefix.toLowerCase())) {
-      // Don't throw UnknownSqlFunctionException here because this is not a dali function
-      // and this method is trying to resolve only Dali functions
-      return ImmutableList.of();
+      // if functionName is not in `databaseName_tableName_udfName` format, we don't require the `databaseName_tableName_` prefix
+      functionPrefix = "";
     }
     String funcBaseName = functionName.substring(functionPrefix.length());
     HiveTable hiveTable = new HiveTable(table);

diff --git a/coral-spark/src/test/java/com/linkedin/coral/spark/CoralSparkTest.java b/coral-spark/src/test/java/com/linkedin/coral/spark/CoralSparkTest.java
@@ -853,6 +853,14 @@ public void testCastOnString() {
     assertEquals(createCoralSpark(relNode).getSparkSql(), targetSql);
   }
 
+  @Test
+  public void testUdfWithoutDatabaseTableNamePrefix() {
+    RelNode relNode = TestUtils.toRelNode("default", "foo_dali_udf_no_prefix");
+
+    String targetSql = "SELECT LessThanHundred(foo.a)\n" + "FROM default.foo foo";
+    assertEquals(createCoralSpark(relNode).getSparkSql(), targetSql);
+  }
+
   private String getCoralSparkTranslatedSqlWithAliasFromCoralSchema(String db, String view) {
     RelNode relNode = TestUtils.toRelNode(db, view);
     Schema schema = TestUtils.getAvroSchemaForView(db, view, false);

diff --git a/coral-spark/src/test/java/com/linkedin/coral/spark/TestUtils.java b/coral-spark/src/test/java/com/linkedin/coral/spark/TestUtils.java
@@ -79,6 +79,7 @@ public static void initializeViews(HiveConf conf) throws HiveException, MetaExce
         "create function default_foo_lateral_udtf_CountOfRow as 'com.linkedin.coral.hive.hive2rel.CoralTestUDTF'");
     run(driver,
         "create function default_foo_duplicate_udf_LessThanHundred as 'com.linkedin.coral.hive.hive2rel.CoralTestUDF'");
+    run(driver, "CREATE FUNCTION LessThanHundred as 'com.linkedin.coral.hive.hive2rel.CoralTestUDF'");
 
     run(driver, String.join("\n", "", "CREATE VIEW IF NOT EXISTS foo_view", "AS", "SELECT b AS bcol, sum(c) AS sum_c",
         "FROM foo", "GROUP BY b"));
@@ -115,6 +116,10 @@ public static void initializeViews(HiveConf conf) throws HiveException, MetaExce
             "              'dependencies' = 'com.linkedin:udf:1.0')", "AS",
             "SELECT default_foo_dali_udf5_UnsupportedUDF(a)", "FROM foo"));
 
+    run(driver, String.join("\n", "", "CREATE VIEW IF NOT EXISTS foo_dali_udf_no_prefix",
+        "tblproperties('functions' = 'LessThanHundred:com.linkedin.coral.hive.hive2rel.CoralTestUDF',",
+        "              'dependencies' = 'ivy://com.linkedin:udf:1.0')", "AS", "SELECT LessThanHundred(a)", "FROM foo"));
+
     run(driver,
         String.join("\n", "", "CREATE VIEW IF NOT EXISTS foo_lateral_udtf",
             "tblproperties('functions' = 'CountOfRow:com.linkedin.coral.hive.hive2rel.CoralTestUDTF')", "AS",