Skip to content

Commit

Permalink
Add support for from_utc_timestamp in Hive views translation
Browse files Browse the repository at this point in the history
  • Loading branch information
@losipiuk
losipiuk committed Aug 11, 2021
1 parent 570c647 commit 33be493
Show file tree
Hide file tree
Showing 4 changed files with 138 additions and 2 deletions.
36 changes: 36 additions & 0 deletions plugin/trino-hive-hadoop2/src/main/java/io/trino/plugin/hive/CanonicalizeHiveTimezoneId.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.trino.plugin.hive;

import io.airlift.slice.Slice;
import io.trino.spi.function.LiteralParameters;
import io.trino.spi.function.ScalarFunction;
import io.trino.spi.function.SqlType;

/**
* Translate timezone id used by Hive to canonical form which is understandable by Trino; used in Hive view translation logic
*/
public final class CanonicalizeHiveTimezoneId
{
private CanonicalizeHiveTimezoneId() {}

@ScalarFunction(value = "$canonicalize_hive_timezone_id", hidden = true)
@LiteralParameters("x")
@SqlType("varchar")
public static Slice canonicalizeHiveTimezoneId(@SqlType("varchar(x)") Slice hiveTimeZoneId)
{
// TODO(https://github.com/trinodb/trino/issues/8853) no-op for now; actual cannicalization logic to be added
return hiveTimeZoneId;
}
}
8 changes: 8 additions & 0 deletions plugin/trino-hive-hadoop2/src/main/java/io/trino/plugin/hive/HivePlugin.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,13 +14,15 @@
package io.trino.plugin.hive;

import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import io.airlift.log.Logger;
import io.trino.spi.Plugin;
import io.trino.spi.connector.Connector;
import io.trino.spi.connector.ConnectorContext;
import io.trino.spi.connector.ConnectorFactory;

import java.util.Map;
import java.util.Set;

public class HivePlugin
implements Plugin
Expand All @@ -33,6 +35,12 @@ public Iterable<ConnectorFactory> getConnectorFactories()
return ImmutableList.of(new HiveConnectorFactory("hive"), new LegacyHiveConnectorFactory());
}

@Override
public Set<Class<?>> getFunctions()
{
return ImmutableSet.of(CanonicalizeHiveTimezoneId.class);
}

private static class LegacyHiveConnectorFactory
extends HiveConnectorFactory
{
Expand Down
4 changes: 2 additions & 2 deletions pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@
<dep.errorprone.version>2.8.0</dep.errorprone.version>
<dep.testcontainers.version>1.16.0</dep.testcontainers.version>
<dep.docker-java.version>3.2.11</dep.docker-java.version>
<dep.coral.version>1.0.60</dep.coral.version>
<dep.coral.version>1.0.77</dep.coral.version>
<dep.confluent.version>5.5.2</dep.confluent.version>
<!-- TODO: update after moving to Airbase 112 -->
<dep.jackson.version>2.12.3</dep.jackson.version>
Expand Down Expand Up @@ -1038,7 +1038,7 @@
<dependency>
<groupId>com.linkedin.calcite</groupId>
<artifactId>calcite-core</artifactId>
<version>1.21.0.146</version>
<version>1.21.0.150</version>
<classifier>shaded</classifier>
<exclusions>
<exclusion>
Expand Down
92 changes: 92 additions & 0 deletions testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/TestHiveViews.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -172,4 +172,96 @@ public void testLateralViewJsonTupleAs()
"SELECT * FROM test_json_tuple_view",
queryAssert -> queryAssert.containsOnly(row(3, "Mateusz", "Gajewski", "true", "1000", null, null)));
}

@Test(groups = HIVE_VIEWS)
public void testFromUtcTimestamp()
{
onTrino().executeQuery("DROP TABLE IF EXISTS test_from_utc_timestamp_source");
onHive().executeQuery("CREATE TABLE test_from_utc_timestamp_source AS SELECT " +
" CAST(123 AS tinyint) source_tinyint, " +
" CAST(10123 AS smallint) source_smallint, " +
" CAST(2592000123 AS int) source_integer, " +
" CAST(2592000123 AS bigint) source_bigint, " +
" CAST(2592000.0 AS float) source_float, " +
" CAST(2592000.123 AS double) source_double, " +
" CAST(2592000.123 AS decimal(10,3)) source_decimal_three," +
" CAST(2592000 AS DECIMAL(10,0)) source_decimal_zero," +
" timestamp '1970-01-30 16:00:00' source_timestamp, " +
" date '1970-01-30' source_date ");

onHive().executeQuery("DROP VIEW IF EXISTS test_from_utc_timestamp_view");
onHive().executeQuery("CREATE VIEW " +
"test_from_utc_timestamp_view " +
"AS SELECT " +
// TODO(https://github.com/trinodb/trino/issues/8853) add testcases with 3-letter tz names (like PST) when we have $canonicalize_hive_timezone_id logic in place
" CAST(from_utc_timestamp(source_tinyint, 'America/Los_Angeles') AS STRING) ts_tinyint, " +
" CAST(from_utc_timestamp(source_smallint, 'America/Los_Angeles') AS STRING) ts_smallint, " +
" CAST(from_utc_timestamp(source_integer, 'America/Los_Angeles') AS STRING) ts_integer, " +
" CAST(from_utc_timestamp(source_bigint, 'America/Los_Angeles') AS STRING) ts_bigint, " +
" CAST(from_utc_timestamp(source_float, 'America/Los_Angeles') AS STRING) ts_float, " +
" CAST(from_utc_timestamp(source_double, 'America/Los_Angeles') AS STRING) ts_double, " +
" CAST(from_utc_timestamp(source_decimal_three, 'America/Los_Angeles') AS STRING) ts_decimal_three, " +
" CAST(from_utc_timestamp(source_decimal_zero, 'America/Los_Angeles') AS STRING) ts_decimal_zero, " +
" CAST(from_utc_timestamp(source_timestamp, 'America/Los_Angeles') AS STRING) ts_timestamp, " +
" CAST(from_utc_timestamp(source_date, 'America/Los_Angeles') AS STRING) ts_date " +
"FROM test_from_utc_timestamp_source");

// check result on Trino
assertThat(query("SELECT * FROM test_from_utc_timestamp_view"))
.containsOnly(row(
"1969-12-31 16:00:00.123",
"1969-12-31 16:00:10.123",
"1969-12-11 22:57:12.827",
"1970-01-30 16:00:00.123",
"1970-01-30 16:00:00.000",
"1970-01-30 16:00:00.123",
"1970-01-30 16:00:00.123",
"1970-01-30 16:00:00.000",
"1970-01-30 08:00:00.000",
"1970-01-29 16:00:00.000"));

// check result on Hive
assertThat(onHive().executeQuery("SELECT * FROM test_from_utc_timestamp_view"))
.containsOnly(row(
"1969-12-31 16:00:00.123",
"1969-12-31 16:00:10.123",
"1969-12-11 22:57:12.827",
"1970-01-30 16:00:00.123",
"1970-01-30 16:00:00",
"1970-01-30 16:00:00.123",
"1970-01-30 16:00:00.123",
"1970-01-30 16:00:00",
"1970-01-30 08:00:00",
"1970-01-29 16:00:00"));
}

@Test(groups = HIVE_VIEWS)
public void testFromUtcTimestampCornerCases()
{
onTrino().executeQuery("DROP TABLE IF EXISTS test_from_utc_timestamp_corner_cases_source");
onTrino().executeQuery("CREATE TABLE test_from_utc_timestamp_corner_cases_source AS SELECT * FROM (VALUES " +
" CAST(-5000000000001 AS BIGINT)," +
" CAST(-1000000000001 AS BIGINT)," +
" -1," +
" 1," +
" 5000000000001" +
")" +
"AS source(source_bigint)");

onHive().executeQuery("DROP VIEW IF EXISTS test_from_utc_timestamp_corner_cases_view");
onHive().executeQuery("CREATE VIEW " +
"test_from_utc_timestamp_corner_cases_view " +
"AS SELECT " +
" CAST(from_utc_timestamp(source_bigint, 'America/Los_Angeles') as STRING) ts_bigint " +
"FROM test_from_utc_timestamp_corner_cases_source");

// check result on Trino
assertViewQuery("SELECT * FROM test_from_utc_timestamp_corner_cases_view",
assertion -> assertion.containsOnly(
row("1811-07-23 07:13:41.999"),
row("1938-04-24 14:13:19.999"),
row("1969-12-31 15:59:59.999"),
row("1969-12-31 16:00:00.001"),
row("2128-06-11 01:53:20.001")));
}
}

0 comments on commit 33be493

Please sign in to comment.