From aa40d7bca748268f45ea2769adeffb6ba4d8028e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Pe=C3=B1a?= Date: Wed, 26 Feb 2020 10:27:57 -0600 Subject: [PATCH] fix: regenerate query plans for join-with-custom-timestamp --- .../6.0.0_1582734336704/plan.json | 246 +++++++++++++++++ .../6.0.0_1582734336704/spec.json | 91 +++++++ .../6.0.0_1582734336704/topology | 42 +++ .../6.0.0_1582734336763/plan.json | 246 +++++++++++++++++ .../6.0.0_1582734336763/spec.json | 91 +++++++ .../6.0.0_1582734336763/topology | 42 +++ .../6.0.0_1582734336880/plan.json | 246 +++++++++++++++++ .../6.0.0_1582734336880/spec.json | 91 +++++++ .../6.0.0_1582734336880/topology | 42 +++ .../6.0.0_1582734336922/plan.json | 252 ++++++++++++++++++ .../6.0.0_1582734336922/spec.json | 93 +++++++ .../6.0.0_1582734336922/topology | 42 +++ .../6.0.0_1582734336984/plan.json | 252 ++++++++++++++++++ .../6.0.0_1582734336984/spec.json | 93 +++++++ .../6.0.0_1582734336984/topology | 42 +++ .../6.0.0_1582734337023/plan.json | 252 ++++++++++++++++++ .../6.0.0_1582734337023/spec.json | 93 +++++++ .../6.0.0_1582734337023/topology | 42 +++ .../6.0.0_1582734337067/plan.json | 239 +++++++++++++++++ .../6.0.0_1582734337067/spec.json | 92 +++++++ .../6.0.0_1582734337067/topology | 33 +++ .../6.0.0_1582734337146/plan.json | 239 +++++++++++++++++ .../6.0.0_1582734337146/spec.json | 92 +++++++ .../6.0.0_1582734337146/topology | 33 +++ .../6.0.0_1582734337225/plan.json | 239 +++++++++++++++++ .../6.0.0_1582734337225/spec.json | 92 +++++++ .../6.0.0_1582734337225/topology | 33 +++ .../6.0.0_1582734337274/plan.json | 222 +++++++++++++++ .../6.0.0_1582734337274/spec.json | 89 +++++++ .../6.0.0_1582734337274/topology | 45 ++++ .../6.0.0_1582734337315/plan.json | 222 +++++++++++++++ .../6.0.0_1582734337315/spec.json | 89 +++++++ .../6.0.0_1582734337315/topology | 45 ++++ .../6.0.0_1582734337353/plan.json | 222 +++++++++++++++ .../6.0.0_1582734337353/spec.json | 89 +++++++ .../6.0.0_1582734337353/topology | 45 ++++ .../6.0.0_1582734337396/plan.json | 228 ++++++++++++++++ .../6.0.0_1582734337396/spec.json | 91 +++++++ .../6.0.0_1582734337396/topology | 45 ++++ .../6.0.0_1582734337431/plan.json | 228 ++++++++++++++++ .../6.0.0_1582734337431/spec.json | 91 +++++++ .../6.0.0_1582734337431/topology | 45 ++++ .../6.0.0_1582734337502/plan.json | 228 ++++++++++++++++ .../6.0.0_1582734337502/spec.json | 91 +++++++ .../6.0.0_1582734337502/topology | 45 ++++ .../6.0.0_1582734342043/plan.json | 151 +++++++++++ .../6.0.0_1582734342043/spec.json | 17 ++ .../6.0.0_1582734342043/topology | 16 ++ .../6.0.0_1582734341981/plan.json | 151 +++++++++++ .../6.0.0_1582734341981/spec.json | 47 ++++ .../6.0.0_1582734341981/topology | 16 ++ .../6.0.0_1582734341990/plan.json | 151 +++++++++++ .../6.0.0_1582734341990/spec.json | 50 ++++ .../6.0.0_1582734341990/topology | 16 ++ .../6.0.0_1582734341998/plan.json | 151 +++++++++++ .../6.0.0_1582734341998/spec.json | 47 ++++ .../6.0.0_1582734341998/topology | 22 ++ .../6.0.0_1582734342008/plan.json | 151 +++++++++++ .../6.0.0_1582734342008/spec.json | 50 ++++ .../6.0.0_1582734342008/topology | 16 ++ .../6.0.0_1582734342024/plan.json | 151 +++++++++++ .../6.0.0_1582734342024/spec.json | 38 +++ .../6.0.0_1582734342024/topology | 13 + 63 files changed, 6804 insertions(+) create mode 100644 ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_stream_inner_join_with_ts_-_AVRO/6.0.0_1582734336704/plan.json create mode 100644 ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_stream_inner_join_with_ts_-_AVRO/6.0.0_1582734336704/spec.json create mode 100644 ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_stream_inner_join_with_ts_-_AVRO/6.0.0_1582734336704/topology create mode 100644 ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_stream_inner_join_with_ts_-_JSON/6.0.0_1582734336763/plan.json create mode 100644 ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_stream_inner_join_with_ts_-_JSON/6.0.0_1582734336763/spec.json create mode 100644 ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_stream_inner_join_with_ts_-_JSON/6.0.0_1582734336763/topology create mode 100644 ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_stream_inner_join_with_ts_-_PROTOBUF/6.0.0_1582734336880/plan.json create mode 100644 ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_stream_inner_join_with_ts_-_PROTOBUF/6.0.0_1582734336880/spec.json create mode 100644 ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_stream_inner_join_with_ts_-_PROTOBUF/6.0.0_1582734336880/topology create mode 100644 ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_stream_inner_join_with_ts_extractor_both_sides_-_AVRO/6.0.0_1582734336922/plan.json create mode 100644 ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_stream_inner_join_with_ts_extractor_both_sides_-_AVRO/6.0.0_1582734336922/spec.json create mode 100644 ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_stream_inner_join_with_ts_extractor_both_sides_-_AVRO/6.0.0_1582734336922/topology create mode 100644 ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_stream_inner_join_with_ts_extractor_both_sides_-_JSON/6.0.0_1582734336984/plan.json create mode 100644 ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_stream_inner_join_with_ts_extractor_both_sides_-_JSON/6.0.0_1582734336984/spec.json create mode 100644 ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_stream_inner_join_with_ts_extractor_both_sides_-_JSON/6.0.0_1582734336984/topology create mode 100644 ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_stream_inner_join_with_ts_extractor_both_sides_-_PROTOBUF/6.0.0_1582734337023/plan.json create mode 100644 ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_stream_inner_join_with_ts_extractor_both_sides_-_PROTOBUF/6.0.0_1582734337023/spec.json create mode 100644 ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_stream_inner_join_with_ts_extractor_both_sides_-_PROTOBUF/6.0.0_1582734337023/topology create mode 100644 ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_table_join_with_ts_extractor_both_sides_-_AVRO/6.0.0_1582734337067/plan.json create mode 100644 ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_table_join_with_ts_extractor_both_sides_-_AVRO/6.0.0_1582734337067/spec.json create mode 100644 ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_table_join_with_ts_extractor_both_sides_-_AVRO/6.0.0_1582734337067/topology create mode 100644 ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_table_join_with_ts_extractor_both_sides_-_JSON/6.0.0_1582734337146/plan.json create mode 100644 ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_table_join_with_ts_extractor_both_sides_-_JSON/6.0.0_1582734337146/spec.json create mode 100644 ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_table_join_with_ts_extractor_both_sides_-_JSON/6.0.0_1582734337146/topology create mode 100644 ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_table_join_with_ts_extractor_both_sides_-_PROTOBUF/6.0.0_1582734337225/plan.json create mode 100644 ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_table_join_with_ts_extractor_both_sides_-_PROTOBUF/6.0.0_1582734337225/spec.json create mode 100644 ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_table_join_with_ts_extractor_both_sides_-_PROTOBUF/6.0.0_1582734337225/topology create mode 100644 ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_-_AVRO/6.0.0_1582734337274/plan.json create mode 100644 ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_-_AVRO/6.0.0_1582734337274/spec.json create mode 100644 ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_-_AVRO/6.0.0_1582734337274/topology create mode 100644 ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_-_JSON/6.0.0_1582734337315/plan.json create mode 100644 ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_-_JSON/6.0.0_1582734337315/spec.json create mode 100644 ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_-_JSON/6.0.0_1582734337315/topology create mode 100644 ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_-_PROTOBUF/6.0.0_1582734337353/plan.json create mode 100644 ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_-_PROTOBUF/6.0.0_1582734337353/spec.json create mode 100644 ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_-_PROTOBUF/6.0.0_1582734337353/topology create mode 100644 ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_extractor_both_sides_-_AVRO/6.0.0_1582734337396/plan.json create mode 100644 ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_extractor_both_sides_-_AVRO/6.0.0_1582734337396/spec.json create mode 100644 ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_extractor_both_sides_-_AVRO/6.0.0_1582734337396/topology create mode 100644 ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_extractor_both_sides_-_JSON/6.0.0_1582734337431/plan.json create mode 100644 ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_extractor_both_sides_-_JSON/6.0.0_1582734337431/spec.json create mode 100644 ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_extractor_both_sides_-_JSON/6.0.0_1582734337431/topology create mode 100644 ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_extractor_both_sides_-_PROTOBUF/6.0.0_1582734337502/plan.json create mode 100644 ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_extractor_both_sides_-_PROTOBUF/6.0.0_1582734337502/spec.json create mode 100644 ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_extractor_both_sides_-_PROTOBUF/6.0.0_1582734337502/topology create mode 100644 ksql-functional-tests/src/test/resources/historical_plans/timestampformat_-_Invalid_timestamp_value_should_throw_an_exception/6.0.0_1582734342043/plan.json create mode 100644 ksql-functional-tests/src/test/resources/historical_plans/timestampformat_-_Invalid_timestamp_value_should_throw_an_exception/6.0.0_1582734342043/spec.json create mode 100644 ksql-functional-tests/src/test/resources/historical_plans/timestampformat_-_Invalid_timestamp_value_should_throw_an_exception/6.0.0_1582734342043/topology create mode 100644 ksql-functional-tests/src/test/resources/historical_plans/timestampformat_-_override_output_timestamp_for_CSAS/6.0.0_1582734341981/plan.json create mode 100644 ksql-functional-tests/src/test/resources/historical_plans/timestampformat_-_override_output_timestamp_for_CSAS/6.0.0_1582734341981/spec.json create mode 100644 ksql-functional-tests/src/test/resources/historical_plans/timestampformat_-_override_output_timestamp_for_CSAS/6.0.0_1582734341981/topology create mode 100644 ksql-functional-tests/src/test/resources/historical_plans/timestampformat_-_override_output_timestamp_for_CSAS_using_a_string_TIMESTAMP_FORMAT/6.0.0_1582734341990/plan.json create mode 100644 ksql-functional-tests/src/test/resources/historical_plans/timestampformat_-_override_output_timestamp_for_CSAS_using_a_string_TIMESTAMP_FORMAT/6.0.0_1582734341990/spec.json create mode 100644 ksql-functional-tests/src/test/resources/historical_plans/timestampformat_-_override_output_timestamp_for_CSAS_using_a_string_TIMESTAMP_FORMAT/6.0.0_1582734341990/topology create mode 100644 ksql-functional-tests/src/test/resources/historical_plans/timestampformat_-_override_output_timestamp_for_CTAS/6.0.0_1582734341998/plan.json create mode 100644 ksql-functional-tests/src/test/resources/historical_plans/timestampformat_-_override_output_timestamp_for_CTAS/6.0.0_1582734341998/spec.json create mode 100644 ksql-functional-tests/src/test/resources/historical_plans/timestampformat_-_override_output_timestamp_for_CTAS/6.0.0_1582734341998/topology create mode 100644 ksql-functional-tests/src/test/resources/historical_plans/timestampformat_-_override_output_timestamp_for_CTAS_using_a_string_TIMESTAMP_FORMAT/6.0.0_1582734342008/plan.json create mode 100644 ksql-functional-tests/src/test/resources/historical_plans/timestampformat_-_override_output_timestamp_for_CTAS_using_a_string_TIMESTAMP_FORMAT/6.0.0_1582734342008/spec.json create mode 100644 ksql-functional-tests/src/test/resources/historical_plans/timestampformat_-_override_output_timestamp_for_CTAS_using_a_string_TIMESTAMP_FORMAT/6.0.0_1582734342008/topology create mode 100644 ksql-functional-tests/src/test/resources/historical_plans/timestampformat_-_timestamp_column_of_source_should_not_influence_sink/6.0.0_1582734342024/plan.json create mode 100644 ksql-functional-tests/src/test/resources/historical_plans/timestampformat_-_timestamp_column_of_source_should_not_influence_sink/6.0.0_1582734342024/spec.json create mode 100644 ksql-functional-tests/src/test/resources/historical_plans/timestampformat_-_timestamp_column_of_source_should_not_influence_sink/6.0.0_1582734342024/topology diff --git a/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_stream_inner_join_with_ts_-_AVRO/6.0.0_1582734336704/plan.json b/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_stream_inner_join_with_ts_-_AVRO/6.0.0_1582734336704/plan.json new file mode 100644 index 000000000000..b611bc7eb5ea --- /dev/null +++ b/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_stream_inner_join_with_ts_-_AVRO/6.0.0_1582734336704/plan.json @@ -0,0 +1,246 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S1 (ROWKEY BIGINT KEY, ID BIGINT, NAME STRING, TS BIGINT) WITH (KAFKA_TOPIC='s1', KEY='ID', TIMESTAMP='TS', VALUE_FORMAT='AVRO');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S1", + "schema" : "`ROWKEY` BIGINT KEY, `ID` BIGINT, `NAME` STRING, `TS` BIGINT", + "keyField" : "ID", + "timestampColumn" : { + "column" : "TS", + "format" : null + }, + "topicName" : "s1", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "AVRO", + "properties" : { } + }, + "options" : [ ] + }, + "windowInfo" : null + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S2 (ROWKEY BIGINT KEY, ID BIGINT, F1 STRING, F2 STRING) WITH (KAFKA_TOPIC='s2', KEY='ID', VALUE_FORMAT='AVRO');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S2", + "schema" : "`ROWKEY` BIGINT KEY, `ID` BIGINT, `F1` STRING, `F2` STRING", + "keyField" : "ID", + "timestampColumn" : null, + "topicName" : "s2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "AVRO", + "properties" : { } + }, + "options" : [ ] + }, + "windowInfo" : null + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S1_JOIN_S2 WITH (TIMESTAMP='TS') AS SELECT\n S1.ID ID,\n S1.NAME NAME,\n S1.TS TS,\n S2.F1 F1,\n S2.F2 F2\nFROM S1 S1\nINNER JOIN S2 S2 WITHIN 11 SECONDS ON ((S1.ID = S2.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S1_JOIN_S2", + "schema" : "`ROWKEY` BIGINT KEY, `ID` BIGINT, `NAME` STRING, `TS` BIGINT, `F1` STRING, `F2` STRING", + "keyField" : "ID", + "timestampColumn" : { + "column" : "TS", + "format" : null + }, + "topicName" : "S1_JOIN_S2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "AVRO", + "properties" : { } + }, + "options" : [ ] + }, + "windowInfo" : null + }, + "queryPlan" : { + "sources" : [ "S1", "S2" ], + "sink" : "S1_JOIN_S2", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "S1_JOIN_S2" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamStreamJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "leftInternalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "AVRO", + "properties" : { } + }, + "options" : [ ] + }, + "rightInternalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "AVRO", + "properties" : { } + }, + "options" : [ ] + }, + "leftSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "s1", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "AVRO", + "properties" : { } + }, + "options" : [ ] + }, + "timestampColumn" : { + "column" : "TS", + "format" : null + }, + "sourceSchema" : "`ROWKEY` BIGINT KEY, `ID` BIGINT, `NAME` STRING, `TS` BIGINT" + }, + "selectExpressions" : [ "ID AS S1_ID", "NAME AS S1_NAME", "TS AS S1_TS", "ROWTIME AS S1_ROWTIME", "ROWKEY AS S1_ROWKEY" ] + }, + "rightSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "s2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "AVRO", + "properties" : { } + }, + "options" : [ ] + }, + "timestampColumn" : null, + "sourceSchema" : "`ROWKEY` BIGINT KEY, `ID` BIGINT, `F1` STRING, `F2` STRING" + }, + "selectExpressions" : [ "ID AS S2_ID", "F1 AS S2_F1", "F2 AS S2_F2", "ROWTIME AS S2_ROWTIME", "ROWKEY AS S2_ROWKEY" ] + }, + "beforeMillis" : 11.000000000, + "afterMillis" : 11.000000000 + }, + "selectExpressions" : [ "S1_ID AS ID", "S1_NAME AS NAME", "S1_TS AS TS", "S2_F1 AS F1", "S2_F2 AS F2" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "AVRO", + "properties" : { } + }, + "options" : [ ] + }, + "topicName" : "S1_JOIN_S2", + "timestampColumn" : { + "column" : "TS", + "format" : null + } + }, + "queryId" : "CSAS_S1_JOIN_S2_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.new.api.enabled" : "false", + "ksql.streams.state.dir" : "/tmp/confluent8609378800848380873", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.query.pull.streamsstore.rebalancing.timeout.ms" : "10000", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647" + } +} \ No newline at end of file diff --git a/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_stream_inner_join_with_ts_-_AVRO/6.0.0_1582734336704/spec.json b/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_stream_inner_join_with_ts_-_AVRO/6.0.0_1582734336704/spec.json new file mode 100644 index 000000000000..eabdb79b4383 --- /dev/null +++ b/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_stream_inner_join_with_ts_-_AVRO/6.0.0_1582734336704/spec.json @@ -0,0 +1,91 @@ +{ + "version" : "6.0.0", + "timestamp" : 1582734336704, + "schemas" : { + "CSAS_S1_JOIN_S2_0.KafkaTopic_Left.Source" : "STRUCT NOT NULL", + "CSAS_S1_JOIN_S2_0.KafkaTopic_Right.Source" : "STRUCT NOT NULL", + "CSAS_S1_JOIN_S2_0.Join.Left" : "STRUCT NOT NULL", + "CSAS_S1_JOIN_S2_0.Join.Right" : "STRUCT NOT NULL", + "CSAS_S1_JOIN_S2_0.S1_JOIN_S2" : "STRUCT NOT NULL" + }, + "inputs" : [ { + "topic" : "s1", + "key" : 0, + "value" : { + "ID" : 0, + "NAME" : "zero", + "TS" : 0 + }, + "timestamp" : 0 + }, { + "topic" : "s2", + "key" : 0, + "value" : { + "ID" : 0, + "F1" : "blah", + "F2" : "foo" + }, + "timestamp" : 10000 + }, { + "topic" : "s2", + "key" : 10, + "value" : { + "ID" : 10, + "F1" : "foo", + "F2" : "bar" + }, + "timestamp" : 13000 + }, { + "topic" : "s1", + "key" : 10, + "value" : { + "ID" : 10, + "NAME" : "100", + "TS" : 11000 + }, + "timestamp" : 22000 + }, { + "topic" : "s1", + "key" : 0, + "value" : { + "ID" : 0, + "NAME" : "jan", + "TS" : 8000 + }, + "timestamp" : 33000 + } ], + "outputs" : [ { + "topic" : "S1_JOIN_S2", + "key" : 0, + "value" : { + "ID" : 0, + "NAME" : "zero", + "TS" : 0, + "F1" : "blah", + "F2" : "foo" + }, + "timestamp" : 0 + }, { + "topic" : "S1_JOIN_S2", + "key" : 10, + "value" : { + "ID" : 10, + "NAME" : "100", + "TS" : 11000, + "F1" : "foo", + "F2" : "bar" + }, + "timestamp" : 11000 + }, { + "topic" : "S1_JOIN_S2", + "key" : 0, + "value" : { + "ID" : 0, + "NAME" : "jan", + "TS" : 8000, + "F1" : "blah", + "F2" : "foo" + }, + "timestamp" : 8000 + } ] +} \ No newline at end of file diff --git a/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_stream_inner_join_with_ts_-_AVRO/6.0.0_1582734336704/topology b/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_stream_inner_join_with_ts_-_AVRO/6.0.0_1582734336704/topology new file mode 100644 index 000000000000..f74bc03af76a --- /dev/null +++ b/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_stream_inner_join_with_ts_-_AVRO/6.0.0_1582734336704/topology @@ -0,0 +1,42 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [s1]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Source: KSTREAM-SOURCE-0000000003 (topics: [s2]) + --> KSTREAM-TRANSFORMVALUES-0000000004 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> PrependAliasLeft + <-- KSTREAM-SOURCE-0000000000 + Processor: KSTREAM-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasRight + <-- KSTREAM-SOURCE-0000000003 + Processor: PrependAliasLeft (stores: []) + --> Join-this-windowed + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: PrependAliasRight (stores: []) + --> Join-other-windowed + <-- KSTREAM-TRANSFORMVALUES-0000000004 + Processor: Join-other-windowed (stores: [KSTREAM-JOINOTHER-0000000009-store]) + --> Join-other-join + <-- PrependAliasRight + Processor: Join-this-windowed (stores: [KSTREAM-JOINTHIS-0000000008-store]) + --> Join-this-join + <-- PrependAliasLeft + Processor: Join-other-join (stores: [KSTREAM-JOINTHIS-0000000008-store]) + --> Join-merge + <-- Join-other-windowed + Processor: Join-this-join (stores: [KSTREAM-JOINOTHER-0000000009-store]) + --> Join-merge + <-- Join-this-windowed + Processor: Join-merge (stores: []) + --> Project + <-- Join-this-join, Join-other-join + Processor: Project (stores: []) + --> ApplyTimestampTransform-S1_JOIN_S2 + <-- Join-merge + Processor: ApplyTimestampTransform-S1_JOIN_S2 (stores: []) + --> KSTREAM-SINK-0000000012 + <-- Project + Sink: KSTREAM-SINK-0000000012 (topic: S1_JOIN_S2) + <-- ApplyTimestampTransform-S1_JOIN_S2 + diff --git a/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_stream_inner_join_with_ts_-_JSON/6.0.0_1582734336763/plan.json b/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_stream_inner_join_with_ts_-_JSON/6.0.0_1582734336763/plan.json new file mode 100644 index 000000000000..ba6acc7623d0 --- /dev/null +++ b/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_stream_inner_join_with_ts_-_JSON/6.0.0_1582734336763/plan.json @@ -0,0 +1,246 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S1 (ROWKEY BIGINT KEY, ID BIGINT, NAME STRING, TS BIGINT) WITH (KAFKA_TOPIC='s1', KEY='ID', TIMESTAMP='TS', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S1", + "schema" : "`ROWKEY` BIGINT KEY, `ID` BIGINT, `NAME` STRING, `TS` BIGINT", + "keyField" : "ID", + "timestampColumn" : { + "column" : "TS", + "format" : null + }, + "topicName" : "s1", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "JSON", + "properties" : { } + }, + "options" : [ ] + }, + "windowInfo" : null + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S2 (ROWKEY BIGINT KEY, ID BIGINT, F1 STRING, F2 STRING) WITH (KAFKA_TOPIC='s2', KEY='ID', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S2", + "schema" : "`ROWKEY` BIGINT KEY, `ID` BIGINT, `F1` STRING, `F2` STRING", + "keyField" : "ID", + "timestampColumn" : null, + "topicName" : "s2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "JSON", + "properties" : { } + }, + "options" : [ ] + }, + "windowInfo" : null + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S1_JOIN_S2 WITH (TIMESTAMP='TS') AS SELECT\n S1.ID ID,\n S1.NAME NAME,\n S1.TS TS,\n S2.F1 F1,\n S2.F2 F2\nFROM S1 S1\nINNER JOIN S2 S2 WITHIN 11 SECONDS ON ((S1.ID = S2.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S1_JOIN_S2", + "schema" : "`ROWKEY` BIGINT KEY, `ID` BIGINT, `NAME` STRING, `TS` BIGINT, `F1` STRING, `F2` STRING", + "keyField" : "ID", + "timestampColumn" : { + "column" : "TS", + "format" : null + }, + "topicName" : "S1_JOIN_S2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "JSON", + "properties" : { } + }, + "options" : [ ] + }, + "windowInfo" : null + }, + "queryPlan" : { + "sources" : [ "S1", "S2" ], + "sink" : "S1_JOIN_S2", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "S1_JOIN_S2" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamStreamJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "leftInternalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "JSON", + "properties" : { } + }, + "options" : [ ] + }, + "rightInternalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "JSON", + "properties" : { } + }, + "options" : [ ] + }, + "leftSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "s1", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "JSON", + "properties" : { } + }, + "options" : [ ] + }, + "timestampColumn" : { + "column" : "TS", + "format" : null + }, + "sourceSchema" : "`ROWKEY` BIGINT KEY, `ID` BIGINT, `NAME` STRING, `TS` BIGINT" + }, + "selectExpressions" : [ "ID AS S1_ID", "NAME AS S1_NAME", "TS AS S1_TS", "ROWTIME AS S1_ROWTIME", "ROWKEY AS S1_ROWKEY" ] + }, + "rightSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "s2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "JSON", + "properties" : { } + }, + "options" : [ ] + }, + "timestampColumn" : null, + "sourceSchema" : "`ROWKEY` BIGINT KEY, `ID` BIGINT, `F1` STRING, `F2` STRING" + }, + "selectExpressions" : [ "ID AS S2_ID", "F1 AS S2_F1", "F2 AS S2_F2", "ROWTIME AS S2_ROWTIME", "ROWKEY AS S2_ROWKEY" ] + }, + "beforeMillis" : 11.000000000, + "afterMillis" : 11.000000000 + }, + "selectExpressions" : [ "S1_ID AS ID", "S1_NAME AS NAME", "S1_TS AS TS", "S2_F1 AS F1", "S2_F2 AS F2" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "JSON", + "properties" : { } + }, + "options" : [ ] + }, + "topicName" : "S1_JOIN_S2", + "timestampColumn" : { + "column" : "TS", + "format" : null + } + }, + "queryId" : "CSAS_S1_JOIN_S2_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.new.api.enabled" : "false", + "ksql.streams.state.dir" : "/tmp/confluent8609378800848380873", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.query.pull.streamsstore.rebalancing.timeout.ms" : "10000", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647" + } +} \ No newline at end of file diff --git a/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_stream_inner_join_with_ts_-_JSON/6.0.0_1582734336763/spec.json b/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_stream_inner_join_with_ts_-_JSON/6.0.0_1582734336763/spec.json new file mode 100644 index 000000000000..519ecf7f5989 --- /dev/null +++ b/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_stream_inner_join_with_ts_-_JSON/6.0.0_1582734336763/spec.json @@ -0,0 +1,91 @@ +{ + "version" : "6.0.0", + "timestamp" : 1582734336763, + "schemas" : { + "CSAS_S1_JOIN_S2_0.KafkaTopic_Left.Source" : "STRUCT NOT NULL", + "CSAS_S1_JOIN_S2_0.KafkaTopic_Right.Source" : "STRUCT NOT NULL", + "CSAS_S1_JOIN_S2_0.Join.Left" : "STRUCT NOT NULL", + "CSAS_S1_JOIN_S2_0.Join.Right" : "STRUCT NOT NULL", + "CSAS_S1_JOIN_S2_0.S1_JOIN_S2" : "STRUCT NOT NULL" + }, + "inputs" : [ { + "topic" : "s1", + "key" : 0, + "value" : { + "ID" : 0, + "NAME" : "zero", + "TS" : 0 + }, + "timestamp" : 0 + }, { + "topic" : "s2", + "key" : 0, + "value" : { + "ID" : 0, + "F1" : "blah", + "F2" : "foo" + }, + "timestamp" : 10000 + }, { + "topic" : "s2", + "key" : 10, + "value" : { + "ID" : 10, + "F1" : "foo", + "F2" : "bar" + }, + "timestamp" : 13000 + }, { + "topic" : "s1", + "key" : 10, + "value" : { + "ID" : 10, + "NAME" : "100", + "TS" : 11000 + }, + "timestamp" : 22000 + }, { + "topic" : "s1", + "key" : 0, + "value" : { + "ID" : 0, + "NAME" : "jan", + "TS" : 8000 + }, + "timestamp" : 33000 + } ], + "outputs" : [ { + "topic" : "S1_JOIN_S2", + "key" : 0, + "value" : { + "ID" : 0, + "NAME" : "zero", + "TS" : 0, + "F1" : "blah", + "F2" : "foo" + }, + "timestamp" : 0 + }, { + "topic" : "S1_JOIN_S2", + "key" : 10, + "value" : { + "ID" : 10, + "NAME" : "100", + "TS" : 11000, + "F1" : "foo", + "F2" : "bar" + }, + "timestamp" : 11000 + }, { + "topic" : "S1_JOIN_S2", + "key" : 0, + "value" : { + "ID" : 0, + "NAME" : "jan", + "TS" : 8000, + "F1" : "blah", + "F2" : "foo" + }, + "timestamp" : 8000 + } ] +} \ No newline at end of file diff --git a/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_stream_inner_join_with_ts_-_JSON/6.0.0_1582734336763/topology b/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_stream_inner_join_with_ts_-_JSON/6.0.0_1582734336763/topology new file mode 100644 index 000000000000..f74bc03af76a --- /dev/null +++ b/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_stream_inner_join_with_ts_-_JSON/6.0.0_1582734336763/topology @@ -0,0 +1,42 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [s1]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Source: KSTREAM-SOURCE-0000000003 (topics: [s2]) + --> KSTREAM-TRANSFORMVALUES-0000000004 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> PrependAliasLeft + <-- KSTREAM-SOURCE-0000000000 + Processor: KSTREAM-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasRight + <-- KSTREAM-SOURCE-0000000003 + Processor: PrependAliasLeft (stores: []) + --> Join-this-windowed + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: PrependAliasRight (stores: []) + --> Join-other-windowed + <-- KSTREAM-TRANSFORMVALUES-0000000004 + Processor: Join-other-windowed (stores: [KSTREAM-JOINOTHER-0000000009-store]) + --> Join-other-join + <-- PrependAliasRight + Processor: Join-this-windowed (stores: [KSTREAM-JOINTHIS-0000000008-store]) + --> Join-this-join + <-- PrependAliasLeft + Processor: Join-other-join (stores: [KSTREAM-JOINTHIS-0000000008-store]) + --> Join-merge + <-- Join-other-windowed + Processor: Join-this-join (stores: [KSTREAM-JOINOTHER-0000000009-store]) + --> Join-merge + <-- Join-this-windowed + Processor: Join-merge (stores: []) + --> Project + <-- Join-this-join, Join-other-join + Processor: Project (stores: []) + --> ApplyTimestampTransform-S1_JOIN_S2 + <-- Join-merge + Processor: ApplyTimestampTransform-S1_JOIN_S2 (stores: []) + --> KSTREAM-SINK-0000000012 + <-- Project + Sink: KSTREAM-SINK-0000000012 (topic: S1_JOIN_S2) + <-- ApplyTimestampTransform-S1_JOIN_S2 + diff --git a/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_stream_inner_join_with_ts_-_PROTOBUF/6.0.0_1582734336880/plan.json b/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_stream_inner_join_with_ts_-_PROTOBUF/6.0.0_1582734336880/plan.json new file mode 100644 index 000000000000..333c666462d1 --- /dev/null +++ b/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_stream_inner_join_with_ts_-_PROTOBUF/6.0.0_1582734336880/plan.json @@ -0,0 +1,246 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S1 (ROWKEY BIGINT KEY, ID BIGINT, NAME STRING, TS BIGINT) WITH (KAFKA_TOPIC='s1', KEY='ID', TIMESTAMP='TS', VALUE_FORMAT='PROTOBUF');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S1", + "schema" : "`ROWKEY` BIGINT KEY, `ID` BIGINT, `NAME` STRING, `TS` BIGINT", + "keyField" : "ID", + "timestampColumn" : { + "column" : "TS", + "format" : null + }, + "topicName" : "s1", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { } + }, + "options" : [ ] + }, + "windowInfo" : null + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S2 (ROWKEY BIGINT KEY, ID BIGINT, F1 STRING, F2 STRING) WITH (KAFKA_TOPIC='s2', KEY='ID', VALUE_FORMAT='PROTOBUF');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S2", + "schema" : "`ROWKEY` BIGINT KEY, `ID` BIGINT, `F1` STRING, `F2` STRING", + "keyField" : "ID", + "timestampColumn" : null, + "topicName" : "s2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { } + }, + "options" : [ ] + }, + "windowInfo" : null + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S1_JOIN_S2 WITH (TIMESTAMP='TS') AS SELECT\n S1.ID ID,\n S1.NAME NAME,\n S1.TS TS,\n S2.F1 F1,\n S2.F2 F2\nFROM S1 S1\nINNER JOIN S2 S2 WITHIN 11 SECONDS ON ((S1.ID = S2.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S1_JOIN_S2", + "schema" : "`ROWKEY` BIGINT KEY, `ID` BIGINT, `NAME` STRING, `TS` BIGINT, `F1` STRING, `F2` STRING", + "keyField" : "ID", + "timestampColumn" : { + "column" : "TS", + "format" : null + }, + "topicName" : "S1_JOIN_S2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { } + }, + "options" : [ ] + }, + "windowInfo" : null + }, + "queryPlan" : { + "sources" : [ "S1", "S2" ], + "sink" : "S1_JOIN_S2", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "S1_JOIN_S2" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamStreamJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "leftInternalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { } + }, + "options" : [ ] + }, + "rightInternalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { } + }, + "options" : [ ] + }, + "leftSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "s1", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { } + }, + "options" : [ ] + }, + "timestampColumn" : { + "column" : "TS", + "format" : null + }, + "sourceSchema" : "`ROWKEY` BIGINT KEY, `ID` BIGINT, `NAME` STRING, `TS` BIGINT" + }, + "selectExpressions" : [ "ID AS S1_ID", "NAME AS S1_NAME", "TS AS S1_TS", "ROWTIME AS S1_ROWTIME", "ROWKEY AS S1_ROWKEY" ] + }, + "rightSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "s2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { } + }, + "options" : [ ] + }, + "timestampColumn" : null, + "sourceSchema" : "`ROWKEY` BIGINT KEY, `ID` BIGINT, `F1` STRING, `F2` STRING" + }, + "selectExpressions" : [ "ID AS S2_ID", "F1 AS S2_F1", "F2 AS S2_F2", "ROWTIME AS S2_ROWTIME", "ROWKEY AS S2_ROWKEY" ] + }, + "beforeMillis" : 11.000000000, + "afterMillis" : 11.000000000 + }, + "selectExpressions" : [ "S1_ID AS ID", "S1_NAME AS NAME", "S1_TS AS TS", "S2_F1 AS F1", "S2_F2 AS F2" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { } + }, + "options" : [ ] + }, + "topicName" : "S1_JOIN_S2", + "timestampColumn" : { + "column" : "TS", + "format" : null + } + }, + "queryId" : "CSAS_S1_JOIN_S2_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.new.api.enabled" : "false", + "ksql.streams.state.dir" : "/tmp/confluent8609378800848380873", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.query.pull.streamsstore.rebalancing.timeout.ms" : "10000", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647" + } +} \ No newline at end of file diff --git a/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_stream_inner_join_with_ts_-_PROTOBUF/6.0.0_1582734336880/spec.json b/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_stream_inner_join_with_ts_-_PROTOBUF/6.0.0_1582734336880/spec.json new file mode 100644 index 000000000000..6ce815b43f16 --- /dev/null +++ b/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_stream_inner_join_with_ts_-_PROTOBUF/6.0.0_1582734336880/spec.json @@ -0,0 +1,91 @@ +{ + "version" : "6.0.0", + "timestamp" : 1582734336880, + "schemas" : { + "CSAS_S1_JOIN_S2_0.KafkaTopic_Left.Source" : "STRUCT NOT NULL", + "CSAS_S1_JOIN_S2_0.KafkaTopic_Right.Source" : "STRUCT NOT NULL", + "CSAS_S1_JOIN_S2_0.Join.Left" : "STRUCT NOT NULL", + "CSAS_S1_JOIN_S2_0.Join.Right" : "STRUCT NOT NULL", + "CSAS_S1_JOIN_S2_0.S1_JOIN_S2" : "STRUCT NOT NULL" + }, + "inputs" : [ { + "topic" : "s1", + "key" : 0, + "value" : { + "ID" : 0, + "NAME" : "zero", + "TS" : 0 + }, + "timestamp" : 0 + }, { + "topic" : "s2", + "key" : 0, + "value" : { + "ID" : 0, + "F1" : "blah", + "F2" : "foo" + }, + "timestamp" : 10000 + }, { + "topic" : "s2", + "key" : 10, + "value" : { + "ID" : 10, + "F1" : "foo", + "F2" : "bar" + }, + "timestamp" : 13000 + }, { + "topic" : "s1", + "key" : 10, + "value" : { + "ID" : 10, + "NAME" : "100", + "TS" : 11000 + }, + "timestamp" : 22000 + }, { + "topic" : "s1", + "key" : 0, + "value" : { + "ID" : 0, + "NAME" : "jan", + "TS" : 8000 + }, + "timestamp" : 33000 + } ], + "outputs" : [ { + "topic" : "S1_JOIN_S2", + "key" : 0, + "value" : { + "ID" : 0, + "NAME" : "zero", + "TS" : 0, + "F1" : "blah", + "F2" : "foo" + }, + "timestamp" : 0 + }, { + "topic" : "S1_JOIN_S2", + "key" : 10, + "value" : { + "ID" : 10, + "NAME" : "100", + "TS" : 11000, + "F1" : "foo", + "F2" : "bar" + }, + "timestamp" : 11000 + }, { + "topic" : "S1_JOIN_S2", + "key" : 0, + "value" : { + "ID" : 0, + "NAME" : "jan", + "TS" : 8000, + "F1" : "blah", + "F2" : "foo" + }, + "timestamp" : 8000 + } ] +} \ No newline at end of file diff --git a/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_stream_inner_join_with_ts_-_PROTOBUF/6.0.0_1582734336880/topology b/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_stream_inner_join_with_ts_-_PROTOBUF/6.0.0_1582734336880/topology new file mode 100644 index 000000000000..f74bc03af76a --- /dev/null +++ b/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_stream_inner_join_with_ts_-_PROTOBUF/6.0.0_1582734336880/topology @@ -0,0 +1,42 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [s1]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Source: KSTREAM-SOURCE-0000000003 (topics: [s2]) + --> KSTREAM-TRANSFORMVALUES-0000000004 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> PrependAliasLeft + <-- KSTREAM-SOURCE-0000000000 + Processor: KSTREAM-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasRight + <-- KSTREAM-SOURCE-0000000003 + Processor: PrependAliasLeft (stores: []) + --> Join-this-windowed + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: PrependAliasRight (stores: []) + --> Join-other-windowed + <-- KSTREAM-TRANSFORMVALUES-0000000004 + Processor: Join-other-windowed (stores: [KSTREAM-JOINOTHER-0000000009-store]) + --> Join-other-join + <-- PrependAliasRight + Processor: Join-this-windowed (stores: [KSTREAM-JOINTHIS-0000000008-store]) + --> Join-this-join + <-- PrependAliasLeft + Processor: Join-other-join (stores: [KSTREAM-JOINTHIS-0000000008-store]) + --> Join-merge + <-- Join-other-windowed + Processor: Join-this-join (stores: [KSTREAM-JOINOTHER-0000000009-store]) + --> Join-merge + <-- Join-this-windowed + Processor: Join-merge (stores: []) + --> Project + <-- Join-this-join, Join-other-join + Processor: Project (stores: []) + --> ApplyTimestampTransform-S1_JOIN_S2 + <-- Join-merge + Processor: ApplyTimestampTransform-S1_JOIN_S2 (stores: []) + --> KSTREAM-SINK-0000000012 + <-- Project + Sink: KSTREAM-SINK-0000000012 (topic: S1_JOIN_S2) + <-- ApplyTimestampTransform-S1_JOIN_S2 + diff --git a/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_stream_inner_join_with_ts_extractor_both_sides_-_AVRO/6.0.0_1582734336922/plan.json b/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_stream_inner_join_with_ts_extractor_both_sides_-_AVRO/6.0.0_1582734336922/plan.json new file mode 100644 index 000000000000..185676b53631 --- /dev/null +++ b/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_stream_inner_join_with_ts_extractor_both_sides_-_AVRO/6.0.0_1582734336922/plan.json @@ -0,0 +1,252 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S1 (ROWKEY BIGINT KEY, ID BIGINT, NAME STRING, TS BIGINT) WITH (KAFKA_TOPIC='s1', KEY='ID', TIMESTAMP='TS', VALUE_FORMAT='AVRO');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S1", + "schema" : "`ROWKEY` BIGINT KEY, `ID` BIGINT, `NAME` STRING, `TS` BIGINT", + "keyField" : "ID", + "timestampColumn" : { + "column" : "TS", + "format" : null + }, + "topicName" : "s1", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "AVRO", + "properties" : { } + }, + "options" : [ ] + }, + "windowInfo" : null + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S2 (ROWKEY BIGINT KEY, ID BIGINT, F1 STRING, F2 STRING, RTS BIGINT) WITH (KAFKA_TOPIC='s2', KEY='ID', TIMESTAMP='RTS', VALUE_FORMAT='AVRO');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S2", + "schema" : "`ROWKEY` BIGINT KEY, `ID` BIGINT, `F1` STRING, `F2` STRING, `RTS` BIGINT", + "keyField" : "ID", + "timestampColumn" : { + "column" : "RTS", + "format" : null + }, + "topicName" : "s2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "AVRO", + "properties" : { } + }, + "options" : [ ] + }, + "windowInfo" : null + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S1_JOIN_S2 WITH (TIMESTAMP='TS') AS SELECT\n S1.ID ID,\n S1.NAME NAME,\n S1.TS TS,\n S2.F1 F1,\n S2.F2 F2\nFROM S1 S1\nINNER JOIN S2 S2 WITHIN 11 SECONDS ON ((S1.ID = S2.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S1_JOIN_S2", + "schema" : "`ROWKEY` BIGINT KEY, `ID` BIGINT, `NAME` STRING, `TS` BIGINT, `F1` STRING, `F2` STRING", + "keyField" : "ID", + "timestampColumn" : { + "column" : "TS", + "format" : null + }, + "topicName" : "S1_JOIN_S2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "AVRO", + "properties" : { } + }, + "options" : [ ] + }, + "windowInfo" : null + }, + "queryPlan" : { + "sources" : [ "S1", "S2" ], + "sink" : "S1_JOIN_S2", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "S1_JOIN_S2" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamStreamJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "leftInternalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "AVRO", + "properties" : { } + }, + "options" : [ ] + }, + "rightInternalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "AVRO", + "properties" : { } + }, + "options" : [ ] + }, + "leftSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "s1", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "AVRO", + "properties" : { } + }, + "options" : [ ] + }, + "timestampColumn" : { + "column" : "TS", + "format" : null + }, + "sourceSchema" : "`ROWKEY` BIGINT KEY, `ID` BIGINT, `NAME` STRING, `TS` BIGINT" + }, + "selectExpressions" : [ "ID AS S1_ID", "NAME AS S1_NAME", "TS AS S1_TS", "ROWTIME AS S1_ROWTIME", "ROWKEY AS S1_ROWKEY" ] + }, + "rightSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "s2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "AVRO", + "properties" : { } + }, + "options" : [ ] + }, + "timestampColumn" : { + "column" : "RTS", + "format" : null + }, + "sourceSchema" : "`ROWKEY` BIGINT KEY, `ID` BIGINT, `F1` STRING, `F2` STRING, `RTS` BIGINT" + }, + "selectExpressions" : [ "ID AS S2_ID", "F1 AS S2_F1", "F2 AS S2_F2", "RTS AS S2_RTS", "ROWTIME AS S2_ROWTIME", "ROWKEY AS S2_ROWKEY" ] + }, + "beforeMillis" : 11.000000000, + "afterMillis" : 11.000000000 + }, + "selectExpressions" : [ "S1_ID AS ID", "S1_NAME AS NAME", "S1_TS AS TS", "S2_F1 AS F1", "S2_F2 AS F2" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "AVRO", + "properties" : { } + }, + "options" : [ ] + }, + "topicName" : "S1_JOIN_S2", + "timestampColumn" : { + "column" : "TS", + "format" : null + } + }, + "queryId" : "CSAS_S1_JOIN_S2_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.new.api.enabled" : "false", + "ksql.streams.state.dir" : "/tmp/confluent8609378800848380873", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.query.pull.streamsstore.rebalancing.timeout.ms" : "10000", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647" + } +} \ No newline at end of file diff --git a/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_stream_inner_join_with_ts_extractor_both_sides_-_AVRO/6.0.0_1582734336922/spec.json b/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_stream_inner_join_with_ts_extractor_both_sides_-_AVRO/6.0.0_1582734336922/spec.json new file mode 100644 index 000000000000..e9fc0f11ed2a --- /dev/null +++ b/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_stream_inner_join_with_ts_extractor_both_sides_-_AVRO/6.0.0_1582734336922/spec.json @@ -0,0 +1,93 @@ +{ + "version" : "6.0.0", + "timestamp" : 1582734336922, + "schemas" : { + "CSAS_S1_JOIN_S2_0.KafkaTopic_Left.Source" : "STRUCT NOT NULL", + "CSAS_S1_JOIN_S2_0.KafkaTopic_Right.Source" : "STRUCT NOT NULL", + "CSAS_S1_JOIN_S2_0.Join.Left" : "STRUCT NOT NULL", + "CSAS_S1_JOIN_S2_0.Join.Right" : "STRUCT NOT NULL", + "CSAS_S1_JOIN_S2_0.S1_JOIN_S2" : "STRUCT NOT NULL" + }, + "inputs" : [ { + "topic" : "s1", + "key" : 0, + "value" : { + "ID" : 0, + "NAME" : "zero", + "TS" : 0 + }, + "timestamp" : 0 + }, { + "topic" : "s2", + "key" : 0, + "value" : { + "ID" : 0, + "F1" : "blah", + "F2" : "foo", + "RTS" : 10000 + }, + "timestamp" : 0 + }, { + "topic" : "s2", + "key" : 10, + "value" : { + "ID" : 10, + "F1" : "foo", + "F2" : "bar", + "RTS" : 13000 + }, + "timestamp" : 0 + }, { + "topic" : "s1", + "key" : 10, + "value" : { + "ID" : 10, + "NAME" : "100", + "TS" : 11000 + }, + "timestamp" : 0 + }, { + "topic" : "s1", + "key" : 0, + "value" : { + "ID" : 0, + "NAME" : "jan", + "TS" : 8000 + }, + "timestamp" : 0 + } ], + "outputs" : [ { + "topic" : "S1_JOIN_S2", + "key" : 0, + "value" : { + "ID" : 0, + "NAME" : "zero", + "TS" : 0, + "F1" : "blah", + "F2" : "foo" + }, + "timestamp" : 0 + }, { + "topic" : "S1_JOIN_S2", + "key" : 10, + "value" : { + "ID" : 10, + "NAME" : "100", + "TS" : 11000, + "F1" : "foo", + "F2" : "bar" + }, + "timestamp" : 11000 + }, { + "topic" : "S1_JOIN_S2", + "key" : 0, + "value" : { + "ID" : 0, + "NAME" : "jan", + "TS" : 8000, + "F1" : "blah", + "F2" : "foo" + }, + "timestamp" : 8000 + } ] +} \ No newline at end of file diff --git a/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_stream_inner_join_with_ts_extractor_both_sides_-_AVRO/6.0.0_1582734336922/topology b/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_stream_inner_join_with_ts_extractor_both_sides_-_AVRO/6.0.0_1582734336922/topology new file mode 100644 index 000000000000..f74bc03af76a --- /dev/null +++ b/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_stream_inner_join_with_ts_extractor_both_sides_-_AVRO/6.0.0_1582734336922/topology @@ -0,0 +1,42 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [s1]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Source: KSTREAM-SOURCE-0000000003 (topics: [s2]) + --> KSTREAM-TRANSFORMVALUES-0000000004 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> PrependAliasLeft + <-- KSTREAM-SOURCE-0000000000 + Processor: KSTREAM-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasRight + <-- KSTREAM-SOURCE-0000000003 + Processor: PrependAliasLeft (stores: []) + --> Join-this-windowed + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: PrependAliasRight (stores: []) + --> Join-other-windowed + <-- KSTREAM-TRANSFORMVALUES-0000000004 + Processor: Join-other-windowed (stores: [KSTREAM-JOINOTHER-0000000009-store]) + --> Join-other-join + <-- PrependAliasRight + Processor: Join-this-windowed (stores: [KSTREAM-JOINTHIS-0000000008-store]) + --> Join-this-join + <-- PrependAliasLeft + Processor: Join-other-join (stores: [KSTREAM-JOINTHIS-0000000008-store]) + --> Join-merge + <-- Join-other-windowed + Processor: Join-this-join (stores: [KSTREAM-JOINOTHER-0000000009-store]) + --> Join-merge + <-- Join-this-windowed + Processor: Join-merge (stores: []) + --> Project + <-- Join-this-join, Join-other-join + Processor: Project (stores: []) + --> ApplyTimestampTransform-S1_JOIN_S2 + <-- Join-merge + Processor: ApplyTimestampTransform-S1_JOIN_S2 (stores: []) + --> KSTREAM-SINK-0000000012 + <-- Project + Sink: KSTREAM-SINK-0000000012 (topic: S1_JOIN_S2) + <-- ApplyTimestampTransform-S1_JOIN_S2 + diff --git a/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_stream_inner_join_with_ts_extractor_both_sides_-_JSON/6.0.0_1582734336984/plan.json b/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_stream_inner_join_with_ts_extractor_both_sides_-_JSON/6.0.0_1582734336984/plan.json new file mode 100644 index 000000000000..b9266faca5d9 --- /dev/null +++ b/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_stream_inner_join_with_ts_extractor_both_sides_-_JSON/6.0.0_1582734336984/plan.json @@ -0,0 +1,252 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S1 (ROWKEY BIGINT KEY, ID BIGINT, NAME STRING, TS BIGINT) WITH (KAFKA_TOPIC='s1', KEY='ID', TIMESTAMP='TS', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S1", + "schema" : "`ROWKEY` BIGINT KEY, `ID` BIGINT, `NAME` STRING, `TS` BIGINT", + "keyField" : "ID", + "timestampColumn" : { + "column" : "TS", + "format" : null + }, + "topicName" : "s1", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "JSON", + "properties" : { } + }, + "options" : [ ] + }, + "windowInfo" : null + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S2 (ROWKEY BIGINT KEY, ID BIGINT, F1 STRING, F2 STRING, RTS BIGINT) WITH (KAFKA_TOPIC='s2', KEY='ID', TIMESTAMP='RTS', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S2", + "schema" : "`ROWKEY` BIGINT KEY, `ID` BIGINT, `F1` STRING, `F2` STRING, `RTS` BIGINT", + "keyField" : "ID", + "timestampColumn" : { + "column" : "RTS", + "format" : null + }, + "topicName" : "s2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "JSON", + "properties" : { } + }, + "options" : [ ] + }, + "windowInfo" : null + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S1_JOIN_S2 WITH (TIMESTAMP='TS') AS SELECT\n S1.ID ID,\n S1.NAME NAME,\n S1.TS TS,\n S2.F1 F1,\n S2.F2 F2\nFROM S1 S1\nINNER JOIN S2 S2 WITHIN 11 SECONDS ON ((S1.ID = S2.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S1_JOIN_S2", + "schema" : "`ROWKEY` BIGINT KEY, `ID` BIGINT, `NAME` STRING, `TS` BIGINT, `F1` STRING, `F2` STRING", + "keyField" : "ID", + "timestampColumn" : { + "column" : "TS", + "format" : null + }, + "topicName" : "S1_JOIN_S2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "JSON", + "properties" : { } + }, + "options" : [ ] + }, + "windowInfo" : null + }, + "queryPlan" : { + "sources" : [ "S1", "S2" ], + "sink" : "S1_JOIN_S2", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "S1_JOIN_S2" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamStreamJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "leftInternalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "JSON", + "properties" : { } + }, + "options" : [ ] + }, + "rightInternalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "JSON", + "properties" : { } + }, + "options" : [ ] + }, + "leftSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "s1", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "JSON", + "properties" : { } + }, + "options" : [ ] + }, + "timestampColumn" : { + "column" : "TS", + "format" : null + }, + "sourceSchema" : "`ROWKEY` BIGINT KEY, `ID` BIGINT, `NAME` STRING, `TS` BIGINT" + }, + "selectExpressions" : [ "ID AS S1_ID", "NAME AS S1_NAME", "TS AS S1_TS", "ROWTIME AS S1_ROWTIME", "ROWKEY AS S1_ROWKEY" ] + }, + "rightSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "s2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "JSON", + "properties" : { } + }, + "options" : [ ] + }, + "timestampColumn" : { + "column" : "RTS", + "format" : null + }, + "sourceSchema" : "`ROWKEY` BIGINT KEY, `ID` BIGINT, `F1` STRING, `F2` STRING, `RTS` BIGINT" + }, + "selectExpressions" : [ "ID AS S2_ID", "F1 AS S2_F1", "F2 AS S2_F2", "RTS AS S2_RTS", "ROWTIME AS S2_ROWTIME", "ROWKEY AS S2_ROWKEY" ] + }, + "beforeMillis" : 11.000000000, + "afterMillis" : 11.000000000 + }, + "selectExpressions" : [ "S1_ID AS ID", "S1_NAME AS NAME", "S1_TS AS TS", "S2_F1 AS F1", "S2_F2 AS F2" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "JSON", + "properties" : { } + }, + "options" : [ ] + }, + "topicName" : "S1_JOIN_S2", + "timestampColumn" : { + "column" : "TS", + "format" : null + } + }, + "queryId" : "CSAS_S1_JOIN_S2_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.new.api.enabled" : "false", + "ksql.streams.state.dir" : "/tmp/confluent8609378800848380873", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.query.pull.streamsstore.rebalancing.timeout.ms" : "10000", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647" + } +} \ No newline at end of file diff --git a/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_stream_inner_join_with_ts_extractor_both_sides_-_JSON/6.0.0_1582734336984/spec.json b/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_stream_inner_join_with_ts_extractor_both_sides_-_JSON/6.0.0_1582734336984/spec.json new file mode 100644 index 000000000000..e25bf17e7907 --- /dev/null +++ b/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_stream_inner_join_with_ts_extractor_both_sides_-_JSON/6.0.0_1582734336984/spec.json @@ -0,0 +1,93 @@ +{ + "version" : "6.0.0", + "timestamp" : 1582734336984, + "schemas" : { + "CSAS_S1_JOIN_S2_0.KafkaTopic_Left.Source" : "STRUCT NOT NULL", + "CSAS_S1_JOIN_S2_0.KafkaTopic_Right.Source" : "STRUCT NOT NULL", + "CSAS_S1_JOIN_S2_0.Join.Left" : "STRUCT NOT NULL", + "CSAS_S1_JOIN_S2_0.Join.Right" : "STRUCT NOT NULL", + "CSAS_S1_JOIN_S2_0.S1_JOIN_S2" : "STRUCT NOT NULL" + }, + "inputs" : [ { + "topic" : "s1", + "key" : 0, + "value" : { + "ID" : 0, + "NAME" : "zero", + "TS" : 0 + }, + "timestamp" : 0 + }, { + "topic" : "s2", + "key" : 0, + "value" : { + "ID" : 0, + "F1" : "blah", + "F2" : "foo", + "RTS" : 10000 + }, + "timestamp" : 0 + }, { + "topic" : "s2", + "key" : 10, + "value" : { + "ID" : 10, + "F1" : "foo", + "F2" : "bar", + "RTS" : 13000 + }, + "timestamp" : 0 + }, { + "topic" : "s1", + "key" : 10, + "value" : { + "ID" : 10, + "NAME" : "100", + "TS" : 11000 + }, + "timestamp" : 0 + }, { + "topic" : "s1", + "key" : 0, + "value" : { + "ID" : 0, + "NAME" : "jan", + "TS" : 8000 + }, + "timestamp" : 0 + } ], + "outputs" : [ { + "topic" : "S1_JOIN_S2", + "key" : 0, + "value" : { + "ID" : 0, + "NAME" : "zero", + "TS" : 0, + "F1" : "blah", + "F2" : "foo" + }, + "timestamp" : 0 + }, { + "topic" : "S1_JOIN_S2", + "key" : 10, + "value" : { + "ID" : 10, + "NAME" : "100", + "TS" : 11000, + "F1" : "foo", + "F2" : "bar" + }, + "timestamp" : 11000 + }, { + "topic" : "S1_JOIN_S2", + "key" : 0, + "value" : { + "ID" : 0, + "NAME" : "jan", + "TS" : 8000, + "F1" : "blah", + "F2" : "foo" + }, + "timestamp" : 8000 + } ] +} \ No newline at end of file diff --git a/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_stream_inner_join_with_ts_extractor_both_sides_-_JSON/6.0.0_1582734336984/topology b/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_stream_inner_join_with_ts_extractor_both_sides_-_JSON/6.0.0_1582734336984/topology new file mode 100644 index 000000000000..f74bc03af76a --- /dev/null +++ b/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_stream_inner_join_with_ts_extractor_both_sides_-_JSON/6.0.0_1582734336984/topology @@ -0,0 +1,42 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [s1]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Source: KSTREAM-SOURCE-0000000003 (topics: [s2]) + --> KSTREAM-TRANSFORMVALUES-0000000004 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> PrependAliasLeft + <-- KSTREAM-SOURCE-0000000000 + Processor: KSTREAM-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasRight + <-- KSTREAM-SOURCE-0000000003 + Processor: PrependAliasLeft (stores: []) + --> Join-this-windowed + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: PrependAliasRight (stores: []) + --> Join-other-windowed + <-- KSTREAM-TRANSFORMVALUES-0000000004 + Processor: Join-other-windowed (stores: [KSTREAM-JOINOTHER-0000000009-store]) + --> Join-other-join + <-- PrependAliasRight + Processor: Join-this-windowed (stores: [KSTREAM-JOINTHIS-0000000008-store]) + --> Join-this-join + <-- PrependAliasLeft + Processor: Join-other-join (stores: [KSTREAM-JOINTHIS-0000000008-store]) + --> Join-merge + <-- Join-other-windowed + Processor: Join-this-join (stores: [KSTREAM-JOINOTHER-0000000009-store]) + --> Join-merge + <-- Join-this-windowed + Processor: Join-merge (stores: []) + --> Project + <-- Join-this-join, Join-other-join + Processor: Project (stores: []) + --> ApplyTimestampTransform-S1_JOIN_S2 + <-- Join-merge + Processor: ApplyTimestampTransform-S1_JOIN_S2 (stores: []) + --> KSTREAM-SINK-0000000012 + <-- Project + Sink: KSTREAM-SINK-0000000012 (topic: S1_JOIN_S2) + <-- ApplyTimestampTransform-S1_JOIN_S2 + diff --git a/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_stream_inner_join_with_ts_extractor_both_sides_-_PROTOBUF/6.0.0_1582734337023/plan.json b/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_stream_inner_join_with_ts_extractor_both_sides_-_PROTOBUF/6.0.0_1582734337023/plan.json new file mode 100644 index 000000000000..7e79c738f433 --- /dev/null +++ b/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_stream_inner_join_with_ts_extractor_both_sides_-_PROTOBUF/6.0.0_1582734337023/plan.json @@ -0,0 +1,252 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S1 (ROWKEY BIGINT KEY, ID BIGINT, NAME STRING, TS BIGINT) WITH (KAFKA_TOPIC='s1', KEY='ID', TIMESTAMP='TS', VALUE_FORMAT='PROTOBUF');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S1", + "schema" : "`ROWKEY` BIGINT KEY, `ID` BIGINT, `NAME` STRING, `TS` BIGINT", + "keyField" : "ID", + "timestampColumn" : { + "column" : "TS", + "format" : null + }, + "topicName" : "s1", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { } + }, + "options" : [ ] + }, + "windowInfo" : null + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S2 (ROWKEY BIGINT KEY, ID BIGINT, F1 STRING, F2 STRING, RTS BIGINT) WITH (KAFKA_TOPIC='s2', KEY='ID', TIMESTAMP='RTS', VALUE_FORMAT='PROTOBUF');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S2", + "schema" : "`ROWKEY` BIGINT KEY, `ID` BIGINT, `F1` STRING, `F2` STRING, `RTS` BIGINT", + "keyField" : "ID", + "timestampColumn" : { + "column" : "RTS", + "format" : null + }, + "topicName" : "s2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { } + }, + "options" : [ ] + }, + "windowInfo" : null + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S1_JOIN_S2 WITH (TIMESTAMP='TS') AS SELECT\n S1.ID ID,\n S1.NAME NAME,\n S1.TS TS,\n S2.F1 F1,\n S2.F2 F2\nFROM S1 S1\nINNER JOIN S2 S2 WITHIN 11 SECONDS ON ((S1.ID = S2.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S1_JOIN_S2", + "schema" : "`ROWKEY` BIGINT KEY, `ID` BIGINT, `NAME` STRING, `TS` BIGINT, `F1` STRING, `F2` STRING", + "keyField" : "ID", + "timestampColumn" : { + "column" : "TS", + "format" : null + }, + "topicName" : "S1_JOIN_S2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { } + }, + "options" : [ ] + }, + "windowInfo" : null + }, + "queryPlan" : { + "sources" : [ "S1", "S2" ], + "sink" : "S1_JOIN_S2", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "S1_JOIN_S2" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamStreamJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "leftInternalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { } + }, + "options" : [ ] + }, + "rightInternalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { } + }, + "options" : [ ] + }, + "leftSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "s1", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { } + }, + "options" : [ ] + }, + "timestampColumn" : { + "column" : "TS", + "format" : null + }, + "sourceSchema" : "`ROWKEY` BIGINT KEY, `ID` BIGINT, `NAME` STRING, `TS` BIGINT" + }, + "selectExpressions" : [ "ID AS S1_ID", "NAME AS S1_NAME", "TS AS S1_TS", "ROWTIME AS S1_ROWTIME", "ROWKEY AS S1_ROWKEY" ] + }, + "rightSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "s2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { } + }, + "options" : [ ] + }, + "timestampColumn" : { + "column" : "RTS", + "format" : null + }, + "sourceSchema" : "`ROWKEY` BIGINT KEY, `ID` BIGINT, `F1` STRING, `F2` STRING, `RTS` BIGINT" + }, + "selectExpressions" : [ "ID AS S2_ID", "F1 AS S2_F1", "F2 AS S2_F2", "RTS AS S2_RTS", "ROWTIME AS S2_ROWTIME", "ROWKEY AS S2_ROWKEY" ] + }, + "beforeMillis" : 11.000000000, + "afterMillis" : 11.000000000 + }, + "selectExpressions" : [ "S1_ID AS ID", "S1_NAME AS NAME", "S1_TS AS TS", "S2_F1 AS F1", "S2_F2 AS F2" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { } + }, + "options" : [ ] + }, + "topicName" : "S1_JOIN_S2", + "timestampColumn" : { + "column" : "TS", + "format" : null + } + }, + "queryId" : "CSAS_S1_JOIN_S2_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.new.api.enabled" : "false", + "ksql.streams.state.dir" : "/tmp/confluent8609378800848380873", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.query.pull.streamsstore.rebalancing.timeout.ms" : "10000", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647" + } +} \ No newline at end of file diff --git a/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_stream_inner_join_with_ts_extractor_both_sides_-_PROTOBUF/6.0.0_1582734337023/spec.json b/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_stream_inner_join_with_ts_extractor_both_sides_-_PROTOBUF/6.0.0_1582734337023/spec.json new file mode 100644 index 000000000000..85f5a8a9e9dc --- /dev/null +++ b/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_stream_inner_join_with_ts_extractor_both_sides_-_PROTOBUF/6.0.0_1582734337023/spec.json @@ -0,0 +1,93 @@ +{ + "version" : "6.0.0", + "timestamp" : 1582734337023, + "schemas" : { + "CSAS_S1_JOIN_S2_0.KafkaTopic_Left.Source" : "STRUCT NOT NULL", + "CSAS_S1_JOIN_S2_0.KafkaTopic_Right.Source" : "STRUCT NOT NULL", + "CSAS_S1_JOIN_S2_0.Join.Left" : "STRUCT NOT NULL", + "CSAS_S1_JOIN_S2_0.Join.Right" : "STRUCT NOT NULL", + "CSAS_S1_JOIN_S2_0.S1_JOIN_S2" : "STRUCT NOT NULL" + }, + "inputs" : [ { + "topic" : "s1", + "key" : 0, + "value" : { + "ID" : 0, + "NAME" : "zero", + "TS" : 0 + }, + "timestamp" : 0 + }, { + "topic" : "s2", + "key" : 0, + "value" : { + "ID" : 0, + "F1" : "blah", + "F2" : "foo", + "RTS" : 10000 + }, + "timestamp" : 0 + }, { + "topic" : "s2", + "key" : 10, + "value" : { + "ID" : 10, + "F1" : "foo", + "F2" : "bar", + "RTS" : 13000 + }, + "timestamp" : 0 + }, { + "topic" : "s1", + "key" : 10, + "value" : { + "ID" : 10, + "NAME" : "100", + "TS" : 11000 + }, + "timestamp" : 0 + }, { + "topic" : "s1", + "key" : 0, + "value" : { + "ID" : 0, + "NAME" : "jan", + "TS" : 8000 + }, + "timestamp" : 0 + } ], + "outputs" : [ { + "topic" : "S1_JOIN_S2", + "key" : 0, + "value" : { + "ID" : 0, + "NAME" : "zero", + "TS" : 0, + "F1" : "blah", + "F2" : "foo" + }, + "timestamp" : 0 + }, { + "topic" : "S1_JOIN_S2", + "key" : 10, + "value" : { + "ID" : 10, + "NAME" : "100", + "TS" : 11000, + "F1" : "foo", + "F2" : "bar" + }, + "timestamp" : 11000 + }, { + "topic" : "S1_JOIN_S2", + "key" : 0, + "value" : { + "ID" : 0, + "NAME" : "jan", + "TS" : 8000, + "F1" : "blah", + "F2" : "foo" + }, + "timestamp" : 8000 + } ] +} \ No newline at end of file diff --git a/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_stream_inner_join_with_ts_extractor_both_sides_-_PROTOBUF/6.0.0_1582734337023/topology b/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_stream_inner_join_with_ts_extractor_both_sides_-_PROTOBUF/6.0.0_1582734337023/topology new file mode 100644 index 000000000000..f74bc03af76a --- /dev/null +++ b/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_stream_inner_join_with_ts_extractor_both_sides_-_PROTOBUF/6.0.0_1582734337023/topology @@ -0,0 +1,42 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [s1]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Source: KSTREAM-SOURCE-0000000003 (topics: [s2]) + --> KSTREAM-TRANSFORMVALUES-0000000004 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> PrependAliasLeft + <-- KSTREAM-SOURCE-0000000000 + Processor: KSTREAM-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasRight + <-- KSTREAM-SOURCE-0000000003 + Processor: PrependAliasLeft (stores: []) + --> Join-this-windowed + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: PrependAliasRight (stores: []) + --> Join-other-windowed + <-- KSTREAM-TRANSFORMVALUES-0000000004 + Processor: Join-other-windowed (stores: [KSTREAM-JOINOTHER-0000000009-store]) + --> Join-other-join + <-- PrependAliasRight + Processor: Join-this-windowed (stores: [KSTREAM-JOINTHIS-0000000008-store]) + --> Join-this-join + <-- PrependAliasLeft + Processor: Join-other-join (stores: [KSTREAM-JOINTHIS-0000000008-store]) + --> Join-merge + <-- Join-other-windowed + Processor: Join-this-join (stores: [KSTREAM-JOINOTHER-0000000009-store]) + --> Join-merge + <-- Join-this-windowed + Processor: Join-merge (stores: []) + --> Project + <-- Join-this-join, Join-other-join + Processor: Project (stores: []) + --> ApplyTimestampTransform-S1_JOIN_S2 + <-- Join-merge + Processor: ApplyTimestampTransform-S1_JOIN_S2 (stores: []) + --> KSTREAM-SINK-0000000012 + <-- Project + Sink: KSTREAM-SINK-0000000012 (topic: S1_JOIN_S2) + <-- ApplyTimestampTransform-S1_JOIN_S2 + diff --git a/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_table_join_with_ts_extractor_both_sides_-_AVRO/6.0.0_1582734337067/plan.json b/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_table_join_with_ts_extractor_both_sides_-_AVRO/6.0.0_1582734337067/plan.json new file mode 100644 index 000000000000..dfdefef210e2 --- /dev/null +++ b/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_table_join_with_ts_extractor_both_sides_-_AVRO/6.0.0_1582734337067/plan.json @@ -0,0 +1,239 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S1 (ROWKEY BIGINT KEY, ID BIGINT, NAME STRING, TS BIGINT) WITH (KAFKA_TOPIC='s1', KEY='ID', TIMESTAMP='TS', VALUE_FORMAT='AVRO');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S1", + "schema" : "`ROWKEY` BIGINT KEY, `ID` BIGINT, `NAME` STRING, `TS` BIGINT", + "keyField" : "ID", + "timestampColumn" : { + "column" : "TS", + "format" : null + }, + "topicName" : "s1", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "AVRO", + "properties" : { } + }, + "options" : [ ] + }, + "windowInfo" : null + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE T1 (ROWKEY BIGINT KEY, ID BIGINT, F1 STRING, F2 STRING, RTS BIGINT) WITH (KAFKA_TOPIC='t1', KEY='ID', TIMESTAMP='RTS', VALUE_FORMAT='AVRO');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "T1", + "schema" : "`ROWKEY` BIGINT KEY, `ID` BIGINT, `F1` STRING, `F2` STRING, `RTS` BIGINT", + "keyField" : "ID", + "timestampColumn" : { + "column" : "RTS", + "format" : null + }, + "topicName" : "t1", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "AVRO", + "properties" : { } + }, + "options" : [ ] + }, + "windowInfo" : null + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S1_JOIN_T1 WITH (TIMESTAMP='TS') AS SELECT\n S1.ID ID,\n S1.NAME NAME,\n S1.TS TS,\n T1.F1 F1,\n T1.F2 F2\nFROM S1 S1\nINNER JOIN T1 T1 ON ((S1.ID = T1.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S1_JOIN_T1", + "schema" : "`ROWKEY` BIGINT KEY, `ID` BIGINT, `NAME` STRING, `TS` BIGINT, `F1` STRING, `F2` STRING", + "keyField" : "ID", + "timestampColumn" : { + "column" : "TS", + "format" : null + }, + "topicName" : "S1_JOIN_T1", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "AVRO", + "properties" : { } + }, + "options" : [ ] + }, + "windowInfo" : null + }, + "queryPlan" : { + "sources" : [ "S1", "T1" ], + "sink" : "S1_JOIN_T1", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "S1_JOIN_T1" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "AVRO", + "properties" : { } + }, + "options" : [ ] + }, + "leftSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "s1", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "AVRO", + "properties" : { } + }, + "options" : [ ] + }, + "timestampColumn" : { + "column" : "TS", + "format" : null + }, + "sourceSchema" : "`ROWKEY` BIGINT KEY, `ID` BIGINT, `NAME` STRING, `TS` BIGINT" + }, + "selectExpressions" : [ "ID AS S1_ID", "NAME AS S1_NAME", "TS AS S1_TS", "ROWTIME AS S1_ROWTIME", "ROWKEY AS S1_ROWKEY" ] + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "t1", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "AVRO", + "properties" : { } + }, + "options" : [ ] + }, + "timestampColumn" : { + "column" : "RTS", + "format" : null + }, + "sourceSchema" : "`ROWKEY` BIGINT KEY, `ID` BIGINT, `F1` STRING, `F2` STRING, `RTS` BIGINT" + }, + "selectExpressions" : [ "ID AS T1_ID", "F1 AS T1_F1", "F2 AS T1_F2", "RTS AS T1_RTS", "ROWTIME AS T1_ROWTIME", "ROWKEY AS T1_ROWKEY" ] + } + }, + "selectExpressions" : [ "S1_ID AS ID", "S1_NAME AS NAME", "S1_TS AS TS", "T1_F1 AS F1", "T1_F2 AS F2" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "AVRO", + "properties" : { } + }, + "options" : [ ] + }, + "topicName" : "S1_JOIN_T1", + "timestampColumn" : { + "column" : "TS", + "format" : null + } + }, + "queryId" : "CSAS_S1_JOIN_T1_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.new.api.enabled" : "false", + "ksql.streams.state.dir" : "/tmp/confluent8609378800848380873", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.query.pull.streamsstore.rebalancing.timeout.ms" : "10000", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647" + } +} \ No newline at end of file diff --git a/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_table_join_with_ts_extractor_both_sides_-_AVRO/6.0.0_1582734337067/spec.json b/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_table_join_with_ts_extractor_both_sides_-_AVRO/6.0.0_1582734337067/spec.json new file mode 100644 index 000000000000..d8c04cdf326a --- /dev/null +++ b/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_table_join_with_ts_extractor_both_sides_-_AVRO/6.0.0_1582734337067/spec.json @@ -0,0 +1,92 @@ +{ + "version" : "6.0.0", + "timestamp" : 1582734337067, + "schemas" : { + "CSAS_S1_JOIN_T1_0.KafkaTopic_Right.Source" : "STRUCT NOT NULL", + "CSAS_S1_JOIN_T1_0.KafkaTopic_Left.Source" : "STRUCT NOT NULL", + "CSAS_S1_JOIN_T1_0.Join.Left" : "STRUCT NOT NULL", + "CSAS_S1_JOIN_T1_0.S1_JOIN_T1" : "STRUCT NOT NULL" + }, + "inputs" : [ { + "topic" : "t1", + "key" : 0, + "value" : { + "ID" : 0, + "F1" : "blah", + "F2" : "foo", + "RTS" : 10000 + }, + "timestamp" : 10000 + }, { + "topic" : "s1", + "key" : 0, + "value" : { + "ID" : 0, + "NAME" : "zero", + "TS" : 0 + }, + "timestamp" : 10000 + }, { + "topic" : "t1", + "key" : 10, + "value" : { + "ID" : 10, + "F1" : "foo", + "F2" : "bar", + "RTS" : 13000 + }, + "timestamp" : 90000 + }, { + "topic" : "s1", + "key" : 10, + "value" : { + "ID" : 10, + "NAME" : "100", + "TS" : 11000 + }, + "timestamp" : 800000 + }, { + "topic" : "s1", + "key" : 0, + "value" : { + "ID" : 0, + "NAME" : "jan", + "TS" : 8000 + }, + "timestamp" : 0 + } ], + "outputs" : [ { + "topic" : "S1_JOIN_T1", + "key" : 0, + "value" : { + "ID" : 0, + "NAME" : "zero", + "TS" : 0, + "F1" : "blah", + "F2" : "foo" + }, + "timestamp" : 0 + }, { + "topic" : "S1_JOIN_T1", + "key" : 10, + "value" : { + "ID" : 10, + "NAME" : "100", + "TS" : 11000, + "F1" : "foo", + "F2" : "bar" + }, + "timestamp" : 11000 + }, { + "topic" : "S1_JOIN_T1", + "key" : 0, + "value" : { + "ID" : 0, + "NAME" : "jan", + "TS" : 8000, + "F1" : "blah", + "F2" : "foo" + }, + "timestamp" : 8000 + } ] +} \ No newline at end of file diff --git a/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_table_join_with_ts_extractor_both_sides_-_AVRO/6.0.0_1582734337067/topology b/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_table_join_with_ts_extractor_both_sides_-_AVRO/6.0.0_1582734337067/topology new file mode 100644 index 000000000000..9834b690229a --- /dev/null +++ b/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_table_join_with_ts_extractor_both_sides_-_AVRO/6.0.0_1582734337067/topology @@ -0,0 +1,33 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000004 (topics: [s1]) + --> KSTREAM-TRANSFORMVALUES-0000000005 + Processor: KSTREAM-TRANSFORMVALUES-0000000005 (stores: []) + --> PrependAliasLeft + <-- KSTREAM-SOURCE-0000000004 + Processor: PrependAliasLeft (stores: []) + --> Join + <-- KSTREAM-TRANSFORMVALUES-0000000005 + Processor: Join (stores: [KafkaTopic_Right-Reduce]) + --> Project + <-- PrependAliasLeft + Source: KSTREAM-SOURCE-0000000000 (topics: [t1]) + --> KTABLE-SOURCE-0000000001 + Processor: KTABLE-SOURCE-0000000001 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000002 + <-- KSTREAM-SOURCE-0000000000 + Processor: Project (stores: []) + --> ApplyTimestampTransform-S1_JOIN_T1 + <-- Join + Processor: ApplyTimestampTransform-S1_JOIN_T1 (stores: []) + --> KSTREAM-SINK-0000000009 + <-- Project + Processor: KTABLE-TRANSFORMVALUES-0000000002 (stores: []) + --> PrependAliasRight + <-- KTABLE-SOURCE-0000000001 + Sink: KSTREAM-SINK-0000000009 (topic: S1_JOIN_T1) + <-- ApplyTimestampTransform-S1_JOIN_T1 + Processor: PrependAliasRight (stores: []) + --> none + <-- KTABLE-TRANSFORMVALUES-0000000002 + diff --git a/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_table_join_with_ts_extractor_both_sides_-_JSON/6.0.0_1582734337146/plan.json b/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_table_join_with_ts_extractor_both_sides_-_JSON/6.0.0_1582734337146/plan.json new file mode 100644 index 000000000000..38e8a4ed4bc9 --- /dev/null +++ b/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_table_join_with_ts_extractor_both_sides_-_JSON/6.0.0_1582734337146/plan.json @@ -0,0 +1,239 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S1 (ROWKEY BIGINT KEY, ID BIGINT, NAME STRING, TS BIGINT) WITH (KAFKA_TOPIC='s1', KEY='ID', TIMESTAMP='TS', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S1", + "schema" : "`ROWKEY` BIGINT KEY, `ID` BIGINT, `NAME` STRING, `TS` BIGINT", + "keyField" : "ID", + "timestampColumn" : { + "column" : "TS", + "format" : null + }, + "topicName" : "s1", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "JSON", + "properties" : { } + }, + "options" : [ ] + }, + "windowInfo" : null + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE T1 (ROWKEY BIGINT KEY, ID BIGINT, F1 STRING, F2 STRING, RTS BIGINT) WITH (KAFKA_TOPIC='t1', KEY='ID', TIMESTAMP='RTS', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "T1", + "schema" : "`ROWKEY` BIGINT KEY, `ID` BIGINT, `F1` STRING, `F2` STRING, `RTS` BIGINT", + "keyField" : "ID", + "timestampColumn" : { + "column" : "RTS", + "format" : null + }, + "topicName" : "t1", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "JSON", + "properties" : { } + }, + "options" : [ ] + }, + "windowInfo" : null + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S1_JOIN_T1 WITH (TIMESTAMP='TS') AS SELECT\n S1.ID ID,\n S1.NAME NAME,\n S1.TS TS,\n T1.F1 F1,\n T1.F2 F2\nFROM S1 S1\nINNER JOIN T1 T1 ON ((S1.ID = T1.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S1_JOIN_T1", + "schema" : "`ROWKEY` BIGINT KEY, `ID` BIGINT, `NAME` STRING, `TS` BIGINT, `F1` STRING, `F2` STRING", + "keyField" : "ID", + "timestampColumn" : { + "column" : "TS", + "format" : null + }, + "topicName" : "S1_JOIN_T1", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "JSON", + "properties" : { } + }, + "options" : [ ] + }, + "windowInfo" : null + }, + "queryPlan" : { + "sources" : [ "S1", "T1" ], + "sink" : "S1_JOIN_T1", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "S1_JOIN_T1" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "JSON", + "properties" : { } + }, + "options" : [ ] + }, + "leftSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "s1", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "JSON", + "properties" : { } + }, + "options" : [ ] + }, + "timestampColumn" : { + "column" : "TS", + "format" : null + }, + "sourceSchema" : "`ROWKEY` BIGINT KEY, `ID` BIGINT, `NAME` STRING, `TS` BIGINT" + }, + "selectExpressions" : [ "ID AS S1_ID", "NAME AS S1_NAME", "TS AS S1_TS", "ROWTIME AS S1_ROWTIME", "ROWKEY AS S1_ROWKEY" ] + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "t1", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "JSON", + "properties" : { } + }, + "options" : [ ] + }, + "timestampColumn" : { + "column" : "RTS", + "format" : null + }, + "sourceSchema" : "`ROWKEY` BIGINT KEY, `ID` BIGINT, `F1` STRING, `F2` STRING, `RTS` BIGINT" + }, + "selectExpressions" : [ "ID AS T1_ID", "F1 AS T1_F1", "F2 AS T1_F2", "RTS AS T1_RTS", "ROWTIME AS T1_ROWTIME", "ROWKEY AS T1_ROWKEY" ] + } + }, + "selectExpressions" : [ "S1_ID AS ID", "S1_NAME AS NAME", "S1_TS AS TS", "T1_F1 AS F1", "T1_F2 AS F2" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "JSON", + "properties" : { } + }, + "options" : [ ] + }, + "topicName" : "S1_JOIN_T1", + "timestampColumn" : { + "column" : "TS", + "format" : null + } + }, + "queryId" : "CSAS_S1_JOIN_T1_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.new.api.enabled" : "false", + "ksql.streams.state.dir" : "/tmp/confluent8609378800848380873", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.query.pull.streamsstore.rebalancing.timeout.ms" : "10000", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647" + } +} \ No newline at end of file diff --git a/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_table_join_with_ts_extractor_both_sides_-_JSON/6.0.0_1582734337146/spec.json b/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_table_join_with_ts_extractor_both_sides_-_JSON/6.0.0_1582734337146/spec.json new file mode 100644 index 000000000000..844bdcadd54e --- /dev/null +++ b/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_table_join_with_ts_extractor_both_sides_-_JSON/6.0.0_1582734337146/spec.json @@ -0,0 +1,92 @@ +{ + "version" : "6.0.0", + "timestamp" : 1582734337146, + "schemas" : { + "CSAS_S1_JOIN_T1_0.KafkaTopic_Right.Source" : "STRUCT NOT NULL", + "CSAS_S1_JOIN_T1_0.KafkaTopic_Left.Source" : "STRUCT NOT NULL", + "CSAS_S1_JOIN_T1_0.Join.Left" : "STRUCT NOT NULL", + "CSAS_S1_JOIN_T1_0.S1_JOIN_T1" : "STRUCT NOT NULL" + }, + "inputs" : [ { + "topic" : "t1", + "key" : 0, + "value" : { + "ID" : 0, + "F1" : "blah", + "F2" : "foo", + "RTS" : 10000 + }, + "timestamp" : 10000 + }, { + "topic" : "s1", + "key" : 0, + "value" : { + "ID" : 0, + "NAME" : "zero", + "TS" : 0 + }, + "timestamp" : 10000 + }, { + "topic" : "t1", + "key" : 10, + "value" : { + "ID" : 10, + "F1" : "foo", + "F2" : "bar", + "RTS" : 13000 + }, + "timestamp" : 90000 + }, { + "topic" : "s1", + "key" : 10, + "value" : { + "ID" : 10, + "NAME" : "100", + "TS" : 11000 + }, + "timestamp" : 800000 + }, { + "topic" : "s1", + "key" : 0, + "value" : { + "ID" : 0, + "NAME" : "jan", + "TS" : 8000 + }, + "timestamp" : 0 + } ], + "outputs" : [ { + "topic" : "S1_JOIN_T1", + "key" : 0, + "value" : { + "ID" : 0, + "NAME" : "zero", + "TS" : 0, + "F1" : "blah", + "F2" : "foo" + }, + "timestamp" : 0 + }, { + "topic" : "S1_JOIN_T1", + "key" : 10, + "value" : { + "ID" : 10, + "NAME" : "100", + "TS" : 11000, + "F1" : "foo", + "F2" : "bar" + }, + "timestamp" : 11000 + }, { + "topic" : "S1_JOIN_T1", + "key" : 0, + "value" : { + "ID" : 0, + "NAME" : "jan", + "TS" : 8000, + "F1" : "blah", + "F2" : "foo" + }, + "timestamp" : 8000 + } ] +} \ No newline at end of file diff --git a/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_table_join_with_ts_extractor_both_sides_-_JSON/6.0.0_1582734337146/topology b/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_table_join_with_ts_extractor_both_sides_-_JSON/6.0.0_1582734337146/topology new file mode 100644 index 000000000000..9834b690229a --- /dev/null +++ b/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_table_join_with_ts_extractor_both_sides_-_JSON/6.0.0_1582734337146/topology @@ -0,0 +1,33 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000004 (topics: [s1]) + --> KSTREAM-TRANSFORMVALUES-0000000005 + Processor: KSTREAM-TRANSFORMVALUES-0000000005 (stores: []) + --> PrependAliasLeft + <-- KSTREAM-SOURCE-0000000004 + Processor: PrependAliasLeft (stores: []) + --> Join + <-- KSTREAM-TRANSFORMVALUES-0000000005 + Processor: Join (stores: [KafkaTopic_Right-Reduce]) + --> Project + <-- PrependAliasLeft + Source: KSTREAM-SOURCE-0000000000 (topics: [t1]) + --> KTABLE-SOURCE-0000000001 + Processor: KTABLE-SOURCE-0000000001 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000002 + <-- KSTREAM-SOURCE-0000000000 + Processor: Project (stores: []) + --> ApplyTimestampTransform-S1_JOIN_T1 + <-- Join + Processor: ApplyTimestampTransform-S1_JOIN_T1 (stores: []) + --> KSTREAM-SINK-0000000009 + <-- Project + Processor: KTABLE-TRANSFORMVALUES-0000000002 (stores: []) + --> PrependAliasRight + <-- KTABLE-SOURCE-0000000001 + Sink: KSTREAM-SINK-0000000009 (topic: S1_JOIN_T1) + <-- ApplyTimestampTransform-S1_JOIN_T1 + Processor: PrependAliasRight (stores: []) + --> none + <-- KTABLE-TRANSFORMVALUES-0000000002 + diff --git a/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_table_join_with_ts_extractor_both_sides_-_PROTOBUF/6.0.0_1582734337225/plan.json b/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_table_join_with_ts_extractor_both_sides_-_PROTOBUF/6.0.0_1582734337225/plan.json new file mode 100644 index 000000000000..2b68f8fb2005 --- /dev/null +++ b/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_table_join_with_ts_extractor_both_sides_-_PROTOBUF/6.0.0_1582734337225/plan.json @@ -0,0 +1,239 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S1 (ROWKEY BIGINT KEY, ID BIGINT, NAME STRING, TS BIGINT) WITH (KAFKA_TOPIC='s1', KEY='ID', TIMESTAMP='TS', VALUE_FORMAT='PROTOBUF');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S1", + "schema" : "`ROWKEY` BIGINT KEY, `ID` BIGINT, `NAME` STRING, `TS` BIGINT", + "keyField" : "ID", + "timestampColumn" : { + "column" : "TS", + "format" : null + }, + "topicName" : "s1", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { } + }, + "options" : [ ] + }, + "windowInfo" : null + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE T1 (ROWKEY BIGINT KEY, ID BIGINT, F1 STRING, F2 STRING, RTS BIGINT) WITH (KAFKA_TOPIC='t1', KEY='ID', TIMESTAMP='RTS', VALUE_FORMAT='PROTOBUF');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "T1", + "schema" : "`ROWKEY` BIGINT KEY, `ID` BIGINT, `F1` STRING, `F2` STRING, `RTS` BIGINT", + "keyField" : "ID", + "timestampColumn" : { + "column" : "RTS", + "format" : null + }, + "topicName" : "t1", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { } + }, + "options" : [ ] + }, + "windowInfo" : null + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S1_JOIN_T1 WITH (TIMESTAMP='TS') AS SELECT\n S1.ID ID,\n S1.NAME NAME,\n S1.TS TS,\n T1.F1 F1,\n T1.F2 F2\nFROM S1 S1\nINNER JOIN T1 T1 ON ((S1.ID = T1.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S1_JOIN_T1", + "schema" : "`ROWKEY` BIGINT KEY, `ID` BIGINT, `NAME` STRING, `TS` BIGINT, `F1` STRING, `F2` STRING", + "keyField" : "ID", + "timestampColumn" : { + "column" : "TS", + "format" : null + }, + "topicName" : "S1_JOIN_T1", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { } + }, + "options" : [ ] + }, + "windowInfo" : null + }, + "queryPlan" : { + "sources" : [ "S1", "T1" ], + "sink" : "S1_JOIN_T1", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "S1_JOIN_T1" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { } + }, + "options" : [ ] + }, + "leftSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "s1", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { } + }, + "options" : [ ] + }, + "timestampColumn" : { + "column" : "TS", + "format" : null + }, + "sourceSchema" : "`ROWKEY` BIGINT KEY, `ID` BIGINT, `NAME` STRING, `TS` BIGINT" + }, + "selectExpressions" : [ "ID AS S1_ID", "NAME AS S1_NAME", "TS AS S1_TS", "ROWTIME AS S1_ROWTIME", "ROWKEY AS S1_ROWKEY" ] + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "t1", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { } + }, + "options" : [ ] + }, + "timestampColumn" : { + "column" : "RTS", + "format" : null + }, + "sourceSchema" : "`ROWKEY` BIGINT KEY, `ID` BIGINT, `F1` STRING, `F2` STRING, `RTS` BIGINT" + }, + "selectExpressions" : [ "ID AS T1_ID", "F1 AS T1_F1", "F2 AS T1_F2", "RTS AS T1_RTS", "ROWTIME AS T1_ROWTIME", "ROWKEY AS T1_ROWKEY" ] + } + }, + "selectExpressions" : [ "S1_ID AS ID", "S1_NAME AS NAME", "S1_TS AS TS", "T1_F1 AS F1", "T1_F2 AS F2" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { } + }, + "options" : [ ] + }, + "topicName" : "S1_JOIN_T1", + "timestampColumn" : { + "column" : "TS", + "format" : null + } + }, + "queryId" : "CSAS_S1_JOIN_T1_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.new.api.enabled" : "false", + "ksql.streams.state.dir" : "/tmp/confluent8609378800848380873", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.query.pull.streamsstore.rebalancing.timeout.ms" : "10000", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647" + } +} \ No newline at end of file diff --git a/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_table_join_with_ts_extractor_both_sides_-_PROTOBUF/6.0.0_1582734337225/spec.json b/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_table_join_with_ts_extractor_both_sides_-_PROTOBUF/6.0.0_1582734337225/spec.json new file mode 100644 index 000000000000..fb9f481de5b8 --- /dev/null +++ b/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_table_join_with_ts_extractor_both_sides_-_PROTOBUF/6.0.0_1582734337225/spec.json @@ -0,0 +1,92 @@ +{ + "version" : "6.0.0", + "timestamp" : 1582734337225, + "schemas" : { + "CSAS_S1_JOIN_T1_0.KafkaTopic_Right.Source" : "STRUCT NOT NULL", + "CSAS_S1_JOIN_T1_0.KafkaTopic_Left.Source" : "STRUCT NOT NULL", + "CSAS_S1_JOIN_T1_0.Join.Left" : "STRUCT NOT NULL", + "CSAS_S1_JOIN_T1_0.S1_JOIN_T1" : "STRUCT NOT NULL" + }, + "inputs" : [ { + "topic" : "t1", + "key" : 0, + "value" : { + "ID" : 0, + "F1" : "blah", + "F2" : "foo", + "RTS" : 10000 + }, + "timestamp" : 10000 + }, { + "topic" : "s1", + "key" : 0, + "value" : { + "ID" : 0, + "NAME" : "zero", + "TS" : 0 + }, + "timestamp" : 10000 + }, { + "topic" : "t1", + "key" : 10, + "value" : { + "ID" : 10, + "F1" : "foo", + "F2" : "bar", + "RTS" : 13000 + }, + "timestamp" : 90000 + }, { + "topic" : "s1", + "key" : 10, + "value" : { + "ID" : 10, + "NAME" : "100", + "TS" : 11000 + }, + "timestamp" : 800000 + }, { + "topic" : "s1", + "key" : 0, + "value" : { + "ID" : 0, + "NAME" : "jan", + "TS" : 8000 + }, + "timestamp" : 0 + } ], + "outputs" : [ { + "topic" : "S1_JOIN_T1", + "key" : 0, + "value" : { + "ID" : 0, + "NAME" : "zero", + "TS" : 0, + "F1" : "blah", + "F2" : "foo" + }, + "timestamp" : 0 + }, { + "topic" : "S1_JOIN_T1", + "key" : 10, + "value" : { + "ID" : 10, + "NAME" : "100", + "TS" : 11000, + "F1" : "foo", + "F2" : "bar" + }, + "timestamp" : 11000 + }, { + "topic" : "S1_JOIN_T1", + "key" : 0, + "value" : { + "ID" : 0, + "NAME" : "jan", + "TS" : 8000, + "F1" : "blah", + "F2" : "foo" + }, + "timestamp" : 8000 + } ] +} \ No newline at end of file diff --git a/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_table_join_with_ts_extractor_both_sides_-_PROTOBUF/6.0.0_1582734337225/topology b/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_table_join_with_ts_extractor_both_sides_-_PROTOBUF/6.0.0_1582734337225/topology new file mode 100644 index 000000000000..9834b690229a --- /dev/null +++ b/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_table_join_with_ts_extractor_both_sides_-_PROTOBUF/6.0.0_1582734337225/topology @@ -0,0 +1,33 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000004 (topics: [s1]) + --> KSTREAM-TRANSFORMVALUES-0000000005 + Processor: KSTREAM-TRANSFORMVALUES-0000000005 (stores: []) + --> PrependAliasLeft + <-- KSTREAM-SOURCE-0000000004 + Processor: PrependAliasLeft (stores: []) + --> Join + <-- KSTREAM-TRANSFORMVALUES-0000000005 + Processor: Join (stores: [KafkaTopic_Right-Reduce]) + --> Project + <-- PrependAliasLeft + Source: KSTREAM-SOURCE-0000000000 (topics: [t1]) + --> KTABLE-SOURCE-0000000001 + Processor: KTABLE-SOURCE-0000000001 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000002 + <-- KSTREAM-SOURCE-0000000000 + Processor: Project (stores: []) + --> ApplyTimestampTransform-S1_JOIN_T1 + <-- Join + Processor: ApplyTimestampTransform-S1_JOIN_T1 (stores: []) + --> KSTREAM-SINK-0000000009 + <-- Project + Processor: KTABLE-TRANSFORMVALUES-0000000002 (stores: []) + --> PrependAliasRight + <-- KTABLE-SOURCE-0000000001 + Sink: KSTREAM-SINK-0000000009 (topic: S1_JOIN_T1) + <-- ApplyTimestampTransform-S1_JOIN_T1 + Processor: PrependAliasRight (stores: []) + --> none + <-- KTABLE-TRANSFORMVALUES-0000000002 + diff --git a/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_-_AVRO/6.0.0_1582734337274/plan.json b/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_-_AVRO/6.0.0_1582734337274/plan.json new file mode 100644 index 000000000000..569b26768230 --- /dev/null +++ b/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_-_AVRO/6.0.0_1582734337274/plan.json @@ -0,0 +1,222 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE S1 (ROWKEY BIGINT KEY, ID BIGINT, NAME STRING, TS BIGINT) WITH (KAFKA_TOPIC='s1', KEY='ID', TIMESTAMP='TS', VALUE_FORMAT='AVRO');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "S1", + "schema" : "`ROWKEY` BIGINT KEY, `ID` BIGINT, `NAME` STRING, `TS` BIGINT", + "keyField" : "ID", + "timestampColumn" : { + "column" : "TS", + "format" : null + }, + "topicName" : "s1", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "AVRO", + "properties" : { } + }, + "options" : [ ] + }, + "windowInfo" : null + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE S2 (ROWKEY BIGINT KEY, ID BIGINT, F1 STRING, F2 STRING) WITH (KAFKA_TOPIC='s2', KEY='ID', VALUE_FORMAT='AVRO');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "S2", + "schema" : "`ROWKEY` BIGINT KEY, `ID` BIGINT, `F1` STRING, `F2` STRING", + "keyField" : "ID", + "timestampColumn" : null, + "topicName" : "s2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "AVRO", + "properties" : { } + }, + "options" : [ ] + }, + "windowInfo" : null + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE S1_JOIN_S2 WITH (TIMESTAMP='TS') AS SELECT\n S1.ID ID,\n S1.NAME NAME,\n S1.TS TS,\n S2.F1 F1,\n S2.F2 F2\nFROM S1 S1\nINNER JOIN S2 S2 ON ((S1.ID = S2.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "S1_JOIN_S2", + "schema" : "`ROWKEY` BIGINT KEY, `ID` BIGINT, `NAME` STRING, `TS` BIGINT, `F1` STRING, `F2` STRING", + "keyField" : "ID", + "timestampColumn" : { + "column" : "TS", + "format" : null + }, + "topicName" : "S1_JOIN_S2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "AVRO", + "properties" : { } + }, + "options" : [ ] + }, + "windowInfo" : null + }, + "queryPlan" : { + "sources" : [ "S1", "S2" ], + "sink" : "S1_JOIN_S2", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "S1_JOIN_S2" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "tableTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "leftSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "s1", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "AVRO", + "properties" : { } + }, + "options" : [ ] + }, + "timestampColumn" : { + "column" : "TS", + "format" : null + }, + "sourceSchema" : "`ROWKEY` BIGINT KEY, `ID` BIGINT, `NAME` STRING, `TS` BIGINT" + }, + "selectExpressions" : [ "ID AS S1_ID", "NAME AS S1_NAME", "TS AS S1_TS", "ROWTIME AS S1_ROWTIME", "ROWKEY AS S1_ROWKEY" ] + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "s2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "AVRO", + "properties" : { } + }, + "options" : [ ] + }, + "timestampColumn" : null, + "sourceSchema" : "`ROWKEY` BIGINT KEY, `ID` BIGINT, `F1` STRING, `F2` STRING" + }, + "selectExpressions" : [ "ID AS S2_ID", "F1 AS S2_F1", "F2 AS S2_F2", "ROWTIME AS S2_ROWTIME", "ROWKEY AS S2_ROWKEY" ] + } + }, + "selectExpressions" : [ "S1_ID AS ID", "S1_NAME AS NAME", "S1_TS AS TS", "S2_F1 AS F1", "S2_F2 AS F2" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "AVRO", + "properties" : { } + }, + "options" : [ ] + }, + "topicName" : "S1_JOIN_S2", + "timestampColumn" : { + "column" : "TS", + "format" : null + } + }, + "queryId" : "CTAS_S1_JOIN_S2_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.new.api.enabled" : "false", + "ksql.streams.state.dir" : "/tmp/confluent8609378800848380873", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.query.pull.streamsstore.rebalancing.timeout.ms" : "10000", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647" + } +} \ No newline at end of file diff --git a/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_-_AVRO/6.0.0_1582734337274/spec.json b/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_-_AVRO/6.0.0_1582734337274/spec.json new file mode 100644 index 000000000000..c5d76337b755 --- /dev/null +++ b/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_-_AVRO/6.0.0_1582734337274/spec.json @@ -0,0 +1,89 @@ +{ + "version" : "6.0.0", + "timestamp" : 1582734337274, + "schemas" : { + "CTAS_S1_JOIN_S2_0.KafkaTopic_Left.Source" : "STRUCT NOT NULL", + "CTAS_S1_JOIN_S2_0.KafkaTopic_Right.Source" : "STRUCT NOT NULL", + "CTAS_S1_JOIN_S2_0.S1_JOIN_S2" : "STRUCT NOT NULL" + }, + "inputs" : [ { + "topic" : "s1", + "key" : 0, + "value" : { + "ID" : 0, + "NAME" : "zero", + "TS" : 0 + }, + "timestamp" : 0 + }, { + "topic" : "s2", + "key" : 0, + "value" : { + "ID" : 0, + "F1" : "blah", + "F2" : "foo" + }, + "timestamp" : 10000 + }, { + "topic" : "s2", + "key" : 10, + "value" : { + "ID" : 10, + "F1" : "foo", + "F2" : "bar" + }, + "timestamp" : 13000 + }, { + "topic" : "s1", + "key" : 10, + "value" : { + "ID" : 10, + "NAME" : "100", + "TS" : 19000 + }, + "timestamp" : 22000 + }, { + "topic" : "s1", + "key" : 0, + "value" : { + "ID" : 0, + "NAME" : "jan", + "TS" : 18000 + }, + "timestamp" : 33000 + } ], + "outputs" : [ { + "topic" : "S1_JOIN_S2", + "key" : 0, + "value" : { + "ID" : 0, + "NAME" : "zero", + "TS" : 0, + "F1" : "blah", + "F2" : "foo" + }, + "timestamp" : 0 + }, { + "topic" : "S1_JOIN_S2", + "key" : 10, + "value" : { + "ID" : 10, + "NAME" : "100", + "TS" : 19000, + "F1" : "foo", + "F2" : "bar" + }, + "timestamp" : 19000 + }, { + "topic" : "S1_JOIN_S2", + "key" : 0, + "value" : { + "ID" : 0, + "NAME" : "jan", + "TS" : 18000, + "F1" : "blah", + "F2" : "foo" + }, + "timestamp" : 18000 + } ] +} \ No newline at end of file diff --git a/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_-_AVRO/6.0.0_1582734337274/topology b/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_-_AVRO/6.0.0_1582734337274/topology new file mode 100644 index 000000000000..3f5c3b5480d2 --- /dev/null +++ b/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_-_AVRO/6.0.0_1582734337274/topology @@ -0,0 +1,45 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [s1]) + --> KTABLE-SOURCE-0000000001 + Source: KSTREAM-SOURCE-0000000004 (topics: [s2]) + --> KTABLE-SOURCE-0000000005 + Processor: KTABLE-SOURCE-0000000001 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000002 + <-- KSTREAM-SOURCE-0000000000 + Processor: KTABLE-SOURCE-0000000005 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000006 + <-- KSTREAM-SOURCE-0000000004 + Processor: KTABLE-TRANSFORMVALUES-0000000002 (stores: []) + --> PrependAliasLeft + <-- KTABLE-SOURCE-0000000001 + Processor: KTABLE-TRANSFORMVALUES-0000000006 (stores: []) + --> PrependAliasRight + <-- KTABLE-SOURCE-0000000005 + Processor: PrependAliasLeft (stores: []) + --> KTABLE-JOINTHIS-0000000009 + <-- KTABLE-TRANSFORMVALUES-0000000002 + Processor: PrependAliasRight (stores: []) + --> KTABLE-JOINOTHER-0000000010 + <-- KTABLE-TRANSFORMVALUES-0000000006 + Processor: KTABLE-JOINOTHER-0000000010 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-MERGE-0000000008 + <-- PrependAliasRight + Processor: KTABLE-JOINTHIS-0000000009 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-MERGE-0000000008 + <-- PrependAliasLeft + Processor: KTABLE-MERGE-0000000008 (stores: []) + --> Project + <-- KTABLE-JOINTHIS-0000000009, KTABLE-JOINOTHER-0000000010 + Processor: Project (stores: []) + --> KTABLE-TOSTREAM-0000000012 + <-- KTABLE-MERGE-0000000008 + Processor: KTABLE-TOSTREAM-0000000012 (stores: []) + --> ApplyTimestampTransform-S1_JOIN_S2 + <-- Project + Processor: ApplyTimestampTransform-S1_JOIN_S2 (stores: []) + --> KSTREAM-SINK-0000000013 + <-- KTABLE-TOSTREAM-0000000012 + Sink: KSTREAM-SINK-0000000013 (topic: S1_JOIN_S2) + <-- ApplyTimestampTransform-S1_JOIN_S2 + diff --git a/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_-_JSON/6.0.0_1582734337315/plan.json b/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_-_JSON/6.0.0_1582734337315/plan.json new file mode 100644 index 000000000000..8e42a0d6c27f --- /dev/null +++ b/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_-_JSON/6.0.0_1582734337315/plan.json @@ -0,0 +1,222 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE S1 (ROWKEY BIGINT KEY, ID BIGINT, NAME STRING, TS BIGINT) WITH (KAFKA_TOPIC='s1', KEY='ID', TIMESTAMP='TS', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "S1", + "schema" : "`ROWKEY` BIGINT KEY, `ID` BIGINT, `NAME` STRING, `TS` BIGINT", + "keyField" : "ID", + "timestampColumn" : { + "column" : "TS", + "format" : null + }, + "topicName" : "s1", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "JSON", + "properties" : { } + }, + "options" : [ ] + }, + "windowInfo" : null + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE S2 (ROWKEY BIGINT KEY, ID BIGINT, F1 STRING, F2 STRING) WITH (KAFKA_TOPIC='s2', KEY='ID', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "S2", + "schema" : "`ROWKEY` BIGINT KEY, `ID` BIGINT, `F1` STRING, `F2` STRING", + "keyField" : "ID", + "timestampColumn" : null, + "topicName" : "s2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "JSON", + "properties" : { } + }, + "options" : [ ] + }, + "windowInfo" : null + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE S1_JOIN_S2 WITH (TIMESTAMP='TS') AS SELECT\n S1.ID ID,\n S1.NAME NAME,\n S1.TS TS,\n S2.F1 F1,\n S2.F2 F2\nFROM S1 S1\nINNER JOIN S2 S2 ON ((S1.ID = S2.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "S1_JOIN_S2", + "schema" : "`ROWKEY` BIGINT KEY, `ID` BIGINT, `NAME` STRING, `TS` BIGINT, `F1` STRING, `F2` STRING", + "keyField" : "ID", + "timestampColumn" : { + "column" : "TS", + "format" : null + }, + "topicName" : "S1_JOIN_S2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "JSON", + "properties" : { } + }, + "options" : [ ] + }, + "windowInfo" : null + }, + "queryPlan" : { + "sources" : [ "S1", "S2" ], + "sink" : "S1_JOIN_S2", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "S1_JOIN_S2" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "tableTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "leftSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "s1", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "JSON", + "properties" : { } + }, + "options" : [ ] + }, + "timestampColumn" : { + "column" : "TS", + "format" : null + }, + "sourceSchema" : "`ROWKEY` BIGINT KEY, `ID` BIGINT, `NAME` STRING, `TS` BIGINT" + }, + "selectExpressions" : [ "ID AS S1_ID", "NAME AS S1_NAME", "TS AS S1_TS", "ROWTIME AS S1_ROWTIME", "ROWKEY AS S1_ROWKEY" ] + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "s2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "JSON", + "properties" : { } + }, + "options" : [ ] + }, + "timestampColumn" : null, + "sourceSchema" : "`ROWKEY` BIGINT KEY, `ID` BIGINT, `F1` STRING, `F2` STRING" + }, + "selectExpressions" : [ "ID AS S2_ID", "F1 AS S2_F1", "F2 AS S2_F2", "ROWTIME AS S2_ROWTIME", "ROWKEY AS S2_ROWKEY" ] + } + }, + "selectExpressions" : [ "S1_ID AS ID", "S1_NAME AS NAME", "S1_TS AS TS", "S2_F1 AS F1", "S2_F2 AS F2" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "JSON", + "properties" : { } + }, + "options" : [ ] + }, + "topicName" : "S1_JOIN_S2", + "timestampColumn" : { + "column" : "TS", + "format" : null + } + }, + "queryId" : "CTAS_S1_JOIN_S2_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.new.api.enabled" : "false", + "ksql.streams.state.dir" : "/tmp/confluent8609378800848380873", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.query.pull.streamsstore.rebalancing.timeout.ms" : "10000", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647" + } +} \ No newline at end of file diff --git a/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_-_JSON/6.0.0_1582734337315/spec.json b/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_-_JSON/6.0.0_1582734337315/spec.json new file mode 100644 index 000000000000..6889e338a501 --- /dev/null +++ b/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_-_JSON/6.0.0_1582734337315/spec.json @@ -0,0 +1,89 @@ +{ + "version" : "6.0.0", + "timestamp" : 1582734337315, + "schemas" : { + "CTAS_S1_JOIN_S2_0.KafkaTopic_Left.Source" : "STRUCT NOT NULL", + "CTAS_S1_JOIN_S2_0.KafkaTopic_Right.Source" : "STRUCT NOT NULL", + "CTAS_S1_JOIN_S2_0.S1_JOIN_S2" : "STRUCT NOT NULL" + }, + "inputs" : [ { + "topic" : "s1", + "key" : 0, + "value" : { + "ID" : 0, + "NAME" : "zero", + "TS" : 0 + }, + "timestamp" : 0 + }, { + "topic" : "s2", + "key" : 0, + "value" : { + "ID" : 0, + "F1" : "blah", + "F2" : "foo" + }, + "timestamp" : 10000 + }, { + "topic" : "s2", + "key" : 10, + "value" : { + "ID" : 10, + "F1" : "foo", + "F2" : "bar" + }, + "timestamp" : 13000 + }, { + "topic" : "s1", + "key" : 10, + "value" : { + "ID" : 10, + "NAME" : "100", + "TS" : 19000 + }, + "timestamp" : 22000 + }, { + "topic" : "s1", + "key" : 0, + "value" : { + "ID" : 0, + "NAME" : "jan", + "TS" : 18000 + }, + "timestamp" : 33000 + } ], + "outputs" : [ { + "topic" : "S1_JOIN_S2", + "key" : 0, + "value" : { + "ID" : 0, + "NAME" : "zero", + "TS" : 0, + "F1" : "blah", + "F2" : "foo" + }, + "timestamp" : 0 + }, { + "topic" : "S1_JOIN_S2", + "key" : 10, + "value" : { + "ID" : 10, + "NAME" : "100", + "TS" : 19000, + "F1" : "foo", + "F2" : "bar" + }, + "timestamp" : 19000 + }, { + "topic" : "S1_JOIN_S2", + "key" : 0, + "value" : { + "ID" : 0, + "NAME" : "jan", + "TS" : 18000, + "F1" : "blah", + "F2" : "foo" + }, + "timestamp" : 18000 + } ] +} \ No newline at end of file diff --git a/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_-_JSON/6.0.0_1582734337315/topology b/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_-_JSON/6.0.0_1582734337315/topology new file mode 100644 index 000000000000..3f5c3b5480d2 --- /dev/null +++ b/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_-_JSON/6.0.0_1582734337315/topology @@ -0,0 +1,45 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [s1]) + --> KTABLE-SOURCE-0000000001 + Source: KSTREAM-SOURCE-0000000004 (topics: [s2]) + --> KTABLE-SOURCE-0000000005 + Processor: KTABLE-SOURCE-0000000001 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000002 + <-- KSTREAM-SOURCE-0000000000 + Processor: KTABLE-SOURCE-0000000005 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000006 + <-- KSTREAM-SOURCE-0000000004 + Processor: KTABLE-TRANSFORMVALUES-0000000002 (stores: []) + --> PrependAliasLeft + <-- KTABLE-SOURCE-0000000001 + Processor: KTABLE-TRANSFORMVALUES-0000000006 (stores: []) + --> PrependAliasRight + <-- KTABLE-SOURCE-0000000005 + Processor: PrependAliasLeft (stores: []) + --> KTABLE-JOINTHIS-0000000009 + <-- KTABLE-TRANSFORMVALUES-0000000002 + Processor: PrependAliasRight (stores: []) + --> KTABLE-JOINOTHER-0000000010 + <-- KTABLE-TRANSFORMVALUES-0000000006 + Processor: KTABLE-JOINOTHER-0000000010 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-MERGE-0000000008 + <-- PrependAliasRight + Processor: KTABLE-JOINTHIS-0000000009 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-MERGE-0000000008 + <-- PrependAliasLeft + Processor: KTABLE-MERGE-0000000008 (stores: []) + --> Project + <-- KTABLE-JOINTHIS-0000000009, KTABLE-JOINOTHER-0000000010 + Processor: Project (stores: []) + --> KTABLE-TOSTREAM-0000000012 + <-- KTABLE-MERGE-0000000008 + Processor: KTABLE-TOSTREAM-0000000012 (stores: []) + --> ApplyTimestampTransform-S1_JOIN_S2 + <-- Project + Processor: ApplyTimestampTransform-S1_JOIN_S2 (stores: []) + --> KSTREAM-SINK-0000000013 + <-- KTABLE-TOSTREAM-0000000012 + Sink: KSTREAM-SINK-0000000013 (topic: S1_JOIN_S2) + <-- ApplyTimestampTransform-S1_JOIN_S2 + diff --git a/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_-_PROTOBUF/6.0.0_1582734337353/plan.json b/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_-_PROTOBUF/6.0.0_1582734337353/plan.json new file mode 100644 index 000000000000..0638f2888e8a --- /dev/null +++ b/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_-_PROTOBUF/6.0.0_1582734337353/plan.json @@ -0,0 +1,222 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE S1 (ROWKEY BIGINT KEY, ID BIGINT, NAME STRING, TS BIGINT) WITH (KAFKA_TOPIC='s1', KEY='ID', TIMESTAMP='TS', VALUE_FORMAT='PROTOBUF');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "S1", + "schema" : "`ROWKEY` BIGINT KEY, `ID` BIGINT, `NAME` STRING, `TS` BIGINT", + "keyField" : "ID", + "timestampColumn" : { + "column" : "TS", + "format" : null + }, + "topicName" : "s1", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { } + }, + "options" : [ ] + }, + "windowInfo" : null + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE S2 (ROWKEY BIGINT KEY, ID BIGINT, F1 STRING, F2 STRING) WITH (KAFKA_TOPIC='s2', KEY='ID', VALUE_FORMAT='PROTOBUF');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "S2", + "schema" : "`ROWKEY` BIGINT KEY, `ID` BIGINT, `F1` STRING, `F2` STRING", + "keyField" : "ID", + "timestampColumn" : null, + "topicName" : "s2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { } + }, + "options" : [ ] + }, + "windowInfo" : null + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE S1_JOIN_S2 WITH (TIMESTAMP='TS') AS SELECT\n S1.ID ID,\n S1.NAME NAME,\n S1.TS TS,\n S2.F1 F1,\n S2.F2 F2\nFROM S1 S1\nINNER JOIN S2 S2 ON ((S1.ID = S2.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "S1_JOIN_S2", + "schema" : "`ROWKEY` BIGINT KEY, `ID` BIGINT, `NAME` STRING, `TS` BIGINT, `F1` STRING, `F2` STRING", + "keyField" : "ID", + "timestampColumn" : { + "column" : "TS", + "format" : null + }, + "topicName" : "S1_JOIN_S2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { } + }, + "options" : [ ] + }, + "windowInfo" : null + }, + "queryPlan" : { + "sources" : [ "S1", "S2" ], + "sink" : "S1_JOIN_S2", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "S1_JOIN_S2" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "tableTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "leftSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "s1", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { } + }, + "options" : [ ] + }, + "timestampColumn" : { + "column" : "TS", + "format" : null + }, + "sourceSchema" : "`ROWKEY` BIGINT KEY, `ID` BIGINT, `NAME` STRING, `TS` BIGINT" + }, + "selectExpressions" : [ "ID AS S1_ID", "NAME AS S1_NAME", "TS AS S1_TS", "ROWTIME AS S1_ROWTIME", "ROWKEY AS S1_ROWKEY" ] + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "s2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { } + }, + "options" : [ ] + }, + "timestampColumn" : null, + "sourceSchema" : "`ROWKEY` BIGINT KEY, `ID` BIGINT, `F1` STRING, `F2` STRING" + }, + "selectExpressions" : [ "ID AS S2_ID", "F1 AS S2_F1", "F2 AS S2_F2", "ROWTIME AS S2_ROWTIME", "ROWKEY AS S2_ROWKEY" ] + } + }, + "selectExpressions" : [ "S1_ID AS ID", "S1_NAME AS NAME", "S1_TS AS TS", "S2_F1 AS F1", "S2_F2 AS F2" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { } + }, + "options" : [ ] + }, + "topicName" : "S1_JOIN_S2", + "timestampColumn" : { + "column" : "TS", + "format" : null + } + }, + "queryId" : "CTAS_S1_JOIN_S2_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.new.api.enabled" : "false", + "ksql.streams.state.dir" : "/tmp/confluent8609378800848380873", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.query.pull.streamsstore.rebalancing.timeout.ms" : "10000", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647" + } +} \ No newline at end of file diff --git a/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_-_PROTOBUF/6.0.0_1582734337353/spec.json b/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_-_PROTOBUF/6.0.0_1582734337353/spec.json new file mode 100644 index 000000000000..285654b5e92d --- /dev/null +++ b/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_-_PROTOBUF/6.0.0_1582734337353/spec.json @@ -0,0 +1,89 @@ +{ + "version" : "6.0.0", + "timestamp" : 1582734337353, + "schemas" : { + "CTAS_S1_JOIN_S2_0.KafkaTopic_Left.Source" : "STRUCT NOT NULL", + "CTAS_S1_JOIN_S2_0.KafkaTopic_Right.Source" : "STRUCT NOT NULL", + "CTAS_S1_JOIN_S2_0.S1_JOIN_S2" : "STRUCT NOT NULL" + }, + "inputs" : [ { + "topic" : "s1", + "key" : 0, + "value" : { + "ID" : 0, + "NAME" : "zero", + "TS" : 0 + }, + "timestamp" : 0 + }, { + "topic" : "s2", + "key" : 0, + "value" : { + "ID" : 0, + "F1" : "blah", + "F2" : "foo" + }, + "timestamp" : 10000 + }, { + "topic" : "s2", + "key" : 10, + "value" : { + "ID" : 10, + "F1" : "foo", + "F2" : "bar" + }, + "timestamp" : 13000 + }, { + "topic" : "s1", + "key" : 10, + "value" : { + "ID" : 10, + "NAME" : "100", + "TS" : 19000 + }, + "timestamp" : 22000 + }, { + "topic" : "s1", + "key" : 0, + "value" : { + "ID" : 0, + "NAME" : "jan", + "TS" : 18000 + }, + "timestamp" : 33000 + } ], + "outputs" : [ { + "topic" : "S1_JOIN_S2", + "key" : 0, + "value" : { + "ID" : 0, + "NAME" : "zero", + "TS" : 0, + "F1" : "blah", + "F2" : "foo" + }, + "timestamp" : 0 + }, { + "topic" : "S1_JOIN_S2", + "key" : 10, + "value" : { + "ID" : 10, + "NAME" : "100", + "TS" : 19000, + "F1" : "foo", + "F2" : "bar" + }, + "timestamp" : 19000 + }, { + "topic" : "S1_JOIN_S2", + "key" : 0, + "value" : { + "ID" : 0, + "NAME" : "jan", + "TS" : 18000, + "F1" : "blah", + "F2" : "foo" + }, + "timestamp" : 18000 + } ] +} \ No newline at end of file diff --git a/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_-_PROTOBUF/6.0.0_1582734337353/topology b/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_-_PROTOBUF/6.0.0_1582734337353/topology new file mode 100644 index 000000000000..3f5c3b5480d2 --- /dev/null +++ b/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_-_PROTOBUF/6.0.0_1582734337353/topology @@ -0,0 +1,45 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [s1]) + --> KTABLE-SOURCE-0000000001 + Source: KSTREAM-SOURCE-0000000004 (topics: [s2]) + --> KTABLE-SOURCE-0000000005 + Processor: KTABLE-SOURCE-0000000001 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000002 + <-- KSTREAM-SOURCE-0000000000 + Processor: KTABLE-SOURCE-0000000005 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000006 + <-- KSTREAM-SOURCE-0000000004 + Processor: KTABLE-TRANSFORMVALUES-0000000002 (stores: []) + --> PrependAliasLeft + <-- KTABLE-SOURCE-0000000001 + Processor: KTABLE-TRANSFORMVALUES-0000000006 (stores: []) + --> PrependAliasRight + <-- KTABLE-SOURCE-0000000005 + Processor: PrependAliasLeft (stores: []) + --> KTABLE-JOINTHIS-0000000009 + <-- KTABLE-TRANSFORMVALUES-0000000002 + Processor: PrependAliasRight (stores: []) + --> KTABLE-JOINOTHER-0000000010 + <-- KTABLE-TRANSFORMVALUES-0000000006 + Processor: KTABLE-JOINOTHER-0000000010 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-MERGE-0000000008 + <-- PrependAliasRight + Processor: KTABLE-JOINTHIS-0000000009 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-MERGE-0000000008 + <-- PrependAliasLeft + Processor: KTABLE-MERGE-0000000008 (stores: []) + --> Project + <-- KTABLE-JOINTHIS-0000000009, KTABLE-JOINOTHER-0000000010 + Processor: Project (stores: []) + --> KTABLE-TOSTREAM-0000000012 + <-- KTABLE-MERGE-0000000008 + Processor: KTABLE-TOSTREAM-0000000012 (stores: []) + --> ApplyTimestampTransform-S1_JOIN_S2 + <-- Project + Processor: ApplyTimestampTransform-S1_JOIN_S2 (stores: []) + --> KSTREAM-SINK-0000000013 + <-- KTABLE-TOSTREAM-0000000012 + Sink: KSTREAM-SINK-0000000013 (topic: S1_JOIN_S2) + <-- ApplyTimestampTransform-S1_JOIN_S2 + diff --git a/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_extractor_both_sides_-_AVRO/6.0.0_1582734337396/plan.json b/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_extractor_both_sides_-_AVRO/6.0.0_1582734337396/plan.json new file mode 100644 index 000000000000..d702ec41c4b8 --- /dev/null +++ b/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_extractor_both_sides_-_AVRO/6.0.0_1582734337396/plan.json @@ -0,0 +1,228 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE S1 (ROWKEY BIGINT KEY, ID BIGINT, NAME STRING, TS BIGINT) WITH (KAFKA_TOPIC='s1', KEY='ID', TIMESTAMP='TS', VALUE_FORMAT='AVRO');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "S1", + "schema" : "`ROWKEY` BIGINT KEY, `ID` BIGINT, `NAME` STRING, `TS` BIGINT", + "keyField" : "ID", + "timestampColumn" : { + "column" : "TS", + "format" : null + }, + "topicName" : "s1", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "AVRO", + "properties" : { } + }, + "options" : [ ] + }, + "windowInfo" : null + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE S2 (ROWKEY BIGINT KEY, ID BIGINT, F1 STRING, F2 STRING, RTS BIGINT) WITH (KAFKA_TOPIC='s2', KEY='ID', TIMESTAMP='RTS', VALUE_FORMAT='AVRO');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "S2", + "schema" : "`ROWKEY` BIGINT KEY, `ID` BIGINT, `F1` STRING, `F2` STRING, `RTS` BIGINT", + "keyField" : "ID", + "timestampColumn" : { + "column" : "RTS", + "format" : null + }, + "topicName" : "s2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "AVRO", + "properties" : { } + }, + "options" : [ ] + }, + "windowInfo" : null + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE S1_JOIN_S2 WITH (TIMESTAMP='TS') AS SELECT\n S1.ID ID,\n S1.NAME NAME,\n S1.TS TS,\n S2.F1 F1,\n S2.F2 F2\nFROM S1 S1\nINNER JOIN S2 S2 ON ((S1.ID = S2.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "S1_JOIN_S2", + "schema" : "`ROWKEY` BIGINT KEY, `ID` BIGINT, `NAME` STRING, `TS` BIGINT, `F1` STRING, `F2` STRING", + "keyField" : "ID", + "timestampColumn" : { + "column" : "TS", + "format" : null + }, + "topicName" : "S1_JOIN_S2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "AVRO", + "properties" : { } + }, + "options" : [ ] + }, + "windowInfo" : null + }, + "queryPlan" : { + "sources" : [ "S1", "S2" ], + "sink" : "S1_JOIN_S2", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "S1_JOIN_S2" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "tableTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "leftSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "s1", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "AVRO", + "properties" : { } + }, + "options" : [ ] + }, + "timestampColumn" : { + "column" : "TS", + "format" : null + }, + "sourceSchema" : "`ROWKEY` BIGINT KEY, `ID` BIGINT, `NAME` STRING, `TS` BIGINT" + }, + "selectExpressions" : [ "ID AS S1_ID", "NAME AS S1_NAME", "TS AS S1_TS", "ROWTIME AS S1_ROWTIME", "ROWKEY AS S1_ROWKEY" ] + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "s2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "AVRO", + "properties" : { } + }, + "options" : [ ] + }, + "timestampColumn" : { + "column" : "RTS", + "format" : null + }, + "sourceSchema" : "`ROWKEY` BIGINT KEY, `ID` BIGINT, `F1` STRING, `F2` STRING, `RTS` BIGINT" + }, + "selectExpressions" : [ "ID AS S2_ID", "F1 AS S2_F1", "F2 AS S2_F2", "RTS AS S2_RTS", "ROWTIME AS S2_ROWTIME", "ROWKEY AS S2_ROWKEY" ] + } + }, + "selectExpressions" : [ "S1_ID AS ID", "S1_NAME AS NAME", "S1_TS AS TS", "S2_F1 AS F1", "S2_F2 AS F2" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "AVRO", + "properties" : { } + }, + "options" : [ ] + }, + "topicName" : "S1_JOIN_S2", + "timestampColumn" : { + "column" : "TS", + "format" : null + } + }, + "queryId" : "CTAS_S1_JOIN_S2_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.new.api.enabled" : "false", + "ksql.streams.state.dir" : "/tmp/confluent8609378800848380873", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.query.pull.streamsstore.rebalancing.timeout.ms" : "10000", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647" + } +} \ No newline at end of file diff --git a/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_extractor_both_sides_-_AVRO/6.0.0_1582734337396/spec.json b/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_extractor_both_sides_-_AVRO/6.0.0_1582734337396/spec.json new file mode 100644 index 000000000000..50c4031e53df --- /dev/null +++ b/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_extractor_both_sides_-_AVRO/6.0.0_1582734337396/spec.json @@ -0,0 +1,91 @@ +{ + "version" : "6.0.0", + "timestamp" : 1582734337396, + "schemas" : { + "CTAS_S1_JOIN_S2_0.KafkaTopic_Left.Source" : "STRUCT NOT NULL", + "CTAS_S1_JOIN_S2_0.KafkaTopic_Right.Source" : "STRUCT NOT NULL", + "CTAS_S1_JOIN_S2_0.S1_JOIN_S2" : "STRUCT NOT NULL" + }, + "inputs" : [ { + "topic" : "s1", + "key" : 0, + "value" : { + "ID" : 0, + "NAME" : "zero", + "TS" : 0 + }, + "timestamp" : 0 + }, { + "topic" : "s2", + "key" : 0, + "value" : { + "ID" : 0, + "F1" : "blah", + "F2" : "foo", + "RTS" : 10000 + }, + "timestamp" : 0 + }, { + "topic" : "s2", + "key" : 10, + "value" : { + "ID" : 10, + "F1" : "foo", + "F2" : "bar", + "RTS" : 13000 + }, + "timestamp" : 0 + }, { + "topic" : "s1", + "key" : 10, + "value" : { + "ID" : 10, + "NAME" : "100", + "TS" : 11000 + }, + "timestamp" : 0 + }, { + "topic" : "s1", + "key" : 0, + "value" : { + "ID" : 0, + "NAME" : "jan", + "TS" : 8000 + }, + "timestamp" : 0 + } ], + "outputs" : [ { + "topic" : "S1_JOIN_S2", + "key" : 0, + "value" : { + "ID" : 0, + "NAME" : "zero", + "TS" : 0, + "F1" : "blah", + "F2" : "foo" + }, + "timestamp" : 0 + }, { + "topic" : "S1_JOIN_S2", + "key" : 10, + "value" : { + "ID" : 10, + "NAME" : "100", + "TS" : 11000, + "F1" : "foo", + "F2" : "bar" + }, + "timestamp" : 11000 + }, { + "topic" : "S1_JOIN_S2", + "key" : 0, + "value" : { + "ID" : 0, + "NAME" : "jan", + "TS" : 8000, + "F1" : "blah", + "F2" : "foo" + }, + "timestamp" : 8000 + } ] +} \ No newline at end of file diff --git a/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_extractor_both_sides_-_AVRO/6.0.0_1582734337396/topology b/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_extractor_both_sides_-_AVRO/6.0.0_1582734337396/topology new file mode 100644 index 000000000000..3f5c3b5480d2 --- /dev/null +++ b/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_extractor_both_sides_-_AVRO/6.0.0_1582734337396/topology @@ -0,0 +1,45 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [s1]) + --> KTABLE-SOURCE-0000000001 + Source: KSTREAM-SOURCE-0000000004 (topics: [s2]) + --> KTABLE-SOURCE-0000000005 + Processor: KTABLE-SOURCE-0000000001 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000002 + <-- KSTREAM-SOURCE-0000000000 + Processor: KTABLE-SOURCE-0000000005 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000006 + <-- KSTREAM-SOURCE-0000000004 + Processor: KTABLE-TRANSFORMVALUES-0000000002 (stores: []) + --> PrependAliasLeft + <-- KTABLE-SOURCE-0000000001 + Processor: KTABLE-TRANSFORMVALUES-0000000006 (stores: []) + --> PrependAliasRight + <-- KTABLE-SOURCE-0000000005 + Processor: PrependAliasLeft (stores: []) + --> KTABLE-JOINTHIS-0000000009 + <-- KTABLE-TRANSFORMVALUES-0000000002 + Processor: PrependAliasRight (stores: []) + --> KTABLE-JOINOTHER-0000000010 + <-- KTABLE-TRANSFORMVALUES-0000000006 + Processor: KTABLE-JOINOTHER-0000000010 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-MERGE-0000000008 + <-- PrependAliasRight + Processor: KTABLE-JOINTHIS-0000000009 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-MERGE-0000000008 + <-- PrependAliasLeft + Processor: KTABLE-MERGE-0000000008 (stores: []) + --> Project + <-- KTABLE-JOINTHIS-0000000009, KTABLE-JOINOTHER-0000000010 + Processor: Project (stores: []) + --> KTABLE-TOSTREAM-0000000012 + <-- KTABLE-MERGE-0000000008 + Processor: KTABLE-TOSTREAM-0000000012 (stores: []) + --> ApplyTimestampTransform-S1_JOIN_S2 + <-- Project + Processor: ApplyTimestampTransform-S1_JOIN_S2 (stores: []) + --> KSTREAM-SINK-0000000013 + <-- KTABLE-TOSTREAM-0000000012 + Sink: KSTREAM-SINK-0000000013 (topic: S1_JOIN_S2) + <-- ApplyTimestampTransform-S1_JOIN_S2 + diff --git a/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_extractor_both_sides_-_JSON/6.0.0_1582734337431/plan.json b/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_extractor_both_sides_-_JSON/6.0.0_1582734337431/plan.json new file mode 100644 index 000000000000..d12b44c04977 --- /dev/null +++ b/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_extractor_both_sides_-_JSON/6.0.0_1582734337431/plan.json @@ -0,0 +1,228 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE S1 (ROWKEY BIGINT KEY, ID BIGINT, NAME STRING, TS BIGINT) WITH (KAFKA_TOPIC='s1', KEY='ID', TIMESTAMP='TS', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "S1", + "schema" : "`ROWKEY` BIGINT KEY, `ID` BIGINT, `NAME` STRING, `TS` BIGINT", + "keyField" : "ID", + "timestampColumn" : { + "column" : "TS", + "format" : null + }, + "topicName" : "s1", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "JSON", + "properties" : { } + }, + "options" : [ ] + }, + "windowInfo" : null + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE S2 (ROWKEY BIGINT KEY, ID BIGINT, F1 STRING, F2 STRING, RTS BIGINT) WITH (KAFKA_TOPIC='s2', KEY='ID', TIMESTAMP='RTS', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "S2", + "schema" : "`ROWKEY` BIGINT KEY, `ID` BIGINT, `F1` STRING, `F2` STRING, `RTS` BIGINT", + "keyField" : "ID", + "timestampColumn" : { + "column" : "RTS", + "format" : null + }, + "topicName" : "s2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "JSON", + "properties" : { } + }, + "options" : [ ] + }, + "windowInfo" : null + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE S1_JOIN_S2 WITH (TIMESTAMP='TS') AS SELECT\n S1.ID ID,\n S1.NAME NAME,\n S1.TS TS,\n S2.F1 F1,\n S2.F2 F2\nFROM S1 S1\nINNER JOIN S2 S2 ON ((S1.ID = S2.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "S1_JOIN_S2", + "schema" : "`ROWKEY` BIGINT KEY, `ID` BIGINT, `NAME` STRING, `TS` BIGINT, `F1` STRING, `F2` STRING", + "keyField" : "ID", + "timestampColumn" : { + "column" : "TS", + "format" : null + }, + "topicName" : "S1_JOIN_S2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "JSON", + "properties" : { } + }, + "options" : [ ] + }, + "windowInfo" : null + }, + "queryPlan" : { + "sources" : [ "S1", "S2" ], + "sink" : "S1_JOIN_S2", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "S1_JOIN_S2" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "tableTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "leftSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "s1", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "JSON", + "properties" : { } + }, + "options" : [ ] + }, + "timestampColumn" : { + "column" : "TS", + "format" : null + }, + "sourceSchema" : "`ROWKEY` BIGINT KEY, `ID` BIGINT, `NAME` STRING, `TS` BIGINT" + }, + "selectExpressions" : [ "ID AS S1_ID", "NAME AS S1_NAME", "TS AS S1_TS", "ROWTIME AS S1_ROWTIME", "ROWKEY AS S1_ROWKEY" ] + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "s2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "JSON", + "properties" : { } + }, + "options" : [ ] + }, + "timestampColumn" : { + "column" : "RTS", + "format" : null + }, + "sourceSchema" : "`ROWKEY` BIGINT KEY, `ID` BIGINT, `F1` STRING, `F2` STRING, `RTS` BIGINT" + }, + "selectExpressions" : [ "ID AS S2_ID", "F1 AS S2_F1", "F2 AS S2_F2", "RTS AS S2_RTS", "ROWTIME AS S2_ROWTIME", "ROWKEY AS S2_ROWKEY" ] + } + }, + "selectExpressions" : [ "S1_ID AS ID", "S1_NAME AS NAME", "S1_TS AS TS", "S2_F1 AS F1", "S2_F2 AS F2" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "JSON", + "properties" : { } + }, + "options" : [ ] + }, + "topicName" : "S1_JOIN_S2", + "timestampColumn" : { + "column" : "TS", + "format" : null + } + }, + "queryId" : "CTAS_S1_JOIN_S2_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.new.api.enabled" : "false", + "ksql.streams.state.dir" : "/tmp/confluent8609378800848380873", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.query.pull.streamsstore.rebalancing.timeout.ms" : "10000", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647" + } +} \ No newline at end of file diff --git a/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_extractor_both_sides_-_JSON/6.0.0_1582734337431/spec.json b/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_extractor_both_sides_-_JSON/6.0.0_1582734337431/spec.json new file mode 100644 index 000000000000..691024818fb2 --- /dev/null +++ b/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_extractor_both_sides_-_JSON/6.0.0_1582734337431/spec.json @@ -0,0 +1,91 @@ +{ + "version" : "6.0.0", + "timestamp" : 1582734337431, + "schemas" : { + "CTAS_S1_JOIN_S2_0.KafkaTopic_Left.Source" : "STRUCT NOT NULL", + "CTAS_S1_JOIN_S2_0.KafkaTopic_Right.Source" : "STRUCT NOT NULL", + "CTAS_S1_JOIN_S2_0.S1_JOIN_S2" : "STRUCT NOT NULL" + }, + "inputs" : [ { + "topic" : "s1", + "key" : 0, + "value" : { + "ID" : 0, + "NAME" : "zero", + "TS" : 0 + }, + "timestamp" : 0 + }, { + "topic" : "s2", + "key" : 0, + "value" : { + "ID" : 0, + "F1" : "blah", + "F2" : "foo", + "RTS" : 10000 + }, + "timestamp" : 0 + }, { + "topic" : "s2", + "key" : 10, + "value" : { + "ID" : 10, + "F1" : "foo", + "F2" : "bar", + "RTS" : 13000 + }, + "timestamp" : 0 + }, { + "topic" : "s1", + "key" : 10, + "value" : { + "ID" : 10, + "NAME" : "100", + "TS" : 11000 + }, + "timestamp" : 0 + }, { + "topic" : "s1", + "key" : 0, + "value" : { + "ID" : 0, + "NAME" : "jan", + "TS" : 8000 + }, + "timestamp" : 0 + } ], + "outputs" : [ { + "topic" : "S1_JOIN_S2", + "key" : 0, + "value" : { + "ID" : 0, + "NAME" : "zero", + "TS" : 0, + "F1" : "blah", + "F2" : "foo" + }, + "timestamp" : 0 + }, { + "topic" : "S1_JOIN_S2", + "key" : 10, + "value" : { + "ID" : 10, + "NAME" : "100", + "TS" : 11000, + "F1" : "foo", + "F2" : "bar" + }, + "timestamp" : 11000 + }, { + "topic" : "S1_JOIN_S2", + "key" : 0, + "value" : { + "ID" : 0, + "NAME" : "jan", + "TS" : 8000, + "F1" : "blah", + "F2" : "foo" + }, + "timestamp" : 8000 + } ] +} \ No newline at end of file diff --git a/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_extractor_both_sides_-_JSON/6.0.0_1582734337431/topology b/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_extractor_both_sides_-_JSON/6.0.0_1582734337431/topology new file mode 100644 index 000000000000..3f5c3b5480d2 --- /dev/null +++ b/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_extractor_both_sides_-_JSON/6.0.0_1582734337431/topology @@ -0,0 +1,45 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [s1]) + --> KTABLE-SOURCE-0000000001 + Source: KSTREAM-SOURCE-0000000004 (topics: [s2]) + --> KTABLE-SOURCE-0000000005 + Processor: KTABLE-SOURCE-0000000001 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000002 + <-- KSTREAM-SOURCE-0000000000 + Processor: KTABLE-SOURCE-0000000005 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000006 + <-- KSTREAM-SOURCE-0000000004 + Processor: KTABLE-TRANSFORMVALUES-0000000002 (stores: []) + --> PrependAliasLeft + <-- KTABLE-SOURCE-0000000001 + Processor: KTABLE-TRANSFORMVALUES-0000000006 (stores: []) + --> PrependAliasRight + <-- KTABLE-SOURCE-0000000005 + Processor: PrependAliasLeft (stores: []) + --> KTABLE-JOINTHIS-0000000009 + <-- KTABLE-TRANSFORMVALUES-0000000002 + Processor: PrependAliasRight (stores: []) + --> KTABLE-JOINOTHER-0000000010 + <-- KTABLE-TRANSFORMVALUES-0000000006 + Processor: KTABLE-JOINOTHER-0000000010 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-MERGE-0000000008 + <-- PrependAliasRight + Processor: KTABLE-JOINTHIS-0000000009 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-MERGE-0000000008 + <-- PrependAliasLeft + Processor: KTABLE-MERGE-0000000008 (stores: []) + --> Project + <-- KTABLE-JOINTHIS-0000000009, KTABLE-JOINOTHER-0000000010 + Processor: Project (stores: []) + --> KTABLE-TOSTREAM-0000000012 + <-- KTABLE-MERGE-0000000008 + Processor: KTABLE-TOSTREAM-0000000012 (stores: []) + --> ApplyTimestampTransform-S1_JOIN_S2 + <-- Project + Processor: ApplyTimestampTransform-S1_JOIN_S2 (stores: []) + --> KSTREAM-SINK-0000000013 + <-- KTABLE-TOSTREAM-0000000012 + Sink: KSTREAM-SINK-0000000013 (topic: S1_JOIN_S2) + <-- ApplyTimestampTransform-S1_JOIN_S2 + diff --git a/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_extractor_both_sides_-_PROTOBUF/6.0.0_1582734337502/plan.json b/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_extractor_both_sides_-_PROTOBUF/6.0.0_1582734337502/plan.json new file mode 100644 index 000000000000..f09c08ac028d --- /dev/null +++ b/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_extractor_both_sides_-_PROTOBUF/6.0.0_1582734337502/plan.json @@ -0,0 +1,228 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE S1 (ROWKEY BIGINT KEY, ID BIGINT, NAME STRING, TS BIGINT) WITH (KAFKA_TOPIC='s1', KEY='ID', TIMESTAMP='TS', VALUE_FORMAT='PROTOBUF');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "S1", + "schema" : "`ROWKEY` BIGINT KEY, `ID` BIGINT, `NAME` STRING, `TS` BIGINT", + "keyField" : "ID", + "timestampColumn" : { + "column" : "TS", + "format" : null + }, + "topicName" : "s1", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { } + }, + "options" : [ ] + }, + "windowInfo" : null + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE S2 (ROWKEY BIGINT KEY, ID BIGINT, F1 STRING, F2 STRING, RTS BIGINT) WITH (KAFKA_TOPIC='s2', KEY='ID', TIMESTAMP='RTS', VALUE_FORMAT='PROTOBUF');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "S2", + "schema" : "`ROWKEY` BIGINT KEY, `ID` BIGINT, `F1` STRING, `F2` STRING, `RTS` BIGINT", + "keyField" : "ID", + "timestampColumn" : { + "column" : "RTS", + "format" : null + }, + "topicName" : "s2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { } + }, + "options" : [ ] + }, + "windowInfo" : null + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE S1_JOIN_S2 WITH (TIMESTAMP='TS') AS SELECT\n S1.ID ID,\n S1.NAME NAME,\n S1.TS TS,\n S2.F1 F1,\n S2.F2 F2\nFROM S1 S1\nINNER JOIN S2 S2 ON ((S1.ID = S2.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "S1_JOIN_S2", + "schema" : "`ROWKEY` BIGINT KEY, `ID` BIGINT, `NAME` STRING, `TS` BIGINT, `F1` STRING, `F2` STRING", + "keyField" : "ID", + "timestampColumn" : { + "column" : "TS", + "format" : null + }, + "topicName" : "S1_JOIN_S2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { } + }, + "options" : [ ] + }, + "windowInfo" : null + }, + "queryPlan" : { + "sources" : [ "S1", "S2" ], + "sink" : "S1_JOIN_S2", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "S1_JOIN_S2" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "tableTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "leftSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "s1", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { } + }, + "options" : [ ] + }, + "timestampColumn" : { + "column" : "TS", + "format" : null + }, + "sourceSchema" : "`ROWKEY` BIGINT KEY, `ID` BIGINT, `NAME` STRING, `TS` BIGINT" + }, + "selectExpressions" : [ "ID AS S1_ID", "NAME AS S1_NAME", "TS AS S1_TS", "ROWTIME AS S1_ROWTIME", "ROWKEY AS S1_ROWKEY" ] + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "s2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { } + }, + "options" : [ ] + }, + "timestampColumn" : { + "column" : "RTS", + "format" : null + }, + "sourceSchema" : "`ROWKEY` BIGINT KEY, `ID` BIGINT, `F1` STRING, `F2` STRING, `RTS` BIGINT" + }, + "selectExpressions" : [ "ID AS S2_ID", "F1 AS S2_F1", "F2 AS S2_F2", "RTS AS S2_RTS", "ROWTIME AS S2_ROWTIME", "ROWKEY AS S2_ROWKEY" ] + } + }, + "selectExpressions" : [ "S1_ID AS ID", "S1_NAME AS NAME", "S1_TS AS TS", "S2_F1 AS F1", "S2_F2 AS F2" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { } + }, + "options" : [ ] + }, + "topicName" : "S1_JOIN_S2", + "timestampColumn" : { + "column" : "TS", + "format" : null + } + }, + "queryId" : "CTAS_S1_JOIN_S2_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.new.api.enabled" : "false", + "ksql.streams.state.dir" : "/tmp/confluent8609378800848380873", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.query.pull.streamsstore.rebalancing.timeout.ms" : "10000", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647" + } +} \ No newline at end of file diff --git a/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_extractor_both_sides_-_PROTOBUF/6.0.0_1582734337502/spec.json b/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_extractor_both_sides_-_PROTOBUF/6.0.0_1582734337502/spec.json new file mode 100644 index 000000000000..a5053dbe1ebb --- /dev/null +++ b/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_extractor_both_sides_-_PROTOBUF/6.0.0_1582734337502/spec.json @@ -0,0 +1,91 @@ +{ + "version" : "6.0.0", + "timestamp" : 1582734337502, + "schemas" : { + "CTAS_S1_JOIN_S2_0.KafkaTopic_Left.Source" : "STRUCT NOT NULL", + "CTAS_S1_JOIN_S2_0.KafkaTopic_Right.Source" : "STRUCT NOT NULL", + "CTAS_S1_JOIN_S2_0.S1_JOIN_S2" : "STRUCT NOT NULL" + }, + "inputs" : [ { + "topic" : "s1", + "key" : 0, + "value" : { + "ID" : 0, + "NAME" : "zero", + "TS" : 0 + }, + "timestamp" : 0 + }, { + "topic" : "s2", + "key" : 0, + "value" : { + "ID" : 0, + "F1" : "blah", + "F2" : "foo", + "RTS" : 10000 + }, + "timestamp" : 0 + }, { + "topic" : "s2", + "key" : 10, + "value" : { + "ID" : 10, + "F1" : "foo", + "F2" : "bar", + "RTS" : 13000 + }, + "timestamp" : 0 + }, { + "topic" : "s1", + "key" : 10, + "value" : { + "ID" : 10, + "NAME" : "100", + "TS" : 11000 + }, + "timestamp" : 0 + }, { + "topic" : "s1", + "key" : 0, + "value" : { + "ID" : 0, + "NAME" : "jan", + "TS" : 8000 + }, + "timestamp" : 0 + } ], + "outputs" : [ { + "topic" : "S1_JOIN_S2", + "key" : 0, + "value" : { + "ID" : 0, + "NAME" : "zero", + "TS" : 0, + "F1" : "blah", + "F2" : "foo" + }, + "timestamp" : 0 + }, { + "topic" : "S1_JOIN_S2", + "key" : 10, + "value" : { + "ID" : 10, + "NAME" : "100", + "TS" : 11000, + "F1" : "foo", + "F2" : "bar" + }, + "timestamp" : 11000 + }, { + "topic" : "S1_JOIN_S2", + "key" : 0, + "value" : { + "ID" : 0, + "NAME" : "jan", + "TS" : 8000, + "F1" : "blah", + "F2" : "foo" + }, + "timestamp" : 8000 + } ] +} \ No newline at end of file diff --git a/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_extractor_both_sides_-_PROTOBUF/6.0.0_1582734337502/topology b/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_extractor_both_sides_-_PROTOBUF/6.0.0_1582734337502/topology new file mode 100644 index 000000000000..3f5c3b5480d2 --- /dev/null +++ b/ksql-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_extractor_both_sides_-_PROTOBUF/6.0.0_1582734337502/topology @@ -0,0 +1,45 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [s1]) + --> KTABLE-SOURCE-0000000001 + Source: KSTREAM-SOURCE-0000000004 (topics: [s2]) + --> KTABLE-SOURCE-0000000005 + Processor: KTABLE-SOURCE-0000000001 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000002 + <-- KSTREAM-SOURCE-0000000000 + Processor: KTABLE-SOURCE-0000000005 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000006 + <-- KSTREAM-SOURCE-0000000004 + Processor: KTABLE-TRANSFORMVALUES-0000000002 (stores: []) + --> PrependAliasLeft + <-- KTABLE-SOURCE-0000000001 + Processor: KTABLE-TRANSFORMVALUES-0000000006 (stores: []) + --> PrependAliasRight + <-- KTABLE-SOURCE-0000000005 + Processor: PrependAliasLeft (stores: []) + --> KTABLE-JOINTHIS-0000000009 + <-- KTABLE-TRANSFORMVALUES-0000000002 + Processor: PrependAliasRight (stores: []) + --> KTABLE-JOINOTHER-0000000010 + <-- KTABLE-TRANSFORMVALUES-0000000006 + Processor: KTABLE-JOINOTHER-0000000010 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-MERGE-0000000008 + <-- PrependAliasRight + Processor: KTABLE-JOINTHIS-0000000009 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-MERGE-0000000008 + <-- PrependAliasLeft + Processor: KTABLE-MERGE-0000000008 (stores: []) + --> Project + <-- KTABLE-JOINTHIS-0000000009, KTABLE-JOINOTHER-0000000010 + Processor: Project (stores: []) + --> KTABLE-TOSTREAM-0000000012 + <-- KTABLE-MERGE-0000000008 + Processor: KTABLE-TOSTREAM-0000000012 (stores: []) + --> ApplyTimestampTransform-S1_JOIN_S2 + <-- Project + Processor: ApplyTimestampTransform-S1_JOIN_S2 (stores: []) + --> KSTREAM-SINK-0000000013 + <-- KTABLE-TOSTREAM-0000000012 + Sink: KSTREAM-SINK-0000000013 (topic: S1_JOIN_S2) + <-- ApplyTimestampTransform-S1_JOIN_S2 + diff --git a/ksql-functional-tests/src/test/resources/historical_plans/timestampformat_-_Invalid_timestamp_value_should_throw_an_exception/6.0.0_1582734342043/plan.json b/ksql-functional-tests/src/test/resources/historical_plans/timestampformat_-_Invalid_timestamp_value_should_throw_an_exception/6.0.0_1582734342043/plan.json new file mode 100644 index 000000000000..031024465e29 --- /dev/null +++ b/ksql-functional-tests/src/test/resources/historical_plans/timestampformat_-_Invalid_timestamp_value_should_throw_an_exception/6.0.0_1582734342043/plan.json @@ -0,0 +1,151 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (ID BIGINT, EVENT_TS STRING) WITH (KAFKA_TOPIC='test_topic', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`ROWKEY` STRING KEY, `ID` BIGINT, `EVENT_TS` STRING", + "keyField" : null, + "timestampColumn" : null, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "JSON", + "properties" : { } + }, + "options" : [ ] + }, + "windowInfo" : null + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TS WITH (TIMESTAMP='event_ts', TIMESTAMP_FORMAT='yyyy-MM-dd''T''HH:mm:ssX') AS SELECT\n TEST.ID ID,\n TEST.EVENT_TS EVENT_TS\nFROM TEST TEST\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TS", + "schema" : "`ROWKEY` STRING KEY, `ID` BIGINT, `EVENT_TS` STRING", + "keyField" : null, + "timestampColumn" : { + "column" : "EVENT_TS", + "format" : "yyyy-MM-dd'T'HH:mm:ssX" + }, + "topicName" : "TS", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "JSON", + "properties" : { } + }, + "options" : [ ] + }, + "windowInfo" : null + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "TS", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "TS" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "JSON", + "properties" : { } + }, + "options" : [ ] + }, + "timestampColumn" : null, + "sourceSchema" : "`ROWKEY` STRING KEY, `ID` BIGINT, `EVENT_TS` STRING" + }, + "selectExpressions" : [ "ID AS ID", "EVENT_TS AS EVENT_TS" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "JSON", + "properties" : { } + }, + "options" : [ ] + }, + "topicName" : "TS", + "timestampColumn" : { + "column" : "EVENT_TS", + "format" : "yyyy-MM-dd'T'HH:mm:ssX" + } + }, + "queryId" : "CSAS_TS_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.new.api.enabled" : "false", + "ksql.streams.state.dir" : "/tmp/confluent8609378800848380873", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.query.pull.streamsstore.rebalancing.timeout.ms" : "10000", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647" + } +} \ No newline at end of file diff --git a/ksql-functional-tests/src/test/resources/historical_plans/timestampformat_-_Invalid_timestamp_value_should_throw_an_exception/6.0.0_1582734342043/spec.json b/ksql-functional-tests/src/test/resources/historical_plans/timestampformat_-_Invalid_timestamp_value_should_throw_an_exception/6.0.0_1582734342043/spec.json new file mode 100644 index 000000000000..72e54279abda --- /dev/null +++ b/ksql-functional-tests/src/test/resources/historical_plans/timestampformat_-_Invalid_timestamp_value_should_throw_an_exception/6.0.0_1582734342043/spec.json @@ -0,0 +1,17 @@ +{ + "version" : "6.0.0", + "timestamp" : 1582734342043, + "schemas" : { + "CSAS_TS_0.KsqlTopic.Source" : "STRUCT NOT NULL", + "CSAS_TS_0.TS" : "STRUCT NOT NULL" + }, + "inputs" : [ { + "topic" : "test_topic", + "key" : "", + "value" : { + "ID" : 1, + "EVENT_TS" : "not a timestamp" + } + } ], + "outputs" : [ ] +} \ No newline at end of file diff --git a/ksql-functional-tests/src/test/resources/historical_plans/timestampformat_-_Invalid_timestamp_value_should_throw_an_exception/6.0.0_1582734342043/topology b/ksql-functional-tests/src/test/resources/historical_plans/timestampformat_-_Invalid_timestamp_value_should_throw_an_exception/6.0.0_1582734342043/topology new file mode 100644 index 000000000000..3b938b733ae6 --- /dev/null +++ b/ksql-functional-tests/src/test/resources/historical_plans/timestampformat_-_Invalid_timestamp_value_should_throw_an_exception/6.0.0_1582734342043/topology @@ -0,0 +1,16 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Project + <-- KSTREAM-SOURCE-0000000000 + Processor: Project (stores: []) + --> ApplyTimestampTransform-TS + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: ApplyTimestampTransform-TS (stores: []) + --> KSTREAM-SINK-0000000003 + <-- Project + Sink: KSTREAM-SINK-0000000003 (topic: TS) + <-- ApplyTimestampTransform-TS + diff --git a/ksql-functional-tests/src/test/resources/historical_plans/timestampformat_-_override_output_timestamp_for_CSAS/6.0.0_1582734341981/plan.json b/ksql-functional-tests/src/test/resources/historical_plans/timestampformat_-_override_output_timestamp_for_CSAS/6.0.0_1582734341981/plan.json new file mode 100644 index 000000000000..8e72067da1e6 --- /dev/null +++ b/ksql-functional-tests/src/test/resources/historical_plans/timestampformat_-_override_output_timestamp_for_CSAS/6.0.0_1582734341981/plan.json @@ -0,0 +1,151 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (ID BIGINT) WITH (KAFKA_TOPIC='test_topic', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`ROWKEY` STRING KEY, `ID` BIGINT", + "keyField" : null, + "timestampColumn" : null, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "JSON", + "properties" : { } + }, + "options" : [ ] + }, + "windowInfo" : null + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TS WITH (TIMESTAMP='sink_ts') AS SELECT\n TEST.ID SINK_TS,\n TEST.ID ID\nFROM TEST TEST\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TS", + "schema" : "`ROWKEY` STRING KEY, `SINK_TS` BIGINT, `ID` BIGINT", + "keyField" : null, + "timestampColumn" : { + "column" : "SINK_TS", + "format" : null + }, + "topicName" : "TS", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "JSON", + "properties" : { } + }, + "options" : [ ] + }, + "windowInfo" : null + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "TS", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "TS" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "JSON", + "properties" : { } + }, + "options" : [ ] + }, + "timestampColumn" : null, + "sourceSchema" : "`ROWKEY` STRING KEY, `ID` BIGINT" + }, + "selectExpressions" : [ "ID AS SINK_TS", "ID AS ID" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "JSON", + "properties" : { } + }, + "options" : [ ] + }, + "topicName" : "TS", + "timestampColumn" : { + "column" : "SINK_TS", + "format" : null + } + }, + "queryId" : "CSAS_TS_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.new.api.enabled" : "false", + "ksql.streams.state.dir" : "/tmp/confluent8609378800848380873", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.query.pull.streamsstore.rebalancing.timeout.ms" : "10000", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647" + } +} \ No newline at end of file diff --git a/ksql-functional-tests/src/test/resources/historical_plans/timestampformat_-_override_output_timestamp_for_CSAS/6.0.0_1582734341981/spec.json b/ksql-functional-tests/src/test/resources/historical_plans/timestampformat_-_override_output_timestamp_for_CSAS/6.0.0_1582734341981/spec.json new file mode 100644 index 000000000000..e7fcb2aa8b25 --- /dev/null +++ b/ksql-functional-tests/src/test/resources/historical_plans/timestampformat_-_override_output_timestamp_for_CSAS/6.0.0_1582734341981/spec.json @@ -0,0 +1,47 @@ +{ + "version" : "6.0.0", + "timestamp" : 1582734341981, + "schemas" : { + "CSAS_TS_0.KsqlTopic.Source" : "STRUCT NOT NULL", + "CSAS_TS_0.TS" : "STRUCT NOT NULL" + }, + "inputs" : [ { + "topic" : "test_topic", + "key" : "", + "value" : { + "ID" : 1 + }, + "timestamp" : 1526075913000 + }, { + "topic" : "test_topic", + "key" : "", + "value" : { + "ID" : -2 + }, + "timestamp" : 1526075913000 + }, { + "topic" : "test_topic", + "key" : "", + "value" : { + "ID" : 3 + }, + "timestamp" : 1589234313000 + } ], + "outputs" : [ { + "topic" : "TS", + "key" : "", + "value" : { + "SINK_TS" : 1, + "ID" : 1 + }, + "timestamp" : 1 + }, { + "topic" : "TS", + "key" : "", + "value" : { + "SINK_TS" : 3, + "ID" : 3 + }, + "timestamp" : 3 + } ] +} \ No newline at end of file diff --git a/ksql-functional-tests/src/test/resources/historical_plans/timestampformat_-_override_output_timestamp_for_CSAS/6.0.0_1582734341981/topology b/ksql-functional-tests/src/test/resources/historical_plans/timestampformat_-_override_output_timestamp_for_CSAS/6.0.0_1582734341981/topology new file mode 100644 index 000000000000..3b938b733ae6 --- /dev/null +++ b/ksql-functional-tests/src/test/resources/historical_plans/timestampformat_-_override_output_timestamp_for_CSAS/6.0.0_1582734341981/topology @@ -0,0 +1,16 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Project + <-- KSTREAM-SOURCE-0000000000 + Processor: Project (stores: []) + --> ApplyTimestampTransform-TS + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: ApplyTimestampTransform-TS (stores: []) + --> KSTREAM-SINK-0000000003 + <-- Project + Sink: KSTREAM-SINK-0000000003 (topic: TS) + <-- ApplyTimestampTransform-TS + diff --git a/ksql-functional-tests/src/test/resources/historical_plans/timestampformat_-_override_output_timestamp_for_CSAS_using_a_string_TIMESTAMP_FORMAT/6.0.0_1582734341990/plan.json b/ksql-functional-tests/src/test/resources/historical_plans/timestampformat_-_override_output_timestamp_for_CSAS_using_a_string_TIMESTAMP_FORMAT/6.0.0_1582734341990/plan.json new file mode 100644 index 000000000000..031024465e29 --- /dev/null +++ b/ksql-functional-tests/src/test/resources/historical_plans/timestampformat_-_override_output_timestamp_for_CSAS_using_a_string_TIMESTAMP_FORMAT/6.0.0_1582734341990/plan.json @@ -0,0 +1,151 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (ID BIGINT, EVENT_TS STRING) WITH (KAFKA_TOPIC='test_topic', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`ROWKEY` STRING KEY, `ID` BIGINT, `EVENT_TS` STRING", + "keyField" : null, + "timestampColumn" : null, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "JSON", + "properties" : { } + }, + "options" : [ ] + }, + "windowInfo" : null + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TS WITH (TIMESTAMP='event_ts', TIMESTAMP_FORMAT='yyyy-MM-dd''T''HH:mm:ssX') AS SELECT\n TEST.ID ID,\n TEST.EVENT_TS EVENT_TS\nFROM TEST TEST\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TS", + "schema" : "`ROWKEY` STRING KEY, `ID` BIGINT, `EVENT_TS` STRING", + "keyField" : null, + "timestampColumn" : { + "column" : "EVENT_TS", + "format" : "yyyy-MM-dd'T'HH:mm:ssX" + }, + "topicName" : "TS", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "JSON", + "properties" : { } + }, + "options" : [ ] + }, + "windowInfo" : null + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "TS", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "TS" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "JSON", + "properties" : { } + }, + "options" : [ ] + }, + "timestampColumn" : null, + "sourceSchema" : "`ROWKEY` STRING KEY, `ID` BIGINT, `EVENT_TS` STRING" + }, + "selectExpressions" : [ "ID AS ID", "EVENT_TS AS EVENT_TS" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "JSON", + "properties" : { } + }, + "options" : [ ] + }, + "topicName" : "TS", + "timestampColumn" : { + "column" : "EVENT_TS", + "format" : "yyyy-MM-dd'T'HH:mm:ssX" + } + }, + "queryId" : "CSAS_TS_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.new.api.enabled" : "false", + "ksql.streams.state.dir" : "/tmp/confluent8609378800848380873", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.query.pull.streamsstore.rebalancing.timeout.ms" : "10000", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647" + } +} \ No newline at end of file diff --git a/ksql-functional-tests/src/test/resources/historical_plans/timestampformat_-_override_output_timestamp_for_CSAS_using_a_string_TIMESTAMP_FORMAT/6.0.0_1582734341990/spec.json b/ksql-functional-tests/src/test/resources/historical_plans/timestampformat_-_override_output_timestamp_for_CSAS_using_a_string_TIMESTAMP_FORMAT/6.0.0_1582734341990/spec.json new file mode 100644 index 000000000000..aff39a0ebf67 --- /dev/null +++ b/ksql-functional-tests/src/test/resources/historical_plans/timestampformat_-_override_output_timestamp_for_CSAS_using_a_string_TIMESTAMP_FORMAT/6.0.0_1582734341990/spec.json @@ -0,0 +1,50 @@ +{ + "version" : "6.0.0", + "timestamp" : 1582734341990, + "schemas" : { + "CSAS_TS_0.KsqlTopic.Source" : "STRUCT NOT NULL", + "CSAS_TS_0.TS" : "STRUCT NOT NULL" + }, + "inputs" : [ { + "topic" : "test_topic", + "key" : "", + "value" : { + "ID" : 1, + "EVENT_TS" : "2018-05-11T21:58:33Z" + }, + "timestamp" : 10 + }, { + "topic" : "test_topic", + "key" : "", + "value" : { + "ID" : 2, + "EVENT_TS" : "not a timestamp" + }, + "timestamp" : 10 + }, { + "topic" : "test_topic", + "key" : "", + "value" : { + "ID" : 3, + "EVENT_TS" : "2019-05-11T21:58:33Z" + }, + "timestamp" : 10 + } ], + "outputs" : [ { + "topic" : "TS", + "key" : "", + "value" : { + "ID" : 1, + "EVENT_TS" : "2018-05-11T21:58:33Z" + }, + "timestamp" : 1526075913000 + }, { + "topic" : "TS", + "key" : "", + "value" : { + "ID" : 3, + "EVENT_TS" : "2019-05-11T21:58:33Z" + }, + "timestamp" : 1557611913000 + } ] +} \ No newline at end of file diff --git a/ksql-functional-tests/src/test/resources/historical_plans/timestampformat_-_override_output_timestamp_for_CSAS_using_a_string_TIMESTAMP_FORMAT/6.0.0_1582734341990/topology b/ksql-functional-tests/src/test/resources/historical_plans/timestampformat_-_override_output_timestamp_for_CSAS_using_a_string_TIMESTAMP_FORMAT/6.0.0_1582734341990/topology new file mode 100644 index 000000000000..3b938b733ae6 --- /dev/null +++ b/ksql-functional-tests/src/test/resources/historical_plans/timestampformat_-_override_output_timestamp_for_CSAS_using_a_string_TIMESTAMP_FORMAT/6.0.0_1582734341990/topology @@ -0,0 +1,16 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Project + <-- KSTREAM-SOURCE-0000000000 + Processor: Project (stores: []) + --> ApplyTimestampTransform-TS + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: ApplyTimestampTransform-TS (stores: []) + --> KSTREAM-SINK-0000000003 + <-- Project + Sink: KSTREAM-SINK-0000000003 (topic: TS) + <-- ApplyTimestampTransform-TS + diff --git a/ksql-functional-tests/src/test/resources/historical_plans/timestampformat_-_override_output_timestamp_for_CTAS/6.0.0_1582734341998/plan.json b/ksql-functional-tests/src/test/resources/historical_plans/timestampformat_-_override_output_timestamp_for_CTAS/6.0.0_1582734341998/plan.json new file mode 100644 index 000000000000..bc17319d0937 --- /dev/null +++ b/ksql-functional-tests/src/test/resources/historical_plans/timestampformat_-_override_output_timestamp_for_CTAS/6.0.0_1582734341998/plan.json @@ -0,0 +1,151 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (ID BIGINT) WITH (KAFKA_TOPIC='test_topic', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`ROWKEY` STRING KEY, `ID` BIGINT", + "keyField" : null, + "timestampColumn" : null, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "JSON", + "properties" : { } + }, + "options" : [ ] + }, + "windowInfo" : null + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TS WITH (TIMESTAMP='sink_ts') AS SELECT\n TEST.ID SINK_TS,\n TEST.ID ID\nFROM TEST TEST\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TS", + "schema" : "`ROWKEY` STRING KEY, `SINK_TS` BIGINT, `ID` BIGINT", + "keyField" : null, + "timestampColumn" : { + "column" : "SINK_TS", + "format" : null + }, + "topicName" : "TS", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "JSON", + "properties" : { } + }, + "options" : [ ] + }, + "windowInfo" : null + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "TS", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "TS" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "tableSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "JSON", + "properties" : { } + }, + "options" : [ ] + }, + "timestampColumn" : null, + "sourceSchema" : "`ROWKEY` STRING KEY, `ID` BIGINT" + }, + "selectExpressions" : [ "ID AS SINK_TS", "ID AS ID" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "JSON", + "properties" : { } + }, + "options" : [ ] + }, + "topicName" : "TS", + "timestampColumn" : { + "column" : "SINK_TS", + "format" : null + } + }, + "queryId" : "CTAS_TS_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.new.api.enabled" : "false", + "ksql.streams.state.dir" : "/tmp/confluent8609378800848380873", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.query.pull.streamsstore.rebalancing.timeout.ms" : "10000", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647" + } +} \ No newline at end of file diff --git a/ksql-functional-tests/src/test/resources/historical_plans/timestampformat_-_override_output_timestamp_for_CTAS/6.0.0_1582734341998/spec.json b/ksql-functional-tests/src/test/resources/historical_plans/timestampformat_-_override_output_timestamp_for_CTAS/6.0.0_1582734341998/spec.json new file mode 100644 index 000000000000..523f8be0a7a2 --- /dev/null +++ b/ksql-functional-tests/src/test/resources/historical_plans/timestampformat_-_override_output_timestamp_for_CTAS/6.0.0_1582734341998/spec.json @@ -0,0 +1,47 @@ +{ + "version" : "6.0.0", + "timestamp" : 1582734341998, + "schemas" : { + "CTAS_TS_0.KsqlTopic.Source" : "STRUCT NOT NULL", + "CTAS_TS_0.TS" : "STRUCT NOT NULL" + }, + "inputs" : [ { + "topic" : "test_topic", + "key" : "", + "value" : { + "ID" : 1 + }, + "timestamp" : 1526075913000 + }, { + "topic" : "test_topic", + "key" : "", + "value" : { + "ID" : -2 + }, + "timestamp" : 1526075913000 + }, { + "topic" : "test_topic", + "key" : "", + "value" : { + "ID" : 3 + }, + "timestamp" : 1589234313000 + } ], + "outputs" : [ { + "topic" : "TS", + "key" : "", + "value" : { + "SINK_TS" : 1, + "ID" : 1 + }, + "timestamp" : 1 + }, { + "topic" : "TS", + "key" : "", + "value" : { + "SINK_TS" : 3, + "ID" : 3 + }, + "timestamp" : 3 + } ] +} \ No newline at end of file diff --git a/ksql-functional-tests/src/test/resources/historical_plans/timestampformat_-_override_output_timestamp_for_CTAS/6.0.0_1582734341998/topology b/ksql-functional-tests/src/test/resources/historical_plans/timestampformat_-_override_output_timestamp_for_CTAS/6.0.0_1582734341998/topology new file mode 100644 index 000000000000..d7fce4d8d45f --- /dev/null +++ b/ksql-functional-tests/src/test/resources/historical_plans/timestampformat_-_override_output_timestamp_for_CTAS/6.0.0_1582734341998/topology @@ -0,0 +1,22 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KTABLE-SOURCE-0000000001 + Processor: KTABLE-SOURCE-0000000001 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000002 + <-- KSTREAM-SOURCE-0000000000 + Processor: KTABLE-TRANSFORMVALUES-0000000002 (stores: []) + --> Project + <-- KTABLE-SOURCE-0000000001 + Processor: Project (stores: []) + --> KTABLE-TOSTREAM-0000000004 + <-- KTABLE-TRANSFORMVALUES-0000000002 + Processor: KTABLE-TOSTREAM-0000000004 (stores: []) + --> ApplyTimestampTransform-TS + <-- Project + Processor: ApplyTimestampTransform-TS (stores: []) + --> KSTREAM-SINK-0000000005 + <-- KTABLE-TOSTREAM-0000000004 + Sink: KSTREAM-SINK-0000000005 (topic: TS) + <-- ApplyTimestampTransform-TS + diff --git a/ksql-functional-tests/src/test/resources/historical_plans/timestampformat_-_override_output_timestamp_for_CTAS_using_a_string_TIMESTAMP_FORMAT/6.0.0_1582734342008/plan.json b/ksql-functional-tests/src/test/resources/historical_plans/timestampformat_-_override_output_timestamp_for_CTAS_using_a_string_TIMESTAMP_FORMAT/6.0.0_1582734342008/plan.json new file mode 100644 index 000000000000..031024465e29 --- /dev/null +++ b/ksql-functional-tests/src/test/resources/historical_plans/timestampformat_-_override_output_timestamp_for_CTAS_using_a_string_TIMESTAMP_FORMAT/6.0.0_1582734342008/plan.json @@ -0,0 +1,151 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (ID BIGINT, EVENT_TS STRING) WITH (KAFKA_TOPIC='test_topic', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`ROWKEY` STRING KEY, `ID` BIGINT, `EVENT_TS` STRING", + "keyField" : null, + "timestampColumn" : null, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "JSON", + "properties" : { } + }, + "options" : [ ] + }, + "windowInfo" : null + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TS WITH (TIMESTAMP='event_ts', TIMESTAMP_FORMAT='yyyy-MM-dd''T''HH:mm:ssX') AS SELECT\n TEST.ID ID,\n TEST.EVENT_TS EVENT_TS\nFROM TEST TEST\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TS", + "schema" : "`ROWKEY` STRING KEY, `ID` BIGINT, `EVENT_TS` STRING", + "keyField" : null, + "timestampColumn" : { + "column" : "EVENT_TS", + "format" : "yyyy-MM-dd'T'HH:mm:ssX" + }, + "topicName" : "TS", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "JSON", + "properties" : { } + }, + "options" : [ ] + }, + "windowInfo" : null + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "TS", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "TS" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "JSON", + "properties" : { } + }, + "options" : [ ] + }, + "timestampColumn" : null, + "sourceSchema" : "`ROWKEY` STRING KEY, `ID` BIGINT, `EVENT_TS` STRING" + }, + "selectExpressions" : [ "ID AS ID", "EVENT_TS AS EVENT_TS" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "JSON", + "properties" : { } + }, + "options" : [ ] + }, + "topicName" : "TS", + "timestampColumn" : { + "column" : "EVENT_TS", + "format" : "yyyy-MM-dd'T'HH:mm:ssX" + } + }, + "queryId" : "CSAS_TS_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.new.api.enabled" : "false", + "ksql.streams.state.dir" : "/tmp/confluent8609378800848380873", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.query.pull.streamsstore.rebalancing.timeout.ms" : "10000", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647" + } +} \ No newline at end of file diff --git a/ksql-functional-tests/src/test/resources/historical_plans/timestampformat_-_override_output_timestamp_for_CTAS_using_a_string_TIMESTAMP_FORMAT/6.0.0_1582734342008/spec.json b/ksql-functional-tests/src/test/resources/historical_plans/timestampformat_-_override_output_timestamp_for_CTAS_using_a_string_TIMESTAMP_FORMAT/6.0.0_1582734342008/spec.json new file mode 100644 index 000000000000..73945ca986b8 --- /dev/null +++ b/ksql-functional-tests/src/test/resources/historical_plans/timestampformat_-_override_output_timestamp_for_CTAS_using_a_string_TIMESTAMP_FORMAT/6.0.0_1582734342008/spec.json @@ -0,0 +1,50 @@ +{ + "version" : "6.0.0", + "timestamp" : 1582734342008, + "schemas" : { + "CSAS_TS_0.KsqlTopic.Source" : "STRUCT NOT NULL", + "CSAS_TS_0.TS" : "STRUCT NOT NULL" + }, + "inputs" : [ { + "topic" : "test_topic", + "key" : "", + "value" : { + "ID" : 1, + "EVENT_TS" : "2018-05-11T21:58:33Z" + }, + "timestamp" : 10 + }, { + "topic" : "test_topic", + "key" : "", + "value" : { + "ID" : 2, + "EVENT_TS" : "not a timestamp" + }, + "timestamp" : 10 + }, { + "topic" : "test_topic", + "key" : "", + "value" : { + "ID" : 3, + "EVENT_TS" : "2019-05-11T21:58:33Z" + }, + "timestamp" : 10 + } ], + "outputs" : [ { + "topic" : "TS", + "key" : "", + "value" : { + "ID" : 1, + "EVENT_TS" : "2018-05-11T21:58:33Z" + }, + "timestamp" : 1526075913000 + }, { + "topic" : "TS", + "key" : "", + "value" : { + "ID" : 3, + "EVENT_TS" : "2019-05-11T21:58:33Z" + }, + "timestamp" : 1557611913000 + } ] +} \ No newline at end of file diff --git a/ksql-functional-tests/src/test/resources/historical_plans/timestampformat_-_override_output_timestamp_for_CTAS_using_a_string_TIMESTAMP_FORMAT/6.0.0_1582734342008/topology b/ksql-functional-tests/src/test/resources/historical_plans/timestampformat_-_override_output_timestamp_for_CTAS_using_a_string_TIMESTAMP_FORMAT/6.0.0_1582734342008/topology new file mode 100644 index 000000000000..3b938b733ae6 --- /dev/null +++ b/ksql-functional-tests/src/test/resources/historical_plans/timestampformat_-_override_output_timestamp_for_CTAS_using_a_string_TIMESTAMP_FORMAT/6.0.0_1582734342008/topology @@ -0,0 +1,16 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Project + <-- KSTREAM-SOURCE-0000000000 + Processor: Project (stores: []) + --> ApplyTimestampTransform-TS + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: ApplyTimestampTransform-TS (stores: []) + --> KSTREAM-SINK-0000000003 + <-- Project + Sink: KSTREAM-SINK-0000000003 (topic: TS) + <-- ApplyTimestampTransform-TS + diff --git a/ksql-functional-tests/src/test/resources/historical_plans/timestampformat_-_timestamp_column_of_source_should_not_influence_sink/6.0.0_1582734342024/plan.json b/ksql-functional-tests/src/test/resources/historical_plans/timestampformat_-_timestamp_column_of_source_should_not_influence_sink/6.0.0_1582734342024/plan.json new file mode 100644 index 000000000000..4e30a6c88c54 --- /dev/null +++ b/ksql-functional-tests/src/test/resources/historical_plans/timestampformat_-_timestamp_column_of_source_should_not_influence_sink/6.0.0_1582734342024/plan.json @@ -0,0 +1,151 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM INPUT (ID BIGINT, EVENT_TS BIGINT) WITH (KAFKA_TOPIC='test_topic', TIMESTAMP='EVENT_TS', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "INPUT", + "schema" : "`ROWKEY` STRING KEY, `ID` BIGINT, `EVENT_TS` BIGINT", + "keyField" : null, + "timestampColumn" : { + "column" : "EVENT_TS", + "format" : null + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "JSON", + "properties" : { } + }, + "options" : [ ] + }, + "windowInfo" : null + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT INPUT.ID EVENT_TS\nFROM INPUT INPUT\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`ROWKEY` STRING KEY, `EVENT_TS` BIGINT", + "keyField" : null, + "timestampColumn" : null, + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "JSON", + "properties" : { } + }, + "options" : [ ] + }, + "windowInfo" : null + }, + "queryPlan" : { + "sources" : [ "INPUT" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "JSON", + "properties" : { } + }, + "options" : [ ] + }, + "timestampColumn" : { + "column" : "EVENT_TS", + "format" : null + }, + "sourceSchema" : "`ROWKEY` STRING KEY, `ID` BIGINT, `EVENT_TS` BIGINT" + }, + "selectExpressions" : [ "ID AS EVENT_TS" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "JSON", + "properties" : { } + }, + "options" : [ ] + }, + "topicName" : "OUTPUT", + "timestampColumn" : null + }, + "queryId" : "CSAS_OUTPUT_0" + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.persistence.wrap.single.values" : "true", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.schema.registry.url" : "", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.output.topic.name.prefix" : "", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.connect.url" : "http://localhost:8083", + "ksql.service.id" : "some.ksql.service.id", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.new.api.enabled" : "false", + "ksql.streams.state.dir" : "/tmp/confluent8609378800848380873", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.metric.reporters" : "", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.metrics.extension" : null, + "ksql.streams.topology.optimization" : "all", + "ksql.query.pull.streamsstore.rebalancing.timeout.ms" : "10000", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.streams.num.stream.threads" : "4", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.metrics.tags.custom" : "", + "ksql.pull.queries.enable" : "true", + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.udf.collect.metrics" : "false", + "ksql.persistent.prefix" : "query_", + "ksql.query.persistent.active.limit" : "2147483647" + } +} \ No newline at end of file diff --git a/ksql-functional-tests/src/test/resources/historical_plans/timestampformat_-_timestamp_column_of_source_should_not_influence_sink/6.0.0_1582734342024/spec.json b/ksql-functional-tests/src/test/resources/historical_plans/timestampformat_-_timestamp_column_of_source_should_not_influence_sink/6.0.0_1582734342024/spec.json new file mode 100644 index 000000000000..20fef1766490 --- /dev/null +++ b/ksql-functional-tests/src/test/resources/historical_plans/timestampformat_-_timestamp_column_of_source_should_not_influence_sink/6.0.0_1582734342024/spec.json @@ -0,0 +1,38 @@ +{ + "version" : "6.0.0", + "timestamp" : 1582734342024, + "schemas" : { + "CSAS_OUTPUT_0.KsqlTopic.Source" : "STRUCT NOT NULL", + "CSAS_OUTPUT_0.OUTPUT" : "STRUCT NOT NULL" + }, + "inputs" : [ { + "topic" : "test_topic", + "key" : "", + "value" : { + "ID" : 1, + "EVENT_TS" : 1526075913000 + } + }, { + "topic" : "test_topic", + "key" : "", + "value" : { + "ID" : 2, + "EVENT_TS" : 1589234313000 + } + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : "", + "value" : { + "EVENT_TS" : 1 + }, + "timestamp" : 1526075913000 + }, { + "topic" : "OUTPUT", + "key" : "", + "value" : { + "EVENT_TS" : 2 + }, + "timestamp" : 1589234313000 + } ] +} \ No newline at end of file diff --git a/ksql-functional-tests/src/test/resources/historical_plans/timestampformat_-_timestamp_column_of_source_should_not_influence_sink/6.0.0_1582734342024/topology b/ksql-functional-tests/src/test/resources/historical_plans/timestampformat_-_timestamp_column_of_source_should_not_influence_sink/6.0.0_1582734342024/topology new file mode 100644 index 000000000000..441a8f282644 --- /dev/null +++ b/ksql-functional-tests/src/test/resources/historical_plans/timestampformat_-_timestamp_column_of_source_should_not_influence_sink/6.0.0_1582734342024/topology @@ -0,0 +1,13 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Project + <-- KSTREAM-SOURCE-0000000000 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Sink: KSTREAM-SINK-0000000003 (topic: OUTPUT) + <-- Project +