From 087807ed592f2c6b0432a690130184d456ede7dc Mon Sep 17 00:00:00 2001 From: Mahdi Dibaiee Date: Mon, 16 Dec 2024 21:45:25 +0000 Subject: [PATCH 1/2] source-oracle-flashback: float types with precision < 18 are not string --- .../acmeCo/FLOW_CAPTURE_TEST.schema.yaml | 9 +++---- .../FLOW_TEST_TEST_ALL_TYPES.schema.yaml | 21 +++++++++++----- .../acmeCo/FLOW_TEST_TEST_CHANGES.schema.yaml | 8 ++---- .../acmeCo/FLOW_TEST_TEST_EMPTY.schema.yaml | 9 +++---- .../source_oracle_flashback/models.py | 13 +++++----- .../tests/db_seeds/create_test_all_types.sql | 2 ++ .../tests/db_seeds/insert_test_all_types.sql | 6 ++++- .../snapshots__capture_all_types__stdout.json | 2 ++ .../snapshots__discover__stdout.json | 25 +++++++++++++++++++ 9 files changed, 64 insertions(+), 31 deletions(-) diff --git a/source-oracle-flashback/acmeCo/FLOW_CAPTURE_TEST.schema.yaml b/source-oracle-flashback/acmeCo/FLOW_CAPTURE_TEST.schema.yaml index a05b213b8b..5da67d3d14 100644 --- a/source-oracle-flashback/acmeCo/FLOW_CAPTURE_TEST.schema.yaml +++ b/source-oracle-flashback/acmeCo/FLOW_CAPTURE_TEST.schema.yaml @@ -11,9 +11,7 @@ $defs: title: Op type: string source: - anyOf: - - $ref: "#/$defs/Source" - - type: "null" + $ref: "#/$defs/Source" required: - op - source @@ -46,9 +44,6 @@ properties: _meta: allOf: - $ref: "#/$defs/Meta" - default: - op: u - source: ~ description: Document metadata ID: anyOf: @@ -57,5 +52,7 @@ properties: default: ~ format: number title: Id +required: + - _meta title: TEST type: object diff --git a/source-oracle-flashback/acmeCo/FLOW_TEST_TEST_ALL_TYPES.schema.yaml b/source-oracle-flashback/acmeCo/FLOW_TEST_TEST_ALL_TYPES.schema.yaml index d99481103c..68b60cc9b5 100644 --- a/source-oracle-flashback/acmeCo/FLOW_TEST_TEST_ALL_TYPES.schema.yaml +++ b/source-oracle-flashback/acmeCo/FLOW_TEST_TEST_ALL_TYPES.schema.yaml @@ -11,9 +11,7 @@ $defs: title: Op type: string source: - anyOf: - - $ref: "#/$defs/Source" - - type: "null" + $ref: "#/$defs/Source" required: - op - source @@ -46,9 +44,6 @@ properties: _meta: allOf: - $ref: "#/$defs/Meta" - default: - op: u - source: ~ description: Document metadata ID: description: Primary Key @@ -75,6 +70,19 @@ properties: default: ~ format: number title: Float 126 + FLOAT_16: + anyOf: + - type: number + - type: "null" + default: ~ + title: Float 16 + FLOAT_63: + anyOf: + - type: string + - type: "null" + default: ~ + format: number + title: Float 63 INTEG: anyOf: - type: string @@ -199,6 +207,7 @@ properties: default: ~ title: Vchar2 required: + - _meta - ID title: TEST_ALL_TYPES type: object diff --git a/source-oracle-flashback/acmeCo/FLOW_TEST_TEST_CHANGES.schema.yaml b/source-oracle-flashback/acmeCo/FLOW_TEST_TEST_CHANGES.schema.yaml index 697db644a0..ff77516f58 100644 --- a/source-oracle-flashback/acmeCo/FLOW_TEST_TEST_CHANGES.schema.yaml +++ b/source-oracle-flashback/acmeCo/FLOW_TEST_TEST_CHANGES.schema.yaml @@ -11,9 +11,7 @@ $defs: title: Op type: string source: - anyOf: - - $ref: "#/$defs/Source" - - type: "null" + $ref: "#/$defs/Source" required: - op - source @@ -46,9 +44,6 @@ properties: _meta: allOf: - $ref: "#/$defs/Meta" - default: - op: u - source: ~ description: Document metadata ID: description: Primary Key @@ -62,6 +57,7 @@ properties: default: ~ title: Str required: + - _meta - ID title: TEST_CHANGES type: object diff --git a/source-oracle-flashback/acmeCo/FLOW_TEST_TEST_EMPTY.schema.yaml b/source-oracle-flashback/acmeCo/FLOW_TEST_TEST_EMPTY.schema.yaml index e74cf32133..541edaaef5 100644 --- a/source-oracle-flashback/acmeCo/FLOW_TEST_TEST_EMPTY.schema.yaml +++ b/source-oracle-flashback/acmeCo/FLOW_TEST_TEST_EMPTY.schema.yaml @@ -11,9 +11,7 @@ $defs: title: Op type: string source: - anyOf: - - $ref: "#/$defs/Source" - - type: "null" + $ref: "#/$defs/Source" required: - op - source @@ -46,9 +44,6 @@ properties: _meta: allOf: - $ref: "#/$defs/Meta" - default: - op: u - source: ~ description: Document metadata ID: anyOf: @@ -57,5 +52,7 @@ properties: default: ~ format: number title: Id +required: + - _meta title: TEST_EMPTY type: object diff --git a/source-oracle-flashback/source_oracle_flashback/models.py b/source-oracle-flashback/source_oracle_flashback/models.py index 329fcdf822..751cbb129e 100644 --- a/source-oracle-flashback/source_oracle_flashback/models.py +++ b/source-oracle-flashback/source_oracle_flashback/models.py @@ -291,18 +291,19 @@ def build_table( field_type: type[int | str | datetime | float | Decimal] field_schema_extra: dict | None = None - if col.data_type == col.Type.NUMBER and col.data_scale == 0: - # datA_precision: 0 defaults to precision 38 + if col.data_type == col.Type.NUMBER and col.data_scale == 0 : + # data_precision: 0 defaults to precision 38 if col.data_precision > 18 or col.data_precision == 0: field_type = Decimal field_schema_extra = {"format": "number"} else: field_type = int - elif col.data_type in (col.Type.DOUBLE, col.Type.NUMBER, col.Type.FLOAT): - # Floats cannot be used as keys, so use {type: string, format: number}. - field_type = Decimal - field_schema_extra = {"format": "number"} + if col.data_precision > 18 or col.data_precision == 0: + field_type = Decimal + field_schema_extra = {"format": "number"} + else: + field_type = float elif col.data_type in (col.Type.CHAR, col.Type.VARCHAR, col.Type.VARCHAR2, col.Type.NCHAR, col.Type.NVARCHAR2): field_type = str elif col.is_datetime and col.has_timezone: diff --git a/source-oracle-flashback/tests/db_seeds/create_test_all_types.sql b/source-oracle-flashback/tests/db_seeds/create_test_all_types.sql index 15b284c794..8269ca0056 100644 --- a/source-oracle-flashback/tests/db_seeds/create_test_all_types.sql +++ b/source-oracle-flashback/tests/db_seeds/create_test_all_types.sql @@ -11,6 +11,8 @@ CREATE TABLE flow_test.test_all_types( integ INTEGER, double_precision DOUBLE PRECISION, float_126 FLOAT(126), + float_63 FLOAT(63), + float_16 FLOAT(16), real_num REAL, datetime DATE, ts TIMESTAMP, diff --git a/source-oracle-flashback/tests/db_seeds/insert_test_all_types.sql b/source-oracle-flashback/tests/db_seeds/insert_test_all_types.sql index 9a4f42b254..ee91f4b228 100644 --- a/source-oracle-flashback/tests/db_seeds/insert_test_all_types.sql +++ b/source-oracle-flashback/tests/db_seeds/insert_test_all_types.sql @@ -11,6 +11,8 @@ INSERT INTO flow_test.test_all_types( integ, double_precision, float_126, + float_63, + float_16, real_num, datetime, ts, @@ -32,7 +34,9 @@ INSERT INTO flow_test.test_all_types( 1234567891234567891, 123456789123456, 123456789, - 18446744073709551615, + 18446744073709551615, + 123456789.123456789, + 123456789.123456789, 123456789.123456789, 123456789.123456789, 123456789.123456789, diff --git a/source-oracle-flashback/tests/snapshots/snapshots__capture_all_types__stdout.json b/source-oracle-flashback/tests/snapshots/snapshots__capture_all_types__stdout.json index 72a29cc708..0bbc108809 100644 --- a/source-oracle-flashback/tests/snapshots/snapshots__capture_all_types__stdout.json +++ b/source-oracle-flashback/tests/snapshots/snapshots__capture_all_types__stdout.json @@ -5,6 +5,8 @@ "DATETIME": "2022-01-01T00:00:00", "DOUBLE_PRECISION": "123456789.123456789", "FLOAT_126": "123456789.123456789", + "FLOAT_16": 123460000.0, + "FLOAT_63": "123456789.123456789", "ID": "1", "INTEG": "18446744073709551615", "INTERVAL_DAY": "+01 02:03:04.567000", diff --git a/source-oracle-flashback/tests/snapshots/snapshots__discover__stdout.json b/source-oracle-flashback/tests/snapshots/snapshots__discover__stdout.json index c945a30d78..17777f10a3 100644 --- a/source-oracle-flashback/tests/snapshots/snapshots__discover__stdout.json +++ b/source-oracle-flashback/tests/snapshots/snapshots__discover__stdout.json @@ -119,6 +119,31 @@ "format": "number", "title": "Float 126" }, + "FLOAT_16": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Float 16" + }, + "FLOAT_63": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "format": "number", + "title": "Float 63" + }, "INTEG": { "anyOf": [ { From 6ceb859811b5ae19602ef658f6e3dccf1feb833b Mon Sep 17 00:00:00 2001 From: Mahdi Dibaiee Date: Tue, 17 Dec 2024 13:43:54 +0000 Subject: [PATCH 2/2] source-oracle-flashback: add testcase for the broken task --- .../acmeCo/FLOW_TEST_TEST_ALL_TYPES.schema.yaml | 6 ++++++ .../source_oracle_flashback/models.py | 2 +- .../tests/db_seeds/create_test_all_types.sql | 1 + .../tests/db_seeds/insert_test_all_types.sql | 2 ++ .../snapshots__capture_all_types__stdout.json | 1 + .../tests/snapshots/snapshots__discover__stdout.json | 12 ++++++++++++ 6 files changed, 23 insertions(+), 1 deletion(-) diff --git a/source-oracle-flashback/acmeCo/FLOW_TEST_TEST_ALL_TYPES.schema.yaml b/source-oracle-flashback/acmeCo/FLOW_TEST_TEST_ALL_TYPES.schema.yaml index 68b60cc9b5..b430bee23b 100644 --- a/source-oracle-flashback/acmeCo/FLOW_TEST_TEST_ALL_TYPES.schema.yaml +++ b/source-oracle-flashback/acmeCo/FLOW_TEST_TEST_ALL_TYPES.schema.yaml @@ -109,6 +109,12 @@ properties: default: ~ format: number title: Num + NUM104: + anyOf: + - type: number + - type: "null" + default: ~ + title: Num104 NUM15: anyOf: - type: integer diff --git a/source-oracle-flashback/source_oracle_flashback/models.py b/source-oracle-flashback/source_oracle_flashback/models.py index 751cbb129e..23f0984755 100644 --- a/source-oracle-flashback/source_oracle_flashback/models.py +++ b/source-oracle-flashback/source_oracle_flashback/models.py @@ -291,7 +291,7 @@ def build_table( field_type: type[int | str | datetime | float | Decimal] field_schema_extra: dict | None = None - if col.data_type == col.Type.NUMBER and col.data_scale == 0 : + if col.data_type == col.Type.NUMBER and col.data_scale == 0: # data_precision: 0 defaults to precision 38 if col.data_precision > 18 or col.data_precision == 0: field_type = Decimal diff --git a/source-oracle-flashback/tests/db_seeds/create_test_all_types.sql b/source-oracle-flashback/tests/db_seeds/create_test_all_types.sql index 8269ca0056..9610595783 100644 --- a/source-oracle-flashback/tests/db_seeds/create_test_all_types.sql +++ b/source-oracle-flashback/tests/db_seeds/create_test_all_types.sql @@ -7,6 +7,7 @@ CREATE TABLE flow_test.test_all_types( num NUMBER (38, 9), num19 NUMBER (19, 0), num15 NUMBER (15, 0), + num104 NUMBER (10, 4), small_int SMALLINT, integ INTEGER, double_precision DOUBLE PRECISION, diff --git a/source-oracle-flashback/tests/db_seeds/insert_test_all_types.sql b/source-oracle-flashback/tests/db_seeds/insert_test_all_types.sql index ee91f4b228..b0ace07756 100644 --- a/source-oracle-flashback/tests/db_seeds/insert_test_all_types.sql +++ b/source-oracle-flashback/tests/db_seeds/insert_test_all_types.sql @@ -7,6 +7,7 @@ INSERT INTO flow_test.test_all_types( num, num19, num15, + num104, small_int, integ, double_precision, @@ -33,6 +34,7 @@ INSERT INTO flow_test.test_all_types( 123456789.123456789, 1234567891234567891, 123456789123456, + 123456.3456, 123456789, 18446744073709551615, 123456789.123456789, diff --git a/source-oracle-flashback/tests/snapshots/snapshots__capture_all_types__stdout.json b/source-oracle-flashback/tests/snapshots/snapshots__capture_all_types__stdout.json index 0bbc108809..98260eb666 100644 --- a/source-oracle-flashback/tests/snapshots/snapshots__capture_all_types__stdout.json +++ b/source-oracle-flashback/tests/snapshots/snapshots__capture_all_types__stdout.json @@ -12,6 +12,7 @@ "INTERVAL_DAY": "+01 02:03:04.567000", "INTERVAL_YEAR": "+1234-05", "NUM": "123456789.123456789", + "NUM104": 123456.3456, "NUM15": 123456789123456, "NUM19": "1234567891234567891", "NVCHAR2": "nvarchar2 value with unicode characters \u2764\ufe0f \ud83d\udd25\ufe0f", diff --git a/source-oracle-flashback/tests/snapshots/snapshots__discover__stdout.json b/source-oracle-flashback/tests/snapshots/snapshots__discover__stdout.json index 17777f10a3..33667bbf3f 100644 --- a/source-oracle-flashback/tests/snapshots/snapshots__discover__stdout.json +++ b/source-oracle-flashback/tests/snapshots/snapshots__discover__stdout.json @@ -194,6 +194,18 @@ "format": "number", "title": "Num" }, + "NUM104": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Num104" + }, "NUM15": { "anyOf": [ {