Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

source-oracle-flashback: float types with precision < 18 are not string #2212

Merged
merged 2 commits into from
Dec 17, 2024
Merged

source-oracle-flashback: float types with precision < 18 are not string #2212

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
087807e
source-oracle-flashback: float types with precision < 18 are not string
mdibaiee Dec 16, 2024
6ceb859
source-oracle-flashback: add testcase for the broken task
mdibaiee Dec 17, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 3 additions & 6 deletions source-oracle-flashback/acmeCo/FLOW_CAPTURE_TEST.schema.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,7 @@ $defs:
title: Op
type: string
source:
anyOf:
- $ref: "#/$defs/Source"
- type: "null"
$ref: "#/$defs/Source"
required:
- op
- source
Expand Down Expand Up @@ -46,9 +44,6 @@ properties:
_meta:
allOf:
- $ref: "#/$defs/Meta"
default:
op: u
source: ~
description: Document metadata
ID:
anyOf:
Expand All @@ -57,5 +52,7 @@ properties:
default: ~
format: number
title: Id
required:
- _meta
title: TEST
type: object
27 changes: 21 additions & 6 deletions source-oracle-flashback/acmeCo/FLOW_TEST_TEST_ALL_TYPES.schema.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,7 @@ $defs:
title: Op
type: string
source:
anyOf:
- $ref: "#/$defs/Source"
- type: "null"
$ref: "#/$defs/Source"
required:
- op
- source
Expand Down Expand Up @@ -46,9 +44,6 @@ properties:
_meta:
allOf:
- $ref: "#/$defs/Meta"
default:
op: u
source: ~
description: Document metadata
ID:
description: Primary Key
Expand All @@ -75,6 +70,19 @@ properties:
default: ~
format: number
title: Float 126
FLOAT_16:
anyOf:
- type: number
- type: "null"
default: ~
title: Float 16
FLOAT_63:
anyOf:
- type: string
- type: "null"
default: ~
format: number
title: Float 63
INTEG:
anyOf:
- type: string
Expand All @@ -101,6 +109,12 @@ properties:
default: ~
format: number
title: Num
NUM104:
anyOf:
- type: number
- type: "null"
default: ~
title: Num104
NUM15:
anyOf:
- type: integer
Expand Down Expand Up @@ -199,6 +213,7 @@ properties:
default: ~
title: Vchar2
required:
- _meta
- ID
title: TEST_ALL_TYPES
type: object
8 changes: 2 additions & 6 deletions source-oracle-flashback/acmeCo/FLOW_TEST_TEST_CHANGES.schema.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,7 @@ $defs:
title: Op
type: string
source:
anyOf:
- $ref: "#/$defs/Source"
- type: "null"
$ref: "#/$defs/Source"
required:
- op
- source
Expand Down Expand Up @@ -46,9 +44,6 @@ properties:
_meta:
allOf:
- $ref: "#/$defs/Meta"
default:
op: u
source: ~
description: Document metadata
ID:
description: Primary Key
Expand All @@ -62,6 +57,7 @@ properties:
default: ~
title: Str
required:
- _meta
- ID
title: TEST_CHANGES
type: object
9 changes: 3 additions & 6 deletions source-oracle-flashback/acmeCo/FLOW_TEST_TEST_EMPTY.schema.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,7 @@ $defs:
title: Op
type: string
source:
anyOf:
- $ref: "#/$defs/Source"
- type: "null"
$ref: "#/$defs/Source"
required:
- op
- source
Expand Down Expand Up @@ -46,9 +44,6 @@ properties:
_meta:
allOf:
- $ref: "#/$defs/Meta"
default:
op: u
source: ~
description: Document metadata
ID:
anyOf:
Expand All @@ -57,5 +52,7 @@ properties:
default: ~
format: number
title: Id
required:
- _meta
title: TEST_EMPTY
type: object
11 changes: 6 additions & 5 deletions source-oracle-flashback/source_oracle_flashback/models.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -292,17 +292,18 @@ def build_table(
field_schema_extra: dict | None = None

if col.data_type == col.Type.NUMBER and col.data_scale == 0:
# datA_precision: 0 defaults to precision 38
# data_precision: 0 defaults to precision 38
if col.data_precision > 18 or col.data_precision == 0:
field_type = Decimal
field_schema_extra = {"format": "number"}
else:
field_type = int

elif col.data_type in (col.Type.DOUBLE, col.Type.NUMBER, col.Type.FLOAT):
# Floats cannot be used as keys, so use {type: string, format: number}.
field_type = Decimal
field_schema_extra = {"format": "number"}
if col.data_precision > 18 or col.data_precision == 0:
field_type = Decimal
field_schema_extra = {"format": "number"}
else:
field_type = float
elif col.data_type in (col.Type.CHAR, col.Type.VARCHAR, col.Type.VARCHAR2, col.Type.NCHAR, col.Type.NVARCHAR2):
field_type = str
elif col.is_datetime and col.has_timezone:
Expand Down
3 changes: 3 additions & 0 deletions source-oracle-flashback/tests/db_seeds/create_test_all_types.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,13 @@ CREATE TABLE flow_test.test_all_types(
num NUMBER (38, 9),
num19 NUMBER (19, 0),
num15 NUMBER (15, 0),
num104 NUMBER (10, 4),
small_int SMALLINT,
integ INTEGER,
double_precision DOUBLE PRECISION,
float_126 FLOAT(126),
float_63 FLOAT(63),
float_16 FLOAT(16),
real_num REAL,
datetime DATE,
ts TIMESTAMP,
Expand Down
8 changes: 7 additions & 1 deletion source-oracle-flashback/tests/db_seeds/insert_test_all_types.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,13 @@ INSERT INTO flow_test.test_all_types(
num,
num19,
num15,
num104,
small_int,
integ,
double_precision,
float_126,
float_63,
float_16,
real_num,
datetime,
ts,
Expand All @@ -31,8 +34,11 @@ INSERT INTO flow_test.test_all_types(
123456789.123456789,
1234567891234567891,
123456789123456,
123456.3456,
123456789,
18446744073709551615,
18446744073709551615,
123456789.123456789,
123456789.123456789,
123456789.123456789,
123456789.123456789,
123456789.123456789,
Expand Down
3 changes: 3 additions & 0 deletions source-oracle-flashback/tests/snapshots/snapshots__capture_all_types__stdout.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,14 @@
"DATETIME": "2022-01-01T00:00:00",
"DOUBLE_PRECISION": "123456789.123456789",
"FLOAT_126": "123456789.123456789",
"FLOAT_16": 123460000.0,
"FLOAT_63": "123456789.123456789",
"ID": "1",
"INTEG": "18446744073709551615",
"INTERVAL_DAY": "+01 02:03:04.567000",
"INTERVAL_YEAR": "+1234-05",
"NUM": "123456789.123456789",
"NUM104": 123456.3456,
"NUM15": 123456789123456,
"NUM19": "1234567891234567891",
"NVCHAR2": "nvarchar2 value with unicode characters \u2764\ufe0f \ud83d\udd25\ufe0f",
Expand Down
37 changes: 37 additions & 0 deletions source-oracle-flashback/tests/snapshots/snapshots__discover__stdout.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,31 @@
"format": "number",
"title": "Float 126"
},
"FLOAT_16": {
"anyOf": [
{
"type": "number"
},
{
"type": "null"
}
],
"default": null,
"title": "Float 16"
},
"FLOAT_63": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"default": null,
"format": "number",
"title": "Float 63"
},
"INTEG": {
"anyOf": [
{
Expand Down Expand Up @@ -169,6 +194,18 @@
"format": "number",
"title": "Num"
},
"NUM104": {
"anyOf": [
{
"type": "number"
},
{
"type": "null"
}
],
"default": null,
"title": "Num104"
},
"NUM15": {
"anyOf": [
{
Expand Down
Loading