Merge pull request #28 from storebrand/numeric_types

Numeric types

.github/workflows/ci.yaml

@@ -61,4 +61,4 @@ jobs:
         poetry install
     - name: Test with pytest
       run: |
-        poetry run pytest --capture=no
+        poetry run pytest --capture=no

.gitignore

@@ -142,4 +142,5 @@ plugins/
 config.json
 poetry.lock
 .dccache
-_jottings.py
+_jottings.py
+.security-notes.json

.pre-commit-config.yaml

@@ -0,0 +1,22 @@
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v2.3.0
+    hooks:
+    - id: check-yaml
+    - id: end-of-file-fixer
+    - id: trailing-whitespace
+    - id: flake8
+  - repo: https://github.com/psf/black
+    rev: 22.10.0
+    hooks:
+    - id: black
+  - repo: https://github.com/Lucas-C/pre-commit-hooks-bandit
+    rev: v1.0.5
+    hooks:
+    - id: python-bandit-vulnerability-check
+      args: [--skip, "B101", --recursive, "./target_mssql/"]
+  - repo: https://github.com/pycqa/isort
+    rev: 5.12.0
+    hooks:
+    - id: isort
+      name: isort (python)

README.md

@@ -61,21 +61,22 @@ Regarding connection info, either the `sqlalchemy_url` or `username`, `password`
 
 ## Settings
 
-| Setting              | Required | Default | Description |
-|:---------------------|:--------:|:-------:|:------------|
-| sqlalchemy_url       | False    | None    | SQLAlchemy connection string |
-| username             | False    | None    | SQL Server username |
-| password             | False    | None    | SQL Server password |
-| host                 | False    | None    | SQL Server host |
-| port                 | False    | 1433    | SQL Server port |
-| database             | False    | None    | SQL Server database |
-| default_target_schema| False    | None    | Default target schema to write to |
-| stream_maps          | False    | None    | Config object for stream maps capability. For more information check out [Stream Maps](https://sdk.meltano.com/en/latest/stream_maps.html). |
-| stream_map_config    | False    | None    | User-defined config values to be used within map expressions. |
-| flattening_enabled   | False    | None    | 'True' to enable schema flattening and automatically expand nested properties. |
-| flattening_max_depth | False    | None    | The max depth to flatten schemas. |
-
-A full list of supported settings and capabilities is available by running: `target-mssql --about`
+| Setting                  | Required | Default | Description |
+|:-------------------------|:--------:|:-------:|:------------|
+| sqlalchemy_url           | False    | None    | SQLAlchemy connection string |
+| username                 | False    | None    | SQL Server username |
+| password                 | False    | None    | SQL Server password |
+| host                     | False    | None    | SQL Server host |
+| port                     | False    | 1433    | SQL Server port |
+| database                 | False    | None    | SQL Server database |
+| default_target_schema    | False    | None    | Default target schema to write to |
+| table_prefix             | False    | None    | Prefix to add to table name |
+| prefer_float_over_numeric| False    |       0 | Use float data type for numbers (otherwise number type is used) |
+| stream_maps              | False    | None    | Config object for stream maps capability. For more information check out [Stream Maps](https://sdk.meltano.com/en/latest/stream_maps.html). |
+| stream_map_config        | False    | None    | User-defined config values to be used within map expressions. |
+| flattening_enabled       | False    | None    | 'True' to enable schema flattening and automatically expand nested properties. |
+| flattening_max_depth     | False    | None    | The max depth to flatten schemas. |
+
 
 A full list of supported settings and capabilities for this
 target is available by running:

meltano.yml

@@ -27,4 +27,4 @@ plugins:
     - name: host
     - name: port
     - name: database
-    - name: default_target_schema
+    - name: default_target_schema

target_mssql/connector.py

@@ -357,8 +357,12 @@ def to_sql_type(self, jsonschema_type: dict) -> sqlalchemy.types.TypeEngine:  #
 
         if self._jsonschema_type_check(jsonschema_type, ("integer",)):
             return cast(sqlalchemy.types.TypeEngine, sqlalchemy.types.BIGINT())
+
         if self._jsonschema_type_check(jsonschema_type, ("number",)):
-            return cast(sqlalchemy.types.TypeEngine, sqlalchemy.types.NUMERIC(22, 16))
+            if self.config.get("prefer_float_over_numeric", False):
+                return cast(sqlalchemy.types.TypeEngine, sqlalchemy.types.FLOAT())
+            return cast(sqlalchemy.types.TypeEngine, sqlalchemy.types.NUMERIC(38, 16))
+
         if self._jsonschema_type_check(jsonschema_type, ("boolean",)):
             return cast(sqlalchemy.types.TypeEngine, mssql.VARCHAR(1))
 
@@ -388,6 +392,6 @@ def create_temp_table_from_table(self, from_table_name):
             SELECT TOP 0 *
             into {tmp_full_table_name}
             FROM {full_table_name}
-        """
+        """  # nosec
 
         self.connection.execute(ddl)

target_mssql/sinks.py

@@ -249,7 +249,7 @@ def merge_upsert_from_table(
             WHEN NOT MATCHED THEN
                 INSERT ({", ".join(schema["properties"].keys())})
                 VALUES ({", ".join([f"temp.{key}" for key in schema["properties"].keys()])});
-        """
+        """  # nosec
 
         with self.connection.begin():
             self.connection.execute(merge_sql)

target_mssql/target.py

@@ -52,6 +52,12 @@ class Targetmssql(SQLTarget):
         th.Property(
             "table_prefix", th.StringType, description="Prefix to add to table name"
         ),
+        th.Property(
+            "prefer_float_over_numeric",
+            th.BooleanType,
+            description="Use float data type for numbers (otherwise number type is used)",
+            default=False,
+        ),
     ).to_dict()
 
     default_sink_class = mssqlSink

target_mssql/tests/data_files/array_data.singer

@@ -3,4 +3,4 @@
 {"type": "RECORD", "stream": "test_carts", "record": {"id": 2, "fruits": [ "banana", "apple" ]}}
 {"type": "RECORD", "stream": "test_carts", "record": {"id": 3, "fruits": [ "pear" ]}}
 {"type": "RECORD", "stream": "test_carts", "record": {"id": 4, "fruits": [ "orange", "banana", "apple", "pear" ]}}
-{"type": "STATE", "value": {"test_carts": 4}}
+{"type": "STATE", "value": {"test_carts": 4}}

target_mssql/tests/data_files/camelcase_complex_schema.singer

@@ -1,2 +1,2 @@
 {"type": "SCHEMA", "stream": "ForecastingTypeToCategory", "schema": {"properties": {"Id": {"type": "string"}, "IsDeleted": {"type": ["null", "boolean"]}, "CreatedDate": {"anyOf": [{"type": "string", "format": "date-time"}, {"type": ["string", "null"]}]}, "CreatedById": {"type": ["null", "string"]}, "LastModifiedDate": {"anyOf": [{"type": "string", "format": "date-time"}, {"type": ["string", "null"]}]}, "LastModifiedById": {"type": ["null", "string"]}, "SystemModstamp": {"anyOf": [{"type": "string", "format": "date-time"}, {"type": ["string", "null"]}]}, "ForecastingTypeId": {"type": ["null", "string"]}, "ForecastingItemCategory": {"type": ["null", "string"]}, "DisplayPosition": {"type": ["null", "integer"]}, "IsAdjustable": {"type": ["null", "boolean"]}, "IsOwnerAdjustable": {"type": ["null", "boolean"]}}, "type": "object", "additionalProperties": false}, "key_properties": ["Id"]}
-{"type": "SCHEMA", "stream": "ForecastingTypeToCategory", "schema": {"properties": {"Id": {"type": "string"}, "IsDeleted": {"type": ["null", "boolean"]}, "CreatedDate": {"anyOf": [{"type": "string", "format": "date-time"}, {"type": ["string", "null"]}]}, "CreatedById": {"type": ["null", "string"]}, "LastModifiedDate": {"anyOf": [{"type": "string", "format": "date-time"}, {"type": ["string", "null"]}]}, "LastModifiedById": {"type": ["null", "string"]}, "SystemModstamp": {"anyOf": [{"type": "string", "format": "date-time"}, {"type": ["string", "null"]}]}, "ForecastingTypeId": {"type": ["null", "string"]}, "ForecastingItemCategory": {"type": ["null", "string"]}, "DisplayPosition": {"type": ["null", "integer"]}, "IsAdjustable": {"type": ["null", "boolean"]}, "IsOwnerAdjustable": {"type": ["null", "boolean"]}, "age": {"type": "integer"}, "NewCamelCasedAttribute": {"type": "string"}}, "type": "object", "additionalProperties": false}, "key_properties": ["Id"]}
+{"type": "SCHEMA", "stream": "ForecastingTypeToCategory", "schema": {"properties": {"Id": {"type": "string"}, "IsDeleted": {"type": ["null", "boolean"]}, "CreatedDate": {"anyOf": [{"type": "string", "format": "date-time"}, {"type": ["string", "null"]}]}, "CreatedById": {"type": ["null", "string"]}, "LastModifiedDate": {"anyOf": [{"type": "string", "format": "date-time"}, {"type": ["string", "null"]}]}, "LastModifiedById": {"type": ["null", "string"]}, "SystemModstamp": {"anyOf": [{"type": "string", "format": "date-time"}, {"type": ["string", "null"]}]}, "ForecastingTypeId": {"type": ["null", "string"]}, "ForecastingItemCategory": {"type": ["null", "string"]}, "DisplayPosition": {"type": ["null", "integer"]}, "IsAdjustable": {"type": ["null", "boolean"]}, "IsOwnerAdjustable": {"type": ["null", "boolean"]}, "age": {"type": "integer"}, "NewCamelCasedAttribute": {"type": "string"}}, "type": "object", "additionalProperties": false}, "key_properties": ["Id"]}

target_mssql/tests/data_files/duplicate_records.singer

@@ -4,4 +4,4 @@
 {"type": "RECORD", "stream": "test_duplicate_records", "record": {"id": 1, "metric": 10}}
 {"type": "RECORD", "stream": "test_duplicate_records", "record": {"id": 2, "metric": 20}}
 {"type": "RECORD", "stream": "test_duplicate_records", "record": {"id": 1, "metric": 100}}
-{"type": "STATE", "value": {"test_duplicate_records": 2}}
+{"type": "STATE", "value": {"test_duplicate_records": 2}}

target_mssql/tests/data_files/encoded_strings.singer

@@ -29,4 +29,4 @@
 {"type": "RECORD", "stream": "test_strings_in_arrays", "record": {"id": 4, "strings": ["\u006D", "\u0101", "\u0199"]}}
 {"type": "RECORD", "stream": "test_strings_in_arrays", "record": {"id": 5, "strings": ["aaa", "Double quoting: \\u0000 \\u0041 \\u0001"]}}
 {"type": "RECORD", "stream": "test_strings_in_arrays", "record": {"id": 6, "strings": ["bbb", "Control Characters in string: \u0000 \u0041 \u0001"]}}
-{"type": "STATE", "value": {"test_strings": 11, "test_strings_in_objects": 11, "test_strings_in_arrays": 6}}
+{"type": "STATE", "value": {"test_strings": 11, "test_strings_in_objects": 11, "test_strings_in_arrays": 6}}

target_mssql/tests/data_files/insert_merge_part2.singer

@@ -1,3 +1,3 @@
 {"type": "SCHEMA", "stream": "insert_merge_stream", "schema": {"required": ["id"], "type": "object", "properties": { "id": {"type": ["string", "null"]}, "client_name": {"type": "string"} }}, "key_properties": ["id"]}
 {"type": "RECORD", "stream": "insert_merge_stream", "record": {"id": "2", "client_name": "Gitter iOS App"}}
-{"type": "RECORD", "stream": "insert_merge_stream", "record": {"id": "3", "client_name": "Generic ios app"}}
+{"type": "RECORD", "stream": "insert_merge_stream", "record": {"id": "3", "client_name": "Generic ios app"}}

target_mssql/tests/data_files/invalid_schema.singer

@@ -1 +1 @@
-{"type": "SCHEMA", "stream": "test_invalid_schema", "schema": {"type": "object"}, "key_properties": []}
+{"type": "SCHEMA", "stream": "test_invalid_schema", "schema": {"type": "object"}, "key_properties": []}

target_mssql/tests/data_files/multiple_state_messages.singer

@@ -16,4 +16,4 @@
 {"type": "RECORD", "stream": "test_multiple_state_messages_b", "record": {"id": 5, "metric": 550}}
 {"type": "RECORD", "stream": "test_multiple_state_messages_b", "record": {"id": 6, "metric": 660}}
 {"type": "STATE", "value": {"test_multiple_state_messages_a": 5, "test_multiple_state_messages_b": 6}}
-{"type": "RECORD", "stream": "test_multiple_state_messages_a", "record": {"id": 6, "metric": 600}}
+{"type": "RECORD", "stream": "test_multiple_state_messages_a", "record": {"id": 6, "metric": 600}}

target_mssql/tests/data_files/no_primary_keys.singer

@@ -2,4 +2,4 @@
 {"type": "RECORD", "stream": "test_no_pk", "record": {"id": 1, "metric": 11}}
 {"type": "RECORD", "stream": "test_no_pk", "record": {"id": 2, "metric": 22}}
 {"type": "RECORD", "stream": "test_no_pk", "record": {"id": 3, "metric": 33}}
-{"type": "STATE", "value": {"test_no_pk": 3}}
+{"type": "STATE", "value": {"test_no_pk": 3}}

target_mssql/tests/data_files/no_primary_keys_append.singer

@@ -4,4 +4,4 @@
 {"type": "RECORD", "stream": "test_no_pk", "record": {"id": 3, "metric": 303}}
 {"type": "RECORD", "stream": "test_no_pk", "record": {"id": 4, "metric": 404}}
 {"type": "RECORD", "stream": "test_no_pk", "record": {"id": 5, "metric": 505}}
-{"type": "STATE", "value": {"test_no_pk": 5}}
+{"type": "STATE", "value": {"test_no_pk": 5}}

target_mssql/tests/data_files/optional_attributes.singer

@@ -3,4 +3,4 @@
 {"type": "RECORD", "stream": "test_optional_attributes", "record": {"id": 2}}
 {"type": "RECORD", "stream": "test_optional_attributes", "record": {"id": 3, "optional": "Also optional"}}
 {"type": "RECORD", "stream": "test_optional_attributes", "record": {"id": 4}}
-{"type": "STATE", "value": {"test_optional_attributes": 4}}
+{"type": "STATE", "value": {"test_optional_attributes": 4}}

target_mssql/tests/data_files/record_missing_key_property.singer

@@ -1,2 +1,2 @@
 {"type": "SCHEMA", "stream": "test_record_missing_key_property", "key_properties": ["id"], "schema": {"type": "object", "properties": {"id": {"type": "integer"}, "metric": {"type": "integer"}}}}
-{"type": "RECORD", "stream": "test_record_missing_key_property", "record": {"metric": 8214}}
+{"type": "RECORD", "stream": "test_record_missing_key_property", "record": {"metric": 8214}}

target_mssql/tests/data_files/record_missing_required_property.singer

@@ -1,2 +1,2 @@
 {"type": "SCHEMA", "stream": "test_record_missing_required_property", "key_properties": [], "schema": {"required": ["id"], "type": "object", "properties": {"id": {"type": "integer"}, "metric": {"type": "integer"}}}}
-{"type": "RECORD", "stream": "test_record_missing_required_property", "record": {"metric": 3215}}
+{"type": "RECORD", "stream": "test_record_missing_required_property", "record": {"metric": 3215}}

target_mssql/tests/data_files/schema_no_properties.singer

@@ -3,4 +3,4 @@
 {"type": "RECORD", "stream": "test_object_schema_with_properties", "record": {"object_store": {"id": 2, "metric": 203}}}
 {"type": "SCHEMA", "stream": "test_object_schema_no_properties", "key_properties": [], "schema": {"type": "object"}}
 {"type": "RECORD", "stream": "test_object_schema_no_properties", "record": {"object_store": {"id": 1, "metric": 1}}}
-{"type": "RECORD", "stream": "test_object_schema_no_properties", "record": {"object_store": {"id": 2, "metric": 2}}}
+{"type": "RECORD", "stream": "test_object_schema_no_properties", "record": {"object_store": {"id": 2, "metric": 2}}}

target_mssql/tests/data_files/schema_updates.singer

@@ -8,4 +8,4 @@
 {"type": "SCHEMA", "stream": "test_schema_updates", "key_properties": ["id"], "schema": {"type": "object", "properties": {"id": {"type": "integer"}, "a1": {"type": "number"}, "a2": {"type": "string"}, "a3": {"type": "boolean"}, "a5": {"type": "string", "maxLength": 164}, "a6": {"type": "integer"}}}}
 {"type": "RECORD", "stream": "test_schema_updates", "record": {"id": 5, "a1": 105, "a2": "string5", "a3": false, "a5": "apple", "a6": 985}}
 {"type": "RECORD", "stream": "test_schema_updates", "record": {"id": 6, "a1": 106, "a2": "string6", "a3": true, "a5": "banana", "a6": 341}}
-{"type": "STATE", "value": {"test_schema_updates": 6}}
+{"type": "STATE", "value": {"test_schema_updates": 6}}

target_mssql/tests/data_files/simple_continents.singer

@@ -7,4 +7,4 @@
 {"type": "RECORD", "stream": "nocontinents", "record": {"code": "NA", "name": "North America"}, "time_extracted": "2022-07-17T20:43:18.860922Z"}
 {"type": "RECORD", "stream": "nocontinents", "record": {"code": "OC", "name": "Oceania"}, "time_extracted": "2022-07-17T20:43:18.860952Z"}
 {"type": "RECORD", "stream": "nocontinents", "record": {"code": "SA", "name": "South America"}, "time_extracted": "2022-07-17T20:43:18.860983Z"}
-{"type": "STATE", "value": {"bookmarks": {"nocontinents": {}}}}
+{"type": "STATE", "value": {"bookmarks": {"nocontinents": {}}}}

target_mssql/tests/data_files/special_chars_in_attributes.singer

@@ -1,2 +1,2 @@
 {"type": "SCHEMA", "stream": "test:SpecialChars!in?attributes", "schema": {"type": "object", "properties": {"_id": {"type": "string"}, "d": {"type": "object", "properties": {"env": {"type": "string"}, "agent__type": {"type": "string"}, "agent__os__version": {"type": "string"}}}}}, "key_properties": ["_id"]}
-{"type": "RECORD", "stream": "test:SpecialChars!in?attributes", "record": {"_id": "a2e98886", "d": {"env": "prod", "agent__type": "desktop", "agent__os__version": "10.13.1"}}, "version": 1541199424491, "time_extracted": "2018-11-02T22:57:04.841020Z"}
+{"type": "RECORD", "stream": "test:SpecialChars!in?attributes", "record": {"_id": "a2e98886", "d": {"env": "prod", "agent__type": "desktop", "agent__os__version": "10.13.1"}}, "version": 1541199424491, "time_extracted": "2018-11-02T22:57:04.841020Z"}

target_mssql/tests/data_files/target_schema.singer

@@ -1,4 +1,4 @@
 {"type": "SCHEMA", "stream": "my_schema-table_in_custom_schema", "key_properties": ["id"], "schema": {"required": ["id", "metric"], "type": "object", "properties": {"id": {"type": "integer"}, "metric": {"type": "integer"}}}}
 {"type": "RECORD", "stream": "my_schema-table_in_custom_schema", "record": {"id": 1, "metric": 1}}
 {"type": "RECORD", "stream": "my_schema-table_in_custom_schema", "record": {"id": 2, "metric": 2}}
-{"type": "STATE", "value": {"table_in_custom_schema": 2}}
+{"type": "STATE", "value": {"table_in_custom_schema": 2}}

target_mssql/tests/data_files/user_location_data.singer

@@ -12,4 +12,4 @@
 {"type": "RECORD", "stream": "test_user_in_location", "record": {"id": 1, "user_id": 1, "location_id": 1, "info": {"weather": "rainy", "mood": "sad"}}}
 {"type": "RECORD", "stream": "test_user_in_location", "record": {"id": 2, "user_id": 1, "location_id": 2, "info": {"weather": "sunny", "mood": "satisfied"}}}
 {"type": "RECORD", "stream": "test_user_in_location", "record": {"id": 3, "user_id": 1, "location_id": 3, "info": {"weather": "sunny", "mood": "happy"}}}
-{"type": "STATE", "value": {"test_users": 5, "test_locations": 3, "test_user_in_location": 3}}
+{"type": "STATE", "value": {"test_users": 5, "test_locations": 3, "test_user_in_location": 3}}

target_mssql/tests/data_files/user_location_upsert_data.singer

@@ -15,4 +15,4 @@
 {"type": "RECORD", "stream": "test_user_in_location", "record": {"id": 2, "user_id": 2, "location_id": 3, "info": {"weather": "sunny", "mood": "satisfied"}}}
 {"type": "RECORD", "stream": "test_user_in_location", "record": {"id": 6, "user_id": 3, "location_id": 2, "info": {"weather": "sunny", "mood": "happy"}}}
 {"type": "RECORD", "stream": "test_user_in_location", "record": {"id": 14, "user_id": 4, "location_id": 1, "info": {"weather": "cloudy", "mood": "ok"}}}
-{"type": "STATE", "value": {"test_users": 13, "test_locations": 8, "test_user_in_location": 14}}
+{"type": "STATE", "value": {"test_users": 13, "test_locations": 8, "test_user_in_location": 14}}

target_mssql/tests/samples/sample_tap_countries/schemas/continents.json

@@ -4,4 +4,4 @@
         "code": { "type": ["null", "string"] },
         "name": { "type": ["null", "string"] }
     }
-}
+}

target_mssql/tests/test_core.py

@@ -148,6 +148,17 @@ def test_schema_no_properties(mssql_target):
 
 # TODO test that data is correct
 def test_schema_updates(mssql_target):
+    base_config = {
+        "schema": "dbo",
+        "username": "sa",
+        "password": "P@55w0rd",
+        "host": "localhost",
+        "port": "1433",
+        "database": "master",
+        "table_prefix": "prfx_",
+        "prefer_float_over_numeric": True,
+    }
+    mssql_target = Targetmssql(config=base_config)
     file_name = "schema_updates.singer"
     singer_file_to_target(file_name, mssql_target)
 

tox.ini

@@ -12,7 +12,7 @@ whitelist_externals = poetry
 commands =
     poetry install -v
     poetry run pytest
-    poetry run black --check target_mssql 
+    poetry run black --check target_mssql
     poetry run flake8  target_mssql
 
 [testenv:pytest]
@@ -38,12 +38,12 @@ commands =
     poetry install -v
     poetry run black --check --diff target_mssql/
     poetry run isort --check target_mssql
-    poetry run flake8 target_mssql 
+    poetry run flake8 target_mssql
 
 [flake8]
 ignore = W503
 max-line-length = 120
 max-complexity = 10
 
 [pydocstyle]
-ignore = D105,D203,D213
+ignore = D105,D203,D213