Skip to content

Commit

Permalink
Fix tests for get_standard_metadata (#134)
Browse files Browse the repository at this point in the history 
* Fix testing of get_standard_metadata

* Fix maxDiff issue

* Add docstring

* Revert change to get_standard_metadata()

* Update how asserts work

* Fix missing import

* Revert changes to breadcrumbs

* Rename test_kp to has_pk

* Add test numbers

* Add missing test

* Fix failing test

Co-authored-by: Jude188 <[email protected]>
  • Loading branch information
@luandy64 @judahrand
luandy64 and judahrand authored Dec 4, 2020
1 parent eeed831 commit 6472683
Showing 1 changed file with 80 additions and 58 deletions.
138 changes: 80 additions & 58 deletions tests/test_metadata.py
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
from pprint import pprint
import unittest
from singer.metadata import get_standard_metadata
from singer.metadata import get_standard_metadata, to_map

def make_expected_metadata(base_obj, dict_of_extras):
def make_expected_metadata(base_obj, dict_of_extras, has_pk=False):
metadata_value = {**base_obj}
metadata_value.update(dict_of_extras)

Expand All @@ -13,7 +13,7 @@ def make_expected_metadata(base_obj, dict_of_extras):
},
{
'metadata': {
'inclusion': 'available',
'inclusion': 'automatic' if has_pk else 'available',
},
'breadcrumb': ('properties', 'id')
},
Expand All @@ -33,9 +33,19 @@ def make_expected_metadata(base_obj, dict_of_extras):

class TestStandardMetadata(unittest.TestCase):

#maxDiff = None

def test_standard_metadata(self):
"""
There's four inputs we want to test: schema, key_properties, replication_method, valid_replication_keys.
When `schema` is a non-null input, we expect `"inclusion": "available"` metadata for the `()` breadcrumb.
When `key_properties` is a non-null input, we expect `table-key-properties` metadata for the `()` breadcrumb.
When `replication_method` is a non-null input, we expect `forced-replication-method` metadata for the `()` breadcrumb.
When `valid_replication_keys` is a non-null input, we expect `valid-replication-keys` metadata for the `()` breadcrumb.
"""
self.maxDiff = None

# Some contants shared by a number of expected metadata objects
tap_stream_id = 'employees'
Expand All @@ -44,7 +54,7 @@ def test_standard_metadata(self):
test_rk = ['id', 'created']
metadata_kp = {'table-key-properties': ['id']}
metadata_rm = {'forced-replication-method': 'INCREMENTAL'}
metadata_rk = {'valid_replication_keys': ['id','created']}
metadata_rk = {'valid-replication-keys': ['id','created']}
schema_present_base_obj = {'inclusion': 'available'}
test_schema = {
'type': ['null', 'object'],
Expand All @@ -61,7 +71,7 @@ def test_standard_metadata(self):
# dictionary of parameters for `get_standard_metadata()` and the
# second element is the expected metadata
test_variables = [
(
( # test_number=0
{
'schema': test_schema,
'schema_name': tap_stream_id,
Expand All @@ -74,7 +84,7 @@ def test_standard_metadata(self):
{'schema-name': tap_stream_id,}
)
),
(
( # test_number=1
{
'schema': test_schema,
'schema_name': tap_stream_id,
Expand All @@ -84,11 +94,11 @@ def test_standard_metadata(self):
},
make_expected_metadata(
schema_present_base_obj,
{'valid_replication_keys': ['id','created'],
{'valid-replication-keys': ['id','created'],
'schema-name':tap_stream_id}
)
),
(
( # test_number=2
{
'schema': test_schema,
'schema_name': tap_stream_id,
Expand All @@ -102,7 +112,7 @@ def test_standard_metadata(self):
'schema-name':tap_stream_id}
)
),
(
( # test_number=3
{
'schema': test_schema,
'schema_name': tap_stream_id,
Expand All @@ -112,12 +122,12 @@ def test_standard_metadata(self):
},
make_expected_metadata(
schema_present_base_obj,
{'valid_replication_keys': ['id','created'],
{'valid-replication-keys': ['id','created'],
'forced-replication-method': 'INCREMENTAL',
'schema-name':tap_stream_id}
)
),
(
( # test_number=4
{
'schema': test_schema,
'schema_name': tap_stream_id,
Expand All @@ -128,10 +138,11 @@ def test_standard_metadata(self):
make_expected_metadata(
schema_present_base_obj,
{'table-key-properties': ['id'],
'schema-name':tap_stream_id}
'schema-name':tap_stream_id},
has_pk=True
)
),
(
( # test_number=5
{
'schema': test_schema,
'schema_name': tap_stream_id,
Expand All @@ -140,14 +151,14 @@ def test_standard_metadata(self):
'valid_replication_keys': test_rk
},
make_expected_metadata(

schema_present_base_obj,
{'table-key-properties': ['id'],
'valid_replication_keys': ['id','created'],
'schema-name':tap_stream_id}
'valid-replication-keys': ['id','created'],
'schema-name':tap_stream_id},
has_pk=True
)
),
(
( # test_number=6
{
'schema': test_schema,
'schema_name': tap_stream_id,
Expand All @@ -159,10 +170,11 @@ def test_standard_metadata(self):
schema_present_base_obj,
{'table-key-properties': ['id'],
'forced-replication-method': 'INCREMENTAL',
'schema-name':tap_stream_id}
'schema-name':tap_stream_id},
has_pk=True
)
),
(
( # test_number=7
{
'schema': test_schema,
'schema_name': tap_stream_id,
Expand All @@ -174,25 +186,21 @@ def test_standard_metadata(self):
schema_present_base_obj,
{'table-key-properties': ['id'],
'forced-replication-method': 'INCREMENTAL',
'valid_replication_keys': ['id','created'],
'schema-name':tap_stream_id}
'valid-replication-keys': ['id','created'],
'schema-name':tap_stream_id},
has_pk=True
)
),
(
( # test_number=8
{
'schema': None,
'key_properties': None,
'replication_method': None,
'valid_replication_keys': None
},
[
{
'metadata': {},
'breadcrumb': []
}
]
[]
),
(
( # test_number=9
{
'schema': None,
'key_properties': None,
Expand All @@ -202,14 +210,13 @@ def test_standard_metadata(self):
[
{
'metadata': {
'inclusion': 'available',
'valid_replication_keys': ['id','created']
'valid-replication-keys': ['id','created']
},
'breadcrumb': []
}
]
),
(
( # test_number=10
{
'schema': None,
'key_properties': None,
Expand All @@ -219,14 +226,13 @@ def test_standard_metadata(self):
[
{
'metadata': {
'inclusion': 'available',
'forced-replication-method': 'INCREMENTAL'
},
'breadcrumb': []
}
]
),
(
( # test_number=11
{
'schema': None,
'key_properties': None,
Expand All @@ -236,15 +242,14 @@ def test_standard_metadata(self):
[
{
'metadata': {
'inclusion': 'available',
'forced-replication-method': 'INCREMENTAL',
'valid_replication_keys': ['id','created']
'valid-replication-keys': ['id','created']
},
'breadcrumb': []
}
]
),
(
( # test_number=12
{
'schema': None,
'key_properties': test_kp,
Expand All @@ -254,14 +259,13 @@ def test_standard_metadata(self):
[
{
'metadata': {
'inclusion': 'available',
'table-key-properties': ['id'],
},
'breadcrumb': []
}
]
),
(
( # test_number=13
{
'schema': None,
'key_properties': test_kp,
Expand All @@ -271,15 +275,31 @@ def test_standard_metadata(self):
[
{
'metadata': {
'inclusion': 'available',
'table-key-properties': ['id'],
'valid_replication_keys': ['id','created']
'valid-replication-keys': ['id','created']
},
'breadcrumb': []
}
]
),
( # test_number=14
{
'schema': None,
'key_properties': test_kp,
'replication_method': test_rm,
'valid_replication_keys': None
},
[
{
'metadata': {
'table-key-properties': ['id'],
'forced-replication-method': 'INCREMENTAL',
},
'breadcrumb': []
}
]
),
(
( # test_number=15
{
'schema': None,
'key_properties': test_kp,
Expand All @@ -289,26 +309,26 @@ def test_standard_metadata(self):
[
{
'metadata': {
'inclusion': 'available',
'table-key-properties': ['id'],
'forced-replication-method': 'INCREMENTAL',
'valid_replication_keys': ['id','created']
'valid-replication-keys': ['id','created']
},
'breadcrumb': []
}
]
)
]

for var in test_variables:
function_params = var[0]
expected_metadata = var[1]
for i, var in enumerate(test_variables):
with self.subTest(test_number=i):
function_params = var[0]
expected_metadata = var[1]

test_value = get_standard_metadata(**function_params)
test_value = get_standard_metadata(**function_params)

for obj in expected_metadata:
if obj in test_value:
self.assertIn(obj, test_value)
expected_value = to_map(expected_metadata)
actual_value = to_map(test_value)
self.assertDictEqual(expected_value, actual_value)

# Test one function call where the parameters are not splat in
test_value = get_standard_metadata(test_schema,
Expand All @@ -320,11 +340,13 @@ def test_standard_metadata(self):
expected_metadata = make_expected_metadata(schema_present_base_obj,
{'table-key-properties': ['id'],
'forced-replication-method': 'INCREMENTAL',
'valid_replication_keys': ['id','created'],
'schema-name':tap_stream_id})
for obj in expected_metadata:
if obj in test_value:
self.assertIn(obj, test_value)
'valid-replication-keys': ['id','created'],
'schema-name':tap_stream_id},
has_pk=True)
self.assertDictEqual(
to_map(expected_metadata),
to_map(test_value)
)

def test_empty_key_properties_are_written(self):
mdata = get_standard_metadata(key_properties=[])
Expand Down

0 comments on commit 6472683

Please sign in to comment.