Skip to content

Commit

Permalink
Merge pull request #4288 from ckan/4288-datastore-tests
Browse files Browse the repository at this point in the history 
Validation Error on datastore_search when sorting timestamp fields
  • Loading branch information
@amercader
amercader authored Jul 20, 2018
2 parents 563a0a1 + b685b0d commit 600be27
Show file tree
Hide file tree
Showing 2 changed files with 138 additions and 2 deletions.
3 changes: 1 addition & 2 deletions ckanext/datastore/backend/postgres.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1672,15 +1672,14 @@ def datastore_search(self, context, data_dict, fields_types, query_dict):

select_cols = []
records_format = data_dict.get(u'records_format')
json_values = records_format in (u'objects', u'lists')
for field_id in field_ids:
fmt = u'to_json({0})' if records_format == u'lists' else u'{0}'
typ = fields_types.get(field_id)
if typ == u'nested':
fmt = u'({0}).json'
elif typ == u'timestamp':
fmt = u"to_char({0}, 'YYYY-MM-DD\"T\"HH24:MI:SS')"
if json_values:
if records_format == u'lists':
fmt = u"to_json({0})".format(fmt)
elif typ.startswith(u'_') or typ.endswith(u'[]'):
fmt = u'array_to_json({0})'
Expand Down
137 changes: 137 additions & 0 deletions ckanext/datastore/tests/test_search.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1035,3 +1035,140 @@ def test_search_sql_enforces_private(self):
'datastore_search_sql',
context=ctx3,
sql=sql3)


class TestDatastoreSearchRecordsFormat(DatastoreFunctionalTestBase):
def test_sort_results_objects(self):
ds = factories.Dataset()
r = helpers.call_action(
u'datastore_create',
resource={u'package_id': ds['id']},
fields=[
{u'id': u'num', u'type': u'numeric'},
{u'id': u'dt', u'type': u'timestamp'},
{u'id': u'txt', u'type': u'text'}],
records=[
{u'num': 10, u'dt': u'2020-01-01', u'txt': 'aaab'},
{u'num': 9, u'dt': u'2020-01-02', u'txt': 'aaab'},
{u'num': 9, u'dt': u'2020-01-01', u'txt': 'aaac'}])
assert_equals(
helpers.call_action(
'datastore_search',
resource_id=r['resource_id'],
sort=u'num, dt')['records'],
[
{u'_id': 3, u'num': 9, u'dt': u'2020-01-01T00:00:00', u'txt': u'aaac'},
{u'_id': 2, u'num': 9, u'dt': u'2020-01-02T00:00:00', u'txt': u'aaab'},
{u'_id': 1, u'num': 10, u'dt': u'2020-01-01T00:00:00', u'txt': u'aaab'},
])
assert_equals(
helpers.call_action(
'datastore_search',
resource_id=r['resource_id'],
sort=u'dt, txt')['records'],
[
{u'_id': 1, u'num': 10, u'dt': u'2020-01-01T00:00:00', u'txt': u'aaab'},
{u'_id': 3, u'num': 9, u'dt': u'2020-01-01T00:00:00', u'txt': u'aaac'},
{u'_id': 2, u'num': 9, u'dt': u'2020-01-02T00:00:00', u'txt': u'aaab'},
])
assert_equals(
helpers.call_action(
'datastore_search',
resource_id=r['resource_id'],
sort=u'txt, num')['records'],
[
{u'_id': 2, u'num': 9, u'dt': u'2020-01-02T00:00:00', u'txt': u'aaab'},
{u'_id': 1, u'num': 10, u'dt': u'2020-01-01T00:00:00', u'txt': u'aaab'},
{u'_id': 3, u'num': 9, u'dt': u'2020-01-01T00:00:00', u'txt': u'aaac'},
])

def test_sort_results_lists(self):
ds = factories.Dataset()
r = helpers.call_action(
u'datastore_create',
resource={u'package_id': ds['id']},
fields=[
{u'id': u'num', u'type': u'numeric'},
{u'id': u'dt', u'type': u'timestamp'},
{u'id': u'txt', u'type': u'text'}],
records=[
{u'num': 10, u'dt': u'2020-01-01', u'txt': u'aaab'},
{u'num': 9, u'dt': u'2020-01-02', u'txt': u'aaab'},
{u'num': 9, u'dt': u'2020-01-01', u'txt': u'aaac'}])
assert_equals(
helpers.call_action(
'datastore_search',
resource_id=r['resource_id'],
records_format=u'lists',
sort=u'num, dt')['records'],
[
[3, 9, u'2020-01-01T00:00:00', u'aaac'],
[2, 9, u'2020-01-02T00:00:00', u'aaab'],
[1, 10, u'2020-01-01T00:00:00', u'aaab'],
])
assert_equals(
helpers.call_action(
'datastore_search',
resource_id=r['resource_id'],
records_format=u'lists',
sort=u'dt, txt')['records'],
[
[1, 10, u'2020-01-01T00:00:00', u'aaab'],
[3, 9, u'2020-01-01T00:00:00', u'aaac'],
[2, 9, u'2020-01-02T00:00:00', u'aaab'],
])
assert_equals(
helpers.call_action(
'datastore_search',
resource_id=r['resource_id'],
records_format=u'lists',
sort=u'txt, num')['records'],
[
[2, 9, u'2020-01-02T00:00:00', u'aaab'],
[1, 10, u'2020-01-01T00:00:00', u'aaab'],
[3, 9, u'2020-01-01T00:00:00', u'aaac'],
])

def test_sort_results_csv(self):
ds = factories.Dataset()
r = helpers.call_action(
u'datastore_create',
resource={u'package_id': ds['id']},
fields=[
{u'id': u'num', u'type': u'numeric'},
{u'id': u'dt', u'type': u'timestamp'},
{u'id': u'txt', u'type': u'text'}],
records=[
{u'num': 10, u'dt': u'2020-01-01', u'txt': u'aaab'},
{u'num': 9, u'dt': u'2020-01-02', u'txt': u'aaab'},
{u'num': 9, u'dt': u'2020-01-01', u'txt': u'aaac'}])
assert_equals(
helpers.call_action(
'datastore_search',
resource_id=r['resource_id'],
records_format=u'csv',
sort=u'num, dt')['records'],
u'3,9,2020-01-01T00:00:00,aaac\n'
u'2,9,2020-01-02T00:00:00,aaab\n'
u'1,10,2020-01-01T00:00:00,aaab\n'
)
assert_equals(
helpers.call_action(
'datastore_search',
resource_id=r['resource_id'],
records_format=u'csv',
sort=u'dt, txt')['records'],
u'1,10,2020-01-01T00:00:00,aaab\n'
u'3,9,2020-01-01T00:00:00,aaac\n'
u'2,9,2020-01-02T00:00:00,aaab\n'
)
assert_equals(
helpers.call_action(
'datastore_search',
resource_id=r['resource_id'],
records_format=u'csv',
sort=u'txt, num')['records'],
u'2,9,2020-01-02T00:00:00,aaab\n'
u'1,10,2020-01-01T00:00:00,aaab\n'
u'3,9,2020-01-01T00:00:00,aaac\n'
)

0 comments on commit 600be27

Please sign in to comment.