Merge pull request #14 from NaturalHistoryMuseum/ckanpackager_duplicates

Allow the use of cursors in Solr searches

ckanext/datasolr/lib/solr_search.py

@@ -107,8 +107,14 @@ def fetch(self):
             fields=fields,
             total=total,
             records=search.results,
+            # indicates that this response came from Solr, this is used by the ckanpackager
+            _backend='datasolr',
         )
 
+        # if there is a next cursor mark in the Solr response, pass it on
+        if hasattr(search, 'nextCursorMark'):
+            response['next_cursor'] = search.nextCursorMark
+
         requested_fields = [f['id'] for f in fields]
         # Date fields are returned as python datetime objects
         # So need to be converted into a string
@@ -161,6 +167,10 @@ def build_query(params, field_names):
             solr_params['group'] = 'true'
             solr_params['group_field'] = fields
             solr_params['group_main'] = 'true'
+        # add cursor
+        cursor = params.get('cursor', None)
+        if cursor:
+            solr_params['cursorMark'] = cursor
 
         # Add facets
         facets = params.get('facets', [])

ckanext/datasolr/plugin.py

@@ -95,14 +95,15 @@ def datasolr_search(self, context, data_dict, fields, query_dict):
             facets_field_limit=data_dict.get('facets_field_limit'),
             limit=data_dict.get('limit', 100),
             sort=data_dict.get('sort'),
-            distinct=data_dict.get('distinct', False)
+            distinct=data_dict.get('distinct', False),
+            cursor=data_dict.get('cursor', None),
         )
         query_params['fields'] = data_dict.get('fields', [f['id'] for f in fields])
         cursor = data_dict.get('cursor', None)
         if cursor:
             # Must be sorted on primary key
             # TODO: We could get the primary field from the solr schema lookup
-            query_params['sort'] = [('_id', 'ASC')]
+            query_params['sort'] = '_id'
         else:
             # If we've specified a paging cursor, then we don't want to use the offset
             query_params['offset'] = data_dict.get('offset', 0)