Merge pull request #5246 from fecgov/feature/add_sort_mur_search

Add sorting by case number and by advisory opinions number in legal/search

tests/integration/test_advisory_opinions.py

@@ -29,6 +29,9 @@ def test_pending_ao(self, get_bucket):
         expected_ao = {
             "type": "advisory_opinions",
             "no": "2017-01",
+            "ao_no": "2017-01",
+            "ao_year": 2017,
+            "ao_serial": 1,
             "doc_id": "advisory_opinions_2017-01",
             "name": "An AO name",
             "summary": "An AO summary",
@@ -330,6 +333,9 @@ def test_ao_offsets(self, get_bucket):
         expected_ao1 = {
             "type": "advisory_opinions",
             "no": "2015-01",
+            "ao_no": "2015-01",
+            "ao_year": 2015,
+            "ao_serial": 1,
             "doc_id": "advisory_opinions_2015-01",
             "name": "AO name1",
             "summary": "AO summary1",
@@ -351,8 +357,11 @@ def test_ao_offsets(self, get_bucket):
             "entities": [],
         }
         expected_ao2 = {
-            "no": "2015-02",
             "type": "advisory_opinions",
+            "no": "2015-02",
+            "ao_no": "2015-02",
+            "ao_year": 2015,
+            "ao_serial": 2,
             "doc_id": "advisory_opinions_2015-02",
             "name": "An AO name2",
             "summary": "An AO summary2",
@@ -376,6 +385,9 @@ def test_ao_offsets(self, get_bucket):
         expected_ao3 = {
             "type": "advisory_opinions",
             "no": "2016-01",
+            "ao_no": "2016-01",
+            "ao_year": 2016,
+            "ao_serial": 1,
             "doc_id": "advisory_opinions_2016-01",
             "name": "An AO name3",
             "summary": "An AO summary3",
@@ -401,13 +413,13 @@ def test_ao_offsets(self, get_bucket):
         self.create_ao(3, expected_ao3)
 
         gen = get_advisory_opinions(None)
-        assert (next(gen)) == expected_ao1
-        assert (next(gen)) == expected_ao2
         assert (next(gen)) == expected_ao3
+        assert (next(gen)) == expected_ao2
+        assert (next(gen)) == expected_ao1
 
         gen = get_advisory_opinions('2015-02')
-        assert (next(gen)) == expected_ao2
         assert (next(gen)) == expected_ao3
+        assert (next(gen)) == expected_ao2
 
     def create_document(self, ao_id, document, filename='201801_C.pdf'):
         self.connection.execute(

tests/integration/test_current_cases.py

@@ -32,6 +32,7 @@ def test_simple_mur(self, get_bucket):
             'mur_type': 'current',
             'election_cycles': [2016],
             'doc_id': 'mur_1',
+            'case_serial': 1,
             'published_flg': True,
             'participants': [],
             'subjects': [mur_subject],
@@ -66,6 +67,7 @@ def test_unpublished_mur(self, get_bucket):
             'mur_type': 'current',
             'election_cycles': [2016],
             'doc_id': 'mur_101',
+            'case_serial': 101,
             'published_flg': False,
             'participants': [],
             'subjects': [mur_subject],
@@ -100,6 +102,7 @@ def test_simple_adr(self, get_bucket):
             'name': 'Simple ADR',
             'election_cycles': [2016],
             'doc_id': 'adr_1',
+            'case_serial': 1,
             'published_flg': True,
             'participants': [],
             'non_monetary_terms': [],
@@ -186,6 +189,7 @@ def test_admin_fine(self, get_bucket):
             'no': '1',
             'name': 'Big Admin Fine',
             'doc_id': 'af_1',
+            'case_serial': 1,
             'published_flg': True,
             'documents': [],
             'commission_votes': [{'action': None, 'vote_date': None}],
@@ -268,6 +272,7 @@ def test_mur_with_participants_and_documents(self, get_bucket, get_credential):
         expected_mur = {
             "type": "murs",
             'no': '1',
+            'case_serial': 1,
             'name': 'MUR with participants',
             'mur_type': 'current',
             'published_flg': True,
@@ -446,6 +451,7 @@ def test_mur_with_disposition(self, get_bucket, get_credential):
             'participants': [],
             'no': '1',
             'doc_id': 'mur_1',
+            'case_serial': 1,
             'published_flg': True,
             'mur_type': 'current',
             'name': 'Open Elections LLC',
@@ -468,6 +474,7 @@ def test_mur_offsets(self, get_bucket):
             'mur_type': 'current',
             'election_cycles': [2016],
             'doc_id': 'mur_1',
+            'case_serial': 1,
             'published_flg': True,
             'participants': [],
             'subjects': [mur_subject],
@@ -488,6 +495,7 @@ def test_mur_offsets(self, get_bucket):
             'mur_type': 'current',
             'election_cycles': [2016],
             'doc_id': 'mur_2',
+            'case_serial': 2,
             'published_flg': True,
             'participants': [],
             'subjects': [mur_subject],
@@ -508,6 +516,7 @@ def test_mur_offsets(self, get_bucket):
             'mur_type': 'current',
             'election_cycles': [2016],
             'doc_id': 'mur_3',
+            'case_serial': 3,
             'published_flg': True,
             'participants': [],
             'subjects': [mur_subject],
@@ -544,9 +553,9 @@ def test_mur_offsets(self, get_bucket):
         )
 
         gen = get_cases('MUR')
-        assert (next(gen)) == expected_mur1
-        assert (next(gen)) == expected_mur2
         assert (next(gen)) == expected_mur3
+        assert (next(gen)) == expected_mur2
+        assert (next(gen)) == expected_mur1
 
         actual_murs = [mur for mur in get_cases('MUR', '2')]
         assert actual_murs == [expected_mur2]

webservices/args.py

@@ -193,7 +193,7 @@ def make_multi_sort_args(default=None, validator=None, default_hide_null=False,
         default_nulls_only=False, default_sort_nulls_last=False):
     args = make_sort_args(default, validator, default_hide_null, default_nulls_only, default_sort_nulls_last)
     args['sort'] = fields.List(fields.Str, missing=default, validate=validator, required=False, allow_none=True,
-        description='Provide a field to sort by. Use - for descending order.',)
+        description=docs.SORT)
     return args
 
 
@@ -264,6 +264,7 @@ def make_seek_args(field=fields.Int, description=None):
     'af_min_fd_date': fields.Date(required=False, description=docs.AF_MIN_FD_DATE),
     'af_max_fd_date': fields.Date(required=False, description=docs.AF_MAX_FD_DATE),
     'af_fd_fine_amount': fields.Int(IStr, required=False, description=docs.AF_FD_FINE_AMOUNT),
+    'sort':fields.Str(IStr, required=False, description=docs.SORT),
 }
 
 candidate_detail = {

webservices/docs.py

@@ -49,6 +49,11 @@
 Number of pages in the document
 '''
 
+SORT = '''
+Provide a field to sort by. Use `-` for descending order. \
+ex: `-case_no`
+'''
+
 # ======== candidate start ===========
 CANDIDATE_TAG = '''
 Candidate endpoints give you access to information about the people running for office.

webservices/legal_docs/advisory_opinions.py

@@ -21,13 +21,15 @@
 
 ALL_AOS = """
     SELECT
-        ao_parsed.ao_id,
-        ao_parsed.ao_no,
-        ao_parsed.name,
-        ao_parsed.summary,
-        ao_parsed.req_date,
-        ao_parsed.issue_date,
-        ao.stage
+        ao_parsed.ao_id as ao_id,
+        ao_parsed.ao_no as ao_no,
+        ao_parsed.ao_year as ao_year,
+        ao_parsed.ao_serial as ao_serial,
+        ao_parsed.name as name,
+        ao_parsed.summary as summary,
+        ao_parsed.req_date as req_date,
+        ao_parsed.issue_date as issue_date,
+        ao.stage as stage
     FROM aouser.aos_with_parsed_numbers ao_parsed
     INNER JOIN aouser.ao ao
         ON ao_parsed.ao_id = ao.ao_id
@@ -36,7 +38,7 @@
         OR
         (ao_parsed.ao_year > %s)
     )
-    ORDER BY ao_parsed.ao_year, ao_parsed.ao_serial
+    ORDER BY ao_parsed.ao_year desc, ao_parsed.ao_serial desc
 """
 
 AO_ENTITIES = """
@@ -173,6 +175,9 @@ def get_advisory_opinions(from_ao_no):
             ao = {
                 "type": AO_DOC_TYPE,
                 "no": row["ao_no"],
+                "ao_no": row["ao_no"],
+                "ao_year": row["ao_year"],
+                "ao_serial": row["ao_serial"],
                 "doc_id": "{0}_{1}".format(AO_DOC_TYPE, row["ao_no"]),
                 "name": row["name"],
                 "summary": row["summary"],

webservices/legal_docs/archived_murs.py

@@ -23,15 +23,17 @@
 ALL_ARCHIVED_MURS = """
     SELECT DISTINCT
         mur_id,
-        mur_number as mur_no
+        mur_number as mur_no,
+        mur_id as case_serial
     FROM MUR_ARCH.ARCHIVED_MURS
-    ORDER BY mur_id
+    ORDER BY mur_id desc
 """
 
 SINGLE_MUR = """
     SELECT DISTINCT
         mur_number as mur_no,
         mur_id,
+        mur_id as case_serial,
         mur_name,
         open_date,
         close_date
@@ -172,6 +174,7 @@ def get_single_mur(mur_no):
                 "type": get_es_type(),
                 "doc_id": "mur_{0}".format(row["mur_no"]),
                 "no": row["mur_no"],
+                "case_serial": row["mur_id"],
                 "url": "/legal/matter-under-review/{0}/".format(row["mur_no"]),
                 "mur_type": "archived",
                 "mur_name": row["mur_name"],

webservices/legal_docs/current_cases.py

@@ -25,18 +25,20 @@
     SELECT
         case_id,
         case_no,
+        case_serial,
         name,
         case_type,
         published_flg
     FROM fecmur.cases_with_parsed_case_serial_numbers_vw
     WHERE case_type = %s
-    ORDER BY case_serial
+    ORDER BY case_serial desc
 """
 
 SINGLE_CASE = """
     SELECT DISTINCT
         case_id,
         case_no,
+        case_serial,
         name,
         case_type,
         published_flg
@@ -237,10 +239,10 @@
 
 """ For ADR's populate case_status based on event_name""" 
 adr_case_status_map = {
-    'Dismissed': 'Case Dismissed',
-    'Settlement Agreement - Complaint Unsubstantiated': 'Negotiated Settlement Approved',
-    'Dismissed - Agreement Rejected': 'Negotiated Settlement Rejected by Commission',
-    'Dismissed - Failed to Approve': 'Case Dismissed'
+    "Dismissed": "Case Dismissed",
+    "Settlement Agreement - Complaint Unsubstantiated": "Negotiated Settlement Approved",
+    "Dismissed - Agreement Rejected": "Negotiated Settlement Rejected by Commission",
+    "Dismissed - Failed to Approve": "Case Dismissed"
 }
 
 STATUTE_REGEX = re.compile(r"(?<!\(|\d)(?P<section>\d+([a-z](-1)?)?)")
@@ -364,6 +366,7 @@ def get_single_case(case_type, case_no, bucket):
                 "type": get_es_type(case_type),
                 "doc_id": "{0}_{1}".format(case_type.lower(), row["case_no"]),
                 "no": row["case_no"],
+                "case_serial": row["case_serial"],
                 "name": row["name"],
                 "published_flg": row["published_flg"],
                 "sort1": sort1,

webservices/legal_docs/es_management.py

@@ -56,6 +56,7 @@
     "type": {"type": "keyword"},
     "doc_id": {"type": "keyword"},
     "no": {"type": "keyword"},
+    "case_serial": {"type": "integer"},
     "name": {"type": "text", "analyzer": "english"},
     "published_flg": {"type": "boolean"},
     "commission_votes": {
@@ -127,6 +128,9 @@
 ADVISORY_OPINIONS = {
     "type": {"type": "keyword"},
     "no": {"type": "keyword"},
+    "ao_no": {"type": "keyword"},
+    "ao_serial": {"type": "integer"},
+    "ao_year": {"type": "integer"},
     "doc_id": {"type": "keyword"},
     "name": {"type": "text", "analyzer": "english"},
     "summary": {"type": "text", "analyzer": "english"},
@@ -198,6 +202,7 @@
     "type": {"type": "keyword"},
     "doc_id": {"type": "keyword"},
     "no": {"type": "keyword"},
+    "case_serial": {"type": "integer"},
     "name": {"type": "text", "analyzer": "english"},
     "published_flg": {"type": "boolean"},
     "commission_votes": {
@@ -244,6 +249,7 @@
     "type": {"type": "keyword"},
     "doc_id": {"type": "keyword"},
     "no": {"type": "keyword"},
+    "case_serial": {"type": "integer"},
     "name": {"type": "text", "analyzer": "english"},
     "published_flg": {"type": "boolean"},
     "complainant": {"type": "text"},
@@ -363,6 +369,7 @@
         "type": {"type": "keyword"},
         "doc_id": {"type": "keyword"},
         "no": {"type": "keyword"},
+        "case_serial": {"type": "integer"},
         "mur_name": {"type": "text"},
         "mur_type": {"type": "keyword"},
         "open_date": {"type": "date", "format": "dateOptionalTime"},

webservices/resources/legal.py

@@ -157,13 +157,25 @@ def generic_query_builder(q, type_, from_hit, hits_returned, **kwargs):
         .index("docs_search")
         .sort("sort1", "sort2")
     )
+
     logger.debug("generic_query_builder =" + json.dumps(query.to_dict(), indent=3, cls=DateTimeEncoder))
     return query
 
 
 def case_query_builder(q, type_, from_hit, hits_returned, **kwargs):
     query = generic_query_builder(None, type_, from_hit, hits_returned, **kwargs)
 
+    # sorting works in all three cases: ('murs','admin_fines','adrs').
+    # so far only be able to sort by 'case_no', default sort is descending order.
+    # descending order: 'sort=-case_no'; ascending order; sort=case_no
+    # https://fec-dev-api.app.cloud.gov/v1/legal/search/?type=murs&sort=-case_no
+    # https://fec-dev-api.app.cloud.gov/v1/legal/search/?type=murs&sort=case_no
+    if kwargs.get("sort"):
+        if kwargs.get("sort").upper() == "CASE_NO":
+            query = query.sort({"case_serial" : {"order" : "asc"}})
+        else:
+            query = query.sort({"case_serial" : {"order" : "desc"}})
+
     should_query = [
         get_case_document_query(q, **kwargs),
         Q("query_string", query=q, fields=["no", "name"]),
@@ -438,6 +450,17 @@ def ao_query_builder(q, type_, from_hit, hits_returned, **kwargs):
     # Only pass query string to document list below
     query = generic_query_builder(None, type_, from_hit, hits_returned, **kwargs)
 
+    # so far only be able to sort by 'ao_no', default sort is descending order.
+    # descending order: 'sort=-ao_no'; ascending order; sort=ao_no
+    # https://fec-dev-api.app.cloud.gov/v1/legal/search/?type=advisory_opinions&sort=-ao_no
+    # https://fec-dev-api.app.cloud.gov/v1/legal/search/?type=advisory_opinions&sort=ao_no
+
+    if kwargs.get("sort"):
+        if kwargs.get("sort").upper() == "AO_NO":
+            query = query.sort({"ao_no" : {"order" : "asc"}})
+        else:
+            query = query.sort({"ao_no" : {"order" : "desc"}})
+
     should_query = [
         get_ao_document_query(q, **kwargs),
         Q("query_string", query=q, fields=["no", "name", "summary"]),