Mattyweb/issue937

docs/data_loading.md

@@ -9,7 +9,7 @@ Data is pulled nightly from the [LA Open Data](https://data.lacity.org/) website
 The data we load essentially comes from a single data source of 311 requests (technically, it's broken up by calendar year) and a GeoJSON file for Neighborhood councils.
 
 1. Data for service requests are **loaded with very little transformation**. Preparation for reporting is mostly done by joining with cleaned dimensions.
-2. The key dimensions are: council (nc), type (requesttype), agency (owner). Councils are further grouped into regions.
+2. The key dimensions are: council (nc), type (requesttype), agency (owner), source (requestsource). Councils are further grouped into regions.
 3. **Not all requests are associated with a neighborhood council**. There are some areas such as Pacific Palisades, Brentwood, and parts of South LA that are not covered by councils. We associate these requests with a "council" with an ID of 0.
 4. These "no-council" requests will appear in some reports and therefore some totals may not add up; I.E., there will always be more requests at the city-wide level than the sum of the councils.
 5. **We only use requests that have a valid latitude and longitude** (this is more than 99% of the data). Some Feedback requests are pure commentary and have no location so they do not appear in reports. However, there are some Feedback requests that are either associated with other requests or are requests on their own (should probably be categorized as Other) which are included.

server/api/alembic/versions/f605be47c1ec_add_agencies.py

@@ -98,5 +98,5 @@ def upgrade():
 def downgrade():
 
     op.execute("DROP MATERIALIZED VIEW service_requests")
-    op.execute("DROP IF EXISTS sources")
-    op.execute("DROP IF EXISTS agencies")
+    op.execute("DROP TABLE IF EXISTS sources")
+    op.execute("DROP TABLE IF EXISTS agencies")

server/api/code/lacity_data_api/routers/reports.py

@@ -9,6 +9,7 @@
 from ..models.service_request import ServiceRequest
 from ..models.request_type import RequestType
 from ..models.council import Council
+from ..models.source import Source
 
 router = APIRouter()
 
@@ -24,6 +25,7 @@
     ).label('created_month'),
     "created_date": ServiceRequest.created_date,
     "council_name": Council.council_name,
+    "source_name": Source.source_name,
     "type_name": RequestType.type_name
 }
 
@@ -36,7 +38,7 @@ async def run_report(
     field: Optional[List[str]] = Query(
         ["type_name", "created_date"],
         description="ex. created_date",
-        regex="(created_year|created_month|created_date|council_name|type_name)"
+        regex="""(created_year|created_month|created_date|council_name|type_name|source_name)"""  # noqa
     ),
     filter: Optional[List[str]] = Query(
         [f"created_date>={str(datetime.date.today() - datetime.timedelta(days=7))}"],
@@ -63,7 +65,7 @@ async def run_report(
         db.select(
             fields
         ).select_from(
-            ServiceRequest.join(RequestType).join(Council)
+            ServiceRequest.join(RequestType).join(Council).join(Source)
         ).where(
             text(' AND '.join(filters))
         ).group_by(

server/api/tests/integration/test_api_models.py

@@ -67,3 +67,17 @@ def test_agency(client):
     response = client.get(url)
     assert response.status_code == 200
     assert response.json()["agency_name"] == "Street Lighting Bureau"
+
+
+def test_sources(client):
+    url = "/sources"
+    response = client.get(url)
+    assert response.status_code == 200
+    assert len(response.json()) == 18
+
+
+def test_source(client):
+    url = "/sources/1"
+    response = client.get(url)
+    assert response.status_code == 200
+    assert response.json()["source_name"] == "City Attorney"

server/api/tests/integration/test_api_requests.py

@@ -64,6 +64,8 @@ def test_service_request(client):
     assert response.json()["typeName"] == "Illegal Dumping"
     assert response.json()["agencyId"] == 2
     assert response.json()["agencyName"] == "Sanitation Bureau"
+    assert response.json()["sourceId"] == 8
+    assert response.json()["sourceName"] == "Phone Call"
     assert response.json()["councilId"] == 44
     assert response.json()["councilName"] == "Lake Balboa"
     assert response.json()["createdDate"] == "2020-01-01"