From b7f64954e77cfc6517a4b73e00618627fef501be Mon Sep 17 00:00:00 2001 From: Matt Webster Date: Mon, 1 Mar 2021 13:48:01 -0800 Subject: [PATCH 1/3] Add Source to API reports and requests Fixes #937 - can report by source_name dimension - can filter by source_name - added tests --- docs/data_loading.md | 2 +- .../alembic/versions/f605be47c1ec_add_agencies.py | 4 ++-- server/api/code/lacity_data_api/routers/reports.py | 6 ++++-- server/api/tests/integration/test_api_models.py | 14 ++++++++++++++ server/api/tests/integration/test_api_requests.py | 2 ++ 5 files changed, 23 insertions(+), 5 deletions(-) diff --git a/docs/data_loading.md b/docs/data_loading.md index c189905c6..b73ecea36 100644 --- a/docs/data_loading.md +++ b/docs/data_loading.md @@ -9,7 +9,7 @@ Data is pulled nightly from the [LA Open Data](https://data.lacity.org/) website The data we load essentially comes from a single data source of 311 requests (technically, it's broken up by calendar year) and a GeoJSON file for Neighborhood councils. 1. Data for service requests are **loaded with very little transformation**. Preparation for reporting is mostly done by joining with cleaned dimensions. -2. The key dimensions are: council (nc), type (requesttype), agency (owner). Councils are further grouped into regions. +2. The key dimensions are: council (nc), type (requesttype), agency (owner), source (requestsource). Councils are further grouped into regions. 3. **Not all requests are associated with a neighborhood council**. There are some areas such as Pacific Palisades, Brentwood, and parts of South LA that are not covered by councils. We associate these requests with a "council" with an ID of 0. 4. These "no-council" requests will appear in some reports and therefore some totals may not add up; I.E., there will always be more requests at the city-wide level than the sum of the councils. 5. **We only use requests that have a valid latitude and longitude** (this is more than 99% of the data). Some Feedback requests are pure commentary and have no location so they do not appear in reports. However, there are some Feedback requests that are either associated with other requests or are requests on their own (should probably be categorized as Other) which are included. diff --git a/server/api/alembic/versions/f605be47c1ec_add_agencies.py b/server/api/alembic/versions/f605be47c1ec_add_agencies.py index 6782782b8..d56e11bc2 100644 --- a/server/api/alembic/versions/f605be47c1ec_add_agencies.py +++ b/server/api/alembic/versions/f605be47c1ec_add_agencies.py @@ -98,5 +98,5 @@ def upgrade(): def downgrade(): op.execute("DROP MATERIALIZED VIEW service_requests") - op.execute("DROP IF EXISTS sources") - op.execute("DROP IF EXISTS agencies") + op.execute("DROP TABLE IF EXISTS sources") + op.execute("DROP TABLE IF EXISTS agencies") diff --git a/server/api/code/lacity_data_api/routers/reports.py b/server/api/code/lacity_data_api/routers/reports.py index 01f07c3e7..5f85786a6 100644 --- a/server/api/code/lacity_data_api/routers/reports.py +++ b/server/api/code/lacity_data_api/routers/reports.py @@ -9,6 +9,7 @@ from ..models.service_request import ServiceRequest from ..models.request_type import RequestType from ..models.council import Council +from ..models.source import Source router = APIRouter() @@ -24,6 +25,7 @@ ).label('created_month'), "created_date": ServiceRequest.created_date, "council_name": Council.council_name, + "source_name": Source.source_name, "type_name": RequestType.type_name } @@ -36,7 +38,7 @@ async def run_report( field: Optional[List[str]] = Query( ["type_name", "created_date"], description="ex. created_date", - regex="(created_year|created_month|created_date|council_name|type_name)" + regex="""(created_year|created_month|created_date|council_name|type_name|source_name)""" # noqa ), filter: Optional[List[str]] = Query( [f"created_date>={str(datetime.date.today() - datetime.timedelta(days=7))}"], @@ -63,7 +65,7 @@ async def run_report( db.select( fields ).select_from( - ServiceRequest.join(RequestType).join(Council) + ServiceRequest.join(RequestType).join(Council).join(Source) ).where( text(' AND '.join(filters)) ).group_by( diff --git a/server/api/tests/integration/test_api_models.py b/server/api/tests/integration/test_api_models.py index 677ab0420..cd6897cf5 100644 --- a/server/api/tests/integration/test_api_models.py +++ b/server/api/tests/integration/test_api_models.py @@ -67,3 +67,17 @@ def test_agency(client): response = client.get(url) assert response.status_code == 200 assert response.json()["agency_name"] == "Street Lighting Bureau" + + +def test_sources(client): + url = "/sources" + response = client.get(url) + assert response.status_code == 200 + assert len(response.json()) == 18 + + +def test_source(client): + url = "/sources/1" + response = client.get(url) + assert response.status_code == 200 + assert response.json()["source_name"] == "City Attorney" diff --git a/server/api/tests/integration/test_api_requests.py b/server/api/tests/integration/test_api_requests.py index 5167cd173..b7248016b 100644 --- a/server/api/tests/integration/test_api_requests.py +++ b/server/api/tests/integration/test_api_requests.py @@ -64,6 +64,8 @@ def test_service_request(client): assert response.json()["typeName"] == "Illegal Dumping" assert response.json()["agencyId"] == 2 assert response.json()["agencyName"] == "Sanitation Bureau" + assert response.json()["sourceId"] == 8 + assert response.json()["sourceName"] == "Phone Call" assert response.json()["councilId"] == 44 assert response.json()["councilName"] == "Lake Balboa" assert response.json()["createdDate"] == "2020-01-01" From 3e5ddab1a7d637d31e0b7909f963b2bfe5621b0a Mon Sep 17 00:00:00 2001 From: Matt Webster Date: Mon, 1 Mar 2021 13:49:22 -0800 Subject: [PATCH 2/3] Add Request Source to API reports and service requests (?) Fixes #937 --- server/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/docker-compose.yml b/server/docker-compose.yml index c1c66c033..04959ae16 100644 --- a/server/docker-compose.yml +++ b/server/docker-compose.yml @@ -3,7 +3,7 @@ version: '3.8' services: db: container_name: 311-postgres - image: postgis/postgis:11-3.1 + image: gangstead/postgis:latest-arm restart: always env_file: .env environment: From af02bf19315b3a7a6230507b97c368983ed14138 Mon Sep 17 00:00:00 2001 From: Matt Webster <10199792+mattyweb@users.noreply.github.com> Date: Mon, 1 Mar 2021 13:52:54 -0800 Subject: [PATCH 3/3] Update docker-compose.yml Reverted postgis version --- server/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/docker-compose.yml b/server/docker-compose.yml index 04959ae16..c1c66c033 100644 --- a/server/docker-compose.yml +++ b/server/docker-compose.yml @@ -3,7 +3,7 @@ version: '3.8' services: db: container_name: 311-postgres - image: gangstead/postgis:latest-arm + image: postgis/postgis:11-3.1 restart: always env_file: .env environment: