Multiprocessing fork bug fix

scripts/us_usda/quickstats/process.py

@@ -28,6 +28,7 @@
 from itertools import repeat
 import json
 import multiprocessing
+from multiprocessing import get_context
 import os
 import sys
 
@@ -93,7 +94,7 @@ def process_survey_data(year, svs, out_dir, usda_api_key):
 
     pool_size = max(2, multiprocessing.cpu_count() - 1)
 
-    with multiprocessing.Pool(pool_size) as pool:
+    with get_context("spawn").Pool(pool_size) as pool:
         pool.starmap(
             fetch_and_write,
             zip(county_names, repeat(year), repeat(svs), repeat(out_dir),
@@ -201,8 +202,8 @@ def get_survey_county_data(year, county, out_dir, usda_api_key):
     print('# api records for', county, '=', len(response['data']))
     return response
 
-
-@limits(calls=10, period=60)
+# TODO: determine if the rate limiter needs to be re-enabled.
+# @limits(calls=10, period=60)
 def get_data(params):
     return requests.get(f'{API_BASE}/api_GET', params=params).json()