diff --git a/django/climate_change_api/climate_data/views.py b/django/climate_change_api/climate_data/views.py index 18364205..791f1ea1 100644 --- a/django/climate_change_api/climate_data/views.py +++ b/django/climate_change_api/climate_data/views.py @@ -6,7 +6,7 @@ from django.http import HttpResponseRedirect from django.utils.http import urlencode -from rest_framework import filters, status, viewsets +from rest_framework import filters, serializers, status, viewsets from rest_framework.decorators import api_view, detail_route, list_route from rest_framework.exceptions import NotFound, ParseError from rest_framework.response import Response @@ -20,7 +20,7 @@ ClimateModelSerializer, ClimateCityScenarioDataSerializer, ScenarioSerializer) -from indicators import indicator_factory, list_available_indicators +from indicators import indicator_factory, list_available_indicators, params logger = logging.getLogger(__name__) @@ -286,36 +286,22 @@ def climate_indicator(request, *args, **kwargs): else: model_list = ClimateModel.objects.all() - agg_param = request.query_params.get('agg', None) - aggregations = agg_param.split(',') if agg_param else None - years_param = request.query_params.get('years', None) - units_param = request.query_params.get('units', None) - indicator_key = kwargs['indicator'] IndicatorClass = indicator_factory(indicator_key) if not IndicatorClass: raise ParseError(detail='Must provide a valid indicator') - data = IndicatorClass(city, - scenario, - models=models_param, - years=years_param, - serializer_aggregations=aggregations, - parameters=request.query_params, - units=units_param).calculate() - - if units_param and units_param not in IndicatorClass.available_units: - raise NotFound(detail='Cannot convert indicator {} to units {}.'.format(indicator_key, - units_param)) - - if not units_param: - units_param = IndicatorClass.default_units + try: + indicator_class = IndicatorClass(city, scenario, parameters=request.query_params) + except params.ValidationError as e: + raise serializers.ValidationError(str(e)) + data = indicator_class.calculate() return Response(OrderedDict([ ('city', CitySerializer(city).data), ('scenario', scenario.name), ('indicator', IndicatorClass.to_dict()), ('climate_models', [m.name for m in model_list]), - ('units', units_param), + ('units', indicator_class.params.units), ('data', data), ])) diff --git a/django/climate_change_api/indicators/abstract_indicators.py b/django/climate_change_api/indicators/abstract_indicators.py index d495cb46..91b980f7 100644 --- a/django/climate_change_api/indicators/abstract_indicators.py +++ b/django/climate_change_api/indicators/abstract_indicators.py @@ -10,6 +10,7 @@ from climate_data.models import ClimateData, ClimateDataSource from climate_data.filters import ClimateDataFilterSet +from .params import IndicatorParams from .serializers import IndicatorSerializer from .unit_converters import DaysUnitsMixin @@ -22,6 +23,7 @@ class Indicator(object): variables = ClimateData.VARIABLE_CHOICES filters = None serializer_class = IndicatorSerializer + params_class = IndicatorParams # Subclasses should use a units mixin from 'unit_converters' to define these units # attributes and any necessary conversion functions @@ -30,8 +32,7 @@ class Indicator(object): available_units = (None,) parameters = None - def __init__(self, city, scenario, models=None, years=None, - serializer_aggregations=None, units=None, parameters=None): + def __init__(self, city, scenario, parameters=None): if not city: raise ValueError('Indicator constructor requires a city instance') if not scenario: @@ -39,18 +40,9 @@ def __init__(self, city, scenario, models=None, years=None, self.city = city self.scenario = scenario - self.models = models - self.years = years - self.serializer_aggregations = serializer_aggregations - # Set and validate desired units - self.units = units if units is not None else self.default_units - if self.units not in self.available_units: - raise ValueError('Cannot convert to requested units ({})'.format(self.units)) - - if self.parameters is not None and parameters is not None: - self.parameters = {key: parameters.get(key, default) - for (key, default) in self.parameters.items()} + self.params = self.params_class(parameters, self.available_units, self.default_units) + self.params.validate() self.queryset = self.get_queryset() self.queryset = self.filter_objects() @@ -88,8 +80,8 @@ def get_queryset(self): filter_set = ClimateDataFilterSet() queryset = (ClimateData.objects.filter(map_cell=self.city.map_cell) .filter(data_source__scenario=self.scenario)) - queryset = filter_set.filter_years(queryset, self.years) - queryset = filter_set.filter_models(queryset, self.models) + queryset = filter_set.filter_years(queryset, self.params.years) + queryset = filter_set.filter_models(queryset, self.params.models) return queryset def filter_objects(self): @@ -114,9 +106,9 @@ def convert_units(self, aggregations): @param aggregations list-of-dicts returned by aggregate method @returns Dict in same format as the aggregations parameter, with values converted - to `self.units` + to `self.params.units` """ - converter = self.getConverter(self.storage_units, self.units) + converter = self.getConverter(self.storage_units, self.params.units) for item in aggregations: if item['value'] is not None: item['value'] = converter(item['value']) @@ -151,7 +143,7 @@ def calculate(self): aggregations = self.convert_units(aggregations) collations = self.collate_results(aggregations) return self.serializer.to_representation(collations, - aggregations=self.serializer_aggregations) + aggregations=self.params.agg) class YearlyIndicator(Indicator): diff --git a/django/climate_change_api/indicators/indicators.py b/django/climate_change_api/indicators/indicators.py index 6acb4e10..f75bf384 100644 --- a/django/climate_change_api/indicators/indicators.py +++ b/django/climate_change_api/indicators/indicators.py @@ -8,6 +8,7 @@ YearlyMaxConsecutiveDaysIndicator, YearlySequenceIndicator, MonthlyAggregationIndicator, MonthlyCountIndicator, DailyRawIndicator) +from .params import PercentileIndicatorParams from .unit_converters import (TemperatureUnitsMixin, PrecipUnitsMixin, DaysUnitsMixin, CountUnitsMixin) @@ -93,39 +94,39 @@ class YearlyExtremePrecipitationEvents(CountUnitsMixin, YearlyCountIndicator): label = 'Yearly Extreme Precipitation Events' description = ('Total number of times per year daily precipitation exceeds the specified ' '(Default 99th) percentile of observations from 1960 to 1995') + params_class = PercentileIndicatorParams variables = ('pr',) - parameters = {'percentile': 99} @property def conditions(self): return {'pr__gt': F('map_cell__baseline__pr'), - 'map_cell__baseline__percentile': self.parameters['percentile']} + 'map_cell__baseline__percentile': self.params.percentile} class YearlyExtremeHeatEvents(CountUnitsMixin, YearlyCountIndicator): label = 'Yearly Extreme Heat Events' description = ('Total number of times per year daily maximum temperature exceeds the specified ' '(Default 99th) percentile of observations from 1960 to 1995') + params_class = PercentileIndicatorParams variables = ('tasmax',) - parameters = {'percentile': 99} @property def conditions(self): return {'tasmax__gt': F('map_cell__baseline__tasmax'), - 'map_cell__baseline__percentile': self.parameters['percentile']} + 'map_cell__baseline__percentile': self.params.percentile} class YearlyExtremeColdEvents(CountUnitsMixin, YearlyCountIndicator): label = 'Yearly Extreme Cold Events' description = ('Total number of times per year daily minimum temperature is below the specified ' '(Default 1st) percentile of observations from 1960 to 1995') + params_class = PercentileIndicatorParams variables = ('tasmin',) - parameters = {'percentile': 1} @property def conditions(self): return {'tasmin__lt': F('map_cell__baseline__tasmin'), - 'map_cell__baseline__percentile': self.parameters['percentile']} + 'map_cell__baseline__percentile': self.params.percentile} class HeatWaveDurationIndex(YearlyMaxConsecutiveDaysIndicator): @@ -195,39 +196,39 @@ class MonthlyExtremePrecipitationEvents(CountUnitsMixin, MonthlyCountIndicator): label = 'Monthly Extreme Precipitation Events' description = ('Total number of times per month daily precipitation exceeds the specified ' '(Default 99th) percentile of observations from 1960 to 1995') + params_class = PercentileIndicatorParams variables = ('pr',) - parameters = {'percentile': 99} @property def conditions(self): return {'pr__gt': F('map_cell__baseline__pr'), - 'map_cell__baseline__percentile': self.parameters['percentile']} + 'map_cell__baseline__percentile': self.params.percentile} class MonthlyExtremeHeatEvents(CountUnitsMixin, MonthlyCountIndicator): label = 'Monthly Extreme Heat Events' description = ('Total number of times per month daily maximum temperature exceeds the specified ' '(Default 99th) percentile of observations from 1960 to 1995') + params_class = PercentileIndicatorParams variables = ('tasmax',) - parameters = {'percentile': 99} @property def conditions(self): return {'tasmax__gt': F('map_cell__baseline__tasmax'), - 'map_cell__baseline__percentile': self.parameters['percentile']} + 'map_cell__baseline__percentile': self.params.percentile} class MonthlyExtremeColdEvents(CountUnitsMixin, MonthlyCountIndicator): label = 'Monthly Extreme Cold Events' description = ('Total number of times per month daily minimum temperature is below the specified ' '(Default 1st) percentile of observations from 1960 to 1995') + params_class = PercentileIndicatorParams variables = ('tasmin',) - parameters = {'percentile': 1} @property def conditions(self): return {'tasmin__lt': F('map_cell__baseline__tasmin'), - 'map_cell__baseline__percentile': self.parameters['percentile']} + 'map_cell__baseline__percentile': self.params.percentile} diff --git a/django/climate_change_api/indicators/params.py b/django/climate_change_api/indicators/params.py new file mode 100644 index 00000000..1b1dabb8 --- /dev/null +++ b/django/climate_change_api/indicators/params.py @@ -0,0 +1,95 @@ + +class ValidationError(Exception): + pass + + +class IndicatorParams(object): + """ Superclass used to define parameters necessary for an Indicator class to function + + This class defines the defaults for all indicators, which are all optional. + + New IndicatorParams subclasses should implement a new validate_paramname(self, value) + method for each parameter they add. This method should: + - Return a default value if the parameter is optional + - Raise indicators.params.ValidationError(message) if the value is invalid + - Ensure the paramname is added to the self.PARAMS iterable + + After calling validate(), the validated parameter values are available as instance attributes. + + """ + PARAMS = ('models', 'years', 'agg', 'units',) + + def __init__(self, query_params, available_units, default_units): + """ + @param query_params (dict[str:str]): Dict of key/value pairs to parse + @param available_units (Iterable[str]): List of unit strings this class should check the + 'units' query_params against + @param default_units (string): Default value for 'units' to set if none provided + in query_params + """ + self._params = query_params + self._available_units = available_units + self._default_units = default_units + self.validated = False + + def validate(self): + """ Validate all parameters found in self.PARAMS """ + for param in self.PARAMS: + value = self._params.get(param, None) + validation_fn = getattr(self, 'validate_' + param) + clean_value = validation_fn(value) + setattr(self, param, clean_value) + self.validated = True + + def validate_models(self, value): + return self._check_is_string('models', value) if value is not None else None + + def validate_years(self, value): + return self._check_is_string('years', value) if value is not None else None + + def validate_agg(self, value): + return self._check_is_string('agg', value) if value is not None else None + + def validate_units(self, value): + if value is None: + value = self._default_units + if value not in self._available_units: + raise ValidationError('Unit {} must be one of {}'.format(value, self._available_units)) + return value + + def __repr__(self): + """ Override repr to improve class self-documentation """ + representation = { + 'validated': self.validated, + 'PARAMS': self.PARAMS, + } + if self.validated: + for p in self.PARAMS: + representation[p] = getattr(self, p) + else: + for k, v in self._params.iteritems(): + representation[k] = v + return str(representation) + + def _check_is_string(self, param, value): + """ Private helper method to raise ValidationError if value is not a string """ + if not isinstance(value, basestring): + raise ValidationError('{} must be a string comma separated list'.format(param)) + return value + + +class PercentileIndicatorParams(IndicatorParams): + """ Extend IndicatorParams with a non-optional 'percentile' parameter """ + + PARAMS = IndicatorParams.PARAMS + ('percentile',) + + def validate_percentile(self, value): + if value is None: + raise ValidationError('percentile param required') + try: + int_value = int(value) + except ValueError: + raise ValidationError('percentile param must be an integer') + if int_value < 1 or int_value > 99: + raise ValidationError('percentile param must be within the range (0-100)') + return int_value diff --git a/django/climate_change_api/indicators/tests/test_indicator_params.py b/django/climate_change_api/indicators/tests/test_indicator_params.py new file mode 100644 index 00000000..f351004e --- /dev/null +++ b/django/climate_change_api/indicators/tests/test_indicator_params.py @@ -0,0 +1,72 @@ +from django.test import TestCase + +from indicators import params +from indicators.utils import merge_dicts + + +class IndicatorParamsTestCase(TestCase): + + default_parameters = {} + params_class = params.IndicatorParams + + def setUp(self): + pass + + def test_validate_valid_only_expected_params(self): + """ it should ensure indicator_params sets values for each of the base params """ + parameters = merge_dicts(self.default_parameters, + {'models': 'CCSM4', 'years': '2050:2060', 'units': 'K', 'agg': 'avg'}) + indicator_params = self.params_class(parameters, ('K',), 'K') + indicator_params.validate() + self.assertEqual(indicator_params.models, 'CCSM4') + self.assertEqual(indicator_params.years, '2050:2060') + self.assertEqual(indicator_params.units, 'K') + self.assertEqual(indicator_params.agg, 'avg') + + def test_validate_valid_some_unused_params(self): + """ it should ensure indicator_params properly ignores params that aren't defined """ + parameters = merge_dicts(self.default_parameters, + {'doesnotexist': 'true', 'years': '2050:2060', 'units': 'K', 'agg': 'avg'}) + indicator_params = self.params_class(parameters, ('K',), 'K') + indicator_params.validate() + with self.assertRaises(AttributeError): + value = indicator_params.doesnotexist + + def test_validate_valid_optional_defaults(self): + """ it should ensure indicator_params properly sets defaults on base params """ + parameters_sets = [ + {'models': None, 'years': None, 'units': None, 'agg': None}, + {} + ] + for parameters in parameters_sets: + p = merge_dicts(self.default_parameters, parameters) + indicator_params = self.params_class(p, ('K',), 'K') + indicator_params.validate() + self.assertIsNone(indicator_params.years) + self.assertIsNone(indicator_params.models) + self.assertIsNone(indicator_params.agg) + self.assertEquals(indicator_params.units, 'K') + + +class PercentileIndicatorParamsTestCase(IndicatorParamsTestCase): + default_parameters = {'percentile': '95'} + params_class = params.PercentileIndicatorParams + + def test_validate_percentile_none(self): + """ It should raise ValidationError if required param is missing """ + parameters = merge_dicts(self.default_parameters, {'percentile': None}) + indicator_params = self.params_class(parameters, ('K',), 'K') + with self.assertRaises(params.ValidationError): + indicator_params.validate() + + def test_validate_percentile_out_of_bounds(self): + """ It should raise ValidationError if percentile outside range 1-99 """ + parameters = merge_dicts(self.default_parameters, {'percentile': '0'}) + indicator_params = self.params_class(parameters, ('K',), 'K') + with self.assertRaises(params.ValidationError): + indicator_params.validate() + + parameters = merge_dicts(self.default_parameters, {'percentile': '100'}) + indicator_params = self.params_class(parameters, ('K',), 'K') + with self.assertRaises(params.ValidationError): + indicator_params.validate() diff --git a/django/climate_change_api/indicators/tests/test_indicators.py b/django/climate_change_api/indicators/tests/test_indicators.py index c89ee430..a0f56bfc 100644 --- a/django/climate_change_api/indicators/tests/test_indicators.py +++ b/django/climate_change_api/indicators/tests/test_indicators.py @@ -3,6 +3,7 @@ from climate_data.models import ClimateModel from climate_data.tests.mixins import ClimateDataSetupMixin from indicators import indicators +from indicators.utils import merge_dicts class IndicatorTests(ClimateDataSetupMixin, object): @@ -10,6 +11,7 @@ class IndicatorTests(ClimateDataSetupMixin, object): indicator_class = None # Override this in subclass to set the indicator to test indicator_name = '' units = None + extra_params = {} test_indicator_no_data_equals = {} test_indicator_rcp85_equals = None test_indicator_rcp45_equals = None @@ -24,28 +26,32 @@ def test_indicator_description(self): self.assertTrue(len(self.indicator_class.description) > 0) def test_indicator_rcp85(self): - indicator = self.indicator_class(self.city1, self.rcp85, units=self.units) + params = merge_dicts(self.extra_params, {'units': self.units}) + indicator = self.indicator_class(self.city1, self.rcp85, parameters=params) data = indicator.calculate() self.assertEqual(data, self.test_indicator_rcp85_equals) def test_indicator_rcp45(self): - indicator = self.indicator_class(self.city1, self.rcp45, units=self.units) + params = merge_dicts(self.extra_params, {'units': self.units}) + indicator = self.indicator_class(self.city1, self.rcp45, parameters=params) data = indicator.calculate() self.assertEqual(data, self.test_indicator_rcp45_equals) def test_indicator_no_data(self): - indicator = self.indicator_class(self.city2, self.rcp85, units=self.units) + params = merge_dicts(self.extra_params, {'units': self.units}) + indicator = self.indicator_class(self.city2, self.rcp85, parameters=params) data = indicator.calculate() self.assertEqual(data, self.test_indicator_no_data_equals) def test_years_filter(self): - indicator = self.indicator_class(self.city1, self.rcp45, - units=self.units, years='2001:2002') + params = merge_dicts(self.extra_params, {'units': self.units, 'years': '2001:2002'}) + indicator = self.indicator_class(self.city1, self.rcp45, parameters=params) data = indicator.calculate() self.assertEqual(data, self.test_years_filter_equals) def test_models_filter(self): - indicator = self.indicator_class(self.city1, self.rcp45, units=self.units, models='CCSM4') + params = merge_dicts(self.extra_params, {'units': self.units, 'models': 'CCSM4'}) + indicator = self.indicator_class(self.city1, self.rcp45, parameters=params) data = indicator.calculate() self.assertEqual(data, self.test_models_filter_equals) @@ -60,7 +66,8 @@ def test_unit_conversion_definitions(self): class TemperatureIndicatorTests(IndicatorTests): def test_unit_conversion(self): - indicator = self.indicator_class(self.city1, self.rcp85, units='F') + params = merge_dicts(self.extra_params, {'units': 'F'}) + indicator = self.indicator_class(self.city1, self.rcp85, parameters=params) data = indicator.calculate() self.assertEqual(data, self.test_units_fahrenheit_equals, 'Temperature should be converted to degrees F') @@ -224,6 +231,7 @@ class YearlyDrySpellsTestCase(IndicatorTests, TestCase): class YearlyExtremePrecipitationEventsTestCase(IndicatorTests, TestCase): indicator_class = indicators.YearlyExtremePrecipitationEvents indicator_name = 'yearly_extreme_precipitation_events' + extra_params = {'percentile': '99'} test_indicator_rcp85_equals = {2000: {'avg': 1, 'min': 1, 'max': 1}} test_indicator_rcp45_equals = {2000: {'avg': 0, 'min': 0, 'max': 0}, 2001: {'avg': 0, 'min': 0, 'max': 0}, @@ -240,6 +248,7 @@ class YearlyExtremePrecipitationEventsTestCase(IndicatorTests, TestCase): class YearlyExtremeHeatEventsTestCase(IndicatorTests, TestCase): indicator_class = indicators.YearlyExtremeHeatEvents indicator_name = 'yearly_extreme_heat_events' + extra_params = {'percentile': '99'} test_indicator_rcp85_equals = {2000: {'avg': 1, 'min': 1, 'max': 1}} test_indicator_rcp45_equals = {2000: {'avg': 0, 'min': 0, 'max': 0}, 2001: {'avg': 0, 'min': 0, 'max': 0}, @@ -256,6 +265,7 @@ class YearlyExtremeHeatEventsTestCase(IndicatorTests, TestCase): class YearlyExtremeColdEventsTestCase(IndicatorTests, TestCase): indicator_class = indicators.YearlyExtremeColdEvents indicator_name = 'yearly_extreme_cold_events' + extra_params = {'percentile': '1'} test_indicator_rcp85_equals = {2000: {'avg': 0, 'min': 0, 'max': 0}} test_indicator_rcp45_equals = {2000: {'avg': 0.5, 'min': 0, 'max': 1}, 2001: {'avg': 0.5, 'min': 0, 'max': 1}, @@ -377,6 +387,7 @@ class MonthlyFrostDaysTestCase(IndicatorTests, TestCase): class MonthlyExtremePrecipitationEventsTestCase(IndicatorTests, TestCase): indicator_class = indicators.MonthlyExtremePrecipitationEvents indicator_name = 'monthly_extreme_precipitation_events' + extra_params = {'percentile': '99'} test_indicator_rcp85_equals = {'2000-01': {'avg': 1.0, 'min': 1, 'max': 1}} test_indicator_rcp45_equals = {'2000-01': {'avg': 0.0, 'min': 0, 'max': 0}, '2001-01': {'avg': 0.0, 'min': 0, 'max': 0}, @@ -393,6 +404,7 @@ class MonthlyExtremePrecipitationEventsTestCase(IndicatorTests, TestCase): class MonthlyExtremeHeatEventsTestCase(IndicatorTests, TestCase): indicator_class = indicators.MonthlyExtremeHeatEvents indicator_name = 'monthly_extreme_heat_events' + extra_params = {'percentile': '99'} test_indicator_rcp85_equals = {'2000-01': {'avg': 1.0, 'min': 1, 'max': 1}} test_indicator_rcp45_equals = {'2000-01': {'avg': 0.0, 'min': 0, 'max': 0}, '2001-01': {'avg': 0.0, 'min': 0, 'max': 0}, @@ -409,6 +421,7 @@ class MonthlyExtremeHeatEventsTestCase(IndicatorTests, TestCase): class MonthlyExtremeColdEventsTestCase(IndicatorTests, TestCase): indicator_class = indicators.MonthlyExtremeColdEvents indicator_name = 'monthly_extreme_cold_events' + extra_params = {'percentile': '1'} test_indicator_rcp85_equals = {'2000-01': {'avg': 0.0, 'min': 0, 'max': 0}} test_indicator_rcp45_equals = {'2000-01': {'avg': 0.5, 'min': 0, 'max': 1}, '2001-01': {'avg': 0.5, 'min': 0, 'max': 1}, diff --git a/django/climate_change_api/indicators/utils.py b/django/climate_change_api/indicators/utils.py new file mode 100644 index 00000000..b63c7158 --- /dev/null +++ b/django/climate_change_api/indicators/utils.py @@ -0,0 +1,6 @@ + +def merge_dicts(x, y): + """ Merge two dicts, returning a shallow copy of both with values in y overwriting x """ + z = x.copy() + z.update(y) + return z