diff --git a/README.rst b/README.rst index a8e60773..c39d6ce7 100644 --- a/README.rst +++ b/README.rst @@ -67,8 +67,8 @@ Run migrations:: Load scenario and climate model data:: - ./scripts/console django loaddata scenarios - ./scripts/console django loaddata climate-models + ./scripts/console django './manage.py loaddata scenarios' + ./scripts/console django './manage.py loaddata climate-models' Load cities:: @@ -116,7 +116,7 @@ Loading From Fixture '''''''''''''''''''' To load pre-computed historic aggregated values from the fixture:: - ./scripts/console django loaddata historic_averages historic_baselines + ./scripts/console django './manage.py loaddata historic_averages historic_baselines' Loading From Remote Instance '''''''''''''''''''''''''''' diff --git a/django/climate_change_api/climate_data/views.py b/django/climate_change_api/climate_data/views.py index 993766da..6b8e7f1d 100644 --- a/django/climate_change_api/climate_data/views.py +++ b/django/climate_change_api/climate_data/views.py @@ -3,6 +3,7 @@ from django.contrib.gis.geos import Point from django.core.exceptions import ObjectDoesNotExist +from django.core.exceptions import ValidationError from django.core.urlresolvers import reverse from django.http import HttpResponseRedirect from django.utils.http import urlencode @@ -23,6 +24,7 @@ ScenarioSerializer) from indicators import indicator_factory, list_available_indicators + logger = logging.getLogger(__name__) @@ -258,7 +260,8 @@ def climate_indicator(request, *args, **kwargs): paramType: query - name: time_aggregation description: Time granularity to group data by for result structure. Valid aggregations - depend on indicator. Can be 'yearly', 'monthly' or 'daily' + depend on indicator. Can be 'yearly', 'monthly' or 'daily'. Defaults to + 'yearly'. required: false type: string paramType: query @@ -270,17 +273,24 @@ def climate_indicator(request, *args, **kwargs): paramType: query - name: percentile description: (Appropriate indicators only) The percentile threshold used to calculate - the number of exceeding events compared to historic levels. + the number of exceeding events compared to historic levels. Default depends on + indicator. required: false type: integer paramType: query - name: basetemp description: (Appropriate indicators only) The base temperature used to calculate the daily - difference for degree days sumations. Can be a number with a unit suffix (Such - as 20C) or a bare number (Such as 65) measured in the request output units. + difference for degree days summations. Defaults to 65. See the 'basetemp_units' + for a discussion of the units this value uses. required: false type: integer paramType: query + - name: basetemp_units + description: (Appropriate indicators only) Units for the value of the 'basetemp' parameter. + Defaults to 'F'. + required: false + type: string + paramType: query """ try: @@ -300,40 +310,28 @@ def climate_indicator(request, *args, **kwargs): else: model_list = ClimateModel.objects.all() - agg_param = request.query_params.get('agg', None) - aggregations = agg_param.split(',') if agg_param else None - years_param = request.query_params.get('years', None) - units_param = request.query_params.get('units', None) - time_aggregation = request.query_params.get('time_aggregation', None) - indicator_key = kwargs['indicator'] IndicatorClass = indicator_factory(indicator_key) if not IndicatorClass: raise ParseError(detail='Must provide a valid indicator') - indicator = IndicatorClass(city, - scenario, - models=models_param, - years=years_param, - time_aggregation=time_aggregation, - serializer_aggregations=aggregations, - parameters=request.query_params, - units=units_param) - data = indicator.calculate() - - if units_param and units_param not in IndicatorClass.available_units: - raise NotFound(detail='Cannot convert indicator {} to units {}.'.format(indicator_key, - units_param)) - - if not units_param: - units_param = IndicatorClass.default_units + try: + indicator_class = IndicatorClass(city, scenario, parameters=request.query_params) + except ValidationError as e: + # If indicator class/params fails validation, return error with help text for + # as much context as possible. + return Response(OrderedDict([ + ('error', str(e)), + ('help', IndicatorClass.init_params_class().to_dict()), + ]), status=status.HTTP_400_BAD_REQUEST) + data = indicator_class.calculate() return Response(OrderedDict([ ('city', CitySerializer(city).data), ('scenario', scenario.name), ('indicator', IndicatorClass.to_dict()), ('climate_models', [m.name for m in model_list]), - ('time_aggregation', indicator.time_aggregation), - ('units', units_param), + ('time_aggregation', indicator_class.params.time_aggregation.value), + ('units', indicator_class.params.units.value), ('data', data), ])) diff --git a/django/climate_change_api/indicators/abstract_indicators.py b/django/climate_change_api/indicators/abstract_indicators.py index 32d9f363..e1e25b30 100644 --- a/django/climate_change_api/indicators/abstract_indicators.py +++ b/django/climate_change_api/indicators/abstract_indicators.py @@ -10,6 +10,7 @@ from climate_data.models import ClimateData, ClimateDataSource from climate_data.filters import ClimateDataFilterSet +from .params import IndicatorParams from .serializers import IndicatorSerializer from .unit_converters import DaysUnitsMixin, TemperatureConverter @@ -70,7 +71,6 @@ class Indicator(object): label = '' description = '' - time_aggregation = None # One of 'daily'|'monthly'|'yearly' valid_aggregations = ('yearly', 'monthly', 'daily', ) variables = ClimateData.VARIABLE_CHOICES @@ -92,6 +92,7 @@ class Indicator(object): agg_function = F serializer_class = IndicatorSerializer + params_class = IndicatorParams # Subclasses should use a units mixin from 'unit_converters' to define these units # attributes and any necessary conversion functions @@ -100,8 +101,7 @@ class Indicator(object): available_units = (None,) parameters = None - def __init__(self, city, scenario, models=None, years=None, time_aggregation=None, - serializer_aggregations=None, units=None, parameters=None): + def __init__(self, city, scenario, parameters=None): if not city: raise ValueError('Indicator constructor requires a city instance') if not scenario: @@ -109,31 +109,23 @@ def __init__(self, city, scenario, models=None, years=None, time_aggregation=Non self.city = city self.scenario = scenario - self.models = models - self.years = years - self.serializer_aggregations = serializer_aggregations - - # Set and validate desired units - self.units = units if units is not None else self.default_units - if self.units not in self.available_units: - raise ValueError('Cannot convert to requested units ({})'.format(self.units)) - - if self.parameters is not None: - # Because degree days changes the parameters object, we need to make sure we make a copy - parameters = parameters if parameters is not None else {} - self.parameters = {key: parameters.get(key, default) - for (key, default) in self.parameters.items()} - - self.time_aggregation = (time_aggregation if time_aggregation is not None - else self.valid_aggregations[0]) - if self.time_aggregation not in self.valid_aggregations: - raise ValueError('Cannot aggregate indicator by requested interval ({})' - .format(self.time_aggregation)) + + self.params = self.init_params_class() + self.params.validate(parameters) self.queryset = self.get_queryset() self.serializer = self.serializer_class() + @classmethod + def init_params_class(cls): + """ Return the instantiated IndicatorParams object for this class + + Should not validate the IndicatorParams + + """ + return cls.params_class(cls.default_units, cls.available_units, cls.valid_aggregations) + @classmethod def name(cls): def convert(name): @@ -153,6 +145,7 @@ def to_dict(cls): ('variables', cls.variables), ('available_units', cls.available_units), ('default_units', cls.default_units), + ('parameters', cls.init_params_class().to_dict()), ]) def get_queryset(self): @@ -165,13 +158,13 @@ def get_queryset(self): filter_set = ClimateDataFilterSet() queryset = (ClimateData.objects.filter(map_cell=self.city.map_cell) .filter(data_source__scenario=self.scenario)) - queryset = filter_set.filter_years(queryset, self.years) - queryset = filter_set.filter_models(queryset, self.models) + queryset = filter_set.filter_years(queryset, self.params.years.value) + queryset = filter_set.filter_models(queryset, self.params.models.value) if self.filters is not None: queryset = queryset.filter(**self.filters) - if self.time_aggregation == 'monthly': + if self.params.time_aggregation.value == 'monthly': queryset = queryset.annotate(month=MonthRangeConfig.monthly_case()) return queryset @@ -182,7 +175,7 @@ def aggregate_keys(self): 'daily': ['data_source__year', 'day_of_year', 'data_source__model'], 'monthly': ['data_source__year', 'month', 'data_source__model'], 'yearly': ['data_source__year', 'data_source__model'] - }.get(self.time_aggregation) + }.get(self.params.time_aggregation.value) @property def expression(self): @@ -211,9 +204,9 @@ def convert_units(self, aggregations): @param aggregations list-of-dicts returned by aggregate method @returns Dict in same format as the aggregations parameter, with values converted - to `self.units` + to `self.params.units.value` """ - converter = self.getConverter(self.storage_units, self.units) + converter = self.getConverter(self.storage_units, self.params.units.value) for item in aggregations: if item['value'] is not None: item['value'] = converter(item['value']) @@ -242,14 +235,14 @@ def key_result(self, result): * YYYY-MM-DD for daily data """ year = result['data_source__year'] - if self.time_aggregation == 'yearly': + if self.params.time_aggregation.value == 'yearly': return year - if self.time_aggregation == 'monthly': + if self.params.time_aggregation.value == 'monthly': month = result['month'] return '{year}-{mo:02d}'.format(year=year, mo=(month+1)) - if self.time_aggregation == 'daily': + if self.params.time_aggregation.value == 'daily': day_of_year = result['day_of_year'] day = date(year, 1, 1) + timedelta(days=day_of_year-1) return day.isoformat() @@ -259,7 +252,7 @@ def calculate(self): aggregations = self.convert_units(aggregations) collations = self.collate_results(aggregations) return self.serializer.to_representation(collations, - aggregations=self.serializer_aggregations) + aggregations=self.params.agg.value.split(',')) class CountIndicator(Indicator): @@ -354,18 +347,10 @@ class BasetempIndicatorMixin(object): def __init__(self, *args, **kwargs): super(BasetempIndicatorMixin, self).__init__(*args, **kwargs) - available_units = TemperatureConverter.available_units - m = re.match(r'^(?P-?\d+(\.\d+)?)(?P%s)?$' % '|'.join(available_units), - self.parameters['basetemp']) - if not m: - raise ValueError('Parameter basetemp must be numeric and optionally end with %s' - % str(available_units)) - - value = float(m.group('value')) - unit = m.group('unit') - if unit is None: - unit = self.parameters.get('units', self.default_units) + value = self.params.basetemp.value + basetemp_units = self.params.basetemp_units.value + unit = basetemp_units if basetemp_units is not None else self.params.units.value converter = TemperatureConverter.get(unit, self.storage_units) - self.parameters['basetemp'] = converter(value) - self.parameters['units'] = self.storage_units + self.params.basetemp.value = converter(float(value)) + self.params.basetemp_units.value = self.storage_units diff --git a/django/climate_change_api/indicators/indicators.py b/django/climate_change_api/indicators/indicators.py index f86a49e3..80f74fe0 100644 --- a/django/climate_change_api/indicators/indicators.py +++ b/django/climate_change_api/indicators/indicators.py @@ -6,6 +6,7 @@ from .abstract_indicators import (Indicator, CountIndicator, BasetempIndicatorMixin, YearlyMaxConsecutiveDaysIndicator, YearlySequenceIndicator) +from .params import DegreeDayIndicatorParams, Percentile1IndicatorParams, Percentile99IndicatorParams from .unit_converters import (TemperatureUnitsMixin, PrecipUnitsMixin, PrecipRateUnitsMixin, DaysUnitsMixin, CountUnitsMixin, TemperatureDeltaUnitsMixin, SECONDS_PER_DAY) @@ -98,90 +99,80 @@ def aggregate(self): class ExtremePrecipitationEvents(CountUnitsMixin, CountIndicator): label = 'Extreme Precipitation Events' description = ('Total number of times per period daily precipitation exceeds the specified ' - '(Default 99th) percentile of observations from 1960 to 1995') + 'percentile of observations from 1960 to 1995') valid_aggregations = ('yearly', 'monthly',) + params_class = Percentile99IndicatorParams variables = ('pr',) - parameters = {'percentile': 99} conditions = {'pr__gt': F('map_cell__baseline__pr')} @property def filters(self): - return {'map_cell__baseline__percentile': self.parameters['percentile']} + return {'map_cell__baseline__percentile': self.params.percentile.value} class ExtremeHeatEvents(CountUnitsMixin, CountIndicator): label = 'Extreme Heat Events' description = ('Total number of times per period daily maximum temperature exceeds the ' - 'specified (Default 99th) percentile of observations from 1960 to 1995') + 'specified percentile of observations from 1960 to 1995') + params_class = Percentile99IndicatorParams valid_aggregations = ('yearly', 'monthly',) variables = ('tasmax',) - parameters = {'percentile': 99} conditions = {'tasmax__gt': F('map_cell__baseline__tasmax')} @property def filters(self): - return {'map_cell__baseline__percentile': self.parameters['percentile']} + return {'map_cell__baseline__percentile': self.params.percentile.value} class ExtremeColdEvents(CountUnitsMixin, CountIndicator): label = 'Extreme Cold Events' description = ('Total number of times per period daily minimum temperature is below the ' - 'specified (Default 1st) percentile of observations from 1960 to 1995') + 'specified percentile of observations from 1960 to 1995') + params_class = Percentile1IndicatorParams valid_aggregations = ('yearly', 'monthly',) variables = ('tasmin',) - parameters = {'percentile': 1} conditions = {'tasmin__lt': F('map_cell__baseline__tasmin')} @property def filters(self): - return {'map_cell__baseline__percentile': self.parameters['percentile']} + return {'map_cell__baseline__percentile': self.params.percentile.value} class HeatingDegreeDays(TemperatureDeltaUnitsMixin, BasetempIndicatorMixin, Indicator): label = 'Heating Degree Days' - description = ('Total difference of daily low temperature to a reference base temperature ' - '(Default 65F)') + description = 'Total difference of daily low temperature to a reference base temperature' valid_aggregations = ('yearly', 'monthly',) variables = ('tasmin',) agg_function = Sum - - # List units as a parameter so it gets updated by the query params if it is overriden. - # This way we can fall back to the units param if we need to handle bare numbers for basetemp - parameters = {'basetemp': '65F', - 'units': 'F'} + params_class = DegreeDayIndicatorParams @property def conditions(self): - return {'tasmin__lte': self.parameters['basetemp']} + return {'tasmin__lte': self.params.basetemp.value} @property def expression(self): - return self.parameters['basetemp'] - F('tasmin') + return self.params.basetemp.value - F('tasmin') class CoolingDegreeDays(TemperatureDeltaUnitsMixin, BasetempIndicatorMixin, Indicator): label = 'Cooling Degree Days' - description = ('Total difference of daily high temperature to a reference base temperature ' - '(Default 65F)') + description = 'Total difference of daily high temperature to a reference base temperature ' valid_aggregations = ('yearly', 'monthly',) variables = ('tasmax',) agg_function = Sum - - # List units as a parameter so it gets updated by the query params if it is overriden. - # This way we can fall back to the units param if we need to handle bare numbers for basetemp - parameters = {'basetemp': '65F', - 'units': 'F'} + params_class = DegreeDayIndicatorParams @property def conditions(self): - return {'tasmax__gte': self.parameters['basetemp']} + return {'tasmax__gte': self.params.basetemp.value} @property def expression(self): - return F('tasmax') - self.parameters['basetemp'] + return F('tasmax') - self.params.basetemp.value class HeatWaveDurationIndex(YearlyMaxConsecutiveDaysIndicator): diff --git a/django/climate_change_api/indicators/params.py b/django/climate_change_api/indicators/params.py new file mode 100644 index 00000000..0208da2d --- /dev/null +++ b/django/climate_change_api/indicators/params.py @@ -0,0 +1,176 @@ +from collections import OrderedDict + +from django.core.exceptions import ValidationError + +from .unit_converters import TemperatureConverter +from .validators import ChoicesValidator, float_validator, percentile_range_validator + +MODELS_PARAM_DOCSTRING = "A list of comma separated model names to filter the indicator by. The indicator values in the response will only use the selected models. If not provided, defaults to all models." + +YEARS_PARAM_DOCSTRING = "A list of comma separated year ranges to filter the response by. Defaults to all years available. A year range is of the form 'start[:end]'. Examples: '2010', '2010:2020', '2010:2020,2030', '2010:2020,2030:2040'" + +AGG_PARAM_DOCSTRING = "A list of comma separated aggregation types to return. Valid choices are 'min', 'max', 'avg', 'median', 'stddev', 'stdev', and 'XXth'. If using 'XXth', replace the XX with a number between 1-99 to return that percentile. For example, '99th' returns the value of the 99th percentile. The 'XXth' option can be provided multiple times with different values. 'stdev' is an alias to 'stddev'. Defaults to 'min,max,avg'." + +TIME_AGGREGATION_PARAM_DOCSTRING = "Time granularity to group data by for result structure. Valid aggregations depend on indicator. Can be 'yearly', 'monthly' or 'daily'. Defaults to 'yearly'." + +UNITS_PARAM_DOCSTRING = "Units in which to return the data. Defaults to Imperial units (Fahrenheit for temperature indicators and inches for precipitation)." + +PERCENTILE_PARAM_DOCSTRING = "The percentile threshold used to calculate the number of exceeding events compared to historic levels. Must be an integer in the range [0,100]." + +PERCENTILE_1_PARAM_DOCSTRING = PERCENTILE_PARAM_DOCSTRING + " Defaults to 1." + +PERCENTILE_99_PARAM_DOCSTRING = PERCENTILE_PARAM_DOCSTRING + " Defaults to 99." + +BASETEMP_PARAM_DOCSTRING = "The base temperature used to calculate the daily difference for degree days summations. Defaults to 65. See the 'basetemp_units' for a discussion of the units this value uses." + +BASETEMP_UNITS_PARAM_DOCSTRING = "Units for the value of the 'basetemp' parameter. Defaults to 'F'." + + +class IndicatorParam(object): + """ Defines an individual query parameter for an Indicator request + + @param name: The name of the query parameter + @param description: Human readable descriptive help string describing use of the parameter + @param required: Is this a required parameter? + @param default: Default value to use for parameter if none provided + @param validators: Array of functions or classes implementing the django.core.validators + interface, used to validate the parameter. + + """ + def __init__(self, name, description='', required=True, default=None, validators=None): + self.name = name + self.description = description + self.required = required + self.default = default + self.validators = validators if validators is not None else [] + self.value = None + + def validate(self, value): + """ Validates the parameter by running all defined validators + + Checks if param is required even if no validators are defined. + + Raises django.core.exceptions.ValidationError on the first failed validation check. + + """ + value = value if value is not None else self.default + if value is None and self.required: + raise ValidationError('{} is required.'.format(self.name)) + + for v in self.validators: + v(value) + self.value = value + + def to_dict(self): + """ Return complete representation of this class as a serializable dict + + Makes the IndicatorParam class self documenting. + + """ + description = OrderedDict([ + ('name', self.name), + ('description', self.description), + ('required', self.required), + ]) + if self.default is not None: + description['default'] = self.default + return description + + def __repr__(self): + return str(self.to_dict()) + + +class IndicatorParams(object): + """ Superclass used to define parameters necessary for an Indicator class to function + + Params can be defined either as class or instance variables. Prefer class variables + if the IndicatorParam in question has no run-time dependencies. + + """ + models = IndicatorParam('models', + description=MODELS_PARAM_DOCSTRING, + required=False, + validators=None) + years = IndicatorParam('years', + description=YEARS_PARAM_DOCSTRING, + required=False, + validators=None) + agg = IndicatorParam('agg', + description=AGG_PARAM_DOCSTRING, + required=False, + default='min,max,avg', + validators=None) + + def __init__(self, default_units, available_units, valid_aggregations): + """ Initialize additional params that are instance specific + + Would love a workaround so that we don't have to do this, and can define all params + statically. But, units has defaults/choices that are specific to the indicator and params + we're validating, so we can't do that here. + + """ + available_units_validator = ChoicesValidator(available_units) + valid_aggregations_validator = ChoicesValidator(valid_aggregations) + self.units = IndicatorParam('units', + description=UNITS_PARAM_DOCSTRING, + required=False, + default=default_units, + validators=[available_units_validator]) + self.time_aggregation = IndicatorParam('time_aggregation', + description=TIME_AGGREGATION_PARAM_DOCSTRING, + required=False, + default='yearly', + validators=[valid_aggregations_validator]) + + def validate(self, parameters): + """ Validate all parameters """ + for param_class in self._get_params_classes(): + value = parameters.get(param_class.name, None) + param_class.validate(value) + + def to_dict(self): + return [c.to_dict() for c in self._get_params_classes()] + + def _get_params_classes(self): + """ Return a list of the IndicatorParam instances defined on this class """ + return sorted([getattr(self, x) for x in dir(self) + if isinstance(getattr(self, x), IndicatorParam)], + key=lambda c: c.name) + + def __repr__(self): + return str(self.to_dict()) + + +class Percentile1IndicatorParams(IndicatorParams): + """ Extend IndicatorParams with a 'percentile' parameter, defaulting to 1 """ + percentile = IndicatorParam('percentile', + description=PERCENTILE_1_PARAM_DOCSTRING, + required=False, + default=1, + validators=[percentile_range_validator]) + + +class Percentile99IndicatorParams(IndicatorParams): + """ Extend IndicatorParams with a 'percentile' parameter, defaulting to 99 """ + percentile = IndicatorParam('percentile', + description=PERCENTILE_99_PARAM_DOCSTRING, + required=False, + default=99, + validators=[percentile_range_validator]) + + +class DegreeDayIndicatorParams(IndicatorParams): + + basetemp_units_validator = ChoicesValidator(TemperatureConverter.available_units) + + basetemp = IndicatorParam('basetemp', + description=BASETEMP_PARAM_DOCSTRING, + required=False, + default=65, + validators=[float_validator]) + + basetemp_units = IndicatorParam('basetemp_units', + description=BASETEMP_UNITS_PARAM_DOCSTRING, + required=False, + default='F', + validators=[basetemp_units_validator]) diff --git a/django/climate_change_api/indicators/tests/test_indicator_params.py b/django/climate_change_api/indicators/tests/test_indicator_params.py new file mode 100644 index 00000000..43400093 --- /dev/null +++ b/django/climate_change_api/indicators/tests/test_indicator_params.py @@ -0,0 +1,120 @@ +from django.core.exceptions import ValidationError +from django.test import TestCase + +from indicators.params import IndicatorParam, IndicatorParams, Percentile99IndicatorParams +from indicators.utils import merge_dicts +from indicators.validators import ChoicesValidator + + +available_units_validator = ChoicesValidator(['K', 'C', 'F']) + + +class IndicatorParamTestCase(TestCase): + + def test_sets_default(self): + param = IndicatorParam('units', default='F') + param.validate('K') + self.assertEqual(param.value, 'K') + + param = IndicatorParam('units', default='F') + param.validate(None) + self.assertEqual(param.value, 'F') + + def test_required_with_default(self): + param = IndicatorParam('units', required=True, default='F') + param.validate(None) + self.assertEqual(param.value, 'F') + + def test_required_no_default(self): + param = IndicatorParam('units', required=True, default=None) + with self.assertRaises(ValidationError): + param.validate(None) + + def test_blank_validators(self): + """ should set validators to empty array if None provided """ + param = IndicatorParam('units', validators=None) + self.assertEqual(param.validators, []) + param = IndicatorParam('units') + self.assertEqual(param.validators, []) + + +class IndicatorParamsTestCase(TestCase): + + def setUp(self): + self.default_parameters = {} + self.default_units = 'F' + self.available_units = ('F', 'C', 'K',) + self.valid_aggregations = ('yearly', 'monthly', 'daily',) + self.params_class = IndicatorParams + + def _get_params_class(self): + return self.params_class(self.default_units, self.available_units, self.valid_aggregations) + + def test_validate_valid_only_expected_params(self): + """ it should ensure indicator_params sets values for each of the base params """ + parameters = merge_dicts(self.default_parameters, + {'models': 'CCSM4', 'years': '2050:2060', 'units': 'K', + 'agg': 'avg', 'time_aggregation': 'monthly'}) + indicator_params = self._get_params_class() + indicator_params.validate(parameters) + self.assertEqual(indicator_params.models.value, 'CCSM4') + self.assertEqual(indicator_params.years.value, '2050:2060') + self.assertEqual(indicator_params.units.value, 'K') + self.assertEqual(indicator_params.agg.value, 'avg') + self.assertEqual(indicator_params.time_aggregation.value, 'monthly') + + def test_validate_valid_some_unused_params(self): + """ it should ensure indicator_params properly ignores params that aren't defined """ + parameters = merge_dicts(self.default_parameters, + {'doesnotexist': 'true', 'years': '2050:2060', 'units': 'K', 'agg': 'avg'}) + indicator_params = self._get_params_class() + indicator_params.validate(parameters) + with self.assertRaises(AttributeError): + value = indicator_params.doesnotexist + + def test_validate_valid_optional_defaults(self): + """ it should ensure indicator_params properly sets defaults on base params + + Force units to have default value by overriding IndicatorParams + + """ + parameters_sets = [ + {'models': None, 'years': None, 'units': None, 'agg': None, 'time_aggregation': None}, + {} + ] + for parameters in parameters_sets: + p = merge_dicts(self.default_parameters, parameters) + indicator_params = self._get_params_class() + indicator_params.validate(p) + self.assertIsNone(indicator_params.years.value) + self.assertIsNone(indicator_params.models.value) + self.assertEqual(indicator_params.agg.value, indicator_params.agg.default) + self.assertEqual(indicator_params.units.value, self.default_units) + self.assertEqual(indicator_params.time_aggregation.value, + indicator_params.time_aggregation.default) + + +class PercentileIndicatorParamsTestCase(IndicatorParamsTestCase): + + def setUp(self): + super(PercentileIndicatorParamsTestCase, self).setUp() + self.params_class = Percentile99IndicatorParams + + def test_validate_percentile_default(self): + """ It should use a default value if required param is missing """ + parameters = {'percentile': None} + indicator_params = self._get_params_class() + indicator_params.validate(parameters) + self.assertEqual(indicator_params.percentile.value, 99) + + def test_validate_percentile_out_of_bounds(self): + """ It should raise ValidationError if percentile outside range [0-100] """ + parameters = merge_dicts(self.default_parameters, {'percentile': '-1'}) + indicator_params = self._get_params_class() + with self.assertRaises(ValidationError): + indicator_params.validate(parameters) + + parameters = merge_dicts(self.default_parameters, {'percentile': '101'}) + indicator_params = self._get_params_class() + with self.assertRaises(ValidationError): + indicator_params.validate(parameters) diff --git a/django/climate_change_api/indicators/tests/test_indicators.py b/django/climate_change_api/indicators/tests/test_indicators.py index 771f87ee..c12c6e60 100644 --- a/django/climate_change_api/indicators/tests/test_indicators.py +++ b/django/climate_change_api/indicators/tests/test_indicators.py @@ -1,17 +1,17 @@ from django.test import TestCase -from climate_data.models import ClimateModel from climate_data.tests.mixins import ClimateDataSetupMixin from indicators import indicators +from indicators.utils import merge_dicts class IndicatorTests(ClimateDataSetupMixin, object): indicator_class = None # Override this in subclass to set the indicator to test - parameters = None indicator_name = '' units = None time_aggregation = None + extra_params = {} test_indicator_no_data_equals = {} test_indicator_rcp85_equals = None test_indicator_rcp45_equals = None @@ -26,34 +26,37 @@ def test_indicator_description(self): self.assertTrue(len(self.indicator_class.description) > 0) def test_indicator_rcp85(self): - indicator = self.indicator_class(self.city1, self.rcp85, parameters=self.parameters, - units=self.units, time_aggregation=self.time_aggregation) + params = merge_dicts(self.extra_params, + {'units': self.units, 'time_aggregation': self.time_aggregation}) + indicator = self.indicator_class(self.city1, self.rcp85, parameters=params) data = indicator.calculate() self.assertEqual(data, self.test_indicator_rcp85_equals) def test_indicator_rcp45(self): - indicator = self.indicator_class(self.city1, self.rcp45, parameters=self.parameters, - units=self.units, time_aggregation=self.time_aggregation) + params = merge_dicts(self.extra_params, + {'units': self.units, 'time_aggregation': self.time_aggregation}) + indicator = self.indicator_class(self.city1, self.rcp45, parameters=params) data = indicator.calculate() self.assertEqual(data, self.test_indicator_rcp45_equals) def test_indicator_no_data(self): - indicator = self.indicator_class(self.city2, self.rcp85, parameters=self.parameters, - units=self.units, time_aggregation=self.time_aggregation) + params = merge_dicts(self.extra_params, + {'units': self.units, 'time_aggregation': self.time_aggregation}) + indicator = self.indicator_class(self.city2, self.rcp85, parameters=params) data = indicator.calculate() self.assertEqual(data, self.test_indicator_no_data_equals) def test_years_filter(self): - indicator = self.indicator_class(self.city1, self.rcp45, parameters=self.parameters, - units=self.units, time_aggregation=self.time_aggregation, - years='2001:2002') + params = merge_dicts(self.extra_params, {'units': self.units, 'years': '2001:2002', + 'time_aggregation': self.time_aggregation}) + indicator = self.indicator_class(self.city1, self.rcp45, parameters=params) data = indicator.calculate() self.assertEqual(data, self.test_years_filter_equals) def test_models_filter(self): - indicator = self.indicator_class(self.city1, self.rcp45, models='CCSM4', - parameters=self.parameters, units=self.units, - time_aggregation=self.time_aggregation) + params = merge_dicts(self.extra_params, {'units': self.units, 'models': 'CCSM4', + 'time_aggregation': self.time_aggregation}) + indicator = self.indicator_class(self.city1, self.rcp45, parameters=params) data = indicator.calculate() self.assertEqual(data, self.test_models_filter_equals) @@ -64,8 +67,9 @@ def test_unit_conversion_definitions(self): class TemperatureIndicatorTests(IndicatorTests): def test_unit_conversion(self): - indicator = self.indicator_class(self.city1, self.rcp85, - time_aggregation=self.time_aggregation, units='F') + params = merge_dicts(self.extra_params, + {'units': 'F', 'time_aggregation': self.time_aggregation}) + indicator = self.indicator_class(self.city1, self.rcp85, parameters=params) data = indicator.calculate() self.assertEqual(data, self.test_units_fahrenheit_equals, 'Temperature should be converted to degrees F') @@ -238,6 +242,7 @@ class YearlyDrySpellsTestCase(IndicatorTests, TestCase): class YearlyExtremePrecipitationEventsTestCase(IndicatorTests, TestCase): indicator_class = indicators.ExtremePrecipitationEvents indicator_name = 'extreme_precipitation_events' + extra_params = {'percentile': '99'} time_aggregation = 'yearly' test_indicator_rcp85_equals = {2000: {'avg': 1, 'min': 1, 'max': 1}} test_indicator_rcp45_equals = {2000: {'avg': 0, 'min': 0, 'max': 0}, @@ -255,6 +260,7 @@ class YearlyExtremePrecipitationEventsTestCase(IndicatorTests, TestCase): class YearlyExtremeHeatEventsTestCase(IndicatorTests, TestCase): indicator_class = indicators.ExtremeHeatEvents indicator_name = 'extreme_heat_events' + extra_params = {'percentile': '99'} time_aggregation = 'yearly' test_indicator_rcp85_equals = {2000: {'avg': 1, 'min': 1, 'max': 1}} test_indicator_rcp45_equals = {2000: {'avg': 0, 'min': 0, 'max': 0}, @@ -272,6 +278,7 @@ class YearlyExtremeHeatEventsTestCase(IndicatorTests, TestCase): class YearlyExtremeColdEventsTestCase(IndicatorTests, TestCase): indicator_class = indicators.ExtremeColdEvents indicator_name = 'extreme_cold_events' + extra_params = {'percentile': '1'} time_aggregation = 'yearly' test_indicator_rcp85_equals = {2000: {'avg': 0, 'min': 0, 'max': 0}} test_indicator_rcp45_equals = {2000: {'avg': 0.5, 'min': 0, 'max': 1}, @@ -290,8 +297,9 @@ class YearlyHeatingDegreeDaysTestCase(IndicatorTests, TestCase): indicator_class = indicators.HeatingDegreeDays indicator_name = 'heating_degree_days' time_aggregation = 'yearly' - parameters = { - 'basetemp': '42.5K' + extra_params = { + 'basetemp': '42.5', + 'basetemp_units': 'K' } test_indicator_rcp85_equals = {2000: {'avg': 13.5, 'min': 4.5, 'max': 22.5}} test_indicator_rcp45_equals = {2000: {'avg': 49.5, 'min': 40.5, 'max': 58.5}, @@ -310,8 +318,9 @@ class YearlyCoolingDegreeDaysTestCase(IndicatorTests, TestCase): indicator_class = indicators.CoolingDegreeDays indicator_name = 'cooling_degree_days' time_aggregation = 'yearly' - parameters = { - 'basetemp': '-265.85C' # 7.3K + extra_params = { + 'basetemp': '-265.85', # 7.3K + 'basetemp_units': 'C' } test_indicator_rcp85_equals = {2000: {'avg': 49.86, 'min': 40.86, 'max': 58.86000000000001}} test_indicator_rcp45_equals = {2000: {'avg': 13.860000000000046, 'min': 4.86000000000009, @@ -450,6 +459,7 @@ class MonthlyFrostDaysTestCase(IndicatorTests, TestCase): class MonthlyExtremePrecipitationEventsTestCase(IndicatorTests, TestCase): indicator_class = indicators.ExtremePrecipitationEvents indicator_name = 'extreme_precipitation_events' + extra_params = {'percentile': '99'} time_aggregation = 'monthly' test_indicator_rcp85_equals = {'2000-01': {'avg': 1.0, 'min': 1, 'max': 1}} test_indicator_rcp45_equals = {'2000-01': {'avg': 0.0, 'min': 0, 'max': 0}, @@ -467,6 +477,7 @@ class MonthlyExtremePrecipitationEventsTestCase(IndicatorTests, TestCase): class MonthlyExtremeHeatEventsTestCase(IndicatorTests, TestCase): indicator_class = indicators.ExtremeHeatEvents indicator_name = 'extreme_heat_events' + extra_params = {'percentile': '99'} time_aggregation = 'monthly' test_indicator_rcp85_equals = {'2000-01': {'avg': 1.0, 'min': 1, 'max': 1}} test_indicator_rcp45_equals = {'2000-01': {'avg': 0.0, 'min': 0, 'max': 0}, @@ -484,6 +495,7 @@ class MonthlyExtremeHeatEventsTestCase(IndicatorTests, TestCase): class MonthlyExtremeColdEventsTestCase(IndicatorTests, TestCase): indicator_class = indicators.ExtremeColdEvents indicator_name = 'extreme_cold_events' + extra_params = {'percentile': '1'} time_aggregation = 'monthly' test_indicator_rcp85_equals = {'2000-01': {'avg': 0.0, 'min': 0, 'max': 0}} test_indicator_rcp45_equals = {'2000-01': {'avg': 0.5, 'min': 0, 'max': 1}, @@ -502,9 +514,9 @@ class MonthlyHeatingDegreeDaysTestCase(IndicatorTests, TestCase): indicator_class = indicators.HeatingDegreeDays indicator_name = 'heating_degree_days' time_aggregation = 'monthly' - parameters = { - 'basetemp': '-384.07', - 'units': 'F' + units = 'F' + extra_params = { + 'basetemp': '-384.07' } test_indicator_rcp85_equals = {'2000-01': {'avg': 12.60000000000001, 'min': 3.6000000000000183, 'max': 21.6}} @@ -524,18 +536,19 @@ class MonthlyCoolingDegreeDaysTestCase(IndicatorTests, TestCase): indicator_class = indicators.CoolingDegreeDays indicator_name = 'cooling_degree_days' time_aggregation = 'monthly' - parameters = { + units = 'C' + extra_params = { 'basetemp': '-273.15', - 'units': 'C' + 'basetemp_units': 'C' } - test_indicator_rcp85_equals = {'2000-01': {'avg': 63.0, 'min': 54.0, 'max': 72.0}} - test_indicator_rcp45_equals = {'2000-01': {'avg': 27.0, 'min': 18.0, 'max': 36.0}, - '2001-01': {'avg': 27.0, 'min': 18.0, 'max': 36.0}, - '2002-01': {'avg': 18.0, 'min': 18.0, 'max': 18.0}, - '2003-01': {'avg': 18.0, 'min': 18.0, 'max': 18.0}} - test_years_filter_equals = {'2001-01': {'avg': 27.0, 'min': 18.0, 'max': 36.0}, - '2002-01': {'avg': 18.0, 'min': 18.0, 'max': 18.0}} - test_models_filter_equals = {'2000-01': {'avg': 18.0, 'min': 18.0, 'max': 18.0}, - '2001-01': {'avg': 18.0, 'min': 18.0, 'max': 18.0}, - '2002-01': {'avg': 18.0, 'min': 18.0, 'max': 18.0}, - '2003-01': {'avg': 18.0, 'min': 18.0, 'max': 18.0}} + test_indicator_rcp85_equals = {'2000-01': {'max': 40.0, 'avg': 35.0, 'min': 30.0}} + test_indicator_rcp45_equals = {'2000-01': {'max': 20.0, 'avg': 15.0, 'min': 10.0}, + '2003-01': {'max': 10.0, 'avg': 10.0, 'min': 10.0}, + '2001-01': {'max': 20.0, 'avg': 15.0, 'min': 10.0}, + '2002-01': {'max': 10.0, 'avg': 10.0, 'min': 10.0}} + test_years_filter_equals = {'2001-01': {'max': 20.0, 'avg': 15.0, 'min': 10.0}, + '2002-01': {'max': 10.0, 'avg': 10.0, 'min': 10.0}} + test_models_filter_equals = {'2000-01': {'max': 10.0, 'avg': 10.0, 'min': 10.0}, + '2003-01': {'max': 10.0, 'avg': 10.0, 'min': 10.0}, + '2001-01': {'max': 10.0, 'avg': 10.0, 'min': 10.0}, + '2002-01': {'max': 10.0, 'avg': 10.0, 'min': 10.0}} diff --git a/django/climate_change_api/indicators/tests/test_validators.py b/django/climate_change_api/indicators/tests/test_validators.py new file mode 100644 index 00000000..9f9d013f --- /dev/null +++ b/django/climate_change_api/indicators/tests/test_validators.py @@ -0,0 +1,76 @@ +from django.core.exceptions import ValidationError +from django.test import TestCase + +from indicators.validators import ChoicesValidator, IntRangeValidator + +class ValidatorTestCase(TestCase): + + def should_succeed_with_value(self, validator, value): + """ Helper method to raise failure if validator(value) should succeed but fails """ + try: + validator(value) + except ValidationError as e: + self.fail(str(e)) + + +class IntRangeValidatorTestCase(ValidatorTestCase): + + def test_valid(self): + v = IntRangeValidator(minimum=0, maximum=5) + self.should_succeed_with_value(v, 3) + + def test_handle_strings(self): + v = IntRangeValidator(minimum=0, maximum=5) + self.should_succeed_with_value(v, '3') + with self.assertRaises(ValidationError): + v('notanumber') + + def test_range_edges(self): + v = IntRangeValidator(minimum=0, maximum=5) + self.should_succeed_with_value(v, 0) + self.should_succeed_with_value(v, 5) + + def test_out_of_range(self): + v = IntRangeValidator(minimum=0, maximum=5) + with self.assertRaises(ValidationError): + v(-1) + with self.assertRaises(ValidationError): + v(6) + + +class ChoicesValidatorTestCase(ValidatorTestCase): + + def setUp(self): + self.default_choices = ('a', 'b', 'c',) + + def test_valid(self): + v = ChoicesValidator(self.default_choices) + self.should_succeed_with_value(v, 'a') + + def test_invalid(self): + v = ChoicesValidator(self.default_choices) + with self.assertRaises(ValidationError): + v('notinchoices') + + def test_int_types(self): + v = ChoicesValidator((1, 2, 3,)) + self.should_succeed_with_value(v, 1) + with self.assertRaises(ValidationError): + v(5) + + def test_null_allowed(self): + v = ChoicesValidator(self.default_choices, is_null_allowed=True) + self.should_succeed_with_value(v, None) + + v = ChoicesValidator(self.default_choices, is_null_allowed=False) + with self.assertRaises(ValidationError): + v(None) + + def test_mixed_types(self): + v = ChoicesValidator(('a', 1,)) + self.should_succeed_with_value(v, 'a') + self.should_succeed_with_value(v, 1) + with self.assertRaises(ValidationError): + v('b') + with self.assertRaises(ValidationError): + v(5) diff --git a/django/climate_change_api/indicators/utils.py b/django/climate_change_api/indicators/utils.py new file mode 100644 index 00000000..b63c7158 --- /dev/null +++ b/django/climate_change_api/indicators/utils.py @@ -0,0 +1,6 @@ + +def merge_dicts(x, y): + """ Merge two dicts, returning a shallow copy of both with values in y overwriting x """ + z = x.copy() + z.update(y) + return z diff --git a/django/climate_change_api/indicators/validators.py b/django/climate_change_api/indicators/validators.py new file mode 100644 index 00000000..34662958 --- /dev/null +++ b/django/climate_change_api/indicators/validators.py @@ -0,0 +1,58 @@ +from django.core.exceptions import ValidationError + + +class IntRangeValidator(object): + """ Validator which verifies that value is an integer and checks it against a range + + The range is inclusive, i.e. the provided minimum and maximum values are considered valid. + + """ + def __init__(self, minimum=0, maximum=100): + self.minimum = minimum + self.maximum = maximum + + def __call__(self, value): + try: + int_value = int(value) + except (TypeError, ValueError) as e: + raise ValidationError('{} is not an integer'.format(value)) + if int_value < self.minimum or int_value > self.maximum: + raise ValidationError('{} must be in the range [{}, {}]' + .format(value, self.minimum, self.maximum)) + + +class TypeClassValidator(object): + """ Validates that a value is of a particular type + + Argument type class should be a callable that takes a value and returns the value coerced + to the desired type. If coercion fails, should raise a TypeError or ValueError. + + """ + def __init__(self, type_class): + self.type_class = type_class + + def __call__(self, value): + try: + typed_value = self.type_class(value) + except (TypeError, ValueError) as e: + raise ValidationError('{} is not {}'.format(value, self.type_class)) + + +class ChoicesValidator(object): + """ Validator that checks to ensure a value is one of a limited set of options """ + def __init__(self, choices, is_null_allowed=False): + self.choices = choices + self.is_null_allowed = is_null_allowed + + def __call__(self, value): + if value not in self.choices and not self.is_null_allowed: + raise ValidationError('{} must be one of {}'.format(value, self.choices)) + + +percentile_range_validator = IntRangeValidator(minimum=0, maximum=100) + + +integer_validator = TypeClassValidator(int) + + +float_validator = TypeClassValidator(float)