diff --git a/django/climate_change_api/climate_data/views.py b/django/climate_change_api/climate_data/views.py index 18364205..f939aac5 100644 --- a/django/climate_change_api/climate_data/views.py +++ b/django/climate_change_api/climate_data/views.py @@ -267,6 +267,13 @@ def climate_indicator(request, *args, **kwargs): required: false type: integer paramType: query + - name: basetemp + description: (Appropriate indicators only) The base temperature used to calculate the daily + difference for degree days sumations. Can be a number with a unit suffix (Such + as 20C) or a bare number (Such as 65) measured in the request output units. + required: false + type: integer + paramType: query """ try: diff --git a/django/climate_change_api/indicators/abstract_indicators.py b/django/climate_change_api/indicators/abstract_indicators.py index d495cb46..1828fbf5 100644 --- a/django/climate_change_api/indicators/abstract_indicators.py +++ b/django/climate_change_api/indicators/abstract_indicators.py @@ -11,7 +11,7 @@ from climate_data.models import ClimateData, ClimateDataSource from climate_data.filters import ClimateDataFilterSet from .serializers import IndicatorSerializer -from .unit_converters import DaysUnitsMixin +from .unit_converters import DaysUnitsMixin, TemperatureConverter class Indicator(object): @@ -174,24 +174,44 @@ def row_group_key(self, row): class YearlyAggregationIndicator(YearlyIndicator): + default = 0 + conditions = None + def aggregate(self): + if self.conditions: + agg_function = self.agg_function( + Case(When(then=self.expression, **self.conditions), + default=self.default, + output_field=IntegerField())) + else: + agg_function = self.agg_function(self.expression) + return (self.queryset.values('data_source__year', 'data_source__model') - .annotate(value=self.agg_function(self.variables[0]))) + .annotate(value=agg_function)) + + @property + def expression(self): + """ Lookup function to get the actual value used for aggregation + + Defaults to the first variable mentioned in the variables variable, but can be overloaded + for complex queries. + + This is necessary because the variables value is serialized for the /indicators/ endpoint, + but complex queries may require non-serializable components. Using the expression property + allows us to subsitute the variable specifically for the serialization instead. + """ + return self.variables[0] -class YearlyCountIndicator(YearlyIndicator): +class YearlyCountIndicator(YearlyAggregationIndicator): """ Class to count days on which a condition is met. - Since using a filter would result in ignoring year/model combinations where the count is zero - and Count doesn't discriminate between values, uses a Case/When clause to return 1 for hits - and 0 for misses then Sum to count them up. + Essentially a specialized version of the YearlyAggregationIndicator where all values count as 1 + if they match the conditions and 0 in all other cases. """ - def aggregate(self): - agg_function = Sum(Case(When(then=1, **self.conditions), - default=0, - output_field=IntegerField())) - return (self.queryset.values('data_source__year', 'data_source__model') - .annotate(value=agg_function)) + agg_function = Sum + default = 0 + expression = 1 class YearlySequenceIndicator(YearlyCountIndicator): @@ -349,13 +369,54 @@ def key_result(self, result): class MonthlyAggregationIndicator(MonthlyIndicator): + default = 0 + conditions = None + def aggregate(self): - return self.monthly_queryset.annotate(value=self.agg_function(self.variables[0])) + if self.conditions: + agg_function = self.agg_function( + Case(When(then=self.expression, **self.conditions), + default=self.default, + output_field=IntegerField())) + else: + agg_function = self.agg_function(self.expression) + + return (self.monthly_queryset.annotate(value=agg_function)) + + @property + def expression(self): + """ Lookup function to get the actual value used for aggregation + + Defaults to the first variable mentioned in the variables variable, but can be overloaded + for complex queries. + + This is necessary because the variables value is serialized for the /indicators/ endpoint, + but complex queries may require non-serializable components. Using the expression property + allows us to subsitute the variable specifically for the serialization instead. + """ + return self.variables[0] class MonthlyCountIndicator(MonthlyAggregationIndicator): - def aggregate(self): - agg_function = Sum(Case(When(then=1, **self.conditions), - default=0, - output_field=IntegerField())) - return self.monthly_queryset.annotate(value=agg_function) + agg_function = Sum + default = 0 + expression = 1 + + +class BasetempIndicatorMixin(object): + """ Framework for pre-processing the basetemp parameter to a native unit + """ + def calculate(self): + m = re.match(r'(?P\d+(\.\d+)?)(?P[FKC])?', self.parameters['basetemp']) + assert m, "Parameter basetemp must be numeric" + + value = float(m.group('value')) + unit = m.group('unit') + if unit is None: + unit = self.parameters.get('units', self.default_units) + + converter = TemperatureConverter.get(unit, self.storage_units) + self.parameters['basetemp'] = converter(value) + self.parameters['units'] = self.storage_units + + return super(BasetempIndicatorMixin, self).calculate() diff --git a/django/climate_change_api/indicators/indicators.py b/django/climate_change_api/indicators/indicators.py index 6acb4e10..98cb632f 100644 --- a/django/climate_change_api/indicators/indicators.py +++ b/django/climate_change_api/indicators/indicators.py @@ -2,14 +2,14 @@ import sys from itertools import groupby -from django.db.models import F, Avg, Max, Min +from django.db.models import F, Sum, Avg, Max, Min from .abstract_indicators import (YearlyAggregationIndicator, YearlyCountIndicator, YearlyMaxConsecutiveDaysIndicator, YearlySequenceIndicator, MonthlyAggregationIndicator, MonthlyCountIndicator, - DailyRawIndicator) -from .unit_converters import (TemperatureUnitsMixin, PrecipUnitsMixin, - DaysUnitsMixin, CountUnitsMixin) + DailyRawIndicator, BasetempIndicatorMixin) +from .unit_converters import (TemperatureUnitsMixin, PrecipUnitsMixin, DaysUnitsMixin, + CountUnitsMixin, TemperatureDeltaUnitsMixin) ########################## @@ -96,10 +96,11 @@ class YearlyExtremePrecipitationEvents(CountUnitsMixin, YearlyCountIndicator): variables = ('pr',) parameters = {'percentile': 99} + conditions = {'pr__gt': F('map_cell__baseline__pr')} + @property - def conditions(self): - return {'pr__gt': F('map_cell__baseline__pr'), - 'map_cell__baseline__percentile': self.parameters['percentile']} + def filters(self): + return {'map_cell__baseline__percentile': self.parameters['percentile']} class YearlyExtremeHeatEvents(CountUnitsMixin, YearlyCountIndicator): @@ -109,10 +110,11 @@ class YearlyExtremeHeatEvents(CountUnitsMixin, YearlyCountIndicator): variables = ('tasmax',) parameters = {'percentile': 99} + conditions = {'tasmax__gt': F('map_cell__baseline__tasmax')} + @property - def conditions(self): - return {'tasmax__gt': F('map_cell__baseline__tasmax'), - 'map_cell__baseline__percentile': self.parameters['percentile']} + def filters(self): + return {'map_cell__baseline__percentile': self.parameters['percentile']} class YearlyExtremeColdEvents(CountUnitsMixin, YearlyCountIndicator): @@ -122,10 +124,55 @@ class YearlyExtremeColdEvents(CountUnitsMixin, YearlyCountIndicator): variables = ('tasmin',) parameters = {'percentile': 1} + conditions = {'tasmin__lt': F('map_cell__baseline__tasmin')} + + @property + def filters(self): + return {'map_cell__baseline__percentile': self.parameters['percentile']} + + +class YearlyHeatingDegreeDays(TemperatureDeltaUnitsMixin, BasetempIndicatorMixin, + YearlyAggregationIndicator): + label = 'Yearly Heating Degree Days' + description = ('Total difference of daily low temperature to a reference base temperature ' + '(Default 65F)') + variables = ('tasmin',) + agg_function = Sum + + # List units as a parameter so it gets updated by the query params if it is overriden. + # This way we can fall back to the units param if we need to handle bare numbers for basetemp + parameters = {'basetemp': '65F', + 'units': 'F'} + @property def conditions(self): - return {'tasmin__lt': F('map_cell__baseline__tasmin'), - 'map_cell__baseline__percentile': self.parameters['percentile']} + return {'tasmin__lte': self.parameters['basetemp']} + + @property + def expression(self): + return self.parameters['basetemp'] - F('tasmin') + + +class YearlyCoolingDegreeDays(TemperatureDeltaUnitsMixin, BasetempIndicatorMixin, + YearlyAggregationIndicator): + label = 'Yearly Cooling Degree Days' + description = ('Total difference of daily high temperature to a reference base temperature ' + '(Default 65F)') + variables = ('tasmax',) + agg_function = Sum + + # List units as a parameter so it gets updated by the query params if it is overriden. + # This way we can fall back to the units param if we need to handle bare numbers for basetemp + parameters = {'basetemp': '65F', + 'units': 'F'} + + @property + def conditions(self): + return {'tasmax__gte': self.parameters['basetemp']} + + @property + def expression(self): + return F('tasmax') - self.parameters['basetemp'] class HeatWaveDurationIndex(YearlyMaxConsecutiveDaysIndicator): @@ -198,10 +245,11 @@ class MonthlyExtremePrecipitationEvents(CountUnitsMixin, MonthlyCountIndicator): variables = ('pr',) parameters = {'percentile': 99} + conditions = {'pr__gt': F('map_cell__baseline__pr')} + @property - def conditions(self): - return {'pr__gt': F('map_cell__baseline__pr'), - 'map_cell__baseline__percentile': self.parameters['percentile']} + def filters(self): + return {'map_cell__baseline__percentile': self.parameters['percentile']} class MonthlyExtremeHeatEvents(CountUnitsMixin, MonthlyCountIndicator): @@ -211,10 +259,11 @@ class MonthlyExtremeHeatEvents(CountUnitsMixin, MonthlyCountIndicator): variables = ('tasmax',) parameters = {'percentile': 99} + conditions = {'tasmax__gt': F('map_cell__baseline__tasmax')} + @property - def conditions(self): - return {'tasmax__gt': F('map_cell__baseline__tasmax'), - 'map_cell__baseline__percentile': self.parameters['percentile']} + def filters(self): + return {'map_cell__baseline__percentile': self.parameters['percentile']} class MonthlyExtremeColdEvents(CountUnitsMixin, MonthlyCountIndicator): @@ -224,11 +273,55 @@ class MonthlyExtremeColdEvents(CountUnitsMixin, MonthlyCountIndicator): variables = ('tasmin',) parameters = {'percentile': 1} + conditions = {'tasmin__lt': F('map_cell__baseline__tasmin')} + + @property + def filters(self): + return {'map_cell__baseline__percentile': self.parameters['percentile']} + + +class MonthlyHeatingDegreeDays(TemperatureDeltaUnitsMixin, BasetempIndicatorMixin, + MonthlyAggregationIndicator): + label = 'Monthly Heating Degree Days' + description = ('Total difference of daily low temperature to a reference base temperature ' + '(Default 65F)') + variables = ('tasmin',) + agg_function = Sum + + # List units as a parameter so it gets updated by the query params if it is overriden. + # This way we can fall back to the units param if we need to handle bare numbers for basetemp + parameters = {'basetemp': '65F', + 'units': 'F'} + @property def conditions(self): - return {'tasmin__lt': F('map_cell__baseline__tasmin'), - 'map_cell__baseline__percentile': self.parameters['percentile']} + return {'tasmin__lte': self.parameters['basetemp']} + @property + def expression(self): + return self.parameters['basetemp'] - F('tasmin') + + +class MonthlyCoolingDegreeDays(TemperatureDeltaUnitsMixin, BasetempIndicatorMixin, + MonthlyAggregationIndicator): + label = 'Monthly Cooling Degree Days' + description = ('Total difference of daily high temperature to a reference base temperature ' + '(Default 65F)') + variables = ('tasmax',) + agg_function = Sum + + # List units as a parameter so it gets updated by the query params if it is overriden. + # This way we can fall back to the units param if we need to handle bare numbers for basetemp + parameters = {'basetemp': '65F', + 'units': 'F'} + + @property + def conditions(self): + return {'tasmax__gte': self.parameters['basetemp']} + + @property + def expression(self): + return F('tasmax') - self.parameters['basetemp'] ########################## diff --git a/django/climate_change_api/indicators/tests/mixins.py b/django/climate_change_api/indicators/tests/mixins.py new file mode 100644 index 00000000..b47b0a54 --- /dev/null +++ b/django/climate_change_api/indicators/tests/mixins.py @@ -0,0 +1,11 @@ +class ConverterTestMixin(): + def test_conversions(self): + for units, tests in self.cases.iteritems(): + converter = self.converter_class.get(*units) + + for start, expected in tests: + value = converter(start) + self.assertAlmostEqual(value, expected, + places=3, + msg='Failed assertion that %f %s = %f %s, got %f %s' % + (start, units[0], expected, units[1], value, units[1])) diff --git a/django/climate_change_api/indicators/tests/test_indicators.py b/django/climate_change_api/indicators/tests/test_indicators.py index c89ee430..bf6ecd4b 100644 --- a/django/climate_change_api/indicators/tests/test_indicators.py +++ b/django/climate_change_api/indicators/tests/test_indicators.py @@ -49,14 +49,6 @@ def test_models_filter(self): data = indicator.calculate() self.assertEqual(data, self.test_models_filter_equals) - def test_unit_conversion_definitions(self): - """ Some sanity checks for unit conversion class attributes """ - self.assertIn(self.indicator_class.default_units, self.indicator_class.available_units) - storage_units = self.indicator_class.storage_units - for units in self.indicator_class.available_units: - self.assertTrue(units == storage_units or callable( - self.indicator_class.conversions[self.indicator_class.storage_units][units])) - class TemperatureIndicatorTests(IndicatorTests): def test_unit_conversion(self): diff --git a/django/climate_change_api/indicators/tests/test_unit_converters.py b/django/climate_change_api/indicators/tests/test_unit_converters.py new file mode 100644 index 00000000..c968804a --- /dev/null +++ b/django/climate_change_api/indicators/tests/test_unit_converters.py @@ -0,0 +1,49 @@ +from django.test import TestCase +from indicators.tests.mixins import ConverterTestMixin +from indicators.unit_converters import TemperatureConverter, PrecipitationConverter + + +class TemperatureConverterTestCase(ConverterTestMixin, TestCase): + converter_class = TemperatureConverter + cases = { + ('F', 'K'): [ + (0, 255.372222), + (32, 273.15), + (73.45, 296.17778) + ], + ('K', 'F'): [ + (0, -459.67), + (276.3, 37.67) + ], + ('K', 'C'): [ + (0, -273.15), + (295.3, 22.15) + ], + ('C', 'F'): [ + (-40, -40), + (0, 32), + (10, 50), + (100, 212), + (23.2, 73.76) + ], + ('F', 'C'): [ + (80, 26.6667), + (73.2, 22.888889), + (-40, -40), + (0, -17.77778), + (212, 100), + (32, 0) + ] + } + + +class PrecipitationConverterTestCase(ConverterTestMixin, TestCase): + converter_class = PrecipitationConverter + cases = { + ('kg/m^2/s', 'kg/m^2/day'): [ + (1, 86400) + ], + ('kg/m^2/day', 'kg/m^2/s'): [ + (86400, 1) + ] + } diff --git a/django/climate_change_api/indicators/unit_converters.py b/django/climate_change_api/indicators/unit_converters.py index 35438d70..58fbee48 100644 --- a/django/climate_change_api/indicators/unit_converters.py +++ b/django/climate_change_api/indicators/unit_converters.py @@ -1,35 +1,76 @@ +from django.utils.decorators import classproperty SECONDS_PER_DAY = 24 * 60 * 60 DAYS_PER_YEAR = 365.25 INCHES_PER_MILLIMETER = 1 / 25.4 -class ConversionMixin(object): - def getConverter(self, start, end): - return lambda x: x +########################## +# Unit converters +class UnitConverter(object): + units = {} -class TemperatureUnitsMixin(ConversionMixin): - """ Define units for temperature conversion. - """ - available_units = ('K', 'F', 'C') - storage_units = 'K' - default_units = 'F' + @classmethod + def create(cls, start, end): + raise NotImplementedError() - conversions = { - 'K': { - 'F': lambda x: x * 1.8 - 459.67, - 'C': lambda x: x - 273.15 - } - } + @classmethod + def get(cls, start, end): + """ Factory method to instantiate a converter if necessary - def getConverter(self, start, end): - if end == self.storage_units: + In cases where we're doing a noop conversion, instead we'll return a pass-through lambda + """ + if start == end: + # We're not doing a conversion, so no need to create a converter class return lambda x: x - return self.conversions[start][end] + # Create a lamdba for converting these specific units + return cls.create(start, end) -class PrecipUnitsMixin(ConversionMixin): + +class LinearConverter(UnitConverter): + @classmethod + def create(cls, start, end): + ratio = 1.0 * cls.units[end] / cls.units[start] + + return lambda x: x * ratio + + +class OffsetLinearConverter(UnitConverter): + @classmethod + def create(cls, start, end): + start_x, start_r = cls.units[start] + end_x, end_r = cls.units[end] + + ratio = 1.0 * end_r / start_r + offset = (1.0 * end_x / ratio) - start_x + + return lambda x: (x + offset) * ratio + + +class TemperatureConverter(OffsetLinearConverter): + units = { + 'F': (-459.67, 1.8), + 'C': (-273.15, 1), + 'K': (0, 1) + } + + +class TemperatureDeltaConverter(LinearConverter): + """ Specialized version of TemperatureConverter to convert a degree temperature as a quantity + + This does not adjust for different 0-points. That is, 1 degree Centigrade is equal to exactly + 1.8 degrees Fahrenheit. + """ + units = { + 'C': 1, + 'F': 1.8, + 'K': 1 + } + + +class PrecipitationConverter(LinearConverter): """ Define units for precipitation The units are rates, so cumulative totals can be had either by averaging the rates then @@ -43,23 +84,47 @@ class PrecipUnitsMixin(ConversionMixin): To convert from mass/area/second to height, we're assuming 1kg water == .001 m^3 which makes kg/m^2 equivalent to millimeters. """ - available_units = ('kg/m^2/s', 'kg/m^2/day', 'kg/m^2/year', 'in/day', 'in/year') - storage_units = 'kg/m^2/s' - default_units = 'in/day' - - conversions = { - 'kg/m^2/s': { - 'kg/m^2/day': lambda x: x * SECONDS_PER_DAY, - 'kg/m^2/year': lambda x: x * SECONDS_PER_DAY * DAYS_PER_YEAR, - 'in/day': lambda x: x * INCHES_PER_MILLIMETER * SECONDS_PER_DAY, - 'in/year': lambda x: x * INCHES_PER_MILLIMETER * SECONDS_PER_DAY * DAYS_PER_YEAR, - } + units = { + 'kg/m^2/s': 1, + 'kg/m^2/day': SECONDS_PER_DAY, + 'kg/m^2/year': SECONDS_PER_DAY * DAYS_PER_YEAR, + 'in/day': INCHES_PER_MILLIMETER * SECONDS_PER_DAY, + 'in/year': INCHES_PER_MILLIMETER * SECONDS_PER_DAY * DAYS_PER_YEAR, } + +########################## +# Mixin classes + +class ConversionMixin(object): + converter_class = UnitConverter + def getConverter(self, start, end): - if end == self.storage_units: - return lambda x: x - return self.conversions[start][end] + return self.converter_class.get(start, end) + + @classproperty + def available_units(cls): + return cls.converter_class.units.keys() + + +class TemperatureUnitsMixin(ConversionMixin): + """ Define units for temperature conversion. + """ + converter_class = TemperatureConverter + storage_units = 'K' + default_units = 'F' + + +class TemperatureDeltaUnitsMixin(TemperatureUnitsMixin): + """ Uses the same units as the TemperatureUnitsMixin, but doesn't adjust for 0-point offset + """ + converter_class = TemperatureDeltaConverter + + +class PrecipUnitsMixin(ConversionMixin): + converter_class = PrecipitationConverter + storage_units = 'kg/m^2/s' + default_units = 'in/day' class DaysUnitsMixin(ConversionMixin):