diff --git a/django/climate_change_api/climate_data/filters.py b/django/climate_change_api/climate_data/filters.py index 2620af7b..cb102e10 100644 --- a/django/climate_change_api/climate_data/filters.py +++ b/django/climate_change_api/climate_data/filters.py @@ -16,6 +16,10 @@ class ClimateDataFilterSet(filters.FilterSet): models = django_filters.MethodFilter() years = django_filters.MethodFilter() + def __init__(self, *args, **kwargs): + self.year_col = kwargs.pop('year_col', 'data_source__year') + super(ClimateDataFilterSet, self).__init__(*args, **kwargs) + def filter_models(self, queryset, value): """ Filter models based on a comma separated list of names @@ -46,13 +50,16 @@ def filter_years(self, queryset, value): for year_range_str in value.split(','): year_range = year_range_str.split(':') if len(year_range) == 2: - start_year = int(year_range[0]) - end_year = int(year_range[1]) - # AND the year range - year_filters.append((Q(data_source__year__gte=start_year) & Q(data_source__year__lte=end_year))) + # Pair the two years with their comparators, gte and lte respectively + bounds = zip(['gte', 'lte'], year_range) + # Create two Q objects with the proper column, comparator and boundary year + start, end = [Q(**{"%s__%s" % (self.year_col, comparator): year}) + for comparator, year in bounds] + # And the checks together + year_filters.append(start & end) if len(year_range) == 1: year = int(year_range[0]) - year_filters.append(Q(data_source__year=year)) + year_filters.append(Q(**{self.year_col: year})) logger.debug(year_filters) # Now OR together all the year filters we've created queryset = queryset.filter(reduce(lambda x, y: x | y, year_filters)) diff --git a/django/climate_change_api/indicators/abstract_indicators.py b/django/climate_change_api/indicators/abstract_indicators.py index 4feddd6d..66d0fe98 100644 --- a/django/climate_change_api/indicators/abstract_indicators.py +++ b/django/climate_change_api/indicators/abstract_indicators.py @@ -1,25 +1,23 @@ from collections import OrderedDict, defaultdict from itertools import groupby import re -from datetime import date, timedelta from django.db.models import F, Case, When, FloatField, Sum from django.db import connection from climate_data.models import ClimateData -from climate_data.filters import ClimateDataFilterSet from .params import IndicatorParams, ThresholdIndicatorParams from .serializers import IndicatorSerializer from .unit_converters import DaysUnitsMixin, TemperatureConverter, PrecipitationConverter -from .query_ranges import MonthRangeConfig, QuarterRangeConfig, CustomRangeConfig +import queryset_generator class Indicator(object): label = '' description = '' - valid_aggregations = ('yearly', 'quarterly', 'monthly', 'daily', 'custom') + valid_aggregations = ('yearly', 'quarterly', 'monthly', 'offset_yearly', 'custom') variables = ClimateData.VARIABLE_CHOICES # Filters define which rows match our query, conditions limit which rows @@ -50,6 +48,9 @@ class Indicator(object): available_units = (None,) parameters = None + # These are the keys that values are identified by when they come from the database. + aggregate_keys = ['agg_key', 'data_source__model'] + def __init__(self, city, scenario, parameters=None): if not city: raise ValueError('Indicator constructor requires a city instance') @@ -105,45 +106,29 @@ def get_queryset(self): by the constructor """ - filter_set = ClimateDataFilterSet() - queryset = (ClimateData.objects.filter(map_cell=self.city.map_cell) - .filter(data_source__scenario=self.scenario)) - queryset = filter_set.filter_years(queryset, self.params.years.value) - queryset = filter_set.filter_models(queryset, self.params.models.value) + # Get the queryset generator for this indicator's time aggregation + generator = queryset_generator.get(self.params.time_aggregation.value) + key_params = {} + + # The custom range config accepts a user-defined parameter to pick which dates to use + if self.params.custom_time_agg.value is not None: + key_params['custom_time_agg'] = self.params.custom_time_agg.value + + # Use the queryset generator classes to construct the initial base climate data queryset + queryset = generator.create_queryset( + years=self.params.years.value, + models=self.params.models.value, + scenario=self.scenario, + key_params=key_params + ).filter( + map_cell=self.city.map_cell + ) if self.filters is not None: queryset = queryset.filter(**self.filters) - # For certain time aggregations, add a field to track which interval a data point is in - time_aggregation_configs = { - 'monthly': MonthRangeConfig, - 'quarterly': QuarterRangeConfig, - 'custom': CustomRangeConfig - } - if self.params.time_aggregation.value in time_aggregation_configs: - config = time_aggregation_configs[self.params.time_aggregation.value] - params = {} - - # The custom range config accepts a user-defined parameter to pick which dates to use - if self.params.custom_time_agg.value is not None: - params['custom_time_agg'] = self.params.custom_time_agg.value - - queryset = (queryset - .annotate(interval=config.cases(**params)) - .filter(interval__isnull=False)) - return queryset - @property - def aggregate_keys(self): - return { - 'daily': ['data_source__year', 'day_of_year', 'data_source__model'], - 'monthly': ['data_source__year', 'interval', 'data_source__model'], - 'quarterly': ['data_source__year', 'interval', 'data_source__model'], - 'custom': ['data_source__year', 'interval', 'data_source__model'], - 'yearly': ['data_source__year', 'data_source__model'] - }.get(self.params.time_aggregation.value) - @property def expression(self): return self.variables[0] @@ -183,40 +168,13 @@ def collate_results(self, aggregations): """ Take results as a series of datapoints and collate them by key @param aggregations list-of-dicts returned by aggregate method - @returns Dict of list of values, keyed by the subclass's key_results implementation + @returns Dict of list of values, keyed by the queryset's agg_key column """ results = defaultdict(list) for result in aggregations: - key = self.key_result(result) - results[key].append(result['value']) + results[result['agg_key']].append(result['value']) return results - def key_result(self, result): - """ Stub function for subclasses to determine how to collate results - - @param result A row of timeseries data generated by aggregate() - @returns The value that row should be keyed by in the final response. - Results should be keyed as one of the following based on aggregation: - * YYYY for yearly data - * YYYY-MM for monthly data - * YYYY-MM-DD for daily data - """ - year = result['data_source__year'] - if self.params.time_aggregation.value == 'yearly': - return year - - if self.params.time_aggregation.value == 'daily': - day_of_year = result['day_of_year'] - day = date(year, 1, 1) + timedelta(days=day_of_year-1) - return day.isoformat() - - template = { - 'monthly': '{year}-{int:02d}', - 'quarterly': '{year}-Q{int:0d}', - 'custom': '{year}-{int:02d}' - }.get(self.params.time_aggregation.value) - return template.format(year=year, int=(result['interval']+1)) - def calculate(self): aggregations = self.aggregate() aggregations = self.convert_units(aggregations) @@ -265,17 +223,17 @@ def get_streaks(self): (base_query, base_query_params) = (self.queryset.select_related('data_source') .query.sql_with_params()) query = """ - SELECT year as data_source__year, model_id as data_source__model, + SELECT agg_key, model_id as data_source__model, count(*) as length, match - FROM (SELECT year, model_id, day_of_year, + FROM (SELECT agg_key, model_id, day_of_year, (CASE WHEN {condition} THEN 1 ELSE 0 END) as match, - ROW_NUMBER() OVER(ORDER BY year, model_id, day_of_year) - + ROW_NUMBER() OVER(ORDER BY agg_key, model_id, day_of_year) - ROW_NUMBER() OVER(PARTITION BY CASE WHEN {condition} THEN 1 ELSE 0 END - ORDER BY year, model_id, day_of_year) + ORDER BY agg_key, model_id, day_of_year) AS grp FROM ({base_query}) orig_query) groups - GROUP BY year, model_id, grp, match - ORDER BY year, model_id + GROUP BY agg_key, model_id, grp, match + ORDER BY agg_key, model_id """.format(base_query=base_query, condition=self.raw_condition) # First run the query and get a list of dicts with one result per sequence with connection.cursor() as cursor: diff --git a/django/climate_change_api/indicators/params.py b/django/climate_change_api/indicators/params.py index ea0ca121..fb7952d6 100644 --- a/django/climate_change_api/indicators/params.py +++ b/django/climate_change_api/indicators/params.py @@ -23,9 +23,9 @@ TIME_AGGREGATION_PARAM_DOCSTRING = ("Time granularity to group data by for result structure. Valid " "aggregations depend on indicator. Can be 'yearly', " - "'quarterly', 'monthly', 'daily' or 'custom'. Defaults to " - "'yearly'. If 'custom', 'custom_time_agg' parameter must be " - "set.") + "'offset_yearly', 'quarterly', 'monthly', or 'custom'. " + "Defaults to 'yearly'. If 'custom', 'custom_time_agg' " + "parameter must be set.") UNITS_PARAM_DOCSTRING = ("Units in which to return the data. Defaults to Imperial units (Fahrenheit" " for temperature indicators and inches for precipitation).") diff --git a/django/climate_change_api/indicators/query_ranges.py b/django/climate_change_api/indicators/query_ranges.py deleted file mode 100644 index 8dabada9..00000000 --- a/django/climate_change_api/indicators/query_ranges.py +++ /dev/null @@ -1,153 +0,0 @@ -from collections import namedtuple -import calendar - -from django.db.models import Case, When, IntegerField, Value -from climate_data.models import ClimateDataSource - - -class QueryRangeConfig(object): - """ Utility class to generate a Django Case object that converts day-of-year to a specific bucket - """ - - CaseRange = namedtuple('CaseRange', ('index', 'start', 'length')) - range_config = None - - @staticmethod - def get_years(): - """ Builds objects that categorize years by a common feature - - By default categorizes years by whether they are a leap year or not - """ - all_years = set(ClimateDataSource.objects.distinct('year') - .values_list('year', flat=True)) - leap_years = set(filter(calendar.isleap, all_years)) - - return [ - ('leap', leap_years), - ('noleap', all_years - leap_years) - ] - - @classmethod - def get_intervals(cls, label): - """ Returns an ordered series intervals to map days to interval segments - - Each value should be a tuple of (start, length), measured in day-of-year - """ - raise NotImplementedError() - - @classmethod - def make_ranges(cls, label): - """ Takes the values of get_intervals and wraps them in CaseRange objects - """ - cases = cls.get_intervals(label) - return [cls.CaseRange(i, start, length) - for (i, (start, length)) in enumerate(cases)] - - @classmethod - def get_ranges(cls): - """ Build mapping from day of year to month. - - Gets the year range by querying what data exists and builds CaseRange objects for each - month. - """ - return [ - { - 'years': years, - 'ranges': cls.make_ranges(label), - } - for (label, years) in cls.get_years() - ] - - @classmethod - def cases(cls): - """ Generates a nested Case aggregation that assigns the month index to each - data point. It first splits on leap year or not then checks day_of_year against ranges. - """ - if cls.range_config is None: - cls.range_config = cls.get_ranges() - - year_whens = [] - for config in cls.range_config: - case_whens = [When(**{ - 'day_of_year__gte': case.start, - 'day_of_year__lt': case.start + case.length, - 'then': Value(case.index) - }) for case in config['ranges']] - year_whens.append(When(data_source__year__in=config['years'], then=Case(*case_whens))) - return Case(*year_whens, output_field=IntegerField()) - - -class LengthRangeConfig(QueryRangeConfig): - """ RangeConfig based on a list of period lengths - - Assumes that the periods are consecutive, and that each period takes place immediately - following the previous period. - """ - lengths = {} - - @classmethod - def get_intervals(cls, label): - lengths = cls.lengths[label] - return [(sum(lengths[:i]) + 1, lengths[i]) for i in range(len(lengths))] - - -class MonthRangeConfig(LengthRangeConfig): - lengths = { - 'leap': [31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31], - 'noleap': [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31] - } - - -class QuarterRangeConfig(LengthRangeConfig): - lengths = { - 'leap': [91, 91, 92, 92], - 'noleap': [90, 91, 92, 92] - } - - -class CustomRangeConfig(QueryRangeConfig): - custom_spans = None - - @classmethod - def day_of_year_from_date(cls, date, label): - starts = { - # These are all zero-based, so, for example, adding 1 for the 1st gives the true DOY - 'leap': [0, 31, 60, 91, 121, 152, 182, 213, 244, 274, 305, 335, 366], - 'noleap': [0, 31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334, 365] - }.get(label) - - # Date is a tuple of month and day of month (DOM) - month, dom = date - - # Offset calculations by a month because weird human dates start with 1 - doy = starts[month - 1] + dom - # Make sure we have a non-zero DOM - assert (dom > 0), "Invalid date provided" - # Make sure this date exists in the month given - assert (doy <= starts[month]), "Invalid date provided" - - return doy - - @classmethod - def get_intervals(cls, label): - # Spans are in the format MM-DD:MM-DD, so break those into nested tuples - spans = [tuple(tuple(int(v) for v in date.split('-')) - for date in span.split(':')) - for span in cls.custom_spans.split(',')] - - for span in spans: - start, end = (cls.day_of_year_from_date(date, label) for date in span) - assert (start <= end), "Dates must be paired start:end" - - # Add one to end date because the end point is inclusive - yield (start, end - start + 1) - - @classmethod - def cases(cls, intervals): - # Cases normally caches the range_config, but that's bad if the custom spans change - # Check if that happened, and if it did clear the cached config - if cls.custom_spans != intervals: - cls.range_config = None - cls.custom_spans = intervals - - return super(CustomRangeConfig, cls).cases() diff --git a/django/climate_change_api/indicators/queryset_generator.py b/django/climate_change_api/indicators/queryset_generator.py new file mode 100644 index 00000000..28b0c8b3 --- /dev/null +++ b/django/climate_change_api/indicators/queryset_generator.py @@ -0,0 +1,274 @@ +from collections import namedtuple +import calendar + +from django.db.models.functions import Concat +from django.db.models import Case, When, CharField, Value, F, Max, Min +from climate_data.models import ClimateData, ClimateDataSource +from climate_data.filters import ClimateDataFilterSet + + +def get(time_aggregation): + """ Provide the correct queryset generator class based on indicator time aggregation """ + return { + 'monthly': MonthQuerysetGenerator, + 'quarterly': QuarterQuerysetGenerator, + 'yearly': YearQuerysetGenerator, + 'offset_yearly': OffsetYearQuerysetGenerator, + 'custom': CustomQuerysetGenerator + }.get(time_aggregation) + + +class QuerysetGenerator(object): + """ Utility class to create querysets for ClimateData for a given time aggregation + + Incorporates filtering by year as the year a data point is associated with can be dependant on + the time aggregation used + """ + + CaseRange = namedtuple('CaseRange', ('key', 'start', 'length')) + range_config = None + filterset_kwargs = {} + + @staticmethod + def get_leap_year_sets(): + """ Builds objects that categorize years by a common feature + + By default categorizes years by whether they are a leap year or not + """ + all_years = set(ClimateDataSource.objects.distinct('year') + .values_list('year', flat=True)) + leap_years = set(filter(calendar.isleap, all_years)) + + return [ + ('leap', leap_years), + ('noleap', all_years - leap_years) + ] + + @classmethod + def create_queryset(cls, scenario, years=None, models=None, key_params=None): + if key_params is None: + key_params = {} + + queryset = (ClimateData.objects.all() + .annotate(agg_key=cls.keys(**key_params)) + .filter(agg_key__isnull=False, + data_source__scenario=scenario)) + + if years is not None or models is not None: + queryset = cls.apply_filters(queryset, years, models) + + return queryset + + @classmethod + def apply_filters(cls, queryset, years, models): + filterset = ClimateDataFilterSet(**cls.filterset_kwargs) + queryset = filterset.filter_years(queryset, years) + queryset = filterset.filter_models(queryset, models) + return queryset + + @classmethod + def get_interval_key(cls, index): + return Concat(F('data_source__year'), Value('-{:02d}'.format(index + 1))) + + @classmethod + def get_intervals(cls, label): + """ Returns an ordered series intervals to map days to interval segments + + Each value should be a tuple of (start, length), measured in day-of-year + """ + raise NotImplementedError() + + @classmethod + def make_ranges(cls, label): + """ Takes the values of get_intervals and wraps them in CaseRange objects + """ + cases = cls.get_intervals(label) + return [cls.CaseRange(cls.get_interval_key(i), start, length) + for (i, (start, length)) in enumerate(cases)] + + @classmethod + def get_ranges(cls): + """ Build mapping from day of year to month. + + Gets the year range by querying what data exists and builds CaseRange objects for each + month. + """ + return [ + { + 'years': years, + 'ranges': cls.make_ranges(label), + } + for (label, years) in cls.get_leap_year_sets() + ] + + @classmethod + def keys(cls): + """ Generates a nested Case aggregation that assigns the range key to each + data point. It first splits on leap year or not then checks day_of_year against ranges. + """ + if cls.range_config is None: + cls.range_config = cls.get_ranges() + + year_whens = [] + for config in cls.range_config: + case_whens = [When(**{ + 'day_of_year__gte': case.start, + 'day_of_year__lt': case.start + case.length, + 'then': case.key + }) for case in config['ranges']] + year_whens.append(When(data_source__year__in=config['years'], then=Case(*case_whens))) + return Case(*year_whens, output_field=CharField()) + + +class YearQuerysetGenerator(QuerysetGenerator): + """ Special case generator for yearly annotations + Yearly annotations don't need any special logic, so we can short-circuit the whole process + """ + + @classmethod + def keys(cls): + return F('data_source__year') + + +class LengthQuerysetGenerator(QuerysetGenerator): + """ QuerysetGenerator based on a list of period lengths + + Assumes that the periods are consecutive, and that each period takes place immediately + following the previous period. + """ + lengths = {} + + @classmethod + def get_intervals(cls, label): + lengths = cls.lengths[label] + return [(sum(lengths[:i]) + 1, lengths[i]) for i in range(len(lengths))] + + +class MonthQuerysetGenerator(LengthQuerysetGenerator): + lengths = { + 'leap': [31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31], + 'noleap': [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31] + } + + +class QuarterQuerysetGenerator(LengthQuerysetGenerator): + @classmethod + def get_interval_key(cls, index): + return Concat(F('data_source__year'), Value('-Q{:0d}'.format(index + 1))) + + lengths = { + 'leap': [91, 91, 92, 92], + 'noleap': [90, 91, 92, 92] + } + + +class CustomQuerysetGenerator(QuerysetGenerator): + """ QuerySet generator for user-defined date ranges within a given calendar year + + Dates cannot span across year, OffsetYearQuerysetGenerator is necessary for that. + """ + custom_spans = None + + @classmethod + def day_of_year_from_date(cls, date, label): + starts = { + # These are all zero-based, so, for example, adding 1 for the 1st gives the true DOY + 'leap': [0, 31, 60, 91, 121, 152, 182, 213, 244, 274, 305, 335, 366], + 'noleap': [0, 31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334, 365] + }.get(label) + + # Date is a tuple of month and day of month (DOM) + month, dom = date + + # Offset calculations by a month because weird human dates start with 1 + doy = starts[month - 1] + dom + # Make sure we have a non-zero DOM + assert (dom > 0), "Invalid date provided" + # Make sure this date exists in the month given + assert (doy <= starts[month]), "Invalid date provided" + + return doy + + @classmethod + def get_intervals(cls, label): + # Spans are in the format MM-DD:MM-DD, so break those into nested tuples + spans = [tuple(tuple(int(v) for v in date.split('-')) + for date in span.split(':')) + for span in cls.custom_spans.split(',')] + + for span in spans: + start, end = (cls.day_of_year_from_date(date, label) for date in span) + assert (start <= end), "Dates must be paired start:end" + + # Add one to end date because the end point is inclusive + yield (start, end - start + 1) + + @classmethod + def cases(cls, intervals): + # Cases normally caches the range_config, but that's bad if the custom spans change + # Check if that happened, and if it did clear the cached config + if cls.custom_spans != intervals: + cls.range_config = None + cls.custom_spans = intervals + + return super(CustomQuerysetGenerator, cls).cases() + + +class OffsetYearQuerysetGenerator(QuerysetGenerator): + # By default place the year divide near the summer solstice to maximize the span that covers + # winter + custom_offset = 180 + filterset_kwargs = {'year_col': 'offset_year'} + + @classmethod + def make_ranges(cls, label): + """ Builds a pair of CaseRanges to represent a 366 day year split across two calendar years + """ + year_len = 366 + offset = cls.custom_offset + return [ + # Include all days from the offset to New Years Eve + cls.CaseRange(Concat(F('data_source__year'), Value('-'), F('data_source__year') + 1), + offset, year_len - offset + 1), + # Start on New Years Day until the offset point the following year + cls.CaseRange(Concat(F('data_source__year') - 1, Value('-'), F('data_source__year')), + 0, offset) + ] + + @classmethod + def create_queryset(cls, *args, **kwargs): + queryset = super(OffsetYearQuerysetGenerator, cls).create_queryset(*args, **kwargs) + + # Since we want to have complete year ranges, we want to conditionally remove the + # beginning part of the first year and ending part of the last year to ensure all + # results have complete data. + # However, because ACCESS1-0, MIROC5, and bcc-csm1-1 don't have data for 2100, we need + # specific logic to determine what constitutes the "first year" and "last year" for each + # model. + # To do this, we build a list of data sources that represent the first and last sources + # for each model, and then filter so that we exclude those sources if the day is either + # too early or too late to have another year to be paired with. + first_year_sources = (ClimateDataSource.objects.all() + .annotate(minYear=Min('model__climatedatasource__year')) + .filter(year=F('minYear')) + .values('id')) + last_year_sources = (ClimateDataSource.objects.all() + .annotate(maxYear=Max('model__climatedatasource__year')) + .filter(year=F('maxYear')) + .values('id')) + queryset = (queryset + # Exclude data from the model's first year before the offset cutoff, + # and data from the model's last year after the cutoff + .exclude(data_source__in=first_year_sources, day_of_year__lt=cls.custom_offset) + .exclude(data_source__in=last_year_sources, day_of_year__gte=cls.custom_offset)) + + return queryset + + @classmethod + def apply_filters(cls, queryset, years, models): + queryset = queryset.annotate(offset_year=Case( + When(day_of_year__lt=cls.custom_offset, + then=F('data_source__year') - 1), + default=F('data_source__year'))) + + return super(OffsetYearQuerysetGenerator, cls).apply_filters(queryset, years, models) diff --git a/django/climate_change_api/indicators/tests/test_indicator_params.py b/django/climate_change_api/indicators/tests/test_indicator_params.py index 2bbac084..f28ad766 100644 --- a/django/climate_change_api/indicators/tests/test_indicator_params.py +++ b/django/climate_change_api/indicators/tests/test_indicator_params.py @@ -47,7 +47,7 @@ def setUp(self): self.params_class_kwargs = {} self.default_units = 'F' self.available_units = ('F', 'C', 'K',) - self.valid_aggregations = ('yearly', 'monthly', 'daily',) + self.valid_aggregations = ('yearly', 'monthly',) self.params_class = IndicatorParams def _get_params_class(self): diff --git a/django/climate_change_api/indicators/tests/test_indicators.py b/django/climate_change_api/indicators/tests/test_indicators.py index 95daaae2..2b074160 100644 --- a/django/climate_change_api/indicators/tests/test_indicators.py +++ b/django/climate_change_api/indicators/tests/test_indicators.py @@ -75,25 +75,6 @@ def test_unit_conversion(self): 'Temperature should be converted to degrees F') -class DailyHighTemperatureTestCase(TemperatureIndicatorTests, TestCase): - indicator_class = indicators.AverageHighTemperature - indicator_name = 'average_high_temperature' - time_aggregation = 'daily' - units = 'K' - test_indicator_rcp85_equals = {'2000-01-01': {'avg': 35.0, 'max': 40.0, 'min': 30.0}} - test_indicator_rcp45_equals = {'2000-01-01': {'max': 20.0, 'avg': 15.0, 'min': 10.0}, - '2001-01-01': {'max': 20.0, 'avg': 15.0, 'min': 10.0}, - '2002-01-01': {'avg': 10.0, 'max': 10.0, 'min': 10.0}, - '2003-01-01': {'avg': 10.0, 'max': 10.0, 'min': 10.0}} - test_years_filter_equals = {'2001-01-01': {'max': 20.0, 'avg': 15.0, 'min': 10.0}, - '2002-01-01': {'avg': 10.0, 'max': 10.0, 'min': 10.0}} - test_models_filter_equals = {'2000-01-01': {'avg': 10.0, 'max': 10.0, 'min': 10.0}, - '2001-01-01': {'avg': 10.0, 'max': 10.0, 'min': 10.0}, - '2002-01-01': {'avg': 10.0, 'max': 10.0, 'min': 10.0}, - '2003-01-01': {'avg': 10.0, 'max': 10.0, 'min': 10.0}} - test_units_fahrenheit_equals = {'2000-01-01': {'avg': -396.67, 'max': -387.67, 'min': -405.67}} - - class YearlyAverageHighTemperatureTestCase(TemperatureIndicatorTests, TestCase): indicator_class = indicators.AverageHighTemperature indicator_name = 'average_high_temperature' @@ -404,6 +385,27 @@ class YearlyHeatingDegreeDaysTestCase(IndicatorTests, TestCase): 2003: {'avg': 58.5, 'min': 58.5, 'max': 58.5}} +class CrossYearlyHeatingDegreeDaysTestCase(IndicatorTests, TestCase): + indicator_class = indicators.HeatingDegreeDays + indicator_name = 'heating_degree_days' + time_aggregation = 'offset_yearly' + extra_params = { + 'basetemp': '42.5', + 'basetemp_units': 'K' + } + test_indicator_rcp85_equals = {} + test_indicator_rcp45_equals = {'2000-2001': {'avg': 49.5, 'min': 40.5, 'max': 58.5}, + '2001-2002': {'avg': 58.5, 'min': 58.5, 'max': 58.5}, + '2002-2003': {'avg': 58.5, 'min': 58.5, 'max': 58.5}} + # Years are filtered by the starting year, so years=2001,2002 gives data for 2001-2002 + # and 2002-2003 + test_years_filter_equals = {'2001-2002': {'avg': 58.5, 'min': 58.5, 'max': 58.5}, + '2002-2003': {'avg': 58.5, 'min': 58.5, 'max': 58.5}} + test_models_filter_equals = {'2000-2001': {'avg': 58.5, 'min': 58.5, 'max': 58.5}, + '2001-2002': {'avg': 58.5, 'min': 58.5, 'max': 58.5}, + '2002-2003': {'avg': 58.5, 'min': 58.5, 'max': 58.5}} + + class YearlyCoolingDegreeDaysTestCase(IndicatorTests, TestCase): indicator_class = indicators.CoolingDegreeDays indicator_name = 'cooling_degree_days' @@ -660,21 +662,3 @@ class MonthlyPrecipitationThresholdTestcase(IndicatorTests, TestCase): '2001-01': {'avg': 1.0, 'max': 1.0, 'min': 1.0}, '2002-01': {'avg': 1.0, 'max': 1.0, 'min': 1.0}, '2003-01': {'avg': 1.0, 'max': 1.0, 'min': 1.0}} - - -class MaxTemperatureThresholdTestCase(IndicatorTests, TestCase): - indicator_class = indicators.MaxTemperatureThreshold - indicator_name = 'max_temperature_threshold' - time_aggregation = 'daily' - extra_params = {'threshold': 20.0, 'threshold_comparator': 'gte', 'threshold_units': 'K'} - test_indicator_rcp85_equals = {'2000-01-01': {'avg': 1.0, 'max': 1.0, 'min': 1.0}} - test_indicator_rcp45_equals = {'2000-01-01': {'max': 1.0, 'avg': 0.5, 'min': 0.0}, - '2001-01-01': {'max': 1.0, 'avg': 0.5, 'min': 0.0}, - '2002-01-01': {'avg': 0.0, 'max': 0.0, 'min': 0.0}, - '2003-01-01': {'avg': 0.0, 'max': 0.0, 'min': 0.0}} - test_years_filter_equals = {'2001-01-01': {'avg': 0.5, 'max': 1.0, 'min': 0.0}, - '2002-01-01': {'avg': 0.0, 'max': 0.0, 'min': 0.0}} - test_models_filter_equals = {'2000-01-01': {'avg': 0.0, 'max': 0.0, 'min': 0.0}, - '2001-01-01': {'avg': 0.0, 'max': 0.0, 'min': 0.0}, - '2002-01-01': {'avg': 0.0, 'max': 0.0, 'min': 0.0}, - '2003-01-01': {'avg': 0.0, 'max': 0.0, 'min': 0.0}} diff --git a/django/climate_change_api/indicators/tests/test_query_ranges.py b/django/climate_change_api/indicators/tests/test_query_ranges.py deleted file mode 100644 index 3227179d..00000000 --- a/django/climate_change_api/indicators/tests/test_query_ranges.py +++ /dev/null @@ -1,68 +0,0 @@ -from django.test import TestCase -from indicators.query_ranges import MonthRangeConfig, QuarterRangeConfig, CustomRangeConfig - - -class QuarterQueryRangeTestCase(TestCase): - def test_non_leap_year(self): - intervals = QuarterRangeConfig.get_intervals('noleap') - self.assertEqual(intervals, [(1, 90), (91, 91), (182, 92), (274, 92)]) - - def test_leap_year(self): - intervals = QuarterRangeConfig.get_intervals('leap') - self.assertEqual(intervals, [(1, 91), (92, 91), (183, 92), (275, 92)]) - - -class MonthQueryRangeTestCase(TestCase): - def test_non_leap_year(self): - intervals = MonthRangeConfig.get_intervals('noleap') - self.assertEqual(intervals, [(1, 31), - (32, 28), - (60, 31), - (91, 30), - (121, 31), - (152, 30), - (182, 31), - (213, 31), - (244, 30), - (274, 31), - (305, 30), - (335, 31)]) - - def test_leap_year(self): - intervals = MonthRangeConfig.get_intervals('leap') - self.assertEqual(intervals, [(1, 31), - (32, 29), - (61, 31), - (92, 30), - (122, 31), - (153, 30), - (183, 31), - (214, 31), - (245, 30), - (275, 31), - (306, 30), - (336, 31)]) - - -class CustomQueryRangeTestCase(TestCase): - def test_non_leap_year(self): - CustomRangeConfig.custom_spans = "1-1:12-31" - intervals = list(CustomRangeConfig.get_intervals('noleap')) - self.assertEqual(intervals, [(1, 365)]) - - CustomRangeConfig.custom_spans = "5-13:5-31,7-8:12-14" - intervals = list(CustomRangeConfig.get_intervals('noleap')) - self.assertEqual(intervals, [(133, 19), (189, 160)]) - - def test_invalid(self): - with self.assertRaises(AssertionError): - CustomRangeConfig.custom_spans = "1-56:1-57" - list(CustomRangeConfig.get_intervals('noleap')) - - with self.assertRaises(AssertionError): - CustomRangeConfig.custom_spans = "7-0:9-2" - list(CustomRangeConfig.get_intervals('noleap')) - - with self.assertRaises(AssertionError): - CustomRangeConfig.custom_spans = "7-1:5-1" - list(CustomRangeConfig.get_intervals('noleap')) diff --git a/django/climate_change_api/indicators/tests/test_queryset_generators.py b/django/climate_change_api/indicators/tests/test_queryset_generators.py new file mode 100644 index 00000000..27526d55 --- /dev/null +++ b/django/climate_change_api/indicators/tests/test_queryset_generators.py @@ -0,0 +1,61 @@ +from django.test import TestCase +from indicators.queryset_generator import (MonthQuerysetGenerator, QuarterQuerysetGenerator, + CustomQuerysetGenerator) + + +class LengthQuerysetGenerator(object): + generator = None + + def test_length_config(self): + total = sum(self.generator.lengths['noleap']) + self.assertEqual(total, 365) + + total = sum(self.generator.lengths['leap']) + self.assertEqual(total, 366) + + def test_get_intervals(self): + for label, result in self.intervals.items(): + intervals = self.generator.get_intervals(label) + self.assertEqual(intervals, result) + + +class QuarterQuerysetGeneratorTestCase(LengthQuerysetGenerator, TestCase): + generator = QuarterQuerysetGenerator + intervals = { + 'leap': [(1, 91), (92, 91), (183, 92), (275, 92)], + 'noleap': [(1, 90), (91, 91), (182, 92), (274, 92)] + } + + +class MonthQuerysetGeneratorTestCase(LengthQuerysetGenerator, TestCase): + generator = MonthQuerysetGenerator + intervals = { + 'leap': [(1, 31), (32, 29), (61, 31), (92, 30), (122, 31), (153, 30), (183, 31), (214, 31), + (245, 30), (275, 31), (306, 30), (336, 31)], + 'noleap': [(1, 31), (32, 28), (60, 31), (91, 30), (121, 31), (152, 30), (182, 31), + (213, 31), (244, 30), (274, 31), (305, 30), (335, 31)] + } + + +class CustomQuerysetGeneratorTestCase(TestCase): + def test_non_leap_year(self): + CustomQuerysetGenerator.custom_spans = "1-1:12-31" + intervals = list(CustomQuerysetGenerator.get_intervals('noleap')) + self.assertEqual(intervals, [(1, 365)]) + + CustomQuerysetGenerator.custom_spans = "5-13:5-31,7-8:12-14" + intervals = list(CustomQuerysetGenerator.get_intervals('noleap')) + self.assertEqual(intervals, [(133, 19), (189, 160)]) + + def test_invalid(self): + with self.assertRaises(AssertionError): + CustomQuerysetGenerator.custom_spans = "1-56:1-57" + list(CustomQuerysetGenerator.get_intervals('noleap')) + + with self.assertRaises(AssertionError): + CustomQuerysetGenerator.custom_spans = "7-0:9-2" + list(CustomQuerysetGenerator.get_intervals('noleap')) + + with self.assertRaises(AssertionError): + CustomQuerysetGenerator.custom_spans = "7-1:5-1" + list(CustomQuerysetGenerator.get_intervals('noleap')) diff --git a/doc/source/overview/how_to_indicator_request.rst b/doc/source/overview/how_to_indicator_request.rst index b429d88a..3ea6d928 100644 --- a/doc/source/overview/how_to_indicator_request.rst +++ b/doc/source/overview/how_to_indicator_request.rst @@ -25,7 +25,6 @@ To begin, `get the full list`_ of actively developed indicators whose response s "yearly", "quarterly", "monthly", - "daily", "custom" ], "variables": [ @@ -56,7 +55,7 @@ To begin, `get the full list`_ of actively developed indicators whose response s }, { "name": "time_aggregation", - "description": "Time granularity to group data by for result structure. Valid aggregations depend on indicator. Can be 'yearly', 'quarterly', 'monthly', 'daily' or 'custom'. Defaults to 'yearly'. If 'custom', 'custom_time_agg' parameter must be set.", + "description": "Time granularity to group data by for result structure. Valid aggregations depend on indicator. Can be 'yearly', 'offset_yearly', 'quarterly', 'monthly' or 'custom'. Defaults to 'yearly'. If 'custom', 'custom_time_agg' parameter must be set.", "required": false, "default": "yearly" }, @@ -208,4 +207,3 @@ Success! To answer our question, we can expect continued, consistent drought mid .. _indicator data endpoint: api_reference.html#indicator-data .. _city: api_reference.html#nearest-city-or-cities .. _scenario: api_reference.html#scenario-list -