Enable areas to be an iterator throughout request.

We subclass dict so that we can create an iterator that json will use.

mapit/iterables.py

@@ -0,0 +1,49 @@
+# If you wanted iterdict() to be more generic, it'd be something like:
+# from django.utils import six
+# import itertools
+# if not args:
+#     self.ITERABLE = iter()
+# elif isinstance(args[0], dict):
+#     self.ITERABLE = six.iteritems(args[0])
+# else:
+#     self.ITERABLE = args[0]
+# if kwargs:
+#     self.ITERABLE = itertools.chain(self.ITERABLE, six.iteritems(kwargs))
+
+
+class iterdict(dict):
+    """This is a fake dict that sticks an iterable on items/iteritems. Why on
+    earth would you want to do such a thing, I hear you cry? Well, if you want
+    to output what you'd like to be a dict of c. 200,000 objects from a
+    database, currently held in a lovely iterable, as JSON, but discover that
+    json.iterencode() doesn't work with iterators, and that wouldn't be much
+    help with a dict anyway, you either need to write your own iterator to
+    output a JSON object, or trick iterencode() into thinking you're passing it
+    a dict when you're not."""
+
+    def __init__(self, *args, **kwargs):
+        self.ITERABLE = args[0]
+        # Create a dict so that "if X" passes.
+        super(iterdict, self).__init__({'Hack': True})
+
+    def iteritems(self):
+        return self.ITERABLE
+
+    def items(self):
+        return self.ITERABLE
+
+
+class iterlist(list):
+    """This is a fake list that uses its own stored iterable. The built in json
+    package, though it can work as an iterator using iterencode() or indeed
+    dump(), can't take one as input, which is annoying if you're trying to save
+    memory. This class can be passed to iterencode() and will work the same as
+    a list, but won't require the list to exist first."""
+
+    def __init__(self, l):
+        self.ITERABLE = l
+        # Create a list so that "if X" passes.
+        super(iterlist, self).__init__("Hack")
+
+    def __iter__(self):
+        return self.ITERABLE

mapit/views/areas.py

@@ -9,6 +9,7 @@
     PYGDAL = False
 
 from django.contrib.gis.geos import Point
+from django.db.models.query import QuerySet
 from django.http import HttpResponse, HttpResponseRedirect
 from django.core.urlresolvers import resolve, reverse
 from django.conf import settings
@@ -19,27 +20,32 @@
 from mapit.middleware import ViewException
 from mapit.ratelimitcache import ratelimit
 from mapit import countries
+from mapit.iterables import iterdict
 
 
 def add_codes(areas):
+    """Given an iterable of areas, return an iterator of those areas with codes
+    attached. We don't use prefetch_related because this can use a lot of
+    memory."""
     codes = Code.objects.select_related('type').filter(area__in=areas)
     lookup = {}
-    for code in codes:
+    for code in codes.iterator():
         lookup.setdefault(code.area_id, {})[code.type.code] = code.code
     if isinstance(areas, QuerySet):
         if hasattr(countries, 'sorted_areas'):
             areas = countries.sorted_areas(areas)
+        areas = areas.iterator()
     for area in areas:
         if area.id in lookup:
             area.all_codes = lookup[area.id]
-    return areas
+        yield area
 
 
 def output_areas(request, title, format, areas, **kwargs):
     areas = add_codes(areas)
     if format == 'html':
         return output_html(request, title, areas, **kwargs)
-    return output_json(dict((area.id, area.as_dict()) for area in areas))
+    return output_json(iterdict((area.id, area.as_dict()) for area in areas))
 
 
 def query_args(request, format, type=None):

mapit/views/postcodes.py

@@ -51,7 +51,7 @@ def postcode(request, postcode, format=None):
     except:
         generation = Generation.objects.current()
     if not hasattr(countries, 'is_special_postcode') or not countries.is_special_postcode(postcode.postcode):
-        areas = add_codes(Area.objects.by_postcode(postcode, generation))
+        areas = list(add_codes(Area.objects.by_postcode(postcode, generation)))
     else:
         areas = []