Add more realistic test data (#1141)

* Add more realistic test data

* Adding OSM licenses; fixing choice creation

* Better names

cadasta/core/fixtures.py → cadasta/core/fixtures/__init__.py

@@ -1,5 +1,5 @@
 import random
-import json
+import os
 from datetime import datetime, timezone
 
 from accounts.models import User
@@ -8,13 +8,12 @@
 from django.contrib.gis.geos import GEOSGeometry
 from faker import Factory
 from jsonattrs.models import Attribute, Schema, AttributeType
-from organization.models import Organization, OrganizationRole, Project
+from organization import models
 from organization.tests.factories import OrganizationFactory, ProjectFactory
 from spatial.tests.factories import (
     SpatialUnitFactory, SpatialRelationshipFactory
 )
 from spatial.choices import TYPE_CHOICES
-
 from tutelary.models import Policy, Role
 
 
@@ -41,11 +40,11 @@ def add_test_users(self):
         for n in range(20):
             if n < len(named_users):
                 users.append(UserFactory.create(
-                    **named_users[n],
                     password='password',
                     email_verified=True,
                     last_login=datetime.now(tz=timezone.utc),
                     is_active=True,
+                    **named_users[n]
                 ))
             else:
                 users.append(UserFactory.create(
@@ -58,7 +57,7 @@ def add_test_users(self):
 
     def add_test_users_and_roles(self):
         users = FixturesData.add_test_users(self)
-        orgs = Organization.objects.all()
+        orgs = models.Organization.objects.all()
 
         pols = {}
         for pol in ['default', 'superuser', 'org-admin', 'org-member',
@@ -72,13 +71,13 @@ def add_test_users_and_roles(self):
 
         for i in [0, 1, 3, 4, 7, 10]:
             admin = i == 3 or i == 4
-            OrganizationRole.objects.create(
+            models.OrganizationRole.objects.create(
                 organization=orgs[0], user=users[i], admin=admin
             )
 
         for i in [0, 1, 2, 4, 8, 10]:
             admin = i == 2 or i == 4
-            OrganizationRole.objects.create(
+            models.OrganizationRole.objects.create(
                 organization=orgs[1], user=users[i], admin=admin
             )
 
@@ -109,12 +108,12 @@ def add_test_organizations(self):
         ))
 
         print('\nSuccessfully added organizations:')
-        for org in Organization.objects.all():
+        for org in models.Organization.objects.all():
             print(org.name)
 
     def add_test_projects(self):
         projs = []
-        orgs = Organization.objects.filter(name__contains='Test')
+        orgs = models.Organization.objects.filter(name__contains='Test')
 
         projs.append(ProjectFactory.create(
             name='Kibera Test Project',
@@ -289,11 +288,11 @@ def add_test_projects(self):
         ))
 
         print('\nSuccessfully added projects:')
-        for proj in Project.objects.all():
+        for proj in models.Project.objects.all():
             print(proj.name)
 
     def add_test_spatial_units(self):
-        project = Project.objects.get(
+        project = models.Project.objects.get(
             name__contains='Pekapuran Laut Test Project')
 
         # add attribute schema
@@ -391,12 +390,12 @@ def add_test_spatial_units(self):
                                   '-4.9383544921875,'
                                   '7.833452408875349'
                                   ']}'),
-            project=Project.objects.get(name='Kibera Test Project'),
+            project=models.Project.objects.get(name='Kibera Test Project'),
             type='MI',
             attributes={})
 
-    def add_huge_project(self):
-        project = Project.objects.get(slug='london-2')
+    def add_huge_project(self, max_num_records=4000):
+        project = models.Project.objects.get(slug='london-2')
         content_type = ContentType.objects.get(
             app_label='spatial', model='spatialunit')
         attr_type = AttributeType.objects.get(name="text")
@@ -409,52 +408,50 @@ def add_huge_project(self):
         )
 
         spatial_units = []
-        for i in range(0, 4000):
-            row = i // 63
-            col = i - (row * 63)
-
-            north_west = [11.67 + (col * 0.025),
-                          50.88 + (row * 0.025)]
-            south_east = [11.67 + ((col + 1) * 0.025),
-                          50.88 + ((row + 1) * 0.025)]
-
-            geometry = {
-                'type': 'Polygon',
-                'coordinates': [[
-                    north_west,
-                    [north_west[0], south_east[1]],
-                    south_east,
-                    [south_east[0], north_west[1]],
-                    north_west
-                ]]
-            }
-            name = 'Spatial Unit #{}'.format(i)
-            type = random.choice([c[0] for c in TYPE_CHOICES])
-
-            spatial_units.append({
-                'geometry': GEOSGeometry(json.dumps(geometry)),
-                'type': type,
-                'project': project,
-                'attributes': {'name': name}
-            })
+        choices = [c[0] for c in TYPE_CHOICES]
+
+        with open(os.path.join(os.path.dirname(__file__),
+                               "londondata.txt"), "r") as ins:
+            for geometry in ins:
+                if not geometry.rstrip() or geometry.startswith('#'):
+                    continue
+
+                num_records = len(spatial_units)
+                if not num_records < max_num_records:
+                    break
+
+                name = 'Spatial Unit #{}'.format(num_records)
+                type = random.choice(choices)
+
+                spatial_units.append({
+                    'geometry': GEOSGeometry(geometry),
+                    'type': type,
+                    'project': project,
+                    'attributes': {'name': name}
+                })
 
         SpatialUnitFactory.create_from_kwargs(spatial_units)
 
     def delete_test_organizations(self):
-        orgs = Organization.objects.filter(name__contains='Test')
+        orgs = models.Organization.objects.filter(name__contains='Test')
         for org in orgs:
             org.delete()
 
     def delete_test_users(self):
         users = User.objects.filter(username__startswith='testuser')
         for user in users:
+            models.ProjectRole.objects.filter(user=user).delete()
+            models.OrganizationRole.objects.filter(user=user).delete()
             user.delete()
         # Specified named users.
         named_users = ['iross', 'oroick']
         for user in named_users:
             if User.objects.filter(username=user).exists():
-                User.objects.get(username=user).delete()
+                u = User.objects.get(username=user)
+                models.ProjectRole.objects.filter(user=u).delete()
+                models.OrganizationRole.objects.filter(user=u).delete()
+                u.delete()
 
     def delete_test_projects(self):
-        projs = Project.objects.filter(name__contains='Test Project')
+        projs = models.Project.objects.filter(name__contains='Test Project')
         projs.delete()