Add Domain model to track project Domains

.travis.yml

@@ -16,6 +16,6 @@ after_success:
 notifications:
   slack:
     rooms:
-      - readthedocs:y3hjODOi7EIz1JAbD1Zb41sz#general
+      - readthedocs:y3hjODOi7EIz1JAbD1Zb41sz#random
     on_success: change
     on_failure: always

readthedocs/core/middleware.py

@@ -7,7 +7,7 @@
 from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned
 from django.http import Http404
 
-from readthedocs.projects.models import Project
+from readthedocs.projects.models import Project, Domain
 
 import redis
 
@@ -71,6 +71,22 @@ def process_request(self, request):
                     log.debug(LOG_TEMPLATE.format(
                         msg='CNAME detetected: %s' % request.slug,
                         **log_kwargs))
+                    try:
+                        proj = Project.objects.get(slug=slug)
+                        domain, created = Domain.objects.get_or_create(
+                            project=proj,
+                            url=host,
+                        )
+                        if created:
+                            domain.machine = True
+                        domain.cname = True
+                        # Track basic domain counts so we know which are heavily used
+                        domain.count = domain.count + 1
+                        domain.save()
+                    except (ObjectDoesNotExist, MultipleObjectsReturned):
+                        log.debug(LOG_TEMPLATE.format(
+                            msg='Project CNAME does not exist: %s' % slug,
+                            **log_kwargs))
                 except:
                     # Some crazy person is CNAMEing to us. 404.
                     log.exception(LOG_TEMPLATE.format(msg='CNAME 404', **log_kwargs))

readthedocs/core/mixins.py

@@ -1,16 +1,29 @@
-'''
+"""
 Common mixin classes for views
-'''
+"""
 
 from django.conf import settings
 
+from vanilla import ListView
+
 
 class StripeMixin(object):
-    '''Adds Stripe publishable key to the context data'''
+
+    """Adds Stripe publishable key to the context data"""
 
     def get_context_data(self, **kwargs):
         context = super(StripeMixin, self).get_context_data(**kwargs)
         context.update({
             'publishable': settings.STRIPE_PUBLISHABLE,
         })
         return context
+
+
+class ListViewWithForm(ListView):
+
+    """List view that also exposes a create form"""
+
+    def get_context_data(self, **kwargs):
+        context = super(ListViewWithForm, self).get_context_data(**kwargs)
+        context['form'] = self.get_form(data=None, files=None)
+        return context

readthedocs/projects/admin.py

@@ -3,9 +3,8 @@
 from readthedocs.builds.models import Version
 from django.contrib import admin
 from readthedocs.redirects.models import Redirect
-from readthedocs.projects.models import (Project, ImportedFile,
-                                         ProjectRelationship, EmailHook,
-                                         WebHook)
+from .models import (Project, ImportedFile,
+                     ProjectRelationship, EmailHook, WebHook, Domain)
 from guardian.admin import GuardedModelAdmin
 
 
@@ -32,6 +31,10 @@ class RedirectInline(admin.TabularInline):
     model = Redirect
 
 
+class DomainInline(admin.TabularInline):
+    model = Domain
+
+
 class ProjectAdmin(GuardedModelAdmin):
 
     """Project model admin view"""
@@ -42,7 +45,7 @@ class ProjectAdmin(GuardedModelAdmin):
                    'documentation_type', 'programming_language')
     list_editable = ('featured',)
     search_fields = ('slug', 'repo')
-    inlines = [ProjectRelationshipInline, RedirectInline, VersionInline]
+    inlines = [ProjectRelationshipInline, RedirectInline, VersionInline, DomainInline]
     raw_id_fields = ('users', 'main_language_project')
 
 
@@ -53,7 +56,12 @@ class ImportedFileAdmin(admin.ModelAdmin):
     list_display = ('path', 'name', 'version')
 
 
+class DomainAdmin(admin.ModelAdmin):
+    list_display = ('url', 'project')
+    model = Domain
+
 admin.site.register(Project, ProjectAdmin)
 admin.site.register(ImportedFile, ImportedFileAdmin)
+admin.site.register(Domain, DomainAdmin)
 admin.site.register(EmailHook)
 admin.site.register(WebHook)

readthedocs/projects/forms.py

@@ -16,7 +16,7 @@
 from readthedocs.core.utils import trigger_build
 from readthedocs.redirects.models import Redirect
 from readthedocs.projects import constants
-from readthedocs.projects.models import Project, EmailHook, WebHook
+from readthedocs.projects.models import Project, EmailHook, WebHook, Domain
 from readthedocs.privacy.loader import AdminPermission
 
 
@@ -126,20 +126,9 @@ class Meta:
             'documentation_type',
             'language', 'programming_language',
             'project_url',
-            'canonical_url',
             'tags',
         )
 
-    def __init__(self, *args, **kwargs):
-        super(ProjectExtraForm, self).__init__(*args, **kwargs)
-        self.fields['canonical_url'].widget.attrs['placeholder'] = self.placehold_canonical_url()
-
-    def placehold_canonical_url(self):
-        return choice([
-            'http://docs.fabfile.org',
-            'http://example.readthedocs.org',
-        ])
-
 
 class ProjectAdvancedForm(ProjectTriggerBuildMixin, ProjectForm):
 
@@ -198,7 +187,6 @@ class Meta:
             'documentation_type',
             'language', 'programming_language',
             'project_url',
-            'canonical_url',
             'tags',
         )
 
@@ -476,3 +464,26 @@ def save(self, **_):
             to_url=self.cleaned_data['to_url'],
         )
         return redirect
+
+
+class DomainForm(forms.ModelForm):
+    project = forms.CharField(widget=forms.HiddenInput(), required=False)
+
+    class Meta:
+        model = Domain
+        exclude = ['machine', 'cname', 'count']
+
+    def __init__(self, *args, **kwargs):
+        self.project = kwargs.pop('project', None)
+        super(DomainForm, self).__init__(*args, **kwargs)
+
+    def clean_project(self):
+        return self.project
+
+    def clean_canonical(self):
+        canonical = self.cleaned_data['canonical']
+        if canonical and Domain.objects.filter(
+            project=self.project, canonical=True
+        ).exclude(url=self.cleaned_data['url']).exists():
+            raise forms.ValidationError(_(u'Only 1 Domain can be canonical at a time.'))
+        return canonical

readthedocs/projects/migrations/0006_add_domain_models.py

@@ -0,0 +1,29 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import models, migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('projects', '0005_sync_project_model'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Domain',
+            fields=[
+                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
+                ('url', models.URLField(unique=True, verbose_name='URL')),
+                ('machine', models.BooleanField(default=False, help_text='This URL was auto-created')),
+                ('cname', models.BooleanField(default=False, help_text='This URL is a CNAME for the project')),
+                ('canonical', models.BooleanField(default=False, help_text='This URL is the primary one where the documentation is served from.')),
+                ('count', models.IntegerField(default=0, help_text='Number of times this domain has been hit.')),
+                ('project', models.ForeignKey(related_name='domains', to='projects.Project')),
+            ],
+            options={
+                'ordering': ('-canonical', '-machine', 'url'),
+            },
+        ),
+    ]

readthedocs/projects/migrations/0007_migrate_canonical_data.py

@@ -0,0 +1,26 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+
+def migrate_canonical(apps, schema_editor):
+    Project = apps.get_model("projects", "Project")
+    for project in Project.objects.all():
+        if project.canonical_url:
+            domain = project.domains.create(
+                url=project.canonical_url,
+                canonical=True,
+            )
+            print "Added {url} to {project}".format(url=domain.url, project=project.name)
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('projects', '0006_add_domain_models'),
+    ]
+
+    operations = [
+        migrations.RunPython(migrate_canonical)
+    ]

readthedocs/projects/models.py

@@ -7,6 +7,7 @@
 
 from django.conf import settings
 from django.contrib.auth.models import User
+from django.core.exceptions import MultipleObjectsReturned
 from django.core.urlresolvers import reverse
 from django.db import models
 from django.template.defaultfilters import slugify
@@ -437,22 +438,26 @@ def canonical_domain(self):
 
     @property
     def clean_canonical_url(self):
-        """Normalize canonical URL field"""
-        if not self.canonical_url:
-            return ""
-        parsed = urlparse(self.canonical_url)
+        try:
+            domain = self.domains.get(canonical=True)
+        except (Domain.DoesNotExist, MultipleObjectsReturned):
+            return ''
+
+        parsed = urlparse(domain.url)
         if parsed.scheme:
             scheme, netloc = parsed.scheme, parsed.netloc
         elif parsed.netloc:
             scheme, netloc = "http", parsed.netloc
         else:
             scheme, netloc = "http", parsed.path
+
         if getattr(settings, 'DONT_HIT_DB', True):
             if parsed.path:
                 netloc = netloc + parsed.path
         else:
             if self.superprojects.count() and parsed.path:
                 netloc = netloc + parsed.path
+
         return "%s://%s/" % (scheme, netloc)
 
     @property
@@ -933,3 +938,35 @@ class WebHook(Notification):
 
     def __unicode__(self):
         return self.url
+
+
+class Domain(models.Model):
+    project = models.ForeignKey(Project, related_name='domains')
+    url = models.URLField(_('URL'), unique=True)
+    machine = models.BooleanField(
+        default=False, help_text=_('This URL was auto-created')
+    )
+    cname = models.BooleanField(
+        default=False, help_text=_('This URL is a CNAME for the project')
+    )
+    canonical = models.BooleanField(
+        default=False,
+        help_text=_('This URL is the primary one where the documentation is served from.')
+    )
+    count = models.IntegerField(default=0, help_text=_('Number of times this domain has been hit.'))
+
+    objects = RelatedProjectManager()
+
+    class Meta:
+        ordering = ('-canonical', '-machine', 'url')
+
+    def __unicode__(self):
+        return "{url} pointed at {project}".format(url=self.url, project=self.project.name)
+
+    @property
+    def clean_host(self):
+        parsed = urlparse(self.url)
+        if parsed.scheme or parsed.netloc:
+            return parsed.netloc
+        else:
+            return parsed.path

readthedocs/projects/symlinks.py

@@ -4,10 +4,10 @@
 import logging
 
 from django.conf import settings
-import redis
 
 from readthedocs.core.utils import run_on_app_servers
 from readthedocs.projects.constants import LOG_TEMPLATE
+from readthedocs.projects.models import Domain
 from readthedocs.restapi.client import api
 
 log = logging.getLogger(__name__)
@@ -23,29 +23,27 @@ def symlink_cnames(version):
     Link from HOME/user_builds/cnametoproject/<cname> ->
               HOME/user_builds/<project>/
     """
-    try:
-        redis_conn = redis.Redis(**settings.REDIS)
-        cnames = redis_conn.smembers('rtd_slug:v1:%s' % version.project.slug)
-    except redis.ConnectionError:
-        log.error(LOG_TEMPLATE
-                  .format(project=version.project.slug, version=version.slug,
-                          msg='Failed to symlink cnames, Redis error.'),
-                  exc_info=True)
-        return
-    for cname in cnames:
-        log.debug(LOG_TEMPLATE
-                  .format(project=version.project.slug, version=version.slug,
-                          msg="Symlinking CNAME: %s" % cname))
+    domains = Domain.objects.filter(project=version.project, cname=True)
+    for domain in domains:
+        log.debug(LOG_TEMPLATE.format(
+            project=version.project.slug,
+            version=version.slug,
+            msg="Symlinking CNAME: %s" % domain.clean_host)
+        )
         docs_dir = version.project.rtd_build_path(version.slug)
         # Chop off the version from the end.
         docs_dir = '/'.join(docs_dir.split('/')[:-1])
         # Old symlink location -- Keep this here til we change nginx over
-        symlink = version.project.cnames_symlink_path(cname)
+        symlink = version.project.cnames_symlink_path(domain.clean_host)
         run_on_app_servers('mkdir -p %s' % '/'.join(symlink.split('/')[:-1]))
         run_on_app_servers('ln -nsf %s %s' % (docs_dir, symlink))
         # New symlink location
         new_docs_dir = version.project.doc_path
-        new_cname_symlink = os.path.join(getattr(settings, 'SITE_ROOT'), 'cnametoproject', cname)
+        new_cname_symlink = os.path.join(
+            getattr(settings, 'SITE_ROOT'),
+            'cnametoproject',
+            domain.clean_host,
+        )
         run_on_app_servers('mkdir -p %s' % '/'.join(new_cname_symlink.split('/')[:-1]))
         run_on_app_servers('ln -nsf %s %s' % (new_docs_dir, new_cname_symlink))