Skip to content

Commit

Permalink
pre-commit run for modified files
Browse files Browse the repository at this point in the history
  • Loading branch information
humitos committed Apr 4, 2018
1 parent 2f2bd2f commit 2e89010
Show file tree
Hide file tree
Showing 21 changed files with 773 additions and 760 deletions.
7 changes: 4 additions & 3 deletions readthedocs/builds/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,15 +7,15 @@
import logging
import os.path
import re
from builtins import object
from shutil import rmtree

from builtins import object
from django.conf import settings
from django.core.urlresolvers import reverse
from django.db import models
from django.utils.encoding import python_2_unicode_compatible
from django.utils.translation import ugettext_lazy as _
from django.utils.translation import ugettext
from django.utils.translation import ugettext_lazy as _
from guardian.shortcuts import assign
from taggit.managers import TaggableManager

Expand Down Expand Up @@ -148,7 +148,8 @@ def commit_name(self):

# If we came that far it's not a special version nor a branch or tag.
# Therefore just return the identifier to make a safe guess.
log.debug('TODO: Raise an exception here. Testing what cases it happens')
log.debug(
'TODO: Raise an exception here. Testing what cases it happens')
return self.identifier

def get_absolute_url(self):
Expand Down
51 changes: 23 additions & 28 deletions readthedocs/builds/syncers.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,13 @@
# -*- coding: utf-8 -*-
"""
Classes to copy files between build and web servers.
"Syncers" copy files from the local machine, while "Pullers" copy files to the
local machine.
"""

from __future__ import absolute_import
from __future__ import (
absolute_import, division, print_function, unicode_literals)

import getpass
import logging
Expand All @@ -15,9 +17,8 @@
from builtins import object
from django.conf import settings

from readthedocs.core.utils.extend import SettingsOverrideObject
from readthedocs.core.utils import safe_makedirs

from readthedocs.core.utils.extend import SettingsOverrideObject

log = logging.getLogger(__name__)

Expand All @@ -27,7 +28,7 @@ class LocalSyncer(object):
@classmethod
def copy(cls, path, target, is_file=False, **__):
"""A copy command that works with files or directories."""
log.info("Local Copy %s to %s", path, target)
log.info('Local Copy %s to %s', path, target)
if is_file:
if path == target:
# Don't copy the same file over itself
Expand All @@ -53,28 +54,26 @@ def copy(cls, path, target, is_file=False, **__):
sync_user = getattr(settings, 'SYNC_USER', getpass.getuser())
app_servers = getattr(settings, 'MULTIPLE_APP_SERVERS', [])
if app_servers:
log.info("Remote Copy %s to %s on %s", path, target, app_servers)
log.info('Remote Copy %s to %s on %s', path, target, app_servers)
for server in app_servers:
mkdir_cmd = ("ssh %s@%s mkdir -p %s" % (sync_user, server, target))
mkdir_cmd = (
'ssh %s@%s mkdir -p %s' % (sync_user, server, target))
ret = os.system(mkdir_cmd)
if ret != 0:
log.error("Copy error to app servers: cmd=%s", mkdir_cmd)
log.error('Copy error to app servers: cmd=%s', mkdir_cmd)
if is_file:
slash = ""
slash = ''
else:
slash = "/"
slash = '/'
# Add a slash when copying directories
sync_cmd = (
"rsync -e 'ssh -T' -av --delete {path}{slash} {user}@{server}:{target}"
.format(
path=path,
slash=slash,
user=sync_user,
server=server,
path=path, slash=slash, user=sync_user, server=server,
target=target))
ret = os.system(sync_cmd)
if ret != 0:
log.error("Copy error to app servers: cmd=%s", sync_cmd)
log.error('Copy error to app servers: cmd=%s', sync_cmd)


class DoubleRemotePuller(object):
Expand All @@ -89,29 +88,25 @@ def copy(cls, path, target, host, is_file=False, **__):
sync_user = getattr(settings, 'SYNC_USER', getpass.getuser())
app_servers = getattr(settings, 'MULTIPLE_APP_SERVERS', [])
if not is_file:
path += "/"
log.info("Remote Copy %s to %s", path, target)
path += '/'
log.info('Remote Copy %s to %s', path, target)
for server in app_servers:
if not is_file:
mkdir_cmd = "ssh {user}@{server} mkdir -p {target}".format(
user=sync_user, server=server, target=target
)
mkdir_cmd = 'ssh {user}@{server} mkdir -p {target}'.format(
user=sync_user, server=server, target=target)
ret = os.system(mkdir_cmd)
if ret != 0:
log.error("MkDir error to app servers: cmd=%s", mkdir_cmd)
log.error('MkDir error to app servers: cmd=%s', mkdir_cmd)
# Add a slash when copying directories
sync_cmd = (
"ssh {user}@{server} 'rsync -av "
"--delete --exclude projects {user}@{host}:{path} {target}'"
.format(
host=host,
path=path,
user=sync_user,
server=server,
host=host, path=path, user=sync_user, server=server,
target=target))
ret = os.system(sync_cmd)
if ret != 0:
log.error("Copy error to app servers: cmd=%s", sync_cmd)
log.error('Copy error to app servers: cmd=%s', sync_cmd)


class RemotePuller(object):
Expand All @@ -125,8 +120,8 @@ def copy(cls, path, target, host, is_file=False, **__):
"""
sync_user = getattr(settings, 'SYNC_USER', getpass.getuser())
if not is_file:
path += "/"
log.info("Remote Pull %s to %s", path, target)
path += '/'
log.info('Remote Pull %s to %s', path, target)
if not is_file and not os.path.exists(target):
safe_makedirs(target)
# Add a slash when copying directories
Expand All @@ -139,7 +134,7 @@ def copy(cls, path, target, host, is_file=False, **__):
ret = os.system(sync_cmd)
if ret != 0:
log.error(
"Copy error to app servers. Command: [%s] Return: [%s]",
'Copy error to app servers. Command: [%s] Return: [%s]',
sync_cmd,
ret,
)
Expand Down
35 changes: 17 additions & 18 deletions readthedocs/core/management/commands/clean_builds.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,11 @@
"""Clean up stable build paths per project version"""
# -*- coding: utf-8 -*-
"""Clean up stable build paths per project version."""

from __future__ import (
absolute_import, division, print_function, unicode_literals)

from __future__ import absolute_import
from datetime import datetime, timedelta
import logging
from datetime import datetime, timedelta
from optparse import make_option

from django.core.management.base import BaseCommand
Expand All @@ -18,25 +21,21 @@ class Command(BaseCommand):
help = __doc__

option_list = BaseCommand.option_list + (
make_option('--days',
dest='days',
type='int',
default=365,
help='Find builds older than DAYS days, default: 365'),
make_option('--dryrun',
action='store_true',
dest='dryrun',
help='Perform dry run on build cleanup'),
make_option(
'--days', dest='days', type='int', default=365,
help='Find builds older than DAYS days, default: 365'),
make_option(
'--dryrun', action='store_true', dest='dryrun',
help='Perform dry run on build cleanup'),
)

def handle(self, *args, **options):
"""Find stale builds and remove build paths"""
"""Find stale builds and remove build paths."""
max_date = datetime.now() - timedelta(days=options['days'])
queryset = (Build.objects
.values('project', 'version')
.annotate(max_date=Max('date'))
.filter(max_date__lt=max_date)
.order_by('-max_date'))
queryset = (
Build.objects.values('project',
'version').annotate(max_date=Max('date'))
.filter(max_date__lt=max_date).order_by('-max_date'))
for build in queryset:
try:
# Get version from build version id, perform sanity check on
Expand Down
25 changes: 11 additions & 14 deletions readthedocs/core/management/commands/reindex_elasticsearch.py
Original file line number Diff line number Diff line change
@@ -1,12 +1,14 @@
"""Reindex Elastic Search indexes"""
# -*- coding: utf-8 -*-
"""Reindex Elastic Search indexes."""

from __future__ import (
absolute_import, division, print_function, unicode_literals)

from __future__ import absolute_import
import logging
from optparse import make_option

from django.core.management.base import BaseCommand
from django.core.management.base import CommandError
from django.conf import settings
from django.core.management.base import BaseCommand, CommandError

from readthedocs.builds.constants import LATEST
from readthedocs.builds.models import Version
Expand All @@ -19,14 +21,10 @@ class Command(BaseCommand):

help = __doc__
option_list = BaseCommand.option_list + (
make_option('-p',
dest='project',
default='',
help='Project to index'),
)
make_option('-p', dest='project', default='', help='Project to index'),)

def handle(self, *args, **options):
"""Build/index all versions or a single project's version"""
"""Build/index all versions or a single project's version."""
project = options['project']

queryset = Version.objects.all()
Expand All @@ -36,12 +34,12 @@ def handle(self, *args, **options):
if not queryset.exists():
raise CommandError(
'No project with slug: {slug}'.format(slug=project))
log.info("Building all versions for %s", project)
log.info('Building all versions for %s', project)
elif getattr(settings, 'INDEX_ONLY_LATEST', True):
queryset = queryset.filter(slug=LATEST)

for version in queryset:
log.info("Reindexing %s", version)
log.info('Reindexing %s', version)
try:
commit = version.project.vcs_repo(version.slug).commit
except: # pylint: disable=bare-except
Expand All @@ -50,7 +48,6 @@ def handle(self, *args, **options):
commit = None

try:
update_search(version.pk, commit,
delete_non_commit_files=False)
update_search(version.pk, commit, delete_non_commit_files=False)
except Exception:
log.exception('Reindex failed for %s', version)
23 changes: 12 additions & 11 deletions readthedocs/core/signals.py
Original file line number Diff line number Diff line change
@@ -1,18 +1,19 @@
# -*- coding: utf-8 -*-
"""Signal handling for core app."""

from __future__ import absolute_import
from __future__ import (
absolute_import, division, print_function, unicode_literals)

import logging

from corsheaders import signals
from django.conf import settings
from django.db.models import Count, Q
from django.db.models.signals import pre_delete
from django.dispatch import Signal
from django.db.models import Q, Count
from django.dispatch import receiver
from django.dispatch import Signal, receiver
from future.backports.urllib.parse import urlparse

from readthedocs.projects.models import Project, Domain
from readthedocs.projects.models import Domain, Project

log = logging.getLogger(__name__)

Expand All @@ -23,7 +24,6 @@
'/api/v2/sustainability',
]


webhook_github = Signal(providing_args=['project', 'data', 'event'])
webhook_gitlab = Signal(providing_args=['project', 'data', 'event'])
webhook_bitbucket = Signal(providing_args=['project', 'data', 'event'])
Expand Down Expand Up @@ -65,8 +65,7 @@ def decide_if_cors(sender, request, **kwargs): # pylint: disable=unused-argumen

domain = Domain.objects.filter(
Q(domain__icontains=host),
Q(project=project) | Q(project__subprojects__child=project)
)
Q(project=project) | Q(project__subprojects__child=project))
if domain.exists():
return True

Expand All @@ -77,12 +76,14 @@ def decide_if_cors(sender, request, **kwargs): # pylint: disable=unused-argumen
def delete_projects_and_organizations(sender, instance, *args, **kwargs):
# Here we count the owner list from the projects that the user own
# Then exclude the projects where there are more than one owner
projects = instance.projects.all().annotate(num_users=Count('users')).exclude(num_users__gt=1)
projects = instance.projects.all().annotate(
num_users=Count('users')).exclude(num_users__gt=1)

# Here we count the users list from the organization that the user belong
# Then exclude the organizations where there are more than one user
oauth_organizations = (instance.oauth_organizations.annotate(num_users=Count('users'))
.exclude(num_users__gt=1))
oauth_organizations = (
instance.oauth_organizations.annotate(num_users=Count('users'))
.exclude(num_users__gt=1))

projects.delete()
oauth_organizations.delete()
Expand Down
Loading

0 comments on commit 2e89010

Please sign in to comment.