Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Use DRF 3.1 pagination_class #3559

Merged
merged 5 commits into from
Feb 15, 2018
Merged

Use DRF 3.1 pagination_class #3559

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
37d97c0
Use DRF 3.1 `pagination_class`
humitos Jan 27, 2018
8ba9e1a
Update pre-commit and styles
humitos Jan 27, 2018
87f046a
Lint
humitos Jan 27, 2018
d066d65
Merge pull request #3560 from rtfd/humitos/style/import/organization
ericholscher Jan 30, 2018
323d650
Pagination tests
humitos Jan 30, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 4 additions & 3 deletions .pre-commit-config.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ exclude: '^$'
fail_fast: false
repos:
- repo: [email protected]:pre-commit/pre-commit-hooks
sha: v1.1.1
sha: v1.2.0
hooks:
- id: autopep8-wrapper
- id: check-added-large-files
Expand Down Expand Up @@ -30,7 +30,7 @@ repos:
- id: trailing-whitespace

- repo: [email protected]:pre-commit/mirrors-yapf.git
sha: v0.20.0
sha: v0.20.1
hooks:
- id: yapf
exclude: 'migrations|settings|scripts'
Expand All @@ -56,9 +56,10 @@ repos:
args: ['--in-place', '--remove-all-unused-imports', '--remove-unused-variable']

- repo: git://github.com/guykisel/prospector-mirror
sha: 'b27f281eb9398fc8504415d7fbdabf119ea8c5e1'
sha: b27f281eb9398fc8504415d7fbdabf119ea8c5e1
hooks:
- id: prospector
# https://github.com/pre-commit/pre-commit/issues/178
language: system
files: '\.py$'
args: ['--profile=prospector']
127 changes: 84 additions & 43 deletions readthedocs/restapi/utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,14 @@
# -*- coding: utf-8 -*-
"""Utility functions that are used by both views and celery tasks."""

from __future__ import absolute_import
from __future__ import (
absolute_import, division, print_function, unicode_literals)

import hashlib
import logging

from rest_framework.pagination import PageNumberPagination

from readthedocs.builds.constants import NON_REPOSITORY_VERSIONS
from readthedocs.builds.models import Version
from readthedocs.search.indexes import PageIndex, ProjectIndex, SectionIndex
Expand Down Expand Up @@ -31,14 +36,17 @@ def sync_versions(project, versions, type): # pylint: disable=redefined-builtin
else:
# Update slug with new identifier
Version.objects.filter(
project=project, verbose_name=version_name
).update(
identifier=version_id,
type=type,
machine=False,
project=project, verbose_name=version_name).update(
identifier=version_id,
type=type,
machine=False,
) # noqa

log.info(
'(Sync Versions) Updated Version: [%s=%s] ',
version['verbose_name'],
version['identifier'],
)
log.info("(Sync Versions) Updated Version: [%s=%s] ",
version['verbose_name'], version['identifier'])
else:
# New Version
created_version = Version.objects.create(
Expand All @@ -49,7 +57,7 @@ def sync_versions(project, versions, type): # pylint: disable=redefined-builtin
)
added.add(created_version.slug)
if added:
log.info("(Sync Versions) Added Versions: [%s] ", ' '.join(added))
log.info('(Sync Versions) Added Versions: [%s] ', ' '.join(added))
return added


Expand All @@ -70,14 +78,15 @@ def delete_versions(project, version_data):

if to_delete_qs.count():
ret_val = {obj.slug for obj in to_delete_qs}
log.info("(Sync Versions) Deleted Versions: [%s]", ' '.join(ret_val))
log.info('(Sync Versions) Deleted Versions: [%s]', ' '.join(ret_val))
to_delete_qs.delete()
return ret_val
return set()


def index_search_request(version, page_list, commit, project_scale, page_scale,
section=True, delete=True):
def index_search_request(
version, page_list, commit, project_scale, page_scale, section=True,
delete=True):
"""
Update search indexes with build output JSON.

Expand All @@ -89,21 +98,25 @@ def index_search_request(version, page_list, commit, project_scale, page_scale,
project = version.project

log_msg = ' '.join([page['path'] for page in page_list])
log.info("Updating search index: project=%s pages=[%s]",
project.slug, log_msg)
log.info(
'Updating search index: project=%s pages=[%s]',
project.slug,
log_msg,
)

project_obj = ProjectIndex()
project_obj.index_document(data={
'id': project.pk,
'name': project.name,
'slug': project.slug,
'description': project.description,
'lang': project.language,
'author': [user.username for user in project.users.all()],
'url': project.get_absolute_url(),
'tags': None,
'weight': project_scale,
})
project_obj.index_document(
data={
'id': project.pk,
'name': project.name,
'slug': project.slug,
'description': project.description,
'lang': project.language,
'author': [user.username for user in project.users.all()],
'url': project.get_absolute_url(),
'tags': None,
'weight': project_scale,
})

page_obj = PageIndex()
section_obj = SectionIndex()
Expand All @@ -112,7 +125,7 @@ def index_search_request(version, page_list, commit, project_scale, page_scale,
routes = [project.slug]
routes.extend([p.parent.slug for p in project.superprojects.all()])
for page in page_list:
log.debug("Indexing page: %s:%s", project.slug, page['path'])
log.debug('Indexing page: %s:%s', project.slug, page['path'])
to_hash = '-'.join([project.slug, version.slug, page['path']])
page_id = hashlib.md5(to_hash.encode('utf-8')).hexdigest()
index_list.append({
Expand All @@ -129,8 +142,12 @@ def index_search_request(version, page_list, commit, project_scale, page_scale,
})
if section:
for sect in page['sections']:
id_to_hash = '-'.join([project.slug, version.slug,
page['path'], sect['id']])
id_to_hash = '-'.join([
project.slug,
version.slug,
page['path'],
sect['id'],
])
section_index_list.append({
'id': (hashlib.md5(id_to_hash.encode('utf-8')).hexdigest()),
'project': project.slug,
Expand All @@ -142,28 +159,52 @@ def index_search_request(version, page_list, commit, project_scale, page_scale,
'weight': page_scale,
})
for route in routes:
section_obj.bulk_index(section_index_list, parent=page_id,
routing=route)
section_obj.bulk_index(
section_index_list,
parent=page_id,
routing=route,
)

for route in routes:
page_obj.bulk_index(index_list, parent=project.slug, routing=route)

if delete:
log.info("Deleting files not in commit: %s", commit)
log.info('Deleting files not in commit: %s', commit)
# TODO: AK Make sure this works
delete_query = {
"query": {
"bool": {
"must": [
{"term": {"project": project.slug, }},
{"term": {"version": version.slug, }},
'query': {
'bool': {
'must': [
{
'term': {
'project': project.slug,
},
},
{
'term': {
'version': version.slug,
},
},
],
"must_not": {
"term": {
"commit": commit
}
}
}
}
'must_not': {
'term': {
'commit': commit,
},
},
},
},
}
page_obj.delete_document(body=delete_query)


class RemoteOrganizationPagination(PageNumberPagination):
page_size = 25


class RemoteProjectPagination(PageNumberPagination):
page_size = 15


class ProjectPagination(PageNumberPagination):
page_size = 100
max_page_size = 1000
Loading