diff --git a/readthedocs/core/views/hooks.py b/readthedocs/core/views/hooks.py index 1110853d0a6..6b00b2d77ef 100644 --- a/readthedocs/core/views/hooks.py +++ b/readthedocs/core/views/hooks.py @@ -1,21 +1,26 @@ """Views pertaining to builds.""" -from __future__ import absolute_import +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import json +import logging import re from django.http import HttpResponse, HttpResponseNotFound from django.shortcuts import redirect from django.views.decorators.csrf import csrf_exempt -from readthedocs.core.utils import trigger_build from readthedocs.builds.constants import LATEST +from readthedocs.core.utils import trigger_build from readthedocs.projects import constants -from readthedocs.projects.models import Project, Feature +from readthedocs.projects.models import Feature, Project from readthedocs.projects.tasks import sync_repository_task -import logging - log = logging.getLogger(__name__) @@ -75,6 +80,35 @@ def build_branches(project, branch_list): return (to_build, not_building) +def sync_versions(project): + """ + Sync the versions of a repo using its latest version. + + This doesn't register a new build, + but clones the repo and syncs the versions. + Due that `sync_repository_task` is bound to a version, + we always pass the default version. + + :returns: The version slug that was used to trigger the clone. + :rtype: str + """ + try: + version_identifier = project.get_default_branch() + version = ( + project.versions + .filter(identifier=version_identifier) + .first() + ) + if not version: + log.info('Unable to sync from %s version', version_identifier) + return None + sync_repository_task.delay(version.pk) + return version.slug + except Exception: + log.exception('Unknown sync versions exception') + return None + + def get_project_from_url(url): if not url: return Project.objects.none() diff --git a/readthedocs/oauth/services/github.py b/readthedocs/oauth/services/github.py index 7f4c9c75786..6106d5cab7b 100644 --- a/readthedocs/oauth/services/github.py +++ b/readthedocs/oauth/services/github.py @@ -172,7 +172,7 @@ def get_webhook_data(self, project, integration): ), 'content_type': 'json', }, - 'events': ['push', 'pull_request'], + 'events': ['push', 'pull_request', 'create', 'delete'], }) def setup_webhook(self, project): diff --git a/readthedocs/restapi/views/integrations.py b/readthedocs/restapi/views/integrations.py index f2ec92dd030..008ae1dd826 100644 --- a/readthedocs/restapi/views/integrations.py +++ b/readthedocs/restapi/views/integrations.py @@ -1,32 +1,44 @@ """Endpoints integrating with Github, Bitbucket, and other webhooks.""" -from __future__ import absolute_import +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import json import logging import re -from builtins import object +import six +from django.shortcuts import get_object_or_404 from rest_framework import permissions -from rest_framework.views import APIView +from rest_framework.exceptions import NotFound, ParseError from rest_framework.renderers import JSONRenderer from rest_framework.response import Response -from rest_framework.exceptions import ParseError, NotFound - -from django.shortcuts import get_object_or_404 +from rest_framework.views import APIView -from readthedocs.core.views.hooks import build_branches -from readthedocs.core.signals import (webhook_github, webhook_bitbucket, - webhook_gitlab) +from readthedocs.core.signals import ( + webhook_bitbucket, + webhook_github, + webhook_gitlab, +) +from readthedocs.core.views.hooks import build_branches, sync_versions from readthedocs.integrations.models import HttpExchange, Integration from readthedocs.integrations.utils import normalize_request_payload from readthedocs.projects.models import Project -import six - log = logging.getLogger(__name__) +GITHUB_EVENT_HEADER = 'HTTP_X_GITHUB_EVENT' GITHUB_PUSH = 'push' +GITHUB_CREATE = 'create' +GITHUB_DELETE = 'delete' GITLAB_PUSH = 'push' +GITLAB_NULL_HASH = '0' * 40 +GITLAB_TAG_PUSH = 'tag_push' +BITBUCKET_EVENT_HEADER = 'HTTP_X_EVENT_KEY' BITBUCKET_PUSH = 'repo:push' @@ -124,6 +136,14 @@ def get_response_push(self, project, branches): 'project': project.slug, 'versions': list(to_build)} + def sync_versions(self, project): + version = sync_versions(project) + return { + 'build_triggered': False, + 'project': project.slug, + 'versions': [version], + } + class GitHubWebhookView(WebhookMixin, APIView): @@ -140,6 +160,12 @@ class GitHubWebhookView(WebhookMixin, APIView): "ref": "branch-name", ... } + + See full payload here: + + - https://developer.github.com/v3/activity/events/types/#pushevent + - https://developer.github.com/v3/activity/events/types/#createevent + - https://developer.github.com/v3/activity/events/types/#deleteevent """ integration_type = Integration.GITHUB_WEBHOOK @@ -154,9 +180,13 @@ def get_data(self): def handle_webhook(self): # Get event and trigger other webhook events - event = self.request.META.get('HTTP_X_GITHUB_EVENT', 'push') - webhook_github.send(Project, project=self.project, - data=self.data, event=event) + event = self.request.META.get(GITHUB_EVENT_HEADER, GITHUB_PUSH) + webhook_github.send( + Project, + project=self.project, + data=self.data, + event=event + ) # Handle push events and trigger builds if event == GITHUB_PUSH: try: @@ -164,6 +194,9 @@ def handle_webhook(self): return self.get_response_push(self.project, branches) except KeyError: raise ParseError('Parameter "ref" is required') + if event in (GITHUB_CREATE, GITHUB_DELETE): + return self.sync_versions(self.project) + return None def _normalize_ref(self, ref): pattern = re.compile(r'^refs/(heads|tags)/') @@ -180,26 +213,55 @@ class GitLabWebhookView(WebhookMixin, APIView): Expects the following JSON:: { + "before": "95790bf891e76fee5e1747ab589903a6a1f80f22", + "after": "da1560886d4f094c3e6c9ef40349f7d38b5d27d7", "object_kind": "push", "ref": "branch-name", ... } + + See full payload here: + + - https://docs.gitlab.com/ce/user/project/integrations/webhooks.html#push-events + - https://docs.gitlab.com/ce/user/project/integrations/webhooks.html#tag-events """ integration_type = Integration.GITLAB_WEBHOOK def handle_webhook(self): - # Get event and trigger other webhook events + """ + Handle GitLab events for push and tag_push. + + GitLab doesn't have a separate event for creation/deletion, + instead, it sets the before/after field to + 0000000000000000000000000000000000000000 ('0' * 40) + """ event = self.request.data.get('object_kind', GITLAB_PUSH) - webhook_gitlab.send(Project, project=self.project, - data=self.request.data, event=event) + webhook_gitlab.send( + Project, + project=self.project, + data=self.request.data, + event=event + ) # Handle push events and trigger builds - if event == GITLAB_PUSH: + if event in (GITLAB_PUSH, GITLAB_TAG_PUSH): + data = self.request.data + before = data['before'] + after = data['after'] + # Tag/branch created/deleted + if GITLAB_NULL_HASH in (before, after): + return self.sync_versions(self.project) + # Normal push to master try: - branches = [self.request.data['ref'].replace('refs/heads/', '')] + branches = [self._normalize_ref(data['ref'])] return self.get_response_push(self.project, branches) except KeyError: raise ParseError('Parameter "ref" is required') + return None + + def _normalize_ref(self, ref): + pattern = re.compile(r'^refs/(heads|tags)/') + return pattern.sub('', ref) class BitbucketWebhookView(WebhookMixin, APIView): @@ -218,31 +280,60 @@ class BitbucketWebhookView(WebhookMixin, APIView): "name": "branch-name", ... }, + "old" { + "name": "branch-name", + ... + }, ... }], ... }, ... } + + See full payload here: + + - https://confluence.atlassian.com/bitbucket/event-payloads-740262817.html#EventPayloads-Push """ integration_type = Integration.BITBUCKET_WEBHOOK def handle_webhook(self): - # Get event and trigger other webhook events - event = self.request.META.get('HTTP_X_EVENT_KEY', BITBUCKET_PUSH) - webhook_bitbucket.send(Project, project=self.project, - data=self.request.data, event=event) - # Handle push events and trigger builds + """ + Handle BitBucket events for push. + + BitBucket doesn't have a separate event for creation/deletion, + instead it sets the new attribute (null if it is a deletion) + and the old attribute (null if it is a creation). + """ + event = self.request.META.get(BITBUCKET_EVENT_HEADER, BITBUCKET_PUSH) + webhook_bitbucket.send( + Project, + project=self.project, + data=self.request.data, + event=event + ) if event == BITBUCKET_PUSH: try: - changes = self.request.data['push']['changes'] - branches = [change['new']['name'] - for change in changes - if change.get('new')] - return self.get_response_push(self.project, branches) + data = self.request.data + changes = data['push']['changes'] + branches = [] + for change in changes: + old = change['old'] + new = change['new'] + # Normal push to master + if old is not None and new is not None: + branches.append(new['name']) + # BitBuck returns an array of changes rather than + # one webhook per change. If we have at least one normal push + # we don't trigger the sync versions, because that + # will be triggered with the normal push. + if branches: + return self.get_response_push(self.project, branches) + return self.sync_versions(self.project) except KeyError: raise ParseError('Invalid request') + return None class IsAuthenticatedOrHasToken(permissions.IsAuthenticated): diff --git a/readthedocs/rtd_tests/tests/test_api.py b/readthedocs/rtd_tests/tests/test_api.py index d169f9bc51d..1a6d09b034b 100644 --- a/readthedocs/rtd_tests/tests/test_api.py +++ b/readthedocs/rtd_tests/tests/test_api.py @@ -26,8 +26,21 @@ from readthedocs.builds.models import Build, BuildCommandResult, Version from readthedocs.integrations.models import Integration from readthedocs.oauth.models import RemoteOrganization, RemoteRepository -from readthedocs.projects.models import APIProject, Feature, Project, EnvironmentVariable -from readthedocs.restapi.views.integrations import GitHubWebhookView +from readthedocs.projects.models import ( + APIProject, + EnvironmentVariable, + Feature, + Project, +) +from readthedocs.restapi.views.integrations import ( + GITHUB_CREATE, + GITHUB_DELETE, + GITHUB_EVENT_HEADER, + GITLAB_NULL_HASH, + GITLAB_PUSH, + GITLAB_TAG_PUSH, + GitHubWebhookView, +) from readthedocs.restapi.views.task_views import get_status_data super_auth = base64.b64encode(b'super:test').decode('utf-8') @@ -825,6 +838,29 @@ def setUp(self): Version, slug='v1.0', verbose_name='v1.0', active=True, project=self.project ) + self.github_payload = { + 'ref': 'master', + } + self.gitlab_payload = { + 'object_kind': GITLAB_PUSH, + 'ref': 'master', + 'before': '95790bf891e76fee5e1747ab589903a6a1f80f22', + 'after': '95790bf891e76fee5e1747ab589903a6a1f80f23', + } + self.bitbucket_payload = { + 'push': { + 'changes': [{ + 'new': { + 'type': 'branch', + 'name': 'master', + }, + 'old': { + 'type': 'branch', + 'name': 'master', + }, + }], + }, + } def test_github_webhook_for_branches(self, trigger_build): """GitHub webhook API.""" @@ -882,6 +918,44 @@ def test_github_webhook_for_tags(self, trigger_build): trigger_build.assert_has_calls( [mock.call(force=True, version=self.version_tag, project=self.project)]) + @mock.patch('readthedocs.core.views.hooks.sync_repository_task') + def test_github_create_event(self, sync_repository_task, trigger_build): + client = APIClient() + + headers = {GITHUB_EVENT_HEADER: GITHUB_CREATE} + resp = client.post( + '/api/v2/webhook/github/{}/'.format(self.project.slug), + self.github_payload, + format='json', + **headers + ) + self.assertEqual(resp.status_code, status.HTTP_200_OK) + self.assertFalse(resp.data['build_triggered']) + self.assertEqual(resp.data['project'], self.project.slug) + self.assertEqual(resp.data['versions'], [LATEST]) + trigger_build.assert_not_called() + latest_version = self.project.versions.get(slug=LATEST) + sync_repository_task.delay.assert_called_with(latest_version.pk) + + @mock.patch('readthedocs.core.views.hooks.sync_repository_task') + def test_github_delete_event(self, sync_repository_task, trigger_build): + client = APIClient() + + headers = {GITHUB_EVENT_HEADER: GITHUB_DELETE} + resp = client.post( + '/api/v2/webhook/github/{}/'.format(self.project.slug), + self.github_payload, + format='json', + **headers + ) + self.assertEqual(resp.status_code, status.HTTP_200_OK) + self.assertFalse(resp.data['build_triggered']) + self.assertEqual(resp.data['project'], self.project.slug) + self.assertEqual(resp.data['versions'], [LATEST]) + trigger_build.assert_not_called() + latest_version = self.project.versions.get(slug=LATEST) + sync_repository_task.delay.assert_called_with(latest_version.pk) + def test_github_parse_ref(self, trigger_build): wh = GitHubWebhookView() @@ -904,23 +978,153 @@ def test_github_invalid_webhook(self, trigger_build): self.assertEqual(resp.status_code, 200) self.assertEqual(resp.data['detail'], 'Unhandled webhook event') - def test_gitlab_webhook(self, trigger_build): + def test_gitlab_webhook_for_branches(self, trigger_build): """GitLab webhook API.""" client = APIClient() client.post( - '/api/v2/webhook/gitlab/{0}/'.format(self.project.slug), - {'object_kind': 'push', 'ref': 'master'}, + '/api/v2/webhook/gitlab/{}/'.format(self.project.slug), + self.gitlab_payload, format='json', ) - trigger_build.assert_has_calls( - [mock.call(force=True, version=mock.ANY, project=self.project)]) + trigger_build.assert_called_with( + force=True, version=mock.ANY, project=self.project + ) + + trigger_build.reset_mock() + self.gitlab_payload.update( + ref='non-existent', + ) client.post( - '/api/v2/webhook/gitlab/{0}/'.format(self.project.slug), - {'object_kind': 'push', 'ref': 'non-existent'}, + '/api/v2/webhook/gitlab/{}/'.format(self.project.slug), + self.gitlab_payload, format='json', ) - trigger_build.assert_has_calls( - [mock.call(force=True, version=mock.ANY, project=self.project)]) + trigger_build.assert_not_called() + + def test_gitlab_webhook_for_tags(self, trigger_build): + client = APIClient() + self.gitlab_payload.update( + object_kind=GITLAB_TAG_PUSH, + ref='v1.0', + ) + client.post( + '/api/v2/webhook/gitlab/{}/'.format(self.project.slug), + self.gitlab_payload, + format='json', + ) + trigger_build.assert_called_with( + force=True, version=self.version_tag, project=self.project + ) + + trigger_build.reset_mock() + self.gitlab_payload.update( + ref='refs/tags/v1.0', + ) + client.post( + '/api/v2/webhook/gitlab/{}/'.format(self.project.slug), + self.gitlab_payload, + format='json', + ) + trigger_build.assert_called_with( + force=True, version=self.version_tag, project=self.project + ) + + trigger_build.reset_mock() + self.gitlab_payload.update( + ref='refs/heads/non-existent', + ) + client.post( + '/api/v2/webhook/gitlab/{}/'.format(self.project.slug), + self.gitlab_payload, + format='json', + ) + trigger_build.assert_not_called() + + @mock.patch('readthedocs.core.views.hooks.sync_repository_task') + def test_gitlab_push_hook_creation( + self, sync_repository_task, trigger_build): + client = APIClient() + self.gitlab_payload.update( + before=GITLAB_NULL_HASH, + after='95790bf891e76fee5e1747ab589903a6a1f80f22', + ) + resp = client.post( + '/api/v2/webhook/gitlab/{}/'.format(self.project.slug), + self.gitlab_payload, + format='json', + ) + self.assertEqual(resp.status_code, status.HTTP_200_OK) + self.assertFalse(resp.data['build_triggered']) + self.assertEqual(resp.data['project'], self.project.slug) + self.assertEqual(resp.data['versions'], [LATEST]) + trigger_build.assert_not_called() + latest_version = self.project.versions.get(slug=LATEST) + sync_repository_task.delay.assert_called_with(latest_version.pk) + + @mock.patch('readthedocs.core.views.hooks.sync_repository_task') + def test_gitlab_push_hook_deletion( + self, sync_repository_task, trigger_build): + client = APIClient() + self.gitlab_payload.update( + before='95790bf891e76fee5e1747ab589903a6a1f80f22', + after=GITLAB_NULL_HASH, + ) + resp = client.post( + '/api/v2/webhook/gitlab/{}/'.format(self.project.slug), + self.gitlab_payload, + format='json', + ) + self.assertEqual(resp.status_code, status.HTTP_200_OK) + self.assertFalse(resp.data['build_triggered']) + self.assertEqual(resp.data['project'], self.project.slug) + self.assertEqual(resp.data['versions'], [LATEST]) + trigger_build.assert_not_called() + latest_version = self.project.versions.get(slug=LATEST) + sync_repository_task.delay.assert_called_with(latest_version.pk) + + @mock.patch('readthedocs.core.views.hooks.sync_repository_task') + def test_gitlab_tag_push_hook_creation( + self, sync_repository_task, trigger_build): + client = APIClient() + self.gitlab_payload.update( + object_kind=GITLAB_TAG_PUSH, + before=GITLAB_NULL_HASH, + after='95790bf891e76fee5e1747ab589903a6a1f80f22', + ) + resp = client.post( + '/api/v2/webhook/gitlab/{}/'.format(self.project.slug), + self.gitlab_payload, + format='json', + ) + self.assertEqual(resp.status_code, status.HTTP_200_OK) + self.assertFalse(resp.data['build_triggered']) + self.assertEqual(resp.data['project'], self.project.slug) + self.assertEqual(resp.data['versions'], [LATEST]) + trigger_build.assert_not_called() + latest_version = self.project.versions.get(slug=LATEST) + sync_repository_task.delay.assert_called_with(latest_version.pk) + + @mock.patch('readthedocs.core.views.hooks.sync_repository_task') + def test_gitlab_tag_push_hook_deletion( + self, sync_repository_task, trigger_build): + client = APIClient() + self.gitlab_payload.update( + object_kind=GITLAB_TAG_PUSH, + before='95790bf891e76fee5e1747ab589903a6a1f80f22', + after=GITLAB_NULL_HASH, + ) + resp = client.post( + '/api/v2/webhook/gitlab/{}/'.format(self.project.slug), + self.gitlab_payload, + format='json', + ) + self.assertEqual(resp.status_code, status.HTTP_200_OK) + self.assertFalse(resp.data['build_triggered']) + self.assertEqual(resp.data['project'], self.project.slug) + self.assertEqual(resp.data['versions'], [LATEST]) + trigger_build.assert_not_called() + latest_version = self.project.versions.get(slug=LATEST) + sync_repository_task.delay.assert_called_with(latest_version.pk) def test_gitlab_invalid_webhook(self, trigger_build): """GitLab webhook unhandled event.""" @@ -937,27 +1141,20 @@ def test_bitbucket_webhook(self, trigger_build): """Bitbucket webhook API.""" client = APIClient() client.post( - '/api/v2/webhook/bitbucket/{0}/'.format(self.project.slug), - { - 'push': { - 'changes': [{ - 'new': { - 'name': 'master', - }, - }], - }, - }, + '/api/v2/webhook/bitbucket/{}/'.format(self.project.slug), + self.bitbucket_payload, format='json', ) trigger_build.assert_has_calls( [mock.call(force=True, version=mock.ANY, project=self.project)]) client.post( - '/api/v2/webhook/bitbucket/{0}/'.format(self.project.slug), + '/api/v2/webhook/bitbucket/{}/'.format(self.project.slug), { 'push': { 'changes': [ { 'new': {'name': 'non-existent'}, + 'old': {'name': 'master'}, }, ], }, @@ -969,7 +1166,7 @@ def test_bitbucket_webhook(self, trigger_build): trigger_build_call_count = trigger_build.call_count client.post( - '/api/v2/webhook/bitbucket/{0}/'.format(self.project.slug), + '/api/v2/webhook/bitbucket/{}/'.format(self.project.slug), { 'push': { 'changes': [ @@ -983,6 +1180,42 @@ def test_bitbucket_webhook(self, trigger_build): ) self.assertEqual(trigger_build_call_count, trigger_build.call_count) + @mock.patch('readthedocs.core.views.hooks.sync_repository_task') + def test_bitbucket_push_hook_creation( + self, sync_repository_task, trigger_build): + client = APIClient() + self.bitbucket_payload['push']['changes'][0]['old'] = None + resp = client.post( + '/api/v2/webhook/bitbucket/{}/'.format(self.project.slug), + self.bitbucket_payload, + format='json', + ) + self.assertEqual(resp.status_code, status.HTTP_200_OK) + self.assertFalse(resp.data['build_triggered']) + self.assertEqual(resp.data['project'], self.project.slug) + self.assertEqual(resp.data['versions'], [LATEST]) + trigger_build.assert_not_called() + latest_version = self.project.versions.get(slug=LATEST) + sync_repository_task.delay.assert_called_with(latest_version.pk) + + @mock.patch('readthedocs.core.views.hooks.sync_repository_task') + def test_bitbucket_push_hook_deletion( + self, sync_repository_task, trigger_build): + client = APIClient() + self.bitbucket_payload['push']['changes'][0]['new'] = None + resp = client.post( + '/api/v2/webhook/bitbucket/{}/'.format(self.project.slug), + self.bitbucket_payload, + format='json', + ) + self.assertEqual(resp.status_code, status.HTTP_200_OK) + self.assertFalse(resp.data['build_triggered']) + self.assertEqual(resp.data['project'], self.project.slug) + self.assertEqual(resp.data['versions'], [LATEST]) + trigger_build.assert_not_called() + latest_version = self.project.versions.get(slug=LATEST) + sync_repository_task.delay.assert_called_with(latest_version.pk) + def test_bitbucket_invalid_webhook(self, trigger_build): """Bitbucket webhook unhandled event.""" client = APIClient()