Skip to content

Commit

Permalink
Build: ability to cancel a running build from dashboard
Browse files Browse the repository at this point in the history
We tried to implement this in another opportunities (see
#7031) but the build process
was really complex and we had to manage the exception in multiple places.

After implementing Celery Handlers, we can just raise the exception when
attending to the proper signal coming from `app.control.revoke` and handle it
properly from `on_failure` task's method.

All the initial local tests were great!
  • Loading branch information
humitos committed Jan 26, 2022
1 parent 1e9c724 commit 60efd91
Show file tree
Hide file tree
Showing 7 changed files with 74 additions and 2 deletions.
18 changes: 18 additions & 0 deletions readthedocs/builds/migrations/0038_track_task_id.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
# Generated by Django 3.2.11 on 2022-01-26 20:10

from django.db import migrations, models


class Migration(migrations.Migration):

dependencies = [
('builds', '0037_alter_build_cold_storage'),
]

operations = [
migrations.AddField(
model_name='build',
name='task_id',
field=models.CharField(blank=True, max_length=36, null=True, verbose_name='Celery task id'),
),
]
8 changes: 7 additions & 1 deletion readthedocs/builds/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@
import os.path
import re
from functools import partial
from shutil import rmtree

import regex
from django.conf import settings
Expand Down Expand Up @@ -667,6 +666,13 @@ class Build(models.Model):
help_text='Build steps stored outside the database.',
)

task_id = models.CharField(
_('Celery task id'),
max_length=36,
null=True,
blank=True,
)

# Managers
objects = BuildQuerySet.as_manager()
# Only include BRANCH, TAG, UNKNOWN type Version builds.
Expand Down
24 changes: 24 additions & 0 deletions readthedocs/builds/views.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
"""Views for builds app."""

import signal

import structlog
import textwrap
from urllib.parse import urlparse
Expand All @@ -21,6 +23,12 @@
from readthedocs.doc_builder.exceptions import BuildAppError
from readthedocs.projects.models import Project

try:
from readthedocsinc.worker import app
except ImportError:
from readthedocs.worker import app


log = structlog.get_logger(__name__)


Expand Down Expand Up @@ -148,6 +156,22 @@ class BuildDetail(BuildBase, DetailView):

pk_url_kwarg = 'build_pk'

@method_decorator(login_required)
def post(self, request, project_slug, build_pk):
project = get_object_or_404(Project, slug=project_slug)
build = get_object_or_404(Build, pk=build_pk)

if not AdminPermission.is_admin(request.user, project):
return HttpResponseForbidden()

# NOTE: `terminate=True` is required for the child to attend our call
# immediately. Otherwise, it finishes the task.
app.control.revoke(build.task_id, signal=signal.SIGINT, terminate=True)

return HttpResponseRedirect(
reverse('builds_detail', args=[project.slug, build.pk]),
)

def get_context_data(self, **kwargs):
context = super().get_context_data(**kwargs)
context['project'] = self.project
Expand Down
8 changes: 7 additions & 1 deletion readthedocs/core/utils/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -239,7 +239,13 @@ def trigger_build(project, version=None, commit=None):
# Build was skipped
return (None, None)

return (update_docs_task.apply_async(), build)
task = update_docs_task.apply_async()

# Store the task_id in the build object to be able to cancel it later.
build.task_id = task.id
build.save()

return task, build


def send_email(
Expand Down
4 changes: 4 additions & 0 deletions readthedocs/doc_builder/exceptions.py
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,10 @@ class DuplicatedBuildError(BuildUserError):
status = BUILD_STATUS_DUPLICATED


class BuildCancelled(BuildUserError):
message = gettext_noop('Build cancelled by user.')


class MkDocsYAMLParseError(BuildUserError):
GENERIC_WITH_PARSE_EXCEPTION = gettext_noop(
'Problem parsing MkDocs YAML configuration. {exception}',
Expand Down
9 changes: 9 additions & 0 deletions readthedocs/projects/tasks/builds.py
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@
BuildUserError,
BuildMaxConcurrencyError,
DuplicatedBuildError,
BuildCancelled,
ProjectBuildsSkippedError,
YAMLParseError,
)
Expand Down Expand Up @@ -206,6 +207,7 @@ class UpdateDocsTask(SyncRepositoryMixin, Task):
ProjectBuildsSkippedError,
ConfigError,
YAMLParseError,
BuildCancelled,
)

acks_late = True
Expand All @@ -221,10 +223,17 @@ def _setup_sigterm(self):
def sigterm_received(*args, **kwargs):
log.warning('SIGTERM received. Waiting for build to stop gracefully after it finishes.')

def sigint_received(*args, **kwargs):
log.warning('SIGINT received. Cancelling the build running.')
raise BuildCancelled

# Do not send the SIGTERM signal to children (pip is automatically killed when
# receives SIGTERM and make the build to fail one command and stop build)
signal.signal(signal.SIGTERM, sigterm_received)


signal.signal(signal.SIGINT, sigint_received)

def _check_concurrency_limit(self):
try:
response = api_v2.build.concurrent.get(project__slug=self.project.slug)
Expand Down
5 changes: 5 additions & 0 deletions readthedocs/templates/builds/build_detail.html
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,11 @@
{% block content %}
<div class="build build-detail" id="build-detail">

<form method="post" action="{% url "builds_detail" build.version.project.slug build.pk %}">
{% csrf_token %}
<input type="submit" value="{% trans "Cancel build" %}">
</form>

<!-- Build meta data -->
<ul class="build-meta">
<div data-bind="visible: finished()">
Expand Down

0 comments on commit 60efd91

Please sign in to comment.