-
Run this check against 10,000 {{ check.hooked_object.value }}s, selected at random. This is used to evaluate the efficacy of a check.
+ {% if check.check_type.value == "event_hook" %}
+
Run this check against {{ evaluation_run_size }} {{ check.hooked_object.value }}s, selected at random. This is used to evaluate the efficacy of a check.
+ {% else %}
+
Execute this check now.
+ {% endif %}
diff --git a/warehouse/admin/templates/admin/malware/checks/index.html b/warehouse/admin/templates/admin/malware/checks/index.html
index 5717849e2579..2601bfe0f62a 100644
--- a/warehouse/admin/templates/admin/malware/checks/index.html
+++ b/warehouse/admin/templates/admin/malware/checks/index.html
@@ -26,7 +26,7 @@
| Check Name |
State |
- Revisions |
+ Type |
Last Modified |
Description |
@@ -38,7 +38,7 @@
{{ check.state.value }} |
-
{{ check.version }} |
+
{{ check.check_type.value }} |
{{ check.created }} |
{{ check.short_description }} |
diff --git a/warehouse/admin/views/checks.py b/warehouse/admin/views/checks.py
index b23465fd4a49..0cd1761c8456 100644
--- a/warehouse/admin/views/checks.py
+++ b/warehouse/admin/views/checks.py
@@ -14,8 +14,10 @@
from pyramid.view import view_config
from sqlalchemy.orm.exc import NoResultFound
-from warehouse.malware.models import MalwareCheck, MalwareCheckState
-from warehouse.malware.tasks import backfill, remove_verdicts
+from warehouse.malware.models import MalwareCheck, MalwareCheckState, MalwareCheckType
+from warehouse.malware.tasks import backfill, remove_verdicts, run_check
+
+EVALUATION_RUN_SIZE = 10000
@view_config(
@@ -52,36 +54,45 @@ def get_check(request):
.all()
)
- return {"check": check, "checks": all_checks, "states": MalwareCheckState}
+ return {
+ "check": check,
+ "checks": all_checks,
+ "states": MalwareCheckState,
+ "evaluation_run_size": EVALUATION_RUN_SIZE,
+ }
@view_config(
- route_name="admin.checks.run_backfill",
+ route_name="admin.checks.run_evaluation",
permission="admin",
request_method="POST",
uses_session=True,
require_methods=False,
require_csrf=True,
)
-def run_backfill(request):
+def run_evaluation(request):
check = get_check_by_name(request.db, request.matchdict["check_name"])
- num_objects = 10000
if check.state not in (MalwareCheckState.Enabled, MalwareCheckState.Evaluation):
request.session.flash(
- f"Check must be in 'enabled' or 'evaluation' state to run a backfill.",
+ f"Check must be in 'enabled' or 'evaluation' state to manually execute.",
queue="error",
)
return HTTPSeeOther(
request.route_path("admin.checks.detail", check_name=check.name)
)
- request.session.flash(
- f"Running {check.name} on {num_objects} {check.hooked_object.value}s!",
- queue="success",
- )
+ if check.check_type == MalwareCheckType.EventHook:
+ request.session.flash(
+ f"Running {check.name} on {EVALUATION_RUN_SIZE} {check.hooked_object.value}s\
+!",
+ queue="success",
+ )
+ request.task(backfill).delay(check.name, EVALUATION_RUN_SIZE)
- request.task(backfill).delay(check.name, num_objects)
+ else:
+ request.session.flash(f"Running {check.name} now!", queue="success")
+ request.task(run_check).delay(check.name, manually_triggered=True)
return HTTPSeeOther(
request.route_path("admin.checks.detail", check_name=check.name)
diff --git a/warehouse/malware/__init__.py b/warehouse/malware/__init__.py
index f54a9e89b4f5..a1b76b82f2b5 100644
--- a/warehouse/malware/__init__.py
+++ b/warehouse/malware/__init__.py
@@ -10,9 +10,16 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+import inspect
+
+from celery.schedules import crontab
+
+import warehouse.malware.checks as checks
+
from warehouse import db
from warehouse.malware import utils
from warehouse.malware.interfaces import IMalwareCheckService
+from warehouse.malware.tasks import run_check
@db.listens_for(db.Session, "after_flush")
@@ -57,3 +64,12 @@ def includeme(config):
config.register_service_factory(
malware_check_class.create_service, IMalwareCheckService
)
+
+ # Add scheduled tasks for every scheduled Malware Check.
+ all_checks = inspect.getmembers(checks, inspect.isclass)
+ for check_obj in all_checks:
+ check = check_obj[1]
+ if check.check_type == "scheduled":
+ config.add_periodic_task(
+ crontab(**check.schedule), run_check, args=(check_obj[0],)
+ )
diff --git a/warehouse/malware/checks/base.py b/warehouse/malware/checks/base.py
index 67810a367203..44f8230a1c2c 100644
--- a/warehouse/malware/checks/base.py
+++ b/warehouse/malware/checks/base.py
@@ -18,7 +18,7 @@ class MalwareCheckBase:
def __init__(self, db):
self.db = db
self._name = self.__class__.__name__
- self._load_check_id()
+ self._load_check_fields()
self._verdicts = []
@classmethod
@@ -26,7 +26,7 @@ def prepare(cls, request, obj_id):
"""
Prepares some context for scanning the given object.
"""
- kwargs = {}
+ kwargs = {"obj_id": obj_id}
model = getattr(models, cls.hooked_object)
kwargs["obj"] = request.db.query(model).get(obj_id)
@@ -60,9 +60,9 @@ def backfill(self, sample=1):
backfill on the entire corpus.
"""
- def _load_check_id(self):
- (self.id,) = (
- self.db.query(MalwareCheck.id)
+ def _load_check_fields(self):
+ self.id, self.state = (
+ self.db.query(MalwareCheck.id, MalwareCheck.state)
.filter(MalwareCheck.name == self._name)
.filter(
MalwareCheck.state.in_(
diff --git a/warehouse/malware/tasks.py b/warehouse/malware/tasks.py
index dc0274cbee98..1fbe46b7a638 100644
--- a/warehouse/malware/tasks.py
+++ b/warehouse/malware/tasks.py
@@ -12,6 +12,8 @@
import inspect
+from sqlalchemy.orm.exc import NoResultFound
+
import warehouse.malware.checks as checks
import warehouse.packaging.models as packaging_models
@@ -21,11 +23,30 @@
@task(bind=True, ignore_result=True, acks_late=True, retry_backoff=True)
-def run_check(task, request, check_name, obj_id):
- check = getattr(checks, check_name)(request.db)
+def run_check(task, request, check_name, obj_id=None, manually_triggered=False):
try:
+ check = getattr(checks, check_name)(request.db)
+ except NoResultFound:
+ request.log.info("Check %s isn't active. Aborting." % check_name)
+ return
+
+ # Don't run scheduled checks if they are in evaluation mode, unless manually
+ # triggered.
+ if check.state == MalwareCheckState.Evaluation and not manually_triggered:
+ request.log.info(
+ "%s is in the `evaluation` state and must be manually triggered to run."
+ % check_name
+ )
+ return
+
+ kwargs = {}
+
+ # Hooked checks require `obj_id`s.
+ if obj_id is not None:
kwargs = check.prepare(request, obj_id)
- check.run(obj_id=obj_id, **kwargs)
+
+ try:
+ check.run(**kwargs)
except Exception as exc:
request.log.error("Error executing check %s: %s" % (check_name, str(exc)))
raise task.retry(exc=exc)
@@ -43,7 +64,7 @@ def backfill(task, request, check_name, num_objects):
request.log.info("Running backfill on %d %ss." % (num_objects, check.hooked_object))
for (elem_id,) in query:
- request.task(run_check).delay(check_name, elem_id)
+ request.task(run_check).delay(check_name, elem_id, manually_triggered=True)
@task(bind=True, ignore_result=True, acks_late=True)