[WIP] Added /seen endpoint, utils.mark_seen, submission type props

WIP because it's based on another feature branch (5474-seen-tables)
and it still needs tests.

Add new API endpoint for listing or marking source conversation items
that have been seen by a journalist.

Add utility method to mark a heterogenous collection of Submission
and Reply objects seen.

Start marking submissions and replies seen when downloaded in the
journalist interface.

Add Submission.is_file and Submission.is_message to encapsulate the
characterization based on filename.

securedrop/create-dev-data.py

@@ -15,7 +15,7 @@
 
 from sdconfig import config
 from db import db
-from models import Journalist, Reply, Source, Submission
+from models import Journalist, Reply, SeenReply, Source, Submission
 from specialstrings import strings
 
 
@@ -138,6 +138,9 @@ def create_source_and_submissions(
             journalist = journalist_who_replied
         reply = Reply(journalist, source, fname)
         db.session.add(reply)
+        db.session.flush()
+        seen_reply = SeenReply(reply_id=reply.id, journalist_id=journalist.id)
+        db.session.add(seen_reply)
 
     db.session.commit()
 

securedrop/journalist_app/api.py

@@ -1,3 +1,4 @@
+import collections.abc
 import json
 
 from datetime import datetime, timedelta
@@ -10,16 +11,17 @@
 
 from db import db
 from journalist_app import utils
-from models import (Journalist, Reply, Source, Submission,
+from models import (Journalist, Reply, SeenReply, Source, Submission,
                     LoginThrottledException, InvalidUsernameException,
                     BadTokenException, WrongPasswordException)
 from store import NotEncrypted
+from typing import Optional
 
 
 TOKEN_EXPIRATION_MINS = 60 * 8
 
 
-def get_user_object(request):
+def get_user_object(request) -> Optional[Journalist]:
     """Helper function to use in token_required views that need a user
     object
     """
@@ -270,6 +272,9 @@ def all_source_replies(source_uuid):
 
             try:
                 db.session.add(reply)
+                db.session.flush()
+                seen_reply = SeenReply(reply_id=reply.id, journalist_id=user.id)
+                db.session.add(seen_reply)
                 db.session.add(source)
                 db.session.commit()
             except IntegrityError as e:
@@ -309,6 +314,57 @@ def get_all_replies():
         return jsonify(
             {'replies': [reply.to_json() for reply in replies if reply.source]}), 200
 
+    @api.route('/seen', methods=['GET', 'POST'])
+    @token_required
+    def seen():
+        """
+        Lists or marks the source conversation items that the journalist has seen.
+        """
+        user = get_user_object(request)
+
+        if request.method == 'GET':
+            return jsonify(
+                {
+                    'files': [f.file.uuid for f in user.seen_files],
+                    'messages': [m.message.uuid for m in user.seen_messages],
+                    'replies': [r.reply.uuid for r in user.seen_replies],
+                }
+            ), 200
+        elif request.method == 'POST':
+            if request.json is None or not isinstance(request.json, collections.abc.Mapping):
+                abort(400, 'please send requests in valid JSON')
+
+            if not any(map(request.json.get, ['files', 'messages', 'replies'])):
+                abort(400, 'please supply the resources to mark seen')
+
+            user = get_user_object(request)
+
+            # gather everything to be marked seen. if any don't exist,
+            # reject the request.
+            targets = set()
+            for file_uuid in request.json.get('files', []):
+                f = Submission.query.filter(Submission.uuid == file_uuid).one_or_none()
+                if f is None or not f.is_file:
+                    abort(404, 'file not found: {}'.format(file_uuid))
+                targets.add(f)
+
+            for message_uuid in request.json.get('messages', []):
+                m = Submission.query.filter(Submission.uuid == message_uuid).one_or_none()
+                if m is None or not m.is_message:
+                    abort(404, 'message not found: {}'.format(message_uuid))
+                targets.add(m)
+
+            for reply_uuid in request.json.get('replies', []):
+                r = Reply.query.filter(Reply.uuid == reply_uuid).one_or_none()
+                if r is None:
+                    abort(404, 'reply not found: {}'.format(reply_uuid))
+                targets.add(r)
+
+            # now mark everything seen.
+            utils.mark_seen(targets, user)
+
+            return jsonify({'message': 'resources marked seen'}), 201
+
     @api.route('/user', methods=['GET'])
     @token_required
     def get_current_user():

securedrop/journalist_app/main.py

@@ -10,7 +10,7 @@
 import store
 
 from db import db
-from models import Source, SourceStar, Submission, Reply
+from models import Source, SourceStar, Submission, Reply, SeenReply
 from journalist_app.forms import ReplyForm
 from journalist_app.utils import (validate_user, bulk_delete, download,
                                   confirm_bulk_delete, get_source)
@@ -113,9 +113,11 @@ def reply():
             output=current_app.storage.path(g.filesystem_id, filename),
         )
         reply = Reply(g.user, g.source, filename)
-
         try:
             db.session.add(reply)
+            db.session.flush()
+            seen_reply = SeenReply(reply_id=reply.id, journalist_id=g.user.id)
+            db.session.add(seen_reply)
             db.session.commit()
             store.async_add_checksum_for_file(reply)
         except Exception as exc:

securedrop/journalist_app/utils.py

@@ -9,7 +9,6 @@
 from flask import (g, flash, current_app, abort, send_file, redirect, url_for,
                    render_template, Markup, sessions, request)
 from flask_babel import gettext, ngettext
-from sqlalchemy.exc import IntegrityError
 from sqlalchemy.sql.expression import false
 
 import i18n
@@ -150,6 +149,43 @@ def validate_hotp_secret(user: Journalist, otp_secret: str) -> bool:
     return True
 
 
+def mark_seen(
+    targets: List[Union[Submission, Reply]], user: Journalist, commit: bool = True
+) -> None:
+    """
+    Marks a list of submissions or replies seen by the given journalist.
+    """
+    for t in targets:
+        if isinstance(t, Submission):
+            t.downloaded = True
+            if t.is_file:
+                sf = SeenFile.query.filter(
+                    SeenFile.file_id == t.id,
+                    SeenFile.journalist_id == user.id
+                ).one_or_none()
+                if not sf:
+                    sf = SeenFile(file_id=t.id, journalist_id=user.id)
+                    db.session.add(sf)
+            elif t.is_message:
+                sm = SeenMessage.query.filter(
+                    SeenMessage.message_id == t.id,
+                    SeenMessage.journalist_id == user.id
+                ).one_or_none()
+                if not sm:
+                    sm = SeenMessage(message_id=t.id, journalist_id=user.id)
+                    db.session.add(sm)
+        elif isinstance(t, Reply):
+            sr = SeenReply.query.filter(
+                SeenReply.reply_id == t.id,
+                SeenReply.journalist_id == user.id
+            ).one_or_none()
+            if not sr:
+                sr = SeenReply(reply_id=t.id, journalist_id=user.id)
+                db.session.add(sr)
+    if commit:
+        db.session.commit()
+
+
 def download(zip_basename: str, submissions: List[Union[Submission, Reply]]) -> werkzeug.Response:
     """Send client contents of ZIP-file *zip_basename*-<timestamp>.zip
     containing *submissions*. The ZIP-file, being a
@@ -166,25 +202,8 @@ def download(zip_basename: str, submissions: List[Union[Submission, Reply]]) ->
     attachment_filename = "{}--{}.zip".format(
         zip_basename, datetime.datetime.utcnow().strftime("%Y-%m-%d--%H-%M-%S"))
 
-    # mark as seen by the current user and update downloaded for submissions
-    journalist_id = g.get('user').id
-    for item in submissions:
-        try:
-            if item.filename.endswith('reply.gpg'):
-                seen_reply = SeenReply(reply_id=item.id, journalist_id=journalist_id)
-                db.session.add(seen_reply)
-            elif item.filename.endswith('-doc.gz.gpg'):
-                seen_file = SeenFile(file_id=item.id, journalist_id=journalist_id)
-                db.session.add(seen_file)
-                item.downloaded = True
-            else:
-                seen_message = SeenMessage(message_id=item.id, journalist_id=journalist_id)
-                db.session.add(seen_message)
-                item.downloaded = True
-
-            db.session.commit()
-        except IntegrityError:
-            pass  # expected not to store that a file was seen by the same user multiple times
+    mark_seen(submissions, g.user, commit=False)
+    db.session.commit()
 
     return send_file(zf.name, mimetype="application/zip",
                      attachment_filename=attachment_filename,

securedrop/models.py

@@ -153,11 +153,9 @@ def journalist_filename(self) -> str:
     def documents_messages_count(self) -> "Dict[str, int]":
         self.docs_msgs_count = {"messages": 0, "documents": 0}
         for submission in self.submissions:
-            if submission.filename.endswith("msg.gpg"):
+            if submission.is_message:
                 self.docs_msgs_count["messages"] += 1
-            elif submission.filename.endswith(
-                "doc.gz.gpg"
-            ) or submission.filename.endswith("doc.zip.gpg"):
+            elif submission.is_file:
                 self.docs_msgs_count["documents"] += 1
         return self.docs_msgs_count
 
@@ -265,7 +263,23 @@ def __init__(self, source: Source, filename: str) -> None:
     def __repr__(self) -> str:
         return "<Submission %r>" % (self.filename)
 
+    @property
+    def is_file(self) -> bool:
+        return self.filename.endswith("doc.gz.gpg") or self.filename.endswith("doc.zip.gpg")
+
+    @property
+    def is_message(self) -> bool:
+        return self.filename.endswith("msg.gpg")
+
     def to_json(self) -> "Dict[str, Union[str, int, bool]]":
+        seen_by = {
+            f.journalist.uuid for f in SeenFile.query.filter(SeenFile.file_id == self.id)
+            if f.journalist
+        }
+        seen_by.update({
+            m.journalist.uuid for m in SeenMessage.query.filter(SeenMessage.message_id == self.id)
+            if m.journalist
+        })
         json_submission = {
             "source_url": url_for("api.single_source", source_uuid=self.source.uuid)
             if self.source
@@ -279,7 +293,9 @@ def to_json(self) -> "Dict[str, Union[str, int, bool]]":
             else None,
             "filename": self.filename,
             "size": self.size,
-            "is_read": self.downloaded,
+            "is_file": self.is_file,
+            "is_message": self.is_message,
+            "is_read": self.downloaded or bool(seen_by),
             "uuid": self.uuid,
             "download_url": url_for(
                 "api.download_submission",
@@ -288,6 +304,7 @@ def to_json(self) -> "Dict[str, Union[str, int, bool]]":
             )
             if self.source
             else None,
+            "seen_by": list(seen_by)
         }
         return json_submission
 
@@ -329,16 +346,20 @@ def __repr__(self) -> str:
         return "<Reply %r>" % (self.filename)
 
     def to_json(self) -> "Dict[str, Union[str, int, bool]]":
-        username = "deleted"
-        first_name = ""
-        last_name = ""
-        uuid = "deleted"
+        journalist_username = "deleted"
+        journalist_first_name = ""
+        journalist_last_name = ""
+        journalist_uuid = "deleted"
         if self.journalist:
-            username = self.journalist.username
-            first_name = self.journalist.first_name
-            last_name = self.journalist.last_name
-            uuid = self.journalist.uuid
-        json_submission = {
+            journalist_username = self.journalist.username
+            journalist_first_name = self.journalist.first_name
+            journalist_last_name = self.journalist.last_name
+            journalist_uuid = self.journalist.uuid
+        seen_by = [
+            r.journalist.uuid for r in SeenReply.query.filter(SeenReply.reply_id == self.id)
+            if r.journalist
+        ]
+        json_reply = {
             "source_url": url_for("api.single_source", source_uuid=self.source.uuid)
             if self.source
             else None,
@@ -349,14 +370,15 @@ def to_json(self) -> "Dict[str, Union[str, int, bool]]":
             else None,
             "filename": self.filename,
             "size": self.size,
-            "journalist_username": username,
-            "journalist_first_name": first_name,
-            "journalist_last_name": last_name,
-            "journalist_uuid": uuid,
+            "journalist_username": journalist_username,
+            "journalist_first_name": journalist_first_name,
+            "journalist_last_name": journalist_last_name,
+            "journalist_uuid": journalist_uuid,
             "uuid": self.uuid,
             "is_deleted_by_source": self.deleted_by_source,
+            "seen_by": seen_by
         }
-        return json_submission
+        return json_reply
 
 
 class SourceStar(db.Model):
@@ -757,7 +779,7 @@ def validate_token_is_not_expired_or_invalid(token):
         return True
 
     @staticmethod
-    def validate_api_token_and_get_user(token: str) -> "Union[Journalist, None]":
+    def validate_api_token_and_get_user(token: str) -> "Optional[Journalist]":
         s = TimedJSONWebSignatureSerializer(current_app.config["SECRET_KEY"])
         try:
             data = s.loads(token)