Timetracking Overview Improvements (#7789)

* WIP: in time tracking, filter by annotation state

* mock frontend part, rewrite query for faster execution

* don’t center the statistics element

* in dropdown, allow searching by project name

* undo conf edit

* changelog

* remove superfluous field

* pr feedback (optional params last)

CHANGELOG.unreleased.md

@@ -12,6 +12,7 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.released
 
 ### Added
 - Within the proofreading tool, the user can now interact with the super voxels of a mesh in the 3D viewport. For example, this allows to merge or cut super voxels from another. As before, the proofreading tool requires an agglomerate file. [#7742](https://github.com/scalableminds/webknossos/pull/7742)
+- Minor improvements for the timetracking overview (faster data loding, styling). [#7789](https://github.com/scalableminds/webknossos/pull/7789)
 
 ### Changed
 - Non-admin or -manager users can no longer start long-running jobs that create datasets. This includes annotation materialization and AI inferrals. [#7753](https://github.com/scalableminds/webknossos/pull/7753)

app/controllers/TimeController.scala

@@ -3,7 +3,7 @@ package controllers
 import play.silhouette.api.Silhouette
 import com.scalableminds.util.time.Instant
 import com.scalableminds.util.tools.{Fox, FoxImplicits}
-import models.annotation.AnnotationType
+import models.annotation.{AnnotationState, AnnotationType}
 
 import scala.collection.immutable.ListMap
 import javax.inject.Inject
@@ -51,19 +51,22 @@ class TimeController @Inject()(userService: UserService,
                                     start: Long,
                                     end: Long,
                                     annotationTypes: String,
+                                    annotationStates: String,
                                     projectIds: Option[String]): Action[AnyContent] = sil.SecuredAction.async {
     implicit request =>
       for {
         userIdValidated <- ObjectId.fromString(userId)
         projectIdsValidated <- ObjectId.fromCommaSeparated(projectIds)
         annotationTypesValidated <- AnnotationType.fromCommaSeparated(annotationTypes) ?~> "invalidAnnotationType"
+        annotationStatesValidated <- AnnotationState.fromCommaSeparated(annotationStates) ?~> "invalidAnnotationState"
         user <- userService.findOneCached(userIdValidated) ?~> "user.notFound" ~> NOT_FOUND
         isTeamManagerOrAdmin <- userService.isTeamManagerOrAdminOf(request.identity, user)
         _ <- bool2Fox(isTeamManagerOrAdmin || user._id == request.identity._id) ?~> "user.notAuthorised" ~> FORBIDDEN
         timesByAnnotation <- timeSpanDAO.summedByAnnotationForUser(user._id,
                                                                    Instant(start),
                                                                    Instant(end),
                                                                    annotationTypesValidated,
+                                                                   annotationStatesValidated,
                                                                    projectIdsValidated)
       } yield Ok(timesByAnnotation)
   }
@@ -72,32 +75,37 @@ class TimeController @Inject()(userService: UserService,
                       start: Long,
                       end: Long,
                       annotationTypes: String,
+                      annotationStates: String,
                       projectIds: Option[String]): Action[AnyContent] =
     sil.SecuredAction.async { implicit request =>
       for {
         userIdValidated <- ObjectId.fromString(userId)
         projectIdsValidated <- ObjectId.fromCommaSeparated(projectIds)
         annotationTypesValidated <- AnnotationType.fromCommaSeparated(annotationTypes) ?~> "invalidAnnotationType"
+        annotationStatesValidated <- AnnotationState.fromCommaSeparated(annotationStates) ?~> "invalidAnnotationState"
         user <- userService.findOneCached(userIdValidated) ?~> "user.notFound" ~> NOT_FOUND
         isTeamManagerOrAdmin <- userService.isTeamManagerOrAdminOf(request.identity, user)
         _ <- bool2Fox(isTeamManagerOrAdmin || user._id == request.identity._id) ?~> "user.notAuthorised" ~> FORBIDDEN
         timeSpansJs <- timeSpanDAO.findAllByUserWithTask(user._id,
                                                          Instant(start),
                                                          Instant(end),
                                                          annotationTypesValidated,
+                                                         annotationStatesValidated,
                                                          projectIdsValidated)
       } yield Ok(timeSpansJs)
     }
 
   def timeOverview(start: Long,
                    end: Long,
                    annotationTypes: String,
+                   annotationStates: String,
                    teamIds: Option[String],
                    projectIds: Option[String]): Action[AnyContent] =
     sil.SecuredAction.async { implicit request =>
       for {
         teamIdsValidated <- ObjectId.fromCommaSeparated(teamIds) ?~> "invalidTeamId"
         annotationTypesValidated <- AnnotationType.fromCommaSeparated(annotationTypes) ?~> "invalidAnnotationType"
+        annotationStatesValidated <- AnnotationState.fromCommaSeparated(annotationStates) ?~> "invalidAnnotationState"
         _ <- bool2Fox(annotationTypesValidated.nonEmpty) ?~> "annotationTypesEmpty"
         _ <- bool2Fox(annotationTypesValidated.forall(typ =>
           typ == AnnotationType.Explorational || typ == AnnotationType.Task)) ?~> "unsupportedAnnotationType"
@@ -109,6 +117,7 @@ class TimeController @Inject()(userService: UserService,
                                                      Instant(end),
                                                      usersFiltered.map(_._id),
                                                      annotationTypesValidated,
+                                                     annotationStatesValidated,
                                                      projectIdsValidated)
       } yield Ok(Json.toJson(usersWithTimesJs))
     }

app/models/user/time/TimeSpan.scala

@@ -3,6 +3,7 @@ package models.user.time
 import com.scalableminds.util.time.Instant
 import com.scalableminds.util.tools.Fox
 import com.scalableminds.webknossos.schema.Tables._
+import models.annotation.AnnotationState.AnnotationState
 import models.annotation.AnnotationType.AnnotationType
 import play.api.libs.json.{JsArray, JsObject, JsValue, Json}
 import slick.lifted.Rep
@@ -73,33 +74,34 @@ class TimeSpanDAO @Inject()(sqlClient: SqlClient)(implicit ec: ExecutionContext)
                                 start: Instant,
                                 end: Instant,
                                 annotationTypes: List[AnnotationType],
+                                annotationStates: List[AnnotationState],
                                 projectIds: List[ObjectId]): Fox[JsValue] =
     if (annotationTypes.isEmpty) Fox.successful(Json.arr())
     else {
       val projectQuery = projectIdsFilterQuery(projectIds)
       for {
         tuples <- run(
-          q"""WITH annotationLayerStatistics AS (
-                SELECT an._id AS _annotation, JSON_AGG(al.statistics) AS layerStatistics
-                FROM webknossos.annotation_layers al
-                JOIN webknossos.annotations an ON al._annotation = an._id
-                GROUP BY an._id
+          q"""WITH timeSummedPerAnnotation AS (
+                SELECT a._id AS _annotation, t._id AS _task, p.name AS projectName, SUM(ts.time) AS timeSummed
+                FROM webknossos.timespans_ ts
+                JOIN webknossos.annotations_ a ON ts._annotation = a._id
+                LEFT JOIN webknossos.tasks_ t ON a._task = t._id
+                LEFT JOIN webknossos.projects_ p ON t._project = p._id
+                WHERE ts._user = $userId
+                AND ts.time > 0
+                AND ts.created >= $start
+                AND ts.created < $end
+                AND $projectQuery
+                AND a.typ IN ${SqlToken.tupleFromList(annotationTypes)}
+                AND a.state IN ${SqlToken.tupleFromList(annotationStates)}
+                GROUP BY a._id, t._id, p.name
               )
-              SELECT a._id, t._id, p.name, SUM(ts.time), JSON_AGG(als.layerStatistics)->0 AS annotationLayerStatistics
-              FROM webknossos.timespans_ ts
-              JOIN webknossos.annotations_ a ON ts._annotation = a._id
-              JOIN annotationLayerStatistics AS als ON als._annotation = a._id
-              LEFT JOIN webknossos.tasks_ t ON a._task = t._id
-              LEFT JOIN webknossos.projects_ p ON t._project = p._id
-              WHERE ts._user = $userId
-              AND ts.time > 0
-              AND ts.created >= $start
-              AND ts.created < $end
-              AND $projectQuery
-              AND a.typ IN ${SqlToken.tupleFromList(annotationTypes)}
-              GROUP BY a._id, t._id, p.name
-              ORDER BY a._id
-         """.as[(String, Option[String], Option[String], Long, String)]
+              SELECT ti._annotation, ti._task, ti.projectName, ti.timeSummed, JSON_AGG(al.statistics) AS layerStatistics
+              FROM timeSummedPerAnnotation ti
+              JOIN webknossos.annotation_layers al ON al._annotation = ti._annotation
+              GROUP BY ti._annotation, ti._task, ti.projectName, ti.timeSummed
+              ORDER BY ti._annotation
+          """.as[(String, Option[String], Option[String], Long, String)]
         )
         parsed = tuples.map { t =>
           val layerStats: JsArray = Json.parse(t._5).validate[JsArray].getOrElse(Json.arr())
@@ -118,6 +120,7 @@ class TimeSpanDAO @Inject()(sqlClient: SqlClient)(implicit ec: ExecutionContext)
                             start: Instant,
                             end: Instant,
                             annotationTypes: List[AnnotationType],
+                            annotationStates: List[AnnotationState],
                             projectIds: List[ObjectId]): Fox[JsValue] =
     if (annotationTypes.isEmpty) Fox.successful(Json.arr())
     else {
@@ -126,20 +129,21 @@ class TimeSpanDAO @Inject()(sqlClient: SqlClient)(implicit ec: ExecutionContext)
         tuples <- run(
           q"""SELECT ts._user, mu.email, o.name, d.name, a._id, t._id, p.name, tt._id, tt.summary, ts._id, ts.created, ts.time
               FROM webknossos.timespans_ ts
-              JOIN webknossos.annotations_ a ON ts._annotation = a._id
-              JOIN webknossos.users_ u ON ts._user = u._id
-              JOIN webknossos.multiUsers_ mu ON u._multiUser = mu._id
-              JOIN webknossos.datasets_ d ON a._dataset = d._id
-              JOIN webknossos.organizations_ o ON d._organization = o._id
-              LEFT JOIN webknossos.tasks_ t ON a._task = t._id
-              LEFT JOIN webknossos.projects_ p ON t._project = p._id
-              LEFT JOIN webknossos.taskTypes_ tt ON t._taskType = tt._id
+              JOIN webknossos.annotations_ a on ts._annotation = a._id
+              JOIN webknossos.users_ u on ts._user = u._id
+              JOIN webknossos.multiUsers_ mu on u._multiUser = mu._id
+              JOIN webknossos.datasets_ d on a._dataset = d._id
+              JOIN webknossos.organizations_ o on d._organization = o._id
+              LEFT JOIN webknossos.tasks_ t on a._task = t._id
+              LEFT JOIN webknossos.projects_ p on t._project = p._id
+              LEFT JOIN webknossos.taskTypes_ tt on t._taskType = tt._id
               WHERE ts._user = $userId
               AND ts.time > 0
               AND ts.created >= $start
               AND ts.created < $end
               AND $projectQuery
               AND a.typ IN ${SqlToken.tupleFromList(annotationTypes)}
+              AND a.state IN ${SqlToken.tupleFromList(annotationStates)}
             """.as[(String,
                     String,
                     String,
@@ -191,6 +195,7 @@ class TimeSpanDAO @Inject()(sqlClient: SqlClient)(implicit ec: ExecutionContext)
                    end: Instant,
                    users: List[ObjectId],
                    annotationTypes: List[AnnotationType],
+                   annotationStates: List[AnnotationState],
                    projectIds: List[ObjectId]): Fox[List[JsObject]] =
     if (users.isEmpty || annotationTypes.isEmpty) Fox.successful(List.empty)
     else {
@@ -207,6 +212,7 @@ class TimeSpanDAO @Inject()(sqlClient: SqlClient)(implicit ec: ExecutionContext)
           WHERE $projectQuery
           AND u._id IN ${SqlToken.tupleFromList(users)}
           AND a.typ IN ${SqlToken.tupleFromList(annotationTypes)}
+          AND a.state IN ${SqlToken.tupleFromList(annotationStates)}
           AND ts.time > 0
           AND ts.created >= $start
           AND ts.created < $end

conf/webknossos.latest.routes

@@ -250,9 +250,9 @@ GET           /termsOfService/acceptanceNeeded
 
 # Time Tracking
 # Note, there is also /users/:id/loggedTime
-GET           /time/user/:userId/spans                                                          controllers.TimeController.timeSpansOfUser(userId: String, start: Long, end: Long, annotationTypes: String, projectIds: Option[String])
-GET           /time/user/:userId/summedByAnnotation                                             controllers.TimeController.timeSummedByAnnotationForUser(userId: String, start: Long, end: Long, annotationTypes: String, projectIds: Option[String])
-GET           /time/overview                                                                    controllers.TimeController.timeOverview(start: Long, end: Long, annotationTypes: String, teamIds: Option[String], projectIds: Option[String])
+GET           /time/user/:userId/spans                                                          controllers.TimeController.timeSpansOfUser(userId: String, start: Long, end: Long, annotationTypes: String, annotationStates: String, projectIds: Option[String])
+GET           /time/user/:userId/summedByAnnotation                                             controllers.TimeController.timeSummedByAnnotationForUser(userId: String, start: Long, end: Long, annotationTypes: String, annotationStates: String, projectIds: Option[String])
+GET           /time/overview                                                                    controllers.TimeController.timeOverview(start: Long, end: Long, annotationTypes: String, annotationStates: String, teamIds: Option[String], projectIds: Option[String])
 
 # Long-Running Jobs
 GET           /jobs/request                                                                     controllers.WKRemoteWorkerController.requestJobs(key: String)

frontend/javascripts/admin/admin_rest_api.ts

@@ -2016,6 +2016,7 @@ export async function getTimeTrackingForUserSummedPerAnnotation(
   if (annotationTypes != null) params.append("annotationTypes", annotationTypes);
   if (projectIds != null && projectIds.length > 0)
     params.append("projectIds", projectIds.join(","));
+  params.append("annotationStates", "Active,Finished");
   const timeTrackingData = await Request.receiveJSON(
     `/api/time/user/${userId}/summedByAnnotation?${params}`,
   );
@@ -2037,6 +2038,7 @@ export async function getTimeTrackingForUserSpans(
   if (annotationTypes != null) params.append("annotationTypes", annotationTypes);
   if (projectIds != null && projectIds.length > 0)
     params.append("projectIds", projectIds.join(","));
+  params.append("annotationStates", "Active,Finished");
   return await Request.receiveJSON(`/api/time/user/${userId}/spans?${params}`);
 }
 
@@ -2055,6 +2057,7 @@ export async function getTimeEntries(
   // Omit empty parameters in request
   if (projectIds.length > 0) params.append("projectIds", projectIds.join(","));
   if (teamIds.length > 0) params.append("teamIds", teamIds.join(","));
+  params.append("annotationStates", "Active,Finished");
   return await Request.receiveJSON(`api/time/overview?${params}`);
 }
 

frontend/javascripts/admin/statistic/project_and_annotation_type_dropdown.tsx

@@ -106,6 +106,7 @@ function ProjectAndAnnotationTypeDropdown({
       placeholder="Filter type or projects"
       style={style}
       options={filterOptions}
+      optionFilterProp="label"
       value={selectedFilters}
       onDeselect={(removedKey: string) => onDeselect(removedKey)}
       onSelect={(newSelection: string) => setSelectedProjects(selectedFilters, newSelection)}

frontend/javascripts/admin/statistic/time_tracking_detail_view.tsx

@@ -17,8 +17,8 @@ type TimeTrackingDetailViewProps = {
   projectIds: string[];
 };
 
-const ANNOTATION_OR_TASK_NAME_SPAN = 12;
-const STATISTICS_SPAN = 8;
+const ANNOTATION_OR_TASK_NAME_SPAN = 16;
+const STATISTICS_SPAN = 4;
 const TIMESPAN_SPAN = 4;
 
 const STYLING_CLASS_NAME = "time-tracking-details";

frontend/javascripts/oxalis/view/right-border-tabs/dataset_info_tab_view.tsx

@@ -212,7 +212,7 @@ export function AnnotationStats({
   const formatLabel = (str: string) => (asInfoBlock ? str : "");
   const useStyleWithMargin = withMargin != null ? withMargin : true;
   const styleWithLargeMarginBottom = { marginBottom: 14 };
-  const styleWithSmallMargin = { margin: "2px auto" };
+  const styleWithSmallMargin = { margin: 2 };
 
   return (
     <div