Improves Admin Analytics, Adds Choropleth

app/controllers/AdminController.scala

@@ -13,14 +13,15 @@ import models.daos.slick.DBTableDefinitions.UserTable
 import models.label.LabelTable.LabelMetadata
 import models.label.{LabelPointTable, LabelTable}
 import models.mission.MissionTable
-import models.region.RegionTable
+import models.region.{RegionCompletionTable, RegionTable}
 import models.street.{StreetEdge, StreetEdgeTable}
 import models.user.User
 import org.geotools.geometry.jts.JTS
 import org.geotools.referencing.CRS
 import play.api.libs.json.{JsArray, JsObject, JsValue, Json}
 import play.extras.geojson
 
+
 import scala.concurrent.Future
 
 /**
@@ -102,29 +103,36 @@ class AdminController @Inject() (implicit val env: Environment[User, SessionAuth
     * @return
     */
   def getNeighborhoodCompletionRate = UserAwareAction.async { implicit request =>
-    if (isAdmin(request.identity)) {
-
-      // http://docs.geotools.org/latest/tutorials/geometry/geometrycrs.html
-      val CRSEpsg4326 = CRS.decode("epsg:4326")
-      val CRSEpsg26918 = CRS.decode("epsg:26918")
-      val transform = CRS.findMathTransform(CRSEpsg4326, CRSEpsg26918)
-
+    RegionCompletionTable.initializeRegionCompletionTable()
+
+    val neighborhoods = RegionCompletionTable.selectAllNamedNeighborhoodCompletions
+    val completionRates: List[JsObject] = for (neighborhood <- neighborhoods) yield {
+      Json.obj("region_id" -> neighborhood.regionId,
+        "total_distance_m" -> neighborhood.totalDistance,
+        "completed_distance_m" -> neighborhood.auditedDistance,
+        "rate" -> (neighborhood.auditedDistance / neighborhood.totalDistance),
+        "name" -> neighborhood.name
+      )
+    }
 
-      val neighborhoods = RegionTable.selectAllNamedNeighborhoods
-      val completionRates: List[JsObject] = for (neighborhood <- neighborhoods) yield {
-        val streets: List[StreetEdge] = StreetEdgeTable.selectStreetsByARegionId(neighborhood.regionId)
-        val auditedStreets: List[StreetEdge] = StreetEdgeTable.selectAuditedStreetsByARegionId(neighborhood.regionId)
+    Future.successful(Ok(JsArray(completionRates)))
+  }
 
-        val completedDistance = auditedStreets.map(s => JTS.transform(s.geom, transform).getLength).sum
-        val totalDistance = streets.map(s => JTS.transform(s.geom, transform).getLength).sum
-        Json.obj("region_id" -> neighborhood.regionId,
-          "total_distance_m" -> totalDistance,
-          "completed_distance_m" -> completedDistance,
-          "name" -> neighborhood.name
+  /**
+    * Returns DC coverage percentage by Date
+    *
+    * @return
+    */
+  def getCompletionRateByDate = UserAwareAction.async { implicit request =>
+    if (isAdmin(request.identity)) {
+      val streets: Seq[(String, Float)] = StreetEdgeTable.streetDistanceCompletionRateByDate(1)
+      val json = Json.arr(streets.map(x => {
+        Json.obj(
+          "date" -> x._1, "completion" -> x._2
         )
-      }
+      }))
 
-      Future.successful(Ok(JsArray(completionRates)))
+      Future.successful(Ok(json))
     } else {
       Future.successful(Redirect("/"))
     }

app/controllers/TaskController.scala

@@ -107,11 +107,19 @@ class TaskController @Inject() (implicit val env: Environment[User, SessionAuthe
   def updateAuditTaskCompleteness(auditTaskId: Int, auditTask: TaskSubmission, incomplete: Option[IncompleteTaskSubmission]): Unit = {
     if (auditTask.completed.isDefined && auditTask.completed.get) {
       AuditTaskTable.updateCompleted(auditTaskId, completed=true)
-      StreetEdgeAssignmentCountTable.incrementCompletion(auditTask.streetEdgeId)
+      val updatedCount: Int = StreetEdgeAssignmentCountTable.incrementCompletion(auditTask.streetEdgeId)
+      // if this was the first completed audit of this street edge, increase total audited distance of that region.
+      if (updatedCount == 1) {
+        RegionCompletionTable.updateAuditedDistance(auditTask.streetEdgeId)
+      }
     } else if (incomplete.isDefined && incomplete.get.issueDescription == "GSVNotAvailable") {
       // If the user skipped with `GSVNotAvailable`, mark the task as completed and increment the task completion
       AuditTaskTable.updateCompleted(auditTaskId, completed=true)
-      StreetEdgeAssignmentCountTable.incrementCompletion(auditTask.streetEdgeId) // Increment task completion
+      val updatedCount: Int = StreetEdgeAssignmentCountTable.incrementCompletion(auditTask.streetEdgeId) // Increment task completion
+      // if this was the first completed audit of this street edge, increase total audited distance of that region.
+      if (updatedCount == 1) {
+        RegionCompletionTable.updateAuditedDistance(auditTask.streetEdgeId)
+      }
     }
   }
 

app/models/audit/AuditTaskTable.scala

@@ -109,10 +109,15 @@ object AuditTaskTable {
     auditTasks.list
   }
 
+  /**
+    * Returns a count of the number of audits performed on each day since the tool was launched (11/17/2015).
+    *
+    * @return
+    */
   def auditCounts: List[AuditCountPerDay] = db.withSession { implicit session =>
     val selectAuditCountQuery =  Q.queryNA[(String, Int)](
       """SELECT calendar_date::date, COUNT(audit_task_id) FROM (SELECT  current_date - (n || ' day')::INTERVAL AS calendar_date
-        |FROM    generate_series(0, 30) n) AS calendar
+        |FROM    generate_series(0, current_date - '11/17/2015') n) AS calendar
         |LEFT JOIN sidewalk.audit_task
         |ON audit_task.task_start::date = calendar_date::date
         |GROUP BY calendar_date

app/models/label/LabelTable.scala

@@ -464,18 +464,23 @@ object LabelTable {
 //    labelLocationList
   }
 
+  /**
+    * Returns a count of the number of labels placed on each day since the tool was launched (11/17/2015).
+    *
+    * @return
+    */
   def selectLabelCountsPerDay: List[LabelCountPerDay] = db.withSession { implicit session =>
-    val selectAuditCountQuery =  Q.queryNA[(String, Int)](
+    val selectLabelCountQuery =  Q.queryNA[(String, Int)](
       """SELECT calendar_date::date, COUNT(label_id) FROM (SELECT  current_date - (n || ' day')::INTERVAL AS calendar_date
-        |FROM    generate_series(0, 30) n) AS calendar
+        |FROM    generate_series(0, current_date - '11/17/2015') n) AS calendar
         |LEFT JOIN sidewalk.audit_task
         |ON audit_task.task_start::date = calendar_date::date
         |LEFT JOIN sidewalk.label
         |ON label.audit_task_id = audit_task.audit_task_id
         |GROUP BY calendar_date
         |ORDER BY calendar_date""".stripMargin
     )
-    selectAuditCountQuery.list.map(x => LabelCountPerDay.tupled(x))
+    selectLabelCountQuery.list.map(x => LabelCountPerDay.tupled(x))
   }
 }
 

app/models/region/RegionCompletionTable.scala

@@ -0,0 +1,138 @@
+package models.region
+
+import java.util.UUID
+
+import com.vividsolutions.jts.geom.Polygon
+import org.geotools.geometry.jts.JTS
+import org.geotools.referencing.CRS
+
+import math._
+import models.street.{StreetEdgeAssignmentCountTable, StreetEdgeRegionTable, StreetEdgeTable, StreetEdge}
+import models.user.UserCurrentRegionTable
+import models.utils.MyPostgresDriver
+import models.utils.MyPostgresDriver.simple._
+import play.api.Play.current
+
+import scala.slick.jdbc.{GetResult, StaticQuery => Q}
+import scala.slick.lifted.ForeignKeyQuery
+
+case class RegionCompletion(regionId: Int, totalDistance: Double, auditedDistance: Double)
+case class NamedRegionCompletion(regionId: Int, name: Option[String], totalDistance: Double, auditedDistance: Double)
+
+class RegionCompletionTable(tag: Tag) extends Table[RegionCompletion](tag, Some("sidewalk"), "region_completion") {
+  def regionId = column[Int]("region_id", O.PrimaryKey)
+  def totalDistance = column[Double]("total_distance")
+  def auditedDistance = column[Double]("audited_distance")
+
+  def * = (regionId, totalDistance, auditedDistance) <> ((RegionCompletion.apply _).tupled, RegionCompletion.unapply)
+}
+
+/**
+  * Data access object for the sidewalk_edge table
+  */
+object RegionCompletionTable {
+  import MyPostgresDriver.plainImplicits._
+
+  implicit val regionCompletionConverter = GetResult[RegionCompletion](r => {
+    RegionCompletion(r.nextInt, r.nextDouble, r.nextDouble)
+  })
+
+//  implicit val namedRegionConverter = GetResult[NamedRegion](r => {
+//    NamedRegion(r.nextInt, r.nextStringOption, r.nextGeometry[Polygon])
+//  })
+
+  case class StreetCompletion(regionId: Int, regionName: String, streetEdgeId: Int, completionCount: Int, distance: Double)
+  implicit val streetCompletionConverter = GetResult[StreetCompletion](r => {
+    StreetCompletion(r.nextInt, r.nextString, r.nextInt, r.nextInt, r.nextDouble)
+  })
+
+  val db = play.api.db.slick.DB
+  val regionCompletions = TableQuery[RegionCompletionTable]
+  val regions = TableQuery[RegionTable]
+  val regionTypes = TableQuery[RegionTypeTable]
+  val regionProperties = TableQuery[RegionPropertyTable]
+  val streetEdges = TableQuery[StreetEdgeTable]
+  val streetEdgeAssignmentCounts = TableQuery[StreetEdgeAssignmentCountTable]
+  val streetEdgeRegion = TableQuery[StreetEdgeRegionTable]
+  val userCurrentRegions = TableQuery[UserCurrentRegionTable]
+
+  val regionsWithoutDeleted = regions.filter(_.deleted === false)
+  val streetEdgesWithoutDeleted = streetEdges.filter(_.deleted === false)
+  val neighborhoods = regionsWithoutDeleted.filter(_.regionTypeId === 2)
+  val streetEdgeNeighborhood = for { (se, n) <- streetEdgeRegion.innerJoin(neighborhoods).on(_.regionId === _.regionId) } yield se
+
+
+  /**
+    * Returns a list of all neighborhoods with names
+    * @return
+    */
+  def selectAllNamedNeighborhoodCompletions: List[NamedRegionCompletion] = db.withSession { implicit session =>
+    val namedRegionCompletions = for {
+      (_neighborhoodCompletions, _regionProperties) <- regionCompletions.leftJoin(regionProperties).on(_.regionId === _.regionId)
+      if _regionProperties.key === "Neighborhood Name"
+    } yield (_neighborhoodCompletions.regionId, _regionProperties.value.?, _neighborhoodCompletions.totalDistance, _neighborhoodCompletions.auditedDistance)
+
+    namedRegionCompletions.list.map(x => NamedRegionCompletion.tupled(x))
+  }
+  /**
+    *
+    */
+
+
+  /**
+    * Increments the `audited_distance` column of the corresponding region by the length of the specified street edge.
+    * Reference: http://slick.lightbend.com/doc/2.0.0/queries.html#updating
+    *
+    * @param streetEdgeId street edge id
+    * @return
+    */
+  def updateAuditedDistance(streetEdgeId: Int) = db.withTransaction { implicit session =>
+    val distToAdd: Float = streetEdgesWithoutDeleted.filter(_.streetEdgeId === streetEdgeId).groupBy(x => x).map(_._1.geom.transform(26918).length).list.head
+    val regionId: Int = streetEdgeNeighborhood.filter(_.streetEdgeId === streetEdgeId).groupBy(x => x).map(_._1.regionId).list.head
+
+    val q = for { regionCompletion <- regionCompletions if regionCompletion.regionId === regionId } yield regionCompletion
+
+    val updatedDist = q.firstOption match {
+      case Some(rC) => q.map(_.auditedDistance).update(rC.auditedDistance + distToAdd)
+      case None => -1
+    }
+    updatedDist
+  }
+
+  def initializeRegionCompletionTable() = db.withTransaction { implicit session =>
+
+    if (regionCompletions.length.run == 0) {
+      // http://docs.geotools.org/latest/tutorials/geometry/geometrycrs.html
+      val CRSEpsg4326 = CRS.decode("epsg:4326")
+      val CRSEpsg26918 = CRS.decode("epsg:26918")
+      val transform = CRS.findMathTransform(CRSEpsg4326, CRSEpsg26918)
+
+      val neighborhoods = RegionTable.selectAllNamedNeighborhoods
+      for (neighborhood <- neighborhoods) yield {
+        val streets: List[StreetEdge] = StreetEdgeTable.selectStreetsByARegionId(neighborhood.regionId)
+        val auditedStreets: List[StreetEdge] = StreetEdgeTable.selectAuditedStreetsByARegionId(neighborhood.regionId)
+
+        val auditedDistance = auditedStreets.map(s => JTS.transform(s.geom, transform).getLength).sum
+        val totalDistance = streets.map(s => JTS.transform(s.geom, transform).getLength).sum
+
+        regionCompletions += RegionCompletion(neighborhood.regionId, totalDistance, auditedDistance)
+      }
+    }
+  }
+
+//
+//  /**
+//    * Update the `task_end` column of the specified audit task row
+//    *
+//    * @param auditTaskId
+//    * @param timestamp
+//    * @return
+//    */
+//  def updateTaskEnd(auditTaskId: Int, timestamp: Timestamp) = db.withTransaction { implicit session =>
+//    val q = for { task <- auditTasks if task.auditTaskId === auditTaskId } yield task.taskEnd
+//    q.update(Some(timestamp))
+//  }
+
+
+
+}

app/models/street/StreetEdgeAssignmentCountTable.scala

@@ -65,7 +65,7 @@ object StreetEdgeAssignmentCountTable {
   def incrementCompletion(edgeId: Int): Int = db.withTransaction { implicit session =>
     val q = for {counts <- streetEdgeAssignmentCounts if counts.streetEdgeId === edgeId} yield counts
     val count = q.firstOption match {
-      case Some(c) => q.map(_.completionCount).update(c.completionCount + 1)
+      case Some(c) => q.map(_.completionCount).update(c.completionCount + 1); c.completionCount + 1 // returns incremented completion count
       case None => 0
     }
     count

app/models/street/StreetEdgeTable.scala

@@ -2,6 +2,8 @@ package models.street
 
 import java.sql.Timestamp
 import java.util.UUID
+import java.util.Calendar
+import java.text.SimpleDateFormat
 
 import com.vividsolutions.jts.geom.LineString
 import models.audit.AuditTaskTable
@@ -101,6 +103,18 @@ object StreetEdgeTable {
     countAuditedStreets(auditCount).toFloat / allEdges.length
   }
 
+  /**
+    * Calculate the proportion of the total miles of DC that have been audited at least auditCount times.
+    *
+    * @param auditCount
+    * @return Float between 0 and 1
+    */
+  def streetDistanceCompletionRate(auditCount: Int): Float = db.withSession { implicit session =>
+    val auditedDistance = auditedStreetDistance(auditCount)
+    val totalDistance = totalStreetDistance()
+    auditedDistance / totalDistance
+  }
+
   /**
     * Get the total distance in miles
     * Reference: http://gis.stackexchange.com/questions/143436/how-do-i-calculate-st-length-in-miles
@@ -130,6 +144,70 @@ object StreetEdgeTable {
     (distances.sum * 0.000621371).toFloat
   }
 
+
+  /**
+    * Computes percentage of DC audited over time.
+    *
+    * author: Mikey Saugstad
+    * date: 06/16/2017
+    *
+    * @param auditCount
+    * @return List[(String,Float)] representing dates and percentages
+    */
+  def streetDistanceCompletionRateByDate(auditCount: Int): Seq[(String, Float)] = db.withSession { implicit session =>
+    // join the street edges and audit tasks
+    // TODO figure out how to do this w/out doing the join twice
+    val edges = for {
+      (_streetEdges, _auditTasks) <- streetEdgesWithoutDeleted.innerJoin(completedAuditTasks).on(_.streetEdgeId === _.streetEdgeId)
+    } yield _streetEdges
+    val audits = for {
+      (_streetEdges, _auditTasks) <- streetEdgesWithoutDeleted.innerJoin(completedAuditTasks).on(_.streetEdgeId === _.streetEdgeId)
+    } yield _auditTasks
+
+    // get distances of street edges associated with their edgeId
+    val edgeDists: Map[Int, Float] = edges.groupBy(x => x).map(g => (g._1.streetEdgeId, g._1.geom.transform(26918).length)).list.toMap
+
+    // Filter out group of edges with the size less than the passed `auditCount`, picking 1 rep from each group
+    // TODO pick audit with earliest timestamp
+    val uniqueEdgeDists: List[(Option[Timestamp], Option[Float])] = (for ((eid, groupedAudits) <- audits.list.groupBy(_.streetEdgeId)) yield {
+      if (auditCount > 0 && groupedAudits.size >= auditCount) {
+        Some((groupedAudits.head.taskEnd, edgeDists.get(eid)))
+      } else {
+        None
+      }
+    }).toList.flatten
+
+    // round the timestamps down to just the date (year-month-day)
+    val dateRoundedDists: List[(Calendar, Double)] = uniqueEdgeDists.map({
+      pair => {
+        var c : Calendar = Calendar.getInstance()
+        c.setTimeInMillis(pair._1.get.getTime)
+        c.set(Calendar.HOUR_OF_DAY, 0)
+        c.set(Calendar.MINUTE, 0)
+        c.set(Calendar.SECOND, 0)
+        c.set(Calendar.MILLISECOND, 0)
+        (c, pair._2.get * 0.000621371)
+      }})
+
+    // sum the distances by date
+    val distsPerDay: List[(Calendar, Double)] = dateRoundedDists.groupBy(_._1).mapValues(_.map(_._2).sum).view.force.toList
+
+    // sort the list by date
+    val sortedEdges: Seq[(Calendar, Double)] =
+      scala.util.Sorting.stableSort(distsPerDay, (e1: (Calendar,Double), e2: (Calendar, Double)) => e1._1.getTimeInMillis < e2._1.getTimeInMillis).toSeq
+
+    // get the cumulative distance over time
+    val cumDistsPerDay: Seq[(Calendar, Double)] = sortedEdges.map({var dist = 0.0; pair => {dist += pair._2; (pair._1, dist)}})
+
+    // calculate the completion percentage for each day
+    val totalDist = totalStreetDistance()
+    val ratePerDay: Seq[(Calendar, Float)] = cumDistsPerDay.map(pair => (pair._1, (100.0 * pair._2 / totalDist).toFloat))
+
+    // format the calendar date in the correct format and return the (date,completionPercentage) pair
+    val format1 = new SimpleDateFormat("yyyy-MM-dd")
+    ratePerDay.map(pair => (format1.format(pair._1.getTime), pair._2))
+  }
+
   /**
     * Count the number of streets that have been audited at least a given number of times
     *