Епик Александр - Phase 1

app/ DTOs/HolderDTO.scala

@@ -0,0 +1,10 @@
+package DTOs
+
+import play.api.libs.json.{Json, Reads}
+
+case class HolderDTO(pages: Int,
+                     items: Option[Seq[VacancyDTO]])
+
+object HolderDTO {
+  implicit val holderDtoReader: Reads[HolderDTO] = Json.reads[HolderDTO]
+}

app/ DTOs/RegionDTO.scala

@@ -0,0 +1,11 @@
+package DTOs
+
+import play.api.libs.json.{Json, Reads}
+
+case class RegionDTO(id: String,
+                     name: String,
+                     areas: Seq[RegionDTO])
+
+object RegionDTO {
+  implicit val regionDtoReader: Reads[RegionDTO] = Json.reads[RegionDTO]
+}

app/ DTOs/SalaryDTO.scala

@@ -0,0 +1,12 @@
+package DTOs
+
+import play.api.libs.json.{Json, Reads}
+
+case class SalaryDTO(currency: Option[String],
+                     from: Option[Int],
+                     gross: Option[Boolean],
+                     to: Option[Int])
+
+object SalaryDTO {
+  implicit val salaryDtoReader: Reads[SalaryDTO] = Json.reads[SalaryDTO]
+}

app/ DTOs/SnippetDTO.scala

@@ -0,0 +1,10 @@
+package DTOs
+
+import play.api.libs.json.{Json, Reads}
+
+case class SnippetDTO(requirement: Option[String],
+                      responsibility: Option[String])
+
+object SnippetDTO {
+  implicit val snippetDtoReader: Reads[SnippetDTO] = Json.reads[SnippetDTO]
+}

app/ DTOs/VacancyDTO.scala

@@ -0,0 +1,13 @@
+package DTOs
+
+import play.api.libs.json.{Json, Reads}
+
+case class VacancyDTO(id: String,
+                      name: Option[String],
+                      alternate_url: Option[String],
+                      snippet: Option[SnippetDTO],
+                      salary: Option[SalaryDTO])
+
+object VacancyDTO {
+  implicit val vacancyDtoReader: Reads[VacancyDTO] = Json.reads[VacancyDTO]
+}

app/controllers/JobAggregatorController.scala

@@ -3,13 +3,17 @@ package controllers
 import javax.inject._
 import play.api.mvc._
 import service.JobAggregatorService
+import scala.util.{Failure, Success}
 
-import scala.concurrent.ExecutionContext
+import scala.concurrent.{ExecutionContext, Future}
 
 @Singleton
-class JobAggregatorController @Inject()(val controllerComponents: ControllerComponents, jobAggregatorService: JobAggregatorService)(implicit ec: ExecutionContext) extends BaseController {
+class JobAggregatorController @Inject()(val controllerComponents: ControllerComponents,
+                                        jobAggregatorService: JobAggregatorService)(implicit ec: ExecutionContext) extends BaseController {
 
-  def index() = Action.async { implicit request: Request[AnyContent] =>
-    jobAggregatorService.addJobTest("Test title").map(_ => Ok(""))
+  def index(text: String, area: Int) = Action.async { implicit request: Request[AnyContent] =>
+
+    jobAggregatorService.aggregateData(text, area).map(_ => Ok(""))
+      .recover(exception => InternalServerError("Following exception has occurred: " + exception.getMessage))
   }
 }

app/model/Job.scala

@@ -1,5 +1,13 @@
 package model
 
-import java.util.UUID
-
-case class Job(id: UUID, title: String)
+case class Job(id: Int,
+               title: Option[String],
+               requirement: Option[String],
+               responsibility: Option[String],
+               alternateUrl: Option[String],
+               salaryFrom: Option[Int],
+               salaryTo: Option[Int],
+               salaryCurrency: Option[String],
+               salaryGross: Option[Boolean],
+               city: Option[String],
+               keyWord: Option[String])

app/model/db/JobTable.scala

@@ -3,13 +3,32 @@ package model.db
 import model.Job
 import slick.jdbc.PostgresProfile.api._
 
-import java.util.UUID
-
 class JobTable(tag: Tag) extends Table[Job](tag, "job") {
-    def id = column[UUID]("id", O.PrimaryKey)
-    def title = column[String]("title")
+  def id = column[Int]("id", O.PrimaryKey)
+
+  def title = column[Option[String]]("title")
+
+  def requirement = column[Option[String]]("requirement")
+
+  def responsibility = column[Option[String]]("responsibility")
+
+  def alternateUrl = column[Option[String]]("alternate_url")
+
+  def salaryFrom = column[Option[Int]]("salary_from")
+
+  def salaryTo = column[Option[Int]]("salary_to")
+
+  def salaryCurrency = column[Option[String]]("salary_currency")
+
+  def salaryGross = column[Option[Boolean]]("salary_gross")
+
+  def city = column[Option[String]]("city")
+
+  def keyWord = column[Option[String]]("key_word")
 
 
-    def * = (id,title) <> (Job.tupled, Job.unapply)
+  def * =
+    (id, title, requirement, responsibility, alternateUrl, salaryFrom, salaryTo, salaryCurrency, salaryGross,
+      city, keyWord) <> (Job.tupled, Job.unapply)
 }
 

app/scheduler/Task.scala

@@ -0,0 +1,41 @@
+package scheduler
+
+import javax.inject.Inject
+import akka.actor.ActorSystem
+
+import scala.concurrent.ExecutionContext
+import scala.concurrent.duration._
+import play.api.{Configuration, Logger, Mode}
+import service.JobAggregatorService
+
+import scala.util.Try
+
+class Task @Inject()(actorSystem: ActorSystem,
+                     configuration: Configuration,
+                     jobAggregatorService: JobAggregatorService)(implicit executionContext: ExecutionContext) {
+
+  if (!configuration.has("initialDelay") || !configuration.has("interval")
+    || !configuration.has("cities") || !configuration.has("keyWords"))
+    throw new NoSuchFieldException("Configuration doesn't have some of these paths:" +
+      " initialDelay, interval, cities, keyWords")
+
+  val initialDelay: Option[String] = configuration.getOptional[String]("initialDelay")
+  val interval: Option[String] = configuration.getOptional[String]("interval")
+  val cities: Option[Seq[String]] = configuration.getOptional[Seq[String]]("cities")
+  val keyWords: Option[Seq[String]] = configuration.getOptional[Seq[String]]("keyWords")
+
+  if (initialDelay.isEmpty || interval.isEmpty || cities.isEmpty || keyWords.isEmpty)
+    throw new ClassCastException("Some of these paths have wrong type: initialDelay, interval, cities, keyWords")
+
+  val initDelay: Try[FiniteDuration] = Try(Duration(initialDelay.get).asInstanceOf[FiniteDuration])
+  val interv: Try[FiniteDuration] = Try(Duration(interval.get).asInstanceOf[FiniteDuration])
+
+  if(initDelay.isFailure || interv.isFailure)
+    throw new ClassCastException("Initial delay or interval have wrong format")
+
+  actorSystem.scheduler.scheduleAtFixedRate(initialDelay = initDelay.get,
+    interval = interv.get) { () =>
+    jobAggregatorService.aggregateData(keyWords.get, cities.get)
+    Logger("play").info("Scheduled task executed")
+  }
+}

app/scheduler/TasksModule.scala

@@ -0,0 +1,6 @@
+package scheduler
+
+import play.api.inject.SimpleModule
+import play.api.inject._
+
+class TasksModule extends SimpleModule(bind[Task].toSelf.eagerly())

app/service/JobAggregatorService.scala

@@ -1,21 +1,184 @@
 package service
 
+import DTOs._
 import model.Job
 import model.db.DBTables.jobTable
 import play.api.db.slick.{DatabaseConfigProvider, HasDatabaseConfigProvider}
 import slick.jdbc.JdbcProfile
 import slick.jdbc.PostgresProfile.api._
-
-import java.util.UUID
 import javax.inject.{Inject, Singleton}
+import play.api.{Configuration, Logger, Mode}
+import play.api.libs.ws._
+import play.api.libs.json._
+
+import scala.util.{Failure, Success, Try}
+import scala.concurrent.ExecutionContext.Implicits.global
+import scala.collection.mutable.ListBuffer
+import scala.concurrent.{Await, Future}
 
-import scala.concurrent.Future
 
 @Singleton
-class JobAggregatorService @Inject()(val dbConfigProvider: DatabaseConfigProvider) extends HasDatabaseConfigProvider[JdbcProfile] {
+class JobAggregatorService @Inject()(ws: WSClient,
+                                     configuration: Configuration,
+                                     val dbConfigProvider: DatabaseConfigProvider) extends HasDatabaseConfigProvider[JdbcProfile] {
+
+  val perPage: Int = configuration.getOptional[Int]("perPage") match {
+    case Some(value) => value
+    case None =>
+      Logger("play").warn("perPage is not configured. The default value(100) will be used")
+      100
+  }
+
+  /**
+   * агрегирует вакансии по ключевому слову и региону
+   *
+   * @param text ключевое слов
+   * @param area индекс региона
+   */
+  def aggregateData(text: String, area: Int) = {
+
+    ws.url(s"https://api.hh.ru/vacancies?text=$text&area=$area&per_page=$perPage&page=0")
+      .get()
+      .flatMap(x => getJobs(x.json, text, area))
+      .map(buff => buff.foreach(x => addToDB(x)))
+  }
+
+  /**
+   * агрегирует вакансии по ключевому слову и региону
+   *
+   * @param keyWords ключевые слова
+   * @param areas    индексы регионов
+   */
+  def aggregateData(keyWords: Seq[String], areas: Seq[String]): Unit = {
+    getRegions() match {
+      case Success(value) => {
+        val regionsMap = value.map(x => x._2)
+        for {
+          keyWord <- keyWords
+          area <- areas
+        } regionsMap.map(x => aggregateData(keyWord, x(area)))
+      }
+      case Failure(exception) => Logger("play").error(exception.getMessage, exception)
+    }
+  }
+
+  /**
+   * делает несколько запросов к hh.ru, если все вакансии не помещаются в один ответ
+   *
+   * @param firstResp первый ответ от hh.ru
+   * @return Future от массива всех полученных вакансий
+   */
+  private def getJobs(firstResp: JsValue, keyWord: String, area: Int): Future[List[Job]] = {
 
-    def addJobTest(title: String): Future[Int] = {
-        db.run(jobTable += Job(UUID.randomUUID(), title))
+    def handleJobs(jobs: Try[List[Job]]) = {
+      jobs match {
+        case Success(value) => value
+        case Failure(exception) => Logger("play").error(exception.getMessage, exception)
+          Nil
+      }
     }
 
+    def parseAllPages(areaText: String) = {
+      val pages = firstResp.asOpt[HolderDTO] match {
+        case Some(holder) => holder.pages
+        case None => throw new ClassCastException("json can't be parsed: " + firstResp.toString)
+      }
+      val jobs: List[Job] = handleJobs(getJobsFromPage(firstResp, keyWord, areaText))
+
+      val requests = for (n <- 1 until pages)
+        yield ws.url(s"https://api.hh.ru/vacancies?text=$keyWord&area=$area&per_page=$perPage&page=$n")
+
+      requests.foldLeft(Future.successful(jobs))((fut, req) =>
+        req.get().map(resp => handleJobs(getJobsFromPage(resp.json, keyWord, areaText)))
+          .flatMap(newJobs => fut.map(oldJobs => oldJobs ++ newJobs)))
+    }
+
+    getRegions() match {
+      case Success(value) => value.flatMap(x => parseAllPages(x._1(area)))
+      case Failure(exception) => Logger("play").error(exception.getMessage, exception)
+        Future.successful(Nil)
+    }
+  }
+
+  /**
+   * парсит вакансии в массив DTO Job
+   *
+   * @param json json, содержащий массив вакансий
+   * @return массив DTO Job
+   */
+  private def getJobsFromPage(json: JsValue, keyWord: String, area: String): Try[List[Job]] = {
+    var jobs = ListBuffer[Job]()
+
+    def addJob(item: VacancyDTO): Unit = {
+      var requirement: Option[String] = None
+      var responsibility: Option[String] = None
+
+      item.snippet match {
+        case Some(value) => {
+          requirement = value.requirement
+          responsibility = value.responsibility
+        }
+        case _ =>
+      }
+
+      Try(item.id.toInt).toOption match {
+        case Some(id) => item.salary match {
+          case Some(salary) => jobs += Job(id, item.name, requirement, responsibility,
+            item.alternate_url, salary.from, salary.to, salary.currency, salary.gross, Option(area), Option(keyWord))
+          case None => jobs += Job(id, item.name, requirement, responsibility,
+            item.alternate_url, None, None, None, None, Option(area), Option(keyWord))
+        }
+        case None => throw new NumberFormatException("id value can't be converted to int" + json.toString())
+      }
+    }
+
+    Try {
+      json.asOpt[HolderDTO] match {
+        case Some(holder) => holder.items match {
+          case Some(items) => items.foreach(addJob)
+          case None => throw new RuntimeException("Items doesn't exist" + json.toString())
+        }
+        case None => throw new RuntimeException("Can't be parsed to HolderDTO" + json.toString())
+      }
+
+      jobs.toList
+    }
+  }
+
+  /**
+   * @return возвращает два словаря: (id -> название_региона) и (название_региона -> id)
+   */
+  private def getRegions() = {
+    val resp = ws.url("https://api.hh.ru/areas").get()
+    val indexToRegion = scala.collection.mutable.Map[Int, String]()
+    val regionToIndex = scala.collection.mutable.Map[String, Int]()
+
+
+    def initialParse(json: JsValue) = {
+      json.asOpt[Seq[RegionDTO]] match {
+        case Some(regions) => regions.foreach(parseRegions)
+        case None => throw new ClassCastException("Can't be parsed to Seq[RegionDTO]: " + json.toString())
+      }
+      (indexToRegion.toMap, regionToIndex.toMap)
+    }
+
+    def parseRegions(regionDTO: RegionDTO): Unit = {
+      Try(regionDTO.id.toInt).toOption match {
+        case Some(id) => {
+          indexToRegion += (id -> regionDTO.name)
+          regionToIndex += (regionDTO.name -> id)
+          if (regionDTO.areas.nonEmpty) {
+            regionDTO.areas.foreach(area => parseRegions(area))
+          }
+        }
+        case None => throw new NumberFormatException("region id can't be converted to int")
+      }
+    }
+
+    Try(resp.map(x => initialParse(x.json)))
+  }
+
+  private def addToDB(job: Job) = {
+    db.run(DBIO.seq(jobTable += job))
+  }
 }

conf/application.conf

@@ -1,6 +1,13 @@
 slick.dbs.default.driver="slick.driver.PostgresDriver$"
 slick.dbs.default.db.driver="org.postgresql.Driver"
-slick.dbs.default.db.url="jdbc:postgresql://10.106.0.26:5434/postgres"
+slick.dbs.default.db.url="jdbc:postgresql://localhost:5434/postgres"
 slick.dbs.default.db.user="postgres"
 slick.dbs.default.db.password="12345678"
 
+//play.modules.enabled += "scheduler.TasksModule"
+
+initialDelay="10 seconds"
+interval="30 seconds"
+cities=["Москва","Екатеринбург"]
+keyWords=["java","scala"]
+perPage=100