From f397574b7bdfa1579f42a2d2d56a12ad5dc6aded Mon Sep 17 00:00:00 2001 From: Roberto Tena Date: Wed, 2 Aug 2017 08:46:50 +0100 Subject: [PATCH 01/22] Add tests for scheduling messages (WIP) --- build.sbt | 5 +- .../message/scheduler/SchedulingActor.scala | 44 ++++++++++++ .../scheduler/SchedulerActorSpec.scala | 70 +++++++++++++++++++ .../message/scheduler/SchedulerIntSpec.scala | 2 +- .../message/scheduler/TestDataUtils.scala | 3 + .../src/test/scala/common/BaseSpec.scala | 4 +- 6 files changed, 124 insertions(+), 4 deletions(-) create mode 100644 scheduler/src/main/scala/com/sky/kafka/message/scheduler/SchedulingActor.scala create mode 100644 scheduler/src/test/scala/com/sky/kafka/message/scheduler/SchedulerActorSpec.scala diff --git a/build.sbt b/build.sbt index 02c558d9..cceca70b 100644 --- a/build.sbt +++ b/build.sbt @@ -22,9 +22,12 @@ val dependencies = Seq( "org.scalatest" %% "scalatest" % "3.0.1" % Test, "com.typesafe.akka" %% "akka-testkit" % akkaVersion % Test, + "com.typesafe.akka" %% "akka-stream-testkit" % akkaVersion % Test, "net.cakesolutions" %% "scala-kafka-client-testkit" % kafkaVersion % Test, "org.slf4j" % "log4j-over-slf4j" % "1.7.21" % Test, - "com.danielasfregola" %% "random-data-generator" % "2.1" % Test + "com.danielasfregola" %% "random-data-generator" % "2.1" % Test, + "com.miguno.akka" %% "akka-mock-scheduler" % "0.5.1" % Test, + "org.mockito" % "mockito-all" % "1.10.19" % Test ) val commonSettings = Seq( diff --git a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/SchedulingActor.scala b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/SchedulingActor.scala new file mode 100644 index 00000000..1a274ad1 --- /dev/null +++ b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/SchedulingActor.scala @@ -0,0 +1,44 @@ +package com.sky.kafka.message.scheduler + +import java.time.OffsetDateTime +import java.time.temporal.ChronoUnit +import java.util.concurrent.TimeUnit + +import akka.actor.{Actor, ActorLogging, Cancellable, Scheduler} +import akka.stream.scaladsl.SourceQueue +import com.sky.kafka.message.scheduler.SchedulingActor.{Ack, CancelSchedule, CreateOrUpdateSchedule} +import com.sky.kafka.message.scheduler.domain.Schedule + +import scala.concurrent.ExecutionContext.Implicits.global +import scala.concurrent.duration.FiniteDuration + +object SchedulingActor { + + case object Ack + + case class CreateOrUpdateSchedule(scheduleId: String, schedule: Schedule) + + case class CancelSchedule(scheduleId: String) + +} + +class SchedulingActor(sourceQueue: SourceQueue[(String, Schedule)], scheduler: Scheduler) extends Actor with ActorLogging { + + override def receive: Receive = manageSchedules(Map.empty) + + def manageSchedules(cancellableSchedules: Map[String, Cancellable]): Receive = { + case CreateOrUpdateSchedule(scheduleId: String, schedule: Schedule) => + val timeFromNow = ChronoUnit.MILLIS.between(OffsetDateTime.now, schedule.time) + val cancellable = scheduler.scheduleOnce(FiniteDuration(timeFromNow, TimeUnit.MILLISECONDS))(sourceQueue.offer((scheduleId, schedule))) + sender ! Ack + context.become(manageSchedules(cancellableSchedules + (scheduleId -> cancellable))) + case CancelSchedule(scheduleId: String) => + val cancelled = cancellableSchedules.get(scheduleId).exists(_.cancel()) + if (cancelled) + log.info(s"Cancelled schedule $scheduleId") + else + log.warning(s"Couldn't cancel $scheduleId") + sender ! Ack + context.become(manageSchedules(cancellableSchedules - scheduleId)) + } +} diff --git a/scheduler/src/test/scala/com/sky/kafka/message/scheduler/SchedulerActorSpec.scala b/scheduler/src/test/scala/com/sky/kafka/message/scheduler/SchedulerActorSpec.scala new file mode 100644 index 00000000..1c32a47a --- /dev/null +++ b/scheduler/src/test/scala/com/sky/kafka/message/scheduler/SchedulerActorSpec.scala @@ -0,0 +1,70 @@ +package com.sky.kafka.message.scheduler + +import java.time.OffsetDateTime +import java.time.temporal.ChronoField +import java.util.UUID +import java.util.concurrent.TimeUnit + +import akka.stream.scaladsl.SourceQueue +import akka.testkit.{ImplicitSender, TestActorRef, TestKit} +import com.miguno.akka.testing.VirtualTime +import com.sky.kafka.message.scheduler.SchedulingActor.{Ack, CancelSchedule, CreateOrUpdateSchedule} +import com.sky.kafka.message.scheduler.TestDataUtils._ +import com.sky.kafka.message.scheduler.domain.Schedule +import common.{BaseSpec, TestActorSystem} +import org.mockito.Mockito +import org.scalatest.mockito.MockitoSugar +import Mockito._ + +import scala.concurrent.duration._ + + +class SchedulingActorSpec extends TestKit(TestActorSystem()) with ImplicitSender with BaseSpec with MockitoSugar { + + case class UnexpectedMessageType(whatever: String) + + override def afterAll(): Unit = + TestKit.shutdownActorSystem(system) + + private class SchedulingActorTest { + val mockSourceQueue = mock[SourceQueue[(String, Schedule)]] + val time = new VirtualTime + + val actorRef = TestActorRef(new SchedulingActor(mockSourceQueue, time.scheduler)) + + def advanceTimeTo(offsetDateTime: OffsetDateTime) = + time.advance(offsetDateTime.toInstant.toEpochMilli - System.currentTimeMillis() + 1 second) + + def createSchedule(scheduleId: String, schedule: Schedule) = { + actorRef ! CreateOrUpdateSchedule(scheduleId, schedule) + expectMsg(Ack) + } + } + + "A scheduler actor" must { + "schedule new messages at the given time" in new SchedulingActorTest { + val (scheduleId, schedule) = generateSchedule + + createSchedule(scheduleId, schedule) + + advanceTimeTo(schedule.time) + verify(mockSourceQueue).offer((scheduleId, schedule)) + } + + "cancel schedules when a cancel message is received" in new SchedulingActorTest { + val (scheduleId, schedule) = generateSchedule + + createSchedule(scheduleId, schedule) + + actorRef ! CancelSchedule(scheduleId) + expectMsg(Ack) + + advanceTimeTo(schedule.time) + verifyZeroInteractions(mockSourceQueue) + } + + } + + private def generateSchedule = + (UUID.randomUUID().toString, random[Schedule].copy(time = OffsetDateTime.now().plusMinutes(5))) +} \ No newline at end of file diff --git a/scheduler/src/test/scala/com/sky/kafka/message/scheduler/SchedulerIntSpec.scala b/scheduler/src/test/scala/com/sky/kafka/message/scheduler/SchedulerIntSpec.scala index f7ccc068..3bb4af57 100644 --- a/scheduler/src/test/scala/com/sky/kafka/message/scheduler/SchedulerIntSpec.scala +++ b/scheduler/src/test/scala/com/sky/kafka/message/scheduler/SchedulerIntSpec.scala @@ -17,7 +17,7 @@ class SchedulerIntSpec extends AkkaStreamIntSpec with KafkaIntSpec { val conf = SchedulerConfig(ScheduleTopic, ShutdownTimeout(10 seconds, 10 seconds)) "Scheduler stream" should { - "publish a given message with a given key to a given topic" in withRunningSchedulerStream { + "schedule a message to be sent to Kafka" in withRunningSchedulerStream { val schedule = random[Schedule] writeToKafka(ScheduleTopic, "scheduleId", schedule.toAvro) diff --git a/scheduler/src/test/scala/com/sky/kafka/message/scheduler/TestDataUtils.scala b/scheduler/src/test/scala/com/sky/kafka/message/scheduler/TestDataUtils.scala index b7030507..a9c88bb9 100644 --- a/scheduler/src/test/scala/com/sky/kafka/message/scheduler/TestDataUtils.scala +++ b/scheduler/src/test/scala/com/sky/kafka/message/scheduler/TestDataUtils.scala @@ -13,6 +13,9 @@ object TestDataUtils extends RandomDataGenerator { implicit val arbAlphaString: Arbitrary[String] = Arbitrary(Gen.alphaStr.suchThat(!_.isEmpty)) + //implicit val chooseOffsetDateTime: Choose[OffsetDateTime] = Choose.xmap[OffsetDateTime, OffsetDateTime](OffsetDateTime.MIN, OffsetDateTime.MAX) + + //TODO java.time.DateTimeException: Invalid value for Year (valid values -999999999 - 999999999): 1000000000 implicit val arbOffsetDateTime: Arbitrary[OffsetDateTime] = Arbitrary(Gen.oneOf(OffsetDateTime.MIN, OffsetDateTime.MAX)) diff --git a/scheduler/src/test/scala/common/BaseSpec.scala b/scheduler/src/test/scala/common/BaseSpec.scala index c79deb68..c6af67d1 100644 --- a/scheduler/src/test/scala/common/BaseSpec.scala +++ b/scheduler/src/test/scala/common/BaseSpec.scala @@ -1,5 +1,5 @@ package common -import org.scalatest.{Matchers, WordSpecLike} +import org.scalatest.{BeforeAndAfterAll, Matchers, WordSpecLike} -trait BaseSpec extends WordSpecLike with Matchers \ No newline at end of file +trait BaseSpec extends WordSpecLike with Matchers with BeforeAndAfterAll \ No newline at end of file From b0d1a6966f32a4e1fae86342f4abd88adf78042f Mon Sep 17 00:00:00 2001 From: "Hubert Behaghel, Lawrence Carvalho, Matthew Pickering, Paolo Ambrosio and Roberto Tena" Date: Wed, 2 Aug 2017 16:52:08 +0100 Subject: [PATCH 02/22] scheduling actor complete, just needs to be wired up. --- .../message/scheduler/SchedulingActor.scala | 61 ++++++++---- .../scheduler/SchedulerActorSpec.scala | 70 -------------- .../message/scheduler/SchedulerIntSpec.scala | 12 +-- .../message/scheduler/SchedulerSpec.scala | 2 +- .../scheduler/SchedulingActorSpec.scala | 92 +++++++++++++++++++ .../src/test/scala/common/AkkaBaseSpec.scala | 13 +++ .../scala/common/AkkaStreamBaseSpec.scala | 8 ++ .../test/scala/common/AkkaStreamIntSpec.scala | 17 ---- .../scheduler => common}/TestDataUtils.scala | 19 ++-- 9 files changed, 172 insertions(+), 122 deletions(-) delete mode 100644 scheduler/src/test/scala/com/sky/kafka/message/scheduler/SchedulerActorSpec.scala create mode 100644 scheduler/src/test/scala/com/sky/kafka/message/scheduler/SchedulingActorSpec.scala create mode 100644 scheduler/src/test/scala/common/AkkaBaseSpec.scala create mode 100644 scheduler/src/test/scala/common/AkkaStreamBaseSpec.scala delete mode 100644 scheduler/src/test/scala/common/AkkaStreamIntSpec.scala rename scheduler/src/test/scala/{com/sky/kafka/message/scheduler => common}/TestDataUtils.scala (59%) diff --git a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/SchedulingActor.scala b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/SchedulingActor.scala index 1a274ad1..f0711b15 100644 --- a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/SchedulingActor.scala +++ b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/SchedulingActor.scala @@ -6,8 +6,8 @@ import java.util.concurrent.TimeUnit import akka.actor.{Actor, ActorLogging, Cancellable, Scheduler} import akka.stream.scaladsl.SourceQueue -import com.sky.kafka.message.scheduler.SchedulingActor.{Ack, CancelSchedule, CreateOrUpdateSchedule} -import com.sky.kafka.message.scheduler.domain.Schedule +import com.sky.kafka.message.scheduler.SchedulingActor.{Ack, Cancel, CreateOrUpdate} +import com.sky.kafka.message.scheduler.domain.{Schedule, ScheduleId} import scala.concurrent.ExecutionContext.Implicits.global import scala.concurrent.duration.FiniteDuration @@ -16,29 +16,50 @@ object SchedulingActor { case object Ack - case class CreateOrUpdateSchedule(scheduleId: String, schedule: Schedule) + case class CreateOrUpdate(scheduleId: ScheduleId, schedule: Schedule) - case class CancelSchedule(scheduleId: String) + case class Cancel(scheduleId: ScheduleId) } class SchedulingActor(sourceQueue: SourceQueue[(String, Schedule)], scheduler: Scheduler) extends Actor with ActorLogging { - override def receive: Receive = manageSchedules(Map.empty) - - def manageSchedules(cancellableSchedules: Map[String, Cancellable]): Receive = { - case CreateOrUpdateSchedule(scheduleId: String, schedule: Schedule) => - val timeFromNow = ChronoUnit.MILLIS.between(OffsetDateTime.now, schedule.time) - val cancellable = scheduler.scheduleOnce(FiniteDuration(timeFromNow, TimeUnit.MILLISECONDS))(sourceQueue.offer((scheduleId, schedule))) - sender ! Ack - context.become(manageSchedules(cancellableSchedules + (scheduleId -> cancellable))) - case CancelSchedule(scheduleId: String) => - val cancelled = cancellableSchedules.get(scheduleId).exists(_.cancel()) - if (cancelled) - log.info(s"Cancelled schedule $scheduleId") - else - log.warning(s"Couldn't cancel $scheduleId") - sender ! Ack - context.become(manageSchedules(cancellableSchedules - scheduleId)) + override def receive: Receive = receiveScheduleMessages(Map.empty) + + def receiveScheduleMessages(schedules: Map[ScheduleId, Cancellable]): Receive = { + + val receiveCreateOrUpdateMessage: PartialFunction[Any, Map[ScheduleId, Cancellable]] = { + case CreateOrUpdate(scheduleId: ScheduleId, schedule: Schedule) => + if (cancel(scheduleId, schedules)) + log.info(s"Updating schedule $scheduleId") + else + log.info(s"Creating schedule $scheduleId") + val cancellable = scheduler.scheduleOnce(timeFromNow(schedule.time))(sourceQueue.offer((scheduleId, schedule))) + schedules + (scheduleId -> cancellable) + } + + val receiveCancelMessage: PartialFunction[Any, Map[ScheduleId, Cancellable]] = { + case Cancel(scheduleId: String) => + if (cancel(scheduleId, schedules)) + log.info(s"Cancelled schedule $scheduleId") + else + log.warning(s"Couldn't cancel $scheduleId") + schedules - scheduleId + } + + receiveCreateOrUpdateMessage orElse receiveCancelMessage andThen updateStateAndAck + } + + def updateStateAndAck(schedules: Map[ScheduleId, Cancellable]): Unit = { + context.become(receiveScheduleMessages(schedules)) + sender ! Ack + } + + def cancel(scheduleId: ScheduleId, schedules: Map[ScheduleId, Cancellable]): Boolean = + schedules.get(scheduleId).exists(_.cancel()) + + def timeFromNow(time: OffsetDateTime): FiniteDuration = { + val offset = ChronoUnit.MILLIS.between(OffsetDateTime.now, time) + FiniteDuration(offset, TimeUnit.MILLISECONDS) } } diff --git a/scheduler/src/test/scala/com/sky/kafka/message/scheduler/SchedulerActorSpec.scala b/scheduler/src/test/scala/com/sky/kafka/message/scheduler/SchedulerActorSpec.scala deleted file mode 100644 index 1c32a47a..00000000 --- a/scheduler/src/test/scala/com/sky/kafka/message/scheduler/SchedulerActorSpec.scala +++ /dev/null @@ -1,70 +0,0 @@ -package com.sky.kafka.message.scheduler - -import java.time.OffsetDateTime -import java.time.temporal.ChronoField -import java.util.UUID -import java.util.concurrent.TimeUnit - -import akka.stream.scaladsl.SourceQueue -import akka.testkit.{ImplicitSender, TestActorRef, TestKit} -import com.miguno.akka.testing.VirtualTime -import com.sky.kafka.message.scheduler.SchedulingActor.{Ack, CancelSchedule, CreateOrUpdateSchedule} -import com.sky.kafka.message.scheduler.TestDataUtils._ -import com.sky.kafka.message.scheduler.domain.Schedule -import common.{BaseSpec, TestActorSystem} -import org.mockito.Mockito -import org.scalatest.mockito.MockitoSugar -import Mockito._ - -import scala.concurrent.duration._ - - -class SchedulingActorSpec extends TestKit(TestActorSystem()) with ImplicitSender with BaseSpec with MockitoSugar { - - case class UnexpectedMessageType(whatever: String) - - override def afterAll(): Unit = - TestKit.shutdownActorSystem(system) - - private class SchedulingActorTest { - val mockSourceQueue = mock[SourceQueue[(String, Schedule)]] - val time = new VirtualTime - - val actorRef = TestActorRef(new SchedulingActor(mockSourceQueue, time.scheduler)) - - def advanceTimeTo(offsetDateTime: OffsetDateTime) = - time.advance(offsetDateTime.toInstant.toEpochMilli - System.currentTimeMillis() + 1 second) - - def createSchedule(scheduleId: String, schedule: Schedule) = { - actorRef ! CreateOrUpdateSchedule(scheduleId, schedule) - expectMsg(Ack) - } - } - - "A scheduler actor" must { - "schedule new messages at the given time" in new SchedulingActorTest { - val (scheduleId, schedule) = generateSchedule - - createSchedule(scheduleId, schedule) - - advanceTimeTo(schedule.time) - verify(mockSourceQueue).offer((scheduleId, schedule)) - } - - "cancel schedules when a cancel message is received" in new SchedulingActorTest { - val (scheduleId, schedule) = generateSchedule - - createSchedule(scheduleId, schedule) - - actorRef ! CancelSchedule(scheduleId) - expectMsg(Ack) - - advanceTimeTo(schedule.time) - verifyZeroInteractions(mockSourceQueue) - } - - } - - private def generateSchedule = - (UUID.randomUUID().toString, random[Schedule].copy(time = OffsetDateTime.now().plusMinutes(5))) -} \ No newline at end of file diff --git a/scheduler/src/test/scala/com/sky/kafka/message/scheduler/SchedulerIntSpec.scala b/scheduler/src/test/scala/com/sky/kafka/message/scheduler/SchedulerIntSpec.scala index 3bb4af57..fdc8bec8 100644 --- a/scheduler/src/test/scala/com/sky/kafka/message/scheduler/SchedulerIntSpec.scala +++ b/scheduler/src/test/scala/com/sky/kafka/message/scheduler/SchedulerIntSpec.scala @@ -1,8 +1,8 @@ package com.sky.kafka.message.scheduler -import com.sky.kafka.message.scheduler.TestDataUtils._ +import common.TestDataUtils._ import com.sky.kafka.message.scheduler.domain.Schedule -import common.{AkkaStreamIntSpec, KafkaIntSpec} +import common.{AkkaStreamBaseSpec, KafkaIntSpec} import org.apache.kafka.common.serialization._ import org.scalatest.Assertion @@ -10,7 +10,7 @@ import scala.concurrent.Await import scala.concurrent.duration._ import avro._ -class SchedulerIntSpec extends AkkaStreamIntSpec with KafkaIntSpec { +class SchedulerIntSpec extends AkkaStreamBaseSpec with KafkaIntSpec { val ScheduleTopic = "scheduleTopic" @@ -25,8 +25,8 @@ class SchedulerIntSpec extends AkkaStreamIntSpec with KafkaIntSpec { val (consumedKey, consumedValue) = consumeFromKafka(schedule.topic, keyDeserializer = new ByteArrayDeserializer).head - consumedKey.get === schedule.key shouldBe true - consumedValue === schedule.value shouldBe true + consumedKey.get should contain theSameElementsInOrderAs schedule.key + consumedValue should contain theSameElementsInOrderAs schedule.value } "publish a delete to the schedule topic after emitting scheduled message" ignore withRunningSchedulerStream { @@ -38,7 +38,7 @@ class SchedulerIntSpec extends AkkaStreamIntSpec with KafkaIntSpec { records.size shouldBe 2 val (consumedKey, consumedValue) = records.last - consumedKey.get === "scheduleId" shouldBe true + consumedKey.get shouldBe "scheduleId" consumedValue shouldBe null } } diff --git a/scheduler/src/test/scala/com/sky/kafka/message/scheduler/SchedulerSpec.scala b/scheduler/src/test/scala/com/sky/kafka/message/scheduler/SchedulerSpec.scala index 5a0543d5..5cd06914 100644 --- a/scheduler/src/test/scala/com/sky/kafka/message/scheduler/SchedulerSpec.scala +++ b/scheduler/src/test/scala/com/sky/kafka/message/scheduler/SchedulerSpec.scala @@ -2,7 +2,7 @@ package com.sky.kafka.message.scheduler import java.time.OffsetDateTime -import com.sky.kafka.message.scheduler.TestDataUtils._ +import common.TestDataUtils._ import com.sky.kafka.message.scheduler.domain._ import common.BaseSpec import org.apache.kafka.clients.consumer.ConsumerRecord diff --git a/scheduler/src/test/scala/com/sky/kafka/message/scheduler/SchedulingActorSpec.scala b/scheduler/src/test/scala/com/sky/kafka/message/scheduler/SchedulingActorSpec.scala new file mode 100644 index 00000000..c2d35501 --- /dev/null +++ b/scheduler/src/test/scala/com/sky/kafka/message/scheduler/SchedulingActorSpec.scala @@ -0,0 +1,92 @@ +package com.sky.kafka.message.scheduler + +import java.util.UUID + +import akka.event.LoggingAdapter +import akka.stream.scaladsl.SourceQueue +import akka.testkit.{ImplicitSender, TestActorRef, TestKit} +import com.miguno.akka.testing.VirtualTime +import com.sky.kafka.message.scheduler.SchedulingActor.{Ack, Cancel, CreateOrUpdate} +import com.sky.kafka.message.scheduler.domain.{Schedule, ScheduleId} +import common.TestDataUtils._ +import common.{BaseSpec, TestActorSystem} +import org.mockito.Mockito._ +import org.scalatest.mockito.MockitoSugar + +import scala.concurrent.duration._ + +class SchedulingActorSpec extends TestKit(TestActorSystem()) with ImplicitSender with BaseSpec with MockitoSugar { + + "A scheduler actor" must { + "schedule new messages at the given time" in new SchedulingActorTest { + val (scheduleId, schedule) = generateSchedule() + + createSchedule(scheduleId, schedule) + + advanceToTimeFrom(schedule) + verify(mockSourceQueue).offer((scheduleId, schedule)) + } + + "cancel schedules when a cancel message is received" in new SchedulingActorTest { + val (scheduleId, schedule) = generateSchedule() + createSchedule(scheduleId, schedule) + + cancelSchedule(scheduleId) + verify(mockLogger).info(s"Cancelled schedule $scheduleId") + + advanceToTimeFrom(schedule) + verifyZeroInteractions(mockSourceQueue) + } + + "warn and do nothing when schedule cancelled twice" in new SchedulingActorTest { + val (scheduleId, schedule) = generateSchedule() + createSchedule(scheduleId, schedule) + cancelSchedule(scheduleId) + + cancelSchedule(scheduleId) + verify(mockLogger).warning(s"Couldn't cancel $scheduleId") + } + + "cancel previous schedule when updating an existing schedule" in new SchedulingActorTest { + val (scheduleId, schedule) = generateSchedule() + createSchedule(scheduleId, schedule) + + val updatedSchedule = schedule.copy(time = schedule.time.plusMinutes(5)) + createSchedule(scheduleId, updatedSchedule) + + advanceToTimeFrom(schedule) + verify(mockSourceQueue, never()).offer((scheduleId, schedule)) + + advanceToTimeFrom(updatedSchedule, schedule.timeInMillis) + verify(mockSourceQueue).offer((scheduleId, updatedSchedule)) + } + + } + + private class SchedulingActorTest { + val mockLogger = mock[LoggingAdapter] + val mockSourceQueue = mock[SourceQueue[(ScheduleId, Schedule)]] + val time = new VirtualTime + + val actorRef = TestActorRef(new SchedulingActor(mockSourceQueue, time.scheduler) { + override def log: LoggingAdapter = mockLogger + }) + + def generateSchedule(): (ScheduleId, Schedule) = + (UUID.randomUUID().toString, random[Schedule]) + + def advanceToTimeFrom(schedule: Schedule, startTime: Long = System.currentTimeMillis()): Unit = + time.advance(schedule.timeInMillis - startTime + 1 second) + + def createSchedule(scheduleId: ScheduleId, schedule: Schedule) = { + actorRef ! CreateOrUpdate(scheduleId, schedule) + expectMsg(Ack) + } + + def cancelSchedule(scheduleId: ScheduleId) = { + actorRef ! Cancel(scheduleId) + expectMsg(Ack) + } + } + +} \ No newline at end of file diff --git a/scheduler/src/test/scala/common/AkkaBaseSpec.scala b/scheduler/src/test/scala/common/AkkaBaseSpec.scala new file mode 100644 index 00000000..5d5f1fec --- /dev/null +++ b/scheduler/src/test/scala/common/AkkaBaseSpec.scala @@ -0,0 +1,13 @@ +package common + +import akka.testkit.TestKit +import org.scalatest.BeforeAndAfterAll + +abstract class AkkaBaseSpec extends TestKit(TestActorSystem()) + with BaseSpec with BeforeAndAfterAll { + + override def afterAll(): Unit = { + TestKit.shutdownActorSystem(system) + super.afterAll() + } +} diff --git a/scheduler/src/test/scala/common/AkkaStreamBaseSpec.scala b/scheduler/src/test/scala/common/AkkaStreamBaseSpec.scala new file mode 100644 index 00000000..8e38f912 --- /dev/null +++ b/scheduler/src/test/scala/common/AkkaStreamBaseSpec.scala @@ -0,0 +1,8 @@ +package common + +import akka.stream.ActorMaterializer + +abstract class AkkaStreamBaseSpec extends AkkaBaseSpec { + + implicit val materializer = ActorMaterializer() +} diff --git a/scheduler/src/test/scala/common/AkkaStreamIntSpec.scala b/scheduler/src/test/scala/common/AkkaStreamIntSpec.scala deleted file mode 100644 index fec6b0d3..00000000 --- a/scheduler/src/test/scala/common/AkkaStreamIntSpec.scala +++ /dev/null @@ -1,17 +0,0 @@ -package common - -import akka.stream.ActorMaterializer -import akka.testkit.TestKit -import org.scalatest.BeforeAndAfterEach - -abstract class AkkaStreamIntSpec extends TestKit(TestActorSystem()) - with BaseSpec with BeforeAndAfterEach { - - implicit val materializer = ActorMaterializer() - - override def afterEach(): Unit = { - TestKit.shutdownActorSystem(system) - super.afterEach() - } - -} diff --git a/scheduler/src/test/scala/com/sky/kafka/message/scheduler/TestDataUtils.scala b/scheduler/src/test/scala/common/TestDataUtils.scala similarity index 59% rename from scheduler/src/test/scala/com/sky/kafka/message/scheduler/TestDataUtils.scala rename to scheduler/src/test/scala/common/TestDataUtils.scala index a9c88bb9..c2288130 100644 --- a/scheduler/src/test/scala/com/sky/kafka/message/scheduler/TestDataUtils.scala +++ b/scheduler/src/test/scala/common/TestDataUtils.scala @@ -1,23 +1,24 @@ -package com.sky.kafka.message.scheduler +package common import java.io.ByteArrayOutputStream -import java.time.OffsetDateTime +import java.time.{Instant, OffsetDateTime, ZoneId} import com.danielasfregola.randomdatagenerator.RandomDataGenerator import com.sksamuel.avro4s.{AvroOutputStream, ToRecord} import com.sky.kafka.message.scheduler.domain.Schedule import org.scalacheck._ -import avro._ +import com.sky.kafka.message.scheduler.avro._ object TestDataUtils extends RandomDataGenerator { implicit val arbAlphaString: Arbitrary[String] = Arbitrary(Gen.alphaStr.suchThat(!_.isEmpty)) - //implicit val chooseOffsetDateTime: Choose[OffsetDateTime] = Choose.xmap[OffsetDateTime, OffsetDateTime](OffsetDateTime.MIN, OffsetDateTime.MAX) - - //TODO java.time.DateTimeException: Invalid value for Year (valid values -999999999 - 999999999): 1000000000 - implicit val arbOffsetDateTime: Arbitrary[OffsetDateTime] = - Arbitrary(Gen.oneOf(OffsetDateTime.MIN, OffsetDateTime.MAX)) + implicit val arbNextMonthOffsetDateTime: Arbitrary[OffsetDateTime] = { + val low = OffsetDateTime.now() + val high = low.plusMonths(1) + Arbitrary(Gen.choose(low.toEpochSecond, high.toEpochSecond) + .map(epoch => OffsetDateTime.ofInstant(Instant.ofEpochSecond(epoch), ZoneId.systemDefault()))) + } implicit class ScheduleOps(val schedule: Schedule) extends AnyVal { def toAvro(implicit toRecord: ToRecord[Schedule]): Array[Byte] = { @@ -27,6 +28,8 @@ object TestDataUtils extends RandomDataGenerator { output.close() baos.toByteArray } + + def timeInMillis: Long = schedule.time.toInstant.toEpochMilli } } From b1156075a8785a3b12ae8baf35868abf9385e1ea Mon Sep 17 00:00:00 2001 From: "Hubert Behaghel, Lawrence Carvalho, Matthew Pickering, Paolo Ambrosio and Roberto Tena" Date: Thu, 3 Aug 2017 15:13:00 +0100 Subject: [PATCH 03/22] e2e test. --- .../message/scheduler/SchedulerStream.scala | 6 +- .../message/scheduler/SchedulingActor.scala | 10 +++- .../scheduler/ScheduleReaderSpec.scala | 24 ++++++++ .../message/scheduler/SchedulerIntSpec.scala | 54 ------------------ .../scheduler/e2e/SchedulerIntSpec.scala | 55 +++++++++++++++++++ .../src/test/scala/common/EmbeddedKafka.scala | 47 ++++++++++++++++ .../src/test/scala/common/KafkaIntSpec.scala | 7 ++- .../src/test/scala/common/TestDataUtils.scala | 3 + 8 files changed, 147 insertions(+), 59 deletions(-) create mode 100644 scheduler/src/test/scala/com/sky/kafka/message/scheduler/ScheduleReaderSpec.scala delete mode 100644 scheduler/src/test/scala/com/sky/kafka/message/scheduler/SchedulerIntSpec.scala create mode 100644 scheduler/src/test/scala/com/sky/kafka/message/scheduler/e2e/SchedulerIntSpec.scala diff --git a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/SchedulerStream.scala b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/SchedulerStream.scala index d852de94..3fcd7d23 100644 --- a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/SchedulerStream.scala +++ b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/SchedulerStream.scala @@ -6,7 +6,7 @@ import akka.stream.ActorMaterializer import com.sky.kafka.message.scheduler.kafka._ import com.typesafe.scalalogging.LazyLogging -case class SchedulerStream(config: SchedulerConfig)(implicit system: ActorSystem, materializer: ActorMaterializer) extends LazyLogging { +case class SchedulerStream(config: SchedulerConfig)(implicit system: ActorSystem, materialzer: ActorMaterializer) extends LazyLogging { def run: Control = consumeFromKafka(config.scheduleTopic) @@ -19,3 +19,7 @@ case class SchedulerStream(config: SchedulerConfig)(implicit system: ActorSystem .run() } + +//object SchedulerStream { +// def reader(implicit system: ActorSystem): Reader[AppConfig, SchedulerStream] = Reader(conf => SchedulerStream(conf.scheduler)) +//} diff --git a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/SchedulingActor.scala b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/SchedulingActor.scala index f0711b15..30a646d7 100644 --- a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/SchedulingActor.scala +++ b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/SchedulingActor.scala @@ -4,8 +4,10 @@ import java.time.OffsetDateTime import java.time.temporal.ChronoUnit import java.util.concurrent.TimeUnit -import akka.actor.{Actor, ActorLogging, Cancellable, Scheduler} -import akka.stream.scaladsl.SourceQueue +import akka.actor.{Actor, ActorLogging, ActorSystem, Cancellable, Scheduler} +import akka.stream.OverflowStrategy +import akka.stream.scaladsl.{Source, SourceQueue} +import cats.data.Reader import com.sky.kafka.message.scheduler.SchedulingActor.{Ack, Cancel, CreateOrUpdate} import com.sky.kafka.message.scheduler.domain.{Schedule, ScheduleId} @@ -20,6 +22,10 @@ object SchedulingActor { case class Cancel(scheduleId: ScheduleId) +// def reader(implicit system: ActorSystem): Reader[AppConfig, SchedulingActor] = +// Reader(_ => new SchedulingActor(Source.queue, system.scheduler)) + // source queue should be using the reader pattern as well, using a materializer + } class SchedulingActor(sourceQueue: SourceQueue[(String, Schedule)], scheduler: Scheduler) extends Actor with ActorLogging { diff --git a/scheduler/src/test/scala/com/sky/kafka/message/scheduler/ScheduleReaderSpec.scala b/scheduler/src/test/scala/com/sky/kafka/message/scheduler/ScheduleReaderSpec.scala new file mode 100644 index 00000000..5df74485 --- /dev/null +++ b/scheduler/src/test/scala/com/sky/kafka/message/scheduler/ScheduleReaderSpec.scala @@ -0,0 +1,24 @@ +//package com.sky.kafka.message.scheduler +// +//import java.util.UUID +// +//import akka.testkit.TestProbe +//import com.sky.kafka.message.scheduler.domain.Schedule +//import common.{AkkaStreamBaseSpec, BaseSpec} +//import common.TestDataUtils._ +// +//class ScheduleReaderSpec extends AkkaStreamBaseSpec { +// +// "stream" should { +// "send schedules to the scheduling actor" in { +// val (scheduleId, schedule) = (UUID.randomUUID().toString, random[Schedule]) +// val probe = TestProbe() +// val scheduleReader = ScheduleReader(source, probe.ref, system) +// +// scheduleReader.stream.run() +// +// probe.expectMsg(SchedulingActor.CreateOrUpdate(scheduleId, schedule)) +// } +// } +// +//} diff --git a/scheduler/src/test/scala/com/sky/kafka/message/scheduler/SchedulerIntSpec.scala b/scheduler/src/test/scala/com/sky/kafka/message/scheduler/SchedulerIntSpec.scala deleted file mode 100644 index fdc8bec8..00000000 --- a/scheduler/src/test/scala/com/sky/kafka/message/scheduler/SchedulerIntSpec.scala +++ /dev/null @@ -1,54 +0,0 @@ -package com.sky.kafka.message.scheduler - -import common.TestDataUtils._ -import com.sky.kafka.message.scheduler.domain.Schedule -import common.{AkkaStreamBaseSpec, KafkaIntSpec} -import org.apache.kafka.common.serialization._ -import org.scalatest.Assertion - -import scala.concurrent.Await -import scala.concurrent.duration._ -import avro._ - -class SchedulerIntSpec extends AkkaStreamBaseSpec with KafkaIntSpec { - - val ScheduleTopic = "scheduleTopic" - - val conf = SchedulerConfig(ScheduleTopic, ShutdownTimeout(10 seconds, 10 seconds)) - - "Scheduler stream" should { - "schedule a message to be sent to Kafka" in withRunningSchedulerStream { - val schedule = random[Schedule] - - writeToKafka(ScheduleTopic, "scheduleId", schedule.toAvro) - - val (consumedKey, consumedValue) = - consumeFromKafka(schedule.topic, keyDeserializer = new ByteArrayDeserializer).head - - consumedKey.get should contain theSameElementsInOrderAs schedule.key - consumedValue should contain theSameElementsInOrderAs schedule.value - } - - "publish a delete to the schedule topic after emitting scheduled message" ignore withRunningSchedulerStream { - val schedule = random[Schedule] - - writeToKafka(ScheduleTopic, "scheduleId", schedule.toAvro) - - val records = consumeFromKafka(ScheduleTopic, 2, new StringDeserializer) - records.size shouldBe 2 - - val (consumedKey, consumedValue) = records.last - consumedKey.get shouldBe "scheduleId" - consumedValue shouldBe null - } - } - - private def withRunningSchedulerStream(scenario: => Assertion) { - val stream = SchedulerStream(conf).run - - scenario - - Await.result(stream.shutdown, 5 seconds) - } - -} diff --git a/scheduler/src/test/scala/com/sky/kafka/message/scheduler/e2e/SchedulerIntSpec.scala b/scheduler/src/test/scala/com/sky/kafka/message/scheduler/e2e/SchedulerIntSpec.scala new file mode 100644 index 00000000..7840ecbc --- /dev/null +++ b/scheduler/src/test/scala/com/sky/kafka/message/scheduler/e2e/SchedulerIntSpec.scala @@ -0,0 +1,55 @@ +package com.sky.kafka.message.scheduler.e2e + +import java.time.OffsetDateTime +import java.util.UUID + +import com.sky.kafka.message.scheduler.domain.Schedule +import com.sky.kafka.message.scheduler.{SchedulerConfig, SchedulerStream, ShutdownTimeout} +import com.sky.kafka.message.scheduler.avro._ +import common.TestDataUtils._ +import common.{AkkaStreamBaseSpec, KafkaIntSpec} +import org.apache.kafka.common.serialization._ +import org.scalactic.TripleEqualsSupport.Spread +import org.scalatest.Assertion + +import scala.concurrent.Await +import scala.concurrent.duration._ +import scala.math.Numeric.LongIsIntegral + +class SchedulerIntSpec extends AkkaStreamBaseSpec with KafkaIntSpec { + + val ScheduleTopic = "scheduleTopic" + + val conf = SchedulerConfig(ScheduleTopic, ShutdownTimeout(10 seconds, 10 seconds)) + + val tolerance = 200 millis + + "Scheduler stream" should { + "schedule a message to be sent to Kafka and delete it after it has been emitted" in withRunningSchedulerStream { + val (scheduleId, schedule) = (UUID.randomUUID().toString, random[Schedule].secondsFromNow(2)) + + writeToKafka(ScheduleTopic, scheduleId, schedule.toAvro) + + val cr = consumeFromKafka(schedule.topic, keyDeserializer = new ByteArrayDeserializer).head + + cr.key() should contain theSameElementsInOrderAs schedule.key + cr.value() should contain theSameElementsInOrderAs schedule.value + cr.timestamp() shouldBe schedule.timeInMillis +- tolerance.toMillis + + val latestMessageOnScheduleTopic = consumeLatestFromScheduleTopic + + latestMessageOnScheduleTopic.key() shouldBe scheduleId + latestMessageOnScheduleTopic.value() shouldBe null + } + } + + private def withRunningSchedulerStream(scenario: => Assertion) { + val stream = SchedulerStream(conf).run + + scenario + + Await.result(stream.shutdown, 5 seconds) + } + + private def consumeLatestFromScheduleTopic = consumeFromKafka(ScheduleTopic, 2, new StringDeserializer).last +} diff --git a/scheduler/src/test/scala/common/EmbeddedKafka.scala b/scheduler/src/test/scala/common/EmbeddedKafka.scala index 375ce11c..a89be042 100644 --- a/scheduler/src/test/scala/common/EmbeddedKafka.scala +++ b/scheduler/src/test/scala/common/EmbeddedKafka.scala @@ -1,6 +1,12 @@ package common import cakesolutions.kafka.testkit.KafkaServer +import cakesolutions.kafka.testkit.KafkaServer.defaultConsumerConfig +import org.apache.kafka.clients.consumer.{ConsumerConfig, ConsumerRecord, KafkaConsumer} +import org.apache.kafka.common.serialization.Deserializer + +import scala.collection.JavaConverters._ +import scala.collection.mutable.ArrayBuffer object EmbeddedKafka { @@ -8,4 +14,45 @@ object EmbeddedKafka { val bootstrapServer = s"localhost:${kafkaServer.kafkaPort}" + implicit class KafkaServerOps(val kafkaServer: KafkaServer) extends AnyVal { + + def consumeRecord[Key, Value, T]( + topic: String, + expectedNumOfRecords: Int, + timeout: Long, + keyDeserializer: Deserializer[Key], + valueDeserializer: Deserializer[Value], + consumerRecordConverter: ConsumerRecord[Key, Value] => T, + consumerConfig: Map[String, String] = defaultConsumerConfig + ): Seq[T] = { + + val extendedConfig: Map[String, Object] = consumerConfig + (ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG -> bootstrapServer) + val consumer = new KafkaConsumer(extendedConfig.asJava, keyDeserializer, valueDeserializer) + + try { + consumer.subscribe(List(topic).asJava) + + var total = 0 + val collected = ArrayBuffer.empty[T] + val start = System.currentTimeMillis() + + while (total <= expectedNumOfRecords && System.currentTimeMillis() < start + timeout) { + val records = consumer.poll(100) + val kvs = records.asScala.map(consumerRecordConverter) + collected ++= kvs + total += records.count() + } + + if (collected.size < expectedNumOfRecords) { + sys.error(s"Did not receive expected amount records. Expected $expectedNumOfRecords but got ${collected.size}.") + } + + collected.toVector + } finally { + consumer.close() + } + } + + } + } diff --git a/scheduler/src/test/scala/common/KafkaIntSpec.scala b/scheduler/src/test/scala/common/KafkaIntSpec.scala index d8d9986b..5bcad0fa 100644 --- a/scheduler/src/test/scala/common/KafkaIntSpec.scala +++ b/scheduler/src/test/scala/common/KafkaIntSpec.scala @@ -1,6 +1,7 @@ package common import EmbeddedKafka._ +import org.apache.kafka.clients.consumer.ConsumerRecord import org.apache.kafka.clients.producer.ProducerRecord import org.apache.kafka.common.serialization.{ByteArrayDeserializer, ByteArraySerializer, Deserializer, StringSerializer} import org.scalatest.{BeforeAndAfterAll, Suite} @@ -16,7 +17,9 @@ trait KafkaIntSpec extends BeforeAndAfterAll { this: Suite => kafkaServer.produce(topic, Iterable(producerRecord), new StringSerializer, new ByteArraySerializer) } - def consumeFromKafka[T](topic: String, numRecords: Int = 1, keyDeserializer: Deserializer[T]): Seq[(Option[T], Array[Byte])] = - kafkaServer.consume(topic, numRecords, 5000, keyDeserializer, new ByteArrayDeserializer) + def consumerRecordConverter[T]: ConsumerRecord[T, Array[Byte]] => ConsumerRecord[T, Array[Byte]] = identity + + def consumeFromKafka[T](topic: String, numRecords: Int = 1, keyDeserializer: Deserializer[T]): Seq[ConsumerRecord[T, Array[Byte]]] = + kafkaServer.consumeRecord(topic, numRecords, 5000, keyDeserializer, new ByteArrayDeserializer, consumerRecordConverter[T]) } diff --git a/scheduler/src/test/scala/common/TestDataUtils.scala b/scheduler/src/test/scala/common/TestDataUtils.scala index c2288130..a8182363 100644 --- a/scheduler/src/test/scala/common/TestDataUtils.scala +++ b/scheduler/src/test/scala/common/TestDataUtils.scala @@ -30,6 +30,9 @@ object TestDataUtils extends RandomDataGenerator { } def timeInMillis: Long = schedule.time.toInstant.toEpochMilli + + def secondsFromNow(seconds: Long) = + schedule.copy(time = OffsetDateTime.now().plusSeconds(seconds)) } } From b9a321bd26a51cee3073a993355ff74f83b31207 Mon Sep 17 00:00:00 2001 From: "Hubert Behaghel, Lawrence Carvalho, Matthew Pickering, Paolo Ambrosio and Roberto Tena" Date: Thu, 3 Aug 2017 17:30:06 +0100 Subject: [PATCH 04/22] Wired up application. TODO: implement publisher stream Flow and reader stream error handling. Metrics? --- .../message/scheduler/SchedulerApp.scala | 19 ++++++-- .../message/scheduler/SchedulerStream.scala | 25 ---------- .../message/scheduler/SchedulingActor.scala | 33 ++++++++----- .../sky/kafka/message/scheduler/package.scala | 14 ++++-- .../scheduler/streams/ScheduleReader.scala | 42 ++++++++++++++++ .../streams/ScheduledMessagePublisher.scala | 25 ++++++++++ .../scheduler/ScheduleReaderSpec.scala | 48 +++++++++---------- .../ScheduledMessagePublisherSpec.scala | 22 +++++++++ .../scheduler/SchedulingActorSpec.scala | 7 ++- .../scheduler/e2e/SchedulerIntSpec.scala | 21 +++++--- 10 files changed, 178 insertions(+), 78 deletions(-) delete mode 100644 scheduler/src/main/scala/com/sky/kafka/message/scheduler/SchedulerStream.scala create mode 100644 scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/ScheduleReader.scala create mode 100644 scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/ScheduledMessagePublisher.scala create mode 100644 scheduler/src/test/scala/com/sky/kafka/message/scheduler/ScheduledMessagePublisherSpec.scala diff --git a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/SchedulerApp.scala b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/SchedulerApp.scala index 79e2332e..137f4e59 100644 --- a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/SchedulerApp.scala +++ b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/SchedulerApp.scala @@ -1,23 +1,32 @@ package com.sky.kafka.message.scheduler +import com.sky.kafka.message.scheduler.streams.ScheduleReader import com.typesafe.scalalogging.LazyLogging import kamon.Kamon +import pureconfig._ import scala.concurrent.Await -import pureconfig._ object SchedulerApp extends App with AkkaComponents with LazyLogging { - val conf = loadConfigOrThrow[AppConfig].scheduler + val conf = loadConfigOrThrow[AppConfig] Kamon.start() logger.info("Kafka Message Scheduler starting up...") - val runningStream = SchedulerStream(conf).run + val app = ScheduleReader.reader(conf) + + val runningApp = app.stream.run() sys.addShutdownHook { + val shutdownTimeout = conf.scheduler.shutdownTimeout logger.info("Kafka Message Scheduler shutting down...") - Await.ready(runningStream.shutdown(), conf.shutdownTimeout.stream) - Await.ready(system.terminate(), conf.shutdownTimeout.system) + + Await.ready(runningApp.shutdown(), shutdownTimeout.stream) + Await.ready({ + app.queue.complete() + app.queue.watchCompletion() + }, shutdownTimeout.stream) + Await.ready(system.terminate(), shutdownTimeout.system) Kamon.shutdown() } diff --git a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/SchedulerStream.scala b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/SchedulerStream.scala deleted file mode 100644 index 3fcd7d23..00000000 --- a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/SchedulerStream.scala +++ /dev/null @@ -1,25 +0,0 @@ -package com.sky.kafka.message.scheduler - -import akka.actor.ActorSystem -import akka.kafka.scaladsl.Consumer.Control -import akka.stream.ActorMaterializer -import com.sky.kafka.message.scheduler.kafka._ -import com.typesafe.scalalogging.LazyLogging - -case class SchedulerStream(config: SchedulerConfig)(implicit system: ActorSystem, materialzer: ActorMaterializer) extends LazyLogging { - - def run: Control = - consumeFromKafka(config.scheduleTopic) - .map { - case Right((scheduleId, Some(schedule))) => - logger.info(s"Publishing scheduled message with ID: $scheduleId to topic: ${schedule.topic}") - schedule - } // match not exhaustive as pending error handling - .writeToKafka - .run() - -} - -//object SchedulerStream { -// def reader(implicit system: ActorSystem): Reader[AppConfig, SchedulerStream] = Reader(conf => SchedulerStream(conf.scheduler)) -//} diff --git a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/SchedulingActor.scala b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/SchedulingActor.scala index 30a646d7..202d67fa 100644 --- a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/SchedulingActor.scala +++ b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/SchedulingActor.scala @@ -4,11 +4,9 @@ import java.time.OffsetDateTime import java.time.temporal.ChronoUnit import java.util.concurrent.TimeUnit -import akka.actor.{Actor, ActorLogging, ActorSystem, Cancellable, Scheduler} -import akka.stream.OverflowStrategy -import akka.stream.scaladsl.{Source, SourceQueue} -import cats.data.Reader -import com.sky.kafka.message.scheduler.SchedulingActor.{Ack, Cancel, CreateOrUpdate} +import akka.actor._ +import akka.stream.scaladsl.SourceQueue +import com.sky.kafka.message.scheduler.SchedulingActor.{Ack, Cancel, CreateOrUpdate, Init} import com.sky.kafka.message.scheduler.domain.{Schedule, ScheduleId} import scala.concurrent.ExecutionContext.Implicits.global @@ -16,15 +14,24 @@ import scala.concurrent.duration.FiniteDuration object SchedulingActor { + sealed trait SchedulingMessage + + case object Init + case object Ack - case class CreateOrUpdate(scheduleId: ScheduleId, schedule: Schedule) + case class CreateOrUpdate(scheduleId: ScheduleId, schedule: Schedule) extends SchedulingMessage + + case class Cancel(scheduleId: ScheduleId) extends SchedulingMessage - case class Cancel(scheduleId: ScheduleId) + def props(queue: SourceQueue[(ScheduleId, Schedule)], scheduler: Scheduler): Props = + Props(new SchedulingActor(queue, scheduler)) -// def reader(implicit system: ActorSystem): Reader[AppConfig, SchedulingActor] = -// Reader(_ => new SchedulingActor(Source.queue, system.scheduler)) - // source queue should be using the reader pattern as well, using a materializer +// def reader(implicit system: ActorSystem): Reader[AppConfig, ActorRef] = { +// SchedulerStreamRunner.reader.map { runner => +// system.actorOf(SchedulingActor.props(runner.runningPublisherStream, system.scheduler)) +// } +// } } @@ -53,7 +60,11 @@ class SchedulingActor(sourceQueue: SourceQueue[(String, Schedule)], scheduler: S schedules - scheduleId } - receiveCreateOrUpdateMessage orElse receiveCancelMessage andThen updateStateAndAck + val receiveInitMessage: PartialFunction[Any, Unit] = { + case Init => sender ! Ack + } + + receiveCreateOrUpdateMessage orElse receiveCancelMessage andThen updateStateAndAck orElse receiveInitMessage } def updateStateAndAck(schedules: Map[ScheduleId, Cancellable]): Unit = { diff --git a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/package.scala b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/package.scala index 8a13480a..f6abb602 100644 --- a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/package.scala +++ b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/package.scala @@ -1,8 +1,11 @@ package com.sky.kafka.message +import cats.data.Reader import cats.syntax.either._ +import cats.syntax.show._ import com.sksamuel.avro4s.AvroInputStream -import com.sky.kafka.message.scheduler.domain._ +import com.sky.kafka.message.scheduler.domain.ApplicationError._ +import com.sky.kafka.message.scheduler.domain.{ApplicationError, _} import com.sky.kafka.message.scheduler.kafka.{ConsumerRecordDecoder, ProducerRecordEncoder} import com.typesafe.scalalogging.LazyLogging import org.apache.kafka.clients.consumer.ConsumerRecord @@ -11,15 +14,16 @@ import com.sky.kafka.message.scheduler.avro._ import scala.concurrent.duration.Duration import scala.util.Try -import com.sky.kafka.message.scheduler.domain.ApplicationError -import ApplicationError._ -import cats.syntax.show._ package object scheduler extends LazyLogging { case class AppConfig(scheduler: SchedulerConfig) - case class SchedulerConfig(scheduleTopic: String, shutdownTimeout: ShutdownTimeout) + case class SchedulerConfig(scheduleTopic: String, shutdownTimeout: ShutdownTimeout, queueBufferSize: Int) + + object SchedulerConfig { + def reader: Reader[AppConfig, SchedulerConfig] = Reader(_.scheduler) + } case class ShutdownTimeout(stream: Duration, system: Duration) diff --git a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/ScheduleReader.scala b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/ScheduleReader.scala new file mode 100644 index 00000000..eec47934 --- /dev/null +++ b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/ScheduleReader.scala @@ -0,0 +1,42 @@ +package com.sky.kafka.message.scheduler.streams + +import akka.Done +import akka.actor.ActorSystem +import akka.kafka.scaladsl.Consumer.Control +import akka.stream.ActorMaterializer +import akka.stream.scaladsl.{RunnableGraph, Sink} +import cats.data.Reader +import com.sky.kafka.message.scheduler.SchedulingActor.{Ack, CreateOrUpdate, SchedulingMessage} +import com.sky.kafka.message.scheduler.kafka._ +import com.sky.kafka.message.scheduler.{AkkaComponents, AppConfig, SchedulerConfig, SchedulerInput, SchedulingActor} +import com.typesafe.scalalogging.LazyLogging + +case class ScheduleReader(config: SchedulerConfig, publisherStream: ScheduledMessagePublisher) + (implicit system: ActorSystem, mat: ActorMaterializer) { + + val queue = publisherStream.stream.run() + + val schedulingActorRef = system.actorOf(SchedulingActor.props(queue, system.scheduler)) + + def stream: RunnableGraph[Control] = + consumeFromKafka(config.scheduleTopic) + .map(ScheduleReader.toSchedulingMessage) + .to(Sink.actorRefWithAck(schedulingActorRef, SchedulingActor.Init, Ack, Done)) + +} + +object ScheduleReader extends LazyLogging with AkkaComponents { + + val toSchedulingMessage: SchedulerInput => SchedulingMessage = { + case Right((scheduleId, Some(schedule))) => + logger.info(s"Publishing scheduled message with ID: $scheduleId to topic: ${schedule.topic}") + CreateOrUpdate(scheduleId, schedule) + // match not exhaustive as pending error handling + } + + def reader: Reader[AppConfig, ScheduleReader] = + for { + conf <- SchedulerConfig.reader + publisher <- ScheduledMessagePublisher.reader + } yield ScheduleReader(conf, publisher) +} diff --git a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/ScheduledMessagePublisher.scala b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/ScheduledMessagePublisher.scala new file mode 100644 index 00000000..b87ae0a7 --- /dev/null +++ b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/ScheduledMessagePublisher.scala @@ -0,0 +1,25 @@ +package com.sky.kafka.message.scheduler.streams + +import akka.actor.ActorSystem +import akka.stream.scaladsl.{RunnableGraph, Source, SourceQueueWithComplete} +import akka.stream.{ActorMaterializer, OverflowStrategy} +import cats.data.Reader +import com.sky.kafka.message.scheduler._ +import com.sky.kafka.message.scheduler.domain.{Schedule, ScheduleId} +import com.sky.kafka.message.scheduler.kafka.{ProducerRecordEncoder, _} +import com.sky.kafka.message.scheduler.{AppConfig, SchedulerConfig} + +case class ScheduledMessagePublisher(schedulerConfig: SchedulerConfig) + (implicit system: ActorSystem) { + + def stream[T: ProducerRecordEncoder]: RunnableGraph[SourceQueueWithComplete[(ScheduleId, Schedule)]] = + Source.queue[(ScheduleId, Schedule)](schedulerConfig.queueBufferSize, OverflowStrategy.backpressure) + .map { case (scheduleId, schedule) => schedule } + .writeToKafka +} + +object ScheduledMessagePublisher { + + def reader(implicit system: ActorSystem): Reader[AppConfig, ScheduledMessagePublisher] = + SchedulerConfig.reader.map(ScheduledMessagePublisher.apply) +} diff --git a/scheduler/src/test/scala/com/sky/kafka/message/scheduler/ScheduleReaderSpec.scala b/scheduler/src/test/scala/com/sky/kafka/message/scheduler/ScheduleReaderSpec.scala index 5df74485..b6a7ead6 100644 --- a/scheduler/src/test/scala/com/sky/kafka/message/scheduler/ScheduleReaderSpec.scala +++ b/scheduler/src/test/scala/com/sky/kafka/message/scheduler/ScheduleReaderSpec.scala @@ -1,24 +1,24 @@ -//package com.sky.kafka.message.scheduler -// -//import java.util.UUID -// -//import akka.testkit.TestProbe -//import com.sky.kafka.message.scheduler.domain.Schedule -//import common.{AkkaStreamBaseSpec, BaseSpec} -//import common.TestDataUtils._ -// -//class ScheduleReaderSpec extends AkkaStreamBaseSpec { -// -// "stream" should { -// "send schedules to the scheduling actor" in { -// val (scheduleId, schedule) = (UUID.randomUUID().toString, random[Schedule]) -// val probe = TestProbe() -// val scheduleReader = ScheduleReader(source, probe.ref, system) -// -// scheduleReader.stream.run() -// -// probe.expectMsg(SchedulingActor.CreateOrUpdate(scheduleId, schedule)) -// } -// } -// -//} +package com.sky.kafka.message.scheduler + +import java.util.UUID + +import akka.stream.testkit.scaladsl.TestSource +import akka.testkit.TestProbe +import com.sky.kafka.message.scheduler.SchedulingActor.{Ack, Init} +import com.sky.kafka.message.scheduler.domain.Schedule +import com.sky.kafka.message.scheduler.streams.ScheduleReader +import common.AkkaStreamBaseSpec +import common.TestDataUtils._ + +class ScheduleReaderSpec extends AkkaStreamBaseSpec { + + "toSchedulingMessage" should { + + "generate a CreateOrUpdate message if there is a schedule" in { + val (scheduleId, schedule) = (UUID.randomUUID().toString, random[Schedule]) + ScheduleReader.toSchedulingMessage(Right((scheduleId, Some(schedule)))) shouldBe + SchedulingActor.CreateOrUpdate(scheduleId, schedule) + } + } + +} diff --git a/scheduler/src/test/scala/com/sky/kafka/message/scheduler/ScheduledMessagePublisherSpec.scala b/scheduler/src/test/scala/com/sky/kafka/message/scheduler/ScheduledMessagePublisherSpec.scala new file mode 100644 index 00000000..4c341930 --- /dev/null +++ b/scheduler/src/test/scala/com/sky/kafka/message/scheduler/ScheduledMessagePublisherSpec.scala @@ -0,0 +1,22 @@ +package com.sky.kafka.message.scheduler + +import java.util.UUID + +import com.sky.kafka.message.scheduler.domain.Schedule +import common.AkkaStreamBaseSpec +import common.TestDataUtils.random +import org.apache.kafka.clients.producer.ProducerRecord +import common.TestDataUtils._ + +class ScheduledMessagePublisherSpec extends AkkaStreamBaseSpec { + + "flow" should { + "send a producer record for the scheduled the message and one with a null payload" in { + val (scheduleId, schedule) = (UUID.randomUUID().toString, random[Schedule]) + ??? + } + } + + def createProducerRecord[K](topic: String, key: K, v: Option[Array[Byte]]): ProducerRecord[K, Array[Byte]] = + new ProducerRecord[K, Array[Byte]](topic, key, v.orNull) +} diff --git a/scheduler/src/test/scala/com/sky/kafka/message/scheduler/SchedulingActorSpec.scala b/scheduler/src/test/scala/com/sky/kafka/message/scheduler/SchedulingActorSpec.scala index c2d35501..03c2e571 100644 --- a/scheduler/src/test/scala/com/sky/kafka/message/scheduler/SchedulingActorSpec.scala +++ b/scheduler/src/test/scala/com/sky/kafka/message/scheduler/SchedulingActorSpec.scala @@ -6,7 +6,7 @@ import akka.event.LoggingAdapter import akka.stream.scaladsl.SourceQueue import akka.testkit.{ImplicitSender, TestActorRef, TestKit} import com.miguno.akka.testing.VirtualTime -import com.sky.kafka.message.scheduler.SchedulingActor.{Ack, Cancel, CreateOrUpdate} +import com.sky.kafka.message.scheduler.SchedulingActor._ import com.sky.kafka.message.scheduler.domain.{Schedule, ScheduleId} import common.TestDataUtils._ import common.{BaseSpec, TestActorSystem} @@ -61,6 +61,11 @@ class SchedulingActorSpec extends TestKit(TestActorSystem()) with ImplicitSender verify(mockSourceQueue).offer((scheduleId, updatedSchedule)) } + "send an Ack to the sender when receiving an Init message" in new SchedulingActorTest { + actorRef ! Init + expectMsg(Ack) + } + } private class SchedulingActorTest { diff --git a/scheduler/src/test/scala/com/sky/kafka/message/scheduler/e2e/SchedulerIntSpec.scala b/scheduler/src/test/scala/com/sky/kafka/message/scheduler/e2e/SchedulerIntSpec.scala index 7840ecbc..9039ae62 100644 --- a/scheduler/src/test/scala/com/sky/kafka/message/scheduler/e2e/SchedulerIntSpec.scala +++ b/scheduler/src/test/scala/com/sky/kafka/message/scheduler/e2e/SchedulerIntSpec.scala @@ -1,26 +1,26 @@ package com.sky.kafka.message.scheduler.e2e -import java.time.OffsetDateTime import java.util.UUID import com.sky.kafka.message.scheduler.domain.Schedule -import com.sky.kafka.message.scheduler.{SchedulerConfig, SchedulerStream, ShutdownTimeout} +import com.sky.kafka.message.scheduler._ import com.sky.kafka.message.scheduler.avro._ +import com.sky.kafka.message.scheduler.streams.ScheduleReader import common.TestDataUtils._ import common.{AkkaStreamBaseSpec, KafkaIntSpec} import org.apache.kafka.common.serialization._ -import org.scalactic.TripleEqualsSupport.Spread import org.scalatest.Assertion import scala.concurrent.Await import scala.concurrent.duration._ -import scala.math.Numeric.LongIsIntegral class SchedulerIntSpec extends AkkaStreamBaseSpec with KafkaIntSpec { val ScheduleTopic = "scheduleTopic" - val conf = SchedulerConfig(ScheduleTopic, ShutdownTimeout(10 seconds, 10 seconds)) + val shutdownTimeout = ShutdownTimeout(10 seconds, 10 seconds) + + val conf = AppConfig(SchedulerConfig(ScheduleTopic, shutdownTimeout, 100)) val tolerance = 200 millis @@ -44,11 +44,18 @@ class SchedulerIntSpec extends AkkaStreamBaseSpec with KafkaIntSpec { } private def withRunningSchedulerStream(scenario: => Assertion) { - val stream = SchedulerStream(conf).run + val app = ScheduleReader.reader(conf) + + val runningStreams = app.stream.run() scenario - Await.result(stream.shutdown, 5 seconds) + Await.ready(runningStreams.shutdown(), shutdownTimeout.stream) + Await.ready({ + app.queue.complete() + app.queue.watchCompletion() + }, shutdownTimeout.stream) + Await.ready(system.terminate(), shutdownTimeout.system) } private def consumeLatestFromScheduleTopic = consumeFromKafka(ScheduleTopic, 2, new StringDeserializer).last From 26c540e9859814b29b4cc2722066d0c1efa213e2 Mon Sep 17 00:00:00 2001 From: "Hubert Behaghel, Lawrence Carvalho, Matthew Pickering, Paolo Ambrosio and Roberto Tena" Date: Thu, 3 Aug 2017 17:45:02 +0100 Subject: [PATCH 05/22] Rename scheduler type alias. --- .../scala/com/sky/kafka/message/scheduler/package.scala | 8 ++++---- .../kafka/message/scheduler/streams/ScheduleReader.scala | 6 +++--- .../scheduler/streams/ScheduledMessagePublisher.scala | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/package.scala b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/package.scala index f6abb602..925869e6 100644 --- a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/package.scala +++ b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/package.scala @@ -27,10 +27,10 @@ package object scheduler extends LazyLogging { case class ShutdownTimeout(stream: Duration, system: Duration) - type SchedulerInput = Either[ApplicationError, (ScheduleId, Option[Schedule])] + type DecodeScheduleResult = Either[ApplicationError, (ScheduleId, Option[Schedule])] - implicit val scheduleConsumerRecordDecoder = new ConsumerRecordDecoder[SchedulerInput] { - def apply(cr: ConsumerRecord[String, Array[Byte]]): SchedulerInput = + implicit val scheduleConsumerRecordDecoder = new ConsumerRecordDecoder[DecodeScheduleResult] { + def apply(cr: ConsumerRecord[String, Array[Byte]]): DecodeScheduleResult = consumerRecordDecoder(cr).leftMap { error => logger.warn(error.show) error @@ -41,7 +41,7 @@ package object scheduler extends LazyLogging { def apply(schedule: Schedule) = new ProducerRecord(schedule.topic, schedule.key, schedule.value) } - def consumerRecordDecoder(cr: ConsumerRecord[String, Array[Byte]]): SchedulerInput = + def consumerRecordDecoder(cr: ConsumerRecord[String, Array[Byte]]): DecodeScheduleResult = Option(cr.value) match { case Some(bytes) => for { diff --git a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/ScheduleReader.scala b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/ScheduleReader.scala index eec47934..cbe5fbf1 100644 --- a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/ScheduleReader.scala +++ b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/ScheduleReader.scala @@ -6,9 +6,9 @@ import akka.kafka.scaladsl.Consumer.Control import akka.stream.ActorMaterializer import akka.stream.scaladsl.{RunnableGraph, Sink} import cats.data.Reader -import com.sky.kafka.message.scheduler.SchedulingActor.{Ack, CreateOrUpdate, SchedulingMessage} +import com.sky.kafka.message.scheduler.SchedulingActor._ import com.sky.kafka.message.scheduler.kafka._ -import com.sky.kafka.message.scheduler.{AkkaComponents, AppConfig, SchedulerConfig, SchedulerInput, SchedulingActor} +import com.sky.kafka.message.scheduler._ import com.typesafe.scalalogging.LazyLogging case class ScheduleReader(config: SchedulerConfig, publisherStream: ScheduledMessagePublisher) @@ -27,7 +27,7 @@ case class ScheduleReader(config: SchedulerConfig, publisherStream: ScheduledMes object ScheduleReader extends LazyLogging with AkkaComponents { - val toSchedulingMessage: SchedulerInput => SchedulingMessage = { + val toSchedulingMessage: DecodeScheduleResult => SchedulingMessage = { case Right((scheduleId, Some(schedule))) => logger.info(s"Publishing scheduled message with ID: $scheduleId to topic: ${schedule.topic}") CreateOrUpdate(scheduleId, schedule) diff --git a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/ScheduledMessagePublisher.scala b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/ScheduledMessagePublisher.scala index b87ae0a7..fd0b14a0 100644 --- a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/ScheduledMessagePublisher.scala +++ b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/ScheduledMessagePublisher.scala @@ -14,7 +14,7 @@ case class ScheduledMessagePublisher(schedulerConfig: SchedulerConfig) def stream[T: ProducerRecordEncoder]: RunnableGraph[SourceQueueWithComplete[(ScheduleId, Schedule)]] = Source.queue[(ScheduleId, Schedule)](schedulerConfig.queueBufferSize, OverflowStrategy.backpressure) - .map { case (scheduleId, schedule) => schedule } + .map { case (scheduleId, schedule) => schedule } //TODO: implement this flow .writeToKafka } From fe9469d287f4291cd350d862dd02a100c6775169 Mon Sep 17 00:00:00 2001 From: "Hubert Behaghel, Lawrence Carvalho, Matthew Pickering, Paolo Ambrosio and Roberto Tena" Date: Fri, 4 Aug 2017 17:41:56 +0100 Subject: [PATCH 06/22] Scheduling messages and deleting them from the input topic after they have been emitted. Tests passing. TODO: error handling in ScheduleReader stream. --- build.sbt | 1 + .../message/scheduler/SchedulerApp.scala | 20 +++------ .../message/scheduler/SchedulingActor.scala | 44 ++++++++----------- .../message/scheduler/domain/package.scala | 2 + .../kafka/ProducerRecordEncoder.scala | 14 ++++++ .../sky/kafka/message/scheduler/package.scala | 12 +++-- .../scheduler/streams/ScheduleReader.scala | 24 +++++----- .../streams/ScheduledMessagePublisher.scala | 27 +++++++----- .../ScheduledMessagePublisherStream.scala | 22 ++++++++++ .../streams/SchedulerReaderStream.scala | 20 +++++++++ .../scheduler/ScheduleReaderSpec.scala | 6 +++ .../ScheduledMessagePublisherSpec.scala | 18 +++++--- .../scheduler/e2e/SchedulerIntSpec.scala | 13 ++---- .../scala/common/AkkaStreamBaseSpec.scala | 5 +++ .../src/test/scala/common/KafkaIntSpec.scala | 10 ++++- 15 files changed, 156 insertions(+), 82 deletions(-) create mode 100644 scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/ScheduledMessagePublisherStream.scala create mode 100644 scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/SchedulerReaderStream.scala diff --git a/build.sbt b/build.sbt index cceca70b..b61b6082 100644 --- a/build.sbt +++ b/build.sbt @@ -15,6 +15,7 @@ val dependencies = Seq( "org.typelevel" %% "cats" % "0.9.0", "ch.qos.logback" % "logback-classic" % "1.2.3" % Runtime, "com.github.pureconfig" %% "pureconfig" % "0.7.2", + "org.zalando" %% "grafter" % "2.0.1", "io.kamon" %% "kamon-jmx" % kamonVersion, "io.kamon" %% "kamon-akka-2.5" % kamonVersion, diff --git a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/SchedulerApp.scala b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/SchedulerApp.scala index 137f4e59..35eadc19 100644 --- a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/SchedulerApp.scala +++ b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/SchedulerApp.scala @@ -3,31 +3,25 @@ package com.sky.kafka.message.scheduler import com.sky.kafka.message.scheduler.streams.ScheduleReader import com.typesafe.scalalogging.LazyLogging import kamon.Kamon +import org.zalando.grafter._ import pureconfig._ import scala.concurrent.Await -object SchedulerApp extends App with AkkaComponents with LazyLogging { +object SchedulerApp extends App with LazyLogging with AkkaComponents { val conf = loadConfigOrThrow[AppConfig] Kamon.start() logger.info("Kafka Message Scheduler starting up...") - val app = ScheduleReader.reader(conf) - - val runningApp = app.stream.run() + val app = ScheduleReader.reader.run(conf) sys.addShutdownHook { - val shutdownTimeout = conf.scheduler.shutdownTimeout logger.info("Kafka Message Scheduler shutting down...") - - Await.ready(runningApp.shutdown(), shutdownTimeout.stream) - Await.ready({ - app.queue.complete() - app.queue.watchCompletion() - }, shutdownTimeout.stream) - Await.ready(system.terminate(), shutdownTimeout.system) + Rewriter.stop(app).value + //TODO shut down in akka components + materializer.shutdown() + Await.result(system.terminate(), conf.scheduler.shutdownTimeout.system) Kamon.shutdown() } - } \ No newline at end of file diff --git a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/SchedulingActor.scala b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/SchedulingActor.scala index 202d67fa..4a1c847f 100644 --- a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/SchedulingActor.scala +++ b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/SchedulingActor.scala @@ -12,29 +12,6 @@ import com.sky.kafka.message.scheduler.domain.{Schedule, ScheduleId} import scala.concurrent.ExecutionContext.Implicits.global import scala.concurrent.duration.FiniteDuration -object SchedulingActor { - - sealed trait SchedulingMessage - - case object Init - - case object Ack - - case class CreateOrUpdate(scheduleId: ScheduleId, schedule: Schedule) extends SchedulingMessage - - case class Cancel(scheduleId: ScheduleId) extends SchedulingMessage - - def props(queue: SourceQueue[(ScheduleId, Schedule)], scheduler: Scheduler): Props = - Props(new SchedulingActor(queue, scheduler)) - -// def reader(implicit system: ActorSystem): Reader[AppConfig, ActorRef] = { -// SchedulerStreamRunner.reader.map { runner => -// system.actorOf(SchedulingActor.props(runner.runningPublisherStream, system.scheduler)) -// } -// } - -} - class SchedulingActor(sourceQueue: SourceQueue[(String, Schedule)], scheduler: Scheduler) extends Actor with ActorLogging { override def receive: Receive = receiveScheduleMessages(Map.empty) @@ -60,11 +37,9 @@ class SchedulingActor(sourceQueue: SourceQueue[(String, Schedule)], scheduler: S schedules - scheduleId } - val receiveInitMessage: PartialFunction[Any, Unit] = { + receiveCreateOrUpdateMessage orElse receiveCancelMessage andThen updateStateAndAck orElse { case Init => sender ! Ack } - - receiveCreateOrUpdateMessage orElse receiveCancelMessage andThen updateStateAndAck orElse receiveInitMessage } def updateStateAndAck(schedules: Map[ScheduleId, Cancellable]): Unit = { @@ -80,3 +55,20 @@ class SchedulingActor(sourceQueue: SourceQueue[(String, Schedule)], scheduler: S FiniteDuration(offset, TimeUnit.MILLISECONDS) } } + +object SchedulingActor { + + sealed trait SchedulingMessage + + case object Init + + case object Ack + + case class CreateOrUpdate(scheduleId: ScheduleId, schedule: Schedule) extends SchedulingMessage + + case class Cancel(scheduleId: ScheduleId) extends SchedulingMessage + + def props(queue: SourceQueue[(ScheduleId, Schedule)], scheduler: Scheduler): Props = + Props(new SchedulingActor(queue, scheduler)) + +} diff --git a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/domain/package.scala b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/domain/package.scala index 73a3b8ff..4f063483 100644 --- a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/domain/package.scala +++ b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/domain/package.scala @@ -8,4 +8,6 @@ package object domain { case class Schedule(time: OffsetDateTime, topic: String, key: Array[Byte], value: Array[Byte]) + case class ScheduleMetadata(scheduleId: ScheduleId, topic: String) + } diff --git a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/kafka/ProducerRecordEncoder.scala b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/kafka/ProducerRecordEncoder.scala index 30a693ca..edc04f1a 100644 --- a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/kafka/ProducerRecordEncoder.scala +++ b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/kafka/ProducerRecordEncoder.scala @@ -5,3 +5,17 @@ import org.apache.kafka.clients.producer.ProducerRecord trait ProducerRecordEncoder[T] { def apply(t: T): ProducerRecord[Array[Byte], Array[Byte]] } + +object ProducerRecordEncoder { + + implicit def eitherProducerRecordEncoder[A, B](implicit leftEncoder: ProducerRecordEncoder[A], rightEncoder: ProducerRecordEncoder[B]): ProducerRecordEncoder[Either[A, B]] = + instance { + case Right(r) => rightEncoder(r) + case Left(l) => leftEncoder(l) + } + + def instance[A, B](f: A => ProducerRecord[Array[Byte], Array[Byte]]) = + new ProducerRecordEncoder[A]() { + final def apply(a: A): ProducerRecord[Array[Byte], Array[Byte]] = f(a) + } +} diff --git a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/package.scala b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/package.scala index 925869e6..f47d2a32 100644 --- a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/package.scala +++ b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/package.scala @@ -1,5 +1,7 @@ package com.sky.kafka.message +import akka.actor.ActorSystem +import akka.stream.ActorMaterializer import cats.data.Reader import cats.syntax.either._ import cats.syntax.show._ @@ -17,7 +19,7 @@ import scala.util.Try package object scheduler extends LazyLogging { - case class AppConfig(scheduler: SchedulerConfig) + case class AppConfig(scheduler: SchedulerConfig)(implicit system: ActorSystem, materialzer: ActorMaterializer) case class SchedulerConfig(scheduleTopic: String, shutdownTimeout: ShutdownTimeout, queueBufferSize: Int) @@ -37,9 +39,11 @@ package object scheduler extends LazyLogging { } } - implicit val scheduleProducerRecordEncoder = new ProducerRecordEncoder[Schedule] { - def apply(schedule: Schedule) = new ProducerRecord(schedule.topic, schedule.key, schedule.value) - } + implicit val scheduleProducerRecordEncoder: ProducerRecordEncoder[Schedule] = + ProducerRecordEncoder.instance(schedule => new ProducerRecord(schedule.topic, schedule.key, schedule.value)) + + implicit val scheduleMedatadataProducerRecordEncoder: ProducerRecordEncoder[ScheduleMetadata] = + ProducerRecordEncoder.instance(schedule => new ProducerRecord(schedule.topic, schedule.scheduleId.getBytes, null)) def consumerRecordDecoder(cr: ConsumerRecord[String, Array[Byte]]): DecodeScheduleResult = Option(cr.value) match { diff --git a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/ScheduleReader.scala b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/ScheduleReader.scala index cbe5fbf1..04b572cd 100644 --- a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/ScheduleReader.scala +++ b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/ScheduleReader.scala @@ -4,39 +4,41 @@ import akka.Done import akka.actor.ActorSystem import akka.kafka.scaladsl.Consumer.Control import akka.stream.ActorMaterializer -import akka.stream.scaladsl.{RunnableGraph, Sink} +import akka.stream.scaladsl.{Sink, SourceQueueWithComplete} import cats.data.Reader import com.sky.kafka.message.scheduler.SchedulingActor._ -import com.sky.kafka.message.scheduler.kafka._ import com.sky.kafka.message.scheduler._ +import com.sky.kafka.message.scheduler.domain.ScheduleId +import com.sky.kafka.message.scheduler.kafka._ import com.typesafe.scalalogging.LazyLogging -case class ScheduleReader(config: SchedulerConfig, publisherStream: ScheduledMessagePublisher) - (implicit system: ActorSystem, mat: ActorMaterializer) { - - val queue = publisherStream.stream.run() +case class ScheduleReader(config: SchedulerConfig, queue: SourceQueueWithComplete[(ScheduleId, domain.Schedule)]) + (implicit system: ActorSystem, materializer: ActorMaterializer) extends SchedulerReaderStream { val schedulingActorRef = system.actorOf(SchedulingActor.props(queue, system.scheduler)) - def stream: RunnableGraph[Control] = + val stream: Control = consumeFromKafka(config.scheduleTopic) .map(ScheduleReader.toSchedulingMessage) .to(Sink.actorRefWithAck(schedulingActorRef, SchedulingActor.Init, Ack, Done)) - + .run() } -object ScheduleReader extends LazyLogging with AkkaComponents { +object ScheduleReader extends LazyLogging { val toSchedulingMessage: DecodeScheduleResult => SchedulingMessage = { case Right((scheduleId, Some(schedule))) => logger.info(s"Publishing scheduled message with ID: $scheduleId to topic: ${schedule.topic}") CreateOrUpdate(scheduleId, schedule) + case Right((scheduleId, None)) => + logger.info(s"Cancelling schedule $scheduleId") + Cancel(scheduleId) // match not exhaustive as pending error handling } - def reader: Reader[AppConfig, ScheduleReader] = + def reader(implicit system: ActorSystem, materializer: ActorMaterializer): Reader[AppConfig, ScheduleReader] = for { conf <- SchedulerConfig.reader publisher <- ScheduledMessagePublisher.reader - } yield ScheduleReader(conf, publisher) + } yield ScheduleReader(conf, publisher.stream) } diff --git a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/ScheduledMessagePublisher.scala b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/ScheduledMessagePublisher.scala index fd0b14a0..92d3cb25 100644 --- a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/ScheduledMessagePublisher.scala +++ b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/ScheduledMessagePublisher.scala @@ -1,25 +1,30 @@ package com.sky.kafka.message.scheduler.streams import akka.actor.ActorSystem -import akka.stream.scaladsl.{RunnableGraph, Source, SourceQueueWithComplete} +import akka.stream.scaladsl.{Source, SourceQueueWithComplete} import akka.stream.{ActorMaterializer, OverflowStrategy} import cats.data.Reader -import com.sky.kafka.message.scheduler._ -import com.sky.kafka.message.scheduler.domain.{Schedule, ScheduleId} -import com.sky.kafka.message.scheduler.kafka.{ProducerRecordEncoder, _} -import com.sky.kafka.message.scheduler.{AppConfig, SchedulerConfig} +import com.sky.kafka.message.scheduler.domain.{Schedule, ScheduleId, ScheduleMetadata} +import com.sky.kafka.message.scheduler.kafka._ +import com.sky.kafka.message.scheduler.{AppConfig, SchedulerConfig, _} -case class ScheduledMessagePublisher(schedulerConfig: SchedulerConfig) - (implicit system: ActorSystem) { +case class ScheduledMessagePublisher(config: SchedulerConfig) + (implicit system: ActorSystem, materializer: ActorMaterializer) extends ScheduledMessagePublisherStream { - def stream[T: ProducerRecordEncoder]: RunnableGraph[SourceQueueWithComplete[(ScheduleId, Schedule)]] = - Source.queue[(ScheduleId, Schedule)](schedulerConfig.queueBufferSize, OverflowStrategy.backpressure) - .map { case (scheduleId, schedule) => schedule } //TODO: implement this flow + def stream: SourceQueueWithComplete[(ScheduleId, Schedule)] = + Source.queue[(ScheduleId, Schedule)](config.queueBufferSize, OverflowStrategy.backpressure) + .mapConcat(splitToScheduleAndMetadata) .writeToKafka + .run() + + val splitToScheduleAndMetadata: ((ScheduleId, Schedule)) => List[Either[ScheduleMetadata, Schedule]] = { + case (scheduleId, schedule) => + List(Right(schedule), Left(ScheduleMetadata(scheduleId, config.scheduleTopic))) + } } object ScheduledMessagePublisher { - def reader(implicit system: ActorSystem): Reader[AppConfig, ScheduledMessagePublisher] = + def reader(implicit system: ActorSystem, materializer: ActorMaterializer): Reader[AppConfig, ScheduledMessagePublisher] = SchedulerConfig.reader.map(ScheduledMessagePublisher.apply) } diff --git a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/ScheduledMessagePublisherStream.scala b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/ScheduledMessagePublisherStream.scala new file mode 100644 index 00000000..f6b2c268 --- /dev/null +++ b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/ScheduledMessagePublisherStream.scala @@ -0,0 +1,22 @@ +package com.sky.kafka.message.scheduler.streams + +import akka.stream.scaladsl.SourceQueueWithComplete +import cats.Eval +import com.sky.kafka.message.scheduler.SchedulerConfig +import com.sky.kafka.message.scheduler.domain.{Schedule, ScheduleId} +import org.zalando.grafter.{Stop, StopResult} + +import scala.concurrent.Await + +trait ScheduledMessagePublisherStream extends Stop { + + def config: SchedulerConfig + + def stream: SourceQueueWithComplete[(ScheduleId, Schedule)] + + override def stop: Eval[StopResult] = StopResult.eval("Shutting down queue...") { + stream.complete() + Await.result(stream.watchCompletion(), config.shutdownTimeout.stream) + } + +} diff --git a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/SchedulerReaderStream.scala b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/SchedulerReaderStream.scala new file mode 100644 index 00000000..bb9b003c --- /dev/null +++ b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/SchedulerReaderStream.scala @@ -0,0 +1,20 @@ +package com.sky.kafka.message.scheduler.streams + +import akka.kafka.scaladsl.Consumer.Control +import cats.Eval +import com.sky.kafka.message.scheduler.SchedulerConfig +import org.zalando.grafter.{Stop, StopResult} + +import scala.concurrent.Await + +trait SchedulerReaderStream extends Stop { + + def config: SchedulerConfig + + def stream: Control + + override def stop: Eval[StopResult] = StopResult.eval("Shutting down reader stream...")( + Await.result(stream.shutdown(), config.shutdownTimeout.stream) + ) + +} diff --git a/scheduler/src/test/scala/com/sky/kafka/message/scheduler/ScheduleReaderSpec.scala b/scheduler/src/test/scala/com/sky/kafka/message/scheduler/ScheduleReaderSpec.scala index b6a7ead6..1504f3d8 100644 --- a/scheduler/src/test/scala/com/sky/kafka/message/scheduler/ScheduleReaderSpec.scala +++ b/scheduler/src/test/scala/com/sky/kafka/message/scheduler/ScheduleReaderSpec.scala @@ -19,6 +19,12 @@ class ScheduleReaderSpec extends AkkaStreamBaseSpec { ScheduleReader.toSchedulingMessage(Right((scheduleId, Some(schedule)))) shouldBe SchedulingActor.CreateOrUpdate(scheduleId, schedule) } + + "generate a Cancel message if there is no schedule" in { + val scheduleId = UUID.randomUUID().toString + ScheduleReader.toSchedulingMessage(Right((scheduleId, None))) shouldBe + SchedulingActor.Cancel(scheduleId) + } } } diff --git a/scheduler/src/test/scala/com/sky/kafka/message/scheduler/ScheduledMessagePublisherSpec.scala b/scheduler/src/test/scala/com/sky/kafka/message/scheduler/ScheduledMessagePublisherSpec.scala index 4c341930..fdb7b990 100644 --- a/scheduler/src/test/scala/com/sky/kafka/message/scheduler/ScheduledMessagePublisherSpec.scala +++ b/scheduler/src/test/scala/com/sky/kafka/message/scheduler/ScheduledMessagePublisherSpec.scala @@ -2,21 +2,27 @@ package com.sky.kafka.message.scheduler import java.util.UUID -import com.sky.kafka.message.scheduler.domain.Schedule +import com.sky.kafka.message.scheduler.domain.{Schedule, ScheduleMetadata} +import com.sky.kafka.message.scheduler.streams.ScheduledMessagePublisher import common.AkkaStreamBaseSpec import common.TestDataUtils.random import org.apache.kafka.clients.producer.ProducerRecord import common.TestDataUtils._ +import scala.concurrent.duration._ + class ScheduledMessagePublisherSpec extends AkkaStreamBaseSpec { - "flow" should { - "send a producer record for the scheduled the message and one with a null payload" in { + "splitToScheduleAndMetadata" should { + "split a (scheduleId, schedule) to a list containing a schedule and a schedule metadata" in { + val testTopic = UUID.randomUUID().toString + val publisher = ScheduledMessagePublisher(SchedulerConfig(testTopic, ShutdownTimeout(1 second, 1 second), 5)) val (scheduleId, schedule) = (UUID.randomUUID().toString, random[Schedule]) - ??? + publisher.splitToScheduleAndMetadata((scheduleId, schedule)) shouldBe List( + Right(schedule), + Left(ScheduleMetadata(scheduleId, testTopic)) + ) } } - def createProducerRecord[K](topic: String, key: K, v: Option[Array[Byte]]): ProducerRecord[K, Array[Byte]] = - new ProducerRecord[K, Array[Byte]](topic, key, v.orNull) } diff --git a/scheduler/src/test/scala/com/sky/kafka/message/scheduler/e2e/SchedulerIntSpec.scala b/scheduler/src/test/scala/com/sky/kafka/message/scheduler/e2e/SchedulerIntSpec.scala index 9039ae62..fc9ba7a0 100644 --- a/scheduler/src/test/scala/com/sky/kafka/message/scheduler/e2e/SchedulerIntSpec.scala +++ b/scheduler/src/test/scala/com/sky/kafka/message/scheduler/e2e/SchedulerIntSpec.scala @@ -10,6 +10,7 @@ import common.TestDataUtils._ import common.{AkkaStreamBaseSpec, KafkaIntSpec} import org.apache.kafka.common.serialization._ import org.scalatest.Assertion +import org.zalando.grafter.Rewriter import scala.concurrent.Await import scala.concurrent.duration._ @@ -44,17 +45,11 @@ class SchedulerIntSpec extends AkkaStreamBaseSpec with KafkaIntSpec { } private def withRunningSchedulerStream(scenario: => Assertion) { - val app = ScheduleReader.reader(conf) - - val runningStreams = app.stream.run() + val app = ScheduleReader.reader.run(conf) scenario - - Await.ready(runningStreams.shutdown(), shutdownTimeout.stream) - Await.ready({ - app.queue.complete() - app.queue.watchCompletion() - }, shutdownTimeout.stream) + Rewriter.stop(app).value + materializer.shutdown() Await.ready(system.terminate(), shutdownTimeout.system) } diff --git a/scheduler/src/test/scala/common/AkkaStreamBaseSpec.scala b/scheduler/src/test/scala/common/AkkaStreamBaseSpec.scala index 8e38f912..6f81180e 100644 --- a/scheduler/src/test/scala/common/AkkaStreamBaseSpec.scala +++ b/scheduler/src/test/scala/common/AkkaStreamBaseSpec.scala @@ -5,4 +5,9 @@ import akka.stream.ActorMaterializer abstract class AkkaStreamBaseSpec extends AkkaBaseSpec { implicit val materializer = ActorMaterializer() + + override def afterAll(): Unit = { + materializer.shutdown() + super.afterAll() + } } diff --git a/scheduler/src/test/scala/common/KafkaIntSpec.scala b/scheduler/src/test/scala/common/KafkaIntSpec.scala index 5bcad0fa..4d7e65d3 100644 --- a/scheduler/src/test/scala/common/KafkaIntSpec.scala +++ b/scheduler/src/test/scala/common/KafkaIntSpec.scala @@ -8,9 +8,15 @@ import org.scalatest.{BeforeAndAfterAll, Suite} trait KafkaIntSpec extends BeforeAndAfterAll { this: Suite => - override def beforeAll() = kafkaServer.startup() + override def beforeAll() = { + kafkaServer.startup() + super.beforeAll() + } - override def afterAll() = kafkaServer.close() + override def afterAll() = { + kafkaServer.close() + super.afterAll() + } def writeToKafka(topic: String, key: String, value: Array[Byte]) { val producerRecord = new ProducerRecord[String, Array[Byte]](topic, key, value) From 5f6bbcd5fe4901ad6ed470382e87e1a1e476c282 Mon Sep 17 00:00:00 2001 From: "Hubert Behaghel, Lawrence Carvalho, Matthew Pickering, Paolo Ambrosio and Roberto Tena" Date: Sun, 6 Aug 2017 17:00:33 +0100 Subject: [PATCH 07/22] Introduced error handling in ScheduleReader. Just need to provide an implicit Sink[ErrorType, _]. --- .../message/scheduler/SchedulerApp.scala | 4 +- .../message/scheduler/SchedulingActor.scala | 20 +++++-- .../message/scheduler/config/package.scala | 21 +++++++ .../scheduler/domain/ApplicationError.scala | 12 +++- .../scheduler/domain/ScheduleData.scala | 27 +++++++++ .../message/scheduler/domain/package.scala | 6 -- .../message/scheduler/kafka/KafkaStream.scala | 26 ++++++++ .../kafka/ProducerRecordEncoder.scala | 6 -- .../message/scheduler/kafka/package.scala | 30 ---------- .../sky/kafka/message/scheduler/package.scala | 38 +++--------- .../scheduler/streams/EitherFanOut.scala | 60 +++++++++++++++++++ .../scheduler/streams/PartitionedSink.scala | 20 +++++++ .../scheduler/streams/ScheduleReader.scala | 49 ++++++++------- ...tream.scala => ScheduleReaderStream.scala} | 4 +- .../streams/ScheduledMessagePublisher.scala | 33 ++++++---- .../ScheduledMessagePublisherStream.scala | 5 +- .../scheduler/PartitionedSinkSpec.scala | 30 ++++++++++ .../scheduler/ScheduleReaderSpec.scala | 9 +-- .../ScheduledMessagePublisherSpec.scala | 17 ++++-- .../message/scheduler/SchedulerSpec.scala | 1 + .../scheduler/SchedulingActorSpec.scala | 3 +- .../scheduler/e2e/SchedulerIntSpec.scala | 4 +- .../src/test/scala/common/TestDataUtils.scala | 2 +- 23 files changed, 294 insertions(+), 133 deletions(-) create mode 100644 scheduler/src/main/scala/com/sky/kafka/message/scheduler/config/package.scala create mode 100644 scheduler/src/main/scala/com/sky/kafka/message/scheduler/domain/ScheduleData.scala create mode 100644 scheduler/src/main/scala/com/sky/kafka/message/scheduler/kafka/KafkaStream.scala delete mode 100644 scheduler/src/main/scala/com/sky/kafka/message/scheduler/kafka/package.scala create mode 100644 scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/EitherFanOut.scala create mode 100644 scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/PartitionedSink.scala rename scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/{SchedulerReaderStream.scala => ScheduleReaderStream.scala} (79%) create mode 100644 scheduler/src/test/scala/com/sky/kafka/message/scheduler/PartitionedSinkSpec.scala diff --git a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/SchedulerApp.scala b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/SchedulerApp.scala index 35eadc19..7ac8fd3b 100644 --- a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/SchedulerApp.scala +++ b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/SchedulerApp.scala @@ -1,5 +1,6 @@ package com.sky.kafka.message.scheduler +import com.sky.kafka.message.scheduler.config.AppConfig import com.sky.kafka.message.scheduler.streams.ScheduleReader import com.typesafe.scalalogging.LazyLogging import kamon.Kamon @@ -19,9 +20,10 @@ object SchedulerApp extends App with LazyLogging with AkkaComponents { sys.addShutdownHook { logger.info("Kafka Message Scheduler shutting down...") Rewriter.stop(app).value - //TODO shut down in akka components + materializer.shutdown() Await.result(system.terminate(), conf.scheduler.shutdownTimeout.system) + Kamon.shutdown() } } \ No newline at end of file diff --git a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/SchedulingActor.scala b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/SchedulingActor.scala index 4a1c847f..2d70866b 100644 --- a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/SchedulingActor.scala +++ b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/SchedulingActor.scala @@ -5,9 +5,14 @@ import java.time.temporal.ChronoUnit import java.util.concurrent.TimeUnit import akka.actor._ +import akka.stream.ActorMaterializer import akka.stream.scaladsl.SourceQueue +import cats.data.Reader import com.sky.kafka.message.scheduler.SchedulingActor.{Ack, Cancel, CreateOrUpdate, Init} -import com.sky.kafka.message.scheduler.domain.{Schedule, ScheduleId} +import com.sky.kafka.message.scheduler.config.AppConfig +import com.sky.kafka.message.scheduler.domain.ScheduleData.Schedule +import com.sky.kafka.message.scheduler.domain.ScheduleId +import com.sky.kafka.message.scheduler.streams.ScheduledMessagePublisher import scala.concurrent.ExecutionContext.Implicits.global import scala.concurrent.duration.FiniteDuration @@ -60,15 +65,20 @@ object SchedulingActor { sealed trait SchedulingMessage + case class CreateOrUpdate(scheduleId: ScheduleId, schedule: Schedule) extends SchedulingMessage + + case class Cancel(scheduleId: ScheduleId) extends SchedulingMessage + case object Init case object Ack - case class CreateOrUpdate(scheduleId: ScheduleId, schedule: Schedule) extends SchedulingMessage - - case class Cancel(scheduleId: ScheduleId) extends SchedulingMessage + def reader(implicit system: ActorSystem, mat: ActorMaterializer): Reader[AppConfig, ActorRef] = + ScheduledMessagePublisher.reader.map(publisher => + system.actorOf(props(publisher.stream, system.scheduler), "scheduling-actor") + ) - def props(queue: SourceQueue[(ScheduleId, Schedule)], scheduler: Scheduler): Props = + private def props(queue: SourceQueue[(ScheduleId, Schedule)], scheduler: Scheduler): Props = Props(new SchedulingActor(queue, scheduler)) } diff --git a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/config/package.scala b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/config/package.scala new file mode 100644 index 00000000..48cbfcf9 --- /dev/null +++ b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/config/package.scala @@ -0,0 +1,21 @@ +package com.sky.kafka.message.scheduler + +import akka.actor.ActorSystem +import akka.stream.ActorMaterializer +import cats.data.Reader + +import scala.concurrent.duration.Duration + +package object config { + + case class AppConfig(scheduler: SchedulerConfig)(implicit system: ActorSystem, materialzer: ActorMaterializer) + + case class SchedulerConfig(scheduleTopic: String, shutdownTimeout: ShutdownTimeout, queueBufferSize: Int) + + object SchedulerConfig { + def reader: Reader[AppConfig, SchedulerConfig] = Reader(_.scheduler) + } + + case class ShutdownTimeout(stream: Duration, system: Duration) + +} diff --git a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/domain/ApplicationError.scala b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/domain/ApplicationError.scala index 3fee96e9..65d1a848 100644 --- a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/domain/ApplicationError.scala +++ b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/domain/ApplicationError.scala @@ -1,10 +1,16 @@ package com.sky.kafka.message.scheduler.domain +import akka.Done +import akka.stream.scaladsl.Sink import cats.Show import cats.Show._ +import cats.syntax.show._ +import com.typesafe.scalalogging.LazyLogging + +import scala.concurrent.Future sealed abstract class ApplicationError(key: String) -object ApplicationError { +object ApplicationError extends LazyLogging { case class InvalidSchemaError(key: String) extends ApplicationError(key) @@ -22,4 +28,8 @@ object ApplicationError { case schemaError: InvalidSchemaError => invalidSchemaErrorShow.show(schemaError) case messageFormatError: AvroMessageFormatError => avroMessageFormatErrorShow.show(messageFormatError) } + + //TODO: is this enough for now? + implicit val errorSink: Sink[ApplicationError, Future[Done]] = + Sink.foreach(error => logger.warn(error.show)) } diff --git a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/domain/ScheduleData.scala b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/domain/ScheduleData.scala new file mode 100644 index 00000000..5c3bdc01 --- /dev/null +++ b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/domain/ScheduleData.scala @@ -0,0 +1,27 @@ +package com.sky.kafka.message.scheduler.domain + +import java.time.OffsetDateTime + +import com.sky.kafka.message.scheduler.kafka.ProducerRecordEncoder +import org.apache.kafka.clients.producer.ProducerRecord + +sealed trait ScheduleData + +object ScheduleData { + + case class Schedule(time: OffsetDateTime, topic: String, key: Array[Byte], value: Array[Byte]) extends ScheduleData + + case class ScheduleMetadata(scheduleId: ScheduleId, topic: String) extends ScheduleData + + implicit val scheduleProducerRecordEncoder: ProducerRecordEncoder[Schedule] = + ProducerRecordEncoder.instance(schedule => new ProducerRecord(schedule.topic, schedule.key, schedule.value)) + + implicit val scheduleMedatadataProducerRecordEncoder: ProducerRecordEncoder[ScheduleMetadata] = + ProducerRecordEncoder.instance(metadata => new ProducerRecord(metadata.topic, metadata.scheduleId.getBytes, null)) + + + implicit def scheduleDataToProducerRecord(scheduleData: ScheduleData): ProducerRecord[Array[Byte], Array[Byte]] = scheduleData match { + case schedule: Schedule => scheduleProducerRecordEncoder(schedule) + case metadata: ScheduleMetadata => scheduleMedatadataProducerRecordEncoder(metadata) + } +} diff --git a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/domain/package.scala b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/domain/package.scala index 4f063483..1e47060b 100644 --- a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/domain/package.scala +++ b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/domain/package.scala @@ -1,13 +1,7 @@ package com.sky.kafka.message.scheduler -import java.time.OffsetDateTime - package object domain { type ScheduleId = String - case class Schedule(time: OffsetDateTime, topic: String, key: Array[Byte], value: Array[Byte]) - - case class ScheduleMetadata(scheduleId: ScheduleId, topic: String) - } diff --git a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/kafka/KafkaStream.scala b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/kafka/KafkaStream.scala new file mode 100644 index 00000000..cc81b363 --- /dev/null +++ b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/kafka/KafkaStream.scala @@ -0,0 +1,26 @@ +package com.sky.kafka.message.scheduler.kafka + +import akka.Done +import akka.actor.ActorSystem +import akka.kafka.scaladsl.Consumer.Control +import akka.kafka.scaladsl.{Consumer, Producer} +import akka.kafka.{ConsumerSettings, ProducerSettings, Subscriptions} +import akka.stream.scaladsl.{Sink, Source} +import com.sky.kafka.message.scheduler.config.SchedulerConfig +import org.apache.kafka.clients.producer.ProducerRecord +import org.apache.kafka.common.serialization.{ByteArrayDeserializer, ByteArraySerializer, StringDeserializer} + +import scala.concurrent.Future + +object KafkaStream { + + def source[T](config: SchedulerConfig)(implicit system: ActorSystem, crDecoder: ConsumerRecordDecoder[T]): Source[T, Control] = + Consumer.plainSource( + ConsumerSettings(system, new StringDeserializer, new ByteArrayDeserializer), + Subscriptions.topics(config.scheduleTopic) + ).map(crDecoder(_)) + + def sink(implicit system: ActorSystem): Sink[ProducerRecord[Array[Byte], Array[Byte]], Future[Done]] = + Producer.plainSink(ProducerSettings(system, new ByteArraySerializer, new ByteArraySerializer)) + +} diff --git a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/kafka/ProducerRecordEncoder.scala b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/kafka/ProducerRecordEncoder.scala index edc04f1a..021ffb70 100644 --- a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/kafka/ProducerRecordEncoder.scala +++ b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/kafka/ProducerRecordEncoder.scala @@ -8,12 +8,6 @@ trait ProducerRecordEncoder[T] { object ProducerRecordEncoder { - implicit def eitherProducerRecordEncoder[A, B](implicit leftEncoder: ProducerRecordEncoder[A], rightEncoder: ProducerRecordEncoder[B]): ProducerRecordEncoder[Either[A, B]] = - instance { - case Right(r) => rightEncoder(r) - case Left(l) => leftEncoder(l) - } - def instance[A, B](f: A => ProducerRecord[Array[Byte], Array[Byte]]) = new ProducerRecordEncoder[A]() { final def apply(a: A): ProducerRecord[Array[Byte], Array[Byte]] = f(a) diff --git a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/kafka/package.scala b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/kafka/package.scala deleted file mode 100644 index 4d96c2ea..00000000 --- a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/kafka/package.scala +++ /dev/null @@ -1,30 +0,0 @@ -package com.sky.kafka.message.scheduler - -import akka.Done -import akka.actor.ActorSystem -import akka.kafka.scaladsl.Consumer.Control -import akka.kafka.scaladsl.{Consumer, Producer} -import akka.kafka.{ConsumerSettings, ProducerSettings, Subscriptions} -import akka.stream.scaladsl.{RunnableGraph, Sink, Source} -import org.apache.kafka.clients.producer.ProducerRecord -import org.apache.kafka.common.serialization._ - -import scala.concurrent.Future - -package object kafka { - - def consumeFromKafka[T](topic: String)(implicit system: ActorSystem, crDecoder: ConsumerRecordDecoder[T]): Source[T, Control] = - Consumer.plainSource( - ConsumerSettings(system, new StringDeserializer, new ByteArrayDeserializer), - Subscriptions.topics(topic) - ).map(crDecoder(_)) - - implicit class SourceOps[Out, Mat](source: Source[Out, Mat]) { - def writeToKafka(implicit system: ActorSystem, recordEncoder: ProducerRecordEncoder[Out]): RunnableGraph[Mat] = - source.map(recordEncoder(_)).to(kafkaSink) - } - - private def kafkaSink(implicit system: ActorSystem): Sink[ProducerRecord[Array[Byte], Array[Byte]], Future[Done]] = - Producer.plainSink(ProducerSettings(system, new ByteArraySerializer, new ByteArraySerializer)) - -} diff --git a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/package.scala b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/package.scala index f47d2a32..030447cd 100644 --- a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/package.scala +++ b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/package.scala @@ -1,51 +1,27 @@ package com.sky.kafka.message -import akka.actor.ActorSystem -import akka.stream.ActorMaterializer -import cats.data.Reader import cats.syntax.either._ -import cats.syntax.show._ import com.sksamuel.avro4s.AvroInputStream import com.sky.kafka.message.scheduler.domain.ApplicationError._ import com.sky.kafka.message.scheduler.domain.{ApplicationError, _} -import com.sky.kafka.message.scheduler.kafka.{ConsumerRecordDecoder, ProducerRecordEncoder} +import com.sky.kafka.message.scheduler.kafka.ConsumerRecordDecoder import com.typesafe.scalalogging.LazyLogging import org.apache.kafka.clients.consumer.ConsumerRecord -import org.apache.kafka.clients.producer.ProducerRecord import com.sky.kafka.message.scheduler.avro._ +import com.sky.kafka.message.scheduler.domain.ScheduleData.Schedule -import scala.concurrent.duration.Duration import scala.util.Try package object scheduler extends LazyLogging { - case class AppConfig(scheduler: SchedulerConfig)(implicit system: ActorSystem, materialzer: ActorMaterializer) + type DecodeResult = Either[ApplicationError, (ScheduleId, Option[Schedule])] - case class SchedulerConfig(scheduleTopic: String, shutdownTimeout: ShutdownTimeout, queueBufferSize: Int) - - object SchedulerConfig { - def reader: Reader[AppConfig, SchedulerConfig] = Reader(_.scheduler) + implicit val scheduleConsumerRecordDecoder = new ConsumerRecordDecoder[DecodeResult] { + def apply(cr: ConsumerRecord[String, Array[Byte]]): DecodeResult = + consumerRecordDecoder(cr) } - case class ShutdownTimeout(stream: Duration, system: Duration) - - type DecodeScheduleResult = Either[ApplicationError, (ScheduleId, Option[Schedule])] - - implicit val scheduleConsumerRecordDecoder = new ConsumerRecordDecoder[DecodeScheduleResult] { - def apply(cr: ConsumerRecord[String, Array[Byte]]): DecodeScheduleResult = - consumerRecordDecoder(cr).leftMap { error => - logger.warn(error.show) - error - } - } - - implicit val scheduleProducerRecordEncoder: ProducerRecordEncoder[Schedule] = - ProducerRecordEncoder.instance(schedule => new ProducerRecord(schedule.topic, schedule.key, schedule.value)) - - implicit val scheduleMedatadataProducerRecordEncoder: ProducerRecordEncoder[ScheduleMetadata] = - ProducerRecordEncoder.instance(schedule => new ProducerRecord(schedule.topic, schedule.scheduleId.getBytes, null)) - - def consumerRecordDecoder(cr: ConsumerRecord[String, Array[Byte]]): DecodeScheduleResult = + def consumerRecordDecoder(cr: ConsumerRecord[String, Array[Byte]]): DecodeResult = Option(cr.value) match { case Some(bytes) => for { diff --git a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/EitherFanOut.scala b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/EitherFanOut.scala new file mode 100644 index 00000000..127ba480 --- /dev/null +++ b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/EitherFanOut.scala @@ -0,0 +1,60 @@ +package com.sky.kafka.message.scheduler.streams + +import akka.stream.FanOutShape2 +import akka.stream.stage.{GraphStage, InHandler, OutHandler} + +//Taken from: https://stackoverflow.com/a/38445121/8424807 +class EitherFanOut[L, R] extends GraphStage[FanOutShape2[Either[L, R], L, R]] { + import akka.stream.Attributes + import akka.stream.stage.GraphStageLogic + + override val shape: FanOutShape2[Either[L, R], L, R] = new FanOutShape2[Either[L, R], L, R]("EitherFanOut") + + override def createLogic(inheritedAttributes: Attributes): GraphStageLogic = new GraphStageLogic(shape) { + + var out0demand = false + var out1demand = false + + setHandler(shape.in, new InHandler { + override def onPush(): Unit = { + + if (out0demand && out1demand) { + grab(shape.in) match { + case Left(l) => + out0demand = false + push(shape.out0, l) + case Right(r) => + out1demand = false + push(shape.out1, r) + } + } + } + }) + + setHandler(shape.out0, new OutHandler { + @scala.throws[Exception](classOf[Exception]) + override def onPull(): Unit = { + if (!out0demand) { + out0demand = true + } + + if (out0demand && out1demand) { + pull(shape.in) + } + } + }) + + setHandler(shape.out1, new OutHandler { + @scala.throws[Exception](classOf[Exception]) + override def onPull(): Unit = { + if (!out1demand) { + out1demand = true + } + + if (out0demand && out1demand) { + pull(shape.in) + } + } + }) + } +} \ No newline at end of file diff --git a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/PartitionedSink.scala b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/PartitionedSink.scala new file mode 100644 index 00000000..ddcca1e4 --- /dev/null +++ b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/PartitionedSink.scala @@ -0,0 +1,20 @@ +package com.sky.kafka.message.scheduler.streams + +import akka.stream.SinkShape +import akka.stream.scaladsl.{GraphDSL, Sink} + +object PartitionedSink { + + def from[A, AMat, B, BMat](rightSink: Sink[B, BMat])(implicit leftSink: Sink[A, AMat]): Sink[Either[A, B], (AMat, BMat)] = + Sink.fromGraph(GraphDSL.create(leftSink, rightSink)((_, _)) { implicit b => + (left, right) => + import GraphDSL.Implicits._ + + val eitherFanOut = b.add(new EitherFanOut[A, B]) + + eitherFanOut.out0 ~> left.in + eitherFanOut.out1 ~> right.in + + SinkShape(eitherFanOut.in) + }) +} diff --git a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/ScheduleReader.scala b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/ScheduleReader.scala index 04b572cd..4c5d43c4 100644 --- a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/ScheduleReader.scala +++ b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/ScheduleReader.scala @@ -1,44 +1,49 @@ package com.sky.kafka.message.scheduler.streams -import akka.Done import akka.actor.ActorSystem import akka.kafka.scaladsl.Consumer.Control -import akka.stream.ActorMaterializer -import akka.stream.scaladsl.{Sink, SourceQueueWithComplete} +import akka.stream._ +import akka.stream.scaladsl.{Sink, Source} +import akka.{Done, NotUsed} import cats.data.Reader import com.sky.kafka.message.scheduler.SchedulingActor._ import com.sky.kafka.message.scheduler._ -import com.sky.kafka.message.scheduler.domain.ScheduleId +import com.sky.kafka.message.scheduler.config.{AppConfig, SchedulerConfig} +import com.sky.kafka.message.scheduler.domain.ApplicationError import com.sky.kafka.message.scheduler.kafka._ import com.typesafe.scalalogging.LazyLogging -case class ScheduleReader(config: SchedulerConfig, queue: SourceQueueWithComplete[(ScheduleId, domain.Schedule)]) - (implicit system: ActorSystem, materializer: ActorMaterializer) extends SchedulerReaderStream { - - val schedulingActorRef = system.actorOf(SchedulingActor.props(queue, system.scheduler)) +case class ScheduleReader(config: SchedulerConfig, scheduleSource: Source[DecodeResult, Control], schedulingSink: Sink[Any, NotUsed]) + (implicit system: ActorSystem, materializer: ActorMaterializer) extends ScheduleReaderStream { val stream: Control = - consumeFromKafka(config.scheduleTopic) + scheduleSource .map(ScheduleReader.toSchedulingMessage) - .to(Sink.actorRefWithAck(schedulingActorRef, SchedulingActor.Init, Ack, Done)) + .to(PartitionedSink.from(schedulingSink)) .run() } object ScheduleReader extends LazyLogging { - val toSchedulingMessage: DecodeScheduleResult => SchedulingMessage = { - case Right((scheduleId, Some(schedule))) => - logger.info(s"Publishing scheduled message with ID: $scheduleId to topic: ${schedule.topic}") - CreateOrUpdate(scheduleId, schedule) - case Right((scheduleId, None)) => - logger.info(s"Cancelling schedule $scheduleId") - Cancel(scheduleId) - // match not exhaustive as pending error handling - } + def toSchedulingMessage(decodeResult: DecodeResult): Either[ApplicationError, SchedulingMessage] = + decodeResult.map { case (scheduleId, scheduleOpt) => + scheduleOpt match { + case Some(schedule) => + logger.info(s"Publishing scheduled message with ID: $scheduleId to topic: ${schedule.topic}") + CreateOrUpdate(scheduleId, schedule) + case None => + logger.info(s"Cancelling schedule $scheduleId") + Cancel(scheduleId) + } + } def reader(implicit system: ActorSystem, materializer: ActorMaterializer): Reader[AppConfig, ScheduleReader] = for { - conf <- SchedulerConfig.reader - publisher <- ScheduledMessagePublisher.reader - } yield ScheduleReader(conf, publisher.stream) + config <- SchedulerConfig.reader + schedulingActorRef <- SchedulingActor.reader + } yield ScheduleReader( + config = config, + scheduleSource = KafkaStream.source(config), + schedulingSink = Sink.actorRefWithAck(schedulingActorRef, SchedulingActor.Init, Ack, Done) + ) } diff --git a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/SchedulerReaderStream.scala b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/ScheduleReaderStream.scala similarity index 79% rename from scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/SchedulerReaderStream.scala rename to scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/ScheduleReaderStream.scala index bb9b003c..b233436a 100644 --- a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/SchedulerReaderStream.scala +++ b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/ScheduleReaderStream.scala @@ -2,12 +2,12 @@ package com.sky.kafka.message.scheduler.streams import akka.kafka.scaladsl.Consumer.Control import cats.Eval -import com.sky.kafka.message.scheduler.SchedulerConfig +import com.sky.kafka.message.scheduler.config.SchedulerConfig import org.zalando.grafter.{Stop, StopResult} import scala.concurrent.Await -trait SchedulerReaderStream extends Stop { +trait ScheduleReaderStream extends Stop { def config: SchedulerConfig diff --git a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/ScheduledMessagePublisher.scala b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/ScheduledMessagePublisher.scala index 92d3cb25..7a5d5d30 100644 --- a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/ScheduledMessagePublisher.scala +++ b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/ScheduledMessagePublisher.scala @@ -1,30 +1,41 @@ package com.sky.kafka.message.scheduler.streams +import akka.Done import akka.actor.ActorSystem -import akka.stream.scaladsl.{Source, SourceQueueWithComplete} +import akka.stream.scaladsl.{Sink, Source, SourceQueueWithComplete} import akka.stream.{ActorMaterializer, OverflowStrategy} import cats.data.Reader -import com.sky.kafka.message.scheduler.domain.{Schedule, ScheduleId, ScheduleMetadata} -import com.sky.kafka.message.scheduler.kafka._ -import com.sky.kafka.message.scheduler.{AppConfig, SchedulerConfig, _} +import com.sky.kafka.message.scheduler.config._ +import com.sky.kafka.message.scheduler.domain.ScheduleData._ +import org.apache.kafka.clients.producer.ProducerRecord +import com.sky.kafka.message.scheduler.domain._ +import com.sky.kafka.message.scheduler.kafka.KafkaStream +import com.sky.kafka.message.scheduler.streams.ScheduledMessagePublisher._ -case class ScheduledMessagePublisher(config: SchedulerConfig) - (implicit system: ActorSystem, materializer: ActorMaterializer) extends ScheduledMessagePublisherStream { +import scala.concurrent.Future + +case class ScheduledMessagePublisher(config: SchedulerConfig, publisherSink: Sink[In, Mat]) + (implicit system: ActorSystem, materializer: ActorMaterializer) + extends ScheduledMessagePublisherStream { def stream: SourceQueueWithComplete[(ScheduleId, Schedule)] = Source.queue[(ScheduleId, Schedule)](config.queueBufferSize, OverflowStrategy.backpressure) .mapConcat(splitToScheduleAndMetadata) - .writeToKafka + .to(publisherSink) .run() - val splitToScheduleAndMetadata: ((ScheduleId, Schedule)) => List[Either[ScheduleMetadata, Schedule]] = { + val splitToScheduleAndMetadata: ((ScheduleId, Schedule)) => List[In] = { case (scheduleId, schedule) => - List(Right(schedule), Left(ScheduleMetadata(scheduleId, config.scheduleTopic))) + List(schedule, ScheduleMetadata(scheduleId, config.scheduleTopic)) } } object ScheduledMessagePublisher { - def reader(implicit system: ActorSystem, materializer: ActorMaterializer): Reader[AppConfig, ScheduledMessagePublisher] = - SchedulerConfig.reader.map(ScheduledMessagePublisher.apply) + type In = ProducerRecord[Array[Byte], Array[Byte]] + type Mat = Future[Done] + + def reader(implicit system: ActorSystem, + materializer: ActorMaterializer): Reader[AppConfig, ScheduledMessagePublisher] = + SchedulerConfig.reader.map(conf => ScheduledMessagePublisher(conf, KafkaStream.sink)) } diff --git a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/ScheduledMessagePublisherStream.scala b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/ScheduledMessagePublisherStream.scala index f6b2c268..dcd3e065 100644 --- a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/ScheduledMessagePublisherStream.scala +++ b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/ScheduledMessagePublisherStream.scala @@ -2,8 +2,9 @@ package com.sky.kafka.message.scheduler.streams import akka.stream.scaladsl.SourceQueueWithComplete import cats.Eval -import com.sky.kafka.message.scheduler.SchedulerConfig -import com.sky.kafka.message.scheduler.domain.{Schedule, ScheduleId} +import com.sky.kafka.message.scheduler.config.SchedulerConfig +import com.sky.kafka.message.scheduler.domain.ScheduleData.Schedule +import com.sky.kafka.message.scheduler.domain.ScheduleId import org.zalando.grafter.{Stop, StopResult} import scala.concurrent.Await diff --git a/scheduler/src/test/scala/com/sky/kafka/message/scheduler/PartitionedSinkSpec.scala b/scheduler/src/test/scala/com/sky/kafka/message/scheduler/PartitionedSinkSpec.scala new file mode 100644 index 00000000..7c7cff24 --- /dev/null +++ b/scheduler/src/test/scala/com/sky/kafka/message/scheduler/PartitionedSinkSpec.scala @@ -0,0 +1,30 @@ +package com.sky.kafka.message.scheduler + +import akka.stream.scaladsl._ +import com.sky.kafka.message.scheduler.streams.PartitionedSink +import common.AkkaStreamBaseSpec + +import scala.concurrent.duration._ +import scala.concurrent.{Await, Future} + +class PartitionedSinkSpec extends AkkaStreamBaseSpec { + + implicit val leftSink: Sink[Int, Future[Seq[Int]]] = + Flow[Int].toMat(Sink.seq)(Keep.right) + + val rightSink: Sink[String, Future[Int]] = + Flow[String].map(_.length).toMat(Sink.fold(0)(_ + _))(Keep.right) + + val source = Source(List(Right("test"), Left(5), Right("someString"))) + + "withRight" should { + "emit Right to rightSink and Left to leftSink" in { + + val (leftFuture, rightFuture) = source.runWith(PartitionedSink.from(rightSink)) + + Await.result(leftFuture, Duration.Inf) shouldBe List(5) + Await.result(rightFuture, Duration.Inf) shouldBe 14 + } + } + +} diff --git a/scheduler/src/test/scala/com/sky/kafka/message/scheduler/ScheduleReaderSpec.scala b/scheduler/src/test/scala/com/sky/kafka/message/scheduler/ScheduleReaderSpec.scala index 1504f3d8..b44f9e19 100644 --- a/scheduler/src/test/scala/com/sky/kafka/message/scheduler/ScheduleReaderSpec.scala +++ b/scheduler/src/test/scala/com/sky/kafka/message/scheduler/ScheduleReaderSpec.scala @@ -2,10 +2,7 @@ package com.sky.kafka.message.scheduler import java.util.UUID -import akka.stream.testkit.scaladsl.TestSource -import akka.testkit.TestProbe -import com.sky.kafka.message.scheduler.SchedulingActor.{Ack, Init} -import com.sky.kafka.message.scheduler.domain.Schedule +import com.sky.kafka.message.scheduler.domain.ScheduleData.Schedule import com.sky.kafka.message.scheduler.streams.ScheduleReader import common.AkkaStreamBaseSpec import common.TestDataUtils._ @@ -17,13 +14,13 @@ class ScheduleReaderSpec extends AkkaStreamBaseSpec { "generate a CreateOrUpdate message if there is a schedule" in { val (scheduleId, schedule) = (UUID.randomUUID().toString, random[Schedule]) ScheduleReader.toSchedulingMessage(Right((scheduleId, Some(schedule)))) shouldBe - SchedulingActor.CreateOrUpdate(scheduleId, schedule) + Right(SchedulingActor.CreateOrUpdate(scheduleId, schedule)) } "generate a Cancel message if there is no schedule" in { val scheduleId = UUID.randomUUID().toString ScheduleReader.toSchedulingMessage(Right((scheduleId, None))) shouldBe - SchedulingActor.Cancel(scheduleId) + Right(SchedulingActor.Cancel(scheduleId)) } } diff --git a/scheduler/src/test/scala/com/sky/kafka/message/scheduler/ScheduledMessagePublisherSpec.scala b/scheduler/src/test/scala/com/sky/kafka/message/scheduler/ScheduledMessagePublisherSpec.scala index fdb7b990..c34d8403 100644 --- a/scheduler/src/test/scala/com/sky/kafka/message/scheduler/ScheduledMessagePublisherSpec.scala +++ b/scheduler/src/test/scala/com/sky/kafka/message/scheduler/ScheduledMessagePublisherSpec.scala @@ -2,7 +2,9 @@ package com.sky.kafka.message.scheduler import java.util.UUID -import com.sky.kafka.message.scheduler.domain.{Schedule, ScheduleMetadata} +import com.sky.kafka.message.scheduler.config.{SchedulerConfig, ShutdownTimeout} +import com.sky.kafka.message.scheduler.domain.ScheduleData.Schedule +import com.sky.kafka.message.scheduler.kafka.KafkaStream import com.sky.kafka.message.scheduler.streams.ScheduledMessagePublisher import common.AkkaStreamBaseSpec import common.TestDataUtils.random @@ -13,16 +15,19 @@ import scala.concurrent.duration._ class ScheduledMessagePublisherSpec extends AkkaStreamBaseSpec { + val testTopic = UUID.randomUUID().toString + val publisher = ScheduledMessagePublisher( + SchedulerConfig(testTopic, ShutdownTimeout(1 second, 1 second), 5), + KafkaStream.sink + ) + "splitToScheduleAndMetadata" should { "split a (scheduleId, schedule) to a list containing a schedule and a schedule metadata" in { - val testTopic = UUID.randomUUID().toString - val publisher = ScheduledMessagePublisher(SchedulerConfig(testTopic, ShutdownTimeout(1 second, 1 second), 5)) val (scheduleId, schedule) = (UUID.randomUUID().toString, random[Schedule]) publisher.splitToScheduleAndMetadata((scheduleId, schedule)) shouldBe List( - Right(schedule), - Left(ScheduleMetadata(scheduleId, testTopic)) + new ProducerRecord(schedule.topic, schedule.key, schedule.value), + new ProducerRecord(testTopic, scheduleId.getBytes, null) ) } } - } diff --git a/scheduler/src/test/scala/com/sky/kafka/message/scheduler/SchedulerSpec.scala b/scheduler/src/test/scala/com/sky/kafka/message/scheduler/SchedulerSpec.scala index 5cd06914..3dd66239 100644 --- a/scheduler/src/test/scala/com/sky/kafka/message/scheduler/SchedulerSpec.scala +++ b/scheduler/src/test/scala/com/sky/kafka/message/scheduler/SchedulerSpec.scala @@ -8,6 +8,7 @@ import common.BaseSpec import org.apache.kafka.clients.consumer.ConsumerRecord import avro._ import com.sky.kafka.message.scheduler.domain.ApplicationError._ +import com.sky.kafka.message.scheduler.domain.ScheduleData.Schedule class SchedulerSpec extends BaseSpec { diff --git a/scheduler/src/test/scala/com/sky/kafka/message/scheduler/SchedulingActorSpec.scala b/scheduler/src/test/scala/com/sky/kafka/message/scheduler/SchedulingActorSpec.scala index 03c2e571..d3ade969 100644 --- a/scheduler/src/test/scala/com/sky/kafka/message/scheduler/SchedulingActorSpec.scala +++ b/scheduler/src/test/scala/com/sky/kafka/message/scheduler/SchedulingActorSpec.scala @@ -7,7 +7,8 @@ import akka.stream.scaladsl.SourceQueue import akka.testkit.{ImplicitSender, TestActorRef, TestKit} import com.miguno.akka.testing.VirtualTime import com.sky.kafka.message.scheduler.SchedulingActor._ -import com.sky.kafka.message.scheduler.domain.{Schedule, ScheduleId} +import com.sky.kafka.message.scheduler.domain.ScheduleData.Schedule +import com.sky.kafka.message.scheduler.domain.ScheduleId import common.TestDataUtils._ import common.{BaseSpec, TestActorSystem} import org.mockito.Mockito._ diff --git a/scheduler/src/test/scala/com/sky/kafka/message/scheduler/e2e/SchedulerIntSpec.scala b/scheduler/src/test/scala/com/sky/kafka/message/scheduler/e2e/SchedulerIntSpec.scala index fc9ba7a0..25a2c47c 100644 --- a/scheduler/src/test/scala/com/sky/kafka/message/scheduler/e2e/SchedulerIntSpec.scala +++ b/scheduler/src/test/scala/com/sky/kafka/message/scheduler/e2e/SchedulerIntSpec.scala @@ -2,9 +2,9 @@ package com.sky.kafka.message.scheduler.e2e import java.util.UUID -import com.sky.kafka.message.scheduler.domain.Schedule -import com.sky.kafka.message.scheduler._ import com.sky.kafka.message.scheduler.avro._ +import com.sky.kafka.message.scheduler.config.{AppConfig, SchedulerConfig, ShutdownTimeout} +import com.sky.kafka.message.scheduler.domain.ScheduleData.Schedule import com.sky.kafka.message.scheduler.streams.ScheduleReader import common.TestDataUtils._ import common.{AkkaStreamBaseSpec, KafkaIntSpec} diff --git a/scheduler/src/test/scala/common/TestDataUtils.scala b/scheduler/src/test/scala/common/TestDataUtils.scala index a8182363..26ace70d 100644 --- a/scheduler/src/test/scala/common/TestDataUtils.scala +++ b/scheduler/src/test/scala/common/TestDataUtils.scala @@ -5,9 +5,9 @@ import java.time.{Instant, OffsetDateTime, ZoneId} import com.danielasfregola.randomdatagenerator.RandomDataGenerator import com.sksamuel.avro4s.{AvroOutputStream, ToRecord} -import com.sky.kafka.message.scheduler.domain.Schedule import org.scalacheck._ import com.sky.kafka.message.scheduler.avro._ +import com.sky.kafka.message.scheduler.domain.ScheduleData.Schedule object TestDataUtils extends RandomDataGenerator { From b18e3ec269982850c2ce717d86df9562d8af0732 Mon Sep 17 00:00:00 2001 From: "Hubert Behaghel, Lawrence Carvalho, Matthew Pickering, Paolo Ambrosio and Roberto Tena" Date: Mon, 7 Aug 2017 08:51:33 +0100 Subject: [PATCH 08/22] Fix splitToScheduleAndMetadata test. --- .../message/scheduler/ScheduledMessagePublisherSpec.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scheduler/src/test/scala/com/sky/kafka/message/scheduler/ScheduledMessagePublisherSpec.scala b/scheduler/src/test/scala/com/sky/kafka/message/scheduler/ScheduledMessagePublisherSpec.scala index c34d8403..4f7ea1a0 100644 --- a/scheduler/src/test/scala/com/sky/kafka/message/scheduler/ScheduledMessagePublisherSpec.scala +++ b/scheduler/src/test/scala/com/sky/kafka/message/scheduler/ScheduledMessagePublisherSpec.scala @@ -22,9 +22,9 @@ class ScheduledMessagePublisherSpec extends AkkaStreamBaseSpec { ) "splitToScheduleAndMetadata" should { - "split a (scheduleId, schedule) to a list containing a schedule and a schedule metadata" in { + "split schedule and convert to producer records" in { val (scheduleId, schedule) = (UUID.randomUUID().toString, random[Schedule]) - publisher.splitToScheduleAndMetadata((scheduleId, schedule)) shouldBe List( + publisher.splitToScheduleAndMetadata((scheduleId, schedule)) === List( new ProducerRecord(schedule.topic, schedule.key, schedule.value), new ProducerRecord(testTopic, scheduleId.getBytes, null) ) From b84b9483df75756017bdc5cbab9a72f0b4607b1e Mon Sep 17 00:00:00 2001 From: "Hubert Behaghel, Lawrence Carvalho, Matthew Pickering, Paolo Ambrosio and Roberto Tena" Date: Mon, 7 Aug 2017 10:48:00 +0100 Subject: [PATCH 09/22] Logging in publisher stream. --- avro/src/main/scala/GenerateSchema.scala | 2 +- .../sky/kafka/message/scheduler/SchedulingActor.scala | 11 +++++++---- .../scheduler/streams/ScheduledMessagePublisher.scala | 1 + .../streams/ScheduledMessagePublisherStream.scala | 3 ++- 4 files changed, 11 insertions(+), 6 deletions(-) diff --git a/avro/src/main/scala/GenerateSchema.scala b/avro/src/main/scala/GenerateSchema.scala index 3ccae207..a152ad48 100644 --- a/avro/src/main/scala/GenerateSchema.scala +++ b/avro/src/main/scala/GenerateSchema.scala @@ -2,7 +2,7 @@ import java.nio.charset.StandardCharsets import java.nio.file.{Files, Paths} import com.sksamuel.avro4s._ -import com.sky.kafka.message.scheduler.domain.Schedule +import com.sky.kafka.message.scheduler.domain.ScheduleData.Schedule object GenerateSchema extends App { diff --git a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/SchedulingActor.scala b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/SchedulingActor.scala index 2d70866b..1986c14e 100644 --- a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/SchedulingActor.scala +++ b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/SchedulingActor.scala @@ -29,7 +29,10 @@ class SchedulingActor(sourceQueue: SourceQueue[(String, Schedule)], scheduler: S log.info(s"Updating schedule $scheduleId") else log.info(s"Creating schedule $scheduleId") - val cancellable = scheduler.scheduleOnce(timeFromNow(schedule.time))(sourceQueue.offer((scheduleId, schedule))) + val cancellable = scheduler.scheduleOnce(timeFromNow(schedule.time)){ + log.info(s"$scheduleId is due. Adding schedule to queue. Scheduled time was ${schedule.time}") + sourceQueue.offer((scheduleId, schedule)) + } schedules + (scheduleId -> cancellable) } @@ -47,15 +50,15 @@ class SchedulingActor(sourceQueue: SourceQueue[(String, Schedule)], scheduler: S } } - def updateStateAndAck(schedules: Map[ScheduleId, Cancellable]): Unit = { + private def updateStateAndAck(schedules: Map[ScheduleId, Cancellable]): Unit = { context.become(receiveScheduleMessages(schedules)) sender ! Ack } - def cancel(scheduleId: ScheduleId, schedules: Map[ScheduleId, Cancellable]): Boolean = + private def cancel(scheduleId: ScheduleId, schedules: Map[ScheduleId, Cancellable]): Boolean = schedules.get(scheduleId).exists(_.cancel()) - def timeFromNow(time: OffsetDateTime): FiniteDuration = { + private def timeFromNow(time: OffsetDateTime): FiniteDuration = { val offset = ChronoUnit.MILLIS.between(OffsetDateTime.now, time) FiniteDuration(offset, TimeUnit.MILLISECONDS) } diff --git a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/ScheduledMessagePublisher.scala b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/ScheduledMessagePublisher.scala index 7a5d5d30..41eb713c 100644 --- a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/ScheduledMessagePublisher.scala +++ b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/ScheduledMessagePublisher.scala @@ -26,6 +26,7 @@ case class ScheduledMessagePublisher(config: SchedulerConfig, publisherSink: Sin val splitToScheduleAndMetadata: ((ScheduleId, Schedule)) => List[In] = { case (scheduleId, schedule) => + logger.info(s"Publishing scheduled message $scheduleId to ${schedule.topic} and deleting it from ${config.scheduleTopic}") List(schedule, ScheduleMetadata(scheduleId, config.scheduleTopic)) } } diff --git a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/ScheduledMessagePublisherStream.scala b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/ScheduledMessagePublisherStream.scala index dcd3e065..c9a9f83b 100644 --- a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/ScheduledMessagePublisherStream.scala +++ b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/ScheduledMessagePublisherStream.scala @@ -5,11 +5,12 @@ import cats.Eval import com.sky.kafka.message.scheduler.config.SchedulerConfig import com.sky.kafka.message.scheduler.domain.ScheduleData.Schedule import com.sky.kafka.message.scheduler.domain.ScheduleId +import com.typesafe.scalalogging.LazyLogging import org.zalando.grafter.{Stop, StopResult} import scala.concurrent.Await -trait ScheduledMessagePublisherStream extends Stop { +trait ScheduledMessagePublisherStream extends Stop with LazyLogging { def config: SchedulerConfig From 3355089ad95a109ace83f204ee8fb9646e974915 Mon Sep 17 00:00:00 2001 From: Lawrence Carvalho and Roberto Tena Date: Mon, 7 Aug 2017 10:58:31 +0100 Subject: [PATCH 10/22] Add some clarity around actor receive. --- .../scala/com/sky/kafka/message/scheduler/SchedulingActor.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/SchedulingActor.scala b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/SchedulingActor.scala index 1986c14e..f0e17a07 100644 --- a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/SchedulingActor.scala +++ b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/SchedulingActor.scala @@ -45,7 +45,7 @@ class SchedulingActor(sourceQueue: SourceQueue[(String, Schedule)], scheduler: S schedules - scheduleId } - receiveCreateOrUpdateMessage orElse receiveCancelMessage andThen updateStateAndAck orElse { + (receiveCreateOrUpdateMessage orElse receiveCancelMessage andThen updateStateAndAck) orElse { case Init => sender ! Ack } } From 1f04e03fb94b6303b0a500d9be572c4b05bf5319 Mon Sep 17 00:00:00 2001 From: Lawrence Carvalho and Roberto Tena Date: Mon, 7 Aug 2017 15:37:58 +0100 Subject: [PATCH 11/22] Fix random data generation error. --- build.sbt | 4 +++- .../src/test/scala/common/TestDataUtils.scala | 16 +++++++++------- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/build.sbt b/build.sbt index b61b6082..859f71e8 100644 --- a/build.sbt +++ b/build.sbt @@ -27,6 +27,7 @@ val dependencies = Seq( "net.cakesolutions" %% "scala-kafka-client-testkit" % kafkaVersion % Test, "org.slf4j" % "log4j-over-slf4j" % "1.7.21" % Test, "com.danielasfregola" %% "random-data-generator" % "2.1" % Test, + "com.47deg" %% "scalacheck-toolbox-datetime"% "0.2.2" % Test, "com.miguno.akka" %% "akka-mock-scheduler" % "0.5.1" % Test, "org.mockito" % "mockito-all" % "1.10.19" % Test ) @@ -77,7 +78,8 @@ lazy val scheduler = (project in file("scheduler")) javaAgents += "org.aspectj" % "aspectjweaver" % "1.8.10", javaOptions in Universal += jmxSettings, buildInfoSettings, - dockerSettings + dockerSettings, + dependencyOverrides += "org.scalacheck" %% "scalacheck" % "1.13.5" ).enablePlugins(DockerPlugin) val schema = inputKey[Unit]("Generate the Avro schema file for the Schedule schema.") diff --git a/scheduler/src/test/scala/common/TestDataUtils.scala b/scheduler/src/test/scala/common/TestDataUtils.scala index 26ace70d..f026254a 100644 --- a/scheduler/src/test/scala/common/TestDataUtils.scala +++ b/scheduler/src/test/scala/common/TestDataUtils.scala @@ -1,23 +1,25 @@ package common import java.io.ByteArrayOutputStream -import java.time.{Instant, OffsetDateTime, ZoneId} +import java.time._ import com.danielasfregola.randomdatagenerator.RandomDataGenerator +import com.fortysevendeg.scalacheck.datetime.GenDateTime.genDateTimeWithinRange +import com.fortysevendeg.scalacheck.datetime.instances.jdk8._ import com.sksamuel.avro4s.{AvroOutputStream, ToRecord} +import com.sky.kafka.message.scheduler.domain.ScheduleData.Schedule import org.scalacheck._ import com.sky.kafka.message.scheduler.avro._ -import com.sky.kafka.message.scheduler.domain.ScheduleData.Schedule object TestDataUtils extends RandomDataGenerator { - implicit val arbAlphaString: Arbitrary[String] = Arbitrary(Gen.alphaStr.suchThat(!_.isEmpty)) + implicit val arbAlphaString: Arbitrary[String] = + Arbitrary(Gen.alphaStr.suchThat(_.nonEmpty).retryUntil(_.nonEmpty)) implicit val arbNextMonthOffsetDateTime: Arbitrary[OffsetDateTime] = { - val low = OffsetDateTime.now() - val high = low.plusMonths(1) - Arbitrary(Gen.choose(low.toEpochSecond, high.toEpochSecond) - .map(epoch => OffsetDateTime.ofInstant(Instant.ofEpochSecond(epoch), ZoneId.systemDefault()))) + val from = ZonedDateTime.now() + val range = Duration.ofDays(20) + Arbitrary(genDateTimeWithinRange(from, range).map(_.toOffsetDateTime)) } implicit class ScheduleOps(val schedule: Schedule) extends AnyVal { From 2958a7b1b315bc72c67095575cc3ee8d7b936db0 Mon Sep 17 00:00:00 2001 From: Lawrence Carvalho and Roberto Tena Date: Mon, 7 Aug 2017 15:58:39 +0100 Subject: [PATCH 12/22] Fix memory leak from consecutive test runs. --- .../sky/kafka/message/scheduler/SchedulingActorSpec.scala | 6 +++--- .../sky/kafka/message/scheduler/e2e/SchedulerIntSpec.scala | 3 --- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/scheduler/src/test/scala/com/sky/kafka/message/scheduler/SchedulingActorSpec.scala b/scheduler/src/test/scala/com/sky/kafka/message/scheduler/SchedulingActorSpec.scala index d3ade969..6e32744a 100644 --- a/scheduler/src/test/scala/com/sky/kafka/message/scheduler/SchedulingActorSpec.scala +++ b/scheduler/src/test/scala/com/sky/kafka/message/scheduler/SchedulingActorSpec.scala @@ -4,19 +4,19 @@ import java.util.UUID import akka.event.LoggingAdapter import akka.stream.scaladsl.SourceQueue -import akka.testkit.{ImplicitSender, TestActorRef, TestKit} +import akka.testkit.{ImplicitSender, TestActorRef} import com.miguno.akka.testing.VirtualTime import com.sky.kafka.message.scheduler.SchedulingActor._ import com.sky.kafka.message.scheduler.domain.ScheduleData.Schedule import com.sky.kafka.message.scheduler.domain.ScheduleId +import common.AkkaBaseSpec import common.TestDataUtils._ -import common.{BaseSpec, TestActorSystem} import org.mockito.Mockito._ import org.scalatest.mockito.MockitoSugar import scala.concurrent.duration._ -class SchedulingActorSpec extends TestKit(TestActorSystem()) with ImplicitSender with BaseSpec with MockitoSugar { +class SchedulingActorSpec extends AkkaBaseSpec with ImplicitSender with MockitoSugar { "A scheduler actor" must { "schedule new messages at the given time" in new SchedulingActorTest { diff --git a/scheduler/src/test/scala/com/sky/kafka/message/scheduler/e2e/SchedulerIntSpec.scala b/scheduler/src/test/scala/com/sky/kafka/message/scheduler/e2e/SchedulerIntSpec.scala index 25a2c47c..0642c169 100644 --- a/scheduler/src/test/scala/com/sky/kafka/message/scheduler/e2e/SchedulerIntSpec.scala +++ b/scheduler/src/test/scala/com/sky/kafka/message/scheduler/e2e/SchedulerIntSpec.scala @@ -12,7 +12,6 @@ import org.apache.kafka.common.serialization._ import org.scalatest.Assertion import org.zalando.grafter.Rewriter -import scala.concurrent.Await import scala.concurrent.duration._ class SchedulerIntSpec extends AkkaStreamBaseSpec with KafkaIntSpec { @@ -49,8 +48,6 @@ class SchedulerIntSpec extends AkkaStreamBaseSpec with KafkaIntSpec { scenario Rewriter.stop(app).value - materializer.shutdown() - Await.ready(system.terminate(), shutdownTimeout.system) } private def consumeLatestFromScheduleTopic = consumeFromKafka(ScheduleTopic, 2, new StringDeserializer).last From a5e304cf207d4d8dbee85e9ebf442e2a55c54a54 Mon Sep 17 00:00:00 2001 From: Lawrence Carvalho and Roberto Tena Date: Mon, 7 Aug 2017 17:18:45 +0100 Subject: [PATCH 13/22] WIP supervision metrics. --- .../message/scheduler/AkkaComponents.scala | 17 ++++++++++++++--- .../message/scheduler/AkkaComponentsSpec.scala | 18 ++++++++++++++++++ 2 files changed, 32 insertions(+), 3 deletions(-) create mode 100644 scheduler/src/test/scala/com/sky/kafka/message/scheduler/AkkaComponentsSpec.scala diff --git a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/AkkaComponents.scala b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/AkkaComponents.scala index 5ab9721a..fca98995 100644 --- a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/AkkaComponents.scala +++ b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/AkkaComponents.scala @@ -1,11 +1,22 @@ package com.sky.kafka.message.scheduler import akka.actor.ActorSystem -import akka.stream.ActorMaterializer +import akka.stream.Supervision.Restart +import akka.stream.{ActorMaterializer, ActorMaterializerSettings, Supervision} +import com.typesafe.scalalogging.LazyLogging -trait AkkaComponents { +trait AkkaComponents extends LazyLogging { implicit val system = ActorSystem("kafka-message-scheduler") - implicit val materializer = ActorMaterializer() + + val decider: Supervision.Decider = { t => + logger.error(s"Supervision failed.", t) + Restart + } + + val settings = ActorMaterializerSettings(system) + .withSupervisionStrategy(decider) + + implicit val materializer = ActorMaterializer(settings) } diff --git a/scheduler/src/test/scala/com/sky/kafka/message/scheduler/AkkaComponentsSpec.scala b/scheduler/src/test/scala/com/sky/kafka/message/scheduler/AkkaComponentsSpec.scala new file mode 100644 index 00000000..61798e2b --- /dev/null +++ b/scheduler/src/test/scala/com/sky/kafka/message/scheduler/AkkaComponentsSpec.scala @@ -0,0 +1,18 @@ +package com.sky.kafka.message.scheduler + +import akka.actor.Props +import akka.actor.SupervisorStrategy.Restart +import akka.testkit.TestActorRef +import common.{AkkaBaseSpec, AkkaStreamBaseSpec} + +class AkkaComponentsSpec extends AkkaBaseSpec { + + "decider" should { + "return a 'Restart' supervision strategy" in { + val supervisor = TestActorRef[SchedulingActor](Props(new SchedulingActor(null, null))) + val strategy = supervisor.underlyingActor.supervisorStrategy.decider + + strategy(new Exception("Any exception")) should be (Restart) + } + } +} From ab143f56c543f88054473259e7a285ef9e78a2d0 Mon Sep 17 00:00:00 2001 From: Lawrence Carvalho and Roberto Tena Date: Tue, 8 Aug 2017 11:41:57 +0100 Subject: [PATCH 14/22] Transform schedules to scheduled messages before adding to the queue. --- avro/src/main/scala/GenerateSchema.scala | 2 +- .../message/scheduler/SchedulingActor.scala | 15 ++++++----- .../scheduler/domain/PublishableMessage.scala | 26 ++++++++++++++++++ .../scheduler/domain/ScheduleData.scala | 27 ------------------- .../message/scheduler/domain/package.scala | 4 +++ .../sky/kafka/message/scheduler/package.scala | 3 +-- .../streams/ScheduledMessagePublisher.scala | 18 ++++++------- .../ScheduledMessagePublisherStream.scala | 4 +-- .../scheduler/ScheduleReaderSpec.scala | 2 +- .../ScheduledMessagePublisherSpec.scala | 4 +-- .../message/scheduler/SchedulerSpec.scala | 1 - .../scheduler/SchedulingActorSpec.scala | 22 +++++++-------- .../scheduler/e2e/SchedulerIntSpec.scala | 4 +-- .../src/test/scala/common/AkkaBaseSpec.scala | 2 +- .../src/test/scala/common/TestDataUtils.scala | 8 ++++-- 15 files changed, 75 insertions(+), 67 deletions(-) create mode 100644 scheduler/src/main/scala/com/sky/kafka/message/scheduler/domain/PublishableMessage.scala delete mode 100644 scheduler/src/main/scala/com/sky/kafka/message/scheduler/domain/ScheduleData.scala diff --git a/avro/src/main/scala/GenerateSchema.scala b/avro/src/main/scala/GenerateSchema.scala index a152ad48..3ccae207 100644 --- a/avro/src/main/scala/GenerateSchema.scala +++ b/avro/src/main/scala/GenerateSchema.scala @@ -2,7 +2,7 @@ import java.nio.charset.StandardCharsets import java.nio.file.{Files, Paths} import com.sksamuel.avro4s._ -import com.sky.kafka.message.scheduler.domain.ScheduleData.Schedule +import com.sky.kafka.message.scheduler.domain.Schedule object GenerateSchema extends App { diff --git a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/SchedulingActor.scala b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/SchedulingActor.scala index f0e17a07..79f7c85f 100644 --- a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/SchedulingActor.scala +++ b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/SchedulingActor.scala @@ -10,14 +10,14 @@ import akka.stream.scaladsl.SourceQueue import cats.data.Reader import com.sky.kafka.message.scheduler.SchedulingActor.{Ack, Cancel, CreateOrUpdate, Init} import com.sky.kafka.message.scheduler.config.AppConfig -import com.sky.kafka.message.scheduler.domain.ScheduleData.Schedule -import com.sky.kafka.message.scheduler.domain.ScheduleId +import com.sky.kafka.message.scheduler.domain.PublishableMessage.ScheduledMessage +import com.sky.kafka.message.scheduler.domain._ import com.sky.kafka.message.scheduler.streams.ScheduledMessagePublisher import scala.concurrent.ExecutionContext.Implicits.global import scala.concurrent.duration.FiniteDuration -class SchedulingActor(sourceQueue: SourceQueue[(String, Schedule)], scheduler: Scheduler) extends Actor with ActorLogging { +class SchedulingActor(sourceQueue: SourceQueue[(String, ScheduledMessage)], scheduler: Scheduler) extends Actor with ActorLogging { override def receive: Receive = receiveScheduleMessages(Map.empty) @@ -29,9 +29,9 @@ class SchedulingActor(sourceQueue: SourceQueue[(String, Schedule)], scheduler: S log.info(s"Updating schedule $scheduleId") else log.info(s"Creating schedule $scheduleId") - val cancellable = scheduler.scheduleOnce(timeFromNow(schedule.time)){ + val cancellable = scheduler.scheduleOnce(timeFromNow(schedule.time)) { log.info(s"$scheduleId is due. Adding schedule to queue. Scheduled time was ${schedule.time}") - sourceQueue.offer((scheduleId, schedule)) + sourceQueue.offer((scheduleId, messageFrom(schedule))) } schedules + (scheduleId -> cancellable) } @@ -62,6 +62,9 @@ class SchedulingActor(sourceQueue: SourceQueue[(String, Schedule)], scheduler: S val offset = ChronoUnit.MILLIS.between(OffsetDateTime.now, time) FiniteDuration(offset, TimeUnit.MILLISECONDS) } + + private def messageFrom(schedule: Schedule) = + ScheduledMessage(schedule.topic, schedule.key, schedule.value) } object SchedulingActor { @@ -81,7 +84,7 @@ object SchedulingActor { system.actorOf(props(publisher.stream, system.scheduler), "scheduling-actor") ) - private def props(queue: SourceQueue[(ScheduleId, Schedule)], scheduler: Scheduler): Props = + private def props(queue: SourceQueue[(ScheduleId, ScheduledMessage)], scheduler: Scheduler): Props = Props(new SchedulingActor(queue, scheduler)) } diff --git a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/domain/PublishableMessage.scala b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/domain/PublishableMessage.scala new file mode 100644 index 00000000..2f69f9aa --- /dev/null +++ b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/domain/PublishableMessage.scala @@ -0,0 +1,26 @@ +package com.sky.kafka.message.scheduler.domain + +import com.sky.kafka.message.scheduler.kafka.ProducerRecordEncoder +import org.apache.kafka.clients.producer.ProducerRecord + +sealed trait PublishableMessage + +object PublishableMessage { + + case class ScheduledMessage(topic: String, key: Array[Byte], value: Array[Byte]) extends PublishableMessage + + case class ScheduleDeletion(scheduleId: ScheduleId, scheduleTopic: String) extends PublishableMessage + + implicit val scheduledMessageProducerRecordEnc: ProducerRecordEncoder[ScheduledMessage] = + ProducerRecordEncoder.instance(message => new ProducerRecord(message.topic, message.key, message.value)) + + implicit val scheduleDeletionProducerRecordEnc: ProducerRecordEncoder[ScheduleDeletion] = + ProducerRecordEncoder.instance(deletion => new ProducerRecord(deletion.scheduleTopic, deletion.scheduleId.getBytes, null)) + + + implicit def scheduleDataToProducerRecord(msg: PublishableMessage): ProducerRecord[Array[Byte], Array[Byte]] = + msg match { + case scheduledMsg: ScheduledMessage => scheduledMessageProducerRecordEnc(scheduledMsg) + case deletion: ScheduleDeletion => scheduleDeletionProducerRecordEnc(deletion) + } +} diff --git a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/domain/ScheduleData.scala b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/domain/ScheduleData.scala deleted file mode 100644 index 5c3bdc01..00000000 --- a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/domain/ScheduleData.scala +++ /dev/null @@ -1,27 +0,0 @@ -package com.sky.kafka.message.scheduler.domain - -import java.time.OffsetDateTime - -import com.sky.kafka.message.scheduler.kafka.ProducerRecordEncoder -import org.apache.kafka.clients.producer.ProducerRecord - -sealed trait ScheduleData - -object ScheduleData { - - case class Schedule(time: OffsetDateTime, topic: String, key: Array[Byte], value: Array[Byte]) extends ScheduleData - - case class ScheduleMetadata(scheduleId: ScheduleId, topic: String) extends ScheduleData - - implicit val scheduleProducerRecordEncoder: ProducerRecordEncoder[Schedule] = - ProducerRecordEncoder.instance(schedule => new ProducerRecord(schedule.topic, schedule.key, schedule.value)) - - implicit val scheduleMedatadataProducerRecordEncoder: ProducerRecordEncoder[ScheduleMetadata] = - ProducerRecordEncoder.instance(metadata => new ProducerRecord(metadata.topic, metadata.scheduleId.getBytes, null)) - - - implicit def scheduleDataToProducerRecord(scheduleData: ScheduleData): ProducerRecord[Array[Byte], Array[Byte]] = scheduleData match { - case schedule: Schedule => scheduleProducerRecordEncoder(schedule) - case metadata: ScheduleMetadata => scheduleMedatadataProducerRecordEncoder(metadata) - } -} diff --git a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/domain/package.scala b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/domain/package.scala index 1e47060b..73a3b8ff 100644 --- a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/domain/package.scala +++ b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/domain/package.scala @@ -1,7 +1,11 @@ package com.sky.kafka.message.scheduler +import java.time.OffsetDateTime + package object domain { type ScheduleId = String + case class Schedule(time: OffsetDateTime, topic: String, key: Array[Byte], value: Array[Byte]) + } diff --git a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/package.scala b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/package.scala index 030447cd..a4b20885 100644 --- a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/package.scala +++ b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/package.scala @@ -3,12 +3,11 @@ package com.sky.kafka.message import cats.syntax.either._ import com.sksamuel.avro4s.AvroInputStream import com.sky.kafka.message.scheduler.domain.ApplicationError._ -import com.sky.kafka.message.scheduler.domain.{ApplicationError, _} +import com.sky.kafka.message.scheduler.domain._ import com.sky.kafka.message.scheduler.kafka.ConsumerRecordDecoder import com.typesafe.scalalogging.LazyLogging import org.apache.kafka.clients.consumer.ConsumerRecord import com.sky.kafka.message.scheduler.avro._ -import com.sky.kafka.message.scheduler.domain.ScheduleData.Schedule import scala.util.Try diff --git a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/ScheduledMessagePublisher.scala b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/ScheduledMessagePublisher.scala index 41eb713c..9b730c83 100644 --- a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/ScheduledMessagePublisher.scala +++ b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/ScheduledMessagePublisher.scala @@ -6,28 +6,28 @@ import akka.stream.scaladsl.{Sink, Source, SourceQueueWithComplete} import akka.stream.{ActorMaterializer, OverflowStrategy} import cats.data.Reader import com.sky.kafka.message.scheduler.config._ -import com.sky.kafka.message.scheduler.domain.ScheduleData._ -import org.apache.kafka.clients.producer.ProducerRecord +import com.sky.kafka.message.scheduler.domain.PublishableMessage._ import com.sky.kafka.message.scheduler.domain._ import com.sky.kafka.message.scheduler.kafka.KafkaStream import com.sky.kafka.message.scheduler.streams.ScheduledMessagePublisher._ +import org.apache.kafka.clients.producer.ProducerRecord import scala.concurrent.Future case class ScheduledMessagePublisher(config: SchedulerConfig, publisherSink: Sink[In, Mat]) - (implicit system: ActorSystem, materializer: ActorMaterializer) + (implicit system: ActorSystem, materializer: ActorMaterializer) extends ScheduledMessagePublisherStream { - def stream: SourceQueueWithComplete[(ScheduleId, Schedule)] = - Source.queue[(ScheduleId, Schedule)](config.queueBufferSize, OverflowStrategy.backpressure) + def stream: SourceQueueWithComplete[(ScheduleId, ScheduledMessage)] = + Source.queue[(ScheduleId, ScheduledMessage)](config.queueBufferSize, OverflowStrategy.backpressure) .mapConcat(splitToScheduleAndMetadata) .to(publisherSink) .run() - val splitToScheduleAndMetadata: ((ScheduleId, Schedule)) => List[In] = { - case (scheduleId, schedule) => - logger.info(s"Publishing scheduled message $scheduleId to ${schedule.topic} and deleting it from ${config.scheduleTopic}") - List(schedule, ScheduleMetadata(scheduleId, config.scheduleTopic)) + val splitToScheduleAndMetadata: ((ScheduleId, ScheduledMessage)) => List[In] = { + case (scheduleId, scheduledMessage) => + logger.info(s"Publishing scheduled message $scheduleId to ${scheduledMessage.topic} and deleting it from ${config.scheduleTopic}") + List(scheduledMessage, ScheduleDeletion(scheduleId, config.scheduleTopic)) } } diff --git a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/ScheduledMessagePublisherStream.scala b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/ScheduledMessagePublisherStream.scala index c9a9f83b..45ce6539 100644 --- a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/ScheduledMessagePublisherStream.scala +++ b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/ScheduledMessagePublisherStream.scala @@ -3,7 +3,7 @@ package com.sky.kafka.message.scheduler.streams import akka.stream.scaladsl.SourceQueueWithComplete import cats.Eval import com.sky.kafka.message.scheduler.config.SchedulerConfig -import com.sky.kafka.message.scheduler.domain.ScheduleData.Schedule +import com.sky.kafka.message.scheduler.domain.PublishableMessage.ScheduledMessage import com.sky.kafka.message.scheduler.domain.ScheduleId import com.typesafe.scalalogging.LazyLogging import org.zalando.grafter.{Stop, StopResult} @@ -14,7 +14,7 @@ trait ScheduledMessagePublisherStream extends Stop with LazyLogging { def config: SchedulerConfig - def stream: SourceQueueWithComplete[(ScheduleId, Schedule)] + def stream: SourceQueueWithComplete[(ScheduleId, ScheduledMessage)] override def stop: Eval[StopResult] = StopResult.eval("Shutting down queue...") { stream.complete() diff --git a/scheduler/src/test/scala/com/sky/kafka/message/scheduler/ScheduleReaderSpec.scala b/scheduler/src/test/scala/com/sky/kafka/message/scheduler/ScheduleReaderSpec.scala index b44f9e19..7d2ccf06 100644 --- a/scheduler/src/test/scala/com/sky/kafka/message/scheduler/ScheduleReaderSpec.scala +++ b/scheduler/src/test/scala/com/sky/kafka/message/scheduler/ScheduleReaderSpec.scala @@ -2,7 +2,7 @@ package com.sky.kafka.message.scheduler import java.util.UUID -import com.sky.kafka.message.scheduler.domain.ScheduleData.Schedule +import com.sky.kafka.message.scheduler.domain._ import com.sky.kafka.message.scheduler.streams.ScheduleReader import common.AkkaStreamBaseSpec import common.TestDataUtils._ diff --git a/scheduler/src/test/scala/com/sky/kafka/message/scheduler/ScheduledMessagePublisherSpec.scala b/scheduler/src/test/scala/com/sky/kafka/message/scheduler/ScheduledMessagePublisherSpec.scala index 4f7ea1a0..83f91249 100644 --- a/scheduler/src/test/scala/com/sky/kafka/message/scheduler/ScheduledMessagePublisherSpec.scala +++ b/scheduler/src/test/scala/com/sky/kafka/message/scheduler/ScheduledMessagePublisherSpec.scala @@ -3,7 +3,7 @@ package com.sky.kafka.message.scheduler import java.util.UUID import com.sky.kafka.message.scheduler.config.{SchedulerConfig, ShutdownTimeout} -import com.sky.kafka.message.scheduler.domain.ScheduleData.Schedule +import com.sky.kafka.message.scheduler.domain._ import com.sky.kafka.message.scheduler.kafka.KafkaStream import com.sky.kafka.message.scheduler.streams.ScheduledMessagePublisher import common.AkkaStreamBaseSpec @@ -24,7 +24,7 @@ class ScheduledMessagePublisherSpec extends AkkaStreamBaseSpec { "splitToScheduleAndMetadata" should { "split schedule and convert to producer records" in { val (scheduleId, schedule) = (UUID.randomUUID().toString, random[Schedule]) - publisher.splitToScheduleAndMetadata((scheduleId, schedule)) === List( + publisher.splitToScheduleAndMetadata((scheduleId, schedule.toScheduledMessage)) === List( new ProducerRecord(schedule.topic, schedule.key, schedule.value), new ProducerRecord(testTopic, scheduleId.getBytes, null) ) diff --git a/scheduler/src/test/scala/com/sky/kafka/message/scheduler/SchedulerSpec.scala b/scheduler/src/test/scala/com/sky/kafka/message/scheduler/SchedulerSpec.scala index 3dd66239..5cd06914 100644 --- a/scheduler/src/test/scala/com/sky/kafka/message/scheduler/SchedulerSpec.scala +++ b/scheduler/src/test/scala/com/sky/kafka/message/scheduler/SchedulerSpec.scala @@ -8,7 +8,6 @@ import common.BaseSpec import org.apache.kafka.clients.consumer.ConsumerRecord import avro._ import com.sky.kafka.message.scheduler.domain.ApplicationError._ -import com.sky.kafka.message.scheduler.domain.ScheduleData.Schedule class SchedulerSpec extends BaseSpec { diff --git a/scheduler/src/test/scala/com/sky/kafka/message/scheduler/SchedulingActorSpec.scala b/scheduler/src/test/scala/com/sky/kafka/message/scheduler/SchedulingActorSpec.scala index 6e32744a..53e7ac4e 100644 --- a/scheduler/src/test/scala/com/sky/kafka/message/scheduler/SchedulingActorSpec.scala +++ b/scheduler/src/test/scala/com/sky/kafka/message/scheduler/SchedulingActorSpec.scala @@ -7,15 +7,13 @@ import akka.stream.scaladsl.SourceQueue import akka.testkit.{ImplicitSender, TestActorRef} import com.miguno.akka.testing.VirtualTime import com.sky.kafka.message.scheduler.SchedulingActor._ -import com.sky.kafka.message.scheduler.domain.ScheduleData.Schedule -import com.sky.kafka.message.scheduler.domain.ScheduleId +import com.sky.kafka.message.scheduler.domain.PublishableMessage.ScheduledMessage +import com.sky.kafka.message.scheduler.domain._ import common.AkkaBaseSpec import common.TestDataUtils._ import org.mockito.Mockito._ import org.scalatest.mockito.MockitoSugar -import scala.concurrent.duration._ - class SchedulingActorSpec extends AkkaBaseSpec with ImplicitSender with MockitoSugar { "A scheduler actor" must { @@ -24,8 +22,8 @@ class SchedulingActorSpec extends AkkaBaseSpec with ImplicitSender with MockitoS createSchedule(scheduleId, schedule) - advanceToTimeFrom(schedule) - verify(mockSourceQueue).offer((scheduleId, schedule)) + advanceToTimeFrom(schedule, now) + verify(mockSourceQueue).offer((scheduleId, schedule.toScheduledMessage)) } "cancel schedules when a cancel message is received" in new SchedulingActorTest { @@ -56,10 +54,10 @@ class SchedulingActorSpec extends AkkaBaseSpec with ImplicitSender with MockitoS createSchedule(scheduleId, updatedSchedule) advanceToTimeFrom(schedule) - verify(mockSourceQueue, never()).offer((scheduleId, schedule)) + verify(mockSourceQueue, never()).offer((scheduleId, schedule.toScheduledMessage)) advanceToTimeFrom(updatedSchedule, schedule.timeInMillis) - verify(mockSourceQueue).offer((scheduleId, updatedSchedule)) + verify(mockSourceQueue).offer((scheduleId, updatedSchedule.toScheduledMessage)) } "send an Ack to the sender when receiving an Init message" in new SchedulingActorTest { @@ -70,8 +68,10 @@ class SchedulingActorSpec extends AkkaBaseSpec with ImplicitSender with MockitoS } private class SchedulingActorTest { + val now = System.currentTimeMillis() + val mockLogger = mock[LoggingAdapter] - val mockSourceQueue = mock[SourceQueue[(ScheduleId, Schedule)]] + val mockSourceQueue = mock[SourceQueue[(ScheduleId, ScheduledMessage)]] val time = new VirtualTime val actorRef = TestActorRef(new SchedulingActor(mockSourceQueue, time.scheduler) { @@ -81,8 +81,8 @@ class SchedulingActorSpec extends AkkaBaseSpec with ImplicitSender with MockitoS def generateSchedule(): (ScheduleId, Schedule) = (UUID.randomUUID().toString, random[Schedule]) - def advanceToTimeFrom(schedule: Schedule, startTime: Long = System.currentTimeMillis()): Unit = - time.advance(schedule.timeInMillis - startTime + 1 second) + def advanceToTimeFrom(schedule: Schedule, startTime: Long = now): Unit = + time.advance(schedule.timeInMillis - startTime) def createSchedule(scheduleId: ScheduleId, schedule: Schedule) = { actorRef ! CreateOrUpdate(scheduleId, schedule) diff --git a/scheduler/src/test/scala/com/sky/kafka/message/scheduler/e2e/SchedulerIntSpec.scala b/scheduler/src/test/scala/com/sky/kafka/message/scheduler/e2e/SchedulerIntSpec.scala index 0642c169..dbcad3bb 100644 --- a/scheduler/src/test/scala/com/sky/kafka/message/scheduler/e2e/SchedulerIntSpec.scala +++ b/scheduler/src/test/scala/com/sky/kafka/message/scheduler/e2e/SchedulerIntSpec.scala @@ -3,8 +3,8 @@ package com.sky.kafka.message.scheduler.e2e import java.util.UUID import com.sky.kafka.message.scheduler.avro._ -import com.sky.kafka.message.scheduler.config.{AppConfig, SchedulerConfig, ShutdownTimeout} -import com.sky.kafka.message.scheduler.domain.ScheduleData.Schedule +import com.sky.kafka.message.scheduler.config._ +import com.sky.kafka.message.scheduler.domain._ import com.sky.kafka.message.scheduler.streams.ScheduleReader import common.TestDataUtils._ import common.{AkkaStreamBaseSpec, KafkaIntSpec} diff --git a/scheduler/src/test/scala/common/AkkaBaseSpec.scala b/scheduler/src/test/scala/common/AkkaBaseSpec.scala index 5d5f1fec..068aefbb 100644 --- a/scheduler/src/test/scala/common/AkkaBaseSpec.scala +++ b/scheduler/src/test/scala/common/AkkaBaseSpec.scala @@ -7,7 +7,7 @@ abstract class AkkaBaseSpec extends TestKit(TestActorSystem()) with BaseSpec with BeforeAndAfterAll { override def afterAll(): Unit = { - TestKit.shutdownActorSystem(system) + shutdown(system, verifySystemShutdown = true) super.afterAll() } } diff --git a/scheduler/src/test/scala/common/TestDataUtils.scala b/scheduler/src/test/scala/common/TestDataUtils.scala index f026254a..6e39c7ed 100644 --- a/scheduler/src/test/scala/common/TestDataUtils.scala +++ b/scheduler/src/test/scala/common/TestDataUtils.scala @@ -7,9 +7,10 @@ import com.danielasfregola.randomdatagenerator.RandomDataGenerator import com.fortysevendeg.scalacheck.datetime.GenDateTime.genDateTimeWithinRange import com.fortysevendeg.scalacheck.datetime.instances.jdk8._ import com.sksamuel.avro4s.{AvroOutputStream, ToRecord} -import com.sky.kafka.message.scheduler.domain.ScheduleData.Schedule +import com.sky.kafka.message.scheduler.domain._ import org.scalacheck._ import com.sky.kafka.message.scheduler.avro._ +import com.sky.kafka.message.scheduler.domain.PublishableMessage.ScheduledMessage object TestDataUtils extends RandomDataGenerator { @@ -33,8 +34,11 @@ object TestDataUtils extends RandomDataGenerator { def timeInMillis: Long = schedule.time.toInstant.toEpochMilli - def secondsFromNow(seconds: Long) = + def secondsFromNow(seconds: Long): Schedule = schedule.copy(time = OffsetDateTime.now().plusSeconds(seconds)) + + def toScheduledMessage: ScheduledMessage = + ScheduledMessage(schedule.topic, schedule.key, schedule.value) } } From 1063f0c06246a160c97cbe3f0a5d5ff90ecc6607 Mon Sep 17 00:00:00 2001 From: Lawrence Carvalho Date: Tue, 8 Aug 2017 12:42:11 +0100 Subject: [PATCH 15/22] Address review comments. --- .../message/scheduler/SchedulingActor.scala | 25 +++++++++++-------- .../message/scheduler/avro/package.scala | 7 ++++-- .../message/scheduler/config/package.scala | 2 +- .../scheduler/domain/ApplicationError.scala | 11 +++----- .../kafka/ConsumerRecordDecoder.scala | 7 ++++++ .../sky/kafka/message/scheduler/package.scala | 10 +++----- .../scheduler/streams/ScheduleReader.scala | 13 +++++++--- .../streams/ScheduledMessagePublisher.scala | 3 +++ .../scheduler/SchedulingActorSpec.scala | 2 +- .../src/test/scala/common/EmbeddedKafka.scala | 7 ++++++ 10 files changed, 55 insertions(+), 32 deletions(-) diff --git a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/SchedulingActor.scala b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/SchedulingActor.scala index 79f7c85f..f7be76e9 100644 --- a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/SchedulingActor.scala +++ b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/SchedulingActor.scala @@ -5,10 +5,10 @@ import java.time.temporal.ChronoUnit import java.util.concurrent.TimeUnit import akka.actor._ -import akka.stream.ActorMaterializer +import akka.stream.{ActorMaterializer, QueueOfferResult} import akka.stream.scaladsl.SourceQueue import cats.data.Reader -import com.sky.kafka.message.scheduler.SchedulingActor.{Ack, Cancel, CreateOrUpdate, Init} +import com.sky.kafka.message.scheduler.SchedulingActor._ import com.sky.kafka.message.scheduler.config.AppConfig import com.sky.kafka.message.scheduler.domain.PublishableMessage.ScheduledMessage import com.sky.kafka.message.scheduler.domain._ @@ -16,6 +16,8 @@ import com.sky.kafka.message.scheduler.streams.ScheduledMessagePublisher import scala.concurrent.ExecutionContext.Implicits.global import scala.concurrent.duration.FiniteDuration +import scala.concurrent.Future +import akka.pattern.pipe class SchedulingActor(sourceQueue: SourceQueue[(String, ScheduledMessage)], scheduler: Scheduler) extends Actor with ActorLogging { @@ -23,20 +25,15 @@ class SchedulingActor(sourceQueue: SourceQueue[(String, ScheduledMessage)], sche def receiveScheduleMessages(schedules: Map[ScheduleId, Cancellable]): Receive = { - val receiveCreateOrUpdateMessage: PartialFunction[Any, Map[ScheduleId, Cancellable]] = { + val receiveSchedulingMessage: PartialFunction[Any, Map[ScheduleId, Cancellable]] = { case CreateOrUpdate(scheduleId: ScheduleId, schedule: Schedule) => if (cancel(scheduleId, schedules)) log.info(s"Updating schedule $scheduleId") else log.info(s"Creating schedule $scheduleId") - val cancellable = scheduler.scheduleOnce(timeFromNow(schedule.time)) { - log.info(s"$scheduleId is due. Adding schedule to queue. Scheduled time was ${schedule.time}") - sourceQueue.offer((scheduleId, messageFrom(schedule))) - } + val cancellable = scheduler.scheduleOnce(timeFromNow(schedule.time))(self ! TriggerMessage(scheduleId, schedule)) schedules + (scheduleId -> cancellable) - } - val receiveCancelMessage: PartialFunction[Any, Map[ScheduleId, Cancellable]] = { case Cancel(scheduleId: String) => if (cancel(scheduleId, schedules)) log.info(s"Cancelled schedule $scheduleId") @@ -45,7 +42,13 @@ class SchedulingActor(sourceQueue: SourceQueue[(String, ScheduledMessage)], sche schedules - scheduleId } - (receiveCreateOrUpdateMessage orElse receiveCancelMessage andThen updateStateAndAck) orElse { + val receiveTriggerMessage: PartialFunction[Any, Unit] = { + case TriggerMessage(scheduleId, schedule) => + log.info(s"$scheduleId is due. Adding schedule to queue. Scheduled time was ${schedule.time}") + sourceQueue.offer((scheduleId, messageFrom(schedule))) + } + + (receiveSchedulingMessage andThen updateStateAndAck) orElse receiveTriggerMessage orElse { case Init => sender ! Ack } } @@ -75,6 +78,8 @@ object SchedulingActor { case class Cancel(scheduleId: ScheduleId) extends SchedulingMessage + case class TriggerMessage(scheduleId: ScheduleId, schedule: Schedule) + case object Init case object Ack diff --git a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/avro/package.scala b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/avro/package.scala index b69d2f51..421349ed 100644 --- a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/avro/package.scala +++ b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/avro/package.scala @@ -1,6 +1,7 @@ package com.sky.kafka.message.scheduler import java.time.OffsetDateTime +import java.time.format.DateTimeFormatter import com.sksamuel.avro4s.{FromValue, ToSchema, ToValue} import org.apache.avro.Schema @@ -8,15 +9,17 @@ import org.apache.avro.Schema.Field package object avro { + private val dateTimeFormatter = DateTimeFormatter.ISO_OFFSET_DATE_TIME + implicit val offsetDateTimeToSchema = new ToSchema[OffsetDateTime] { override val schema: Schema = Schema.create(Schema.Type.STRING) } implicit val offsetDateTimeFromValue = new FromValue[OffsetDateTime] { - override def apply(value: Any, field: Field): OffsetDateTime = OffsetDateTime.parse(value.toString) + override def apply(value: Any, field: Field): OffsetDateTime = OffsetDateTime.parse(value.toString, dateTimeFormatter) } implicit val offsetDateTimeToValue = new ToValue[OffsetDateTime] { - override def apply(value: OffsetDateTime): String = value.toString + override def apply(value: OffsetDateTime): String = value.format(dateTimeFormatter) } } diff --git a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/config/package.scala b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/config/package.scala index 48cbfcf9..5b55610d 100644 --- a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/config/package.scala +++ b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/config/package.scala @@ -8,7 +8,7 @@ import scala.concurrent.duration.Duration package object config { - case class AppConfig(scheduler: SchedulerConfig)(implicit system: ActorSystem, materialzer: ActorMaterializer) + case class AppConfig(scheduler: SchedulerConfig)(implicit system: ActorSystem, materializer: ActorMaterializer) case class SchedulerConfig(scheduleTopic: String, shutdownTimeout: ShutdownTimeout, queueBufferSize: Int) diff --git a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/domain/ApplicationError.scala b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/domain/ApplicationError.scala index 65d1a848..23c24d08 100644 --- a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/domain/ApplicationError.scala +++ b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/domain/ApplicationError.scala @@ -14,22 +14,19 @@ object ApplicationError extends LazyLogging { case class InvalidSchemaError(key: String) extends ApplicationError(key) - implicit val invalidSchemaErrorShow: Show[InvalidSchemaError] = + implicit val showInvalidSchemaError: Show[InvalidSchemaError] = show(error => s"Invalid schema used to produce message with key: ${error.key}") - case class AvroMessageFormatError(key: String, cause: Throwable) extends ApplicationError(key) - implicit val avroMessageFormatErrorShow: Show[AvroMessageFormatError] = + implicit val showAvroMessageFormatError: Show[AvroMessageFormatError] = show(error => s"Error when processing message with key: ${error.key}. Error message: ${error.cause.getMessage}") - implicit val showError: Show[ApplicationError] = show { - case schemaError: InvalidSchemaError => invalidSchemaErrorShow.show(schemaError) - case messageFormatError: AvroMessageFormatError => avroMessageFormatErrorShow.show(messageFormatError) + case schemaError: InvalidSchemaError => showInvalidSchemaError.show(schemaError) + case messageFormatError: AvroMessageFormatError => showAvroMessageFormatError.show(messageFormatError) } - //TODO: is this enough for now? implicit val errorSink: Sink[ApplicationError, Future[Done]] = Sink.foreach(error => logger.warn(error.show)) } diff --git a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/kafka/ConsumerRecordDecoder.scala b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/kafka/ConsumerRecordDecoder.scala index 66e29361..ea07559a 100644 --- a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/kafka/ConsumerRecordDecoder.scala +++ b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/kafka/ConsumerRecordDecoder.scala @@ -5,3 +5,10 @@ import org.apache.kafka.clients.consumer.ConsumerRecord trait ConsumerRecordDecoder[T] { def apply(cr: ConsumerRecord[String, Array[Byte]]): T } + +object ConsumerRecordDecoder { + def instance[T](f: ConsumerRecord[String, Array[Byte]] => T) = new ConsumerRecordDecoder[T] { + final def apply(cr: ConsumerRecord[String, Array[Byte]]): T = + f(cr) + } +} diff --git a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/package.scala b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/package.scala index a4b20885..83d80e98 100644 --- a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/package.scala +++ b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/package.scala @@ -13,14 +13,10 @@ import scala.util.Try package object scheduler extends LazyLogging { - type DecodeResult = Either[ApplicationError, (ScheduleId, Option[Schedule])] + implicit val scheduleConsumerRecordDecoder: ConsumerRecordDecoder[Either[ApplicationError, (ScheduleId, Option[Schedule])]] = + ConsumerRecordDecoder.instance(consumerRecordDecoder) - implicit val scheduleConsumerRecordDecoder = new ConsumerRecordDecoder[DecodeResult] { - def apply(cr: ConsumerRecord[String, Array[Byte]]): DecodeResult = - consumerRecordDecoder(cr) - } - - def consumerRecordDecoder(cr: ConsumerRecord[String, Array[Byte]]): DecodeResult = + def consumerRecordDecoder(cr: ConsumerRecord[String, Array[Byte]]): Either[ApplicationError, (ScheduleId, Option[Schedule])] = Option(cr.value) match { case Some(bytes) => for { diff --git a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/ScheduleReader.scala b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/ScheduleReader.scala index 4c5d43c4..0bf60d57 100644 --- a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/ScheduleReader.scala +++ b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/ScheduleReader.scala @@ -9,11 +9,16 @@ import cats.data.Reader import com.sky.kafka.message.scheduler.SchedulingActor._ import com.sky.kafka.message.scheduler._ import com.sky.kafka.message.scheduler.config.{AppConfig, SchedulerConfig} -import com.sky.kafka.message.scheduler.domain.ApplicationError +import com.sky.kafka.message.scheduler.domain.{ApplicationError, Schedule, ScheduleId} import com.sky.kafka.message.scheduler.kafka._ import com.typesafe.scalalogging.LazyLogging -case class ScheduleReader(config: SchedulerConfig, scheduleSource: Source[DecodeResult, Control], schedulingSink: Sink[Any, NotUsed]) +/** + * Provides stream from the schedule source to the scheduling actor. + */ +case class ScheduleReader(config: SchedulerConfig, + scheduleSource: Source[Either[ApplicationError, (ScheduleId, Option[Schedule])], Control], + schedulingSink: Sink[Any, NotUsed]) (implicit system: ActorSystem, materializer: ActorMaterializer) extends ScheduleReaderStream { val stream: Control = @@ -25,8 +30,8 @@ case class ScheduleReader(config: SchedulerConfig, scheduleSource: Source[Decode object ScheduleReader extends LazyLogging { - def toSchedulingMessage(decodeResult: DecodeResult): Either[ApplicationError, SchedulingMessage] = - decodeResult.map { case (scheduleId, scheduleOpt) => + def toSchedulingMessage[T](either: Either[ApplicationError, (ScheduleId, Option[Schedule])]): Either[ApplicationError, SchedulingMessage] = + either.map { case (scheduleId, scheduleOpt) => scheduleOpt match { case Some(schedule) => logger.info(s"Publishing scheduled message with ID: $scheduleId to topic: ${schedule.topic}") diff --git a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/ScheduledMessagePublisher.scala b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/ScheduledMessagePublisher.scala index 9b730c83..3b448e70 100644 --- a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/ScheduledMessagePublisher.scala +++ b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/ScheduledMessagePublisher.scala @@ -14,6 +14,9 @@ import org.apache.kafka.clients.producer.ProducerRecord import scala.concurrent.Future +/** + * Provides stream from the queue of due messages to the sink + */ case class ScheduledMessagePublisher(config: SchedulerConfig, publisherSink: Sink[In, Mat]) (implicit system: ActorSystem, materializer: ActorMaterializer) extends ScheduledMessagePublisherStream { diff --git a/scheduler/src/test/scala/com/sky/kafka/message/scheduler/SchedulingActorSpec.scala b/scheduler/src/test/scala/com/sky/kafka/message/scheduler/SchedulingActorSpec.scala index 53e7ac4e..9f25ee8b 100644 --- a/scheduler/src/test/scala/com/sky/kafka/message/scheduler/SchedulingActorSpec.scala +++ b/scheduler/src/test/scala/com/sky/kafka/message/scheduler/SchedulingActorSpec.scala @@ -60,7 +60,7 @@ class SchedulingActorSpec extends AkkaBaseSpec with ImplicitSender with MockitoS verify(mockSourceQueue).offer((scheduleId, updatedSchedule.toScheduledMessage)) } - "send an Ack to the sender when receiving an Init message" in new SchedulingActorTest { + "does nothing when an Init message is received" in new SchedulingActorTest { actorRef ! Init expectMsg(Ack) } diff --git a/scheduler/src/test/scala/common/EmbeddedKafka.scala b/scheduler/src/test/scala/common/EmbeddedKafka.scala index a89be042..68fd7013 100644 --- a/scheduler/src/test/scala/common/EmbeddedKafka.scala +++ b/scheduler/src/test/scala/common/EmbeddedKafka.scala @@ -14,6 +14,13 @@ object EmbeddedKafka { val bootstrapServer = s"localhost:${kafkaServer.kafkaPort}" + /* The consume method provided by [cakesolutions kafka testkit](https://github.com/cakesolutions/scala-kafka-client) + * doesn't provide a way of extracting a consumer record, we have added this so we can access the timestamp + * from a consumer record. + * + * This should be contributed to the library. + * + */ implicit class KafkaServerOps(val kafkaServer: KafkaServer) extends AnyVal { def consumeRecord[Key, Value, T]( From a81702d39617ce18e5867f3cb0b04cc0cd4439ce Mon Sep 17 00:00:00 2001 From: Lawrence Carvalho Date: Tue, 8 Aug 2017 14:50:27 +0100 Subject: [PATCH 16/22] Replace EitherFanOut with PartitionWith. --- build.sbt | 1 + .../scheduler/streams/EitherFanOut.scala | 60 ------------------- .../scheduler/streams/PartitionedSink.scala | 9 +-- 3 files changed, 6 insertions(+), 64 deletions(-) delete mode 100644 scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/EitherFanOut.scala diff --git a/build.sbt b/build.sbt index 859f71e8..305973a2 100644 --- a/build.sbt +++ b/build.sbt @@ -9,6 +9,7 @@ val dependencies = Seq( "com.typesafe.akka" %% "akka-stream" % akkaVersion, "com.typesafe.akka" %% "akka-slf4j" % akkaVersion, "com.typesafe.akka" %% "akka-stream-kafka" % "0.16", + "com.typesafe.akka" %% "akka-stream-contrib" % "0.8", "com.typesafe.scala-logging" %% "scala-logging" % "3.5.0", "com.sksamuel.avro4s" %% "avro4s-core" % "1.7.0", diff --git a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/EitherFanOut.scala b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/EitherFanOut.scala deleted file mode 100644 index 127ba480..00000000 --- a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/EitherFanOut.scala +++ /dev/null @@ -1,60 +0,0 @@ -package com.sky.kafka.message.scheduler.streams - -import akka.stream.FanOutShape2 -import akka.stream.stage.{GraphStage, InHandler, OutHandler} - -//Taken from: https://stackoverflow.com/a/38445121/8424807 -class EitherFanOut[L, R] extends GraphStage[FanOutShape2[Either[L, R], L, R]] { - import akka.stream.Attributes - import akka.stream.stage.GraphStageLogic - - override val shape: FanOutShape2[Either[L, R], L, R] = new FanOutShape2[Either[L, R], L, R]("EitherFanOut") - - override def createLogic(inheritedAttributes: Attributes): GraphStageLogic = new GraphStageLogic(shape) { - - var out0demand = false - var out1demand = false - - setHandler(shape.in, new InHandler { - override def onPush(): Unit = { - - if (out0demand && out1demand) { - grab(shape.in) match { - case Left(l) => - out0demand = false - push(shape.out0, l) - case Right(r) => - out1demand = false - push(shape.out1, r) - } - } - } - }) - - setHandler(shape.out0, new OutHandler { - @scala.throws[Exception](classOf[Exception]) - override def onPull(): Unit = { - if (!out0demand) { - out0demand = true - } - - if (out0demand && out1demand) { - pull(shape.in) - } - } - }) - - setHandler(shape.out1, new OutHandler { - @scala.throws[Exception](classOf[Exception]) - override def onPull(): Unit = { - if (!out1demand) { - out1demand = true - } - - if (out0demand && out1demand) { - pull(shape.in) - } - } - }) - } -} \ No newline at end of file diff --git a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/PartitionedSink.scala b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/PartitionedSink.scala index ddcca1e4..c2d91017 100644 --- a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/PartitionedSink.scala +++ b/scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/PartitionedSink.scala @@ -1,6 +1,7 @@ package com.sky.kafka.message.scheduler.streams import akka.stream.SinkShape +import akka.stream.contrib.PartitionWith import akka.stream.scaladsl.{GraphDSL, Sink} object PartitionedSink { @@ -10,11 +11,11 @@ object PartitionedSink { (left, right) => import GraphDSL.Implicits._ - val eitherFanOut = b.add(new EitherFanOut[A, B]) + val partition = b.add(PartitionWith[Either[A, B], A, B](identity)) - eitherFanOut.out0 ~> left.in - eitherFanOut.out1 ~> right.in + partition.out0 ~> left.in + partition.out1 ~> right.in - SinkShape(eitherFanOut.in) + SinkShape(partition.in) }) } From 33e3e35eaa935ae9bb70e094e34d02cd9b4b92c3 Mon Sep 17 00:00:00 2001 From: Lawrence Carvalho Date: Tue, 8 Aug 2017 15:24:11 +0100 Subject: [PATCH 17/22] Rename package. --- avro/src/main/scala/GenerateSchema.scala | 2 +- scheduler/src/main/resources/application.conf | 3 ++- .../scheduler/AkkaComponents.scala | 2 +- .../scheduler/SchedulerApp.scala | 6 ++--- .../scheduler/SchedulingActor.scala | 22 +++++++++---------- .../scheduler/avro/package.scala | 2 +- .../scheduler/config/package.scala | 2 +- .../scheduler/domain/ApplicationError.scala | 3 ++- .../scheduler/domain/PublishableMessage.scala | 4 ++-- .../scheduler/domain/package.scala | 2 +- .../kafka/ConsumerRecordDecoder.scala | 2 +- .../scheduler/kafka/KafkaStream.scala | 4 ++-- .../kafka/ProducerRecordEncoder.scala | 2 +- .../scheduler/package.scala | 10 ++++----- .../scheduler/streams/PartitionedSink.scala | 2 +- .../scheduler/streams/ScheduleReader.scala | 12 +++++----- .../streams/ScheduleReaderStream.scala | 4 ++-- .../streams/ScheduledMessagePublisher.scala | 14 ++++++------ .../ScheduledMessagePublisherStream.scala | 8 +++---- .../scheduler/AkkaComponentsSpec.scala | 2 +- .../scheduler/PartitionedSinkSpec.scala | 4 ++-- .../scheduler/ScheduleReaderSpec.scala | 6 ++--- .../ScheduledMessagePublisherSpec.scala | 10 ++++----- .../scheduler/SchedulerSpec.scala | 6 ++--- .../scheduler/SchedulingActorSpec.scala | 8 +++---- .../scheduler/avro/AvroSpec.scala | 2 +- .../scheduler/e2e/SchedulerIntSpec.scala | 10 ++++----- .../src/test/scala/common/TestDataUtils.scala | 6 ++--- 28 files changed, 80 insertions(+), 80 deletions(-) rename scheduler/src/main/scala/com/sky/{kafka/message => kafkamessage}/scheduler/AkkaComponents.scala (93%) rename scheduler/src/main/scala/com/sky/{kafka/message => kafkamessage}/scheduler/SchedulerApp.scala (79%) rename scheduler/src/main/scala/com/sky/{kafka/message => kafkamessage}/scheduler/SchedulingActor.scala (81%) rename scheduler/src/main/scala/com/sky/{kafka/message => kafkamessage}/scheduler/avro/package.scala (95%) rename scheduler/src/main/scala/com/sky/{kafka/message => kafkamessage}/scheduler/config/package.scala (93%) rename scheduler/src/main/scala/com/sky/{kafka/message => kafkamessage}/scheduler/domain/ApplicationError.scala (96%) rename scheduler/src/main/scala/com/sky/{kafka/message => kafkamessage}/scheduler/domain/PublishableMessage.scala (90%) rename scheduler/src/main/scala/com/sky/{kafka/message => kafkamessage}/scheduler/domain/package.scala (82%) rename scheduler/src/main/scala/com/sky/{kafka/message => kafkamessage}/scheduler/kafka/ConsumerRecordDecoder.scala (88%) rename scheduler/src/main/scala/com/sky/{kafka/message => kafkamessage}/scheduler/kafka/KafkaStream.scala (90%) rename scheduler/src/main/scala/com/sky/{kafka/message => kafkamessage}/scheduler/kafka/ProducerRecordEncoder.scala (89%) rename scheduler/src/main/scala/com/sky/{kafka/message => kafkamessage}/scheduler/package.scala (82%) rename scheduler/src/main/scala/com/sky/{kafka/message => kafkamessage}/scheduler/streams/PartitionedSink.scala (92%) rename scheduler/src/main/scala/com/sky/{kafka/message => kafkamessage}/scheduler/streams/ScheduleReader.scala (83%) rename scheduler/src/main/scala/com/sky/{kafka/message => kafkamessage}/scheduler/streams/ScheduleReaderStream.scala (78%) rename scheduler/src/main/scala/com/sky/{kafka/message => kafkamessage}/scheduler/streams/ScheduledMessagePublisher.scala (78%) rename scheduler/src/main/scala/com/sky/{kafka/message => kafkamessage}/scheduler/streams/ScheduledMessagePublisherStream.scala (68%) rename scheduler/src/test/scala/com/sky/{kafka/message => kafkamessage}/scheduler/AkkaComponentsSpec.scala (92%) rename scheduler/src/test/scala/com/sky/{kafka/message => kafkamessage}/scheduler/PartitionedSinkSpec.scala (88%) rename scheduler/src/test/scala/com/sky/{kafka/message => kafkamessage}/scheduler/ScheduleReaderSpec.scala (83%) rename scheduler/src/test/scala/com/sky/{kafka/message => kafkamessage}/scheduler/ScheduledMessagePublisherSpec.scala (75%) rename scheduler/src/test/scala/com/sky/{kafka/message => kafkamessage}/scheduler/SchedulerSpec.scala (92%) rename scheduler/src/test/scala/com/sky/{kafka/message => kafkamessage}/scheduler/SchedulingActorSpec.scala (93%) rename scheduler/src/test/scala/com/sky/{kafka/message => kafkamessage}/scheduler/avro/AvroSpec.scala (96%) rename scheduler/src/test/scala/com/sky/{kafka/message => kafkamessage}/scheduler/e2e/SchedulerIntSpec.scala (86%) diff --git a/avro/src/main/scala/GenerateSchema.scala b/avro/src/main/scala/GenerateSchema.scala index 3ccae207..654b0f56 100644 --- a/avro/src/main/scala/GenerateSchema.scala +++ b/avro/src/main/scala/GenerateSchema.scala @@ -2,7 +2,7 @@ import java.nio.charset.StandardCharsets import java.nio.file.{Files, Paths} import com.sksamuel.avro4s._ -import com.sky.kafka.message.scheduler.domain.Schedule +import com.sky.kafkamessage.scheduler.domain.Schedule object GenerateSchema extends App { diff --git a/scheduler/src/main/resources/application.conf b/scheduler/src/main/resources/application.conf index 39f38193..5bec25d7 100644 --- a/scheduler/src/main/resources/application.conf +++ b/scheduler/src/main/resources/application.conf @@ -25,7 +25,8 @@ akka { use-dispatcher = "akka.kafka.default-dispatcher" kafka-clients { - enable.auto.commit = false + // until we use a commitableSource + enable.auto.commit = true group.id = "com.sky.kafka.scheduler" bootstrap.servers = ${scheduler.kafka-brokers} diff --git a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/AkkaComponents.scala b/scheduler/src/main/scala/com/sky/kafkamessage/scheduler/AkkaComponents.scala similarity index 93% rename from scheduler/src/main/scala/com/sky/kafka/message/scheduler/AkkaComponents.scala rename to scheduler/src/main/scala/com/sky/kafkamessage/scheduler/AkkaComponents.scala index fca98995..591ed9cb 100644 --- a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/AkkaComponents.scala +++ b/scheduler/src/main/scala/com/sky/kafkamessage/scheduler/AkkaComponents.scala @@ -1,4 +1,4 @@ -package com.sky.kafka.message.scheduler +package com.sky.kafkamessage.scheduler import akka.actor.ActorSystem import akka.stream.Supervision.Restart diff --git a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/SchedulerApp.scala b/scheduler/src/main/scala/com/sky/kafkamessage/scheduler/SchedulerApp.scala similarity index 79% rename from scheduler/src/main/scala/com/sky/kafka/message/scheduler/SchedulerApp.scala rename to scheduler/src/main/scala/com/sky/kafkamessage/scheduler/SchedulerApp.scala index 7ac8fd3b..659f8171 100644 --- a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/SchedulerApp.scala +++ b/scheduler/src/main/scala/com/sky/kafkamessage/scheduler/SchedulerApp.scala @@ -1,7 +1,7 @@ -package com.sky.kafka.message.scheduler +package com.sky.kafkamessage.scheduler -import com.sky.kafka.message.scheduler.config.AppConfig -import com.sky.kafka.message.scheduler.streams.ScheduleReader +import com.sky.kafkamessage.scheduler.config.AppConfig +import com.sky.kafkamessage.scheduler.streams.ScheduleReader import com.typesafe.scalalogging.LazyLogging import kamon.Kamon import org.zalando.grafter._ diff --git a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/SchedulingActor.scala b/scheduler/src/main/scala/com/sky/kafkamessage/scheduler/SchedulingActor.scala similarity index 81% rename from scheduler/src/main/scala/com/sky/kafka/message/scheduler/SchedulingActor.scala rename to scheduler/src/main/scala/com/sky/kafkamessage/scheduler/SchedulingActor.scala index f7be76e9..5a90edd4 100644 --- a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/SchedulingActor.scala +++ b/scheduler/src/main/scala/com/sky/kafkamessage/scheduler/SchedulingActor.scala @@ -1,23 +1,21 @@ -package com.sky.kafka.message.scheduler +package com.sky.kafkamessage.scheduler import java.time.OffsetDateTime import java.time.temporal.ChronoUnit import java.util.concurrent.TimeUnit import akka.actor._ -import akka.stream.{ActorMaterializer, QueueOfferResult} +import akka.stream.ActorMaterializer import akka.stream.scaladsl.SourceQueue import cats.data.Reader -import com.sky.kafka.message.scheduler.SchedulingActor._ -import com.sky.kafka.message.scheduler.config.AppConfig -import com.sky.kafka.message.scheduler.domain.PublishableMessage.ScheduledMessage -import com.sky.kafka.message.scheduler.domain._ -import com.sky.kafka.message.scheduler.streams.ScheduledMessagePublisher +import com.sky.kafkamessage.scheduler.SchedulingActor._ +import com.sky.kafkamessage.scheduler.config.AppConfig +import com.sky.kafkamessage.scheduler.domain.PublishableMessage.ScheduledMessage +import com.sky.kafkamessage.scheduler.domain._ +import com.sky.kafkamessage.scheduler.streams.ScheduledMessagePublisher import scala.concurrent.ExecutionContext.Implicits.global import scala.concurrent.duration.FiniteDuration -import scala.concurrent.Future -import akka.pattern.pipe class SchedulingActor(sourceQueue: SourceQueue[(String, ScheduledMessage)], scheduler: Scheduler) extends Actor with ActorLogging { @@ -31,7 +29,7 @@ class SchedulingActor(sourceQueue: SourceQueue[(String, ScheduledMessage)], sche log.info(s"Updating schedule $scheduleId") else log.info(s"Creating schedule $scheduleId") - val cancellable = scheduler.scheduleOnce(timeFromNow(schedule.time))(self ! TriggerMessage(scheduleId, schedule)) + val cancellable = scheduler.scheduleOnce(timeFromNow(schedule.time))(self ! Trigger(scheduleId, schedule)) schedules + (scheduleId -> cancellable) case Cancel(scheduleId: String) => @@ -43,7 +41,7 @@ class SchedulingActor(sourceQueue: SourceQueue[(String, ScheduledMessage)], sche } val receiveTriggerMessage: PartialFunction[Any, Unit] = { - case TriggerMessage(scheduleId, schedule) => + case Trigger(scheduleId, schedule) => log.info(s"$scheduleId is due. Adding schedule to queue. Scheduled time was ${schedule.time}") sourceQueue.offer((scheduleId, messageFrom(schedule))) } @@ -78,7 +76,7 @@ object SchedulingActor { case class Cancel(scheduleId: ScheduleId) extends SchedulingMessage - case class TriggerMessage(scheduleId: ScheduleId, schedule: Schedule) + private case class Trigger(scheduleId: ScheduleId, schedule: Schedule) case object Init diff --git a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/avro/package.scala b/scheduler/src/main/scala/com/sky/kafkamessage/scheduler/avro/package.scala similarity index 95% rename from scheduler/src/main/scala/com/sky/kafka/message/scheduler/avro/package.scala rename to scheduler/src/main/scala/com/sky/kafkamessage/scheduler/avro/package.scala index 421349ed..1216e23f 100644 --- a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/avro/package.scala +++ b/scheduler/src/main/scala/com/sky/kafkamessage/scheduler/avro/package.scala @@ -1,4 +1,4 @@ -package com.sky.kafka.message.scheduler +package com.sky.kafkamessage.scheduler import java.time.OffsetDateTime import java.time.format.DateTimeFormatter diff --git a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/config/package.scala b/scheduler/src/main/scala/com/sky/kafkamessage/scheduler/config/package.scala similarity index 93% rename from scheduler/src/main/scala/com/sky/kafka/message/scheduler/config/package.scala rename to scheduler/src/main/scala/com/sky/kafkamessage/scheduler/config/package.scala index 5b55610d..533c7141 100644 --- a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/config/package.scala +++ b/scheduler/src/main/scala/com/sky/kafkamessage/scheduler/config/package.scala @@ -1,4 +1,4 @@ -package com.sky.kafka.message.scheduler +package com.sky.kafkamessage.scheduler import akka.actor.ActorSystem import akka.stream.ActorMaterializer diff --git a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/domain/ApplicationError.scala b/scheduler/src/main/scala/com/sky/kafkamessage/scheduler/domain/ApplicationError.scala similarity index 96% rename from scheduler/src/main/scala/com/sky/kafka/message/scheduler/domain/ApplicationError.scala rename to scheduler/src/main/scala/com/sky/kafkamessage/scheduler/domain/ApplicationError.scala index 23c24d08..28052dc7 100644 --- a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/domain/ApplicationError.scala +++ b/scheduler/src/main/scala/com/sky/kafkamessage/scheduler/domain/ApplicationError.scala @@ -1,4 +1,5 @@ -package com.sky.kafka.message.scheduler.domain +package com.sky.kafkamessage.scheduler.domain + import akka.Done import akka.stream.scaladsl.Sink import cats.Show diff --git a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/domain/PublishableMessage.scala b/scheduler/src/main/scala/com/sky/kafkamessage/scheduler/domain/PublishableMessage.scala similarity index 90% rename from scheduler/src/main/scala/com/sky/kafka/message/scheduler/domain/PublishableMessage.scala rename to scheduler/src/main/scala/com/sky/kafkamessage/scheduler/domain/PublishableMessage.scala index 2f69f9aa..0fd322ff 100644 --- a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/domain/PublishableMessage.scala +++ b/scheduler/src/main/scala/com/sky/kafkamessage/scheduler/domain/PublishableMessage.scala @@ -1,6 +1,6 @@ -package com.sky.kafka.message.scheduler.domain +package com.sky.kafkamessage.scheduler.domain -import com.sky.kafka.message.scheduler.kafka.ProducerRecordEncoder +import com.sky.kafkamessage.scheduler.kafka.ProducerRecordEncoder import org.apache.kafka.clients.producer.ProducerRecord sealed trait PublishableMessage diff --git a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/domain/package.scala b/scheduler/src/main/scala/com/sky/kafkamessage/scheduler/domain/package.scala similarity index 82% rename from scheduler/src/main/scala/com/sky/kafka/message/scheduler/domain/package.scala rename to scheduler/src/main/scala/com/sky/kafkamessage/scheduler/domain/package.scala index 73a3b8ff..6b36913a 100644 --- a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/domain/package.scala +++ b/scheduler/src/main/scala/com/sky/kafkamessage/scheduler/domain/package.scala @@ -1,4 +1,4 @@ -package com.sky.kafka.message.scheduler +package com.sky.kafkamessage.scheduler import java.time.OffsetDateTime diff --git a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/kafka/ConsumerRecordDecoder.scala b/scheduler/src/main/scala/com/sky/kafkamessage/scheduler/kafka/ConsumerRecordDecoder.scala similarity index 88% rename from scheduler/src/main/scala/com/sky/kafka/message/scheduler/kafka/ConsumerRecordDecoder.scala rename to scheduler/src/main/scala/com/sky/kafkamessage/scheduler/kafka/ConsumerRecordDecoder.scala index ea07559a..3827d452 100644 --- a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/kafka/ConsumerRecordDecoder.scala +++ b/scheduler/src/main/scala/com/sky/kafkamessage/scheduler/kafka/ConsumerRecordDecoder.scala @@ -1,4 +1,4 @@ -package com.sky.kafka.message.scheduler.kafka +package com.sky.kafkamessage.scheduler.kafka import org.apache.kafka.clients.consumer.ConsumerRecord diff --git a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/kafka/KafkaStream.scala b/scheduler/src/main/scala/com/sky/kafkamessage/scheduler/kafka/KafkaStream.scala similarity index 90% rename from scheduler/src/main/scala/com/sky/kafka/message/scheduler/kafka/KafkaStream.scala rename to scheduler/src/main/scala/com/sky/kafkamessage/scheduler/kafka/KafkaStream.scala index cc81b363..121614da 100644 --- a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/kafka/KafkaStream.scala +++ b/scheduler/src/main/scala/com/sky/kafkamessage/scheduler/kafka/KafkaStream.scala @@ -1,4 +1,4 @@ -package com.sky.kafka.message.scheduler.kafka +package com.sky.kafkamessage.scheduler.kafka import akka.Done import akka.actor.ActorSystem @@ -6,7 +6,7 @@ import akka.kafka.scaladsl.Consumer.Control import akka.kafka.scaladsl.{Consumer, Producer} import akka.kafka.{ConsumerSettings, ProducerSettings, Subscriptions} import akka.stream.scaladsl.{Sink, Source} -import com.sky.kafka.message.scheduler.config.SchedulerConfig +import com.sky.kafkamessage.scheduler.config.SchedulerConfig import org.apache.kafka.clients.producer.ProducerRecord import org.apache.kafka.common.serialization.{ByteArrayDeserializer, ByteArraySerializer, StringDeserializer} diff --git a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/kafka/ProducerRecordEncoder.scala b/scheduler/src/main/scala/com/sky/kafkamessage/scheduler/kafka/ProducerRecordEncoder.scala similarity index 89% rename from scheduler/src/main/scala/com/sky/kafka/message/scheduler/kafka/ProducerRecordEncoder.scala rename to scheduler/src/main/scala/com/sky/kafkamessage/scheduler/kafka/ProducerRecordEncoder.scala index 021ffb70..b9dd0e33 100644 --- a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/kafka/ProducerRecordEncoder.scala +++ b/scheduler/src/main/scala/com/sky/kafkamessage/scheduler/kafka/ProducerRecordEncoder.scala @@ -1,4 +1,4 @@ -package com.sky.kafka.message.scheduler.kafka +package com.sky.kafkamessage.scheduler.kafka import org.apache.kafka.clients.producer.ProducerRecord diff --git a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/package.scala b/scheduler/src/main/scala/com/sky/kafkamessage/scheduler/package.scala similarity index 82% rename from scheduler/src/main/scala/com/sky/kafka/message/scheduler/package.scala rename to scheduler/src/main/scala/com/sky/kafkamessage/scheduler/package.scala index 83d80e98..f94efd80 100644 --- a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/package.scala +++ b/scheduler/src/main/scala/com/sky/kafkamessage/scheduler/package.scala @@ -1,13 +1,13 @@ -package com.sky.kafka.message +package com.sky.kafkamessage import cats.syntax.either._ import com.sksamuel.avro4s.AvroInputStream -import com.sky.kafka.message.scheduler.domain.ApplicationError._ -import com.sky.kafka.message.scheduler.domain._ -import com.sky.kafka.message.scheduler.kafka.ConsumerRecordDecoder +import com.sky.kafkamessage.scheduler.domain.ApplicationError._ +import com.sky.kafkamessage.scheduler.domain._ import com.typesafe.scalalogging.LazyLogging import org.apache.kafka.clients.consumer.ConsumerRecord -import com.sky.kafka.message.scheduler.avro._ +import com.sky.kafkamessage.scheduler.avro._ +import com.sky.kafkamessage.scheduler.kafka.ConsumerRecordDecoder import scala.util.Try diff --git a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/PartitionedSink.scala b/scheduler/src/main/scala/com/sky/kafkamessage/scheduler/streams/PartitionedSink.scala similarity index 92% rename from scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/PartitionedSink.scala rename to scheduler/src/main/scala/com/sky/kafkamessage/scheduler/streams/PartitionedSink.scala index c2d91017..4c631570 100644 --- a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/PartitionedSink.scala +++ b/scheduler/src/main/scala/com/sky/kafkamessage/scheduler/streams/PartitionedSink.scala @@ -1,4 +1,4 @@ -package com.sky.kafka.message.scheduler.streams +package com.sky.kafkamessage.scheduler.streams import akka.stream.SinkShape import akka.stream.contrib.PartitionWith diff --git a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/ScheduleReader.scala b/scheduler/src/main/scala/com/sky/kafkamessage/scheduler/streams/ScheduleReader.scala similarity index 83% rename from scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/ScheduleReader.scala rename to scheduler/src/main/scala/com/sky/kafkamessage/scheduler/streams/ScheduleReader.scala index 0bf60d57..cf31678a 100644 --- a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/ScheduleReader.scala +++ b/scheduler/src/main/scala/com/sky/kafkamessage/scheduler/streams/ScheduleReader.scala @@ -1,4 +1,4 @@ -package com.sky.kafka.message.scheduler.streams +package com.sky.kafkamessage.scheduler.streams import akka.actor.ActorSystem import akka.kafka.scaladsl.Consumer.Control @@ -6,11 +6,11 @@ import akka.stream._ import akka.stream.scaladsl.{Sink, Source} import akka.{Done, NotUsed} import cats.data.Reader -import com.sky.kafka.message.scheduler.SchedulingActor._ -import com.sky.kafka.message.scheduler._ -import com.sky.kafka.message.scheduler.config.{AppConfig, SchedulerConfig} -import com.sky.kafka.message.scheduler.domain.{ApplicationError, Schedule, ScheduleId} -import com.sky.kafka.message.scheduler.kafka._ +import com.sky.kafkamessage.scheduler.SchedulingActor._ +import com.sky.kafkamessage.scheduler._ +import com.sky.kafkamessage.scheduler.config.{AppConfig, SchedulerConfig} +import com.sky.kafkamessage.scheduler.domain.{ApplicationError, Schedule, ScheduleId} +import com.sky.kafkamessage.scheduler.kafka._ import com.typesafe.scalalogging.LazyLogging /** diff --git a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/ScheduleReaderStream.scala b/scheduler/src/main/scala/com/sky/kafkamessage/scheduler/streams/ScheduleReaderStream.scala similarity index 78% rename from scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/ScheduleReaderStream.scala rename to scheduler/src/main/scala/com/sky/kafkamessage/scheduler/streams/ScheduleReaderStream.scala index b233436a..0b2d2a66 100644 --- a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/ScheduleReaderStream.scala +++ b/scheduler/src/main/scala/com/sky/kafkamessage/scheduler/streams/ScheduleReaderStream.scala @@ -1,8 +1,8 @@ -package com.sky.kafka.message.scheduler.streams +package com.sky.kafkamessage.scheduler.streams import akka.kafka.scaladsl.Consumer.Control import cats.Eval -import com.sky.kafka.message.scheduler.config.SchedulerConfig +import com.sky.kafkamessage.scheduler.config.SchedulerConfig import org.zalando.grafter.{Stop, StopResult} import scala.concurrent.Await diff --git a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/ScheduledMessagePublisher.scala b/scheduler/src/main/scala/com/sky/kafkamessage/scheduler/streams/ScheduledMessagePublisher.scala similarity index 78% rename from scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/ScheduledMessagePublisher.scala rename to scheduler/src/main/scala/com/sky/kafkamessage/scheduler/streams/ScheduledMessagePublisher.scala index 3b448e70..63298497 100644 --- a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/ScheduledMessagePublisher.scala +++ b/scheduler/src/main/scala/com/sky/kafkamessage/scheduler/streams/ScheduledMessagePublisher.scala @@ -1,21 +1,21 @@ -package com.sky.kafka.message.scheduler.streams +package com.sky.kafkamessage.scheduler.streams import akka.Done import akka.actor.ActorSystem import akka.stream.scaladsl.{Sink, Source, SourceQueueWithComplete} import akka.stream.{ActorMaterializer, OverflowStrategy} import cats.data.Reader -import com.sky.kafka.message.scheduler.config._ -import com.sky.kafka.message.scheduler.domain.PublishableMessage._ -import com.sky.kafka.message.scheduler.domain._ -import com.sky.kafka.message.scheduler.kafka.KafkaStream -import com.sky.kafka.message.scheduler.streams.ScheduledMessagePublisher._ +import com.sky.kafkamessage.scheduler.config._ +import com.sky.kafkamessage.scheduler.domain.PublishableMessage._ +import com.sky.kafkamessage.scheduler.domain._ +import com.sky.kafkamessage.scheduler.streams.ScheduledMessagePublisher._ +import com.sky.kafkamessage.scheduler.kafka.KafkaStream import org.apache.kafka.clients.producer.ProducerRecord import scala.concurrent.Future /** - * Provides stream from the queue of due messages to the sink + * Provides stream from the queue of due messages to kafka */ case class ScheduledMessagePublisher(config: SchedulerConfig, publisherSink: Sink[In, Mat]) (implicit system: ActorSystem, materializer: ActorMaterializer) diff --git a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/ScheduledMessagePublisherStream.scala b/scheduler/src/main/scala/com/sky/kafkamessage/scheduler/streams/ScheduledMessagePublisherStream.scala similarity index 68% rename from scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/ScheduledMessagePublisherStream.scala rename to scheduler/src/main/scala/com/sky/kafkamessage/scheduler/streams/ScheduledMessagePublisherStream.scala index 45ce6539..16b50c2c 100644 --- a/scheduler/src/main/scala/com/sky/kafka/message/scheduler/streams/ScheduledMessagePublisherStream.scala +++ b/scheduler/src/main/scala/com/sky/kafkamessage/scheduler/streams/ScheduledMessagePublisherStream.scala @@ -1,10 +1,10 @@ -package com.sky.kafka.message.scheduler.streams +package com.sky.kafkamessage.scheduler.streams import akka.stream.scaladsl.SourceQueueWithComplete import cats.Eval -import com.sky.kafka.message.scheduler.config.SchedulerConfig -import com.sky.kafka.message.scheduler.domain.PublishableMessage.ScheduledMessage -import com.sky.kafka.message.scheduler.domain.ScheduleId +import com.sky.kafkamessage.scheduler.config.SchedulerConfig +import com.sky.kafkamessage.scheduler.domain.PublishableMessage.ScheduledMessage +import com.sky.kafkamessage.scheduler.domain.ScheduleId import com.typesafe.scalalogging.LazyLogging import org.zalando.grafter.{Stop, StopResult} diff --git a/scheduler/src/test/scala/com/sky/kafka/message/scheduler/AkkaComponentsSpec.scala b/scheduler/src/test/scala/com/sky/kafkamessage/scheduler/AkkaComponentsSpec.scala similarity index 92% rename from scheduler/src/test/scala/com/sky/kafka/message/scheduler/AkkaComponentsSpec.scala rename to scheduler/src/test/scala/com/sky/kafkamessage/scheduler/AkkaComponentsSpec.scala index 61798e2b..e1ece2a7 100644 --- a/scheduler/src/test/scala/com/sky/kafka/message/scheduler/AkkaComponentsSpec.scala +++ b/scheduler/src/test/scala/com/sky/kafkamessage/scheduler/AkkaComponentsSpec.scala @@ -1,4 +1,4 @@ -package com.sky.kafka.message.scheduler +package com.sky.kafkamessage.scheduler import akka.actor.Props import akka.actor.SupervisorStrategy.Restart diff --git a/scheduler/src/test/scala/com/sky/kafka/message/scheduler/PartitionedSinkSpec.scala b/scheduler/src/test/scala/com/sky/kafkamessage/scheduler/PartitionedSinkSpec.scala similarity index 88% rename from scheduler/src/test/scala/com/sky/kafka/message/scheduler/PartitionedSinkSpec.scala rename to scheduler/src/test/scala/com/sky/kafkamessage/scheduler/PartitionedSinkSpec.scala index 7c7cff24..b8a76d02 100644 --- a/scheduler/src/test/scala/com/sky/kafka/message/scheduler/PartitionedSinkSpec.scala +++ b/scheduler/src/test/scala/com/sky/kafkamessage/scheduler/PartitionedSinkSpec.scala @@ -1,7 +1,7 @@ -package com.sky.kafka.message.scheduler +package com.sky.kafkamessage.scheduler import akka.stream.scaladsl._ -import com.sky.kafka.message.scheduler.streams.PartitionedSink +import com.sky.kafkamessage.scheduler.streams.PartitionedSink import common.AkkaStreamBaseSpec import scala.concurrent.duration._ diff --git a/scheduler/src/test/scala/com/sky/kafka/message/scheduler/ScheduleReaderSpec.scala b/scheduler/src/test/scala/com/sky/kafkamessage/scheduler/ScheduleReaderSpec.scala similarity index 83% rename from scheduler/src/test/scala/com/sky/kafka/message/scheduler/ScheduleReaderSpec.scala rename to scheduler/src/test/scala/com/sky/kafkamessage/scheduler/ScheduleReaderSpec.scala index 7d2ccf06..1675cdfb 100644 --- a/scheduler/src/test/scala/com/sky/kafka/message/scheduler/ScheduleReaderSpec.scala +++ b/scheduler/src/test/scala/com/sky/kafkamessage/scheduler/ScheduleReaderSpec.scala @@ -1,9 +1,9 @@ -package com.sky.kafka.message.scheduler +package com.sky.kafkamessage.scheduler import java.util.UUID -import com.sky.kafka.message.scheduler.domain._ -import com.sky.kafka.message.scheduler.streams.ScheduleReader +import com.sky.kafkamessage.scheduler.domain._ +import com.sky.kafkamessage.scheduler.streams.ScheduleReader import common.AkkaStreamBaseSpec import common.TestDataUtils._ diff --git a/scheduler/src/test/scala/com/sky/kafka/message/scheduler/ScheduledMessagePublisherSpec.scala b/scheduler/src/test/scala/com/sky/kafkamessage/scheduler/ScheduledMessagePublisherSpec.scala similarity index 75% rename from scheduler/src/test/scala/com/sky/kafka/message/scheduler/ScheduledMessagePublisherSpec.scala rename to scheduler/src/test/scala/com/sky/kafkamessage/scheduler/ScheduledMessagePublisherSpec.scala index 83f91249..8e014817 100644 --- a/scheduler/src/test/scala/com/sky/kafka/message/scheduler/ScheduledMessagePublisherSpec.scala +++ b/scheduler/src/test/scala/com/sky/kafkamessage/scheduler/ScheduledMessagePublisherSpec.scala @@ -1,11 +1,11 @@ -package com.sky.kafka.message.scheduler +package com.sky.kafkamessage.scheduler import java.util.UUID -import com.sky.kafka.message.scheduler.config.{SchedulerConfig, ShutdownTimeout} -import com.sky.kafka.message.scheduler.domain._ -import com.sky.kafka.message.scheduler.kafka.KafkaStream -import com.sky.kafka.message.scheduler.streams.ScheduledMessagePublisher +import com.sky.kafkamessage.scheduler.config._ +import com.sky.kafkamessage.scheduler.domain._ +import com.sky.kafkamessage.scheduler.kafka.KafkaStream +import com.sky.kafkamessage.scheduler.streams.ScheduledMessagePublisher import common.AkkaStreamBaseSpec import common.TestDataUtils.random import org.apache.kafka.clients.producer.ProducerRecord diff --git a/scheduler/src/test/scala/com/sky/kafka/message/scheduler/SchedulerSpec.scala b/scheduler/src/test/scala/com/sky/kafkamessage/scheduler/SchedulerSpec.scala similarity index 92% rename from scheduler/src/test/scala/com/sky/kafka/message/scheduler/SchedulerSpec.scala rename to scheduler/src/test/scala/com/sky/kafkamessage/scheduler/SchedulerSpec.scala index 5cd06914..e288f61d 100644 --- a/scheduler/src/test/scala/com/sky/kafka/message/scheduler/SchedulerSpec.scala +++ b/scheduler/src/test/scala/com/sky/kafkamessage/scheduler/SchedulerSpec.scala @@ -1,13 +1,13 @@ -package com.sky.kafka.message.scheduler +package com.sky.kafkamessage.scheduler import java.time.OffsetDateTime import common.TestDataUtils._ -import com.sky.kafka.message.scheduler.domain._ +import com.sky.kafkamessage.scheduler.domain._ import common.BaseSpec import org.apache.kafka.clients.consumer.ConsumerRecord import avro._ -import com.sky.kafka.message.scheduler.domain.ApplicationError._ +import com.sky.kafkamessage.scheduler.domain.ApplicationError._ class SchedulerSpec extends BaseSpec { diff --git a/scheduler/src/test/scala/com/sky/kafka/message/scheduler/SchedulingActorSpec.scala b/scheduler/src/test/scala/com/sky/kafkamessage/scheduler/SchedulingActorSpec.scala similarity index 93% rename from scheduler/src/test/scala/com/sky/kafka/message/scheduler/SchedulingActorSpec.scala rename to scheduler/src/test/scala/com/sky/kafkamessage/scheduler/SchedulingActorSpec.scala index 9f25ee8b..a59f5ab8 100644 --- a/scheduler/src/test/scala/com/sky/kafka/message/scheduler/SchedulingActorSpec.scala +++ b/scheduler/src/test/scala/com/sky/kafkamessage/scheduler/SchedulingActorSpec.scala @@ -1,4 +1,4 @@ -package com.sky.kafka.message.scheduler +package com.sky.kafkamessage.scheduler import java.util.UUID @@ -6,9 +6,9 @@ import akka.event.LoggingAdapter import akka.stream.scaladsl.SourceQueue import akka.testkit.{ImplicitSender, TestActorRef} import com.miguno.akka.testing.VirtualTime -import com.sky.kafka.message.scheduler.SchedulingActor._ -import com.sky.kafka.message.scheduler.domain.PublishableMessage.ScheduledMessage -import com.sky.kafka.message.scheduler.domain._ +import com.sky.kafkamessage.scheduler.SchedulingActor._ +import com.sky.kafkamessage.scheduler.domain.PublishableMessage.ScheduledMessage +import com.sky.kafkamessage.scheduler.domain._ import common.AkkaBaseSpec import common.TestDataUtils._ import org.mockito.Mockito._ diff --git a/scheduler/src/test/scala/com/sky/kafka/message/scheduler/avro/AvroSpec.scala b/scheduler/src/test/scala/com/sky/kafkamessage/scheduler/avro/AvroSpec.scala similarity index 96% rename from scheduler/src/test/scala/com/sky/kafka/message/scheduler/avro/AvroSpec.scala rename to scheduler/src/test/scala/com/sky/kafkamessage/scheduler/avro/AvroSpec.scala index 88787fa3..9ad29605 100644 --- a/scheduler/src/test/scala/com/sky/kafka/message/scheduler/avro/AvroSpec.scala +++ b/scheduler/src/test/scala/com/sky/kafkamessage/scheduler/avro/AvroSpec.scala @@ -1,4 +1,4 @@ -package com.sky.kafka.message.scheduler.avro +package com.sky.kafkamessage.scheduler.avro import java.time.OffsetDateTime diff --git a/scheduler/src/test/scala/com/sky/kafka/message/scheduler/e2e/SchedulerIntSpec.scala b/scheduler/src/test/scala/com/sky/kafkamessage/scheduler/e2e/SchedulerIntSpec.scala similarity index 86% rename from scheduler/src/test/scala/com/sky/kafka/message/scheduler/e2e/SchedulerIntSpec.scala rename to scheduler/src/test/scala/com/sky/kafkamessage/scheduler/e2e/SchedulerIntSpec.scala index dbcad3bb..3c830b3e 100644 --- a/scheduler/src/test/scala/com/sky/kafka/message/scheduler/e2e/SchedulerIntSpec.scala +++ b/scheduler/src/test/scala/com/sky/kafkamessage/scheduler/e2e/SchedulerIntSpec.scala @@ -1,11 +1,11 @@ -package com.sky.kafka.message.scheduler.e2e +package com.sky.kafkamessage.scheduler.e2e import java.util.UUID -import com.sky.kafka.message.scheduler.avro._ -import com.sky.kafka.message.scheduler.config._ -import com.sky.kafka.message.scheduler.domain._ -import com.sky.kafka.message.scheduler.streams.ScheduleReader +import com.sky.kafkamessage.scheduler.avro._ +import com.sky.kafkamessage.scheduler.config._ +import com.sky.kafkamessage.scheduler.domain._ +import com.sky.kafkamessage.scheduler.streams.ScheduleReader import common.TestDataUtils._ import common.{AkkaStreamBaseSpec, KafkaIntSpec} import org.apache.kafka.common.serialization._ diff --git a/scheduler/src/test/scala/common/TestDataUtils.scala b/scheduler/src/test/scala/common/TestDataUtils.scala index 6e39c7ed..97bba887 100644 --- a/scheduler/src/test/scala/common/TestDataUtils.scala +++ b/scheduler/src/test/scala/common/TestDataUtils.scala @@ -7,10 +7,10 @@ import com.danielasfregola.randomdatagenerator.RandomDataGenerator import com.fortysevendeg.scalacheck.datetime.GenDateTime.genDateTimeWithinRange import com.fortysevendeg.scalacheck.datetime.instances.jdk8._ import com.sksamuel.avro4s.{AvroOutputStream, ToRecord} -import com.sky.kafka.message.scheduler.domain._ +import com.sky.kafkamessage.scheduler.domain._ import org.scalacheck._ -import com.sky.kafka.message.scheduler.avro._ -import com.sky.kafka.message.scheduler.domain.PublishableMessage.ScheduledMessage +import com.sky.kafkamessage.scheduler.avro._ +import com.sky.kafkamessage.scheduler.domain.PublishableMessage.ScheduledMessage object TestDataUtils extends RandomDataGenerator { From 229013a88625cdf681206199bac2589d0185f942 Mon Sep 17 00:00:00 2001 From: Roberto Tena Date: Tue, 8 Aug 2017 15:44:34 +0100 Subject: [PATCH 18/22] Add custom metrics component --- avro/src/main/scala/GenerateSchema.scala | 2 +- .../scheduler => kms}/AkkaComponents.scala | 5 +- .../main/scala/com/sky/kms/Monitoring.scala | 20 ++++++++ .../scheduler => kms}/SchedulerApp.scala | 6 +-- .../scheduler => kms}/SchedulingActor.scala | 12 ++--- .../scheduler => kms}/avro/package.scala | 2 +- .../scheduler => kms}/config/package.scala | 2 +- .../domain/ApplicationError.scala | 2 +- .../domain/PublishableMessage.scala | 4 +- .../scheduler => kms}/domain/package.scala | 2 +- .../kafka/ConsumerRecordDecoder.scala | 2 +- .../scheduler => kms}/kafka/KafkaStream.scala | 4 +- .../kafka/ProducerRecordEncoder.scala | 2 +- .../scheduler => kms}/package.scala | 12 ++--- .../streams/PartitionedSink.scala | 4 +- .../streams/ScheduleReader.scala | 14 +++--- .../streams/ScheduleReaderStream.scala | 4 +- .../streams/ScheduledMessagePublisher.scala | 14 +++--- .../ScheduledMessagePublisherStream.scala | 8 ++-- scheduler/src/test/resources/log4j.properties | 5 -- scheduler/src/test/resources/logback-test.xml | 11 +++++ .../scheduler/AkkaComponentsSpec.scala | 18 -------- .../com/sky/kms/AkkaComponentsSpec.scala | 38 +++++++++++++++ .../scala/com/sky/kms/MonitoringSpec.scala | 46 +++++++++++++++++++ .../PartitionedSinkSpec.scala | 6 +-- .../ScheduleReaderSpec.scala | 6 +-- .../ScheduledMessagePublisherSpec.scala | 10 ++-- .../scheduler => kms}/SchedulerSpec.scala | 6 +-- .../SchedulingActorSpec.scala | 8 ++-- .../scheduler => kms}/avro/AvroSpec.scala | 4 +- .../sky/kms}/common/AkkaBaseSpec.scala | 2 +- .../sky/kms}/common/AkkaStreamBaseSpec.scala | 2 +- .../{ => com/sky/kms}/common/BaseSpec.scala | 2 +- .../sky/kms}/common/EmbeddedKafka.scala | 7 ++- .../sky/kms}/common/KafkaIntSpec.scala | 2 +- .../sky/kms}/common/TestActorSystem.scala | 2 +- .../sky/kms}/common/TestDataUtils.scala | 8 ++-- .../e2e/SchedulerIntSpec.scala | 14 +++--- 38 files changed, 205 insertions(+), 113 deletions(-) rename scheduler/src/main/scala/com/sky/{kafkamessage/scheduler => kms}/AkkaComponents.scala (83%) create mode 100644 scheduler/src/main/scala/com/sky/kms/Monitoring.scala rename scheduler/src/main/scala/com/sky/{kafkamessage/scheduler => kms}/SchedulerApp.scala (80%) rename scheduler/src/main/scala/com/sky/{kafkamessage/scheduler => kms}/SchedulingActor.scala (89%) rename scheduler/src/main/scala/com/sky/{kafkamessage/scheduler => kms}/avro/package.scala (95%) rename scheduler/src/main/scala/com/sky/{kafkamessage/scheduler => kms}/config/package.scala (93%) rename scheduler/src/main/scala/com/sky/{kafkamessage/scheduler => kms}/domain/ApplicationError.scala (96%) rename scheduler/src/main/scala/com/sky/{kafkamessage/scheduler => kms}/domain/PublishableMessage.scala (90%) rename scheduler/src/main/scala/com/sky/{kafkamessage/scheduler => kms}/domain/package.scala (82%) rename scheduler/src/main/scala/com/sky/{kafkamessage/scheduler => kms}/kafka/ConsumerRecordDecoder.scala (88%) rename scheduler/src/main/scala/com/sky/{kafkamessage/scheduler => kms}/kafka/KafkaStream.scala (90%) rename scheduler/src/main/scala/com/sky/{kafkamessage/scheduler => kms}/kafka/ProducerRecordEncoder.scala (89%) rename scheduler/src/main/scala/com/sky/{kafkamessage/scheduler => kms}/package.scala (79%) rename scheduler/src/main/scala/com/sky/{kafkamessage/scheduler => kms}/streams/PartitionedSink.scala (73%) rename scheduler/src/main/scala/com/sky/{kafkamessage/scheduler => kms}/streams/ScheduleReader.scala (81%) rename scheduler/src/main/scala/com/sky/{kafkamessage/scheduler => kms}/streams/ScheduleReaderStream.scala (78%) rename scheduler/src/main/scala/com/sky/{kafkamessage/scheduler => kms}/streams/ScheduledMessagePublisher.scala (78%) rename scheduler/src/main/scala/com/sky/{kafkamessage/scheduler => kms}/streams/ScheduledMessagePublisherStream.scala (69%) delete mode 100644 scheduler/src/test/resources/log4j.properties create mode 100644 scheduler/src/test/resources/logback-test.xml delete mode 100644 scheduler/src/test/scala/com/sky/kafkamessage/scheduler/AkkaComponentsSpec.scala create mode 100644 scheduler/src/test/scala/com/sky/kms/AkkaComponentsSpec.scala create mode 100644 scheduler/src/test/scala/com/sky/kms/MonitoringSpec.scala rename scheduler/src/test/scala/com/sky/{kafkamessage/scheduler => kms}/PartitionedSinkSpec.scala (86%) rename scheduler/src/test/scala/com/sky/{kafkamessage/scheduler => kms}/ScheduleReaderSpec.scala (83%) rename scheduler/src/test/scala/com/sky/{kafkamessage/scheduler => kms}/ScheduledMessagePublisherSpec.scala (77%) rename scheduler/src/test/scala/com/sky/{kafkamessage/scheduler => kms}/SchedulerSpec.scala (92%) rename scheduler/src/test/scala/com/sky/{kafkamessage/scheduler => kms}/SchedulingActorSpec.scala (93%) rename scheduler/src/test/scala/com/sky/{kafkamessage/scheduler => kms}/avro/AvroSpec.scala (94%) rename scheduler/src/test/scala/{ => com/sky/kms}/common/AkkaBaseSpec.scala (91%) rename scheduler/src/test/scala/{ => com/sky/kms}/common/AkkaStreamBaseSpec.scala (89%) rename scheduler/src/test/scala/{ => com/sky/kms}/common/BaseSpec.scala (83%) rename scheduler/src/test/scala/{ => com/sky/kms}/common/EmbeddedKafka.scala (88%) rename scheduler/src/test/scala/{ => com/sky/kms}/common/KafkaIntSpec.scala (97%) rename scheduler/src/test/scala/{ => com/sky/kms}/common/TestActorSystem.scala (96%) rename scheduler/src/test/scala/{ => com/sky/kms}/common/TestDataUtils.scala (88%) rename scheduler/src/test/scala/com/sky/{kafkamessage/scheduler => kms}/e2e/SchedulerIntSpec.scala (82%) diff --git a/avro/src/main/scala/GenerateSchema.scala b/avro/src/main/scala/GenerateSchema.scala index 654b0f56..61da6b3f 100644 --- a/avro/src/main/scala/GenerateSchema.scala +++ b/avro/src/main/scala/GenerateSchema.scala @@ -2,7 +2,7 @@ import java.nio.charset.StandardCharsets import java.nio.file.{Files, Paths} import com.sksamuel.avro4s._ -import com.sky.kafkamessage.scheduler.domain.Schedule +import com.sky.kms.domain.Schedule object GenerateSchema extends App { diff --git a/scheduler/src/main/scala/com/sky/kafkamessage/scheduler/AkkaComponents.scala b/scheduler/src/main/scala/com/sky/kms/AkkaComponents.scala similarity index 83% rename from scheduler/src/main/scala/com/sky/kafkamessage/scheduler/AkkaComponents.scala rename to scheduler/src/main/scala/com/sky/kms/AkkaComponents.scala index 591ed9cb..f726cbb3 100644 --- a/scheduler/src/main/scala/com/sky/kafkamessage/scheduler/AkkaComponents.scala +++ b/scheduler/src/main/scala/com/sky/kms/AkkaComponents.scala @@ -1,15 +1,16 @@ -package com.sky.kafkamessage.scheduler +package com.sky.kms import akka.actor.ActorSystem import akka.stream.Supervision.Restart import akka.stream.{ActorMaterializer, ActorMaterializerSettings, Supervision} import com.typesafe.scalalogging.LazyLogging -trait AkkaComponents extends LazyLogging { +trait AkkaComponents extends LazyLogging with Monitoring { implicit val system = ActorSystem("kafka-message-scheduler") val decider: Supervision.Decider = { t => + recordException(t) logger.error(s"Supervision failed.", t) Restart } diff --git a/scheduler/src/main/scala/com/sky/kms/Monitoring.scala b/scheduler/src/main/scala/com/sky/kms/Monitoring.scala new file mode 100644 index 00000000..945d078a --- /dev/null +++ b/scheduler/src/main/scala/com/sky/kms/Monitoring.scala @@ -0,0 +1,20 @@ +package com.sky.kms + +import kamon.Kamon +import kamon.metric.MetricsModule + +trait Monitoring { + + val metrics: MetricsModule = Kamon.metrics + + def increment(key: String) = metrics.counter(key).increment() + + def recordException(throwable: Throwable) = { + val key = generateKeyFromException(throwable) + metrics.counter(key).increment() + } + + private def generateKeyFromException(throwable: Throwable): String = { + return s"exception.${throwable.getClass.getName.replace(".", "_")}" + } +} diff --git a/scheduler/src/main/scala/com/sky/kafkamessage/scheduler/SchedulerApp.scala b/scheduler/src/main/scala/com/sky/kms/SchedulerApp.scala similarity index 80% rename from scheduler/src/main/scala/com/sky/kafkamessage/scheduler/SchedulerApp.scala rename to scheduler/src/main/scala/com/sky/kms/SchedulerApp.scala index 659f8171..ccd0c8b1 100644 --- a/scheduler/src/main/scala/com/sky/kafkamessage/scheduler/SchedulerApp.scala +++ b/scheduler/src/main/scala/com/sky/kms/SchedulerApp.scala @@ -1,7 +1,7 @@ -package com.sky.kafkamessage.scheduler +package com.sky.kms -import com.sky.kafkamessage.scheduler.config.AppConfig -import com.sky.kafkamessage.scheduler.streams.ScheduleReader +import com.sky.kms.config.AppConfig +import com.sky.kms.streams.ScheduleReader import com.typesafe.scalalogging.LazyLogging import kamon.Kamon import org.zalando.grafter._ diff --git a/scheduler/src/main/scala/com/sky/kafkamessage/scheduler/SchedulingActor.scala b/scheduler/src/main/scala/com/sky/kms/SchedulingActor.scala similarity index 89% rename from scheduler/src/main/scala/com/sky/kafkamessage/scheduler/SchedulingActor.scala rename to scheduler/src/main/scala/com/sky/kms/SchedulingActor.scala index 5a90edd4..fe14a83c 100644 --- a/scheduler/src/main/scala/com/sky/kafkamessage/scheduler/SchedulingActor.scala +++ b/scheduler/src/main/scala/com/sky/kms/SchedulingActor.scala @@ -1,4 +1,4 @@ -package com.sky.kafkamessage.scheduler +package com.sky.kms import java.time.OffsetDateTime import java.time.temporal.ChronoUnit @@ -8,11 +8,11 @@ import akka.actor._ import akka.stream.ActorMaterializer import akka.stream.scaladsl.SourceQueue import cats.data.Reader -import com.sky.kafkamessage.scheduler.SchedulingActor._ -import com.sky.kafkamessage.scheduler.config.AppConfig -import com.sky.kafkamessage.scheduler.domain.PublishableMessage.ScheduledMessage -import com.sky.kafkamessage.scheduler.domain._ -import com.sky.kafkamessage.scheduler.streams.ScheduledMessagePublisher +import com.sky.kms.SchedulingActor._ +import com.sky.kms.config.AppConfig +import com.sky.kms.domain.PublishableMessage.ScheduledMessage +import com.sky.kms.domain._ +import com.sky.kms.streams.ScheduledMessagePublisher import scala.concurrent.ExecutionContext.Implicits.global import scala.concurrent.duration.FiniteDuration diff --git a/scheduler/src/main/scala/com/sky/kafkamessage/scheduler/avro/package.scala b/scheduler/src/main/scala/com/sky/kms/avro/package.scala similarity index 95% rename from scheduler/src/main/scala/com/sky/kafkamessage/scheduler/avro/package.scala rename to scheduler/src/main/scala/com/sky/kms/avro/package.scala index 1216e23f..a33e40e9 100644 --- a/scheduler/src/main/scala/com/sky/kafkamessage/scheduler/avro/package.scala +++ b/scheduler/src/main/scala/com/sky/kms/avro/package.scala @@ -1,4 +1,4 @@ -package com.sky.kafkamessage.scheduler +package com.sky.kms import java.time.OffsetDateTime import java.time.format.DateTimeFormatter diff --git a/scheduler/src/main/scala/com/sky/kafkamessage/scheduler/config/package.scala b/scheduler/src/main/scala/com/sky/kms/config/package.scala similarity index 93% rename from scheduler/src/main/scala/com/sky/kafkamessage/scheduler/config/package.scala rename to scheduler/src/main/scala/com/sky/kms/config/package.scala index 533c7141..0234f15a 100644 --- a/scheduler/src/main/scala/com/sky/kafkamessage/scheduler/config/package.scala +++ b/scheduler/src/main/scala/com/sky/kms/config/package.scala @@ -1,4 +1,4 @@ -package com.sky.kafkamessage.scheduler +package com.sky.kms import akka.actor.ActorSystem import akka.stream.ActorMaterializer diff --git a/scheduler/src/main/scala/com/sky/kafkamessage/scheduler/domain/ApplicationError.scala b/scheduler/src/main/scala/com/sky/kms/domain/ApplicationError.scala similarity index 96% rename from scheduler/src/main/scala/com/sky/kafkamessage/scheduler/domain/ApplicationError.scala rename to scheduler/src/main/scala/com/sky/kms/domain/ApplicationError.scala index 28052dc7..357197ca 100644 --- a/scheduler/src/main/scala/com/sky/kafkamessage/scheduler/domain/ApplicationError.scala +++ b/scheduler/src/main/scala/com/sky/kms/domain/ApplicationError.scala @@ -1,4 +1,4 @@ -package com.sky.kafkamessage.scheduler.domain +package com.sky.kms.domain import akka.Done import akka.stream.scaladsl.Sink diff --git a/scheduler/src/main/scala/com/sky/kafkamessage/scheduler/domain/PublishableMessage.scala b/scheduler/src/main/scala/com/sky/kms/domain/PublishableMessage.scala similarity index 90% rename from scheduler/src/main/scala/com/sky/kafkamessage/scheduler/domain/PublishableMessage.scala rename to scheduler/src/main/scala/com/sky/kms/domain/PublishableMessage.scala index 0fd322ff..ff349f68 100644 --- a/scheduler/src/main/scala/com/sky/kafkamessage/scheduler/domain/PublishableMessage.scala +++ b/scheduler/src/main/scala/com/sky/kms/domain/PublishableMessage.scala @@ -1,6 +1,6 @@ -package com.sky.kafkamessage.scheduler.domain +package com.sky.kms.domain -import com.sky.kafkamessage.scheduler.kafka.ProducerRecordEncoder +import com.sky.kms.kafka.ProducerRecordEncoder import org.apache.kafka.clients.producer.ProducerRecord sealed trait PublishableMessage diff --git a/scheduler/src/main/scala/com/sky/kafkamessage/scheduler/domain/package.scala b/scheduler/src/main/scala/com/sky/kms/domain/package.scala similarity index 82% rename from scheduler/src/main/scala/com/sky/kafkamessage/scheduler/domain/package.scala rename to scheduler/src/main/scala/com/sky/kms/domain/package.scala index 6b36913a..530df003 100644 --- a/scheduler/src/main/scala/com/sky/kafkamessage/scheduler/domain/package.scala +++ b/scheduler/src/main/scala/com/sky/kms/domain/package.scala @@ -1,4 +1,4 @@ -package com.sky.kafkamessage.scheduler +package com.sky.kms import java.time.OffsetDateTime diff --git a/scheduler/src/main/scala/com/sky/kafkamessage/scheduler/kafka/ConsumerRecordDecoder.scala b/scheduler/src/main/scala/com/sky/kms/kafka/ConsumerRecordDecoder.scala similarity index 88% rename from scheduler/src/main/scala/com/sky/kafkamessage/scheduler/kafka/ConsumerRecordDecoder.scala rename to scheduler/src/main/scala/com/sky/kms/kafka/ConsumerRecordDecoder.scala index 3827d452..c4848fa5 100644 --- a/scheduler/src/main/scala/com/sky/kafkamessage/scheduler/kafka/ConsumerRecordDecoder.scala +++ b/scheduler/src/main/scala/com/sky/kms/kafka/ConsumerRecordDecoder.scala @@ -1,4 +1,4 @@ -package com.sky.kafkamessage.scheduler.kafka +package com.sky.kms.kafka import org.apache.kafka.clients.consumer.ConsumerRecord diff --git a/scheduler/src/main/scala/com/sky/kafkamessage/scheduler/kafka/KafkaStream.scala b/scheduler/src/main/scala/com/sky/kms/kafka/KafkaStream.scala similarity index 90% rename from scheduler/src/main/scala/com/sky/kafkamessage/scheduler/kafka/KafkaStream.scala rename to scheduler/src/main/scala/com/sky/kms/kafka/KafkaStream.scala index 121614da..7a1498c9 100644 --- a/scheduler/src/main/scala/com/sky/kafkamessage/scheduler/kafka/KafkaStream.scala +++ b/scheduler/src/main/scala/com/sky/kms/kafka/KafkaStream.scala @@ -1,4 +1,4 @@ -package com.sky.kafkamessage.scheduler.kafka +package com.sky.kms.kafka import akka.Done import akka.actor.ActorSystem @@ -6,7 +6,7 @@ import akka.kafka.scaladsl.Consumer.Control import akka.kafka.scaladsl.{Consumer, Producer} import akka.kafka.{ConsumerSettings, ProducerSettings, Subscriptions} import akka.stream.scaladsl.{Sink, Source} -import com.sky.kafkamessage.scheduler.config.SchedulerConfig +import com.sky.kms.config.SchedulerConfig import org.apache.kafka.clients.producer.ProducerRecord import org.apache.kafka.common.serialization.{ByteArrayDeserializer, ByteArraySerializer, StringDeserializer} diff --git a/scheduler/src/main/scala/com/sky/kafkamessage/scheduler/kafka/ProducerRecordEncoder.scala b/scheduler/src/main/scala/com/sky/kms/kafka/ProducerRecordEncoder.scala similarity index 89% rename from scheduler/src/main/scala/com/sky/kafkamessage/scheduler/kafka/ProducerRecordEncoder.scala rename to scheduler/src/main/scala/com/sky/kms/kafka/ProducerRecordEncoder.scala index b9dd0e33..ff835cc6 100644 --- a/scheduler/src/main/scala/com/sky/kafkamessage/scheduler/kafka/ProducerRecordEncoder.scala +++ b/scheduler/src/main/scala/com/sky/kms/kafka/ProducerRecordEncoder.scala @@ -1,4 +1,4 @@ -package com.sky.kafkamessage.scheduler.kafka +package com.sky.kms.kafka import org.apache.kafka.clients.producer.ProducerRecord diff --git a/scheduler/src/main/scala/com/sky/kafkamessage/scheduler/package.scala b/scheduler/src/main/scala/com/sky/kms/package.scala similarity index 79% rename from scheduler/src/main/scala/com/sky/kafkamessage/scheduler/package.scala rename to scheduler/src/main/scala/com/sky/kms/package.scala index f94efd80..a25cc8cd 100644 --- a/scheduler/src/main/scala/com/sky/kafkamessage/scheduler/package.scala +++ b/scheduler/src/main/scala/com/sky/kms/package.scala @@ -1,17 +1,17 @@ -package com.sky.kafkamessage +package com.sky import cats.syntax.either._ import com.sksamuel.avro4s.AvroInputStream -import com.sky.kafkamessage.scheduler.domain.ApplicationError._ -import com.sky.kafkamessage.scheduler.domain._ +import com.sky.kms.domain.ApplicationError._ +import com.sky.kms.domain._ import com.typesafe.scalalogging.LazyLogging import org.apache.kafka.clients.consumer.ConsumerRecord -import com.sky.kafkamessage.scheduler.avro._ -import com.sky.kafkamessage.scheduler.kafka.ConsumerRecordDecoder +import com.sky.kms.avro._ +import com.sky.kms.kafka.ConsumerRecordDecoder import scala.util.Try -package object scheduler extends LazyLogging { +package object kms extends LazyLogging { implicit val scheduleConsumerRecordDecoder: ConsumerRecordDecoder[Either[ApplicationError, (ScheduleId, Option[Schedule])]] = ConsumerRecordDecoder.instance(consumerRecordDecoder) diff --git a/scheduler/src/main/scala/com/sky/kafkamessage/scheduler/streams/PartitionedSink.scala b/scheduler/src/main/scala/com/sky/kms/streams/PartitionedSink.scala similarity index 73% rename from scheduler/src/main/scala/com/sky/kafkamessage/scheduler/streams/PartitionedSink.scala rename to scheduler/src/main/scala/com/sky/kms/streams/PartitionedSink.scala index 4c631570..3f1ceb6d 100644 --- a/scheduler/src/main/scala/com/sky/kafkamessage/scheduler/streams/PartitionedSink.scala +++ b/scheduler/src/main/scala/com/sky/kms/streams/PartitionedSink.scala @@ -1,4 +1,4 @@ -package com.sky.kafkamessage.scheduler.streams +package com.sky.kms.streams import akka.stream.SinkShape import akka.stream.contrib.PartitionWith @@ -6,7 +6,7 @@ import akka.stream.scaladsl.{GraphDSL, Sink} object PartitionedSink { - def from[A, AMat, B, BMat](rightSink: Sink[B, BMat])(implicit leftSink: Sink[A, AMat]): Sink[Either[A, B], (AMat, BMat)] = + def withRight[A, AMat, B, BMat](rightSink: Sink[B, BMat])(implicit leftSink: Sink[A, AMat]): Sink[Either[A, B], (AMat, BMat)] = Sink.fromGraph(GraphDSL.create(leftSink, rightSink)((_, _)) { implicit b => (left, right) => import GraphDSL.Implicits._ diff --git a/scheduler/src/main/scala/com/sky/kafkamessage/scheduler/streams/ScheduleReader.scala b/scheduler/src/main/scala/com/sky/kms/streams/ScheduleReader.scala similarity index 81% rename from scheduler/src/main/scala/com/sky/kafkamessage/scheduler/streams/ScheduleReader.scala rename to scheduler/src/main/scala/com/sky/kms/streams/ScheduleReader.scala index cf31678a..5ea982ad 100644 --- a/scheduler/src/main/scala/com/sky/kafkamessage/scheduler/streams/ScheduleReader.scala +++ b/scheduler/src/main/scala/com/sky/kms/streams/ScheduleReader.scala @@ -1,4 +1,4 @@ -package com.sky.kafkamessage.scheduler.streams +package com.sky.kms.streams import akka.actor.ActorSystem import akka.kafka.scaladsl.Consumer.Control @@ -6,11 +6,11 @@ import akka.stream._ import akka.stream.scaladsl.{Sink, Source} import akka.{Done, NotUsed} import cats.data.Reader -import com.sky.kafkamessage.scheduler.SchedulingActor._ -import com.sky.kafkamessage.scheduler._ -import com.sky.kafkamessage.scheduler.config.{AppConfig, SchedulerConfig} -import com.sky.kafkamessage.scheduler.domain.{ApplicationError, Schedule, ScheduleId} -import com.sky.kafkamessage.scheduler.kafka._ +import com.sky.kms.SchedulingActor._ +import com.sky.kms._ +import com.sky.kms.config.{AppConfig, SchedulerConfig} +import com.sky.kms.domain._ +import com.sky.kms.kafka._ import com.typesafe.scalalogging.LazyLogging /** @@ -24,7 +24,7 @@ case class ScheduleReader(config: SchedulerConfig, val stream: Control = scheduleSource .map(ScheduleReader.toSchedulingMessage) - .to(PartitionedSink.from(schedulingSink)) + .to(PartitionedSink.withRight(schedulingSink)) .run() } diff --git a/scheduler/src/main/scala/com/sky/kafkamessage/scheduler/streams/ScheduleReaderStream.scala b/scheduler/src/main/scala/com/sky/kms/streams/ScheduleReaderStream.scala similarity index 78% rename from scheduler/src/main/scala/com/sky/kafkamessage/scheduler/streams/ScheduleReaderStream.scala rename to scheduler/src/main/scala/com/sky/kms/streams/ScheduleReaderStream.scala index 0b2d2a66..0afba905 100644 --- a/scheduler/src/main/scala/com/sky/kafkamessage/scheduler/streams/ScheduleReaderStream.scala +++ b/scheduler/src/main/scala/com/sky/kms/streams/ScheduleReaderStream.scala @@ -1,8 +1,8 @@ -package com.sky.kafkamessage.scheduler.streams +package com.sky.kms.streams import akka.kafka.scaladsl.Consumer.Control import cats.Eval -import com.sky.kafkamessage.scheduler.config.SchedulerConfig +import com.sky.kms.config.SchedulerConfig import org.zalando.grafter.{Stop, StopResult} import scala.concurrent.Await diff --git a/scheduler/src/main/scala/com/sky/kafkamessage/scheduler/streams/ScheduledMessagePublisher.scala b/scheduler/src/main/scala/com/sky/kms/streams/ScheduledMessagePublisher.scala similarity index 78% rename from scheduler/src/main/scala/com/sky/kafkamessage/scheduler/streams/ScheduledMessagePublisher.scala rename to scheduler/src/main/scala/com/sky/kms/streams/ScheduledMessagePublisher.scala index 63298497..e1d73cfb 100644 --- a/scheduler/src/main/scala/com/sky/kafkamessage/scheduler/streams/ScheduledMessagePublisher.scala +++ b/scheduler/src/main/scala/com/sky/kms/streams/ScheduledMessagePublisher.scala @@ -1,15 +1,15 @@ -package com.sky.kafkamessage.scheduler.streams +package com.sky.kms.streams import akka.Done import akka.actor.ActorSystem -import akka.stream.scaladsl.{Sink, Source, SourceQueueWithComplete} +import akka.stream.scaladsl._ import akka.stream.{ActorMaterializer, OverflowStrategy} import cats.data.Reader -import com.sky.kafkamessage.scheduler.config._ -import com.sky.kafkamessage.scheduler.domain.PublishableMessage._ -import com.sky.kafkamessage.scheduler.domain._ -import com.sky.kafkamessage.scheduler.streams.ScheduledMessagePublisher._ -import com.sky.kafkamessage.scheduler.kafka.KafkaStream +import com.sky.kms.config._ +import com.sky.kms.domain.PublishableMessage._ +import com.sky.kms.domain._ +import com.sky.kms.kafka.KafkaStream +import com.sky.kms.streams.ScheduledMessagePublisher._ import org.apache.kafka.clients.producer.ProducerRecord import scala.concurrent.Future diff --git a/scheduler/src/main/scala/com/sky/kafkamessage/scheduler/streams/ScheduledMessagePublisherStream.scala b/scheduler/src/main/scala/com/sky/kms/streams/ScheduledMessagePublisherStream.scala similarity index 69% rename from scheduler/src/main/scala/com/sky/kafkamessage/scheduler/streams/ScheduledMessagePublisherStream.scala rename to scheduler/src/main/scala/com/sky/kms/streams/ScheduledMessagePublisherStream.scala index 16b50c2c..c36f3c31 100644 --- a/scheduler/src/main/scala/com/sky/kafkamessage/scheduler/streams/ScheduledMessagePublisherStream.scala +++ b/scheduler/src/main/scala/com/sky/kms/streams/ScheduledMessagePublisherStream.scala @@ -1,10 +1,10 @@ -package com.sky.kafkamessage.scheduler.streams +package com.sky.kms.streams import akka.stream.scaladsl.SourceQueueWithComplete import cats.Eval -import com.sky.kafkamessage.scheduler.config.SchedulerConfig -import com.sky.kafkamessage.scheduler.domain.PublishableMessage.ScheduledMessage -import com.sky.kafkamessage.scheduler.domain.ScheduleId +import com.sky.kms.config.SchedulerConfig +import com.sky.kms.domain.PublishableMessage.ScheduledMessage +import com.sky.kms.domain.ScheduleId import com.typesafe.scalalogging.LazyLogging import org.zalando.grafter.{Stop, StopResult} diff --git a/scheduler/src/test/resources/log4j.properties b/scheduler/src/test/resources/log4j.properties deleted file mode 100644 index 93b0195a..00000000 --- a/scheduler/src/test/resources/log4j.properties +++ /dev/null @@ -1,5 +0,0 @@ -log4j.rootLogger=WARN, stderr - -log4j.appender.stderr=org.apache.log4j.varia.NullAppender -log4j.appender.stderr.layout=org.apache.log4j.PatternLayout -log4j.appender.stderr.layout.ConversionPattern=[%d] %p %m (%c)%n \ No newline at end of file diff --git a/scheduler/src/test/resources/logback-test.xml b/scheduler/src/test/resources/logback-test.xml new file mode 100644 index 00000000..67ff0731 --- /dev/null +++ b/scheduler/src/test/resources/logback-test.xml @@ -0,0 +1,11 @@ + + + + %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n + + + + + + + \ No newline at end of file diff --git a/scheduler/src/test/scala/com/sky/kafkamessage/scheduler/AkkaComponentsSpec.scala b/scheduler/src/test/scala/com/sky/kafkamessage/scheduler/AkkaComponentsSpec.scala deleted file mode 100644 index e1ece2a7..00000000 --- a/scheduler/src/test/scala/com/sky/kafkamessage/scheduler/AkkaComponentsSpec.scala +++ /dev/null @@ -1,18 +0,0 @@ -package com.sky.kafkamessage.scheduler - -import akka.actor.Props -import akka.actor.SupervisorStrategy.Restart -import akka.testkit.TestActorRef -import common.{AkkaBaseSpec, AkkaStreamBaseSpec} - -class AkkaComponentsSpec extends AkkaBaseSpec { - - "decider" should { - "return a 'Restart' supervision strategy" in { - val supervisor = TestActorRef[SchedulingActor](Props(new SchedulingActor(null, null))) - val strategy = supervisor.underlyingActor.supervisorStrategy.decider - - strategy(new Exception("Any exception")) should be (Restart) - } - } -} diff --git a/scheduler/src/test/scala/com/sky/kms/AkkaComponentsSpec.scala b/scheduler/src/test/scala/com/sky/kms/AkkaComponentsSpec.scala new file mode 100644 index 00000000..f5f644a8 --- /dev/null +++ b/scheduler/src/test/scala/com/sky/kms/AkkaComponentsSpec.scala @@ -0,0 +1,38 @@ +package com.sky.kms + +import akka.stream.Supervision.Restart +import common.BaseSpec +import kamon.metric.MetricsModule +import kamon.metric.instrument.Counter +import org.mockito.Mockito.{verify, when} +import org.scalatest.mockito.MockitoSugar + +import scala.concurrent.Await +import scala.concurrent.duration._ + +class AkkaComponentsSpec extends BaseSpec with MockitoSugar { + + "decider" should { + "collect metrics from failures" in new AkkaComponentsFixture { + val key = "exception.java_lang_Exception" + val exception = new Exception("Any exception") + + when(metrics.counter(key)).thenReturn(mockCounter) + + materializer + .settings + .supervisionDecider(exception) shouldBe Restart + + verify(metrics).counter(key) + verify(mockCounter).increment() + + materializer.shutdown() + Await.ready(system.terminate(), 5 seconds) + } + } + + private class AkkaComponentsFixture extends AkkaComponents { + val mockCounter = mock[Counter] + override val metrics: MetricsModule = mock[MetricsModule] + } +} diff --git a/scheduler/src/test/scala/com/sky/kms/MonitoringSpec.scala b/scheduler/src/test/scala/com/sky/kms/MonitoringSpec.scala new file mode 100644 index 00000000..d20dc84e --- /dev/null +++ b/scheduler/src/test/scala/com/sky/kms/MonitoringSpec.scala @@ -0,0 +1,46 @@ +package com.sky.kms + +import common.BaseSpec +import kamon.metric.MetricsModule +import kamon.metric.instrument.Counter +import org.mockito.Mockito._ +import org.scalatest.mockito.MockitoSugar + +class MonitoringSpec extends BaseSpec with MockitoSugar { + + "increment" should { + "increment the counter for the given key" in new MonitoringFixtures { + val key = "test-key" + + when(mockMetrics.counter(key)).thenReturn(mockCounter) + + monitoring.increment(key) + + verify(mockMetrics).counter(key) + verify(mockCounter).increment() + } + } + + "recordException" should { + "increment the counter for a key based on the exception type" in new MonitoringFixtures { + val key = "exception.java_lang_Exception" + val exception = new Exception + when(mockMetrics.counter(key)).thenReturn(mockCounter) + + monitoring.recordException(exception) + + verify(mockMetrics).counter(key) + verify(mockCounter).increment() + } + } + + private class MonitoringFixtures { + val mockCounter = mock[Counter] + val mockMetrics = mock[MetricsModule] + + val monitoring = new Monitoring(){ + override val metrics: MetricsModule = mockMetrics + } + } + +} diff --git a/scheduler/src/test/scala/com/sky/kafkamessage/scheduler/PartitionedSinkSpec.scala b/scheduler/src/test/scala/com/sky/kms/PartitionedSinkSpec.scala similarity index 86% rename from scheduler/src/test/scala/com/sky/kafkamessage/scheduler/PartitionedSinkSpec.scala rename to scheduler/src/test/scala/com/sky/kms/PartitionedSinkSpec.scala index b8a76d02..feeeb70b 100644 --- a/scheduler/src/test/scala/com/sky/kafkamessage/scheduler/PartitionedSinkSpec.scala +++ b/scheduler/src/test/scala/com/sky/kms/PartitionedSinkSpec.scala @@ -1,7 +1,7 @@ -package com.sky.kafkamessage.scheduler +package com.sky.kms import akka.stream.scaladsl._ -import com.sky.kafkamessage.scheduler.streams.PartitionedSink +import com.sky.kms.streams.PartitionedSink import common.AkkaStreamBaseSpec import scala.concurrent.duration._ @@ -20,7 +20,7 @@ class PartitionedSinkSpec extends AkkaStreamBaseSpec { "withRight" should { "emit Right to rightSink and Left to leftSink" in { - val (leftFuture, rightFuture) = source.runWith(PartitionedSink.from(rightSink)) + val (leftFuture, rightFuture) = source.runWith(PartitionedSink.withRight(rightSink)) Await.result(leftFuture, Duration.Inf) shouldBe List(5) Await.result(rightFuture, Duration.Inf) shouldBe 14 diff --git a/scheduler/src/test/scala/com/sky/kafkamessage/scheduler/ScheduleReaderSpec.scala b/scheduler/src/test/scala/com/sky/kms/ScheduleReaderSpec.scala similarity index 83% rename from scheduler/src/test/scala/com/sky/kafkamessage/scheduler/ScheduleReaderSpec.scala rename to scheduler/src/test/scala/com/sky/kms/ScheduleReaderSpec.scala index 1675cdfb..36b24b77 100644 --- a/scheduler/src/test/scala/com/sky/kafkamessage/scheduler/ScheduleReaderSpec.scala +++ b/scheduler/src/test/scala/com/sky/kms/ScheduleReaderSpec.scala @@ -1,9 +1,9 @@ -package com.sky.kafkamessage.scheduler +package com.sky.kms import java.util.UUID -import com.sky.kafkamessage.scheduler.domain._ -import com.sky.kafkamessage.scheduler.streams.ScheduleReader +import com.sky.kms.domain._ +import com.sky.kms.streams.ScheduleReader import common.AkkaStreamBaseSpec import common.TestDataUtils._ diff --git a/scheduler/src/test/scala/com/sky/kafkamessage/scheduler/ScheduledMessagePublisherSpec.scala b/scheduler/src/test/scala/com/sky/kms/ScheduledMessagePublisherSpec.scala similarity index 77% rename from scheduler/src/test/scala/com/sky/kafkamessage/scheduler/ScheduledMessagePublisherSpec.scala rename to scheduler/src/test/scala/com/sky/kms/ScheduledMessagePublisherSpec.scala index 8e014817..65b518ab 100644 --- a/scheduler/src/test/scala/com/sky/kafkamessage/scheduler/ScheduledMessagePublisherSpec.scala +++ b/scheduler/src/test/scala/com/sky/kms/ScheduledMessagePublisherSpec.scala @@ -1,11 +1,11 @@ -package com.sky.kafkamessage.scheduler +package com.sky.kms import java.util.UUID -import com.sky.kafkamessage.scheduler.config._ -import com.sky.kafkamessage.scheduler.domain._ -import com.sky.kafkamessage.scheduler.kafka.KafkaStream -import com.sky.kafkamessage.scheduler.streams.ScheduledMessagePublisher +import com.sky.kms.config._ +import com.sky.kms.domain._ +import com.sky.kms.kafka.KafkaStream +import com.sky.kms.streams.ScheduledMessagePublisher import common.AkkaStreamBaseSpec import common.TestDataUtils.random import org.apache.kafka.clients.producer.ProducerRecord diff --git a/scheduler/src/test/scala/com/sky/kafkamessage/scheduler/SchedulerSpec.scala b/scheduler/src/test/scala/com/sky/kms/SchedulerSpec.scala similarity index 92% rename from scheduler/src/test/scala/com/sky/kafkamessage/scheduler/SchedulerSpec.scala rename to scheduler/src/test/scala/com/sky/kms/SchedulerSpec.scala index e288f61d..5222bebc 100644 --- a/scheduler/src/test/scala/com/sky/kafkamessage/scheduler/SchedulerSpec.scala +++ b/scheduler/src/test/scala/com/sky/kms/SchedulerSpec.scala @@ -1,13 +1,13 @@ -package com.sky.kafkamessage.scheduler +package com.sky.kms import java.time.OffsetDateTime import common.TestDataUtils._ -import com.sky.kafkamessage.scheduler.domain._ +import com.sky.kms.domain._ import common.BaseSpec import org.apache.kafka.clients.consumer.ConsumerRecord import avro._ -import com.sky.kafkamessage.scheduler.domain.ApplicationError._ +import com.sky.kms.domain.ApplicationError._ class SchedulerSpec extends BaseSpec { diff --git a/scheduler/src/test/scala/com/sky/kafkamessage/scheduler/SchedulingActorSpec.scala b/scheduler/src/test/scala/com/sky/kms/SchedulingActorSpec.scala similarity index 93% rename from scheduler/src/test/scala/com/sky/kafkamessage/scheduler/SchedulingActorSpec.scala rename to scheduler/src/test/scala/com/sky/kms/SchedulingActorSpec.scala index a59f5ab8..9f33218b 100644 --- a/scheduler/src/test/scala/com/sky/kafkamessage/scheduler/SchedulingActorSpec.scala +++ b/scheduler/src/test/scala/com/sky/kms/SchedulingActorSpec.scala @@ -1,4 +1,4 @@ -package com.sky.kafkamessage.scheduler +package com.sky.kms import java.util.UUID @@ -6,9 +6,9 @@ import akka.event.LoggingAdapter import akka.stream.scaladsl.SourceQueue import akka.testkit.{ImplicitSender, TestActorRef} import com.miguno.akka.testing.VirtualTime -import com.sky.kafkamessage.scheduler.SchedulingActor._ -import com.sky.kafkamessage.scheduler.domain.PublishableMessage.ScheduledMessage -import com.sky.kafkamessage.scheduler.domain._ +import com.sky.kms.SchedulingActor._ +import com.sky.kms.domain.PublishableMessage.ScheduledMessage +import com.sky.kms.domain._ import common.AkkaBaseSpec import common.TestDataUtils._ import org.mockito.Mockito._ diff --git a/scheduler/src/test/scala/com/sky/kafkamessage/scheduler/avro/AvroSpec.scala b/scheduler/src/test/scala/com/sky/kms/avro/AvroSpec.scala similarity index 94% rename from scheduler/src/test/scala/com/sky/kafkamessage/scheduler/avro/AvroSpec.scala rename to scheduler/src/test/scala/com/sky/kms/avro/AvroSpec.scala index 9ad29605..53e5a492 100644 --- a/scheduler/src/test/scala/com/sky/kafkamessage/scheduler/avro/AvroSpec.scala +++ b/scheduler/src/test/scala/com/sky/kms/avro/AvroSpec.scala @@ -1,9 +1,9 @@ -package com.sky.kafkamessage.scheduler.avro +package com.sky.kms.avro import java.time.OffsetDateTime import com.sksamuel.avro4s._ -import common.BaseSpec +import com.sky.kms.common.BaseSpec import org.apache.avro.Schema import org.apache.avro.generic.GenericData diff --git a/scheduler/src/test/scala/common/AkkaBaseSpec.scala b/scheduler/src/test/scala/com/sky/kms/common/AkkaBaseSpec.scala similarity index 91% rename from scheduler/src/test/scala/common/AkkaBaseSpec.scala rename to scheduler/src/test/scala/com/sky/kms/common/AkkaBaseSpec.scala index 068aefbb..0437ab2c 100644 --- a/scheduler/src/test/scala/common/AkkaBaseSpec.scala +++ b/scheduler/src/test/scala/com/sky/kms/common/AkkaBaseSpec.scala @@ -1,4 +1,4 @@ -package common +package com.sky.kms.common import akka.testkit.TestKit import org.scalatest.BeforeAndAfterAll diff --git a/scheduler/src/test/scala/common/AkkaStreamBaseSpec.scala b/scheduler/src/test/scala/com/sky/kms/common/AkkaStreamBaseSpec.scala similarity index 89% rename from scheduler/src/test/scala/common/AkkaStreamBaseSpec.scala rename to scheduler/src/test/scala/com/sky/kms/common/AkkaStreamBaseSpec.scala index 6f81180e..ba2038b2 100644 --- a/scheduler/src/test/scala/common/AkkaStreamBaseSpec.scala +++ b/scheduler/src/test/scala/com/sky/kms/common/AkkaStreamBaseSpec.scala @@ -1,4 +1,4 @@ -package common +package com.sky.kms.common import akka.stream.ActorMaterializer diff --git a/scheduler/src/test/scala/common/BaseSpec.scala b/scheduler/src/test/scala/com/sky/kms/common/BaseSpec.scala similarity index 83% rename from scheduler/src/test/scala/common/BaseSpec.scala rename to scheduler/src/test/scala/com/sky/kms/common/BaseSpec.scala index c6af67d1..5a4dbcab 100644 --- a/scheduler/src/test/scala/common/BaseSpec.scala +++ b/scheduler/src/test/scala/com/sky/kms/common/BaseSpec.scala @@ -1,4 +1,4 @@ -package common +package com.sky.kms.common import org.scalatest.{BeforeAndAfterAll, Matchers, WordSpecLike} diff --git a/scheduler/src/test/scala/common/EmbeddedKafka.scala b/scheduler/src/test/scala/com/sky/kms/common/EmbeddedKafka.scala similarity index 88% rename from scheduler/src/test/scala/common/EmbeddedKafka.scala rename to scheduler/src/test/scala/com/sky/kms/common/EmbeddedKafka.scala index 68fd7013..0df7c5e4 100644 --- a/scheduler/src/test/scala/common/EmbeddedKafka.scala +++ b/scheduler/src/test/scala/com/sky/kms/common/EmbeddedKafka.scala @@ -1,4 +1,4 @@ -package common +package com.sky.kms.common import cakesolutions.kafka.testkit.KafkaServer import cakesolutions.kafka.testkit.KafkaServer.defaultConsumerConfig @@ -14,9 +14,8 @@ object EmbeddedKafka { val bootstrapServer = s"localhost:${kafkaServer.kafkaPort}" - /* The consume method provided by [cakesolutions kafka testkit](https://github.com/cakesolutions/scala-kafka-client) - * doesn't provide a way of extracting a consumer record, we have added this so we can access the timestamp - * from a consumer record. + /** The consume method provided by [[cakesolutions.kafka.testkit.KafkaServer]] doesn't provide a way of + * extracting a consumer record, we have added this so we can access the timestamp from a consumer record. * * This should be contributed to the library. * diff --git a/scheduler/src/test/scala/common/KafkaIntSpec.scala b/scheduler/src/test/scala/com/sky/kms/common/KafkaIntSpec.scala similarity index 97% rename from scheduler/src/test/scala/common/KafkaIntSpec.scala rename to scheduler/src/test/scala/com/sky/kms/common/KafkaIntSpec.scala index 4d7e65d3..8b660308 100644 --- a/scheduler/src/test/scala/common/KafkaIntSpec.scala +++ b/scheduler/src/test/scala/com/sky/kms/common/KafkaIntSpec.scala @@ -1,4 +1,4 @@ -package common +package com.sky.kms.common import EmbeddedKafka._ import org.apache.kafka.clients.consumer.ConsumerRecord diff --git a/scheduler/src/test/scala/common/TestActorSystem.scala b/scheduler/src/test/scala/com/sky/kms/common/TestActorSystem.scala similarity index 96% rename from scheduler/src/test/scala/common/TestActorSystem.scala rename to scheduler/src/test/scala/com/sky/kms/common/TestActorSystem.scala index af1eeaa5..51d6999b 100644 --- a/scheduler/src/test/scala/common/TestActorSystem.scala +++ b/scheduler/src/test/scala/com/sky/kms/common/TestActorSystem.scala @@ -1,4 +1,4 @@ -package common +package com.sky.kms.common import java.util.UUID diff --git a/scheduler/src/test/scala/common/TestDataUtils.scala b/scheduler/src/test/scala/com/sky/kms/common/TestDataUtils.scala similarity index 88% rename from scheduler/src/test/scala/common/TestDataUtils.scala rename to scheduler/src/test/scala/com/sky/kms/common/TestDataUtils.scala index 97bba887..d35e96ff 100644 --- a/scheduler/src/test/scala/common/TestDataUtils.scala +++ b/scheduler/src/test/scala/com/sky/kms/common/TestDataUtils.scala @@ -1,4 +1,4 @@ -package common +package com.sky.kms.common import java.io.ByteArrayOutputStream import java.time._ @@ -7,10 +7,10 @@ import com.danielasfregola.randomdatagenerator.RandomDataGenerator import com.fortysevendeg.scalacheck.datetime.GenDateTime.genDateTimeWithinRange import com.fortysevendeg.scalacheck.datetime.instances.jdk8._ import com.sksamuel.avro4s.{AvroOutputStream, ToRecord} -import com.sky.kafkamessage.scheduler.domain._ +import com.sky.kms.domain._ import org.scalacheck._ -import com.sky.kafkamessage.scheduler.avro._ -import com.sky.kafkamessage.scheduler.domain.PublishableMessage.ScheduledMessage +import com.sky.kms.avro._ +import com.sky.kms.domain.PublishableMessage.ScheduledMessage object TestDataUtils extends RandomDataGenerator { diff --git a/scheduler/src/test/scala/com/sky/kafkamessage/scheduler/e2e/SchedulerIntSpec.scala b/scheduler/src/test/scala/com/sky/kms/e2e/SchedulerIntSpec.scala similarity index 82% rename from scheduler/src/test/scala/com/sky/kafkamessage/scheduler/e2e/SchedulerIntSpec.scala rename to scheduler/src/test/scala/com/sky/kms/e2e/SchedulerIntSpec.scala index 3c830b3e..46d65dc8 100644 --- a/scheduler/src/test/scala/com/sky/kafkamessage/scheduler/e2e/SchedulerIntSpec.scala +++ b/scheduler/src/test/scala/com/sky/kms/e2e/SchedulerIntSpec.scala @@ -1,13 +1,13 @@ -package com.sky.kafkamessage.scheduler.e2e +package com.sky.kms.e2e import java.util.UUID -import com.sky.kafkamessage.scheduler.avro._ -import com.sky.kafkamessage.scheduler.config._ -import com.sky.kafkamessage.scheduler.domain._ -import com.sky.kafkamessage.scheduler.streams.ScheduleReader -import common.TestDataUtils._ -import common.{AkkaStreamBaseSpec, KafkaIntSpec} +import com.sky.kms.avro._ +import com.sky.kms.config._ +import com.sky.kms.domain._ +import com.sky.kms.streams.ScheduleReader +import com.sky.kms.common.TestDataUtils._ +import com.sky.kms.common.{AkkaStreamBaseSpec, KafkaIntSpec} import org.apache.kafka.common.serialization._ import org.scalatest.Assertion import org.zalando.grafter.Rewriter From 3b12464fa88042c2e6b495fcc8e23bee149083bb Mon Sep 17 00:00:00 2001 From: Lawrence Carvalho Date: Tue, 8 Aug 2017 16:34:31 +0100 Subject: [PATCH 19/22] Remove unused implicits from AppConfig, put some default config in reference.conf. --- scheduler/src/main/resources/application.conf | 4 ---- scheduler/src/main/resources/reference.conf | 7 +++++++ scheduler/src/main/scala/com/sky/kms/config/package.scala | 4 +--- .../main/scala/com/sky/kms/domain/PublishableMessage.scala | 3 +-- 4 files changed, 9 insertions(+), 9 deletions(-) create mode 100644 scheduler/src/main/resources/reference.conf diff --git a/scheduler/src/main/resources/application.conf b/scheduler/src/main/resources/application.conf index 5bec25d7..b2bffe47 100644 --- a/scheduler/src/main/resources/application.conf +++ b/scheduler/src/main/resources/application.conf @@ -2,10 +2,6 @@ scheduler { schedule-topic = ${?SCHEDULE_TOPIC} kafka-brokers = "localhost:9092" kafka-brokers = ${?KAFKA_BROKERS} - shutdown-timeout { - stream = 10 seconds - system = 10 seconds - } } akka { diff --git a/scheduler/src/main/resources/reference.conf b/scheduler/src/main/resources/reference.conf new file mode 100644 index 00000000..82ac6297 --- /dev/null +++ b/scheduler/src/main/resources/reference.conf @@ -0,0 +1,7 @@ +scheduler { + shutdown-timeout { + stream = 10 seconds + system = 10 seconds + } + queue-buffer-size = 100 +} \ No newline at end of file diff --git a/scheduler/src/main/scala/com/sky/kms/config/package.scala b/scheduler/src/main/scala/com/sky/kms/config/package.scala index 0234f15a..06bcaeb6 100644 --- a/scheduler/src/main/scala/com/sky/kms/config/package.scala +++ b/scheduler/src/main/scala/com/sky/kms/config/package.scala @@ -1,14 +1,12 @@ package com.sky.kms -import akka.actor.ActorSystem -import akka.stream.ActorMaterializer import cats.data.Reader import scala.concurrent.duration.Duration package object config { - case class AppConfig(scheduler: SchedulerConfig)(implicit system: ActorSystem, materializer: ActorMaterializer) + case class AppConfig(scheduler: SchedulerConfig) case class SchedulerConfig(scheduleTopic: String, shutdownTimeout: ShutdownTimeout, queueBufferSize: Int) diff --git a/scheduler/src/main/scala/com/sky/kms/domain/PublishableMessage.scala b/scheduler/src/main/scala/com/sky/kms/domain/PublishableMessage.scala index ff349f68..a83d1d82 100644 --- a/scheduler/src/main/scala/com/sky/kms/domain/PublishableMessage.scala +++ b/scheduler/src/main/scala/com/sky/kms/domain/PublishableMessage.scala @@ -16,8 +16,7 @@ object PublishableMessage { implicit val scheduleDeletionProducerRecordEnc: ProducerRecordEncoder[ScheduleDeletion] = ProducerRecordEncoder.instance(deletion => new ProducerRecord(deletion.scheduleTopic, deletion.scheduleId.getBytes, null)) - - + implicit def scheduleDataToProducerRecord(msg: PublishableMessage): ProducerRecord[Array[Byte], Array[Byte]] = msg match { case scheduledMsg: ScheduledMessage => scheduledMessageProducerRecordEnc(scheduledMsg) From 39063b96611fac5151115105e3ba494aa0df8766 Mon Sep 17 00:00:00 2001 From: Lawrence Carvalho Date: Tue, 8 Aug 2017 19:53:34 +0100 Subject: [PATCH 20/22] Removed monitoring stuff as it has been agreed that it will be done separately. --- README.md | 6 +++ build.sbt | 8 +++- .../scala/com/sky/kms/AkkaComponents.scala | 16 ++----- .../main/scala/com/sky/kms/Monitoring.scala | 20 -------- .../com/sky/kms/streams/ScheduleReader.scala | 13 ++++-- .../streams/ScheduledMessagePublisher.scala | 7 +-- .../com/sky/kms/AkkaComponentsSpec.scala | 38 --------------- .../scala/com/sky/kms/MonitoringSpec.scala | 46 ------------------- .../kms/ScheduledMessagePublisherSpec.scala | 4 +- .../com/sky/kms/SchedulingActorSpec.scala | 2 +- 10 files changed, 32 insertions(+), 128 deletions(-) delete mode 100644 scheduler/src/main/scala/com/sky/kms/Monitoring.scala delete mode 100644 scheduler/src/test/scala/com/sky/kms/AkkaComponentsSpec.scala delete mode 100644 scheduler/src/test/scala/com/sky/kms/MonitoringSpec.scala diff --git a/README.md b/README.md index 71e62ccd..44bbf637 100644 --- a/README.md +++ b/README.md @@ -57,3 +57,9 @@ for generating the Avro schema. JMX metrics are exposed using Kamon. Port 9186 has to be exposed to obtain them. +### Topic configuration + +The `schedule-topic` must be configured to use [log compaction](https://kafka.apache.org/documentation/#compaction). +This is to allow the scheduler to delete the schedule after it has been written to its destination topic. This is very +important because the scheduler uses the `schedule-topic` to reconstruct its state. + diff --git a/build.sbt b/build.sbt index 305973a2..647cea16 100644 --- a/build.sbt +++ b/build.sbt @@ -74,7 +74,13 @@ lazy val scheduler = (project in file("scheduler")) libraryDependencies ++= dependencies, resolvers += Resolver.bintrayRepo("cakesolutions", "maven"), addCompilerPlugin("org.scalamacros" % "paradise" % "2.1.0" cross CrossVersion.full), - scalacOptions += "-language:implicitConversions", + scalacOptions ++= Seq( + "-language:implicitConversions", + "-language:postfixOps", + "-Xfatal-warnings", + "-Ywarn-dead-code", + "-encoding", "utf-8" + ), fork in run := true, javaAgents += "org.aspectj" % "aspectjweaver" % "1.8.10", javaOptions in Universal += jmxSettings, diff --git a/scheduler/src/main/scala/com/sky/kms/AkkaComponents.scala b/scheduler/src/main/scala/com/sky/kms/AkkaComponents.scala index f726cbb3..80ed9d14 100644 --- a/scheduler/src/main/scala/com/sky/kms/AkkaComponents.scala +++ b/scheduler/src/main/scala/com/sky/kms/AkkaComponents.scala @@ -1,23 +1,13 @@ package com.sky.kms import akka.actor.ActorSystem -import akka.stream.Supervision.Restart -import akka.stream.{ActorMaterializer, ActorMaterializerSettings, Supervision} +import akka.stream.ActorMaterializer import com.typesafe.scalalogging.LazyLogging -trait AkkaComponents extends LazyLogging with Monitoring { +trait AkkaComponents extends LazyLogging { implicit val system = ActorSystem("kafka-message-scheduler") - val decider: Supervision.Decider = { t => - recordException(t) - logger.error(s"Supervision failed.", t) - Restart - } - - val settings = ActorMaterializerSettings(system) - .withSupervisionStrategy(decider) - - implicit val materializer = ActorMaterializer(settings) + implicit val materializer = ActorMaterializer() } diff --git a/scheduler/src/main/scala/com/sky/kms/Monitoring.scala b/scheduler/src/main/scala/com/sky/kms/Monitoring.scala deleted file mode 100644 index 945d078a..00000000 --- a/scheduler/src/main/scala/com/sky/kms/Monitoring.scala +++ /dev/null @@ -1,20 +0,0 @@ -package com.sky.kms - -import kamon.Kamon -import kamon.metric.MetricsModule - -trait Monitoring { - - val metrics: MetricsModule = Kamon.metrics - - def increment(key: String) = metrics.counter(key).increment() - - def recordException(throwable: Throwable) = { - val key = generateKeyFromException(throwable) - metrics.counter(key).increment() - } - - private def generateKeyFromException(throwable: Throwable): String = { - return s"exception.${throwable.getClass.getName.replace(".", "_")}" - } -} diff --git a/scheduler/src/main/scala/com/sky/kms/streams/ScheduleReader.scala b/scheduler/src/main/scala/com/sky/kms/streams/ScheduleReader.scala index 5ea982ad..b4697127 100644 --- a/scheduler/src/main/scala/com/sky/kms/streams/ScheduleReader.scala +++ b/scheduler/src/main/scala/com/sky/kms/streams/ScheduleReader.scala @@ -11,17 +11,18 @@ import com.sky.kms._ import com.sky.kms.config.{AppConfig, SchedulerConfig} import com.sky.kms.domain._ import com.sky.kms.kafka._ +import com.sky.kms.streams.ScheduleReader.{In, Mat} import com.typesafe.scalalogging.LazyLogging /** * Provides stream from the schedule source to the scheduling actor. */ case class ScheduleReader(config: SchedulerConfig, - scheduleSource: Source[Either[ApplicationError, (ScheduleId, Option[Schedule])], Control], + scheduleSource: Source[In, Mat], schedulingSink: Sink[Any, NotUsed]) (implicit system: ActorSystem, materializer: ActorMaterializer) extends ScheduleReaderStream { - val stream: Control = + val stream: Mat = scheduleSource .map(ScheduleReader.toSchedulingMessage) .to(PartitionedSink.withRight(schedulingSink)) @@ -30,8 +31,12 @@ case class ScheduleReader(config: SchedulerConfig, object ScheduleReader extends LazyLogging { - def toSchedulingMessage[T](either: Either[ApplicationError, (ScheduleId, Option[Schedule])]): Either[ApplicationError, SchedulingMessage] = - either.map { case (scheduleId, scheduleOpt) => + type In = Either[ApplicationError, (ScheduleId, Option[Schedule])] + + type Mat = Control + + def toSchedulingMessage[T](readResult: In): Either[ApplicationError, SchedulingMessage] = + readResult.map { case (scheduleId, scheduleOpt) => scheduleOpt match { case Some(schedule) => logger.info(s"Publishing scheduled message with ID: $scheduleId to topic: ${schedule.topic}") diff --git a/scheduler/src/main/scala/com/sky/kms/streams/ScheduledMessagePublisher.scala b/scheduler/src/main/scala/com/sky/kms/streams/ScheduledMessagePublisher.scala index e1d73cfb..a2d56e50 100644 --- a/scheduler/src/main/scala/com/sky/kms/streams/ScheduledMessagePublisher.scala +++ b/scheduler/src/main/scala/com/sky/kms/streams/ScheduledMessagePublisher.scala @@ -15,7 +15,8 @@ import org.apache.kafka.clients.producer.ProducerRecord import scala.concurrent.Future /** - * Provides stream from the queue of due messages to kafka + * Provides a stream that consumes from the queue of triggered messages, + * writes the messages to Kafka and then deletes the schedules from Kafka */ case class ScheduledMessagePublisher(config: SchedulerConfig, publisherSink: Sink[In, Mat]) (implicit system: ActorSystem, materializer: ActorMaterializer) @@ -23,11 +24,11 @@ case class ScheduledMessagePublisher(config: SchedulerConfig, publisherSink: Sin def stream: SourceQueueWithComplete[(ScheduleId, ScheduledMessage)] = Source.queue[(ScheduleId, ScheduledMessage)](config.queueBufferSize, OverflowStrategy.backpressure) - .mapConcat(splitToScheduleAndMetadata) + .mapConcat(splitToMessageAndDeletion) .to(publisherSink) .run() - val splitToScheduleAndMetadata: ((ScheduleId, ScheduledMessage)) => List[In] = { + val splitToMessageAndDeletion: ((ScheduleId, ScheduledMessage)) => List[In] = { case (scheduleId, scheduledMessage) => logger.info(s"Publishing scheduled message $scheduleId to ${scheduledMessage.topic} and deleting it from ${config.scheduleTopic}") List(scheduledMessage, ScheduleDeletion(scheduleId, config.scheduleTopic)) diff --git a/scheduler/src/test/scala/com/sky/kms/AkkaComponentsSpec.scala b/scheduler/src/test/scala/com/sky/kms/AkkaComponentsSpec.scala deleted file mode 100644 index f5f644a8..00000000 --- a/scheduler/src/test/scala/com/sky/kms/AkkaComponentsSpec.scala +++ /dev/null @@ -1,38 +0,0 @@ -package com.sky.kms - -import akka.stream.Supervision.Restart -import common.BaseSpec -import kamon.metric.MetricsModule -import kamon.metric.instrument.Counter -import org.mockito.Mockito.{verify, when} -import org.scalatest.mockito.MockitoSugar - -import scala.concurrent.Await -import scala.concurrent.duration._ - -class AkkaComponentsSpec extends BaseSpec with MockitoSugar { - - "decider" should { - "collect metrics from failures" in new AkkaComponentsFixture { - val key = "exception.java_lang_Exception" - val exception = new Exception("Any exception") - - when(metrics.counter(key)).thenReturn(mockCounter) - - materializer - .settings - .supervisionDecider(exception) shouldBe Restart - - verify(metrics).counter(key) - verify(mockCounter).increment() - - materializer.shutdown() - Await.ready(system.terminate(), 5 seconds) - } - } - - private class AkkaComponentsFixture extends AkkaComponents { - val mockCounter = mock[Counter] - override val metrics: MetricsModule = mock[MetricsModule] - } -} diff --git a/scheduler/src/test/scala/com/sky/kms/MonitoringSpec.scala b/scheduler/src/test/scala/com/sky/kms/MonitoringSpec.scala deleted file mode 100644 index d20dc84e..00000000 --- a/scheduler/src/test/scala/com/sky/kms/MonitoringSpec.scala +++ /dev/null @@ -1,46 +0,0 @@ -package com.sky.kms - -import common.BaseSpec -import kamon.metric.MetricsModule -import kamon.metric.instrument.Counter -import org.mockito.Mockito._ -import org.scalatest.mockito.MockitoSugar - -class MonitoringSpec extends BaseSpec with MockitoSugar { - - "increment" should { - "increment the counter for the given key" in new MonitoringFixtures { - val key = "test-key" - - when(mockMetrics.counter(key)).thenReturn(mockCounter) - - monitoring.increment(key) - - verify(mockMetrics).counter(key) - verify(mockCounter).increment() - } - } - - "recordException" should { - "increment the counter for a key based on the exception type" in new MonitoringFixtures { - val key = "exception.java_lang_Exception" - val exception = new Exception - when(mockMetrics.counter(key)).thenReturn(mockCounter) - - monitoring.recordException(exception) - - verify(mockMetrics).counter(key) - verify(mockCounter).increment() - } - } - - private class MonitoringFixtures { - val mockCounter = mock[Counter] - val mockMetrics = mock[MetricsModule] - - val monitoring = new Monitoring(){ - override val metrics: MetricsModule = mockMetrics - } - } - -} diff --git a/scheduler/src/test/scala/com/sky/kms/ScheduledMessagePublisherSpec.scala b/scheduler/src/test/scala/com/sky/kms/ScheduledMessagePublisherSpec.scala index 65b518ab..53779a0d 100644 --- a/scheduler/src/test/scala/com/sky/kms/ScheduledMessagePublisherSpec.scala +++ b/scheduler/src/test/scala/com/sky/kms/ScheduledMessagePublisherSpec.scala @@ -21,10 +21,10 @@ class ScheduledMessagePublisherSpec extends AkkaStreamBaseSpec { KafkaStream.sink ) - "splitToScheduleAndMetadata" should { + "splitToMessageAndDeletion" should { "split schedule and convert to producer records" in { val (scheduleId, schedule) = (UUID.randomUUID().toString, random[Schedule]) - publisher.splitToScheduleAndMetadata((scheduleId, schedule.toScheduledMessage)) === List( + publisher.splitToMessageAndDeletion((scheduleId, schedule.toScheduledMessage)) === List( new ProducerRecord(schedule.topic, schedule.key, schedule.value), new ProducerRecord(testTopic, scheduleId.getBytes, null) ) diff --git a/scheduler/src/test/scala/com/sky/kms/SchedulingActorSpec.scala b/scheduler/src/test/scala/com/sky/kms/SchedulingActorSpec.scala index 9f33218b..5ffadf9f 100644 --- a/scheduler/src/test/scala/com/sky/kms/SchedulingActorSpec.scala +++ b/scheduler/src/test/scala/com/sky/kms/SchedulingActorSpec.scala @@ -16,7 +16,7 @@ import org.scalatest.mockito.MockitoSugar class SchedulingActorSpec extends AkkaBaseSpec with ImplicitSender with MockitoSugar { - "A scheduler actor" must { + "A scheduling actor" must { "schedule new messages at the given time" in new SchedulingActorTest { val (scheduleId, schedule) = generateSchedule() From 3a81593891f22676b30c2517c434fd03f78785f1 Mon Sep 17 00:00:00 2001 From: Lawrence Carvalho Date: Tue, 8 Aug 2017 20:46:55 +0100 Subject: [PATCH 21/22] wait for Init message in actor before handling other messages. --- .../scala/com/sky/kms/SchedulingActor.scala | 32 +++++++++-------- .../sky/kms/kafka/ConsumerRecordDecoder.scala | 3 +- .../com/sky/kms/SchedulingActorSpec.scala | 34 ++++++++++++++----- 3 files changed, 45 insertions(+), 24 deletions(-) diff --git a/scheduler/src/main/scala/com/sky/kms/SchedulingActor.scala b/scheduler/src/main/scala/com/sky/kms/SchedulingActor.scala index fe14a83c..e7b87572 100644 --- a/scheduler/src/main/scala/com/sky/kms/SchedulingActor.scala +++ b/scheduler/src/main/scala/com/sky/kms/SchedulingActor.scala @@ -17,13 +17,19 @@ import com.sky.kms.streams.ScheduledMessagePublisher import scala.concurrent.ExecutionContext.Implicits.global import scala.concurrent.duration.FiniteDuration -class SchedulingActor(sourceQueue: SourceQueue[(String, ScheduledMessage)], scheduler: Scheduler) extends Actor with ActorLogging { +class SchedulingActor(queue: SourceQueue[(String, ScheduledMessage)], scheduler: Scheduler) extends Actor with ActorLogging { - override def receive: Receive = receiveScheduleMessages(Map.empty) + override def receive: Receive = waitForInit - def receiveScheduleMessages(schedules: Map[ScheduleId, Cancellable]): Receive = { + private val waitForInit: Receive = { + case Init => + context.become(receiveWithSchedules(Map.empty)) + sender ! Ack + } + + private def receiveWithSchedules(schedules: Map[ScheduleId, Cancellable]): Receive = { - val receiveSchedulingMessage: PartialFunction[Any, Map[ScheduleId, Cancellable]] = { + val handleSchedulingMessage: PartialFunction[Any, Map[ScheduleId, Cancellable]] = { case CreateOrUpdate(scheduleId: ScheduleId, schedule: Schedule) => if (cancel(scheduleId, schedules)) log.info(s"Updating schedule $scheduleId") @@ -40,22 +46,20 @@ class SchedulingActor(sourceQueue: SourceQueue[(String, ScheduledMessage)], sche schedules - scheduleId } - val receiveTriggerMessage: PartialFunction[Any, Unit] = { - case Trigger(scheduleId, schedule) => - log.info(s"$scheduleId is due. Adding schedule to queue. Scheduled time was ${schedule.time}") - sourceQueue.offer((scheduleId, messageFrom(schedule))) - } - - (receiveSchedulingMessage andThen updateStateAndAck) orElse receiveTriggerMessage orElse { - case Init => sender ! Ack - } + (handleSchedulingMessage andThen updateStateAndAck) orElse handleTrigger } private def updateStateAndAck(schedules: Map[ScheduleId, Cancellable]): Unit = { - context.become(receiveScheduleMessages(schedules)) + context.become(receiveWithSchedules(schedules)) sender ! Ack } + private val handleTrigger: Receive = { + case Trigger(scheduleId, schedule) => + log.info(s"$scheduleId is due. Adding schedule to queue. Scheduled time was ${schedule.time}") + queue.offer((scheduleId, messageFrom(schedule))) + } + private def cancel(scheduleId: ScheduleId, schedules: Map[ScheduleId, Cancellable]): Boolean = schedules.get(scheduleId).exists(_.cancel()) diff --git a/scheduler/src/main/scala/com/sky/kms/kafka/ConsumerRecordDecoder.scala b/scheduler/src/main/scala/com/sky/kms/kafka/ConsumerRecordDecoder.scala index c4848fa5..922793b6 100644 --- a/scheduler/src/main/scala/com/sky/kms/kafka/ConsumerRecordDecoder.scala +++ b/scheduler/src/main/scala/com/sky/kms/kafka/ConsumerRecordDecoder.scala @@ -8,7 +8,6 @@ trait ConsumerRecordDecoder[T] { object ConsumerRecordDecoder { def instance[T](f: ConsumerRecord[String, Array[Byte]] => T) = new ConsumerRecordDecoder[T] { - final def apply(cr: ConsumerRecord[String, Array[Byte]]): T = - f(cr) + final def apply(cr: ConsumerRecord[String, Array[Byte]]): T = f(cr) } } diff --git a/scheduler/src/test/scala/com/sky/kms/SchedulingActorSpec.scala b/scheduler/src/test/scala/com/sky/kms/SchedulingActorSpec.scala index 5ffadf9f..206a7e4c 100644 --- a/scheduler/src/test/scala/com/sky/kms/SchedulingActorSpec.scala +++ b/scheduler/src/test/scala/com/sky/kms/SchedulingActorSpec.scala @@ -2,20 +2,24 @@ package com.sky.kms import java.util.UUID +import akka.actor.ActorRef import akka.event.LoggingAdapter import akka.stream.scaladsl.SourceQueue import akka.testkit.{ImplicitSender, TestActorRef} import com.miguno.akka.testing.VirtualTime import com.sky.kms.SchedulingActor._ +import com.sky.kms.common.AkkaBaseSpec +import com.sky.kms.common.TestDataUtils._ import com.sky.kms.domain.PublishableMessage.ScheduledMessage import com.sky.kms.domain._ -import common.AkkaBaseSpec -import common.TestDataUtils._ import org.mockito.Mockito._ import org.scalatest.mockito.MockitoSugar class SchedulingActorSpec extends AkkaBaseSpec with ImplicitSender with MockitoSugar { + val mockLogger = mock[LoggingAdapter] + val mockSourceQueue = mock[SourceQueue[(ScheduleId, ScheduledMessage)]] + "A scheduling actor" must { "schedule new messages at the given time" in new SchedulingActorTest { val (scheduleId, schedule) = generateSchedule() @@ -60,8 +64,17 @@ class SchedulingActorSpec extends AkkaBaseSpec with ImplicitSender with MockitoS verify(mockSourceQueue).offer((scheduleId, updatedSchedule.toScheduledMessage)) } - "does nothing when an Init message is received" in new SchedulingActorTest { - actorRef ! Init + "accept scheduling messages only after it has received an Init" in { + + val actorRef = TestActorRef(new SchedulingActor(mockSourceQueue, system.scheduler)) + val (scheduleId, schedule) = generateSchedule() + + actorRef ! CreateOrUpdate(scheduleId, schedule) + expectNoMsg() + + init(actorRef) + + actorRef ! CreateOrUpdate(scheduleId, schedule) expectMsg(Ack) } @@ -70,16 +83,13 @@ class SchedulingActorSpec extends AkkaBaseSpec with ImplicitSender with MockitoS private class SchedulingActorTest { val now = System.currentTimeMillis() - val mockLogger = mock[LoggingAdapter] - val mockSourceQueue = mock[SourceQueue[(ScheduleId, ScheduledMessage)]] val time = new VirtualTime val actorRef = TestActorRef(new SchedulingActor(mockSourceQueue, time.scheduler) { override def log: LoggingAdapter = mockLogger }) - def generateSchedule(): (ScheduleId, Schedule) = - (UUID.randomUUID().toString, random[Schedule]) + init(actorRef) def advanceToTimeFrom(schedule: Schedule, startTime: Long = now): Unit = time.advance(schedule.timeInMillis - startTime) @@ -95,4 +105,12 @@ class SchedulingActorSpec extends AkkaBaseSpec with ImplicitSender with MockitoS } } + private def generateSchedule(): (ScheduleId, Schedule) = + (UUID.randomUUID().toString, random[Schedule]) + + private def init(actorRef: ActorRef) = { + actorRef ! Init + expectMsg(Ack) + } + } \ No newline at end of file From 837e36cd02af28506f08baad26656bce1d906e0b Mon Sep 17 00:00:00 2001 From: Lawrence Carvalho Date: Wed, 9 Aug 2017 08:51:00 +0100 Subject: [PATCH 22/22] Address review comments. --- README.md | 5 +++-- build.sbt | 2 +- .../src/main/scala/com/sky/kms/SchedulingActor.scala | 4 ++-- .../scala/com/sky/kms/domain/PublishableMessage.scala | 2 +- .../com/sky/kms/streams/ScheduledMessagePublisher.scala | 3 ++- .../test/scala/com/sky/kms/common/EmbeddedKafka.scala | 9 ++++----- 6 files changed, 13 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 44bbf637..ce32733d 100644 --- a/README.md +++ b/README.md @@ -61,5 +61,6 @@ JMX metrics are exposed using Kamon. Port 9186 has to be exposed to obtain them. The `schedule-topic` must be configured to use [log compaction](https://kafka.apache.org/documentation/#compaction). This is to allow the scheduler to delete the schedule after it has been written to its destination topic. This is very -important because the scheduler uses the `schedule-topic` to reconstruct its state. - +important because the scheduler uses the `schedule-topic` to reconstruct its state. This application will also support +longer-term schedules so log compaction is required to ensure they are not prematurely removed from Kafka allowing the +application to recover them after a restart. \ No newline at end of file diff --git a/build.sbt b/build.sbt index 647cea16..dce93366 100644 --- a/build.sbt +++ b/build.sbt @@ -26,7 +26,7 @@ val dependencies = Seq( "com.typesafe.akka" %% "akka-testkit" % akkaVersion % Test, "com.typesafe.akka" %% "akka-stream-testkit" % akkaVersion % Test, "net.cakesolutions" %% "scala-kafka-client-testkit" % kafkaVersion % Test, - "org.slf4j" % "log4j-over-slf4j" % "1.7.21" % Test, + "org.slf4j" % "log4j-over-slf4j" % "1.7.25" % Test, "com.danielasfregola" %% "random-data-generator" % "2.1" % Test, "com.47deg" %% "scalacheck-toolbox-datetime"% "0.2.2" % Test, "com.miguno.akka" %% "akka-mock-scheduler" % "0.5.1" % Test, diff --git a/scheduler/src/main/scala/com/sky/kms/SchedulingActor.scala b/scheduler/src/main/scala/com/sky/kms/SchedulingActor.scala index e7b87572..7fad2840 100644 --- a/scheduler/src/main/scala/com/sky/kms/SchedulingActor.scala +++ b/scheduler/src/main/scala/com/sky/kms/SchedulingActor.scala @@ -17,7 +17,7 @@ import com.sky.kms.streams.ScheduledMessagePublisher import scala.concurrent.ExecutionContext.Implicits.global import scala.concurrent.duration.FiniteDuration -class SchedulingActor(queue: SourceQueue[(String, ScheduledMessage)], scheduler: Scheduler) extends Actor with ActorLogging { +class SchedulingActor(queue: SourceQueue[(String, ScheduledMessage)], akkaScheduler: Scheduler) extends Actor with ActorLogging { override def receive: Receive = waitForInit @@ -35,7 +35,7 @@ class SchedulingActor(queue: SourceQueue[(String, ScheduledMessage)], scheduler: log.info(s"Updating schedule $scheduleId") else log.info(s"Creating schedule $scheduleId") - val cancellable = scheduler.scheduleOnce(timeFromNow(schedule.time))(self ! Trigger(scheduleId, schedule)) + val cancellable = akkaScheduler.scheduleOnce(timeFromNow(schedule.time))(self ! Trigger(scheduleId, schedule)) schedules + (scheduleId -> cancellable) case Cancel(scheduleId: String) => diff --git a/scheduler/src/main/scala/com/sky/kms/domain/PublishableMessage.scala b/scheduler/src/main/scala/com/sky/kms/domain/PublishableMessage.scala index a83d1d82..d7f5b6a1 100644 --- a/scheduler/src/main/scala/com/sky/kms/domain/PublishableMessage.scala +++ b/scheduler/src/main/scala/com/sky/kms/domain/PublishableMessage.scala @@ -17,7 +17,7 @@ object PublishableMessage { implicit val scheduleDeletionProducerRecordEnc: ProducerRecordEncoder[ScheduleDeletion] = ProducerRecordEncoder.instance(deletion => new ProducerRecord(deletion.scheduleTopic, deletion.scheduleId.getBytes, null)) - implicit def scheduleDataToProducerRecord(msg: PublishableMessage): ProducerRecord[Array[Byte], Array[Byte]] = + implicit def publishableMessageToProducerRecord(msg: PublishableMessage): ProducerRecord[Array[Byte], Array[Byte]] = msg match { case scheduledMsg: ScheduledMessage => scheduledMessageProducerRecordEnc(scheduledMsg) case deletion: ScheduleDeletion => scheduleDeletionProducerRecordEnc(deletion) diff --git a/scheduler/src/main/scala/com/sky/kms/streams/ScheduledMessagePublisher.scala b/scheduler/src/main/scala/com/sky/kms/streams/ScheduledMessagePublisher.scala index a2d56e50..d2900ed4 100644 --- a/scheduler/src/main/scala/com/sky/kms/streams/ScheduledMessagePublisher.scala +++ b/scheduler/src/main/scala/com/sky/kms/streams/ScheduledMessagePublisher.scala @@ -16,7 +16,8 @@ import scala.concurrent.Future /** * Provides a stream that consumes from the queue of triggered messages, - * writes the messages to Kafka and then deletes the schedules from Kafka + * writes the scheduled messages to the specified Kafka topics and then deletes the schedules + * from the scheduling Kafka topic to mark completion */ case class ScheduledMessagePublisher(config: SchedulerConfig, publisherSink: Sink[In, Mat]) (implicit system: ActorSystem, materializer: ActorMaterializer) diff --git a/scheduler/src/test/scala/com/sky/kms/common/EmbeddedKafka.scala b/scheduler/src/test/scala/com/sky/kms/common/EmbeddedKafka.scala index 0df7c5e4..4b01df3d 100644 --- a/scheduler/src/test/scala/com/sky/kms/common/EmbeddedKafka.scala +++ b/scheduler/src/test/scala/com/sky/kms/common/EmbeddedKafka.scala @@ -14,11 +14,10 @@ object EmbeddedKafka { val bootstrapServer = s"localhost:${kafkaServer.kafkaPort}" - /** The consume method provided by [[cakesolutions.kafka.testkit.KafkaServer]] doesn't provide a way of - * extracting a consumer record, we have added this so we can access the timestamp from a consumer record. - * - * This should be contributed to the library. - * + /** + * The consume method provided by [[cakesolutions.kafka.testkit.KafkaServer]] doesn't provide a way of + * extracting a consumer record, we have added this so we can access the timestamp from a consumer record. In the + * future we should contribute this to the library as it provides better flexibility. */ implicit class KafkaServerOps(val kafkaServer: KafkaServer) extends AnyVal {