diff --git a/.gitignore b/.gitignore index 8e01bdaa..c53d5ef3 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ build/ /Android/src/main/assets/LondonTravel.data.*.sql /.idea/ +!/.idea/runConfigurations/ *.ipr *.iml *.iws diff --git a/.idea/runConfigurations/Run_status_history_server.xml b/.idea/runConfigurations/Run_status_history_server.xml new file mode 100644 index 00000000..86ac92da --- /dev/null +++ b/.idea/runConfigurations/Run_status_history_server.xml @@ -0,0 +1,21 @@ + + + + + + + + + + + \ No newline at end of file diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md index da93e156..4af75b10 100644 --- a/docs/DEVELOPMENT.md +++ b/docs/DEVELOPMENT.md @@ -19,13 +19,13 @@ After this runs hit: ### Update AppEngine ```shell -$ gradlew appengineRun -> Task :AppEngine:checkCloudSdk FAILED +$ gradlew :web:status-history:appengineRun +> Task :web:status-history:checkCloudSdk FAILED FAILURE: Build failed with an exception. * What went wrong: - Execution failed for task ':AppEngine:checkCloudSdk'. + Execution failed for task ':web:status-history:checkCloudSdk'. > Specified Cloud SDK version (347.0.0) does not match installed version (319.0.0). ``` diff --git a/domain/status/build.gradle b/domain/status/build.gradle new file mode 100644 index 00000000..2ba2d29f --- /dev/null +++ b/domain/status/build.gradle @@ -0,0 +1,30 @@ +plugins { + id("net.twisterrob.blt.convention") + id("org.jetbrains.kotlin.multiplatform") + id("com.google.devtools.ksp") +} + +kotlin { + jvm() + sourceSets { + commonMain { + dependencies { + api(libs.kotlin.datetime) + api(libs.kotlin.serialization) + //implementation("io.github.oshai:kotlin-logging:6.0.1") + } + } + commonTest { + dependencies { + implementation(libs.kotlin.test) + implementation(libs.test.mockative) + } + } + } +} + +configurations + .matching { Configuration it -> it.name.startsWith("ksp") && it.name.endsWith("Test") } + .configureEach { Configuration it -> + project.dependencies.add(it.name, libs.test.mockative.processor) + } diff --git a/domain/status/src/commonMain/kotlin/net/twisterrob/travel/domain/london/status/DomainHistoryUseCase.kt b/domain/status/src/commonMain/kotlin/net/twisterrob/travel/domain/london/status/DomainHistoryUseCase.kt new file mode 100644 index 00000000..2d47750f --- /dev/null +++ b/domain/status/src/commonMain/kotlin/net/twisterrob/travel/domain/london/status/DomainHistoryUseCase.kt @@ -0,0 +1,44 @@ +package net.twisterrob.travel.domain.london.status + +import net.twisterrob.travel.domain.london.status.api.FeedParser +import net.twisterrob.travel.domain.london.status.api.HistoryUseCase +import net.twisterrob.travel.domain.london.status.api.ParsedStatusItem +import net.twisterrob.travel.domain.london.status.api.StatusHistoryRepository +import net.twisterrob.travel.domain.london.status.api.StatusInteractor + +class DomainHistoryUseCase( + private val statusHistoryRepository: StatusHistoryRepository, + private val statusInteractor: StatusInteractor, + private val feedParser: FeedParser, +) : HistoryUseCase { + + /** + * @param max maximum number of items to return, current is not included in the count. + * @param includeCurrent whether to include the current status in the result. + */ + override fun history(feed: Feed, max: Int, includeCurrent: Boolean): List { + val result = mutableListOf() + if (includeCurrent) { + result.add(statusInteractor.getCurrent(feed)) + } + val history = statusHistoryRepository.getAll(feed, max) + result.addAll(history) + return result.map { it.parse() } + } + + private fun StatusItem.parse(): ParsedStatusItem = + when (this) { + is StatusItem.SuccessfulStatusItem -> { + try { + val feedContents = feedParser.parse(this.feed, this.content) + ParsedStatusItem.ParsedFeed(this, feedContents) + } catch (ex: Exception) { + ParsedStatusItem.ParseFailed(this, Stacktrace(ex.stackTraceToString())) + } + } + + is StatusItem.FailedStatusItem -> { + ParsedStatusItem.AlreadyFailed(this) + } + } +} diff --git a/domain/status/src/commonMain/kotlin/net/twisterrob/travel/domain/london/status/DomainRefreshUseCase.kt b/domain/status/src/commonMain/kotlin/net/twisterrob/travel/domain/london/status/DomainRefreshUseCase.kt new file mode 100644 index 00000000..54d136c5 --- /dev/null +++ b/domain/status/src/commonMain/kotlin/net/twisterrob/travel/domain/london/status/DomainRefreshUseCase.kt @@ -0,0 +1,51 @@ +package net.twisterrob.travel.domain.london.status + +import net.twisterrob.travel.domain.london.status.api.RefreshResult +import net.twisterrob.travel.domain.london.status.api.RefreshUseCase +import net.twisterrob.travel.domain.london.status.api.StatusHistoryRepository +import net.twisterrob.travel.domain.london.status.api.StatusInteractor + +class DomainRefreshUseCase( + private val statusHistoryRepository: StatusHistoryRepository, + private val statusInteractor: StatusInteractor, +) : RefreshUseCase { + + override fun refreshLatest(feed: Feed): RefreshResult { + val current = statusInteractor.getCurrent(feed) + val latest = statusHistoryRepository.getAll(feed, 1).singleOrNull() + + return when { + latest == null -> { + statusHistoryRepository.add(current) + RefreshResult.Created(current) + } + + sameContent(latest, current) -> { + RefreshResult.NoChange(current, latest) + } + + sameError(latest, current) -> { + RefreshResult.NoChange(current, latest) + } + + else -> { + statusHistoryRepository.add(current) + RefreshResult.Refreshed(current, latest) + } + } + } + + private fun sameContent(latest: StatusItem, current: StatusItem): Boolean = + if (latest is StatusItem.SuccessfulStatusItem && current is StatusItem.SuccessfulStatusItem) { + latest.content == current.content + } else { + false + } + + private fun sameError(latest: StatusItem, current: StatusItem): Boolean = + if (latest is StatusItem.FailedStatusItem && current is StatusItem.FailedStatusItem) { + latest.error == current.error + } else { + false + } +} diff --git a/domain/status/src/commonMain/kotlin/net/twisterrob/travel/domain/london/status/Feed.kt b/domain/status/src/commonMain/kotlin/net/twisterrob/travel/domain/london/status/Feed.kt new file mode 100644 index 00000000..907fbb62 --- /dev/null +++ b/domain/status/src/commonMain/kotlin/net/twisterrob/travel/domain/london/status/Feed.kt @@ -0,0 +1,6 @@ +package net.twisterrob.travel.domain.london.status + +enum class Feed { + TubeDepartureBoardsLineStatus, + TubeDepartureBoardsLineStatusIncidents, +} diff --git a/domain/status/src/commonMain/kotlin/net/twisterrob/travel/domain/london/status/Stacktrace.kt b/domain/status/src/commonMain/kotlin/net/twisterrob/travel/domain/london/status/Stacktrace.kt new file mode 100644 index 00000000..eedc68c6 --- /dev/null +++ b/domain/status/src/commonMain/kotlin/net/twisterrob/travel/domain/london/status/Stacktrace.kt @@ -0,0 +1,5 @@ +package net.twisterrob.travel.domain.london.status + +data class Stacktrace( + val stacktrace: String, +) diff --git a/domain/status/src/commonMain/kotlin/net/twisterrob/travel/domain/london/status/StatusContent.kt b/domain/status/src/commonMain/kotlin/net/twisterrob/travel/domain/london/status/StatusContent.kt new file mode 100644 index 00000000..bf2314dd --- /dev/null +++ b/domain/status/src/commonMain/kotlin/net/twisterrob/travel/domain/london/status/StatusContent.kt @@ -0,0 +1,5 @@ +package net.twisterrob.travel.domain.london.status + +data class StatusContent( + val content: String, +) diff --git a/domain/status/src/commonMain/kotlin/net/twisterrob/travel/domain/london/status/StatusItem.kt b/domain/status/src/commonMain/kotlin/net/twisterrob/travel/domain/london/status/StatusItem.kt new file mode 100644 index 00000000..1e426ab5 --- /dev/null +++ b/domain/status/src/commonMain/kotlin/net/twisterrob/travel/domain/london/status/StatusItem.kt @@ -0,0 +1,21 @@ +package net.twisterrob.travel.domain.london.status + +import kotlinx.datetime.Instant + +sealed class StatusItem { + + abstract val retrievedDate: Instant + abstract val feed: Feed + + data class SuccessfulStatusItem( + override val feed: Feed, + val content: StatusContent, + override val retrievedDate: Instant, + ) : StatusItem() + + data class FailedStatusItem( + override val feed: Feed, + val error: Stacktrace, + override val retrievedDate: Instant, + ) : StatusItem() +} diff --git a/domain/status/src/commonMain/kotlin/net/twisterrob/travel/domain/london/status/api/FeedParser.kt b/domain/status/src/commonMain/kotlin/net/twisterrob/travel/domain/london/status/api/FeedParser.kt new file mode 100644 index 00000000..d10a30cc --- /dev/null +++ b/domain/status/src/commonMain/kotlin/net/twisterrob/travel/domain/london/status/api/FeedParser.kt @@ -0,0 +1,10 @@ +package net.twisterrob.travel.domain.london.status.api + +import net.twisterrob.travel.domain.london.status.Feed +import net.twisterrob.travel.domain.london.status.StatusContent + +interface FeedParser { + + @Throws(Exception::class) + fun parse(feed: Feed, content: StatusContent): Any +} diff --git a/domain/status/src/commonMain/kotlin/net/twisterrob/travel/domain/london/status/api/HistoryUseCase.kt b/domain/status/src/commonMain/kotlin/net/twisterrob/travel/domain/london/status/api/HistoryUseCase.kt new file mode 100644 index 00000000..59aa02e7 --- /dev/null +++ b/domain/status/src/commonMain/kotlin/net/twisterrob/travel/domain/london/status/api/HistoryUseCase.kt @@ -0,0 +1,29 @@ +package net.twisterrob.travel.domain.london.status.api + +import net.twisterrob.travel.domain.london.status.Feed +import net.twisterrob.travel.domain.london.status.Stacktrace +import net.twisterrob.travel.domain.london.status.StatusItem + +interface HistoryUseCase { + + fun history(feed: Feed, max: Int, includeCurrent: Boolean): List +} + +sealed class ParsedStatusItem { + + abstract val item: StatusItem + + data class ParsedFeed( + override val item: StatusItem.SuccessfulStatusItem, + val content: Any, + ) : ParsedStatusItem() + + data class AlreadyFailed( + override val item: StatusItem.FailedStatusItem, + ) : ParsedStatusItem() + + data class ParseFailed( + override val item: StatusItem.SuccessfulStatusItem, + val error: Stacktrace, + ) : ParsedStatusItem() +} diff --git a/domain/status/src/commonMain/kotlin/net/twisterrob/travel/domain/london/status/api/RefreshUseCase.kt b/domain/status/src/commonMain/kotlin/net/twisterrob/travel/domain/london/status/api/RefreshUseCase.kt new file mode 100644 index 00000000..43320408 --- /dev/null +++ b/domain/status/src/commonMain/kotlin/net/twisterrob/travel/domain/london/status/api/RefreshUseCase.kt @@ -0,0 +1,26 @@ +package net.twisterrob.travel.domain.london.status.api + +import net.twisterrob.travel.domain.london.status.Feed +import net.twisterrob.travel.domain.london.status.StatusItem + +interface RefreshUseCase { + + fun refreshLatest(feed: Feed): RefreshResult +} + +sealed class RefreshResult { + + data class Created( + val current: StatusItem, + ) : RefreshResult() + + data class Refreshed( + val current: StatusItem, + val latest: StatusItem, + ) : RefreshResult() + + data class NoChange( + val current: StatusItem, + val latest: StatusItem, + ) : RefreshResult() +} diff --git a/domain/status/src/commonMain/kotlin/net/twisterrob/travel/domain/london/status/api/StatusHistoryRepository.kt b/domain/status/src/commonMain/kotlin/net/twisterrob/travel/domain/london/status/api/StatusHistoryRepository.kt new file mode 100644 index 00000000..7400b902 --- /dev/null +++ b/domain/status/src/commonMain/kotlin/net/twisterrob/travel/domain/london/status/api/StatusHistoryRepository.kt @@ -0,0 +1,11 @@ +package net.twisterrob.travel.domain.london.status.api + +import net.twisterrob.travel.domain.london.status.Feed +import net.twisterrob.travel.domain.london.status.StatusItem + +interface StatusHistoryRepository { + + fun add(current: StatusItem) + + fun getAll(feed: Feed, max: Int): List +} diff --git a/domain/status/src/commonMain/kotlin/net/twisterrob/travel/domain/london/status/api/StatusInteractor.kt b/domain/status/src/commonMain/kotlin/net/twisterrob/travel/domain/london/status/api/StatusInteractor.kt new file mode 100644 index 00000000..48507243 --- /dev/null +++ b/domain/status/src/commonMain/kotlin/net/twisterrob/travel/domain/london/status/api/StatusInteractor.kt @@ -0,0 +1,9 @@ +package net.twisterrob.travel.domain.london.status.api + +import net.twisterrob.travel.domain.london.status.Feed +import net.twisterrob.travel.domain.london.status.StatusItem + +interface StatusInteractor { + + fun getCurrent(feed: Feed): StatusItem +} diff --git a/domain/status/src/commonTest/kotlin/net/twisterrob/travel/domain/london/status/DomainHistoryUseCaseUnitTest.kt b/domain/status/src/commonTest/kotlin/net/twisterrob/travel/domain/london/status/DomainHistoryUseCaseUnitTest.kt new file mode 100644 index 00000000..6686e8d3 --- /dev/null +++ b/domain/status/src/commonTest/kotlin/net/twisterrob/travel/domain/london/status/DomainHistoryUseCaseUnitTest.kt @@ -0,0 +1,184 @@ +package net.twisterrob.travel.domain.london.status + +import io.mockative.any +import io.mockative.every +import io.mockative.mock +import io.mockative.verify +import io.mockative.verifyNoUnmetExpectations +import io.mockative.verifyNoUnverifiedExpectations +import net.twisterrob.travel.domain.london.status.api.FeedParser +import net.twisterrob.travel.domain.london.status.api.HistoryUseCase +import net.twisterrob.travel.domain.london.status.api.ParsedStatusItem +import net.twisterrob.travel.domain.london.status.api.StatusHistoryRepository +import net.twisterrob.travel.domain.london.status.api.StatusInteractor +import java.io.IOException +import kotlin.test.AfterTest +import kotlin.test.Test +import kotlin.test.assertEquals + +class DomainHistoryUseCaseUnitTest { + + private val mockRepo: StatusHistoryRepository = mock() + private val mockInteractor: StatusInteractor = mock() + private val mockParser: FeedParser = mock() + private val subject: HistoryUseCase = DomainHistoryUseCase(mockRepo, mockInteractor, mockParser) + private val feed = Feed.TubeDepartureBoardsLineStatus + + @AfterTest + fun verify() { + listOf(mockRepo, mockInteractor, mockParser).forEach { + verifyNoUnverifiedExpectations(it) + verifyNoUnmetExpectations(it) + } + } + + @Test fun `empty history is processed`() { + every { mockRepo.getAll(any(), any()) }.returns(emptyList()) + + val result = subject.history(feed, max = 123, includeCurrent = false) + assertEquals(emptyList(), result) + + verify { mockRepo.getAll(feed, 123) }.wasInvoked() + } + + @Test fun `empty history with current is returned`() { + val current = SuccessfulStatusItem() + every { mockRepo.getAll(any(), any()) }.returns(emptyList()) + every { mockInteractor.getCurrent(any()) }.returns(current) + val parsed = Any() + every { mockParser.parse(any(), any()) }.returns(parsed) + + val result = subject.history(feed, max = 123, includeCurrent = true) + val expected = listOf( + current.toParsed(parsed) + ) + assertEquals(expected, result) + + verify { mockRepo.getAll(feed, 123) }.wasInvoked() + verify { mockInteractor.getCurrent(feed) }.wasInvoked() + verify { mockParser.parse(current.feed, current.content) }.wasInvoked() + } + + @Test fun `some history with current is prepended`() { + val current = SuccessfulStatusItem() + val existing1 = SuccessfulStatusItem() + val existing2 = SuccessfulStatusItem() + every { mockRepo.getAll(any(), any()) }.returns(listOf(existing1, existing2)) + every { mockInteractor.getCurrent(any()) }.returns(current) + val currentParsed = Any() + val existingParsed1 = Any() + val existingParsed2 = Any() + every { mockParser.parse(any(), any()) }.returnsMany(currentParsed, existingParsed1, existingParsed2) + + val result = subject.history(feed, max = 123, includeCurrent = true) + val expected = listOf( + current.toParsed(currentParsed), + existing1.toParsed(existingParsed1), + existing2.toParsed(existingParsed2), + ) + assertEquals(expected, result) + + verify { mockRepo.getAll(feed, 123) }.wasInvoked() + verify { mockInteractor.getCurrent(feed) }.wasInvoked() + verify { mockParser.parse(current.feed, current.content) }.wasInvoked() + verify { mockParser.parse(existing1.feed, existing1.content) }.wasInvoked() + verify { mockParser.parse(existing2.feed, existing2.content) }.wasInvoked() + } + + @Test fun `failed existing is returned`() { + val existing1 = FailedStatusItem() + val existing2 = SuccessfulStatusItem() + every { mockRepo.getAll(any(), any()) }.returns(listOf(existing1, existing2)) + val existingParsed2 = Any() + every { mockParser.parse(any(), any()) }.returns(existingParsed2) + + val result = subject.history(feed, max = 123, includeCurrent = false) + val expected = listOf( + existing1.toParsed(), + existing2.toParsed(existingParsed2), + ) + assertEquals(expected, result) + + verify { mockRepo.getAll(feed, 123) }.wasInvoked() + verify { mockParser.parse(existing2.feed, existing2.content) }.wasInvoked() + } + + @Test fun `failed current is returned`() { + val current = FailedStatusItem() + val existing1 = SuccessfulStatusItem() + val existing2 = SuccessfulStatusItem() + every { mockRepo.getAll(any(), any()) }.returns(listOf(existing1, existing2)) + every { mockInteractor.getCurrent(any()) }.returns(current) + val existingParsed1 = Any() + val existingParsed2 = Any() + every { mockParser.parse(any(), any()) }.returnsMany(existingParsed1, existingParsed2) + + val result = subject.history(feed, max = 123, includeCurrent = true) + val expected = listOf( + current.toParsed(), + existing1.toParsed(existingParsed1), + existing2.toParsed(existingParsed2), + ) + assertEquals(expected, result) + + verify { mockRepo.getAll(feed, 123) }.wasInvoked() + verify { mockInteractor.getCurrent(feed) }.wasInvoked() + verify { mockParser.parse(existing1.feed, existing1.content) }.wasInvoked() + verify { mockParser.parse(existing2.feed, existing2.content) }.wasInvoked() + } + + @Test fun `existing failing to parse is returned`() { + val existing1 = SuccessfulStatusItem() + val existing2 = SuccessfulStatusItem() + every { mockRepo.getAll(any(), any()) }.returns(listOf(existing1, existing2)) + val existingParsed1 = Any() + val ex = IOException("Failed to parse XML") + every { mockParser.parse(any(), any()) }.invokesMany({ existingParsed1 }, { throw ex }) + + val result = subject.history(feed, max = 123, includeCurrent = false) + val expected = listOf( + existing1.toParsed(existingParsed1), + existing2.toParsed(ex), + ) + assertEquals(expected, result) + + verify { mockRepo.getAll(feed, 123) }.wasInvoked() + verify { mockParser.parse(existing1.feed, existing1.content) }.wasInvoked() + verify { mockParser.parse(existing2.feed, existing2.content) }.wasInvoked() + } + + @Test fun `current failing to parse is returned`() { + val current = SuccessfulStatusItem() + val existing1 = SuccessfulStatusItem() + val existing2 = SuccessfulStatusItem() + every { mockRepo.getAll(any(), any()) }.returns(listOf(existing1, existing2)) + every { mockInteractor.getCurrent(any()) }.returns(current) + val existingParsed1 = Any() + val existingParsed2 = Any() + val ex = IOException("Failed to parse XML") + every { mockParser.parse(any(), any()) }.invokesMany({ throw ex }, { existingParsed1 }, { existingParsed2 }) + + val result = subject.history(feed, max = 123, includeCurrent = true) + val expected = listOf( + current.toParsed(ex), + existing1.toParsed(existingParsed1), + existing2.toParsed(existingParsed2), + ) + assertEquals(expected, result) + + verify { mockRepo.getAll(feed, 123) }.wasInvoked() + verify { mockInteractor.getCurrent(feed) }.wasInvoked() + verify { mockParser.parse(current.feed, current.content) }.wasInvoked() + verify { mockParser.parse(existing1.feed, existing1.content) }.wasInvoked() + verify { mockParser.parse(existing2.feed, existing2.content) }.wasInvoked() + } +} + +private fun StatusItem.SuccessfulStatusItem.toParsed(parsed: Any): ParsedStatusItem = + ParsedStatusItem.ParsedFeed(this, parsed) + +private fun StatusItem.FailedStatusItem.toParsed(): ParsedStatusItem = + ParsedStatusItem.AlreadyFailed(this) + +private fun StatusItem.SuccessfulStatusItem.toParsed(ex: Throwable): ParsedStatusItem = + ParsedStatusItem.ParseFailed(this, Stacktrace(ex.stackTraceToString())) diff --git a/domain/status/src/commonTest/kotlin/net/twisterrob/travel/domain/london/status/DomainRefreshUseCaseUnitTest.kt b/domain/status/src/commonTest/kotlin/net/twisterrob/travel/domain/london/status/DomainRefreshUseCaseUnitTest.kt new file mode 100644 index 00000000..6dafba85 --- /dev/null +++ b/domain/status/src/commonTest/kotlin/net/twisterrob/travel/domain/london/status/DomainRefreshUseCaseUnitTest.kt @@ -0,0 +1,128 @@ +package net.twisterrob.travel.domain.london.status + +import io.mockative.any +import io.mockative.every +import io.mockative.mock +import io.mockative.verify +import io.mockative.verifyNoUnmetExpectations +import io.mockative.verifyNoUnverifiedExpectations +import net.twisterrob.travel.domain.london.status.api.RefreshResult +import net.twisterrob.travel.domain.london.status.api.RefreshUseCase +import net.twisterrob.travel.domain.london.status.api.StatusHistoryRepository +import net.twisterrob.travel.domain.london.status.api.StatusInteractor +import kotlin.test.AfterTest +import kotlin.test.Test +import kotlin.test.assertEquals + +class DomainRefreshUseCaseUnitTest { + + private val mockRepo: StatusHistoryRepository = mock() + private val mockInteractor: StatusInteractor = mock() + private val subject: RefreshUseCase = DomainRefreshUseCase(mockRepo, mockInteractor) + private val feed = Feed.TubeDepartureBoardsLineStatus + + @AfterTest + fun verify() { + listOf(mockRepo, mockInteractor).forEach { + verifyNoUnverifiedExpectations(it) + verifyNoUnmetExpectations(it) + } + } + + @Test fun `current will be saved when there are no previous statuses`() { + val current = SuccessfulStatusItem() + every { mockInteractor.getCurrent(any()) }.returns(current) + every { mockRepo.getAll(any(), any()) }.returns(emptyList()) + + val result = subject.refreshLatest(feed) + assertEquals(RefreshResult.Created(current), result) + + verify { mockInteractor.getCurrent(feed) }.wasInvoked() + verify { mockRepo.getAll(feed, 1) }.wasInvoked() + verify { mockRepo.add(current) }.wasInvoked() + } + + @Test fun `current will not be saved when it is the same as the latest status`() { + val current = SuccessfulStatusItem() + val latest = SuccessfulStatusItem().copy(content = current.content) + every { mockInteractor.getCurrent(any()) }.returns(current) + every { mockRepo.getAll(any(), any()) }.returns(listOf(latest)) + + val result = subject.refreshLatest(feed) + assertEquals(RefreshResult.NoChange(current, latest), result) + + verify { mockInteractor.getCurrent(feed) }.wasInvoked() + verify { mockRepo.getAll(feed, 1) }.wasInvoked() + verify { mockRepo.add(any()) }.wasNotInvoked() + } + + @Test fun `current will be saved when it differs from the latest status`() { + val current = SuccessfulStatusItem() + val latest = SuccessfulStatusItem() + every { mockInteractor.getCurrent(any()) }.returns(current) + every { mockRepo.getAll(any(), any()) }.returns(listOf(latest)) + + val result = subject.refreshLatest(feed) + assertEquals(RefreshResult.Refreshed(current, latest), result) + + verify { mockInteractor.getCurrent(feed) }.wasInvoked() + verify { mockRepo.getAll(feed, 1) }.wasInvoked() + verify { mockRepo.add(current) }.wasInvoked() + } + + @Test fun `current error will be saved when it differs from the latest status`() { + val current = FailedStatusItem() + val latest = SuccessfulStatusItem() + every { mockInteractor.getCurrent(any()) }.returns(current) + every { mockRepo.getAll(any(), any()) }.returns(listOf(latest)) + + val result = subject.refreshLatest(feed) + assertEquals(RefreshResult.Refreshed(current, latest), result) + + verify { mockInteractor.getCurrent(feed) }.wasInvoked() + verify { mockRepo.getAll(feed, 1) }.wasInvoked() + verify { mockRepo.add(current) }.wasInvoked() + } + + @Test fun `current success will be saved when it differs from the latest error`() { + val current = SuccessfulStatusItem() + val latest = FailedStatusItem() + every { mockInteractor.getCurrent(any()) }.returns(current) + every { mockRepo.getAll(any(), any()) }.returns(listOf(latest)) + + val result = subject.refreshLatest(feed) + assertEquals(RefreshResult.Refreshed(current, latest), result) + + verify { mockInteractor.getCurrent(feed) }.wasInvoked() + verify { mockRepo.getAll(feed, 1) }.wasInvoked() + verify { mockRepo.add(current) }.wasInvoked() + } + + @Test fun `current error will be saved when it differs from the latest error`() { + val current = FailedStatusItem() + val latest = FailedStatusItem() + every { mockInteractor.getCurrent(any()) }.returns(current) + every { mockRepo.getAll(any(), any()) }.returns(listOf(latest)) + + val result = subject.refreshLatest(feed) + assertEquals(RefreshResult.Refreshed(current, latest), result) + + verify { mockInteractor.getCurrent(feed) }.wasInvoked() + verify { mockRepo.getAll(feed, 1) }.wasInvoked() + verify { mockRepo.add(current) }.wasInvoked() + } + + @Test fun `current error will not be saved when it is the same as the latest error`() { + val current = FailedStatusItem() + val latest = FailedStatusItem().copy(error = current.error) + every { mockInteractor.getCurrent(any()) }.returns(current) + every { mockRepo.getAll(any(), any()) }.returns(listOf(latest)) + + val result = subject.refreshLatest(feed) + assertEquals(RefreshResult.NoChange(current, latest), result) + + verify { mockInteractor.getCurrent(feed) }.wasInvoked() + verify { mockRepo.getAll(feed, 1) }.wasInvoked() + verify { mockRepo.add(current) }.wasNotInvoked() + } +} diff --git a/domain/status/src/commonTest/kotlin/net/twisterrob/travel/domain/london/status/Mocks.kt b/domain/status/src/commonTest/kotlin/net/twisterrob/travel/domain/london/status/Mocks.kt new file mode 100644 index 00000000..0d52ee98 --- /dev/null +++ b/domain/status/src/commonTest/kotlin/net/twisterrob/travel/domain/london/status/Mocks.kt @@ -0,0 +1,21 @@ +package net.twisterrob.travel.domain.london.status + +import io.mockative.Mock +import io.mockative.classOf +import io.mockative.mock +import net.twisterrob.travel.domain.london.status.api.FeedParser +import net.twisterrob.travel.domain.london.status.api.StatusHistoryRepository +import net.twisterrob.travel.domain.london.status.api.StatusInteractor + +@Suppress("unused") // Used by mockative KSP. +object Mocks { + + @Mock + private val statusHistoryRepository = mock(classOf()) + + @Mock + private val statusInteractor = mock(classOf()) + + @Mock + private val feedParser = mock(classOf()) +} diff --git a/domain/status/src/commonTest/kotlin/net/twisterrob/travel/domain/london/status/StatusItemFake.kt b/domain/status/src/commonTest/kotlin/net/twisterrob/travel/domain/london/status/StatusItemFake.kt new file mode 100644 index 00000000..5973947b --- /dev/null +++ b/domain/status/src/commonTest/kotlin/net/twisterrob/travel/domain/london/status/StatusItemFake.kt @@ -0,0 +1,19 @@ +package net.twisterrob.travel.domain.london.status + +import kotlinx.datetime.Instant +import java.util.UUID +import kotlin.random.Random + +fun SuccessfulStatusItem(): StatusItem.SuccessfulStatusItem = + StatusItem.SuccessfulStatusItem( + Feed.entries[Random.nextInt(Feed.entries.size)], + StatusContent(UUID.randomUUID().toString()), + Instant.fromEpochMilliseconds(Random.nextLong()), + ) + +fun FailedStatusItem(): StatusItem.FailedStatusItem = + StatusItem.FailedStatusItem( + Feed.entries[Random.nextInt(Feed.entries.size)], + Stacktrace(UUID.randomUUID().toString()), + Instant.fromEpochMilliseconds(Random.nextLong()), + ) diff --git a/domain/status/src/commonTest/kotlin/net/twisterrob/travel/domain/london/status/mock.kt b/domain/status/src/commonTest/kotlin/net/twisterrob/travel/domain/london/status/mock.kt new file mode 100644 index 00000000..e70c1c53 --- /dev/null +++ b/domain/status/src/commonTest/kotlin/net/twisterrob/travel/domain/london/status/mock.kt @@ -0,0 +1,15 @@ +@file:Suppress("PackageDirectoryMismatch") // Keep it close to Mocks.kt + +package io.mockative + +import net.twisterrob.travel.domain.london.status.api.FeedParser +import net.twisterrob.travel.domain.london.status.api.StatusHistoryRepository +import net.twisterrob.travel.domain.london.status.api.StatusInteractor + +inline fun mock(): T = + when (T::class) { + StatusHistoryRepository::class -> mock(StatusHistoryRepository::class) as T + StatusInteractor::class -> mock(StatusInteractor::class) as T + FeedParser::class -> mock(FeedParser::class) as T + else -> throw IllegalArgumentException("No mock for ${T::class}") + } diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 40bd6950..2810cc69 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -6,6 +6,10 @@ java-appengine = "17" agp = "8.2.0" twisterrob = "0.16" gretty = "4.1.1" +kotlin = "1.9.22" +kotlin-ksp = "1.9.21-1.0.15" +kotlin-datetime = "0.5.0" +kotlin-serialization = "1.6.2" jsr305 = "3.0.2" kxml2 = "2.3.0" @@ -38,6 +42,7 @@ snakeyaml = "2.2" junit4 = "4.13.2" mockito = "5.8.0" +mockative = "2.0.1" # Use this instead of 1.3 # If `hamcrest-1.3` appears in the dependency list, check if it's excluded from all usages. hamcrest = "2.0.0.0" @@ -50,6 +55,8 @@ plugin-agp = { module = "com.android.tools.build:gradle", version.ref = "agp" } plugin-appengine = { module = "com.google.cloud.tools:appengine-gradle-plugin", version.ref = "google-appengine-plugin" } plugin-twisterrob = { module = "net.twisterrob.gradle:twister-convention-plugins", version.ref = "twisterrob" } plugin-gretty = { module = "org.gretty:gretty", version.ref = "gretty" } +plugin-kotlin = { module = "org.jetbrains.kotlin:kotlin-gradle-plugin", version.ref = "kotlin" } +plugin-ksp = { module = "com.google.devtools.ksp:com.google.devtools.ksp.gradle.plugin", version.ref = "kotlin-ksp" } plugin-micronaut = { module = "io.micronaut.application:io.micronaut.application.gradle.plugin", version.ref = "micronaut" } #noinspection unused versions.micronaut is used for micronaut { version = ... }, micronaut-platform is added here to help Renovate. @@ -62,6 +69,9 @@ snakeyaml = { module = "org.yaml:snakeyaml", version.ref = "snakeyaml" } jsr305 = { module = "com.google.code.findbugs:jsr305", version.ref = "jsr305" } kxml2 = { module = "net.sf.kxml:kxml2", version.ref = "kxml2" } sqlite-jdbc = { module = "org.xerial:sqlite-jdbc", version.ref = "sqlite-jdbc" } +kotlin-datetime = { module = "org.jetbrains.kotlinx:kotlinx-datetime", version.ref = "kotlin-datetime" } +kotlin-serialization = { module = "org.jetbrains.kotlinx:kotlinx-serialization-core", version.ref = "kotlin-serialization" } +kotlin-test = { module = "org.jetbrains.kotlin:kotlin-test", version.ref = "kotlin" } androidx-fragment = { module = "androidx.fragment:fragment", version.ref = "androidx-fragment" } androidx-swiperefreshlayout = { module = "androidx.swiperefreshlayout:swiperefreshlayout", version.ref = "androidx-swiperefreshlayout" } @@ -77,6 +87,8 @@ gms-places = { module = "com.google.android.libraries.places:places", version.re gms-maps = { module = "com.google.android.gms:play-services-maps", version.ref = "google-maps" } test-junit4 = { module = "junit:junit", version.ref = "junit4" } +test-mockative = { module = "io.mockative:mockative", version.ref = "mockative" } +test-mockative-processor = { module = "io.mockative:mockative-processor", version.ref = "mockative" } test-mockito = { module = "org.mockito:mockito-core", version.ref = "mockito" } test-hamcrest = { module = "org.hamcrest:hamcrest-junit", version.ref = "hamcrest" } test-gwen = { module = "com.shazam:gwen", version.ref = "gwen" } diff --git a/gradle/plugins/build.gradle b/gradle/plugins/build.gradle index d0e3c1b6..614e2c4e 100644 --- a/gradle/plugins/build.gradle +++ b/gradle/plugins/build.gradle @@ -7,6 +7,8 @@ dependencies { implementation(libs.plugin.agp) implementation(libs.plugin.appengine) implementation(libs.plugin.micronaut) + implementation(libs.plugin.kotlin) + implementation(libs.plugin.ksp) implementation(libs.plugin.gretty) { // This causes an error, which is locally not reproducible, but happens on GHA CI. // org.gretty:gretty:3.0.5 -> org.bouncycastle:bcprov-jdk15on:1.60 diff --git a/settings.gradle b/settings.gradle index 0e1cb9cb..767b8f18 100644 --- a/settings.gradle +++ b/settings.gradle @@ -39,6 +39,7 @@ include(":common:log-console") include(":common:model") include(":common:maptiler") include(":common:test-helpers") +include(":domain:status") include(":desktop:routes") include(":web:status-history") diff --git a/web/status-history/README.md b/web/status-history/README.md index a514c2f5..74f2b155 100644 --- a/web/status-history/README.md +++ b/web/status-history/README.md @@ -17,6 +17,6 @@ ``` gcloud beta emulators datastore env-init ``` -5. Set the output environment variables before running `gradlew :AppEngine:run`. +5. Set the output environment variables before running `gradlew :web:status-history:run`. -Note: step 4 and 5 are automated in `build.gradle`. +Note: step 4 and 5 are automated in `build.gradle` and in "Run status-history server" IDEA run configuration. diff --git a/web/status-history/build.gradle b/web/status-history/build.gradle index 15ccbcb6..63a75e9c 100644 --- a/web/status-history/build.gradle +++ b/web/status-history/build.gradle @@ -22,6 +22,7 @@ dependencies { runtimeOnly(libs.micronaut.jackson) implementation(libs.micronaut.handlebars) + implementation(projects.domain.status) implementation(projects.common.data.static) implementation(projects.common.feed.feeds) implementation(projects.common.feed.trackernet) diff --git a/web/status-history/src/main/java/net/twisterrob/blt/gapp/DatastoreStatusHistoryRepository.java b/web/status-history/src/main/java/net/twisterrob/blt/gapp/DatastoreStatusHistoryRepository.java new file mode 100644 index 00000000..90b0f07e --- /dev/null +++ b/web/status-history/src/main/java/net/twisterrob/blt/gapp/DatastoreStatusHistoryRepository.java @@ -0,0 +1,148 @@ +package net.twisterrob.blt.gapp; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +import com.google.cloud.Timestamp; +import com.google.cloud.datastore.BaseEntity; +import com.google.cloud.datastore.Datastore; +import com.google.cloud.datastore.Entity; +import com.google.cloud.datastore.FullEntity; +import com.google.cloud.datastore.IncompleteKey; +import com.google.cloud.datastore.KeyFactory; +import com.google.cloud.datastore.NullValue; +import com.google.cloud.datastore.Query; +import com.google.cloud.datastore.QueryResults; +import com.google.cloud.datastore.StringValue; +import com.google.cloud.datastore.StructuredQuery.OrderBy; +import com.google.cloud.datastore.Value; + +import io.micronaut.context.annotation.Bean; +import kotlinx.datetime.Instant; + +import net.twisterrob.travel.domain.london.status.Feed; +import net.twisterrob.travel.domain.london.status.Stacktrace; +import net.twisterrob.travel.domain.london.status.StatusContent; +import net.twisterrob.travel.domain.london.status.StatusItem; +import net.twisterrob.travel.domain.london.status.api.StatusHistoryRepository; + +import static net.twisterrob.blt.gapp.DatastoreStatusHistoryRepository.DS_PROP_CONTENT; +import static net.twisterrob.blt.gapp.DatastoreStatusHistoryRepository.DS_PROP_ERROR; +import static net.twisterrob.blt.gapp.DatastoreStatusHistoryRepository.DS_PROP_RETRIEVED_DATE; + +@Bean(typed = StatusHistoryRepository.class) +public class DatastoreStatusHistoryRepository implements StatusHistoryRepository { + + static final String DS_PROP_RETRIEVED_DATE = "retrievedDate"; + static final String DS_PROP_CONTENT = "content"; + static final String DS_PROP_ERROR = "error"; + + private final Datastore datastore; + private final StatusItemToEntityConverter statusItemConverter; + private final EntityToStatusItemConverter entityConverter; + + public DatastoreStatusHistoryRepository(Datastore datastore) { + this.datastore = datastore; + this.statusItemConverter = new StatusItemToEntityConverter(datastore); + this.entityConverter = new EntityToStatusItemConverter(); + } + + @Override public void add(@Nonnull StatusItem current) { + datastore.add(statusItemConverter.toEntity(current)); + } + + @Override public @Nonnull List getAll(@Nonnull Feed feed, int max) { + Iterator entries = queryAllMostRecentFirst(feed, max); + List results = new ArrayList<>(); + while (entries.hasNext()) { + if (--max < 0) { + break; // We've had enough, don't read more. + } + Entity entry = entries.next(); + StatusItem result = entityConverter.toItem(entry); + results.add(result); + } + return results; + } + + private @Nonnull QueryResults queryAllMostRecentFirst(@Nonnull Feed feed, int limit) { + Query query = Query + .newEntityQueryBuilder() + .setKind(feed.name()) + .setLimit(limit) + .addOrderBy(OrderBy.desc(DS_PROP_RETRIEVED_DATE)) + .build() + ; + return datastore.run(query); + } +} + +class StatusItemToEntityConverter { + + private final Datastore datastore; + + StatusItemToEntityConverter(Datastore datastore) { + this.datastore = datastore; + } + + public @Nonnull FullEntity toEntity(@Nonnull StatusItem current) { + KeyFactory keyFactory = datastore.newKeyFactory().setKind(current.getFeed().name()); + FullEntity.Builder newEntry = Entity.newBuilder(keyFactory.newKey()); + newEntry.set(DS_PROP_RETRIEVED_DATE, toTimestamp(current.getRetrievedDate())); + if (current instanceof StatusItem.SuccessfulStatusItem success) { + newEntry.set(DS_PROP_CONTENT, unindexedString(success.getContent().getContent())); + } else if (current instanceof StatusItem.FailedStatusItem error) { + newEntry.set(DS_PROP_ERROR, unindexedString(error.getError().getStacktrace())); + } else { + throw new IllegalArgumentException("Unknown item: " + current); + } + return newEntry.build(); + } + + private static @Nonnull Timestamp toTimestamp(@Nonnull Instant instant) { + return Timestamp.ofTimeSecondsAndNanos(instant.getEpochSeconds(), instant.getNanosecondsOfSecond()); + } + + /** + * Strings have a limitation of 1500 bytes when indexed. This removes that limitation. + * @see Entities + */ + static @Nonnull Value unindexedString(@Nullable String value) { + return value == null + ? NullValue.of() + : StringValue.newBuilder(value).setExcludeFromIndexes(true).build(); + } +} + +class EntityToStatusItemConverter { + + public @Nonnull StatusItem toItem(@Nonnull Entity entity) { + if (hasProperty(entity, DS_PROP_CONTENT)) { + return new StatusItem.SuccessfulStatusItem( + Feed.valueOf(entity.getKey().getKind()), + new StatusContent(entity.getString(DS_PROP_CONTENT)), + toInstant(entity.getTimestamp(DS_PROP_RETRIEVED_DATE)) + ); + } else if (hasProperty(entity, DS_PROP_ERROR)) { + return new StatusItem.FailedStatusItem( + Feed.valueOf(entity.getKey().getKind()), + new Stacktrace(entity.getString(DS_PROP_ERROR)), + toInstant(entity.getTimestamp(DS_PROP_RETRIEVED_DATE)) + ); + } else { + throw new IllegalArgumentException("Unknown entity: " + entity); + } + } + + private static @Nonnull Instant toInstant(@Nonnull Timestamp timestamp) { + return Instant.Companion.fromEpochSeconds(timestamp.getSeconds(), timestamp.getNanos()); + } + + static boolean hasProperty(BaseEntity entry, String propName) { + return entry.getProperties().containsKey(propName); + } +} diff --git a/web/status-history/src/main/java/net/twisterrob/blt/gapp/Dependencies.java b/web/status-history/src/main/java/net/twisterrob/blt/gapp/Dependencies.java new file mode 100644 index 00000000..c99a7c8a --- /dev/null +++ b/web/status-history/src/main/java/net/twisterrob/blt/gapp/Dependencies.java @@ -0,0 +1,51 @@ +package net.twisterrob.blt.gapp; + +import com.google.cloud.datastore.Datastore; +import com.google.cloud.datastore.DatastoreOptions; + +import io.micronaut.context.annotation.Factory; +import io.micronaut.runtime.http.scope.RequestScope; +import jakarta.inject.Singleton; + +import net.twisterrob.travel.domain.london.status.DomainHistoryUseCase; +import net.twisterrob.travel.domain.london.status.DomainRefreshUseCase; +import net.twisterrob.travel.domain.london.status.api.FeedParser; +import net.twisterrob.travel.domain.london.status.api.HistoryUseCase; +import net.twisterrob.travel.domain.london.status.api.RefreshUseCase; +import net.twisterrob.travel.domain.london.status.api.StatusHistoryRepository; +import net.twisterrob.travel.domain.london.status.api.StatusInteractor; + +@Factory +public class Dependencies { + + /** + * External dependency from Google. + */ + @Singleton + public Datastore datastore() { + return DatastoreOptions.getDefaultInstance().getService(); + } + + /** + * External dependency from domain layer in common KMP code. + */ + @RequestScope + public HistoryUseCase historyUseCase( + StatusHistoryRepository statusHistoryRepository, + StatusInteractor statusInteractor, + FeedParser feedParser + ) { + return new DomainHistoryUseCase(statusHistoryRepository, statusInteractor, feedParser); + } + + /** + * External dependency from domain layer in common KMP code. + */ + @RequestScope + public RefreshUseCase refreshUseCase( + StatusHistoryRepository statusHistoryRepository, + StatusInteractor statusInteractor + ) { + return new DomainRefreshUseCase(statusHistoryRepository, statusInteractor); + } +} diff --git a/web/status-history/src/main/java/net/twisterrob/blt/gapp/FeedConsts.java b/web/status-history/src/main/java/net/twisterrob/blt/gapp/FeedConsts.java index 3355ca12..07fe26a8 100644 --- a/web/status-history/src/main/java/net/twisterrob/blt/gapp/FeedConsts.java +++ b/web/status-history/src/main/java/net/twisterrob/blt/gapp/FeedConsts.java @@ -10,10 +10,4 @@ public interface FeedConsts { URLBuilder URL_BUILDER = new TFLUrlBuilder(EMAIL); StaticData STATIC_DATA = new SharedStaticData(); - - String ENCODING = "UTF-8"; - - String DS_PROP_RETRIEVED_DATE = "retrievedDate"; - String DS_PROP_CONTENT = "content"; - String DS_PROP_ERROR = "error"; } diff --git a/web/status-history/src/main/java/net/twisterrob/blt/gapp/FeedCronServlet.java b/web/status-history/src/main/java/net/twisterrob/blt/gapp/FeedCronServlet.java index e29600d1..3f39b26d 100644 --- a/web/status-history/src/main/java/net/twisterrob/blt/gapp/FeedCronServlet.java +++ b/web/status-history/src/main/java/net/twisterrob/blt/gapp/FeedCronServlet.java @@ -1,128 +1,76 @@ package net.twisterrob.blt.gapp; import java.io.*; -import java.net.*; import java.util.*; +import javax.annotation.Nullable; + import io.micronaut.http.annotation.Controller; import io.micronaut.http.annotation.Get; import jakarta.servlet.http.*; import org.slf4j.*; -import com.google.cloud.Timestamp; -import com.google.cloud.datastore.*; -import com.google.cloud.datastore.StructuredQuery.OrderBy; - -import net.twisterrob.blt.io.feeds.Feed; -import net.twisterrob.java.io.IOTools; -import net.twisterrob.java.utils.ObjectTools; - -import static net.twisterrob.blt.gapp.FeedConsts.*; +import net.twisterrob.travel.domain.london.status.Feed; +import net.twisterrob.travel.domain.london.status.StatusItem; +import net.twisterrob.travel.domain.london.status.api.RefreshResult; +import net.twisterrob.travel.domain.london.status.api.RefreshUseCase; @Controller @SuppressWarnings("serial") public class FeedCronServlet extends HttpServlet { + private static final Logger LOG = LoggerFactory.getLogger(FeedCronServlet.class); private static final String QUERY_FEED = "feed"; - private final Datastore datastore = DatastoreOptions.getDefaultInstance().getService(); + private final RefreshUseCase useCase; + + public FeedCronServlet(RefreshUseCase useCase) { + this.useCase = useCase; + } @Get("/FeedCron") @Override public void doGet(HttpServletRequest req, HttpServletResponse resp) throws IOException { - String feedString = String.valueOf(req.getParameter(QUERY_FEED)); - Feed feed; - try { - feed = Feed.valueOf(feedString); - } catch (IllegalArgumentException ex) { + String feedString = req.getParameter(QUERY_FEED); + Feed feed = parseFeed(feedString); + if (feed == null) { String message = String.format(Locale.getDefault(), "No such feed: '%s'.", feedString); LOG.warn(message); resp.getWriter().println(message); resp.setStatus(HttpServletResponse.SC_NOT_FOUND); return; } - Marker marker = MarkerFactory.getMarker(feed.name()); - FullEntity newEntry = downloadNewEntry(datastore, feed); - Entity oldEntry = readLatest(datastore, feed); - if (oldEntry != null) { - if (sameProp(DS_PROP_CONTENT, oldEntry, newEntry)) { + RefreshResult result = useCase.refreshLatest(feed); + Marker marker = MarkerFactory.getMarker(feed.name()); + if (result instanceof RefreshResult.NoChange noChange) { + if (noChange.getLatest() instanceof StatusItem.SuccessfulStatusItem) { LOG.info(marker, "They have the same content."); resp.setStatus(HttpServletResponse.SC_NO_CONTENT); - } else if (sameProp(DS_PROP_ERROR, oldEntry, newEntry)) { + } else { LOG.info(marker, "They have the same error."); resp.setStatus(HttpServletResponse.SC_NON_AUTHORITATIVE_INFORMATION); - } else { - LOG.info(marker, "They're different, storing..."); - resp.setStatus(HttpServletResponse.SC_ACCEPTED); - datastore.put(newEntry); } - } else { - LOG.info(marker, "It's new, storing..."); + } else if (result instanceof RefreshResult.Created) { + LOG.info(marker, "It's new, stored..."); resp.setStatus(HttpServletResponse.SC_CREATED); - datastore.put(newEntry); + } else if (result instanceof RefreshResult.Refreshed) { + LOG.info(marker, "They're different, stored..."); + resp.setStatus(HttpServletResponse.SC_ACCEPTED); + } else { + throw new IllegalStateException("Unknown result: " + result); } } - private static Entity readLatest(Datastore datastore, Feed feed) { - Query q = Query - .newEntityQueryBuilder() - .setKind(feed.name()) - .addOrderBy(OrderBy.desc(DS_PROP_RETRIEVED_DATE)) - .build() - ; - // We're only concerned about the latest one, if any. - QueryResults result = datastore.run(q); - return result.hasNext()? result.next() : null; - } - - private static boolean sameProp(String propName, BaseEntity oldEntry, BaseEntity newEntry) { - return hasProperty(oldEntry, propName) && hasProperty(newEntry, propName) - && ObjectTools.equals(oldEntry.getValue(propName), newEntry.getValue(propName)); - } - - static boolean hasProperty(BaseEntity entry, String propName) { - return entry.getProperties().containsKey(propName); - } - - public static FullEntity downloadNewEntry(Datastore datastore, Feed feed) { - KeyFactory keyFactory = datastore.newKeyFactory().setKind(feed.name()); - FullEntity.Builder newEntry = Entity.newBuilder(keyFactory.newKey()); - try { - String feedResult = downloadFeed(feed); - newEntry.set(DS_PROP_CONTENT, unindexedString(feedResult)); - } catch (Exception ex) { - LOG.error("Cannot load '{}'!", feed, ex); - newEntry.set(DS_PROP_ERROR, unindexedString(ObjectTools.getFullStackTrace(ex))); + static @Nullable Feed parseFeed(@Nullable String feedString) { + if (feedString == null) { + return null; } - newEntry.set(DS_PROP_RETRIEVED_DATE, Timestamp.now()); - return newEntry.build(); - } - - /** - * Strings have a limitation of 1500 bytes when indexed. This removes that limitation. - * @see https://cloud.google.com/datastore/docs/concepts/entities#text_string - */ - private static Value unindexedString(String value) { - return value == null - ? NullValue.of() - : StringValue.newBuilder(value).setExcludeFromIndexes(true).build(); - } - - public static String downloadFeed(Feed feed) throws IOException { - InputStream input = null; - String result; try { - URL url = URL_BUILDER.getFeedUrl(feed, Collections.emptyMap()); - LOG.debug("Requesting feed '{}': '{}'...", feed, url); - HttpURLConnection connection = (HttpURLConnection)url.openConnection(); - connection.connect(); - input = connection.getInputStream(); - result = IOTools.readAll(input); - } finally { - IOTools.ignorantClose(input); + return Feed.valueOf(feedString); + } catch (IllegalArgumentException ex) { + return null; } - return result; } } diff --git a/web/status-history/src/main/java/net/twisterrob/blt/gapp/FeedHandlerFeedParser.java b/web/status-history/src/main/java/net/twisterrob/blt/gapp/FeedHandlerFeedParser.java new file mode 100644 index 00000000..bffcebb5 --- /dev/null +++ b/web/status-history/src/main/java/net/twisterrob/blt/gapp/FeedHandlerFeedParser.java @@ -0,0 +1,23 @@ +package net.twisterrob.blt.gapp; + +import java.io.ByteArrayInputStream; +import java.io.InputStream; + +import javax.annotation.Nonnull; + +import io.micronaut.context.annotation.Bean; +import kotlin.text.Charsets; + +import net.twisterrob.travel.domain.london.status.Feed; +import net.twisterrob.travel.domain.london.status.StatusContent; +import net.twisterrob.travel.domain.london.status.api.FeedParser; + +@Bean(typed = FeedParser.class) +public class FeedHandlerFeedParser implements FeedParser { + + @Override public @Nonnull Object parse(@Nonnull Feed feed, @Nonnull StatusContent content) throws Exception { + try (InputStream stream = new ByteArrayInputStream(content.getContent().getBytes(Charsets.UTF_8))) { + return net.twisterrob.blt.io.feeds.Feed.valueOf(feed.name()).getHandler().parse(stream); + } + } +} diff --git a/web/status-history/src/main/java/net/twisterrob/blt/gapp/HttpStatusInteractor.java b/web/status-history/src/main/java/net/twisterrob/blt/gapp/HttpStatusInteractor.java new file mode 100644 index 00000000..08cb1e08 --- /dev/null +++ b/web/status-history/src/main/java/net/twisterrob/blt/gapp/HttpStatusInteractor.java @@ -0,0 +1,61 @@ +package net.twisterrob.blt.gapp; + +import java.io.IOException; +import java.io.InputStream; +import java.net.HttpURLConnection; +import java.net.URL; +import java.util.Collections; + +import javax.annotation.Nonnull; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import io.micronaut.context.annotation.Bean; +import kotlinx.datetime.Clock; +import kotlinx.datetime.Instant; + +import net.twisterrob.blt.io.feeds.Feed; +import net.twisterrob.java.io.IOTools; +import net.twisterrob.java.utils.ObjectTools; +import net.twisterrob.travel.domain.london.status.Stacktrace; +import net.twisterrob.travel.domain.london.status.StatusContent; +import net.twisterrob.travel.domain.london.status.StatusItem; +import net.twisterrob.travel.domain.london.status.api.StatusInteractor; + +import static net.twisterrob.blt.gapp.FeedConsts.URL_BUILDER; + +@Bean(typed = StatusInteractor.class) +class HttpStatusInteractor implements StatusInteractor { + private static final Logger LOG = LoggerFactory.getLogger(HttpStatusInteractor.class); + + @Override public @Nonnull StatusItem getCurrent(@Nonnull net.twisterrob.travel.domain.london.status.Feed feed) { + Instant now = Clock.System.INSTANCE.now(); + try { + StatusContent content = new StatusContent(downloadFeed(Feed.valueOf(feed.name()))); + return new StatusItem.SuccessfulStatusItem(feed, content, now); + } catch (Exception ex) { + LOG.error("Cannot load '{}'!", feed, ex); + Stacktrace stacktrace = new Stacktrace(ObjectTools.getFullStackTrace(ex)); + return new StatusItem.FailedStatusItem(feed, stacktrace, now); + } + } + + public static String downloadFeed(Feed feed) throws IOException { + InputStream input = null; + String result; + try { + URL url = URL_BUILDER.getFeedUrl(feed, Collections.emptyMap()); + LOG.debug("Requesting feed '{}': '{}'...", feed, url); + HttpURLConnection connection = (HttpURLConnection)url.openConnection(); + connection.setConnectTimeout(5000); + connection.setReadTimeout(5000); + connection.connect(); + input = connection.getInputStream(); + result = IOTools.readAll(input); + } finally { + IOTools.ignorantClose(input); + } + return result; + } +} diff --git a/web/status-history/src/main/java/net/twisterrob/blt/gapp/LineStatusHistoryServlet.java b/web/status-history/src/main/java/net/twisterrob/blt/gapp/LineStatusHistoryServlet.java index caafe976..76c56f05 100644 --- a/web/status-history/src/main/java/net/twisterrob/blt/gapp/LineStatusHistoryServlet.java +++ b/web/status-history/src/main/java/net/twisterrob/blt/gapp/LineStatusHistoryServlet.java @@ -1,6 +1,5 @@ package net.twisterrob.blt.gapp; -import java.io.*; import java.util.*; import io.micronaut.http.HttpResponse; @@ -11,64 +10,41 @@ import io.micronaut.views.View; import jakarta.servlet.http.*; -import com.google.cloud.datastore.*; -import com.google.cloud.datastore.StructuredQuery.OrderBy; - import net.twisterrob.blt.gapp.viewmodel.*; -import net.twisterrob.blt.io.feeds.Feed; import net.twisterrob.blt.io.feeds.trackernet.LineStatusFeed; -import net.twisterrob.java.utils.ObjectTools; - -import static net.twisterrob.blt.gapp.FeedConsts.*; -import static net.twisterrob.blt.gapp.FeedCronServlet.hasProperty; +import net.twisterrob.travel.domain.london.status.Feed; +import net.twisterrob.travel.domain.london.status.api.HistoryUseCase; +import net.twisterrob.travel.domain.london.status.api.ParsedStatusItem; @Controller -@SuppressWarnings("serial") public class LineStatusHistoryServlet { - //private static final Logger LOG = LoggerFactory.getLogger(LineStatusHistoryServlet.class); private static final String QUERY_DISPLAY_CURRENT = "current"; private static final String QUERY_DISPLAY_ERRORS = "errors"; private static final String QUERY_DISPLAY_MAX = "max"; private static final int DISPLAY_MAX_DEFAULT = 100; - private final Datastore datastore = DatastoreOptions.getDefaultInstance().getService(); + private final HistoryUseCase useCase; + + public LineStatusHistoryServlet(HistoryUseCase useCase) { + this.useCase = useCase; + } @Get("/LineStatusHistory") @View("LineStatus") @Produces(MediaType.TEXT_HTML) public HttpResponse doGet(HttpServletRequest req, HttpServletResponse resp) { + int max = parseInt(req.getParameter(QUERY_DISPLAY_MAX), DISPLAY_MAX_DEFAULT); + boolean displayCurrent = Boolean.parseBoolean(req.getParameter(QUERY_DISPLAY_CURRENT)); + boolean displayErrors = Boolean.parseBoolean(req.getParameter(QUERY_DISPLAY_ERRORS)); Feed feed = Feed.TubeDepartureBoardsLineStatus; - List results = new LinkedList<>(); - // Params - int max; - try { - max = Integer.parseInt(req.getParameter(QUERY_DISPLAY_MAX)); - } catch (NumberFormatException ex) { - max = DISPLAY_MAX_DEFAULT; - } - if (Boolean.parseBoolean(req.getParameter(QUERY_DISPLAY_CURRENT))) { - FullEntity entry = FeedCronServlet.downloadNewEntry(datastore, feed); - results.add(toResult(entry)); - } - boolean skipErrors = true; - if (Boolean.parseBoolean(req.getParameter(QUERY_DISPLAY_ERRORS))) { - skipErrors = false; - } - - // process them - Iterator entries = fetchEntries(feed); - while (entries.hasNext()) { - Entity entry = entries.next(); - if (--max < 0) { - break; // we've had enough - } - Result result = toResult(entry); - results.add(result); - } - - List differences = getDifferences(results, skipErrors); + List history = useCase.history(feed, max, displayCurrent); + List results = history + .stream() + .filter((it) -> displayErrors || !(it instanceof ParsedStatusItem.ParseFailed)) + .map(LineStatusHistoryServlet::toResult).toList(); + List differences = getDifferences(results); return HttpResponse.ok( new LineStatusHistoryModel( @@ -85,47 +61,25 @@ private record LineStatusHistoryModel( } - private static Result toResult(BaseEntity entry) { + private static Result toResult(ParsedStatusItem parsed) { Result result; - String content = hasProperty(entry, DS_PROP_CONTENT) ? entry.getString(DS_PROP_CONTENT) : null; - String error = hasProperty(entry, DS_PROP_ERROR) ? entry.getString(DS_PROP_ERROR) : null; - Date date = entry.getTimestamp(DS_PROP_RETRIEVED_DATE).toDate(); - if (content != null) { - try { - Feed feed = Feed.valueOf(entry.getKey().getKind()); - InputStream stream = new ByteArrayInputStream(content.getBytes(ENCODING)); - @SuppressWarnings({"RedundantTypeArguments", "RedundantSuppression"}) // False positive. - LineStatusFeed feedContents = feed.getHandler().parse(stream); - result = new Result(date, feedContents); - } catch (Exception ex) { - result = new Result(date, "Error while displaying loaded XML: " + ObjectTools.getFullStackTrace(ex)); - } - } else if (error != null) { - result = new Result(date, error); + Date date = new Date(parsed.getItem().getRetrievedDate().toEpochMilliseconds()); + if (parsed instanceof ParsedStatusItem.ParsedFeed feed) { + result = new Result(date, (LineStatusFeed)feed.getContent()); + } else if (parsed instanceof ParsedStatusItem.AlreadyFailed failure) { + result = new Result(date, failure.getItem().getError().getStacktrace()); + } else if (parsed instanceof ParsedStatusItem.ParseFailed parseFailure) { + result = new Result(date, "Error while displaying loaded XML: " + parseFailure.getError().getStacktrace()); } else { - result = new Result(date, "Empty entity"); + throw new IllegalArgumentException("Unsupported parse result: " + parsed); } return result; } - protected Iterator fetchEntries(Feed feed) { - Query q = Query - .newEntityQueryBuilder() - .setKind(feed.name()) - .addOrderBy(OrderBy.desc(DS_PROP_RETRIEVED_DATE)) - .build() - ; - QueryResults results = datastore.run(q); - return results; - } - - private static List getDifferences(List results, boolean skipErrors) { + private static List getDifferences(List results) { List resultChanges = new ArrayList<>(results.size()); Result newResult = null; for (Result oldResult : results) { // we're going forward, but the list is backwards - if (skipErrors && oldResult.getFullError() != null) { - continue; - } resultChanges.add(new ResultChange(oldResult, newResult)); newResult = oldResult; } @@ -133,4 +87,14 @@ private static List getDifferences(List results, boolean s resultChanges.remove(0); return resultChanges; } + + private static int parseInt(String value, int def) { + int max; + try { + max = Integer.parseInt(value); + } catch (NumberFormatException ex) { + max = def; + } + return max; + } } diff --git a/web/status-history/src/test/java/net/twisterrob/blt/gapp/DatastoreStatusHistoryRepositoryUnitTest.java b/web/status-history/src/test/java/net/twisterrob/blt/gapp/DatastoreStatusHistoryRepositoryUnitTest.java new file mode 100644 index 00000000..c3867a99 --- /dev/null +++ b/web/status-history/src/test/java/net/twisterrob/blt/gapp/DatastoreStatusHistoryRepositoryUnitTest.java @@ -0,0 +1,268 @@ +package net.twisterrob.blt.gapp; + +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.UUID; + +import org.junit.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.ArgumentMatchers; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.hasEntry; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.instanceOf; +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.google.cloud.Timestamp; +import com.google.cloud.datastore.BaseKey; +import com.google.cloud.datastore.Datastore; +import com.google.cloud.datastore.Entity; +import com.google.cloud.datastore.EntityQuery; +import com.google.cloud.datastore.FullEntity; +import com.google.cloud.datastore.Key; +import com.google.cloud.datastore.KeyFactory; +import com.google.cloud.datastore.QueryResults; +import com.google.cloud.datastore.StringValue; +import com.google.cloud.datastore.StructuredQuery.OrderBy; +import com.google.cloud.datastore.TimestampValue; +import com.google.cloud.datastore.Value; + +import kotlin.random.Random; +import kotlinx.datetime.Instant; + +import net.twisterrob.travel.domain.london.status.Feed; +import net.twisterrob.travel.domain.london.status.Stacktrace; +import net.twisterrob.travel.domain.london.status.StatusContent; +import net.twisterrob.travel.domain.london.status.StatusItem; +import net.twisterrob.travel.domain.london.status.StatusItem.FailedStatusItem; +import net.twisterrob.travel.domain.london.status.StatusItem.SuccessfulStatusItem; +import net.twisterrob.travel.domain.london.status.api.StatusHistoryRepository; + +public class DatastoreStatusHistoryRepositoryUnitTest { + + private final Datastore mockDatastore = mock(Datastore.class); + private final StatusHistoryRepository subject = new DatastoreStatusHistoryRepository(mockDatastore); + + @Test public void testSavesSuccessfulItem() { + when(mockDatastore.newKeyFactory()).thenReturn(new KeyFactory("test-project-id")); + + SuccessfulStatusItem item = SuccessfulStatusItem(); + + subject.add(item); + + FullEntity entity = captureAddedEntity(mockDatastore); + assertEquals(item.getFeed().name(), entity.getKey().getKind()); + Map> props = entity.getProperties(); + assertThat(props.entrySet(), hasSize(2)); + Instant now = item.getRetrievedDate(); + Timestamp ts = Timestamp.ofTimeSecondsAndNanos(now.getEpochSeconds(), now.getNanosecondsOfSecond()); + assertThat(props, hasEntry("retrievedDate", TimestampValue.of(ts))); + String content = item.getContent().getContent(); + assertThat(props, hasEntry("content", StringValue.newBuilder(content).setExcludeFromIndexes(true).build())); + } + + @Test public void testSavesFailedItem() { + when(mockDatastore.newKeyFactory()).thenReturn(new KeyFactory("test-project-id")); + FailedStatusItem item = FailedStatusItem(); + + subject.add(item); + + FullEntity entity = captureAddedEntity(mockDatastore); + assertEquals(item.getFeed().name(), entity.getKey().getKind()); + Map> props = entity.getProperties(); + assertThat(props.entrySet(), hasSize(2)); + Instant now = item.getRetrievedDate(); + Timestamp ts = Timestamp.ofTimeSecondsAndNanos(now.getEpochSeconds(), now.getNanosecondsOfSecond()); + assertThat(props, hasEntry("retrievedDate", TimestampValue.of(ts))); + String error = item.getError().getStacktrace(); + assertThat(props, hasEntry("error", StringValue.newBuilder(error).setExcludeFromIndexes(true).build())); + } + + @Test public void testQuery() { + @SuppressWarnings("unchecked") + QueryResults mockResults = mock(QueryResults.class); + doAnswerIterator(mockResults, Collections.emptyIterator()); + when(mockDatastore.run(ArgumentMatchers.any())).thenReturn(mockResults); + + Feed feed = randomFeed(); + subject.getAll(feed, 2); + + EntityQuery query = captureRanQuery(mockDatastore); + assertEquals(feed.name(), query.getKind()); + assertEquals(List.of(OrderBy.desc("retrievedDate")), query.getOrderBy()); + assertEquals(Integer.valueOf(2), query.getLimit()); + } + + @Test public void testLimitShortensLongerList() { + List list = List.of(SuccessfulEntity(), SuccessfulEntity(), SuccessfulEntity()); + @SuppressWarnings("unchecked") + QueryResults mockResults = mock(QueryResults.class); + doAnswerIterator(mockResults, list.iterator()); + when(mockDatastore.run(ArgumentMatchers.any())).thenReturn(mockResults); + + List result = subject.getAll(randomFeed(), 2); + + assertThat(result, hasSize(2)); + } + + @Test public void testReturnsSuccessfulEntityData() { + Feed feed = randomFeed(); + Entity entity = SuccessfulEntity(feed); + @SuppressWarnings("unchecked") + QueryResults mockResults = mock(QueryResults.class); + List list = List.of(entity); + doAnswerIterator(mockResults, list.iterator()); + when(mockDatastore.run(ArgumentMatchers.any())).thenReturn(mockResults); + + List result = subject.getAll(randomFeed(), 2); + + StatusItem item = result.get(0); + assertEquals(feed, item.getFeed()); + Timestamp ts = entity.getTimestamp("retrievedDate"); + Instant instant = Instant.Companion.fromEpochSeconds(ts.getSeconds(), ts.getNanos()); + assertEquals(instant, item.getRetrievedDate()); + assertThat(item, instanceOf(SuccessfulStatusItem.class)); + assertEquals(entity.getString("content"), ((SuccessfulStatusItem)item).getContent().getContent()); + } + + @Test public void testReturnsFailedEntityData() { + Feed feed = randomFeed(); + Entity entity = FailedEntity(feed); + List list = List.of(entity); + @SuppressWarnings("unchecked") + QueryResults mockResults = mock(QueryResults.class); + doAnswerIterator(mockResults, list.iterator()); + when(mockDatastore.run(ArgumentMatchers.any())).thenReturn(mockResults); + + List result = subject.getAll(randomFeed(), 2); + + StatusItem item = result.get(0); + assertEquals(feed, item.getFeed()); + Timestamp ts = entity.getTimestamp("retrievedDate"); + Instant instant = Instant.Companion.fromEpochSeconds(ts.getSeconds(), ts.getNanos()); + assertEquals(instant, item.getRetrievedDate()); + assertThat(item, instanceOf(FailedStatusItem.class)); + assertEquals(entity.getString("error"), ((FailedStatusItem)item).getError().getStacktrace()); + } + + @Test public void testSymmetricSaveLoadSuccessfulItem() { + SuccessfulStatusItem item = SuccessfulStatusItem(); + when(mockDatastore.newKeyFactory()).thenReturn(new KeyFactory("test-project-id")); + subject.add(item); + FullEntity savedEntity = captureAddedEntity(mockDatastore); + Entity entity = Entity.newBuilder(randomKey(item.getFeed().name()), savedEntity).build(); + + @SuppressWarnings("unchecked") + QueryResults mockResults = mock(QueryResults.class); + doAnswerIterator(mockResults, List.of(entity).iterator()); + when(mockDatastore.run(ArgumentMatchers.any())).thenReturn(mockResults); + + List result = subject.getAll(randomFeed(), 1); + assertEquals(item, result.get(0)); + } + + private static void doAnswerIterator(Iterator mock, Iterator value) { + doAnswer(invocation -> value.next()).when(mock).next(); + doAnswer(invocation -> value.hasNext()).when(mock).hasNext(); + } + + private static EntityQuery captureRanQuery(Datastore mockDatastore) { + ArgumentCaptor captor = ArgumentCaptor.forClass(EntityQuery.class); + verify(mockDatastore).run(captor.capture()); + assertThat(captor.getAllValues(), hasSize(1)); + return captor.getAllValues().get(0); + } + + private static FullEntity captureAddedEntity(Datastore mockDatastore) { + @SuppressWarnings("unchecked") + ArgumentCaptor> captor = ArgumentCaptor.forClass(FullEntity.class); + verify(mockDatastore).add(captor.capture()); + assertThat(captor.getAllValues(), hasSize(1)); + return captor.getAllValues().get(0); + } + + private static Entity SuccessfulEntity() { + return SuccessfulEntity(randomFeed()); + } + + private static Entity SuccessfulEntity(Feed feed) { + return Entity + .newBuilder(randomKey(feed.name())) + .set("retrievedDate", randomTimestamp()) + .set("content", UUID.randomUUID().toString()) + .build(); + } + + private static Entity FailedEntity(Feed feed) { + return Entity + .newBuilder(randomKey(feed.name())) + .set("retrievedDate", randomTimestamp()) + .set("error", UUID.randomUUID().toString()) + .build(); + } + + private static StatusItem.SuccessfulStatusItem SuccessfulStatusItem() { + return new StatusItem.SuccessfulStatusItem( + randomFeed(), + new StatusContent(UUID.randomUUID().toString()), + randomInstant() + ); + } + + private static StatusItem.FailedStatusItem FailedStatusItem() { + return new StatusItem.FailedStatusItem( + randomFeed(), + new Stacktrace(UUID.randomUUID().toString()), + randomInstant() + ); + } + + private static Feed randomFeed() { + return Feed.getEntries().get(Random.Default.nextInt(Feed.getEntries().size())); + } + + private static Key randomKey(String kind) { + KeyFactory keyFactory = new KeyFactory("test-project-id").setKind(kind); + return keyFactory.newKey(UUID.randomUUID().toString()); + } + + private static Timestamp randomTimestamp() { + return Timestamp.ofTimeSecondsAndNanos( + Random.Default.nextLong( + Timestamp.MIN_VALUE.getSeconds(), + Timestamp.MAX_VALUE.getSeconds() + ), + Random.Default.nextInt( + Timestamp.MIN_VALUE.getNanos(), + Timestamp.MAX_VALUE.getNanos() + ) + ); + } + + private static Instant randomInstant() { + System.out.println(Instant.Companion.getDISTANT_PAST()); + System.out.println(Instant.Companion.getDISTANT_FUTURE()); + System.out.println(Timestamp.MIN_VALUE); + System.out.println(Timestamp.MAX_VALUE); + // Note: Have to use the range from Google's Timestamp, because Instant.DISTANT_PAST/DISTANT_FUTURE + // are out of range (-100001-12-31T23:59:59.999999999Z - +100000-01-01T00:00:00Z), + // compared to Timestamp's 0001-01-01T00:00:00Z - 9999-12-31T23:59:59.999999999Z range. + return Instant.Companion.fromEpochSeconds( + Random.Default.nextLong( + Timestamp.MIN_VALUE.getSeconds(), + Timestamp.MAX_VALUE.getSeconds() + ), + Random.Default.nextInt( + Timestamp.MIN_VALUE.getNanos(), + Timestamp.MAX_VALUE.getNanos() + ) + ); + } +}