From 9d2ae15dff48ffa6b5fb26e386caf7ad25d76427 Mon Sep 17 00:00:00 2001 From: ekuzmichev Date: Tue, 17 Sep 2024 19:13:02 +0300 Subject: [PATCH] Add support for sharing product from app (containing text and short URL) --- build.sbt | 6 +- src/main/scala/app/AppLayers.scala | 2 + .../OzonPriceCheckerUpdateConsumer.scala | 3 +- .../scala/product/OzonProductIdParser.scala | 41 ++++++-- src/main/scala/product/ProductIdParser.scala | 4 +- .../scala/product/ProductIdParserLayers.scala | 6 +- .../util/ozon/OzonShortUrlResolver.scala | 3 +- .../util/ozon/OzonShortUrlResolverImpl.scala | 8 +- .../scala/util/ozon/OzonShortUrlUtils.scala | 5 - .../OzonProductIdParserIntegrationTest.scala | 94 +++++++++++++++++++ .../product/OzonProductIdParserTest.scala | 56 ----------- .../OzonShortUrlResolverImplTestApp.scala | 3 +- .../util/ozon/OzonShortUrlUtilsTest.scala | 15 --- ...pec.scala => UrlExtractionUtilsTest.scala} | 4 +- 14 files changed, 151 insertions(+), 99 deletions(-) delete mode 100644 src/main/scala/util/ozon/OzonShortUrlUtils.scala create mode 100644 src/test/scala/product/OzonProductIdParserIntegrationTest.scala delete mode 100644 src/test/scala/product/OzonProductIdParserTest.scala delete mode 100644 src/test/scala/util/ozon/OzonShortUrlUtilsTest.scala rename src/test/scala/util/uri/{UrlExtractionUtilsSpec.scala => UrlExtractionUtilsTest.scala} (93%) diff --git a/build.sbt b/build.sbt index 18dcbc4..d8a10db 100644 --- a/build.sbt +++ b/build.sbt @@ -14,6 +14,8 @@ ThisBuild / assemblyMergeStrategy := { case x => MergeStrategy.defaultMergeStrategy(x) } +ThisBuild / Test / testOptions += Tests.Filter(s => !s.endsWith("IntegrationTest")) + lazy val root = (project in file(".")) .settings( name := "ozon-price-checker", @@ -33,8 +35,8 @@ lazy val root = (project in file(".")) "org.jasypt" % "jasypt" % "1.9.3", "com.github.pathikrit" %% "better-files" % "3.9.2", "org.scalatest" %% "scalatest" % "3.2.19" % Test, - "dev.zio" %% "zio-test" % "2.1.6" % Test, - "dev.zio" %% "zio-test-sbt" % "2.1.6" % Test, + "dev.zio" %% "zio-test" % "2.1.9" % Test, + "dev.zio" %% "zio-test-sbt" % "2.1.9" % Test, "com.stephenn" %% "scalatest-circe" % "0.2.5" % Test ) ++ Seq( diff --git a/src/main/scala/app/AppLayers.scala b/src/main/scala/app/AppLayers.scala index 3c27c07..e7e7fa3 100644 --- a/src/main/scala/app/AppLayers.scala +++ b/src/main/scala/app/AppLayers.scala @@ -12,6 +12,7 @@ import schedule.{JobIdGeneratorLayers, ZioSchedulerLayers} import store.{CacheStateRepository, CacheStateRepositoryLayers, ProductStore, ProductStoreLayers} import telegram.TelegramClientLayers import util.lang.Throwables.failure +import util.ozon.OzonShortUrlResolverLayers import org.telegram.telegrambots.longpolling.interfaces.LongPollingUpdateConsumer import zio.{RLayer, Task, TaskLayer, URLayer, ZIO, ZIOAppArgs, ZLayer} @@ -46,6 +47,7 @@ object AppLayers: BrowserLayers.jsoup, JobIdGeneratorLayers.alphaNumeric, ProductIdParserLayers.ozon, + OzonShortUrlResolverLayers.impl, CommandProcessorLayers.ozonPriceChecker, EncDecLayers.aes256(encryptionPassword.value), ProductWatchingJobSchedulerLayers.impl, diff --git a/src/main/scala/consumer/OzonPriceCheckerUpdateConsumer.scala b/src/main/scala/consumer/OzonPriceCheckerUpdateConsumer.scala index 5313446..77383a3 100644 --- a/src/main/scala/consumer/OzonPriceCheckerUpdateConsumer.scala +++ b/src/main/scala/consumer/OzonPriceCheckerUpdateConsumer.scala @@ -63,12 +63,13 @@ class OzonPriceCheckerUpdateConsumer( case Some(productCandidate) => productCandidate match case WaitingProductId => - productIdParser.parse(text) match + productIdParser.parse(text).flatMap { case Right(productId) => onProductId(sourceId, productId) case Left(error) => ZIO.log(s"Failed to parse productId from text '$text'. Cause: $error") *> sendTextMessage(sourceId.chatId, s"Send me valid URL or product ID") + } case WaitingPriceThreshold(productId, productPrice) => text.toIntOption match case Some(priceThreshold) => diff --git a/src/main/scala/product/OzonProductIdParser.scala b/src/main/scala/product/OzonProductIdParser.scala index 29e4cc3..0351b49 100644 --- a/src/main/scala/product/OzonProductIdParser.scala +++ b/src/main/scala/product/OzonProductIdParser.scala @@ -2,21 +2,40 @@ package ru.ekuzmichev package product import common.ProductId -import product.OzonProductIdParser.OzonHostRegex -import util.lang.Throwables -import util.lang.Throwables.makeCauseSeqMessage +import product.OzonProductIdParser.{OzonHostRegex, OzonProductUrlPathPart, OzonShortUrlPathPart} +import util.ozon.OzonShortUrlResolver +import util.uri.UrlExtractionUtils -import cats.syntax.either.* import io.lemonlabs.uri.{Url, UrlPath} +import zio.{Task, ZIO} import scala.util.matching.Regex -class OzonProductIdParser extends ProductIdParser: - override def parse(s: String): Either[String, ProductId] = - parseUrlEither(s).flatMap(extractProductId(s, _)) +class OzonProductIdParser(ozonShortUrlResolver: OzonShortUrlResolver) extends ProductIdParser: + override def parse(s: String): Task[Either[String, ProductId]] = + UrlExtractionUtils.extractUrl(s).flatMap { + case Some(url) => + if isOzonUrl(url) then + if isShortOzonUrl(url) then + ozonShortUrlResolver + .resolveShortUrl(url) + .flatMap(extractProductId) + else extractProductId(url) + else ZIO.left(s"URL $url contains host other than *ozon.ru") + case None => + ZIO.logDebug(s"Returning $s as product id") *> ZIO.right(s) + } + + private def extractProductId(url: Url) = + if isProductUrl(url) then + takeProductIdFromPath(url.path) match + case Some(productId) => ZIO.right(productId) + case None => ZIO.left(s"Not found product ID in URL path. URL: $url") + else ZIO.left(s"URL $url is not product OZON URL") + + private def isShortOzonUrl(url: Url): Boolean = url.path.parts.contains(OzonShortUrlPathPart) - private def parseUrlEither(s: String): Either[String, Url] = - Url.parseTry(s).toEither.leftMap(makeCauseSeqMessage(_)) + private def isProductUrl(url: Url) = url.path.parts.contains(OzonProductUrlPathPart) private def extractProductId(s: ProductId, url: Url): Either[String, ProductId] = if isOzonUrl(url) then @@ -33,4 +52,6 @@ class OzonProductIdParser extends ProductIdParser: urlPath.parts.drop(1).filter(!_.isBlank).headOption object OzonProductIdParser: - val OzonHostRegex: Regex = ".*ozon.ru".r + val OzonHostRegex: Regex = ".*ozon.ru".r + val OzonShortUrlPathPart: String = "t" + val OzonProductUrlPathPart: String = "product" diff --git a/src/main/scala/product/ProductIdParser.scala b/src/main/scala/product/ProductIdParser.scala index 354f174..f7979c9 100644 --- a/src/main/scala/product/ProductIdParser.scala +++ b/src/main/scala/product/ProductIdParser.scala @@ -3,5 +3,7 @@ package product import common.ProductId +import zio.Task + trait ProductIdParser: - def parse(s: String): Either[String, ProductId] + def parse(s: String): Task[Either[String, ProductId]] diff --git a/src/main/scala/product/ProductIdParserLayers.scala b/src/main/scala/product/ProductIdParserLayers.scala index 204c8e6..e20dcd5 100644 --- a/src/main/scala/product/ProductIdParserLayers.scala +++ b/src/main/scala/product/ProductIdParserLayers.scala @@ -1,7 +1,9 @@ package ru.ekuzmichev package product -import zio.{ULayer, ZLayer} +import util.ozon.OzonShortUrlResolver + +import zio.{RLayer, ZLayer} object ProductIdParserLayers: - val ozon: ULayer[ProductIdParser] = ZLayer.succeed(new OzonProductIdParser) + val ozon: RLayer[OzonShortUrlResolver, ProductIdParser] = ZLayer.fromFunction(new OzonProductIdParser(_)) diff --git a/src/main/scala/util/ozon/OzonShortUrlResolver.scala b/src/main/scala/util/ozon/OzonShortUrlResolver.scala index e98fa1e..fbb3495 100644 --- a/src/main/scala/util/ozon/OzonShortUrlResolver.scala +++ b/src/main/scala/util/ozon/OzonShortUrlResolver.scala @@ -1,7 +1,8 @@ package ru.ekuzmichev package util.ozon +import io.lemonlabs.uri.Url import zio.Task trait OzonShortUrlResolver: - def resolveShortUrl(url: String): Task[String] + def resolveShortUrl(url: Url): Task[Url] diff --git a/src/main/scala/util/ozon/OzonShortUrlResolverImpl.scala b/src/main/scala/util/ozon/OzonShortUrlResolverImpl.scala index 52354df..d4c262e 100644 --- a/src/main/scala/util/ozon/OzonShortUrlResolverImpl.scala +++ b/src/main/scala/util/ozon/OzonShortUrlResolverImpl.scala @@ -3,6 +3,7 @@ package util.ozon import util.ozon.OzonShortUrlResolverImpl.ProductUrlExtractor +import io.lemonlabs.uri.Url import net.ruippeixotog.scalascraper.browser.Browser import net.ruippeixotog.scalascraper.dsl.DSL.* import net.ruippeixotog.scalascraper.dsl.DSL.Extract.* @@ -13,11 +14,12 @@ import zio.{Task, ZIO} class OzonShortUrlResolverImpl(browser: Browser) extends OzonShortUrlResolver: private type BrowserDocument = browser.DocumentType - override def resolveShortUrl(url: String): Task[String] = + override def resolveShortUrl(url: Url): Task[Url] = ZIO.attempt { - val responseHtml: BrowserDocument = browser.get(url) + val responseHtml: BrowserDocument = browser.get(url.toStringPunycode) responseHtml >> ProductUrlExtractor - } + }.flatMap(fullUrlString => ZIO.fromTry(Url.parseTry(fullUrlString))) + .tap{fullUrl => ZIO.logDebug(s"Resolved short OZON URL $url into full OZON URL $fullUrl")} object OzonShortUrlResolverImpl: val ProductUrlExtractor: HtmlExtractor[Element, String] = attr("content")("meta[data-hid=property::og:url]") diff --git a/src/main/scala/util/ozon/OzonShortUrlUtils.scala b/src/main/scala/util/ozon/OzonShortUrlUtils.scala deleted file mode 100644 index 116e254..0000000 --- a/src/main/scala/util/ozon/OzonShortUrlUtils.scala +++ /dev/null @@ -1,5 +0,0 @@ -package ru.ekuzmichev -package util.ozon - -object OzonShortUrlUtils: - def isShortUrl(url: String): Boolean = ".*ozon.ru/t/.*".r.matches(url) diff --git a/src/test/scala/product/OzonProductIdParserIntegrationTest.scala b/src/test/scala/product/OzonProductIdParserIntegrationTest.scala new file mode 100644 index 0000000..85b4434 --- /dev/null +++ b/src/test/scala/product/OzonProductIdParserIntegrationTest.scala @@ -0,0 +1,94 @@ +package ru.ekuzmichev +package product + +import util.ozon.OzonShortUrlResolverImpl + +import net.ruippeixotog.scalascraper.browser.JsoupBrowser +import zio.test.* +import zio.test.Assertion.* + +object OzonProductIdParserIntegrationTest extends ZIOSpecDefault: + def spec: Spec[Any, Throwable] = + suite("OzonProductIdParser.parse")( + test("parse Ozon product id from Ozon URL 'shared' from browser") { + val parser = makeProductIdParser + parser + .parse( + "https://www.ozon.ru/product/kanistra-dlya-smeshivaniya-benzina-i-masla-dde-2-l-1422144661/?__rr=1&from=share_ios&utm_campaign=productpage_link&utm_medium=share_button&utm_source=smm" + ) + .map { parseResult => + assert(parseResult)(isRight(equalTo("kanistra-dlya-smeshivaniya-benzina-i-masla-dde-2-l-1422144661"))) + } + }, + test("parse take product id from plain non-URL string") { + val parser = makeProductIdParser + parser + .parse( + "akusticheskaya-gitara-donner-hush-i-silent-guitar-sunburst-6-strunnaya-988766503" + ) + .map { parseResult => + assert(parseResult)( + isRight(equalTo("akusticheskaya-gitara-donner-hush-i-silent-guitar-sunburst-6-strunnaya-988766503")) + ) + } + }, + test("parse Ozon product id from Ozon link URL without query parameters") { + val parser = makeProductIdParser + parser + .parse( + "https://www.ozon.ru/product/kanistra-dlya-smeshivaniya-benzina-i-masla-dde-2-l-1422144661" + ) + .map { parseResult => + assert(parseResult)(isRight(equalTo("kanistra-dlya-smeshivaniya-benzina-i-masla-dde-2-l-1422144661"))) + } + }, + test("parse Ozon product id from Ozon short URL") { + val parser = makeProductIdParser + parser + .parse("https://ozon.ru/t/YKknAE4") + .map { parseResult => + assert(parseResult)( + isRight( + equalTo( + "kovrik-samonaduvayushchiysya-naturehike-yugu-ultralight-automatic-inflatable-cushion-mummy-blue-b-r-1449496528" + ) + ) + ) + } + }, + test("parse Ozon product id from Ozon short URL wrapped with text") { + val parser = makeProductIdParser + parser + .parse( + "Коврик Самонадувающийся Naturehike Yugu Ultralight Automatic Inflatable Cushion Mummy Blue (Б/Р) https://ozon.ru/t/YKknAE4" + ) + .map { parseResult => + assert(parseResult)( + isRight( + equalTo( + "kovrik-samonaduvayushchiysya-naturehike-yugu-ultralight-automatic-inflatable-cushion-mummy-blue-b-r-1449496528" + ) + ) + ) + } + }, + test("fail if link has no path") { + val parser = makeProductIdParser + parser + .parse("https://www.ozon.ru/product/") + .map { parseResult => + assert(parseResult)(isLeft) + } + }, + test("fail if link has invalid host") { + val parser = makeProductIdParser + parser + .parse("https://www.vk.ru/product/kanistra-dlya-smeshivaniya-benzina-i-masla-dde-2-l-1422144661") + .map { parseResult => + assert(parseResult)(isLeft) + } + } + ) + + private def makeProductIdParser: ProductIdParser = + new OzonProductIdParser(new OzonShortUrlResolverImpl(new JsoupBrowser())) diff --git a/src/test/scala/product/OzonProductIdParserTest.scala b/src/test/scala/product/OzonProductIdParserTest.scala deleted file mode 100644 index fb45628..0000000 --- a/src/test/scala/product/OzonProductIdParserTest.scala +++ /dev/null @@ -1,56 +0,0 @@ -package ru.ekuzmichev -package product - -import org.scalatest.Inside.* -import org.scalatest.flatspec.AnyFlatSpecLike -import org.scalatest.matchers.should.Matchers.* - -class OzonProductIdParserTest extends AnyFlatSpecLike: - - it should "parse Ozon product id from Ozon URL 'shared' from browser" in { - val parser = new OzonProductIdParser - val parseResult = parser.parse( - "https://www.ozon.ru/product/kanistra-dlya-smeshivaniya-benzina-i-masla-dde-2-l-1422144661/?__rr=1&from=share_ios&utm_campaign=productpage_link&utm_medium=share_button&utm_source=smm" - ) - - inside(parseResult) { case Right(productId) => - productId should be("kanistra-dlya-smeshivaniya-benzina-i-masla-dde-2-l-1422144661") - } - } - - it should "parse take product id from plain non-URL string" in { - val parser = new OzonProductIdParser - val parseResult = parser.parse( - "akusticheskaya-gitara-donner-hush-i-silent-guitar-sunburst-6-strunnaya-988766503" - ) - - inside(parseResult) { case Right(productId) => - productId should be("akusticheskaya-gitara-donner-hush-i-silent-guitar-sunburst-6-strunnaya-988766503") - } - } - - it should "parse Ozon product id from Ozon link URL without query parameters" in { - val parser = new OzonProductIdParser - val parseResult = parser.parse( - "https://www.ozon.ru/product/kanistra-dlya-smeshivaniya-benzina-i-masla-dde-2-l-1422144661" - ) - - inside(parseResult) { case Right(productId) => - productId should be("kanistra-dlya-smeshivaniya-benzina-i-masla-dde-2-l-1422144661") - } - } - - it should "fail if link has no path" in { - val parser = new OzonProductIdParser - val parseResult = parser.parse("https://www.ozon.ru/product/") - - parseResult.isLeft should be(true) - } - - it should "fail if link has invalid host" in { - val parser = new OzonProductIdParser - val parseResult = - parser.parse("https://www.vk.ru/product/kanistra-dlya-smeshivaniya-benzina-i-masla-dde-2-l-1422144661") - - parseResult.isLeft should be(true) - } diff --git a/src/test/scala/util/ozon/OzonShortUrlResolverImplTestApp.scala b/src/test/scala/util/ozon/OzonShortUrlResolverImplTestApp.scala index a716afb..55b9b68 100644 --- a/src/test/scala/util/ozon/OzonShortUrlResolverImplTestApp.scala +++ b/src/test/scala/util/ozon/OzonShortUrlResolverImplTestApp.scala @@ -1,12 +1,13 @@ package ru.ekuzmichev package util.ozon +import io.lemonlabs.uri.Url import net.ruippeixotog.scalascraper.browser.JsoupBrowser import zio.{Scope, ZIO, ZIOAppArgs, ZIOAppDefault} object OzonShortUrlResolverImplTestApp extends ZIOAppDefault: override def run: ZIO[Any with ZIOAppArgs with Scope, Any, Any] = - val productShortUrl = "https://ozon.ru/t/YKknAE4" + val productShortUrl: Url = Url.parse("https://ozon.ru/t/YKknAE4") val ozonShortUrlResolver = new OzonShortUrlResolverImpl(new JsoupBrowser()) ozonShortUrlResolver .resolveShortUrl(productShortUrl) diff --git a/src/test/scala/util/ozon/OzonShortUrlUtilsTest.scala b/src/test/scala/util/ozon/OzonShortUrlUtilsTest.scala deleted file mode 100644 index c305beb..0000000 --- a/src/test/scala/util/ozon/OzonShortUrlUtilsTest.scala +++ /dev/null @@ -1,15 +0,0 @@ -package ru.ekuzmichev -package util.ozon - -import org.scalatest.flatspec.AnyFlatSpecLike -import org.scalatest.matchers.should.Matchers.* - -class OzonShortUrlUtilsTest extends AnyFlatSpecLike: - - "isShortUrl" should "return true if the format matches .*ozon.ru/t/.*" in { - OzonShortUrlUtils.isShortUrl("https://ozon.ru/t/YKknAE4") should be(true) - OzonShortUrlUtils.isShortUrl("https://another-site.ru/t/YKknAE4") should be(false) - OzonShortUrlUtils.isShortUrl( - "https://www.ozon.ru/product/leska-dlya-trimmera-2-4-mm-denzel-extra-cord-vitoy-kvadrat-243-m-dvuhkomponentnaya-iz-poliamida-932200216" - ) should be(false) - } diff --git a/src/test/scala/util/uri/UrlExtractionUtilsSpec.scala b/src/test/scala/util/uri/UrlExtractionUtilsTest.scala similarity index 93% rename from src/test/scala/util/uri/UrlExtractionUtilsSpec.scala rename to src/test/scala/util/uri/UrlExtractionUtilsTest.scala index c7a4e7f..31e15bf 100644 --- a/src/test/scala/util/uri/UrlExtractionUtilsSpec.scala +++ b/src/test/scala/util/uri/UrlExtractionUtilsTest.scala @@ -4,9 +4,9 @@ package util.uri import io.lemonlabs.uri.{QueryString, Url} import zio.test.{Spec, ZIOSpecDefault, assertTrue} -object UrlExtractionUtilsSpec extends ZIOSpecDefault: +object UrlExtractionUtilsTest extends ZIOSpecDefault: def spec: Spec[Any, Throwable] = - suite("UrlExtractionUtilsSpec.extractUrl")( + suite("UrlExtractionUtils.extractUrl")( test("should extract URL from text containing it") { for maybeUrl <- UrlExtractionUtils.extractUrl( s"Коврик Самонадувающийся Naturehike Yugu Ultralight Automatic Inflatable Cushion Mummy Blue (Б/Р) https://ozon.ru/t/YKknAE4"