Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add trait for audio and podcast. #550

Merged
merged 8 commits into from
Dec 16, 2024
Merged
Show file tree
Hide file tree
Changes from 7 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ import no.ndla.searchapi.integration.SearchApiClient
import no.ndla.searchapi.model.api.grep.GrepSearchResultsDTO
import no.ndla.searchapi.model.api.{ErrorHandling, GroupSearchResultDTO, MultiSearchResultDTO, SubjectAggregationsDTO}
import no.ndla.searchapi.model.domain.{LearningResourceType, Sort}
import no.ndla.searchapi.model.search.SearchType
import no.ndla.searchapi.model.search.{SearchTrait, SearchType}
import no.ndla.searchapi.model.search.settings.{MultiDraftSearchSettings, SearchSettings}
import no.ndla.searchapi.service.search.{
GrepSearchService,
Expand Down Expand Up @@ -125,19 +125,15 @@ trait SearchController {
|If subjects are specified the learning resource must have specified relevances in relation to a specified subject.
|If levels are specified the learning resource must have specified relevances in relation to a specified level.""".stripMargin
)
private val contextFilters = listQuery[String]("context-filters")
.description(
"""A comma separated list of resource-types the learning resources should be filtered by.
|Used in conjunction with the parameter resource-types to filter additional resource-types.
""".stripMargin
)
private val includeMissingResourceTypeGroup = query[Boolean]("missing-group")
.description(
"Whether to include group without resource-types for group-search. Defaults to false."
)
.default(false)
private val grepCodes = listQuery[String]("grep-codes")
.description("A comma separated list of codes from GREP API the resources should be filtered by.")
private val traits = listQuery[String]("traits")
.description("A comma separated list of traits the resources should be filtered by.")
private val scrollId = query[Option[String]]("search-context")
.description(
s"""A unique string obtained from a search you want to keep scrolling in. To obtain one from a search, provide one of the following values: ${InitialScrollContextKeywords
Expand Down Expand Up @@ -199,10 +195,10 @@ trait SearchController {
.in(contextTypes)
.in(languageFilter)
.in(relevanceFilter)
.in(contextFilters)
.in(includeMissingResourceTypeGroup)
.in(aggregatePaths)
.in(grepCodes)
.in(traits)
.in(embedResource)
.in(embedId)
.in(filterInactive)
Expand All @@ -223,10 +219,10 @@ trait SearchController {
contextTypes,
languageFilter,
relevanceFilter,
anotherResourceTypes,
includeMissingResourceTypeGroup,
aggregatePaths,
grepCodes,
traits,
embedResource,
embedId,
filterInactive,
Expand All @@ -249,11 +245,12 @@ trait SearchController {
sort = sort,
withIdIn = learningResourceIds.values,
subjects = subjects.values,
resourceTypes = groupTypes.values ++ anotherResourceTypes.values,
resourceTypes = groupTypes.values,
learningResourceTypes = contextTypes.values.flatMap(LearningResourceType.valueOf),
supportedLanguages = languageFilter.values,
relevanceIds = relevanceFilter.values,
grepCodes = grepCodes.values,
traits = traits.values.flatMap(SearchTrait.valueOf),
shouldScroll = false,
filterByNoResourceType = false,
aggregatePaths = aggregatePaths.values,
Expand Down Expand Up @@ -380,9 +377,9 @@ trait SearchController {
.in(subjects)
.in(languageFilter)
.in(relevanceFilter)
.in(contextFilters)
.in(scrollId)
.in(grepCodes)
.in(traits)
.in(aggregatePaths)
.in(embedResource)
.in(embedId)
Expand All @@ -404,9 +401,9 @@ trait SearchController {
subjects,
languageFilter,
relevanceFilter,
contextFilters,
scrollId,
grepCodes,
traits,
aggregatePaths,
embedResource,
embedId,
Expand All @@ -427,11 +424,12 @@ trait SearchController {
sort = sort.getOrElse(Sort.ByRelevanceDesc),
withIdIn = learningResourceIds.values,
subjects = subjects.values,
resourceTypes = resourceTypes.values ++ contextFilters.values,
resourceTypes = resourceTypes.values,
learningResourceTypes = contextTypes.values.flatMap(LearningResourceType.valueOf),
supportedLanguages = languageFilter.values,
relevanceIds = relevanceFilter.values,
grepCodes = grepCodes.values,
traits = traits.values.flatMap(SearchTrait.valueOf),
shouldScroll = shouldScroll,
filterByNoResourceType = false,
aggregatePaths = aggregatePaths.values,
Expand Down Expand Up @@ -552,6 +550,7 @@ trait SearchController {
draftStatus = stringListParam("draft-status").some,
users = stringListParam("users").some,
grepCodes = stringListParam("grep-codes").some,
traits = stringListParam("traits").flatMap(SearchTrait.withNameOption).some,
aggregatePaths = stringListParam("aggregate-paths").some,
embedResource = stringListParam("embed-resource").some,
embedId = stringParamOrNone("embed-id"),
Expand Down Expand Up @@ -652,6 +651,7 @@ trait SearchController {
supportedLanguages = params.languageFilter.getOrElse(List.empty),
relevanceIds = params.relevance.getOrElse(List.empty),
grepCodes = params.grepCodes.getOrElse(List.empty),
traits = params.traits.getOrElse(List.empty),
shouldScroll = shouldScroll,
filterByNoResourceType = false,
aggregatePaths = params.aggregatePaths.getOrElse(List.empty),
Expand Down Expand Up @@ -690,6 +690,7 @@ trait SearchController {
statusFilter = params.draftStatus.getOrElse(List.empty).flatMap(DraftStatus.valueOf),
userFilter = params.users.getOrElse(List.empty),
grepCodes = params.grepCodes.getOrElse(List.empty),
traits = params.traits.getOrElse(List.empty),
shouldScroll = shouldScroll,
searchDecompounded = false,
aggregatePaths = params.aggregatePaths.getOrElse(List.empty),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ import io.circe.{Decoder, Encoder}
import no.ndla.common.model.NDLADate
import no.ndla.network.tapir.NonEmptyString
import no.ndla.searchapi.model.domain.Sort
import no.ndla.searchapi.model.search.SearchType
import no.ndla.searchapi.model.search.{SearchTrait, SearchType}
import sttp.tapir.Schema
import sttp.tapir.Schema.annotations.description

Expand Down Expand Up @@ -68,6 +68,8 @@ case class DraftSearchParamsDTO(
users: Option[List[String]],
@description("A list of codes from GREP API the resources should be filtered by.")
grepCodes: Option[List[String]],
@description("A comma separated list of traits the resources should be filtered by.")
traits: Option[List[SearchTrait]],
@description("List of index-paths that should be term-aggregated and returned in result.")
aggregatePaths: Option[List[String]],
@description(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import io.circe.generic.semiauto.{deriveDecoder, deriveEncoder}
import io.circe.{Decoder, Encoder}
import no.ndla.network.tapir.NonEmptyString
import no.ndla.searchapi.model.domain.Sort
import no.ndla.searchapi.model.search.SearchTrait
import sttp.tapir.Schema
import sttp.tapir.Schema.annotations.description

Expand Down Expand Up @@ -58,6 +59,8 @@ case class SearchParamsDTO(
languageFilter: Option[List[String]],
@description("A list of codes from GREP API the resources should be filtered by.")
grepCodes: Option[List[String]],
@description("A comma separated list of traits the resources should be filtered by.")
traits: Option[List[SearchTrait]],
@description("List of index-paths that should be term-aggregated and returned in result.")
aggregatePaths: Option[List[String]],
@description(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ import io.circe.generic.semiauto.{deriveDecoder, deriveEncoder}
import no.ndla.common.model.NDLADate
import no.ndla.common.model.api.draft.CommentDTO
import no.ndla.searchapi.model.domain.LearningResourceType
import no.ndla.searchapi.model.search.SearchType
import no.ndla.searchapi.model.search.{SearchTrait, SearchType}
import sttp.tapir.Schema.annotations.description

@description("Object describing matched field with matching words emphasized")
Expand All @@ -38,7 +38,7 @@ case class MultiSearchSummaryDTO(
@description("Languages the resource exists in") supportedLanguages: Seq[String],
@description("Learning resource type") learningResourceType: LearningResourceType,
@description("Status information of the resource") status: Option[StatusDTO],
@description("Traits for the resource") traits: List[String],
@description("Traits for the resource") traits: List[SearchTrait],
@description("Relevance score. The higher the score, the better the document matches your search criteria.") score: Float,
@description("List of objects describing matched field with matching words emphasized") highlights: List[HighlightedFieldDTO],
@description("The taxonomy paths for the resource") paths: List[String],
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
/*
* Part of NDLA search-api
* Copyright (C) 2018 NDLA
*
* See LICENSE
*/

package no.ndla.searchapi.model.search

import com.scalatsi.TypescriptType.{TSLiteralString, TSUnion}
import com.scalatsi.{TSNamedType, TSType}
import enumeratum.*
import no.ndla.common.CirceUtil.CirceEnumWithErrors
import sttp.tapir.Schema
import sttp.tapir.codec.enumeratum.*

sealed abstract class SearchTrait(override val entryName: String) extends EnumEntry {
override def toString: String = entryName
}
object SearchTrait extends Enum[SearchTrait] with CirceEnumWithErrors[SearchTrait] {
case object Video extends SearchTrait("VIDEO")
case object H5p extends SearchTrait("H5P")
case object Audio extends SearchTrait("AUDIO")
case object Podcast extends SearchTrait("PODCAST")

def all: List[String] = SearchTrait.values.map(_.toString).toList
override def values: IndexedSeq[SearchTrait] = findValues
def valueOf(s: String): Option[SearchTrait] = SearchTrait.values.find(_.entryName == s)
implicit def schema: Schema[SearchTrait] = schemaForEnumEntry[SearchTrait]

implicit val enumTsType: TSNamedType[SearchTrait] =
TSType.alias[SearchTrait]("SearchTrait", TSUnion(values.map(e => TSLiteralString(e.entryName))))
}
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ case class SearchableArticle(
supportedLanguages: List[String],
contexts: List[SearchableTaxonomyContext],
grepContexts: List[SearchableGrepContext],
traits: List[String],
traits: List[SearchTrait],
embedAttributes: SearchableLanguageList,
embedResourcesAndIds: List[EmbedValues],
availability: String,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ case class SearchableDraft(
users: List[String],
previousVersionsNotes: List[String],
grepContexts: List[SearchableGrepContext],
traits: List[String],
traits: List[SearchTrait],
embedAttributes: SearchableLanguageList,
embedResourcesAndIds: List[EmbedValues],
revisionMeta: List[RevisionMeta],
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ import no.ndla.common.model.domain.draft.DraftStatus
import no.ndla.language.Language
import no.ndla.network.tapir.NonEmptyString
import no.ndla.searchapi.model.domain.{LearningResourceType, Sort}
import no.ndla.searchapi.model.search.SearchType
import no.ndla.searchapi.model.search.{SearchTrait, SearchType}

case class MultiDraftSearchSettings(
query: Option[NonEmptyString],
Expand All @@ -33,6 +33,7 @@ case class MultiDraftSearchSettings(
statusFilter: List[DraftStatus],
userFilter: List[String],
grepCodes: List[String],
traits: List[SearchTrait],
shouldScroll: Boolean,
searchDecompounded: Boolean,
aggregatePaths: List[String],
Expand Down Expand Up @@ -72,6 +73,7 @@ object MultiDraftSearchSettings {
statusFilter = List.empty,
userFilter = List.empty,
grepCodes = List.empty,
traits = List.empty,
shouldScroll = false,
searchDecompounded = false,
aggregatePaths = List.empty,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import no.ndla.common.model.domain.Availability
import no.ndla.language.Language
import no.ndla.network.tapir.NonEmptyString
import no.ndla.searchapi.model.domain.{LearningResourceType, Sort}
import no.ndla.searchapi.model.search.SearchTrait

case class SearchSettings(
query: Option[NonEmptyString],
Expand All @@ -27,6 +28,7 @@ case class SearchSettings(
supportedLanguages: List[String],
relevanceIds: List[String],
grepCodes: List[String],
traits: List[SearchTrait],
shouldScroll: Boolean,
filterByNoResourceType: Boolean,
aggregatePaths: List[String],
Expand All @@ -53,6 +55,7 @@ object SearchSettings {
supportedLanguages = List.empty,
relevanceIds = List.empty,
grepCodes = List.empty,
traits = List.empty,
shouldScroll = false,
filterByNoResourceType = false,
aggregatePaths = List.empty,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -274,6 +274,10 @@ trait MultiDraftSearchService {
if (settings.grepCodes.nonEmpty) Some(termsQuery("grepContexts.code", settings.grepCodes))
else None

val traitsFilter =
if (settings.traits.nonEmpty) Some(termsQuery("traits", settings.traits.map(_.entryName)))
else None

val embedResourceAndIdFilter =
buildNestedEmbedField(settings.embedResource, settings.embedId, settings.language, settings.fallback)

Expand Down Expand Up @@ -321,6 +325,7 @@ trait MultiDraftSearchService {
statusFilter,
usersFilter,
grepCodesFilter,
traitsFilter,
embedResourceAndIdFilter,
revisionDateFilter,
publishedDateFilter,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -146,6 +146,11 @@ trait MultiSearchService {
Some(termsQuery("grepContexts.code", settings.grepCodes))
else None

val traitsFilter =
if (settings.traits.nonEmpty)
Some(termsQuery("traits", settings.traits.map(_.entryName)))
else None

val embedResourceAndIdFilter =
buildNestedEmbedField(settings.embedResource, settings.embedId, settings.language, settings.fallback)

Expand Down Expand Up @@ -181,6 +186,7 @@ trait MultiSearchService {
taxonomyRelevanceFilter,
taxonomyContextActiveFilter,
grepCodesFilter,
traitsFilter,
embedResourceAndIdFilter,
availabilityFilter
).flatten
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -67,26 +67,28 @@ trait SearchConverterService {
document.body()
}

private def getArticleTraits(contents: Seq[ArticleContent]): Seq[String] = {
private def getArticleTraits(contents: Seq[ArticleContent]): Seq[SearchTrait] = {
contents.flatMap(content => {
val traits = ListBuffer[String]()
val traits = ListBuffer[SearchTrait]()
parseHtml(content.content)
.select(EmbedTagName)
.forEach(embed => {
val dataResource = embed.attr("data-resource")
dataResource match {
case "h5p" => traits += "H5P"
case "brightcove" | "nrk" => traits += "VIDEO"
case "h5p" => traits += SearchTrait.H5p
case "brightcove" | "nrk" => traits += SearchTrait.Video
case "external" | "iframe" =>
val dataUrl = embed.attr("data-url")
if (
dataUrl.contains("youtu") || dataUrl.contains("vimeo") || dataUrl
.contains("filmiundervisning") || dataUrl.contains("imdb") || dataUrl
.contains("nrk") || dataUrl.contains("khanacademy")
) {
traits += "VIDEO"
traits += SearchTrait.Video
}
case _ => // Do nothing
case "audio" => traits += SearchTrait.Audio
case "podcast" => traits += SearchTrait.Podcast
case _ => // Do nothing
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Er det egentlig noen grunn til at vi ikke bare kan indeksere alt og bruke typene fra validation?

}
})
traits
Expand Down
2 changes: 2 additions & 0 deletions search-api/src/test/scala/no/ndla/searchapi/TestData.scala
Original file line number Diff line number Diff line change
Expand Up @@ -1599,6 +1599,7 @@ object TestData {
supportedLanguages = List.empty,
relevanceIds = List.empty,
grepCodes = List.empty,
traits = List.empty,
shouldScroll = false,
filterByNoResourceType = false,
aggregatePaths = List.empty,
Expand Down Expand Up @@ -1628,6 +1629,7 @@ object TestData {
statusFilter = List.empty,
userFilter = List.empty,
grepCodes = List.empty,
traits = List.empty,
shouldScroll = false,
searchDecompounded = false,
aggregatePaths = List.empty,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@

package no.ndla.searchapi.service.search

import com.sksamuel.elastic4s.ElasticDsl._
import com.sksamuel.elastic4s.ElasticDsl.*
import no.ndla.scalatestsuite.IntegrationSuite
import no.ndla.searchapi.TestEnvironment

Expand Down
Loading
Loading