diff --git a/build.sbt b/build.sbt index a7c15d7872..f055d63232 100755 --- a/build.sbt +++ b/build.sbt @@ -713,6 +713,7 @@ lazy val ship = project compositeViewsPlugin % "compile->compile", elasticsearchPlugin % "compile->compile", storagePlugin % "compile->compile;test->test", + searchPlugin, tests % "test->compile;test->test" ) .settings( diff --git a/ship/src/main/resources/ship-default.conf b/ship/src/main/resources/ship-default.conf index b3f26954a1..93c734017a 100644 --- a/ship/src/main/resources/ship-default.conf +++ b/ship/src/main/resources/ship-default.conf @@ -67,6 +67,16 @@ ship { name = "Default Sparql view" description = "A Sparql view of all resources in the project." } + + search { + name = "Default global search view" + description = "An Elasticsearch view of configured resources for the global search." + } + } + + search { + commit = "master" + rebuild-interval = 10 minutes } organizations { diff --git a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/config/InputConfig.scala b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/config/InputConfig.scala index 36d9a5d4d3..d860086f31 100644 --- a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/config/InputConfig.scala +++ b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/config/InputConfig.scala @@ -19,6 +19,7 @@ final case class InputConfig( organizations: OrganizationCreationConfig, projectMapping: ProjectMapping = Map.empty, viewDefaults: ViewDefaults, + search: SearchConfig, serviceAccount: ServiceAccountConfig, storages: StoragesConfig, files: FileProcessingConfig, diff --git a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/config/SearchConfig.scala b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/config/SearchConfig.scala new file mode 100644 index 0000000000..110ce5f3cc --- /dev/null +++ b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/config/SearchConfig.scala @@ -0,0 +1,13 @@ +package ch.epfl.bluebrain.nexus.ship.config + +import pureconfig.ConfigReader +import pureconfig.generic.semiauto.deriveReader + +import scala.concurrent.duration.FiniteDuration + +final case class SearchConfig(commit: String, rebuildInterval: FiniteDuration) + +object SearchConfig { + implicit val searchConfigReader: ConfigReader[SearchConfig] = + deriveReader[SearchConfig] +} diff --git a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/config/ViewDefaults.scala b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/config/ViewDefaults.scala index 6c638c57a3..c2f7b4ae45 100644 --- a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/config/ViewDefaults.scala +++ b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/config/ViewDefaults.scala @@ -6,7 +6,8 @@ import pureconfig.generic.semiauto.deriveReader case class ViewDefaults( elasticsearch: Defaults, - blazegraph: Defaults + blazegraph: Defaults, + search: Defaults ) object ViewDefaults { diff --git a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/projects/ScopeInitializerWiring.scala b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/projects/ScopeInitializerWiring.scala index 1e1baefcd3..70871f0f33 100644 --- a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/projects/ScopeInitializerWiring.scala +++ b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/projects/ScopeInitializerWiring.scala @@ -4,14 +4,16 @@ import cats.effect.IO import ch.epfl.bluebrain.nexus.delta.kernel.utils.UUIDF import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.api.JsonLdApi import ch.epfl.bluebrain.nexus.delta.sdk.ScopeInitializer +import ch.epfl.bluebrain.nexus.delta.sdk.model.BaseUri import ch.epfl.bluebrain.nexus.delta.sdk.projects.{FetchContext, ScopeInitializationErrorStore} import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.ResolverContextResolution import ch.epfl.bluebrain.nexus.delta.sourcing.Transactors import ch.epfl.bluebrain.nexus.ship.EventClock import ch.epfl.bluebrain.nexus.ship.config.InputConfig +import ch.epfl.bluebrain.nexus.ship.search.SearchWiring import ch.epfl.bluebrain.nexus.ship.storages.StorageWiring import ch.epfl.bluebrain.nexus.ship.storages.StorageWiring.s3StorageInitializer -import ch.epfl.bluebrain.nexus.ship.views.ViewWiring.{blazegraphViews, elasticSearchViews, viewInitializers} +import ch.epfl.bluebrain.nexus.ship.views.ViewWiring.{blazegraphViews, compositeViews, elasticSearchViews, viewInitializers} object ScopeInitializerWiring { @@ -21,14 +23,21 @@ object ScopeInitializerWiring { config: InputConfig, clock: EventClock, xas: Transactors - )(implicit jsonLdApi: JsonLdApi): IO[ScopeInitializer] = + )(implicit jsonLdApi: JsonLdApi, baseUri: BaseUri): IO[ScopeInitializer] = for { - esViews <- elasticSearchViews(fetchContext, rcr, config.eventLog, clock, UUIDF.random, xas) - bgViews <- blazegraphViews(fetchContext, rcr, config.eventLog, clock, UUIDF.random, xas) - storages <- StorageWiring.storages(fetchContext, rcr, config, clock, xas) - storageInit <- s3StorageInitializer(storages, config) - allInits = viewInitializers(esViews, bgViews, config) + storageInit - errorStore = ScopeInitializationErrorStore(xas, clock) + esViews <- elasticSearchViews(fetchContext, rcr, config.eventLog, clock, UUIDF.random, xas) + bgViews <- blazegraphViews(fetchContext, rcr, config.eventLog, clock, UUIDF.random, xas) + compositeViews <- compositeViews(fetchContext, rcr, config.eventLog, clock, UUIDF.random, xas) + searchInit <- SearchWiring.searchInitializer( + compositeViews, + config.serviceAccount.value, + config.search, + config.viewDefaults.search + ) + storages <- StorageWiring.storages(fetchContext, rcr, config, clock, xas) + storageInit <- s3StorageInitializer(storages, config) + allInits = viewInitializers(esViews, bgViews, config) + searchInit + storageInit + errorStore = ScopeInitializationErrorStore(xas, clock) } yield ScopeInitializer(allInits, errorStore) } diff --git a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/search/SearchWiring.scala b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/search/SearchWiring.scala new file mode 100644 index 0000000000..d09832f385 --- /dev/null +++ b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/search/SearchWiring.scala @@ -0,0 +1,82 @@ +package ch.epfl.bluebrain.nexus.ship.search + +import cats.effect.IO +import cats.syntax.all._ +import ch.epfl.bluebrain.nexus.delta.plugins.compositeviews.CompositeViews +import ch.epfl.bluebrain.nexus.delta.plugins.compositeviews.model.CompositeView.Interval +import ch.epfl.bluebrain.nexus.delta.plugins.compositeviews.model.TemplateSparqlConstructQuery +import ch.epfl.bluebrain.nexus.delta.plugins.search.SearchScopeInitialization +import ch.epfl.bluebrain.nexus.delta.plugins.search.model.SearchConfig.IndexingConfig +import ch.epfl.bluebrain.nexus.delta.plugins.search.model.SearchConfigError.{InvalidJsonError, InvalidSparqlConstructQuery, LoadingFileError} +import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.ContextValue.ContextObject +import ch.epfl.bluebrain.nexus.delta.rdf.query.SparqlQuery.SparqlConstructQuery +import ch.epfl.bluebrain.nexus.delta.sdk.Defaults +import ch.epfl.bluebrain.nexus.delta.sdk.identities.model.ServiceAccount +import ch.epfl.bluebrain.nexus.delta.sdk.model.BaseUri +import ch.epfl.bluebrain.nexus.delta.sourcing.model.IriFilter +import ch.epfl.bluebrain.nexus.ship.config.SearchConfig +import io.circe.parser.decode +import io.circe.{Decoder, JsonObject} + +import java.net.URI +import java.net.http.{HttpClient, HttpRequest, HttpResponse} +import scala.concurrent.duration._ +import scala.util.Try + +object SearchWiring { + + private val client = HttpClient.newHttpClient() + + private def githubPrefix(commit: String) = + s"https://raw.githubusercontent.com/BlueBrain/nexus/$commit/tests/docker/config/search" + + private def getAsString(url: String) = { + val request = HttpRequest.newBuilder().uri(URI.create(url)).GET().build() + IO.fromEither( + Try(client.send(request, HttpResponse.BodyHandlers.ofString())).toEither.leftMap(LoadingFileError(url, _)) + ) + } + + private def loadExternalConfig[A: Decoder](url: String): IO[A] = + for { + content <- getAsString(url) + value <- IO.fromEither(decode[A](content.body()).leftMap { e => InvalidJsonError(url, e.getMessage) }) + } yield value + + private def loadSparqlQuery(url: String): IO[SparqlConstructQuery] = + for { + content <- getAsString(url) + value <- IO.fromEither(TemplateSparqlConstructQuery(content.body()).leftMap { e => + InvalidSparqlConstructQuery(url, e) + }) + } yield value + + private def indexingConfig(commit: String, rebuildInterval: FiniteDuration) = { + val prefix = githubPrefix(commit) + for { + resourceTypes <- loadExternalConfig[IriFilter](s"$prefix/resource-types.json") + mapping <- loadExternalConfig[JsonObject](s"$prefix/mapping.json") + settings <- loadExternalConfig[JsonObject](s"$prefix/settings.json") + query <- loadSparqlQuery(s"$prefix/construct-query.sparql") + context <- loadExternalConfig[JsonObject](s"$prefix/search-context.json") + } yield IndexingConfig( + resourceTypes, + mapping, + settings = Some(settings), + query = query, + context = ContextObject(context), + rebuildStrategy = Some(Interval(rebuildInterval)) + ) + } + + def searchInitializer( + compositeViews: CompositeViews, + serviceAccount: ServiceAccount, + config: SearchConfig, + defaults: Defaults + )(implicit baseUri: BaseUri): IO[SearchScopeInitialization] = + indexingConfig(config.commit, config.rebuildInterval).map { config => + new SearchScopeInitialization(compositeViews, config, serviceAccount, defaults) + } + +} diff --git a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/views/CompositeViewProcessor.scala b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/views/CompositeViewProcessor.scala index ee4e1172c5..1b2aca75ce 100644 --- a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/views/CompositeViewProcessor.scala +++ b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/views/CompositeViewProcessor.scala @@ -2,6 +2,7 @@ package ch.epfl.bluebrain.nexus.ship.views import cats.effect.IO import ch.epfl.bluebrain.nexus.delta.kernel.Logger +import ch.epfl.bluebrain.nexus.delta.kernel.utils.UUIDF import ch.epfl.bluebrain.nexus.delta.plugins.compositeviews.CompositeViews import ch.epfl.bluebrain.nexus.delta.plugins.compositeviews.model.CompositeViewEvent import ch.epfl.bluebrain.nexus.delta.plugins.compositeviews.model.CompositeViewEvent._ @@ -85,7 +86,7 @@ object CompositeViewProcessor { )(implicit jsonLdApi: JsonLdApi ): CompositeViewProcessor = { - val views = ViewWiring.compositeViews(fetchContext, rcr, config, clock, xas) + val views = (uuid: UUID) => ViewWiring.compositeViews(fetchContext, rcr, config, clock, UUIDF.fixed(uuid), xas) new CompositeViewProcessor(views, projectMapper, clock) } diff --git a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/views/ViewWiring.scala b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/views/ViewWiring.scala index 3be2bd1b4d..cf180e613c 100644 --- a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/views/ViewWiring.scala +++ b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/views/ViewWiring.scala @@ -82,21 +82,21 @@ object ViewWiring { rcr: ResolverContextResolution, config: EventLogConfig, clock: EventClock, + uuidF: UUIDF, xas: Transactors )(implicit jsonLdApi: JsonLdApi) = { val noValidation = new ValidateCompositeView { override def apply(uuid: UUID, value: CompositeViewValue): IO[Unit] = IO.unit } - (uuid: UUID) => - CompositeViews( - fetchContext, - rcr, - noValidation, - 3.seconds, // TODO: use the config? - config, - xas, - clock - )(jsonLdApi, UUIDF.fixed(uuid)) + CompositeViews( + fetchContext, + rcr, + noValidation, + 3.seconds, // TODO: use the config? + config, + xas, + clock + )(jsonLdApi, uuidF) } def viewInitializers( diff --git a/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/RunShipSuite.scala b/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/RunShipSuite.scala index 44fcc436af..4f84257767 100644 --- a/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/RunShipSuite.scala +++ b/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/RunShipSuite.scala @@ -93,6 +93,7 @@ class RunShipSuite _ <- RunShip(events, s3Client, inputConfig, xas).assertEquals(expectedImportReport) _ <- checkFor("elasticsearch", nxv + "defaultElasticSearchIndex", xas).assertEquals(1) _ <- checkFor("blazegraph", nxv + "defaultSparqlIndex", xas).assertEquals(1) + _ <- checkFor("compositeviews", nxv + "searchView", xas).assertEquals(1) _ <- checkFor("storage", nxv + "defaultS3Storage", xas).assertEquals(1) } yield () } diff --git a/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/ShipIntegrationSpec.scala b/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/ShipIntegrationSpec.scala index 6d41615fde..dab74af7a4 100644 --- a/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/ShipIntegrationSpec.scala +++ b/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/ShipIntegrationSpec.scala @@ -8,6 +8,7 @@ import ch.epfl.bluebrain.nexus.delta.plugins.blazegraph.model.BlazegraphViewType import ch.epfl.bluebrain.nexus.delta.plugins.blazegraph.model.{defaultViewId => bgDefaultViewId} import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.model.ElasticSearchViewType.AggregateElasticSearch import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.model.{defaultViewId => esDefaultViewId} +import ch.epfl.bluebrain.nexus.delta.plugins.search.model.{defaultViewId => searchViewId} import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.Iri import ch.epfl.bluebrain.nexus.delta.rdf.Vocabulary.nxv import ch.epfl.bluebrain.nexus.delta.sourcing.exporter.ExportEventQuery @@ -53,6 +54,10 @@ class ShipIntegrationSpec extends BaseIntegrationSpec { weFixThePermissions(project) thereShouldBeAProject(project, projectJson) + + thereShouldBeAViewWithId(project, bgDefaultViewId) + thereShouldBeAViewWithId(project, esDefaultViewId) + thereShouldBeAViewWithId(project, searchViewId) } "transfer multiple revisions of a project" in { @@ -185,6 +190,15 @@ class ShipIntegrationSpec extends BaseIntegrationSpec { thereShouldBeAView(project, bgView, patchedSource) } + def thereShouldBeAViewWithId(project: ProjectRef, view: Iri): Assertion = { + val encodedIri = UrlUtils.encode(view.toString) + deltaClient + .get[Json](s"/views/${project.organization}/${project.project}/$encodedIri", writer) { (_, response) => + response.status shouldEqual StatusCodes.OK + } + .accepted + } + def thereShouldBeAView(project: ProjectRef, view: Iri, expectedJson: Json): Assertion = { val encodedIri = UrlUtils.encode(view.toString) deltaClient diff --git a/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/config/ShipConfigFixtures.scala b/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/config/ShipConfigFixtures.scala index e07ec064fb..bc37842944 100644 --- a/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/config/ShipConfigFixtures.scala +++ b/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/config/ShipConfigFixtures.scala @@ -14,6 +14,7 @@ import ch.epfl.bluebrain.nexus.delta.sdk.{ConfigFixtures, Defaults} import ch.epfl.bluebrain.nexus.delta.sourcing.model.Identity.User import ch.epfl.bluebrain.nexus.delta.sourcing.model.Label import ch.epfl.bluebrain.nexus.testkit.scalatest.ClasspathResources +import concurrent.duration._ trait ShipConfigFixtures extends ConfigFixtures with StorageFixtures with ClasspathResources { @@ -25,7 +26,8 @@ trait ShipConfigFixtures extends ConfigFixtures with StorageFixtures with Classp private val viewDefaults = ViewDefaults( Defaults("Default ES View", "Description ES View"), - Defaults("Default EBG View", "Description BG View") + Defaults("Default EBG View", "Description BG View"), + Defaults("Default Search View", "Description Search View") ) private val serviceAccount: ServiceAccountConfig = ServiceAccountConfig( @@ -57,6 +59,7 @@ trait ShipConfigFixtures extends ConfigFixtures with StorageFixtures with Classp organizationsCreation, Map.empty, viewDefaults, + SearchConfig("master", 10.minutes), serviceAccount, StoragesConfig(eventLogConfig, pagination, config.copy(amazon = Some(amazonConfig))), FileProcessingConfig(