Create search views when creating project (#5227)

Co-authored-by: Simon Dumas <[email protected]>

build.sbt

@@ -713,6 +713,7 @@ lazy val ship = project
     compositeViewsPlugin % "compile->compile",
     elasticsearchPlugin  % "compile->compile",
     storagePlugin        % "compile->compile;test->test",
+    searchPlugin,
     tests                % "test->compile;test->test"
   )
   .settings(

ship/src/main/resources/ship-default.conf

@@ -67,6 +67,16 @@ ship {
         name = "Default Sparql view"
         description = "A Sparql view of all resources in the project."
       }
+
+      search {
+        name = "Default global search view"
+        description = "An Elasticsearch view of configured resources for the global search."
+      }
+    }
+
+    search {
+      commit = "master"
+      rebuild-interval = 10 minutes
     }
 
     organizations {

ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/config/InputConfig.scala

@@ -19,6 +19,7 @@ final case class InputConfig(
     organizations: OrganizationCreationConfig,
     projectMapping: ProjectMapping = Map.empty,
     viewDefaults: ViewDefaults,
+    search: SearchConfig,
     serviceAccount: ServiceAccountConfig,
     storages: StoragesConfig,
     files: FileProcessingConfig,

ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/config/SearchConfig.scala

@@ -0,0 +1,13 @@
+package ch.epfl.bluebrain.nexus.ship.config
+
+import pureconfig.ConfigReader
+import pureconfig.generic.semiauto.deriveReader
+
+import scala.concurrent.duration.FiniteDuration
+
+final case class SearchConfig(commit: String, rebuildInterval: FiniteDuration)
+
+object SearchConfig {
+  implicit val searchConfigReader: ConfigReader[SearchConfig] =
+    deriveReader[SearchConfig]
+}

ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/config/ViewDefaults.scala

@@ -6,7 +6,8 @@ import pureconfig.generic.semiauto.deriveReader
 
 case class ViewDefaults(
     elasticsearch: Defaults,
-    blazegraph: Defaults
+    blazegraph: Defaults,
+    search: Defaults
 )
 
 object ViewDefaults {

ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/projects/ScopeInitializerWiring.scala

@@ -4,14 +4,16 @@ import cats.effect.IO
 import ch.epfl.bluebrain.nexus.delta.kernel.utils.UUIDF
 import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.api.JsonLdApi
 import ch.epfl.bluebrain.nexus.delta.sdk.ScopeInitializer
+import ch.epfl.bluebrain.nexus.delta.sdk.model.BaseUri
 import ch.epfl.bluebrain.nexus.delta.sdk.projects.{FetchContext, ScopeInitializationErrorStore}
 import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.ResolverContextResolution
 import ch.epfl.bluebrain.nexus.delta.sourcing.Transactors
 import ch.epfl.bluebrain.nexus.ship.EventClock
 import ch.epfl.bluebrain.nexus.ship.config.InputConfig
+import ch.epfl.bluebrain.nexus.ship.search.SearchWiring
 import ch.epfl.bluebrain.nexus.ship.storages.StorageWiring
 import ch.epfl.bluebrain.nexus.ship.storages.StorageWiring.s3StorageInitializer
-import ch.epfl.bluebrain.nexus.ship.views.ViewWiring.{blazegraphViews, elasticSearchViews, viewInitializers}
+import ch.epfl.bluebrain.nexus.ship.views.ViewWiring.{blazegraphViews, compositeViews, elasticSearchViews, viewInitializers}
 
 object ScopeInitializerWiring {
 
@@ -21,14 +23,21 @@ object ScopeInitializerWiring {
       config: InputConfig,
       clock: EventClock,
       xas: Transactors
-  )(implicit jsonLdApi: JsonLdApi): IO[ScopeInitializer] =
+  )(implicit jsonLdApi: JsonLdApi, baseUri: BaseUri): IO[ScopeInitializer] =
     for {
-      esViews     <- elasticSearchViews(fetchContext, rcr, config.eventLog, clock, UUIDF.random, xas)
-      bgViews     <- blazegraphViews(fetchContext, rcr, config.eventLog, clock, UUIDF.random, xas)
-      storages    <- StorageWiring.storages(fetchContext, rcr, config, clock, xas)
-      storageInit <- s3StorageInitializer(storages, config)
-      allInits     = viewInitializers(esViews, bgViews, config) + storageInit
-      errorStore   = ScopeInitializationErrorStore(xas, clock)
+      esViews        <- elasticSearchViews(fetchContext, rcr, config.eventLog, clock, UUIDF.random, xas)
+      bgViews        <- blazegraphViews(fetchContext, rcr, config.eventLog, clock, UUIDF.random, xas)
+      compositeViews <- compositeViews(fetchContext, rcr, config.eventLog, clock, UUIDF.random, xas)
+      searchInit     <- SearchWiring.searchInitializer(
+                          compositeViews,
+                          config.serviceAccount.value,
+                          config.search,
+                          config.viewDefaults.search
+                        )
+      storages       <- StorageWiring.storages(fetchContext, rcr, config, clock, xas)
+      storageInit    <- s3StorageInitializer(storages, config)
+      allInits        = viewInitializers(esViews, bgViews, config) + searchInit + storageInit
+      errorStore      = ScopeInitializationErrorStore(xas, clock)
     } yield ScopeInitializer(allInits, errorStore)
 
 }

ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/search/SearchWiring.scala

@@ -0,0 +1,82 @@
+package ch.epfl.bluebrain.nexus.ship.search
+
+import cats.effect.IO
+import cats.syntax.all._
+import ch.epfl.bluebrain.nexus.delta.plugins.compositeviews.CompositeViews
+import ch.epfl.bluebrain.nexus.delta.plugins.compositeviews.model.CompositeView.Interval
+import ch.epfl.bluebrain.nexus.delta.plugins.compositeviews.model.TemplateSparqlConstructQuery
+import ch.epfl.bluebrain.nexus.delta.plugins.search.SearchScopeInitialization
+import ch.epfl.bluebrain.nexus.delta.plugins.search.model.SearchConfig.IndexingConfig
+import ch.epfl.bluebrain.nexus.delta.plugins.search.model.SearchConfigError.{InvalidJsonError, InvalidSparqlConstructQuery, LoadingFileError}
+import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.ContextValue.ContextObject
+import ch.epfl.bluebrain.nexus.delta.rdf.query.SparqlQuery.SparqlConstructQuery
+import ch.epfl.bluebrain.nexus.delta.sdk.Defaults
+import ch.epfl.bluebrain.nexus.delta.sdk.identities.model.ServiceAccount
+import ch.epfl.bluebrain.nexus.delta.sdk.model.BaseUri
+import ch.epfl.bluebrain.nexus.delta.sourcing.model.IriFilter
+import ch.epfl.bluebrain.nexus.ship.config.SearchConfig
+import io.circe.parser.decode
+import io.circe.{Decoder, JsonObject}
+
+import java.net.URI
+import java.net.http.{HttpClient, HttpRequest, HttpResponse}
+import scala.concurrent.duration._
+import scala.util.Try
+
+object SearchWiring {
+
+  private val client = HttpClient.newHttpClient()
+
+  private def githubPrefix(commit: String) =
+    s"https://raw.githubusercontent.com/BlueBrain/nexus/$commit/tests/docker/config/search"
+
+  private def getAsString(url: String) = {
+    val request = HttpRequest.newBuilder().uri(URI.create(url)).GET().build()
+    IO.fromEither(
+      Try(client.send(request, HttpResponse.BodyHandlers.ofString())).toEither.leftMap(LoadingFileError(url, _))
+    )
+  }
+
+  private def loadExternalConfig[A: Decoder](url: String): IO[A] =
+    for {
+      content <- getAsString(url)
+      value   <- IO.fromEither(decode[A](content.body()).leftMap { e => InvalidJsonError(url, e.getMessage) })
+    } yield value
+
+  private def loadSparqlQuery(url: String): IO[SparqlConstructQuery] =
+    for {
+      content <- getAsString(url)
+      value   <- IO.fromEither(TemplateSparqlConstructQuery(content.body()).leftMap { e =>
+                   InvalidSparqlConstructQuery(url, e)
+                 })
+    } yield value
+
+  private def indexingConfig(commit: String, rebuildInterval: FiniteDuration) = {
+    val prefix = githubPrefix(commit)
+    for {
+      resourceTypes <- loadExternalConfig[IriFilter](s"$prefix/resource-types.json")
+      mapping       <- loadExternalConfig[JsonObject](s"$prefix/mapping.json")
+      settings      <- loadExternalConfig[JsonObject](s"$prefix/settings.json")
+      query         <- loadSparqlQuery(s"$prefix/construct-query.sparql")
+      context       <- loadExternalConfig[JsonObject](s"$prefix/search-context.json")
+    } yield IndexingConfig(
+      resourceTypes,
+      mapping,
+      settings = Some(settings),
+      query = query,
+      context = ContextObject(context),
+      rebuildStrategy = Some(Interval(rebuildInterval))
+    )
+  }
+
+  def searchInitializer(
+      compositeViews: CompositeViews,
+      serviceAccount: ServiceAccount,
+      config: SearchConfig,
+      defaults: Defaults
+  )(implicit baseUri: BaseUri): IO[SearchScopeInitialization] =
+    indexingConfig(config.commit, config.rebuildInterval).map { config =>
+      new SearchScopeInitialization(compositeViews, config, serviceAccount, defaults)
+    }
+
+}

ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/views/CompositeViewProcessor.scala

@@ -2,6 +2,7 @@ package ch.epfl.bluebrain.nexus.ship.views
 
 import cats.effect.IO
 import ch.epfl.bluebrain.nexus.delta.kernel.Logger
+import ch.epfl.bluebrain.nexus.delta.kernel.utils.UUIDF
 import ch.epfl.bluebrain.nexus.delta.plugins.compositeviews.CompositeViews
 import ch.epfl.bluebrain.nexus.delta.plugins.compositeviews.model.CompositeViewEvent
 import ch.epfl.bluebrain.nexus.delta.plugins.compositeviews.model.CompositeViewEvent._
@@ -85,7 +86,7 @@ object CompositeViewProcessor {
   )(implicit
       jsonLdApi: JsonLdApi
   ): CompositeViewProcessor = {
-    val views = ViewWiring.compositeViews(fetchContext, rcr, config, clock, xas)
+    val views = (uuid: UUID) => ViewWiring.compositeViews(fetchContext, rcr, config, clock, UUIDF.fixed(uuid), xas)
     new CompositeViewProcessor(views, projectMapper, clock)
   }
 

ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/views/ViewWiring.scala

@@ -82,21 +82,21 @@ object ViewWiring {
       rcr: ResolverContextResolution,
       config: EventLogConfig,
       clock: EventClock,
+      uuidF: UUIDF,
       xas: Transactors
   )(implicit jsonLdApi: JsonLdApi) = {
     val noValidation = new ValidateCompositeView {
       override def apply(uuid: UUID, value: CompositeViewValue): IO[Unit] = IO.unit
     }
-    (uuid: UUID) =>
-      CompositeViews(
-        fetchContext,
-        rcr,
-        noValidation,
-        3.seconds, // TODO: use the config?
-        config,
-        xas,
-        clock
-      )(jsonLdApi, UUIDF.fixed(uuid))
+    CompositeViews(
+      fetchContext,
+      rcr,
+      noValidation,
+      3.seconds, // TODO: use the config?
+      config,
+      xas,
+      clock
+    )(jsonLdApi, uuidF)
   }
 
   def viewInitializers(

ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/RunShipSuite.scala

@@ -93,6 +93,7 @@ class RunShipSuite
       _      <- RunShip(events, s3Client, inputConfig, xas).assertEquals(expectedImportReport)
       _      <- checkFor("elasticsearch", nxv + "defaultElasticSearchIndex", xas).assertEquals(1)
       _      <- checkFor("blazegraph", nxv + "defaultSparqlIndex", xas).assertEquals(1)
+      _      <- checkFor("compositeviews", nxv + "searchView", xas).assertEquals(1)
       _      <- checkFor("storage", nxv + "defaultS3Storage", xas).assertEquals(1)
     } yield ()
   }

ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/ShipIntegrationSpec.scala

@@ -8,6 +8,7 @@ import ch.epfl.bluebrain.nexus.delta.plugins.blazegraph.model.BlazegraphViewType
 import ch.epfl.bluebrain.nexus.delta.plugins.blazegraph.model.{defaultViewId => bgDefaultViewId}
 import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.model.ElasticSearchViewType.AggregateElasticSearch
 import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.model.{defaultViewId => esDefaultViewId}
+import ch.epfl.bluebrain.nexus.delta.plugins.search.model.{defaultViewId => searchViewId}
 import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.Iri
 import ch.epfl.bluebrain.nexus.delta.rdf.Vocabulary.nxv
 import ch.epfl.bluebrain.nexus.delta.sourcing.exporter.ExportEventQuery
@@ -53,6 +54,10 @@ class ShipIntegrationSpec extends BaseIntegrationSpec {
       weFixThePermissions(project)
 
       thereShouldBeAProject(project, projectJson)
+
+      thereShouldBeAViewWithId(project, bgDefaultViewId)
+      thereShouldBeAViewWithId(project, esDefaultViewId)
+      thereShouldBeAViewWithId(project, searchViewId)
     }
 
     "transfer multiple revisions of a project" in {
@@ -185,6 +190,15 @@ class ShipIntegrationSpec extends BaseIntegrationSpec {
       thereShouldBeAView(project, bgView, patchedSource)
     }
 
+    def thereShouldBeAViewWithId(project: ProjectRef, view: Iri): Assertion = {
+      val encodedIri = UrlUtils.encode(view.toString)
+      deltaClient
+        .get[Json](s"/views/${project.organization}/${project.project}/$encodedIri", writer) { (_, response) =>
+          response.status shouldEqual StatusCodes.OK
+        }
+        .accepted
+    }
+
     def thereShouldBeAView(project: ProjectRef, view: Iri, expectedJson: Json): Assertion = {
       val encodedIri = UrlUtils.encode(view.toString)
       deltaClient

ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/config/ShipConfigFixtures.scala

@@ -14,6 +14,7 @@ import ch.epfl.bluebrain.nexus.delta.sdk.{ConfigFixtures, Defaults}
 import ch.epfl.bluebrain.nexus.delta.sourcing.model.Identity.User
 import ch.epfl.bluebrain.nexus.delta.sourcing.model.Label
 import ch.epfl.bluebrain.nexus.testkit.scalatest.ClasspathResources
+import concurrent.duration._
 
 trait ShipConfigFixtures extends ConfigFixtures with StorageFixtures with ClasspathResources {
 
@@ -25,7 +26,8 @@ trait ShipConfigFixtures extends ConfigFixtures with StorageFixtures with Classp
 
   private val viewDefaults = ViewDefaults(
     Defaults("Default ES View", "Description ES View"),
-    Defaults("Default EBG View", "Description BG View")
+    Defaults("Default EBG View", "Description BG View"),
+    Defaults("Default Search View", "Description Search View")
   )
 
   private val serviceAccount: ServiceAccountConfig = ServiceAccountConfig(
@@ -57,6 +59,7 @@ trait ShipConfigFixtures extends ConfigFixtures with StorageFixtures with Classp
       organizationsCreation,
       Map.empty,
       viewDefaults,
+      SearchConfig("master", 10.minutes),
       serviceAccount,
       StoragesConfig(eventLogConfig, pagination, config.copy(amazon = Some(amazonConfig))),
       FileProcessingConfig(