diff --git a/modules/core/src/main/scala/com.snowplowanalytics.iglu/client/resolver/Resolver.scala b/modules/core/src/main/scala/com.snowplowanalytics.iglu/client/resolver/Resolver.scala index 771b71f0..f80bd9d4 100644 --- a/modules/core/src/main/scala/com.snowplowanalytics.iglu/client/resolver/Resolver.scala +++ b/modules/core/src/main/scala/com.snowplowanalytics.iglu/client/resolver/Resolver.scala @@ -38,6 +38,12 @@ final case class Resolver[F[_]](repos: List[Registry], cache: Option[ResolverCac private[client] val allRepos: NonEmptyList[Registry] = NonEmptyList[Registry](Registry.EmbeddedRegistry, repos) + private val allIgluCentral: Set[String] = repos.collect { + case Registry.Http(config, connection) + if connection.uri.getHost.matches(""".*\biglucentral\b.*""") => + config.name + }.toSet + /** * Tries to find the given schema in any of the provided repository refs * If any of repositories gives non-non-found error, lookup will retried @@ -181,6 +187,71 @@ final case class Resolver[F[_]](repos: List[Registry], cache: Option[ResolverCac ): F[Either[ResolutionError, Json]] = lookupSchemaResult(schemaKey).map(_.map(_.value.schema)) + /** + * If Iglu Central or any of its mirrors doesn't have a schema, + * it should be considered NotFound, even if one of them returned an error. + */ + private[resolver] def isNotFound(error: ResolutionError): Boolean = { + val (igluCentral, custom) = error.value.partition { case (repo, _) => + allIgluCentral.contains(repo) + } + (igluCentral.isEmpty || igluCentral.values.exists( + _.errors.exists(_ == RegistryError.NotFound) + )) && custom.values.flatMap(_.errors).forall(_ == RegistryError.NotFound) + } + + /** + * Looks up all the schemas with the same model until `maxSchemaKey`. + * For the schemas of previous revisions, it starts with addition = 0 + * and increments it until a NotFound. + * + * @param maxSchemaKey The SchemaKey until which schemas of the same model should get returned + * @return All the schemas if all went well, [[Resolver.SchemaResolutionError]] with the first error that happened + * while looking up the schemas if something went wrong. + */ + def lookupSchemasUntil( + maxSchemaKey: SchemaKey + )(implicit + F: Monad[F], + L: RegistryLookup[F], + C: Clock[F] + ): F[Either[SchemaResolutionError, NonEmptyList[SelfDescribingSchema[Json]]]] = { + def go( + current: SchemaVer.Full, + acc: List[SelfDescribingSchema[Json]] + ): F[Either[SchemaResolutionError, NonEmptyList[SelfDescribingSchema[Json]]]] = { + val currentSchemaKey = maxSchemaKey.copy(version = current) + lookupSchema(currentSchemaKey).flatMap { + case Left(e) => + if (current.addition === 0) + Monad[F].pure(Left(SchemaResolutionError(currentSchemaKey, e))) + else if (current.revision < maxSchemaKey.version.revision && isNotFound(e)) + go(current.copy(revision = current.revision + 1, addition = 0), acc) + else + Monad[F].pure(Left(SchemaResolutionError(currentSchemaKey, e))) + case Right(json) => + if (current.revision < maxSchemaKey.version.revision) + go( + current.copy(addition = current.addition + 1), + SelfDescribingSchema(SchemaMap(currentSchemaKey), json) :: acc + ) + else if (current.addition < maxSchemaKey.version.addition) + go( + current.copy(addition = current.addition + 1), + SelfDescribingSchema(SchemaMap(currentSchemaKey), json) :: acc + ) + else + Monad[F].pure( + Right( + NonEmptyList(SelfDescribingSchema(SchemaMap(currentSchemaKey), json), acc).reverse + ) + ) + } + } + + go(SchemaVer.Full(maxSchemaKey.version.model, 0, 0), Nil) + } + /** * Get list of available schemas for particular vendor and name part * Server supposed to return them in proper order @@ -389,6 +460,8 @@ object Resolver { */ case class SchemaItem(schema: Json, supersededBy: SupersededBy) + case class SchemaResolutionError(schemaKey: SchemaKey, error: ResolutionError) + /** The result of doing a lookup with the resolver, carrying information on whether the cache was used */ sealed trait ResolverResult[+K, +A] { def value: A diff --git a/modules/core/src/test/resources/iglu-client-embedded/schemas/com.snowplowanalytics.iglu-test/lookup-schemas-until/jsonschema/3-1-0 b/modules/core/src/test/resources/iglu-client-embedded/schemas/com.snowplowanalytics.iglu-test/lookup-schemas-until/jsonschema/3-1-0 new file mode 100644 index 00000000..39afe4e5 --- /dev/null +++ b/modules/core/src/test/resources/iglu-client-embedded/schemas/com.snowplowanalytics.iglu-test/lookup-schemas-until/jsonschema/3-1-0 @@ -0,0 +1,23 @@ +{ + "$schema": "http://iglucentral.com/schemas/com.snowplowanalytics.self-desc/schema/jsonschema/1-0-0#", + "description": "Schema to test lookupSchemasUntil function", + "self": { + "vendor": "com.snowplowanalytics", + "name": "lookup-schemas-until", + "format": "jsonschema", + "version": "3-1-0" + }, + "type": "object", + "properties": { + "field___1": { + "type": "string" + }, + "field___2": { + "type": ["string", "null"] + } + }, + "required": [ + "field___1" + ], + "additionalProperties": false +} \ No newline at end of file diff --git a/modules/core/src/test/resources/iglu-test-embedded/schemas/com.snowplowanalytics.iglu-test/lookup-schemas-until/jsonschema/1-0-0 b/modules/core/src/test/resources/iglu-test-embedded/schemas/com.snowplowanalytics.iglu-test/lookup-schemas-until/jsonschema/1-0-0 new file mode 100644 index 00000000..87182efd --- /dev/null +++ b/modules/core/src/test/resources/iglu-test-embedded/schemas/com.snowplowanalytics.iglu-test/lookup-schemas-until/jsonschema/1-0-0 @@ -0,0 +1,20 @@ +{ + "$schema": "http://iglucentral.com/schemas/com.snowplowanalytics.self-desc/schema/jsonschema/1-0-0#", + "description": "Schema to test lookupSchemasUntil function", + "self": { + "vendor": "com.snowplowanalytics", + "name": "lookup-schemas-until", + "format": "jsonschema", + "version": "1-0-0" + }, + "type": "object", + "properties": { + "field_1": { + "type": "string" + } + }, + "required": [ + "field_1" + ], + "additionalProperties": false +} \ No newline at end of file diff --git a/modules/core/src/test/resources/iglu-test-embedded/schemas/com.snowplowanalytics.iglu-test/lookup-schemas-until/jsonschema/1-1-0 b/modules/core/src/test/resources/iglu-test-embedded/schemas/com.snowplowanalytics.iglu-test/lookup-schemas-until/jsonschema/1-1-0 new file mode 100644 index 00000000..83cb728e --- /dev/null +++ b/modules/core/src/test/resources/iglu-test-embedded/schemas/com.snowplowanalytics.iglu-test/lookup-schemas-until/jsonschema/1-1-0 @@ -0,0 +1,23 @@ +{ + "$schema": "http://iglucentral.com/schemas/com.snowplowanalytics.self-desc/schema/jsonschema/1-0-0#", + "description": "Schema to test lookupSchemasUntil function", + "self": { + "vendor": "com.snowplowanalytics", + "name": "lookup-schemas-until", + "format": "jsonschema", + "version": "1-1-0" + }, + "type": "object", + "properties": { + "field_1": { + "type": "string" + }, + "field_2": { + "type": ["string", "null"] + } + }, + "required": [ + "field_1" + ], + "additionalProperties": false +} \ No newline at end of file diff --git a/modules/core/src/test/resources/iglu-test-embedded/schemas/com.snowplowanalytics.iglu-test/lookup-schemas-until/jsonschema/1-1-1 b/modules/core/src/test/resources/iglu-test-embedded/schemas/com.snowplowanalytics.iglu-test/lookup-schemas-until/jsonschema/1-1-1 new file mode 100644 index 00000000..463be1cf --- /dev/null +++ b/modules/core/src/test/resources/iglu-test-embedded/schemas/com.snowplowanalytics.iglu-test/lookup-schemas-until/jsonschema/1-1-1 @@ -0,0 +1,26 @@ +{ + "$schema": "http://iglucentral.com/schemas/com.snowplowanalytics.self-desc/schema/jsonschema/1-0-0#", + "description": "Schema to test lookupSchemasUntil function", + "self": { + "vendor": "com.snowplowanalytics", + "name": "lookup-schemas-until", + "format": "jsonschema", + "version": "1-1-1" + }, + "type": "object", + "properties": { + "field_1": { + "type": "string" + }, + "field_2": { + "type": ["string", "null"] + }, + "field_3": { + "type": ["string", "null"] + } + }, + "required": [ + "field_1" + ], + "additionalProperties": false +} \ No newline at end of file diff --git a/modules/core/src/test/resources/iglu-test-embedded/schemas/com.snowplowanalytics.iglu-test/lookup-schemas-until/jsonschema/1-1-2 b/modules/core/src/test/resources/iglu-test-embedded/schemas/com.snowplowanalytics.iglu-test/lookup-schemas-until/jsonschema/1-1-2 new file mode 100644 index 00000000..4199f23c --- /dev/null +++ b/modules/core/src/test/resources/iglu-test-embedded/schemas/com.snowplowanalytics.iglu-test/lookup-schemas-until/jsonschema/1-1-2 @@ -0,0 +1,29 @@ +{ + "$schema": "http://iglucentral.com/schemas/com.snowplowanalytics.self-desc/schema/jsonschema/1-0-0#", + "description": "Schema to test lookupSchemasUntil function", + "self": { + "vendor": "com.snowplowanalytics", + "name": "lookup-schemas-until", + "format": "jsonschema", + "version": "1-1-2" + }, + "type": "object", + "properties": { + "field_1": { + "type": "string" + }, + "field_2": { + "type": ["string", "null"] + }, + "field_3": { + "type": ["string", "null"] + }, + "field_4": { + "type": ["string", "null"] + } + }, + "required": [ + "field_1" + ], + "additionalProperties": false +} \ No newline at end of file diff --git a/modules/core/src/test/resources/iglu-test-embedded/schemas/com.snowplowanalytics.iglu-test/lookup-schemas-until/jsonschema/1-2-0 b/modules/core/src/test/resources/iglu-test-embedded/schemas/com.snowplowanalytics.iglu-test/lookup-schemas-until/jsonschema/1-2-0 new file mode 100644 index 00000000..12309351 --- /dev/null +++ b/modules/core/src/test/resources/iglu-test-embedded/schemas/com.snowplowanalytics.iglu-test/lookup-schemas-until/jsonschema/1-2-0 @@ -0,0 +1,32 @@ +{ + "$schema": "http://iglucentral.com/schemas/com.snowplowanalytics.self-desc/schema/jsonschema/1-0-0#", + "description": "Schema to test lookupSchemasUntil function", + "self": { + "vendor": "com.snowplowanalytics", + "name": "lookup-schemas-until", + "format": "jsonschema", + "version": "1-2-0" + }, + "type": "object", + "properties": { + "field_1": { + "type": "string" + }, + "field_2": { + "type": ["string", "null"] + }, + "field_3": { + "type": ["string", "null"] + }, + "field_4": { + "type": ["string", "null"] + }, + "field_5": { + "type": ["string", "null"] + } + }, + "required": [ + "field_1" + ], + "additionalProperties": false +} \ No newline at end of file diff --git a/modules/core/src/test/resources/iglu-test-embedded/schemas/com.snowplowanalytics.iglu-test/lookup-schemas-until/jsonschema/1-2-1 b/modules/core/src/test/resources/iglu-test-embedded/schemas/com.snowplowanalytics.iglu-test/lookup-schemas-until/jsonschema/1-2-1 new file mode 100644 index 00000000..08883fb3 --- /dev/null +++ b/modules/core/src/test/resources/iglu-test-embedded/schemas/com.snowplowanalytics.iglu-test/lookup-schemas-until/jsonschema/1-2-1 @@ -0,0 +1,35 @@ +{ + "$schema": "http://iglucentral.com/schemas/com.snowplowanalytics.self-desc/schema/jsonschema/1-0-0#", + "description": "Schema to test lookupSchemasUntil function", + "self": { + "vendor": "com.snowplowanalytics", + "name": "lookup-schemas-until", + "format": "jsonschema", + "version": "1-2-1" + }, + "type": "object", + "properties": { + "field_1": { + "type": "string" + }, + "field_2": { + "type": ["string", "null"] + }, + "field_3": { + "type": ["string", "null"] + }, + "field_4": { + "type": ["string", "null"] + }, + "field_5": { + "type": ["string", "null"] + }, + "field_6": { + "type": ["string", "null"] + } + }, + "required": [ + "field_1" + ], + "additionalProperties": false +} \ No newline at end of file diff --git a/modules/core/src/test/resources/iglu-test-embedded/schemas/com.snowplowanalytics.iglu-test/lookup-schemas-until/jsonschema/1-2-2 b/modules/core/src/test/resources/iglu-test-embedded/schemas/com.snowplowanalytics.iglu-test/lookup-schemas-until/jsonschema/1-2-2 new file mode 100644 index 00000000..24daa689 --- /dev/null +++ b/modules/core/src/test/resources/iglu-test-embedded/schemas/com.snowplowanalytics.iglu-test/lookup-schemas-until/jsonschema/1-2-2 @@ -0,0 +1,38 @@ +{ + "$schema": "http://iglucentral.com/schemas/com.snowplowanalytics.self-desc/schema/jsonschema/1-0-0#", + "description": "Schema to test lookupSchemasUntil function", + "self": { + "vendor": "com.snowplowanalytics", + "name": "lookup-schemas-until", + "format": "jsonschema", + "version": "1-2-2" + }, + "type": "object", + "properties": { + "field_1": { + "type": "string" + }, + "field_2": { + "type": ["string", "null"] + }, + "field_3": { + "type": ["string", "null"] + }, + "field_4": { + "type": ["string", "null"] + }, + "field_5": { + "type": ["string", "null"] + }, + "field_6": { + "type": ["string", "null"] + }, + "field_7": { + "type": ["string", "null"] + } + }, + "required": [ + "field_1" + ], + "additionalProperties": false +} \ No newline at end of file diff --git a/modules/core/src/test/resources/iglu-test-embedded/schemas/com.snowplowanalytics.iglu-test/lookup-schemas-until/jsonschema/1-3-0 b/modules/core/src/test/resources/iglu-test-embedded/schemas/com.snowplowanalytics.iglu-test/lookup-schemas-until/jsonschema/1-3-0 new file mode 100644 index 00000000..88b7c0af --- /dev/null +++ b/modules/core/src/test/resources/iglu-test-embedded/schemas/com.snowplowanalytics.iglu-test/lookup-schemas-until/jsonschema/1-3-0 @@ -0,0 +1,41 @@ +{ + "$schema": "http://iglucentral.com/schemas/com.snowplowanalytics.self-desc/schema/jsonschema/1-0-0#", + "description": "Invalid schema to test lookupSchemasUntil function - Misses comma after properties", + "self": { + "vendor": "com.snowplowanalytics", + "name": "lookup-schemas-until", + "format": "jsonschema", + "version": "1-3-0" + }, + "type": "object", + "properties": { + "field_1": { + "type": "string" + }, + "field_2": { + "type": ["string", "null"] + }, + "field_3": { + "type": ["string", "null"] + }, + "field_4": { + "type": ["string", "null"] + }, + "field_5": { + "type": ["string", "null"] + }, + "field_6": { + "type": ["string", "null"] + }, + "field_7": { + "type": ["string", "null"] + }, + "field_8": { + "type": ["string", "null"] + } + } + "required": [ + "field_1" + ], + "additionalProperties": false +} \ No newline at end of file diff --git a/modules/core/src/test/resources/iglu-test-embedded/schemas/com.snowplowanalytics.iglu-test/lookup-schemas-until/jsonschema/1-4-0 b/modules/core/src/test/resources/iglu-test-embedded/schemas/com.snowplowanalytics.iglu-test/lookup-schemas-until/jsonschema/1-4-0 new file mode 100644 index 00000000..4c9be2a1 --- /dev/null +++ b/modules/core/src/test/resources/iglu-test-embedded/schemas/com.snowplowanalytics.iglu-test/lookup-schemas-until/jsonschema/1-4-0 @@ -0,0 +1,44 @@ +{ + "$schema": "http://iglucentral.com/schemas/com.snowplowanalytics.self-desc/schema/jsonschema/1-0-0#", + "description": "Schema to test lookupSchemasUntil function", + "self": { + "vendor": "com.snowplowanalytics", + "name": "lookup-schemas-until", + "format": "jsonschema", + "version": "1-4-0" + }, + "type": "object", + "properties": { + "field_1": { + "type": "string" + }, + "field_2": { + "type": ["string", "null"] + }, + "field_3": { + "type": ["string", "null"] + }, + "field_4": { + "type": ["string", "null"] + }, + "field_5": { + "type": ["string", "null"] + }, + "field_6": { + "type": ["string", "null"] + }, + "field_7": { + "type": ["string", "null"] + }, + "field_8": { + "type": ["string", "null"] + }, + "field_9": { + "type": ["string", "null"] + } + }, + "required": [ + "field_1" + ], + "additionalProperties": false +} \ No newline at end of file diff --git a/modules/core/src/test/resources/iglu-test-embedded/schemas/com.snowplowanalytics.iglu-test/lookup-schemas-until/jsonschema/2-0-0 b/modules/core/src/test/resources/iglu-test-embedded/schemas/com.snowplowanalytics.iglu-test/lookup-schemas-until/jsonschema/2-0-0 new file mode 100644 index 00000000..b6cd49c2 --- /dev/null +++ b/modules/core/src/test/resources/iglu-test-embedded/schemas/com.snowplowanalytics.iglu-test/lookup-schemas-until/jsonschema/2-0-0 @@ -0,0 +1,20 @@ +{ + "$schema": "http://iglucentral.com/schemas/com.snowplowanalytics.self-desc/schema/jsonschema/1-0-0#", + "description": "Schema to test lookupSchemasUntil function", + "self": { + "vendor": "com.snowplowanalytics", + "name": "lookup-schemas-until", + "format": "jsonschema", + "version": "2-0-0" + }, + "type": "object", + "properties": { + "field__1": { + "type": "string" + } + }, + "required": [ + "field__1" + ], + "additionalProperties": false +} \ No newline at end of file diff --git a/modules/core/src/test/resources/iglu-test-embedded/schemas/com.snowplowanalytics.iglu-test/lookup-schemas-until/jsonschema/2-0-1 b/modules/core/src/test/resources/iglu-test-embedded/schemas/com.snowplowanalytics.iglu-test/lookup-schemas-until/jsonschema/2-0-1 new file mode 100644 index 00000000..4c9a26b9 --- /dev/null +++ b/modules/core/src/test/resources/iglu-test-embedded/schemas/com.snowplowanalytics.iglu-test/lookup-schemas-until/jsonschema/2-0-1 @@ -0,0 +1,23 @@ +{ + "$schema": "http://iglucentral.com/schemas/com.snowplowanalytics.self-desc/schema/jsonschema/1-0-0#", + "description": "Invalid schema to test lookupSchemasUntil function - Misses comma after properties", + "self": { + "vendor": "com.snowplowanalytics", + "name": "lookup-schemas-until", + "format": "jsonschema", + "version": "2-0-0" + }, + "type": "object", + "properties": { + "field__1": { + "type": "string" + }, + "field__2": { + "type": ["string", "null"] + } + } + "required": [ + "field__1" + ], + "additionalProperties": false +} \ No newline at end of file diff --git a/modules/core/src/test/resources/iglu-test-embedded/schemas/com.snowplowanalytics.iglu-test/lookup-schemas-until/jsonschema/2-1-0 b/modules/core/src/test/resources/iglu-test-embedded/schemas/com.snowplowanalytics.iglu-test/lookup-schemas-until/jsonschema/2-1-0 new file mode 100644 index 00000000..13475e62 --- /dev/null +++ b/modules/core/src/test/resources/iglu-test-embedded/schemas/com.snowplowanalytics.iglu-test/lookup-schemas-until/jsonschema/2-1-0 @@ -0,0 +1,26 @@ +{ + "$schema": "http://iglucentral.com/schemas/com.snowplowanalytics.self-desc/schema/jsonschema/1-0-0#", + "description": "Schema to test lookupSchemasUntil function", + "self": { + "vendor": "com.snowplowanalytics", + "name": "lookup-schemas-until", + "format": "jsonschema", + "version": "2-1-0" + }, + "type": "object", + "properties": { + "field__1": { + "type": "string" + }, + "field__2": { + "type": ["string", "null"] + }, + "field__3": { + "type": ["string", "null"] + } + }, + "required": [ + "field__1" + ], + "additionalProperties": false +} \ No newline at end of file diff --git a/modules/core/src/test/resources/iglu-test-embedded/schemas/com.snowplowanalytics.iglu-test/lookup-schemas-until/jsonschema/3-0-0 b/modules/core/src/test/resources/iglu-test-embedded/schemas/com.snowplowanalytics.iglu-test/lookup-schemas-until/jsonschema/3-0-0 new file mode 100644 index 00000000..cc90f021 --- /dev/null +++ b/modules/core/src/test/resources/iglu-test-embedded/schemas/com.snowplowanalytics.iglu-test/lookup-schemas-until/jsonschema/3-0-0 @@ -0,0 +1,20 @@ +{ + "$schema": "http://iglucentral.com/schemas/com.snowplowanalytics.self-desc/schema/jsonschema/1-0-0#", + "description": "Schema to test lookupSchemasUntil function", + "self": { + "vendor": "com.snowplowanalytics", + "name": "lookup-schemas-until", + "format": "jsonschema", + "version": "3-0-0" + }, + "type": "object", + "properties": { + "field___1": { + "type": "string" + } + }, + "required": [ + "field___1" + ], + "additionalProperties": false +} \ No newline at end of file diff --git a/modules/core/src/test/scala/com.snowplowanalytics.iglu.client/SpecHelpers.scala b/modules/core/src/test/scala/com.snowplowanalytics.iglu.client/SpecHelpers.scala index a5d906d6..0f41093a 100644 --- a/modules/core/src/test/scala/com.snowplowanalytics.iglu.client/SpecHelpers.scala +++ b/modules/core/src/test/scala/com.snowplowanalytics.iglu.client/SpecHelpers.scala @@ -52,6 +52,12 @@ object SpecHelpers { Registry.HttpConnection(URI.create("http://iglucentral.com"), None) ) + val IgluCentralMirror: Registry = + Registry.Http( + Registry.Config("Iglu Central - GCP Mirror", 10, List("com.snowplowanalytics")), + Registry.HttpConnection(URI.create("http://mirror01.iglucentral.com"), None) + ) + case class TrackingRegistry( lookupState: AtomicReference[List[String]], listState: AtomicReference[List[String]] diff --git a/modules/core/src/test/scala/com.snowplowanalytics.iglu.client/resolver/ResolverResultSpec.scala b/modules/core/src/test/scala/com.snowplowanalytics.iglu.client/resolver/ResolverResultSpec.scala index d67950d9..3f051a7c 100644 --- a/modules/core/src/test/scala/com.snowplowanalytics.iglu.client/resolver/ResolverResultSpec.scala +++ b/modules/core/src/test/scala/com.snowplowanalytics.iglu.client/resolver/ResolverResultSpec.scala @@ -23,6 +23,7 @@ import scala.concurrent.duration._ // Cats import cats.Id +import cats.data.NonEmptyList import cats.effect.IO import cats.effect.implicits._ import cats.implicits._ @@ -32,7 +33,7 @@ import io.circe.Json import io.circe.literal._ // Iglu Core -import com.snowplowanalytics.iglu.core.{SchemaKey, SchemaVer} +import com.snowplowanalytics.iglu.core.{SchemaKey, SchemaVer, SelfDescribingSchema} // This project import com.snowplowanalytics.iglu.client.ClientError._ @@ -44,7 +45,7 @@ import com.snowplowanalytics.iglu.client.resolver.registries.{ RegistryError, RegistryLookup } -import com.snowplowanalytics.iglu.client.resolver.Resolver.SchemaItem +import com.snowplowanalytics.iglu.client.resolver.Resolver.{SchemaItem, SchemaResolutionError} // Specs2 import com.snowplowanalytics.iglu.client.SpecHelpers._ @@ -76,6 +77,20 @@ class ResolverResultSpec extends Specification with ValidatedMatchers with CatsE a Resolver shouldn't return superseding schema if resolveSupersedingSchema is false $e17 a Resolver should return cached "not found" when ttl not exceeded $e18 a Resolver should update cached "not found" when ttl exceeded $e19 + lookupSchemasUntil should + return 1-0-0 $e20 + return 1-0-0 and 1-1-0 $e21 + return 1-0-0, 1-1-0 and 1-1-1 $e22 + return 1-0-0, 1-1-0, 1-1-1 and 1-1-2 $e23 + return 1-0-0, 1-1-0, 1-1-1, 1-1-2 and 1-2-0 $e24 + return 1-0-0, 1-1-0, 1-1-1, 1-1-2, 1-2-0 and 1-2-1 $e25 + return 1-0-0, 1-1-0, 1-1-1, 1-1-2, 1-2-0, 1-2-1 and 1-2-2 $e26 + return an error if the first schema of current revision is invalid $e27 + return an error if the first schema of previous revision is invalid $e28 + return an error if the second schema of current revision is invalid $e29 + return an error if the second schema of previous revision is invalid $e30 + return 3-0-0 (no 1-*-* and 2-*-* schemas) $e31 + return 3-0-0 from a registry and 3-1-0 from another one $e32 """ import ResolverSpec._ @@ -214,16 +229,13 @@ class ResolverResultSpec extends Specification with ValidatedMatchers with CatsE val time = Instant.ofEpochMilli(3L) val responses = List(timeoutError, correctResult) - val httpRep = - Registry.Http(Registry.Config("Mock Repo", 1, List("com.snowplowanalytics.iglu-test")), null) - implicit val cache = ResolverSpecHelpers.staticResolverCache implicit val clock = ResolverSpecHelpers.staticClock implicit val registryLookup: RegistryLookup[StaticLookup] = ResolverSpecHelpers.getLookup(responses, Nil) val result = for { - resolver <- Resolver.init[StaticLookup](10, None, httpRep) + resolver <- Resolver.init[StaticLookup](10, None, Repos.httpRep) response1 <- resolver.lookupSchemaResult(schemaKey) response2 <- resolver.lookupSchemaResult(schemaKey) _ <- StaticLookup.addTime(600.milliseconds) @@ -270,9 +282,6 @@ class ResolverResultSpec extends Specification with ValidatedMatchers with CatsE RegistryError.RepoFailure("Should never be reached").asLeft ) - val httpRep = - Registry.Http(Registry.Config("Mock Repo", 1, List("com.snowplowanalytics.iglu-test")), null) - implicit val cache = ResolverSpecHelpers.staticResolverCache implicit val clock = ResolverSpecHelpers.staticClock implicit val registryLookup: RegistryLookup[StaticLookup] = @@ -284,7 +293,7 @@ class ResolverResultSpec extends Specification with ValidatedMatchers with CatsE .init[StaticLookup]( 10, Some(1.seconds), - httpRep + Repos.httpRep ) _ <- resolver.lookupSchemaResult(schemaKey) _ <- StaticLookup.addTime(2.seconds) @@ -323,14 +332,8 @@ class ResolverResultSpec extends Specification with ValidatedMatchers with CatsE val error4 = RegistryError.RepoFailure("Server segfault") // Mocking repositories - val httpRep1 = Registry.Http( - Registry.Config("Mock Repo 1", 1, List("com.snowplowanalytics.iglu-test")), - null - ) - val httpRep2 = Registry.Http( - Registry.Config("Mock Repo 2", 1, List("com.snowplowanalytics.iglu-test")), - null - ) + val httpRep1 = Repos.httpRep.copy(config = Repos.httpRep.config.copy(name = "Mock Repo 1")) + val httpRep2 = Repos.httpRep.copy(config = Repos.httpRep.config.copy(name = "Mock Repo 2")) implicit val cache = ResolverSpecHelpers.staticResolverCache implicit val clock = ResolverSpecHelpers.staticClock @@ -465,16 +468,13 @@ class ResolverResultSpec extends Specification with ValidatedMatchers with CatsE Json.Null.asRight[RegistryError] val responses = List(schema, schema) - val httpRep = - Registry.Http(Registry.Config("Mock Repo", 1, List("com.snowplowanalytics.iglu-test")), null) - implicit val cache = ResolverSpecHelpers.staticResolverCache implicit val clock = ResolverSpecHelpers.staticClock implicit val registryLookup: RegistryLookup[StaticLookup] = ResolverSpecHelpers.getLookup(responses, Nil) val result = for { - resolver <- Resolver.init[StaticLookup](10, Some(200.seconds), httpRep) + resolver <- Resolver.init[StaticLookup](10, Some(200.seconds), Repos.httpRep) response1 <- resolver.lookupSchemaResult(schemaKey) _ <- StaticLookup.addTime(150.seconds) // ttl 200, delay 150 response2 <- resolver.lookupSchemaResult(schemaKey) @@ -503,16 +503,13 @@ class ResolverResultSpec extends Specification with ValidatedMatchers with CatsE Json.Null.asRight[RegistryError] val responses = List(schema, schema) - val httpRep = - Registry.Http(Registry.Config("Mock Repo", 1, List("com.snowplowanalytics.iglu-test")), null) - implicit val cache = ResolverSpecHelpers.staticResolverCache implicit val clock = ResolverSpecHelpers.staticClock implicit val registryLookup: RegistryLookup[StaticLookup] = ResolverSpecHelpers.getLookup(responses, Nil) val result = for { - resolver <- Resolver.init[StaticLookup](10, Some(200.seconds), httpRep) + resolver <- Resolver.init[StaticLookup](10, Some(200.seconds), Repos.httpRep) response1 <- resolver.lookupSchemaResult(schemaKey) _ <- StaticLookup.addTime(250.seconds) // ttl 200, delay 250 response2 <- resolver.lookupSchemaResult(schemaKey) @@ -654,16 +651,13 @@ class ResolverResultSpec extends Specification with ValidatedMatchers with CatsE val notFound = RegistryError.NotFound.asLeft[Json] val responses = List(notFound, schema) - val httpRep = - Registry.Http(Registry.Config("Mock Repo", 1, List("com.snowplowanalytics.iglu-test")), null) - implicit val cache = ResolverSpecHelpers.staticResolverCache implicit val clock = ResolverSpecHelpers.staticClock implicit val registryLookup: RegistryLookup[StaticLookup] = ResolverSpecHelpers.getLookup(responses, Nil) val result = for { - resolver <- Resolver.init[StaticLookup](10, Some(200.seconds), httpRep) + resolver <- Resolver.init[StaticLookup](10, Some(200.seconds), Repos.httpRep) response1 <- resolver.lookupSchemaResult(schemaKey) _ <- StaticLookup.addTime(150.seconds) // ttl 200, delay 150 response2 <- resolver.lookupSchemaResult(schemaKey) @@ -705,9 +699,7 @@ class ResolverResultSpec extends Specification with ValidatedMatchers with CatsE val notFound = RegistryError.NotFound.asLeft[Json] val responses = List(notFound, notFound, schema) - val repoName = "Mock Repo" - val httpRep = - Registry.Http(Registry.Config(repoName, 1, List("com.snowplowanalytics.iglu-test")), null) + val repoName = Repos.httpRep.config.name implicit val cache = ResolverSpecHelpers.staticResolverCache implicit val clock = ResolverSpecHelpers.staticClock @@ -715,7 +707,7 @@ class ResolverResultSpec extends Specification with ValidatedMatchers with CatsE ResolverSpecHelpers.getLookup(responses, Nil) val result = for { - resolver <- Resolver.init[StaticLookup](10, Some(200.seconds), httpRep) + resolver <- Resolver.init[StaticLookup](10, Some(200.seconds), Repos.httpRep) response1 <- resolver.lookupSchemaResult(schemaKey) _ <- StaticLookup.addTime(250.seconds) // ttl 200, delay 250 response2 <- resolver.lookupSchemaResult(schemaKey) @@ -747,7 +739,86 @@ class ResolverResultSpec extends Specification with ValidatedMatchers with CatsE val secondAndThirdEqual = response2 must beEqualTo(response3) firstNotFound and secondNotCached and secondAndThirdEqual + } + import ResolverSpecHelpers.LookupSchemasUntil._ + + def testLookupUntil(maxSchemaKey: SchemaKey, expected: NonEmptyList[SelfDescribingSchema[Json]]) = + for { + resolver <- mkResolver + result <- resolver.lookupSchemasUntil(maxSchemaKey) + } yield result must beRight.like { case schemas => schemas must beEqualTo(expected) } + + def e20 = testLookupUntil( + getUntilSchemaKey(1, 0, 0), + NonEmptyList.one(until100) + ) + + def e21 = testLookupUntil( + getUntilSchemaKey(1, 1, 0), + NonEmptyList.of(until100, until110) + ) + + def e22 = testLookupUntil( + getUntilSchemaKey(1, 1, 1), + NonEmptyList.of(until100, until110, until111) + ) + + def e23 = testLookupUntil( + getUntilSchemaKey(1, 1, 2), + NonEmptyList.of(until100, until110, until111, until112) + ) + + def e24 = testLookupUntil( + getUntilSchemaKey(1, 2, 0), + NonEmptyList.of(until100, until110, until111, until112, until120) + ) + + def e25 = testLookupUntil( + getUntilSchemaKey(1, 2, 1), + NonEmptyList.of(until100, until110, until111, until112, until120, until121) + ) + + def e26 = testLookupUntil( + getUntilSchemaKey(1, 2, 2), + NonEmptyList.of(until100, until110, until111, until112, until120, until121, until122) + ) + + def e27 = for { + resolver <- mkResolver + result <- resolver.lookupSchemasUntil(getUntilSchemaKey(1, 3, 0)) + } yield result must beLeft.like { case SchemaResolutionError(schemaKey, _) => + schemaKey must beEqualTo(getUntilSchemaKey(1, 3, 0)) } + def e28 = for { + resolver <- mkResolver + result <- resolver.lookupSchemasUntil(getUntilSchemaKey(1, 4, 0)) + } yield result must beLeft.like { case SchemaResolutionError(schemaKey, _) => + schemaKey must beEqualTo(getUntilSchemaKey(1, 3, 0)) + } + + def e29 = for { + resolver <- mkResolver + result <- resolver.lookupSchemasUntil(getUntilSchemaKey(2, 0, 1)) + } yield result must beLeft.like { case SchemaResolutionError(schemaKey, _) => + schemaKey must beEqualTo(getUntilSchemaKey(2, 0, 1)) + } + + def e30 = for { + resolver <- mkResolver + result <- resolver.lookupSchemasUntil(getUntilSchemaKey(2, 1, 0)) + } yield result must beLeft.like { case SchemaResolutionError(schemaKey, _) => + schemaKey must beEqualTo(getUntilSchemaKey(2, 0, 1)) + } + + def e31 = testLookupUntil( + getUntilSchemaKey(3, 0, 0), + NonEmptyList.one(until300) + ) + + def e32 = testLookupUntil( + getUntilSchemaKey(3, 1, 0), + NonEmptyList.of(until300, until310) + ) } diff --git a/modules/core/src/test/scala/com.snowplowanalytics.iglu.client/resolver/ResolverSpec.scala b/modules/core/src/test/scala/com.snowplowanalytics.iglu.client/resolver/ResolverSpec.scala index ff4296b6..d064dff5 100644 --- a/modules/core/src/test/scala/com.snowplowanalytics.iglu.client/resolver/ResolverSpec.scala +++ b/modules/core/src/test/scala/com.snowplowanalytics.iglu.client/resolver/ResolverSpec.scala @@ -56,6 +56,18 @@ object ResolverSpec { val one: Registry.Embedded = embedRef("com.acme", 0) val two: Registry.Embedded = embedRef("de.acompany.snowplow", 40) val three: Registry.Embedded = embedRef("de.acompany.snowplow", 100) + + val custom: Registry = + Registry.Http( + Registry.Config("Iglu Custom Repo", 10, List("com.acme")), + Registry.HttpConnection(URI.create("http://iglu.acme.com"), None) + ) + + val httpRep = + Registry.Http( + Registry.Config("Mock Repo", 1, List("com.snowplowanalytics.iglu-test")), + Registry.HttpConnection(URI.create("http://iglu.mock.org"), None) + ) } } @@ -76,6 +88,15 @@ class ResolverSpec extends Specification with CatsEffect { a Resolver should cache SchemaLists with different models separately $e11 a Resolver should use schemaKey provided in SchemaListLike for result validation $e12 result from SchemaListLike should contain the exact schemaKey provided $e13 + isNotFound should + return true if custom repo and Iglu Central repos don't have a schema $e14 + return true if one Iglu Central repo returns an error and the other one NotFound $e15 + return false if custom repo returns an error $e16 + return true if there is no custom repo, one Iglu Central repo returns an error and the other one NotFound $e17 + return false if there is no custom repo and Iglu Central ones return an error $e18 + return true if there is just one custom repo that returns NotFound $e19 + return false if there is just one custom repo that returns an error $e20 + return true if one Iglu Central repo returns 2 errors and the other one returns one error and one NotFound $e21 """ import ResolverSpec._ @@ -208,16 +229,13 @@ class ResolverSpec extends Specification with CatsEffect { val time = Instant.ofEpochMilli(3L) val responses = List(timeoutError, correctSchema) - val httpRep = - Registry.Http(Registry.Config("Mock Repo", 1, List("com.snowplowanalytics.iglu-test")), null) - implicit val cache: CreateResolverCache[StaticLookup] = ResolverSpecHelpers.staticResolverCache implicit val clock: Clock[StaticLookup] = ResolverSpecHelpers.staticClock implicit val registryLookup: RegistryLookup[StaticLookup] = ResolverSpecHelpers.getLookup(responses, Nil) val result = for { - resolver <- Resolver.init[StaticLookup](10, None, httpRep) + resolver <- Resolver.init[StaticLookup](10, None, Repos.httpRep) response1 <- resolver.lookupSchema(schemaKey) response2 <- resolver.lookupSchema(schemaKey) _ <- StaticLookup.addTime(600.millis) @@ -261,9 +279,6 @@ class ResolverSpec extends Specification with CatsEffect { RegistryError.RepoFailure("Should never be reached").asLeft ) - val httpRep = - Registry.Http(Registry.Config("Mock Repo", 1, List("com.snowplowanalytics.iglu-test")), null) - implicit val cache: CreateResolverCache[StaticLookup] = ResolverSpecHelpers.staticResolverCache implicit val clock: Clock[StaticLookup] = ResolverSpecHelpers.staticClock implicit val registryLookup: RegistryLookup[StaticLookup] = @@ -272,7 +287,7 @@ class ResolverSpec extends Specification with CatsEffect { val result = for { resolver <- Resolver - .init[StaticLookup](10, Some(1.second), httpRep) + .init[StaticLookup](10, Some(1.second), Repos.httpRep) _ <- resolver.lookupSchema(schemaKey) _ <- StaticLookup.addTime(2.seconds) _ <- resolver.lookupSchema(schemaKey) @@ -307,14 +322,8 @@ class ResolverSpec extends Specification with CatsEffect { val error4 = RegistryError.RepoFailure("Server segfault") // Mocking repositories - val httpRep1 = Registry.Http( - Registry.Config("Mock Repo 1", 1, List("com.snowplowanalytics.iglu-test")), - null - ) - val httpRep2 = Registry.Http( - Registry.Config("Mock Repo 2", 1, List("com.snowplowanalytics.iglu-test")), - null - ) + val httpRep1 = Repos.httpRep.copy(config = Repos.httpRep.config.copy(name = "Mock Repo 1")) + val httpRep2 = Repos.httpRep.copy(config = Repos.httpRep.config.copy(name = "Mock Repo 2")) implicit val cache: CreateResolverCache[StaticLookup] = ResolverSpecHelpers.staticResolverCache implicit val clock: Clock[StaticLookup] = ResolverSpecHelpers.staticClock @@ -480,4 +489,178 @@ class ResolverSpec extends Specification with CatsEffect { result must beRight(SchemaList.parseUnsafe(List(schema100, schema101, schema102))) } + + def e14 = { + val resolver: Resolver[Id] = + Resolver + .init[Id](0, None, SpecHelpers.IgluCentral, SpecHelpers.IgluCentralMirror, Repos.custom) + + val resolutionError = ResolutionError( + SortedMap( + SpecHelpers.IgluCentral.config.name -> LookupHistory( + Set(RegistryError.NotFound), + 1, + Instant.now() + ), + SpecHelpers.IgluCentralMirror.config.name -> LookupHistory( + Set(RegistryError.NotFound), + 1, + Instant.now() + ), + Repos.custom.config.name -> LookupHistory(Set(RegistryError.NotFound), 1, Instant.now()) + ) + ) + + resolver.isNotFound(resolutionError) should beTrue + } + + def e15 = { + val resolver: Resolver[Id] = + Resolver + .init[Id](0, None, SpecHelpers.IgluCentral, SpecHelpers.IgluCentralMirror, Repos.custom) + + val resolutionError = ResolutionError( + SortedMap( + SpecHelpers.IgluCentral.config.name -> LookupHistory( + Set(RegistryError.RepoFailure("Problem")), + 1, + Instant.now() + ), + SpecHelpers.IgluCentralMirror.config.name -> LookupHistory( + Set(RegistryError.NotFound), + 1, + Instant.now() + ), + Repos.custom.config.name -> LookupHistory(Set(RegistryError.NotFound), 1, Instant.now()) + ) + ) + + resolver.isNotFound(resolutionError) should beTrue + } + + def e16 = { + val resolver: Resolver[Id] = + Resolver + .init[Id](0, None, SpecHelpers.IgluCentral, SpecHelpers.IgluCentralMirror, Repos.custom) + + val resolutionError = ResolutionError( + SortedMap( + SpecHelpers.IgluCentral.config.name -> LookupHistory( + Set(RegistryError.NotFound), + 1, + Instant.now() + ), + SpecHelpers.IgluCentralMirror.config.name -> LookupHistory( + Set(RegistryError.NotFound), + 1, + Instant.now() + ), + Repos.custom.config.name -> LookupHistory( + Set(RegistryError.ClientFailure("Something went wrong")), + 1, + Instant.now() + ) + ) + ) + + resolver.isNotFound(resolutionError) should beFalse + } + + def e17 = { + val resolver: Resolver[Id] = + Resolver.init[Id](0, None, SpecHelpers.IgluCentral, SpecHelpers.IgluCentralMirror) + + val resolutionError = ResolutionError( + SortedMap( + SpecHelpers.IgluCentral.config.name -> LookupHistory( + Set(RegistryError.RepoFailure("Problem")), + 1, + Instant.now() + ), + SpecHelpers.IgluCentralMirror.config.name -> LookupHistory( + Set(RegistryError.NotFound), + 1, + Instant.now() + ) + ) + ) + + resolver.isNotFound(resolutionError) should beTrue + } + + def e18 = { + val resolver: Resolver[Id] = + Resolver.init[Id](0, None, SpecHelpers.IgluCentral, SpecHelpers.IgluCentralMirror) + + val resolutionError = ResolutionError( + SortedMap( + SpecHelpers.IgluCentral.config.name -> LookupHistory( + Set(RegistryError.RepoFailure("Problem")), + 1, + Instant.now() + ), + SpecHelpers.IgluCentralMirror.config.name -> LookupHistory( + Set(RegistryError.ClientFailure("Network issue")), + 1, + Instant.now() + ) + ) + ) + + resolver.isNotFound(resolutionError) should beFalse + } + + def e19 = { + val resolver: Resolver[Id] = + Resolver.init[Id](0, None, Repos.custom) + + val resolutionError = ResolutionError( + SortedMap( + Repos.custom.config.name -> LookupHistory(Set(RegistryError.NotFound), 1, Instant.now()) + ) + ) + + resolver.isNotFound(resolutionError) should beTrue + } + + def e20 = { + val resolver: Resolver[Id] = + Resolver.init[Id](0, None, Repos.custom) + + val resolutionError = ResolutionError( + SortedMap( + Repos.custom.config.name -> LookupHistory( + Set(RegistryError.ClientFailure("Boom")), + 1, + Instant.now() + ) + ) + ) + + resolver.isNotFound(resolutionError) should beFalse + } + + def e21 = { + val resolver: Resolver[Id] = + Resolver + .init[Id](0, None, SpecHelpers.IgluCentral, SpecHelpers.IgluCentralMirror, Repos.custom) + + val resolutionError = ResolutionError( + SortedMap( + SpecHelpers.IgluCentral.config.name -> LookupHistory( + Set(RegistryError.RepoFailure("Problem"), RegistryError.ClientFailure("Boom")), + 2, + Instant.now() + ), + SpecHelpers.IgluCentralMirror.config.name -> LookupHistory( + Set(RegistryError.RepoFailure("Problem"), RegistryError.NotFound), + 2, + Instant.now() + ), + Repos.custom.config.name -> LookupHistory(Set(RegistryError.NotFound), 1, Instant.now()) + ) + ) + + resolver.isNotFound(resolutionError) should beTrue + } } diff --git a/modules/core/src/test/scala/com.snowplowanalytics.iglu.client/resolver/ResolverSpecHelpers.scala b/modules/core/src/test/scala/com.snowplowanalytics.iglu.client/resolver/ResolverSpecHelpers.scala index 202feac7..7edeb4de 100644 --- a/modules/core/src/test/scala/com.snowplowanalytics.iglu.client/resolver/ResolverSpecHelpers.scala +++ b/modules/core/src/test/scala/com.snowplowanalytics.iglu.client/resolver/ResolverSpecHelpers.scala @@ -13,24 +13,34 @@ package com.snowplowanalytics.iglu.client package resolver +import scala.io.Source + // Cats import cats.Applicative import cats.data.State -import cats.effect.Clock +import cats.effect.{Clock, IO} import cats.syntax.either._ import scala.concurrent.duration.{DurationInt, FiniteDuration} // Circe import io.circe.Json +import io.circe.parser._ // LRU Map import com.snowplowanalytics.iglu.core.circe.implicits._ -import com.snowplowanalytics.iglu.core.{SchemaKey, SchemaList} +import com.snowplowanalytics.iglu.core.{ + SchemaKey, + SchemaList, + SchemaMap, + SchemaVer, + SelfDescribingSchema +} import com.snowplowanalytics.lrumap.LruMap // This project import com.snowplowanalytics.iglu.client.resolver.registries.{ + JavaNetRegistryLookup, Registry, RegistryError, RegistryLookup @@ -223,4 +233,48 @@ object ResolverSpecHelpers { def withLockOn[A](key: K)(f: => StaticLookup[A]): StaticLookup[A] = f } + + object LookupSchemasUntil { + val vendor = "com.snowplowanalytics.iglu-test" + val name = "lookup-schemas-until" + val format = "jsonschema" + + val until100 = parseSchemaUntil(1, 0, 0) + val until110 = parseSchemaUntil(1, 1, 0) + val until111 = parseSchemaUntil(1, 1, 1) + val until112 = parseSchemaUntil(1, 1, 2) + val until120 = parseSchemaUntil(1, 2, 0) + val until121 = parseSchemaUntil(1, 2, 1) + val until122 = parseSchemaUntil(1, 2, 2) + val until300 = parseSchemaUntil(3, 0, 0) + val until310 = parseSchemaUntil(3, 1, 0, "iglu-client-embedded") + + implicit val lookup: RegistryLookup[IO] = JavaNetRegistryLookup.ioLookupInstance[IO] + def mkResolver = Resolver.init[IO](0, None, SpecHelpers.EmbeddedTest) + + def getUntilSchemaKey(model: Int, revision: Int, addition: Int): SchemaKey = + SchemaKey( + vendor, + name, + format, + SchemaVer.Full(model, revision, addition) + ) + + def parseSchemaUntil( + model: Int, + revision: Int, + addition: Int, + embeddedFolder: String = "iglu-test-embedded" + ): SelfDescribingSchema[Json] = { + val path = + s"/$embeddedFolder/schemas/$vendor/$name/$format/$model-$revision-$addition" + val content = Source.fromInputStream(getClass.getResourceAsStream(path)).mkString + parse(content) match { + case Right(json) => + SelfDescribingSchema(SchemaMap(getUntilSchemaKey(model, revision, addition)), json) + case Left(err) => + throw new IllegalArgumentException(s"$path can't be parsed as JSON : [$err]") + } + } + } }