From 122f5af36d96693c3ada3d1594ea500ff3316fb0 Mon Sep 17 00:00:00 2001 From: Gregorius Soedharmo Date: Wed, 12 Jan 2022 03:48:50 +0700 Subject: [PATCH] Add type filtering feature (#281) * Add type filtering feature * fix implicit typed variable and lambda expression build error * Add documentation * Deserializer type filter should work on downcasted type * add XML-DOC * Update API Approval list --- README.md | 58 ++++++++++++++++- .../CoreApiSpec.ApproveApi.approved.txt | 35 ++++++++++- .../UnsafeDeserializationExclusionTests.cs | 63 ++++++++++++++++++- src/Hyperion/Extensions/TypeEx.cs | 13 ++-- src/Hyperion/Hyperion.csproj | 1 + src/Hyperion/ITypeFilter.cs | 15 +++++ src/Hyperion/Internal/Annotations.cs | 7 +++ src/Hyperion/SerializerOptions.cs | 42 ++++++++----- src/Hyperion/TypeFilter.cs | 34 ++++++++++ src/Hyperion/TypeFilterBuilder.cs | 36 +++++++++++ .../ValueSerializers/TypeSerializer.cs | 3 +- 11 files changed, 281 insertions(+), 26 deletions(-) create mode 100644 src/Hyperion/ITypeFilter.cs create mode 100644 src/Hyperion/TypeFilter.cs create mode 100644 src/Hyperion/TypeFilterBuilder.cs diff --git a/README.md b/README.md index 78b53d59..168de438 100644 --- a/README.md +++ b/README.md @@ -48,6 +48,60 @@ var serializer = new Serializer(options); This is essential for frameworks like Akka.NET where we need to be able to resolve live Actor References in the deserializing system. +## Whitelisting Types On Deserialization + +Sometimes we need to limit the types that are allowed to be deserialized for security reasons. For this reason, you can either pass a class instance that implements the `ITypeFilter` interface into the `SerializerOptions` or use the `TypeFilterBuilder` class to build a `TypeFilter` that Hyperion can use to filter out any possibly harmful injection attack during deserialization. + +using the `ITypeFilter` interface: + +```c# +public sealed class TypeFilter : ITypeFilter +{ + public ImmutableHashSet FilteredTypes { get; } + + internal TypeFilter(IEnumerable types) + { + FilteredTypes = types.Select(t => t.GetShortAssemblyQualifiedName()).ToImmutableHashSet(); + } + + public bool IsAllowed(string typeName) + => FilteredTypes.Any(t => t == typeName); +} +``` + +using the `TypeFilterBuilder` convenience builder: + +```c# +var typeFilter = TypeFilterBuilder.Create() + .Include() + .Include() + .Build(); + +var options = SerializerOptions.Default + .WithTypeFilter(typeFilter); + +var serializer = new Serializer(options); +``` + +### Convert Whitelist To Blacklist + +To do blacklisting instead of whitelisting a list of types, you will need to do a slight modification to the TypeFilter class. + +```c# +public sealed class TypeFilter : ITypeFilter +{ + public ImmutableHashSet FilteredTypes { get; } + + internal TypeFilter(IEnumerable types) + { + FilteredTypes = types.Select(t => t.GetShortAssemblyQualifiedName()).ToImmutableHashSet(); + } + + public bool IsAllowed(string typeName) + => FilteredTypes.All(t => t != typeName); +} +``` + ## Version Tolerance Hyperion has been designed to work in multiple modes in terms of version tolerance vs. performance. @@ -55,13 +109,13 @@ Hyperion has been designed to work in multiple modes in terms of version toleran 1. Pre Register Types, when using "Pre registered types", Hyperion will only emit a type ID in the output stream. This results in the best performance, but is also fragile if different clients have different versions of the contract types. 2. Non Versioned, this is largely the same as the above, but the serializer does not need to know about your types up front. it will embed the fully qualified typename -in the outputstream. this results in a larger payload and some performance overhead. +in the output stream. this results in a larger payload and some performance overhead. 3. Versioned, in this mode, Hyperion will emit both type names and field information in the output stream. This allows systems to have slightly different versions of the contract types where some fields may have been added or removed. Hyperion has been designed as a wire format, point to point for soft realtime scenarios. If you need a format that is durable for persistence over time. -e.g. EventSourcing or for message queues, then Protobuf or MS Bond is probably a better choise as those formats have been designed for true version tolerance. +e.g. EventSourcing or for message queues, then Protobuf or MS Bond is probably a better choice as those formats have been designed for true version tolerance. ## Performance diff --git a/src/Hyperion.API.Tests/CoreApiSpec.ApproveApi.approved.txt b/src/Hyperion.API.Tests/CoreApiSpec.ApproveApi.approved.txt index d5fe90b4..bca91930 100644 --- a/src/Hyperion.API.Tests/CoreApiSpec.ApproveApi.approved.txt +++ b/src/Hyperion.API.Tests/CoreApiSpec.ApproveApi.approved.txt @@ -41,6 +41,11 @@ namespace Hyperion public void TrackDeserializedType([Hyperion.Internal.NotNull] System.Type type) { } public void TrackDeserializedTypeWithVersion([Hyperion.Internal.NotNull] System.Type type, [Hyperion.Internal.NotNull] Hyperion.TypeVersionInfo versionInfo) { } } + public sealed class DisabledTypeFilter : Hyperion.ITypeFilter + { + public static readonly Hyperion.DisabledTypeFilter Instance; + public bool IsAllowed(string typeName) { } + } public delegate object FieldInfoReader(object obj); public delegate void FieldInfoWriter(object obj, object value); public delegate void FieldReader(System.IO.Stream stream, object obj, Hyperion.DeserializerSession session); @@ -49,6 +54,10 @@ namespace Hyperion { void BuildSerializer(Hyperion.Serializer serializer, Hyperion.ValueSerializers.ObjectSerializer objectSerializer); } + public interface ITypeFilter + { + bool IsAllowed(string typeName); + } [System.AttributeUsage(System.AttributeTargets.Field | System.AttributeTargets.All, AllowMultiple=false, Inherited=true)] public sealed class IgnoreAttribute : System.Attribute { @@ -90,11 +99,16 @@ namespace Hyperion public class SerializerOptions { public static readonly Hyperion.SerializerOptions Default; - [System.Obsolete] + [System.Obsolete("This constructor is deprecated and will be removed in the future, please use the " + + "one with the packageNameOverrides argument")] public SerializerOptions(bool versionTolerance = false, bool preserveObjectReferences = false, System.Collections.Generic.IEnumerable surrogates = null, System.Collections.Generic.IEnumerable serializerFactories = null, System.Collections.Generic.IEnumerable knownTypes = null, bool ignoreISerializable = false) { } - [System.Obsolete] + [System.Obsolete("This constructor is deprecated and will be removed in the future, please use the " + + "one with the disallowUnsafeTypes argument")] public SerializerOptions(bool versionTolerance, bool preserveObjectReferences, System.Collections.Generic.IEnumerable surrogates, System.Collections.Generic.IEnumerable serializerFactories, System.Collections.Generic.IEnumerable knownTypes, bool ignoreISerializable, System.Collections.Generic.IEnumerable> packageNameOverrides) { } + [System.Obsolete("This constructor is deprecated and will be removed in the future, please use the " + + "one with the typeFilter argument")] public SerializerOptions(bool versionTolerance, bool preserveObjectReferences, System.Collections.Generic.IEnumerable surrogates, System.Collections.Generic.IEnumerable serializerFactories, System.Collections.Generic.IEnumerable knownTypes, bool ignoreISerializable, System.Collections.Generic.IEnumerable> packageNameOverrides, bool disallowUnsafeTypes) { } + public SerializerOptions(bool versionTolerance, bool preserveObjectReferences, System.Collections.Generic.IEnumerable surrogates, System.Collections.Generic.IEnumerable serializerFactories, System.Collections.Generic.IEnumerable knownTypes, bool ignoreISerializable, System.Collections.Generic.IEnumerable> packageNameOverrides, bool disallowUnsafeTypes, Hyperion.ITypeFilter typeFilter) { } public Hyperion.SerializerOptions WithDisallowUnsafeType(bool disallowUnsafeType) { } public Hyperion.SerializerOptions WithIgnoreSerializable(bool ignoreISerializable) { } public Hyperion.SerializerOptions WithKnownTypes(System.Collections.Generic.IEnumerable knownTypes) { } @@ -102,6 +116,7 @@ namespace Hyperion public Hyperion.SerializerOptions WithPreserveObjectReferences(bool preserveObjectReferences) { } public Hyperion.SerializerOptions WithSerializerFactory(System.Collections.Generic.IEnumerable serializerFactories) { } public Hyperion.SerializerOptions WithSurrogates(System.Collections.Generic.IEnumerable surrogates) { } + public Hyperion.SerializerOptions WithTypeFilter(Hyperion.ITypeFilter typeFilter) { } public Hyperion.SerializerOptions WithVersionTolerance(bool versionTolerance) { } } public class SerializerSession @@ -130,6 +145,18 @@ namespace Hyperion { public Surrogate(System.Func toSurrogate, System.Func fromSurrogate) { } } + public sealed class TypeFilter : Hyperion.ITypeFilter + { + public System.Collections.Immutable.ImmutableHashSet FilteredTypes { get; } + public bool IsAllowed(string typeName) { } + } + public class TypeFilterBuilder + { + public Hyperion.TypeFilter Build() { } + public Hyperion.TypeFilterBuilder Include(System.Type type) { } + public Hyperion.TypeFilterBuilder Include() { } + public static Hyperion.TypeFilterBuilder Create() { } + } public class TypeVersionInfo { public TypeVersionInfo() { } @@ -591,6 +618,10 @@ namespace Hyperion.Internal public Hyperion.Internal.ImplicitUseTargetFlags TargetFlags { get; } public Hyperion.Internal.ImplicitUseKindFlags UseKindFlags { get; } } + public class UserEvilDeserializationException : Hyperion.Internal.EvilDeserializationException + { + public UserEvilDeserializationException(string message, string typeString) { } + } [System.AttributeUsage(System.AttributeTargets.Property | System.AttributeTargets.Field | System.AttributeTargets.Parameter | System.AttributeTargets.All)] public sealed class ValueProviderAttribute : System.Attribute { diff --git a/src/Hyperion.Tests/UnsafeDeserializationExclusionTests.cs b/src/Hyperion.Tests/UnsafeDeserializationExclusionTests.cs index 0444f377..e69a58bd 100644 --- a/src/Hyperion.Tests/UnsafeDeserializationExclusionTests.cs +++ b/src/Hyperion.Tests/UnsafeDeserializationExclusionTests.cs @@ -1,7 +1,9 @@ -using System.IO; +using System; +using System.IO; using Hyperion.Extensions; using Hyperion.Internal; using Xunit; +using FluentAssertions; namespace Hyperion.Tests { @@ -22,5 +24,64 @@ public void CantDeserializeANaughtyType() serializer.Deserialize(stream)); } } + + internal class ClassA + { } + + internal class ClassB + { } + + internal class ClassC + { } + + [Fact] + public void TypeFilterShouldThrowOnNaughtyType() + { + var typeFilter = TypeFilterBuilder.Create() + .Include() + .Include() + .Build(); + + var options = SerializerOptions.Default + .WithTypeFilter(typeFilter); + + var serializer = new Serializer(options); + + using (var stream = new MemoryStream()) + { + serializer.Serialize(new ClassA(), stream); + stream.Position = 0; + Action act = () => serializer.Deserialize(stream); + act.Should().NotThrow(); + + stream.Position = 0; + Action actObj = () => serializer.Deserialize(stream); + actObj.Should().NotThrow(); + } + + using (var stream = new MemoryStream()) + { + serializer.Serialize(new ClassB(), stream); + stream.Position = 0; + Action act = () => serializer.Deserialize(stream); + act.Should().NotThrow(); + + stream.Position = 0; + Action actObj = () => serializer.Deserialize(stream); + actObj.Should().NotThrow(); + } + + using (var stream = new MemoryStream()) + { + serializer.Serialize(new ClassC(), stream); + stream.Position = 0; + Action act = () => serializer.Deserialize(stream); + act.Should().Throw(); + + stream.Position = 0; + Action actObj = () => serializer.Deserialize(stream); + actObj.Should().Throw(); + } + } } } \ No newline at end of file diff --git a/src/Hyperion/Extensions/TypeEx.cs b/src/Hyperion/Extensions/TypeEx.cs index 59a7105e..f323b133 100644 --- a/src/Hyperion/Extensions/TypeEx.cs +++ b/src/Hyperion/Extensions/TypeEx.cs @@ -162,7 +162,8 @@ private static Type GetTypeFromManifestName(Stream stream, DeserializerSession s break; } - return LoadTypeByName(shortName, session.Serializer.Options.DisallowUnsafeTypes); + var options = session.Serializer.Options; + return LoadTypeByName(shortName, options.DisallowUnsafeTypes, options.TypeFilter); }); } @@ -192,12 +193,14 @@ private static bool UnsafeInheritanceCheck(Type type) return false; } - public static Type LoadTypeByName(string name, bool disallowUnsafeTypes) + public static Type LoadTypeByName(string name, bool disallowUnsafeTypes, ITypeFilter typeFilter) { - if (disallowUnsafeTypes && UnsafeTypesDenySet.Any(name.Contains)) + if (disallowUnsafeTypes) { - throw new EvilDeserializationException( - "Unsafe Type Deserialization Detected!", name); + if(UnsafeTypesDenySet.Any(name.Contains)) + throw new EvilDeserializationException("Unsafe Type Deserialization Detected!", name); + if(!typeFilter.IsAllowed(name)) + throw new UserEvilDeserializationException("Unsafe Type Deserialization Detected!", name); } try { diff --git a/src/Hyperion/Hyperion.csproj b/src/Hyperion/Hyperion.csproj index 803258d0..214e2837 100644 --- a/src/Hyperion/Hyperion.csproj +++ b/src/Hyperion/Hyperion.csproj @@ -33,6 +33,7 @@ + diff --git a/src/Hyperion/ITypeFilter.cs b/src/Hyperion/ITypeFilter.cs new file mode 100644 index 00000000..038a3e1a --- /dev/null +++ b/src/Hyperion/ITypeFilter.cs @@ -0,0 +1,15 @@ +namespace Hyperion +{ + /// + /// Provide a callback to allow a user defined Type filter during certain operations + /// + public interface ITypeFilter + { + /// + /// Determines if a fully qualified class name is allowed to be processed or not + /// + /// The fully qualified class name of the type to be filtered + /// true if a type is allowed + bool IsAllowed(string typeName); + } +} \ No newline at end of file diff --git a/src/Hyperion/Internal/Annotations.cs b/src/Hyperion/Internal/Annotations.cs index b1c5c0f6..dd4a8525 100644 --- a/src/Hyperion/Internal/Annotations.cs +++ b/src/Hyperion/Internal/Annotations.cs @@ -30,6 +30,13 @@ public EvilDeserializationException(string message, public string BadTypeString { get; } } + + public class UserEvilDeserializationException : EvilDeserializationException + { + public UserEvilDeserializationException(string message, string typeString) : base(message, typeString) + { } + } + /// /// Indicates that the value of the marked element could be null sometimes, /// so the check for null is necessary before its usage. diff --git a/src/Hyperion/SerializerOptions.cs b/src/Hyperion/SerializerOptions.cs index 6b655097..0c8694dd 100644 --- a/src/Hyperion/SerializerOptions.cs +++ b/src/Hyperion/SerializerOptions.cs @@ -16,15 +16,7 @@ namespace Hyperion { public class SerializerOptions { - public static readonly SerializerOptions Default = new SerializerOptions( - versionTolerance: false, - preserveObjectReferences: false, - surrogates: null, - serializerFactories: null, - knownTypes: null, - ignoreISerializable: false, - packageNameOverrides: null, - disallowUnsafeTypes: true); + public static readonly SerializerOptions Default = new SerializerOptions(); internal static List> DefaultPackageNameOverrides() { @@ -81,8 +73,9 @@ internal static List> DefaultPackageNameOverrides() internal readonly Dictionary KnownTypesDict = new Dictionary(); internal readonly List> CrossFrameworkPackageNameOverrides = DefaultPackageNameOverrides(); internal readonly bool DisallowUnsafeTypes; - - [Obsolete] + internal readonly ITypeFilter TypeFilter; + + [Obsolete(message:"This constructor is deprecated and will be removed in the future, please use the one with the packageNameOverrides argument")] public SerializerOptions( bool versionTolerance = false, bool preserveObjectReferences = false, @@ -93,7 +86,7 @@ public SerializerOptions( : this(versionTolerance, preserveObjectReferences, surrogates, serializerFactories, knownTypes, ignoreISerializable, null) { } - [Obsolete] + [Obsolete(message:"This constructor is deprecated and will be removed in the future, please use the one with the disallowUnsafeTypes argument")] public SerializerOptions( bool versionTolerance, bool preserveObjectReferences, @@ -102,9 +95,10 @@ public SerializerOptions( IEnumerable knownTypes, bool ignoreISerializable, IEnumerable> packageNameOverrides) - : this(versionTolerance, preserveObjectReferences, surrogates, serializerFactories, knownTypes, ignoreISerializable, null, true) + : this(versionTolerance, preserveObjectReferences, surrogates, serializerFactories, knownTypes, ignoreISerializable, packageNameOverrides, true) { } + [Obsolete(message:"This constructor is deprecated and will be removed in the future, please use the one with the typeFilter argument")] public SerializerOptions( bool versionTolerance, bool preserveObjectReferences, @@ -114,6 +108,19 @@ public SerializerOptions( bool ignoreISerializable, IEnumerable> packageNameOverrides, bool disallowUnsafeTypes) + : this(versionTolerance, preserveObjectReferences, surrogates, serializerFactories, knownTypes, ignoreISerializable, packageNameOverrides, disallowUnsafeTypes, DisabledTypeFilter.Instance) + { } + + public SerializerOptions( + bool versionTolerance, + bool preserveObjectReferences, + IEnumerable surrogates, + IEnumerable serializerFactories, + IEnumerable knownTypes, + bool ignoreISerializable, + IEnumerable> packageNameOverrides, + bool disallowUnsafeTypes, + ITypeFilter typeFilter) { VersionTolerance = versionTolerance; Surrogates = surrogates?.ToArray() ?? EmptySurrogates; @@ -136,6 +143,7 @@ public SerializerOptions( CrossFrameworkPackageNameOverrides.AddRange(packageNameOverrides); DisallowUnsafeTypes = disallowUnsafeTypes; + TypeFilter = typeFilter ?? DisabledTypeFilter.Instance; } public SerializerOptions WithVersionTolerance(bool versionTolerance) @@ -154,6 +162,8 @@ public SerializerOptions WithPackageNameOverrides(IEnumerable Copy(packageNameOverrides: packageNameOverrides); public SerializerOptions WithDisallowUnsafeType(bool disallowUnsafeType) => Copy(disallowUnsafeType: disallowUnsafeType); + public SerializerOptions WithTypeFilter(ITypeFilter typeFilter) + => Copy(typeFilter: typeFilter); private SerializerOptions Copy( bool? versionTolerance = null, @@ -163,7 +173,8 @@ private SerializerOptions Copy( IEnumerable knownTypes = null, bool? ignoreISerializable = null, IEnumerable> packageNameOverrides = null, - bool? disallowUnsafeType = null) + bool? disallowUnsafeType = null, + ITypeFilter typeFilter = null) => new SerializerOptions( versionTolerance ?? VersionTolerance, preserveObjectReferences ?? PreserveObjectReferences, @@ -172,7 +183,8 @@ private SerializerOptions Copy( knownTypes ?? KnownTypes, ignoreISerializable ?? IgnoreISerializable, packageNameOverrides ?? CrossFrameworkPackageNameOverrides, - disallowUnsafeType ?? DisallowUnsafeTypes + disallowUnsafeType ?? DisallowUnsafeTypes, + typeFilter ?? TypeFilter ); } } \ No newline at end of file diff --git a/src/Hyperion/TypeFilter.cs b/src/Hyperion/TypeFilter.cs new file mode 100644 index 00000000..d337bf04 --- /dev/null +++ b/src/Hyperion/TypeFilter.cs @@ -0,0 +1,34 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using Hyperion.Extensions; + +namespace Hyperion +{ + /// + public sealed class TypeFilter : ITypeFilter + { + public ImmutableHashSet FilteredTypes { get; } + + internal TypeFilter(IEnumerable types) + { + FilteredTypes = types.Select(t => t.GetShortAssemblyQualifiedName()).ToImmutableHashSet(); + } + + public bool IsAllowed(string typeName) + => FilteredTypes.Any(t => t == typeName); + } + + /// + /// A disabled type filter that always returns true + /// + public sealed class DisabledTypeFilter : ITypeFilter + { + public static readonly DisabledTypeFilter Instance = new DisabledTypeFilter(); + + private DisabledTypeFilter() { } + + public bool IsAllowed(string typeName) => true; + } +} \ No newline at end of file diff --git a/src/Hyperion/TypeFilterBuilder.cs b/src/Hyperion/TypeFilterBuilder.cs new file mode 100644 index 00000000..569625d4 --- /dev/null +++ b/src/Hyperion/TypeFilterBuilder.cs @@ -0,0 +1,36 @@ +using System; +using System.Collections.Generic; + +namespace Hyperion +{ + /// + /// Helper class to programatically create a using fluent builder pattern. + /// + public class TypeFilterBuilder + { + /// + /// Create a new instance of + /// + /// a new instance of + public static TypeFilterBuilder Create() => new TypeFilterBuilder(); + + private readonly List _types = new List(); + + private TypeFilterBuilder() + { } + + public TypeFilterBuilder Include() + { + return Include(typeof(T)); + } + + public TypeFilterBuilder Include(Type type) + { + _types.Add(type); + return this; + } + + public TypeFilter Build() + => new TypeFilter(_types); + } +} \ No newline at end of file diff --git a/src/Hyperion/ValueSerializers/TypeSerializer.cs b/src/Hyperion/ValueSerializers/TypeSerializer.cs index 633d2cc7..cbf24857 100644 --- a/src/Hyperion/ValueSerializers/TypeSerializer.cs +++ b/src/Hyperion/ValueSerializers/TypeSerializer.cs @@ -69,8 +69,9 @@ public override object ReadValue(Stream stream, DeserializerSession session) if (shortname == null) return null; + var options = session.Serializer.Options; var type = TypeNameLookup.GetOrAdd(shortname, - name => TypeEx.LoadTypeByName(shortname, session.Serializer.Options.DisallowUnsafeTypes)); + name => TypeEx.LoadTypeByName(shortname, options.DisallowUnsafeTypes, options.TypeFilter)); //add the deserialized type to lookup if (session.Serializer.Options.PreserveObjectReferences)