Addresses a denial of service security vulnerability where FlatSharp could be induced to stack overflow given a schema with a loop (or simply too deep) along with a malicious input.
This commit is contained in:
James Courtney 2022-04-23 23:43:03 -07:00 коммит произвёл GitHub
Родитель 593a94498c
Коммит 7a97087e31
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
33 изменённых файлов: 420 добавлений и 67 удалений

Просмотреть файл

@ -18,6 +18,7 @@ namespace Benchmark
{
using BenchmarkDotNet.Attributes;
using FlatSharp;
using FlatSharp.Internal;
using System;
using System.Collections.Generic;

Просмотреть файл

@ -14,6 +14,8 @@
* limitations under the License.
*/
using System.IO;
namespace FlatSharp;
internal static class FlatSharpInternal

Просмотреть файл

@ -30,6 +30,7 @@ internal class GeneratedSerializerWrapper<T> : ISerializer<T>, ISerializer where
private readonly ThreadLocal<ISharedStringWriter>? sharedStringWriter;
private readonly bool enableMemoryCopySerialization;
private readonly string? fileIdentifier;
private readonly short remainingDepthLimit;
public GeneratedSerializerWrapper(
IGeneratedSerializer<T>? innerSerializer,
@ -45,6 +46,7 @@ internal class GeneratedSerializerWrapper<T> : ISerializer<T>, ISerializer where
var tableAttribute = typeof(T).GetCustomAttribute<Attributes.FlatBufferTableAttribute>();
this.fileIdentifier = tableAttribute?.FileIdentifier;
this.sharedStringWriter = new ThreadLocal<ISharedStringWriter>(() => new SharedStringWriter());
this.remainingDepthLimit = 1000; // sane default.
}
private GeneratedSerializerWrapper(GeneratedSerializerWrapper<T> template, SerializerSettings settings)
@ -54,6 +56,7 @@ internal class GeneratedSerializerWrapper<T> : ISerializer<T>, ISerializer where
this.AssemblyBytes = template.AssemblyBytes;
this.innerSerializer = template.innerSerializer;
this.fileIdentifier = template.fileIdentifier;
this.remainingDepthLimit = template.remainingDepthLimit;
this.enableMemoryCopySerialization = settings.EnableMemoryCopySerialization;
@ -62,6 +65,16 @@ internal class GeneratedSerializerWrapper<T> : ISerializer<T>, ISerializer where
{
this.sharedStringWriter = new ThreadLocal<ISharedStringWriter>(writerFactory);
}
if (settings.ObjectDepthLimit is not null)
{
if (settings.ObjectDepthLimit <= 0)
{
throw new ArgumentException("ObjectDepthLimit must be nonnegative.");
}
this.remainingDepthLimit = settings.ObjectDepthLimit.Value;
}
}
Type ISerializer.RootType => typeof(T);
@ -120,7 +133,7 @@ internal class GeneratedSerializerWrapper<T> : ISerializer<T>, ISerializer where
}
// In case buffer is a reference type or is a boxed value, this allows it the opportunity to "wrap" itself in a value struct for efficiency.
return buffer.InvokeParse(this.innerSerializer, 0);
return buffer.InvokeParse(this.innerSerializer, new GeneratedSerializerParseArguments(0, this.remainingDepthLimit));
}
object ISerializer.Parse<TInputBuffer>(TInputBuffer buffer) => this.Parse(buffer);

Просмотреть файл

@ -1,5 +1,5 @@
/*
* Copyright 2018 James Courtney
* Copyright 2022 James Courtney
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -14,7 +14,23 @@
* limitations under the License.
*/
namespace FlatSharp;
namespace FlatSharp.Internal;
/// <summary>
/// Wrapper struct to pass arguments into <see cref="IGeneratedSerializer{T}.Parse{TInputBuffer}(TInputBuffer, GeneratedSerializerParseArguments)"/>.
/// </summary>
public readonly struct GeneratedSerializerParseArguments
{
public GeneratedSerializerParseArguments(int offset, short depthLimit)
{
this.Offset = offset;
this.DepthLimit = depthLimit;
}
public int Offset { get; }
public short DepthLimit { get; }
}
/// <summary>
/// An interface implemented dynamically by FlatSharp for reading and writing data from a buffer.
@ -49,5 +65,7 @@ public interface IGeneratedSerializer<T>
/// <summary>
/// Parses the given buffer as an instance of <typeparamref name="T"/> from the given offset.
/// </summary>
T Parse<TInputBuffer>(TInputBuffer buffer, int offset) where TInputBuffer : IInputBuffer;
T Parse<TInputBuffer>(
TInputBuffer buffer,
in GeneratedSerializerParseArguments arguments) where TInputBuffer : IInputBuffer;
}

Просмотреть файл

@ -143,9 +143,8 @@ public struct ArrayInputBuffer : IInputBuffer, IInputBuffer2
{
return this.memory;
}
public T InvokeParse<T>(IGeneratedSerializer<T> serializer, int offset)
public T InvokeParse<T>(IGeneratedSerializer<T> serializer, in GeneratedSerializerParseArguments arguments)
{
return serializer.Parse(this, offset);
return serializer.Parse(this, arguments);
}
}

Просмотреть файл

@ -149,10 +149,9 @@ public struct ArraySegmentInputBuffer : IInputBuffer, IInputBuffer2
{
return this.pointer.segment;
}
public T InvokeParse<T>(IGeneratedSerializer<T> serializer, int offset)
public T InvokeParse<T>(IGeneratedSerializer<T> serializer, in GeneratedSerializerParseArguments arguments)
{
return serializer.Parse(this, offset);
return serializer.Parse(this, arguments);
}
// Array Segment is a relatively heavy struct. It contains an array pointer, an int offset, and and int length.

Просмотреть файл

@ -97,7 +97,7 @@ public interface IInputBuffer
/// Invokes the parse method on the <see cref="IGeneratedSerializer{T}"/> parameter. Allows passing
/// generic parameters.
/// </summary>
TItem InvokeParse<TItem>(IGeneratedSerializer<TItem> serializer, int offset);
TItem InvokeParse<TItem>(IGeneratedSerializer<TItem> serializer, in GeneratedSerializerParseArguments arguments);
}
/// <summary>

Просмотреть файл

@ -148,9 +148,9 @@ public struct MemoryInputBuffer : IInputBuffer, IInputBuffer2
return this.pointer.memory;
}
public T InvokeParse<T>(IGeneratedSerializer<T> serializer, int offset)
public T InvokeParse<T>(IGeneratedSerializer<T> serializer, in GeneratedSerializerParseArguments arguments)
{
return serializer.Parse(this, offset);
return serializer.Parse(this, arguments);
}
// Memory<byte> is a relatively heavy struct. It's cheaper to wrap it in a

Просмотреть файл

@ -151,9 +151,9 @@ public struct ReadOnlyMemoryInputBuffer : IInputBuffer, IInputBuffer2
throw new InvalidOperationException(ErrorMessage);
}
public T InvokeParse<T>(IGeneratedSerializer<T> serializer, int offset)
public T InvokeParse<T>(IGeneratedSerializer<T> serializer, in GeneratedSerializerParseArguments arguments)
{
return serializer.Parse(this, offset);
return serializer.Parse(this, arguments);
}
// Memory<byte> is a relatively heavy struct. It's cheaper to wrap it in a

Просмотреть файл

@ -16,7 +16,7 @@
using System.Threading;
namespace FlatSharp;
namespace FlatSharp.Internal;
/// <summary>
/// A context object for a FlatBuffer serialize operation. The context is responsible for allocating space in the buffer

Просмотреть файл

@ -17,7 +17,7 @@
using System.IO;
using System.Text;
namespace FlatSharp;
namespace FlatSharp.Internal;
/// <summary>
/// Collection of methods that help to serialize objects. It's kind of a hodge-podge,
@ -92,4 +92,19 @@ public static class SerializationHelpers
{
throw new InvalidDataException("FlatSharp encountered a null reference in an invalid context, such as a vector. Vectors are not permitted to have null objects.");
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void EnsureDepthLimit(short remainingDepth)
{
if (remainingDepth < 0)
{
ThrowDepthLimitExceededException();
}
}
[MethodImpl(MethodImplOptions.NoInlining)]
private static void ThrowDepthLimitExceededException()
{
throw new InvalidDataException($"FlatSharp passed the configured depth limit when deserializing. This can be configured with 'IGeneratedSerializer.WithSettings'.");
}
}

Просмотреть файл

@ -41,4 +41,15 @@ public class SerializerSettings
get;
set;
}
/// <summary>
/// When set, specifies a depth limit for nested objects. Enforced at deserialization time.
/// If set to <c>null</c>, a default value of <c>1000</c> will be used. This setting may be used to prevent
/// stack overflow errors and otherwise guard against malicious inputs.
/// </summary>
public short? ObjectDepthLimit
{
get;
set;
}
}

Просмотреть файл

@ -29,7 +29,8 @@ public abstract class FlatBufferVector<T, TInputBuffer> : FlatBufferVectorBase<T
TInputBuffer memory,
int offset,
int itemSize,
in TableFieldContext fieldContext) : base(memory, fieldContext)
short remainingDepth,
in TableFieldContext fieldContext) : base(memory, remainingDepth, fieldContext)
{
this.offset = offset;
this.itemSize = itemSize;
@ -75,10 +76,10 @@ public abstract class FlatBufferVector<T, TInputBuffer> : FlatBufferVectorBase<T
protected override void ParseItem(int index, out T item)
{
int offset = checked(this.offset + (this.itemSize * index));
this.ParseItem(this.memory, offset, this.fieldContext, out item);
this.ParseItem(this.memory, offset, base.remainingDepth, this.fieldContext, out item);
}
protected abstract void ParseItem(TInputBuffer buffer, int offset, TableFieldContext context, out T item);
protected abstract void ParseItem(TInputBuffer buffer, int offset, short remainingDepthLimit, TableFieldContext context, out T item);
protected abstract void WriteThrough(T item, Span<byte> data);
}

Просмотреть файл

@ -24,12 +24,15 @@ public abstract class FlatBufferVectorBase<T, TInputBuffer> : IList<T>, IReadOnl
{
protected readonly TInputBuffer memory;
protected readonly TableFieldContext fieldContext;
protected readonly short remainingDepth;
protected FlatBufferVectorBase(
TInputBuffer memory,
short remainingDepth,
TableFieldContext fieldContext)
{
this.memory = memory;
this.remainingDepth = remainingDepth;
this.fieldContext = fieldContext;
}

Просмотреть файл

@ -32,7 +32,8 @@ public abstract class FlatBufferVectorOfUnion<T, TInputBuffer> : FlatBufferVecto
TInputBuffer memory,
int discriminatorOffset,
int offsetVectorOffset,
in TableFieldContext fieldContext) : base(memory, fieldContext)
short remainingDepth,
in TableFieldContext fieldContext) : base(memory, remainingDepth, fieldContext)
{
uint discriminatorCount = memory.ReadUInt(discriminatorOffset);
uint offsetCount = memory.ReadUInt(offsetVectorOffset);
@ -58,6 +59,7 @@ public abstract class FlatBufferVectorOfUnion<T, TInputBuffer> : FlatBufferVecto
this.memory,
this.discriminatorVectorOffset + index,
this.offsetVectorOffset + (index * sizeof(int)),
base.remainingDepth,
this.fieldContext,
out item);
}
@ -67,6 +69,7 @@ public abstract class FlatBufferVectorOfUnion<T, TInputBuffer> : FlatBufferVecto
TInputBuffer buffer,
int discriminatorOffset,
int offsetOffset,
short objectDepth,
TableFieldContext fieldContext,
out T item);
}

Просмотреть файл

@ -31,6 +31,7 @@ internal static class FlatBufferVectorHelpers
InputBufferVariableName = "memory",
IsOffsetByRef = false,
TableFieldContextVariableName = "fieldContext",
RemainingDepthVariableName = "remainingDepth",
};
var serializeContext = parseContext.GetWriteThroughContext("data", "item", "0");
@ -48,13 +49,15 @@ internal static class FlatBufferVectorHelpers
{parseContext.InputBufferTypeName} memory,
int offset,
int itemSize,
{nameof(TableFieldContext)} fieldContext) : base(memory, offset, itemSize, fieldContext)
short remainingDepth,
{nameof(TableFieldContext)} fieldContext) : base(memory, offset, itemSize, remainingDepth, fieldContext)
{{
}}
protected override void ParseItem(
{parseContext.InputBufferTypeName} memory,
int offset,
short remainingDepth,
{nameof(TableFieldContext)} fieldContext,
out {itemType.GetGlobalCompilableTypeName()} item)
{{
@ -84,6 +87,7 @@ internal static class FlatBufferVectorHelpers
IsOffsetByRef = true,
TableFieldContextVariableName = "fieldContext",
OffsetVariableName = "temp",
RemainingDepthVariableName = "remainingDepth",
};
string classDef = $@"
@ -94,7 +98,8 @@ internal static class FlatBufferVectorHelpers
{context.InputBufferTypeName} memory,
int discriminatorOffset,
int offsetVectorOffset,
{nameof(TableFieldContext)} fieldContext) : base(memory, discriminatorOffset, offsetVectorOffset, fieldContext)
short remainingDepth,
{nameof(TableFieldContext)} fieldContext) : base(memory, discriminatorOffset, offsetVectorOffset, remainingDepth, fieldContext)
{{
}}
@ -102,6 +107,7 @@ internal static class FlatBufferVectorHelpers
{context.InputBufferTypeName} memory,
int discriminatorOffset,
int offsetOffset,
short remainingDepth,
{nameof(TableFieldContext)} {context.TableFieldContextVariableName},
out {typeModel.GetGlobalCompilableTypeName()} item)
{{

Просмотреть файл

@ -23,6 +23,7 @@ internal class DeserializeClassDefinition
protected const string InputBufferVariableName = "__buffer";
protected const string OffsetVariableName = "__offset";
protected const string VTableVariableName = "__vtable";
protected const string RemainingDepthVariableName = "__remainingDepth";
protected readonly ITypeModel typeModel;
protected readonly FlatBufferSerializerOptions options;
@ -37,6 +38,7 @@ internal class DeserializeClassDefinition
protected readonly MethodInfo? onDeserializeMethod;
protected readonly string vtableTypeName;
protected readonly string vtableAccessor;
protected readonly string remainingDepthAccessor;
private DeserializeClassDefinition(
string className,
@ -51,27 +53,35 @@ internal class DeserializeClassDefinition
this.vtableTypeName = GetVTableTypeName(maxVtableIndex);
this.onDeserializeMethod = onDeserializeMethod;
if (!this.options.GreedyDeserialize)
{
// maintain reference to buffer.
this.instanceFieldDefinitions[InputBufferVariableName] = $"private TInputBuffer {InputBufferVariableName};";
this.instanceFieldDefinitions[OffsetVariableName] = $"private int {OffsetVariableName};";
this.initializeStatements.Add($"this.{InputBufferVariableName} = buffer;");
this.initializeStatements.Add($"this.{OffsetVariableName} = offset;");
}
this.vtableAccessor = "default";
if (this.typeModel.SchemaType == FlatBufferSchemaType.Table)
if (this.options.GreedyDeserialize)
{
if (this.options.GreedyDeserialize)
this.remainingDepthAccessor = "remainingDepth";
if (this.typeModel.SchemaType == FlatBufferSchemaType.Table)
{
// Greedy tables decode a vtable in the constructor but don't stor eit.
// Greedy tables decode a vtable in the constructor but don't store it.
this.initializeStatements.Add($"{this.vtableTypeName}.Create<TInputBuffer>(buffer, offset, out var vtable);");
this.vtableAccessor = "vtable";
}
else
}
else
{
this.remainingDepthAccessor = $"this.{RemainingDepthVariableName}";
// maintain reference to buffer.
this.instanceFieldDefinitions[InputBufferVariableName] = $"private TInputBuffer {InputBufferVariableName};";
this.instanceFieldDefinitions[OffsetVariableName] = $"private int {OffsetVariableName};";
this.instanceFieldDefinitions[RemainingDepthVariableName] = $"private short {RemainingDepthVariableName};";
this.initializeStatements.Add($"this.{InputBufferVariableName} = buffer;");
this.initializeStatements.Add($"this.{OffsetVariableName} = offset;");
this.initializeStatements.Add($"{this.remainingDepthAccessor} = remainingDepth;");
if (this.typeModel.SchemaType == FlatBufferSchemaType.Table)
{
// non-greedy tables also carry a vtable.
// Non-greedy tables store the vtable.
this.vtableAccessor = $"this.{VTableVariableName}";
this.initializeStatements.Add($"{this.vtableTypeName}.Create<TInputBuffer>(buffer, offset, out {this.vtableAccessor});");
this.instanceFieldDefinitions[VTableVariableName] = $"private {this.vtableTypeName} {VTableVariableName};";
@ -139,6 +149,7 @@ internal class DeserializeClassDefinition
InputBufferTypeName = "TInputBuffer",
OffsetVariableName = "offset",
InputBufferVariableName = "buffer",
RemainingDepthVariableName = "remainingDepth",
};
string body = itemModel.CreateReadItemBody(
@ -151,7 +162,8 @@ internal class DeserializeClassDefinition
private static {typeName} {GetReadIndexMethodName(itemModel)}(
TInputBuffer buffer,
int offset,
{this.vtableTypeName} vtable)
{this.vtableTypeName} vtable,
short remainingDepth)
{{
{body}
}}");
@ -227,7 +239,7 @@ internal class DeserializeClassDefinition
if (this.options.GreedyDeserialize || !itemModel.IsVirtual)
{
this.initializeStatements.Add($"{assignment} = {GetReadIndexMethodName(itemModel)}(buffer, offset, {this.vtableAccessor});");
this.initializeStatements.Add($"{assignment} = {GetReadIndexMethodName(itemModel)}(buffer, offset, {this.vtableAccessor}, {this.remainingDepthAccessor});");
}
else if (!this.options.Lazy)
{
@ -271,7 +283,7 @@ internal class DeserializeClassDefinition
{string.Join("\r\n", this.instanceFieldDefinitions.Values)}
public static {this.ClassName}<TInputBuffer> GetOrCreate(TInputBuffer buffer, int offset)
public static {this.ClassName}<TInputBuffer> GetOrCreate(TInputBuffer buffer, int offset, short remainingDepth)
{{
{this.GetGetOrCreateMethodBody()}
}}
@ -325,7 +337,7 @@ internal class DeserializeClassDefinition
protected virtual string GetGetterBody(ItemMemberModel itemModel)
{
string readUnderlyingInvocation = $"{GetReadIndexMethodName(itemModel)}(this.{InputBufferVariableName}, this.{OffsetVariableName}, {this.vtableAccessor})";
string readUnderlyingInvocation = $"{GetReadIndexMethodName(itemModel)}(this.{InputBufferVariableName}, this.{OffsetVariableName}, {this.vtableAccessor}, {this.remainingDepthAccessor})";
if (this.options.GreedyDeserialize)
{
return $"return this.{GetFieldName(itemModel)};";
@ -350,7 +362,7 @@ internal class DeserializeClassDefinition
protected virtual string GetGetOrCreateMethodBody()
{
return $@"
var item = new {this.ClassName}<TInputBuffer>(buffer, offset);
var item = new {this.ClassName}<TInputBuffer>(buffer, offset, remainingDepth);
return item;
";
}
@ -358,7 +370,7 @@ internal class DeserializeClassDefinition
protected virtual string GetCtorMethodDefinition(string onDeserializedStatement, string baseCtorParams)
{
return $@"
private {this.ClassName}(TInputBuffer buffer, int offset) : base({baseCtorParams})
private {this.ClassName}(TInputBuffer buffer, int offset, short remainingDepth) : base({baseCtorParams})
{{
{string.Join("\r\n", this.initializeStatements)}
{onDeserializedStatement}

Просмотреть файл

@ -27,6 +27,7 @@ public record ParserCodeGenContext
public ParserCodeGenContext(
string inputBufferVariableName,
string offsetVariableName,
string remainingDepthVariableName,
string inputBufferTypeName,
bool isOffsetByRef,
string tableFieldContextVariableName,
@ -39,6 +40,7 @@ public record ParserCodeGenContext
this.InputBufferVariableName = inputBufferVariableName;
this.OffsetVariableName = offsetVariableName;
this.InputBufferTypeName = inputBufferTypeName;
this.RemainingDepthVariableName = remainingDepthVariableName;
this.MethodNameMap = methodNameMap;
this.SerializeMethodNameMap = serializeMethodNameMap;
this.IsOffsetByRef = isOffsetByRef;
@ -63,6 +65,11 @@ public record ParserCodeGenContext
/// </summary>
public string OffsetVariableName { get; init; }
/// <summary>
/// The name of the variable that tracks the remaining depth limit. Decremented down the stack.
/// </summary>
public string RemainingDepthVariableName { get; init; }
/// <summary>
/// Indicates if the offset variable is passed by reference.
/// </summary>
@ -112,6 +119,8 @@ public record ParserCodeGenContext
}
sb.Append(this.OffsetVariableName);
sb.Append(", ");
sb.Append(this.RemainingDepthVariableName);
if (typeModel.TableFieldContextRequirements.HasFlag(TableFieldContextRequirements.Parse))
{

Просмотреть файл

@ -173,11 +173,12 @@ $@"
this.Write<ISpanWriter>(default!, new byte[10], default!, default!, default!);
this.Write<SpanWriter>(default!, new byte[10], default!, default!, default!);
this.Parse<IInputBuffer>(default!, 0);
this.Parse<MemoryInputBuffer>(default!, 0);
this.Parse<ReadOnlyMemoryInputBuffer>(default!, 0);
this.Parse<ArrayInputBuffer>(default!, 0);
this.Parse<ArraySegmentInputBuffer>(default!, 0);
this.Parse<IInputBuffer>(default!, default);
this.Parse<IInputBuffer2>(default!, default);
this.Parse<MemoryInputBuffer>(default!, default);
this.Parse<ReadOnlyMemoryInputBuffer>(default!, default);
this.Parse<ArrayInputBuffer>(default!, default);
this.Parse<ArraySegmentInputBuffer>(default!, default);
throw new InvalidOperationException(""__AotHelper is not intended to be invoked"");
}}
@ -396,7 +397,7 @@ $@"
/// </summary>
private void DefineMethods(ITypeModel rootModel)
{
HashSet<Type> types = new HashSet<Type>();
HashSet<Type> types = new();
rootModel.TraverseObjectGraph(types);
foreach (var type in types)
@ -413,7 +414,7 @@ $@"
var rootModel = this.typeModelContainer.CreateTypeModel(typeof(TRoot));
// all type model types.
HashSet<Type> types = new HashSet<Type>();
HashSet<Type> types = new();
rootModel.TraverseObjectGraph(types);
foreach (var type in types.ToArray())
@ -485,10 +486,10 @@ $@"
{
string methodText =
$@"
public {CSharpHelpers.GetGlobalCompilableTypeName(rootType)} Parse<TInputBuffer>(TInputBuffer buffer, int offset)
public {CSharpHelpers.GetGlobalCompilableTypeName(rootType)} Parse<TInputBuffer>(TInputBuffer buffer, in {typeof(GeneratedSerializerParseArguments).GetGlobalCompilableTypeName()} args)
where TInputBuffer : IInputBuffer
{{
return {this.readMethods[rootType]}(buffer, offset);
return {this.readMethods[rootType]}(buffer, args.{nameof(GeneratedSerializerParseArguments.Offset)}, args.{nameof(GeneratedSerializerParseArguments.DepthLimit)});
}}
";
this.methodDeclarations.Add(CSharpSyntaxTree.ParseText(methodText, ParseOptions).GetRoot());
@ -497,6 +498,7 @@ $@"
private void ImplementMethods(ITypeModel rootTypeModel)
{
bool requiresDepthTracking = rootTypeModel.IsDeepEnoughToRequireDepthTracking();
List<(ITypeModel, TableFieldContext)> allContexts = rootTypeModel.GetAllTableFieldContexts();
Dictionary<ITypeModel, List<TableFieldContext>> allContextsMap = new();
@ -531,7 +533,7 @@ $@"
: string.Empty;
var maxSizeContext = new GetMaxSizeCodeGenContext("value", getMaxSizeFieldContextVariableName, this.maxSizeMethods, this.options, this.typeModelContainer, allContextsMap);
var parseContext = new ParserCodeGenContext("buffer", "offset", "TInputBuffer", isOffsetByRef, parseFieldContextVariableName, this.readMethods, this.writeMethods, this.options, this.typeModelContainer, allContextsMap);
var parseContext = new ParserCodeGenContext("buffer", "offset", "remainingDepth", "TInputBuffer", isOffsetByRef, parseFieldContextVariableName, this.readMethods, this.writeMethods, this.options, this.typeModelContainer, allContextsMap);
var serializeContext = new SerializationCodeGenContext("context", "span", "spanWriter", "value", "offset", serializeFieldContextVariableName, isOffsetByRef, this.writeMethods, this.typeModelContainer, this.options, allContextsMap);
var maxSizeMethod = typeModel.CreateGetMaxSizeMethodBody(maxSizeContext);
@ -539,7 +541,7 @@ $@"
var writeMethod = typeModel.CreateSerializeMethodBody(serializeContext);
this.GenerateGetMaxSizeMethod(typeModel, maxSizeMethod, maxSizeContext);
this.GenerateParseMethod(typeModel, parseMethod, parseContext);
this.GenerateParseMethod(requiresDepthTracking, typeModel, parseMethod, parseContext);
this.GenerateSerializeMethod(typeModel, writeMethod, serializeContext);
string? extraClasses = typeModel.CreateExtraClasses();
@ -615,7 +617,7 @@ $@"
this.AddMethod(method, declaration);
}
private void GenerateParseMethod(ITypeModel typeModel, CodeGeneratedMethod method, ParserCodeGenContext context)
private void GenerateParseMethod(bool requiresDepthTracking, ITypeModel typeModel, CodeGeneratedMethod method, ParserCodeGenContext context)
{
string tableFieldContextParameter = string.Empty;
if (typeModel.TableFieldContextRequirements.HasFlag(TableFieldContextRequirements.Parse))
@ -625,14 +627,26 @@ $@"
string clrType = typeModel.GetGlobalCompilableTypeName();
// If we require depth tracking due to the schema, inject the if statement and the decrement instruction.
string depthCheck = string.Empty;
if (requiresDepthTracking)
{
depthCheck = $@"
--{context.RemainingDepthVariableName};
{typeof(SerializationHelpers).GetGlobalCompilableTypeName()}.{nameof(SerializationHelpers.EnsureDepthLimit)}({context.RemainingDepthVariableName});
";
}
string declaration =
$@"
{method.GetMethodImplAttribute()}
private static {clrType} {this.readMethods[typeModel.ClrType]}<TInputBuffer>(
TInputBuffer {context.InputBufferVariableName},
{GetVTableOffsetVariableType(typeModel.PhysicalLayout.Length)} {context.OffsetVariableName}
{GetVTableOffsetVariableType(typeModel.PhysicalLayout.Length)} {context.OffsetVariableName},
short {context.RemainingDepthVariableName}
{tableFieldContextParameter}) where TInputBuffer : IInputBuffer
{{
{depthCheck}
{method.MethodBody}
}}";

Просмотреть файл

@ -14,6 +14,8 @@
* limitations under the License.
*/
using System.Linq;
namespace FlatSharp.TypeModel;
[Flags]
@ -190,16 +192,50 @@ internal static class ITypeModelExtensions
}
}
/// <summary>
/// Indicates if the given type model has enough recursive depth to require object depth tracking (ie, there
/// is a risk of stack overflow). This can be due to an excessively deep object graph or a cycle (we do not care which).
/// </summary>
public static bool IsDeepEnoughToRequireDepthTracking(this ITypeModel typeModel)
{
static bool Recurse(ITypeModel model, int depthRemaining)
{
if (depthRemaining <= 0)
{
return true;
}
foreach (var child in model.Children)
{
if (Recurse(child, depthRemaining - 1))
{
return true;
}
}
return false;
}
return Recurse(typeModel, 500);
}
/// <summary>
/// Recursively traverses the full object graph for the given type model.
/// </summary>
public static void TraverseObjectGraph(this ITypeModel model, HashSet<Type> seenTypes)
{
if (seenTypes.Add(model.ClrType))
Queue<ITypeModel> discoveryQueue = new();
discoveryQueue.Enqueue(model);
while (discoveryQueue.Count > 0)
{
foreach (var child in model.Children)
ITypeModel next = discoveryQueue.Dequeue();
if (seenTypes.Add(next.ClrType))
{
child.TraverseObjectGraph(seenTypes);
foreach (var child in next.Children)
{
discoveryQueue.Enqueue(child);
}
}
}
}

Просмотреть файл

@ -124,7 +124,7 @@ public class StructTypeModel : RuntimeTypeModel
classDef.AddProperty(value, context);
}
return new CodeGeneratedMethod($"return {className}<{context.InputBufferTypeName}>.GetOrCreate({context.InputBufferVariableName}, {context.OffsetVariableName});")
return new CodeGeneratedMethod($"return {className}<{context.InputBufferTypeName}>.GetOrCreate({context.InputBufferVariableName}, {context.OffsetVariableName}, {context.RemainingDepthVariableName});")
{
ClassDefinition = classDef.ToString(),
};

Просмотреть файл

@ -696,7 +696,7 @@ $@"
classDef.AddProperty(value, tempContext);
}
string body = $"return {this.tableReaderClassName}<{context.InputBufferTypeName}>.GetOrCreate({context.InputBufferVariableName}, {context.OffsetVariableName} + {context.InputBufferVariableName}.{nameof(InputBufferExtensions.ReadUOffset)}({context.OffsetVariableName}));";
string body = $"return {this.tableReaderClassName}<{context.InputBufferTypeName}>.GetOrCreate({context.InputBufferVariableName}, {context.OffsetVariableName} + {context.InputBufferVariableName}.{nameof(InputBufferExtensions.ReadUOffset)}({context.OffsetVariableName}), {context.RemainingDepthVariableName});";
return new CodeGeneratedMethod(body)
{
ClassDefinition = classDef.ToString(),

Просмотреть файл

@ -110,7 +110,8 @@ public class ValueStructTypeModel : RuntimeTypeModel
propertyStatements.Add($@"
item.{member.accessor} = {context.MethodNameMap[member.model.ClrType]}<{context.InputBufferTypeName}>(
{context.InputBufferVariableName},
{context.OffsetVariableName} + {member.offset});");
{context.OffsetVariableName} + {member.offset},
{context.RemainingDepthVariableName});");
}
string nonMarshalBody = $@"

Просмотреть файл

@ -93,6 +93,7 @@ public class ArrayVectorTypeModel : BaseVectorTypeModel
{context.InputBufferVariableName},
{context.OffsetVariableName} + {context.InputBufferVariableName}.{nameof(InputBufferExtensions.ReadUOffset)}({context.OffsetVariableName}),
{this.PaddedMemberInlineSize},
{context.RemainingDepthVariableName},
{context.TableFieldContextVariableName})";
body = $"return ({createFlatBufferVector}).ToArray();";

Просмотреть файл

@ -106,6 +106,7 @@ public class IndexedVectorTypeModel : BaseVectorTypeModel
{context.InputBufferVariableName},
{context.OffsetVariableName} + {context.InputBufferVariableName}.{nameof(InputBufferExtensions.ReadUOffset)}({context.OffsetVariableName}),
{this.PaddedMemberInlineSize},
{context.RemainingDepthVariableName},
{context.TableFieldContextVariableName})";
string mutable = context.Options.GenerateMutableObjects.ToString().ToLowerInvariant();

Просмотреть файл

@ -114,6 +114,7 @@ public class ListVectorTypeModel : BaseVectorTypeModel
{context.InputBufferVariableName},
{context.OffsetVariableName} + {context.InputBufferVariableName}.{nameof(InputBufferExtensions.ReadUOffset)}({context.OffsetVariableName}),
{this.PaddedMemberInlineSize},
{context.RemainingDepthVariableName},
{context.TableFieldContextVariableName})";
return new CodeGeneratedMethod(CreateParseBody(this.ItemTypeModel, createFlatBufferVector, context)) { ClassDefinition = vectorClassDef };

Просмотреть файл

@ -46,6 +46,7 @@ public class ArrayVectorOfUnionTypeModel : BaseVectorOfUnionTypeModel
{context.InputBufferVariableName},
{context.OffsetVariableName}.offset0 + {context.InputBufferVariableName}.{nameof(InputBufferExtensions.ReadUOffset)}({context.OffsetVariableName}.offset0),
{context.OffsetVariableName}.offset1 + {context.InputBufferVariableName}.{nameof(InputBufferExtensions.ReadUOffset)}({context.OffsetVariableName}.offset1),
{context.RemainingDepthVariableName},
{context.TableFieldContextVariableName})";
string body = $"return ({createFlatBufferVector}).ToArray();";

Просмотреть файл

@ -39,6 +39,7 @@ public class ListVectorOfUnionTypeModel : BaseVectorOfUnionTypeModel
{context.InputBufferVariableName},
{context.OffsetVariableName}.offset0 + {context.InputBufferVariableName}.{nameof(InputBufferExtensions.ReadUOffset)}({context.OffsetVariableName}.offset0),
{context.OffsetVariableName}.offset1 + {context.InputBufferVariableName}.{nameof(InputBufferExtensions.ReadUOffset)}({context.OffsetVariableName}.offset1),
{context.RemainingDepthVariableName},
{context.TableFieldContextVariableName})";
return new CodeGeneratedMethod(ListVectorTypeModel.CreateParseBody(

Просмотреть файл

@ -0,0 +1,124 @@
/*
* Copyright 2020 James Courtney
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
using FlatSharp.TypeModel;
using System.Linq;
using System.Text;
namespace FlatSharpTests.Compiler;
public class DepthLimitTests
{
[Fact]
public void TableReferencesSelf()
{
string fbs = $@"
{MetadataHelpers.AllAttributes}
namespace Foo.Bar;
table ListNode {{ Next : ListNode; }}
";
CompileAndVerify(fbs, "Foo.Bar.ListNode", true);
}
[Fact]
public void TableCycle()
{
string fbs = $@"
{MetadataHelpers.AllAttributes}
namespace Foo.Bar;
table A {{ Next : B; }}
table B {{ Next : C; }}
table C {{ Next : A; }}
";
CompileAndVerify(fbs, "Foo.Bar.A", true);
}
[Theory]
[InlineData(499, false)]
[InlineData(500, true)]
public void DeepTable_NoCycle(int depth, bool expectCycleTracking)
{
StringBuilder sb = new();
sb.Append($@"
{MetadataHelpers.AllAttributes}
namespace Foo.Bar;
table T0 {{ Value : int; }}
");
for (int i = 1; i < depth; ++i)
{
sb.AppendLine($"table T{i} {{ Previous : T{i - 1}; }}");
}
CompileAndVerify(sb.ToString(), $"Foo.Bar.T{depth - 1}", expectCycleTracking);
}
[Fact]
public void TableCycleWithListVector()
{
string fbs = $@"
{MetadataHelpers.AllAttributes}
namespace Foo.Bar;
table A {{ Next : B; }}
table B {{ Next : C; }}
table C {{ Next : [A] (fs_vector:""IList""); }}
";
CompileAndVerify(fbs, "Foo.Bar.A", true);
}
[Fact]
public void TableCycleWithArrayVector()
{
string fbs = $@"
{MetadataHelpers.AllAttributes}
namespace Foo.Bar;
table A {{ Next : B; }}
table B {{ Next : C; }}
table C {{ Next : [A] (fs_vector:""Array""); }}
";
CompileAndVerify(fbs, "Foo.Bar.A", true);
}
/*
[Fact]
public void TableCycleWithIndexedVector()
{
string fbs = $@"
{MetadataHelpers.AllAttributes}
namespace Foo.Bar;
table A {{ Next : B; Key : string (key); }}
table B {{ Next : C; }}
table C {{ Next : [A] (fs_vector:""IIndexedVector""); }}
";
CompileAndVerify(fbs, "Foo.Bar.A", true);
}
*/
private static void CompileAndVerify(string fbs, string typeName, bool needsTracking)
{
Assembly asm = FlatSharpCompiler.CompileAndLoadAssembly(fbs, new());
Type t = asm.GetTypes().Single(x => x.FullName == typeName);
Assert.NotNull(t);
ITypeModel typeModel = RuntimeTypeModel.CreateFrom(t);
Assert.Equal(needsTracking, typeModel.IsDeepEnoughToRequireDepthTracking());
}
}

Просмотреть файл

@ -421,9 +421,9 @@ public class InputBufferTests
return ((IInputBuffer)innerBuffer).GetReadOnlyByteMemory(start, length);
}
public TItem InvokeParse<TItem>(IGeneratedSerializer<TItem> serializer, int offset)
public TItem InvokeParse<TItem>(IGeneratedSerializer<TItem> serializer, in GeneratedSerializerParseArguments arguments)
{
return ((IInputBuffer)innerBuffer).InvokeParse<TItem>(serializer, offset);
return ((IInputBuffer)innerBuffer).InvokeParse<TItem>(serializer, arguments);
}
public byte ReadByte(int offset)

Просмотреть файл

@ -0,0 +1,80 @@
/*
* Copyright 2021 James Courtney
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
namespace FlatSharpTests;
public class DepthLimitTests
{
[Fact]
public void InvalidDepthLimit()
{
Assert.Throws<ArgumentException>(() =>
FlatBufferSerializer.Default
.Compile<LinkedListNode>()
.WithSettings(new SerializerSettings { ObjectDepthLimit = -1 }));
}
[Theory]
[InlineData(FlatBufferDeserializationOption.GreedyMutable, 1000, 999, false)]
[InlineData(FlatBufferDeserializationOption.GreedyMutable, 1000, 1000, true)]
[InlineData(FlatBufferDeserializationOption.Lazy, 1000, 999, false)]
[InlineData(FlatBufferDeserializationOption.Lazy, 1000, 1000, true)]
public void LinkedListDepth(FlatBufferDeserializationOption option, short limit, int nodes, bool expectException)
{
FlatBufferSerializer fbs = new FlatBufferSerializer(option);
ISerializer<LinkedListNode> serializer = fbs.Compile<LinkedListNode>().WithSettings(new SerializerSettings { ObjectDepthLimit = limit });
LinkedListNode head = new LinkedListNode();
LinkedListNode current = head;
for (int i = 0; i < nodes; ++i)
{
current.Next = new LinkedListNode();
current = current.Next;
}
byte[] buffer = new byte[serializer.GetMaxSize(head)];
serializer.Write(buffer, head);
Action callback = () =>
{
LinkedListNode node = serializer.Parse(buffer);
while (node != null)
{
node = node.Next;
}
};
if (expectException)
{
Assert.Throws<System.IO.InvalidDataException>(callback);
}
else
{
callback();
}
}
[FlatBufferTable]
public class LinkedListNode
{
[FlatBufferItem(0)]
public virtual LinkedListNode? Next { get; set; }
[FlatBufferItem(1)]
public virtual int Value { get; set; }
}
}

Просмотреть файл

@ -26,6 +26,7 @@ public static class ContextHelpers
return new ParserCodeGenContext(
"a",
"b",
"e",
"c",
false,
"d",

Просмотреть файл

@ -9,12 +9,12 @@
<PropertyGroup>
<GeneratePackageOnBuild>true</GeneratePackageOnBuild>
<Version>6.2.1</Version>
<PackageVersion>6.2.1</PackageVersion>
<Version>6.3.0</Version>
<PackageVersion>6.3.0</PackageVersion>
<AssemblyVersion>$(Version)</AssemblyVersion>
<Authors>James Courtney</Authors>
<Description>FlatSharp is a fast, idiomatic implementation of the FlatBuffer binary format.</Description>
<Copyright>2021</Copyright>
<Copyright>2022</Copyright>
<RepositoryUrl>https://github.com/jamescourtney/FlatSharp/</RepositoryUrl>
<PackageTags>flatbuffers serialization flatbuffer flatsharp</PackageTags>
<PackageReleaseNotes>Release notes at https://github.com/jamescourtney/FlatSharp/releases</PackageReleaseNotes>