From b500a02b1e323b2d337cc1859eddf662a0ebb84f Mon Sep 17 00:00:00 2001 From: Calum Grant Date: Tue, 13 Aug 2019 18:50:25 +0100 Subject: [PATCH] C#: Compare CIL entities directly by handle rather than by label. C#: Remove IDs from the CIL extractor and make consistent with C# extractor. C#: Fix method collisions. --- .../Semmle.Extraction.CIL/Context.cs | 14 +- .../Entities/Assembly.cs | 21 +- .../Entities/Attribute.cs | 12 +- .../Semmle.Extraction.CIL/Entities/Event.cs | 25 +- .../Semmle.Extraction.CIL/Entities/Field.cs | 47 +- .../Semmle.Extraction.CIL/Entities/File.cs | 18 +- .../Semmle.Extraction.CIL/Entities/Folder.cs | 18 +- .../Entities/LocalVariable.cs | 13 +- .../Semmle.Extraction.CIL/Entities/Method.cs | 107 ++- .../Entities/Namespace.cs | 45 +- .../Entities/Parameter.cs | 25 +- .../Entities/Property.cs | 36 +- .../Entities/SourceLocation.cs | 26 +- .../Semmle.Extraction.CIL/Entities/Type.cs | 620 +++++++++++------- .../ExtractionProduct.cs | 25 +- .../Semmle.Extraction.CIL/Factories.cs | 155 +++-- csharp/extractor/Semmle.Extraction.CIL/Id.cs | 211 +----- .../Semmle.Extraction.CIL.csproj | 6 +- .../Entities/Compilation.cs | 4 +- .../Entities/Method.cs | 2 +- .../Entities/Modifier.cs | 2 +- .../Entities/Types/Type.cs | 8 +- .../Entities/UsingDirective.cs | 1 - .../Semmle.Extraction.CSharp/Tuples.cs | 6 + .../Semmle.Extraction/CommentProcessing.cs | 2 +- csharp/extractor/Semmle.Extraction/Context.cs | 16 +- .../Semmle.Extraction/Entities/Assembly.cs | 2 +- csharp/extractor/Semmle.Extraction/Entity.cs | 12 +- csharp/extractor/Semmle.Extraction/Id.cs | 2 +- csharp/extractor/Semmle.Extraction/Options.cs | 3 + csharp/extractor/Semmle.Extraction/Symbol.cs | 8 +- .../Semmle.Extraction/TrapExtensions.cs | 6 + csharp/extractor/Semmle.Extraction/Tuple.cs | 8 +- .../Semmle.Util/IEnumerableExtensions.cs | 14 + 34 files changed, 878 insertions(+), 642 deletions(-) diff --git a/csharp/extractor/Semmle.Extraction.CIL/Context.cs b/csharp/extractor/Semmle.Extraction.CIL/Context.cs index dc2801e5bc7..0809ded9650 100644 --- a/csharp/extractor/Semmle.Extraction.CIL/Context.cs +++ b/csharp/extractor/Semmle.Extraction.CIL/Context.cs @@ -31,7 +31,7 @@ namespace Semmle.Extraction.CIL mdReader = peReader.GetMetadataReader(); TypeSignatureDecoder = new Entities.TypeSignatureDecoder(this); - globalNamespace = new Lazy(() => Populate(new Entities.Namespace(this, GetId(""), null))); + globalNamespace = new Lazy(() => Populate(new Entities.Namespace(this, "", null))); systemNamespace = new Lazy(() => Populate(new Entities.Namespace(this, "System"))); genericHandleFactory = new CachedFunction(CreateGenericHandle); namespaceFactory = new CachedFunction(n => CreateNamespace(mdReader.GetString(n))); @@ -42,9 +42,6 @@ namespace Semmle.Extraction.CIL defaultGenericContext = new EmptyContext(this); - var def = mdReader.GetAssemblyDefinition(); - AssemblyPrefix = GetId(def.Name) + "_" + def.Version.ToString() + "::"; - if (extractPdbs) { pdb = PDB.PdbReader.Create(assemblyPath, peReader); @@ -75,7 +72,14 @@ namespace Semmle.Extraction.CIL } } - public readonly Id AssemblyPrefix; + public void WriteAssemblyPrefix(TextWriter trapFile) + { + var def = mdReader.GetAssemblyDefinition(); + trapFile.Write(GetString(def.Name)); + trapFile.Write('_'); + trapFile.Write(def.Version.ToString()); + trapFile.Write("::"); + } public readonly Entities.TypeSignatureDecoder TypeSignatureDecoder; diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/Assembly.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/Assembly.cs index 7d88f34f153..255d37699f2 100644 --- a/csharp/extractor/Semmle.Extraction.CIL/Entities/Assembly.cs +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/Assembly.cs @@ -4,6 +4,7 @@ using System.Collections.Generic; using Semmle.Util.Logging; using System; using Semmle.Extraction.Entities; +using System.IO; namespace Semmle.Extraction.CIL.Entities { @@ -20,8 +21,6 @@ namespace Semmle.Extraction.CIL.Entities /// public class Assembly : LabelledEntity, IAssembly { - public override Id IdSuffix => suffix; - readonly File file; readonly AssemblyName assemblyName; @@ -38,12 +37,24 @@ namespace Semmle.Extraction.CIL.Entities if (!def.PublicKey.IsNil) assemblyName.SetPublicKey(cx.mdReader.GetBlobBytes(def.PublicKey)); - ShortId = cx.GetId(FullName) + "#file:///" + cx.assemblyPath.Replace("\\", "/"); - file = new File(cx, cx.assemblyPath); } - static readonly Id suffix = new StringId(";assembly"); + public override void WriteId(TextWriter trapFile) + { + trapFile.Write(FullName); + trapFile.Write("#file:///"); + trapFile.Write(cx.assemblyPath.Replace("\\", "/")); + } + + public override bool Equals(object obj) + { + return GetType() == obj.GetType() && Equals(file, ((Assembly)obj).file); + } + + public override int GetHashCode() => 7 * file.GetHashCode(); + + public override string IdSuffix => ";assembly"; string FullName => assemblyName.GetPublicKey() is null ? assemblyName.FullName + ", PublicKeyToken=null" : assemblyName.FullName; diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/Attribute.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/Attribute.cs index 774de6cf145..a9ba1a845b8 100644 --- a/csharp/extractor/Semmle.Extraction.CIL/Entities/Attribute.cs +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/Attribute.cs @@ -14,8 +14,9 @@ namespace Semmle.Extraction.CIL.Entities /// /// Entity representing a CIL attribute. /// - class Attribute : UnlabelledEntity, IAttribute + sealed class Attribute : UnlabelledEntity, IAttribute { + readonly CustomAttributeHandle handle; readonly CustomAttribute attrib; readonly IEntity @object; @@ -25,6 +26,13 @@ namespace Semmle.Extraction.CIL.Entities this.@object = @object; } + public override bool Equals(object obj) + { + return obj is Attribute attribute && handle.Equals(attribute.handle); + } + + public override int GetHashCode() => handle.GetHashCode(); + public override IEnumerable Contents { get @@ -78,7 +86,7 @@ namespace Semmle.Extraction.CIL.Entities readonly Context cx; public CustomAttributeDecoder(Context cx) { this.cx = cx; } - public Type GetPrimitiveType(PrimitiveTypeCode typeCode) => cx.Populate(new PrimitiveType(cx, typeCode)); + public Type GetPrimitiveType(PrimitiveTypeCode typeCode) => cx.Create(typeCode); public Type GetSystemType() => throw new NotImplementedException(); diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/Event.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/Event.cs index 009afc1b1e9..8f5554dd300 100644 --- a/csharp/extractor/Semmle.Extraction.CIL/Entities/Event.cs +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/Event.cs @@ -1,4 +1,5 @@ using System.Collections.Generic; +using System.IO; using System.Reflection.Metadata; namespace Semmle.Extraction.CIL.Entities @@ -13,19 +14,35 @@ namespace Semmle.Extraction.CIL.Entities /// /// An event entity. /// - class Event : LabelledEntity, IEvent + sealed class Event : LabelledEntity, IEvent { + readonly EventDefinitionHandle handle; readonly Type parent; readonly EventDefinition ed; - static readonly Id suffix = CIL.Id.Create(";cil-event"); public Event(Context cx, Type parent, EventDefinitionHandle handle) : base(cx) { + this.handle = handle; this.parent = parent; ed = cx.mdReader.GetEventDefinition(handle); - ShortId = parent.ShortId + cx.Dot + cx.ShortName(ed.Name) + suffix; } + public override void WriteId(TextWriter trapFile) + { + parent.WriteId(trapFile); + trapFile.Write('.'); + trapFile.Write(cx.ShortName(ed.Name)); + } + + public override string IdSuffix => ";cil-event"; + + public override bool Equals(object obj) + { + return obj is Event e && handle.Equals(e.handle); + } + + public override int GetHashCode() => handle.GetHashCode(); + public override IEnumerable Contents { get @@ -61,7 +78,5 @@ namespace Semmle.Extraction.CIL.Entities yield return c; } } - - public override Id IdSuffix => suffix; } } diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/Field.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/Field.cs index e3789d3d547..d77ff955e63 100644 --- a/csharp/extractor/Semmle.Extraction.CIL/Entities/Field.cs +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/Field.cs @@ -37,28 +37,24 @@ namespace Semmle.Extraction.CIL.Entities public Label Label { get; set; } - public IId Id => ShortId + IdSuffix; - public void WriteId(TextWriter trapFile) { - trapFile.WriteIId(Id); + trapFile.WriteSubId(DeclaringType); + trapFile.Write('.'); + trapFile.Write(Name); } public void WriteQuotedId(TextWriter trapFile) { + trapFile.Write("@\""); WriteId(trapFile); + trapFile.Write(IdSuffix); + trapFile.Write('\"'); } - public Id IdSuffix => fieldSuffix; + public string IdSuffix => ";cil-field"; - static readonly StringId fieldSuffix = new StringId(";cil-field"); - - public Id ShortId - { - get; set; - } - - public abstract Id Name { get; } + public abstract string Name { get; } public abstract Type DeclaringType { get; } @@ -70,7 +66,7 @@ namespace Semmle.Extraction.CIL.Entities { get { - yield return Tuples.cil_field(this, DeclaringType, Name.Value, Type); + yield return Tuples.cil_field(this, DeclaringType, Name, Type); } } @@ -93,9 +89,15 @@ namespace Semmle.Extraction.CIL.Entities this.handle = handle; this.gc = gc; fd = cx.mdReader.GetFieldDefinition(handle); - ShortId = DeclaringType.ShortId + cx.Dot + Name; } + public override bool Equals(object obj) + { + return obj is DefinitionField field && handle.Equals(field.handle); + } + + public override int GetHashCode() => handle.GetHashCode(); + public override IEnumerable Contents { get @@ -125,7 +127,7 @@ namespace Semmle.Extraction.CIL.Entities } } - public override Id Name => cx.GetId(fd.Name); + public override string Name => cx.GetString(fd.Name); public override Type DeclaringType => (Type)cx.Create(fd.GetDeclaringType()); @@ -138,19 +140,30 @@ namespace Semmle.Extraction.CIL.Entities sealed class MemberReferenceField : Field { + readonly MemberReferenceHandle Handle; readonly MemberReference mr; readonly GenericContext gc; readonly Type declType; public MemberReferenceField(GenericContext gc, MemberReferenceHandle handle) : base(gc.cx) { + Handle = handle; this.gc = gc; mr = cx.mdReader.GetMemberReference(handle); declType = (Type)cx.CreateGeneric(gc, mr.Parent); - ShortId = declType.ShortId + cx.Dot + Name; } - public override Id Name => cx.GetId(mr.Name); + public override bool Equals(object obj) + { + return obj is MemberReferenceField field && Handle.Equals(field.Handle); + } + + public override int GetHashCode() + { + return Handle.GetHashCode(); + } + + public override string Name => cx.GetString(mr.Name); public override Type DeclaringType => declType; diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/File.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/File.cs index 185d12df294..bc8c4c8c76d 100644 --- a/csharp/extractor/Semmle.Extraction.CIL/Entities/File.cs +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/File.cs @@ -1,4 +1,5 @@ using System.Collections.Generic; +using System.IO; namespace Semmle.Extraction.CIL.Entities { @@ -17,9 +18,20 @@ namespace Semmle.Extraction.CIL.Entities public File(Context cx, string path) : base(cx) { this.path = Semmle.Extraction.Entities.File.PathAsDatabaseString(path); - ShortId = new StringId(Semmle.Extraction.Entities.File.PathAsDatabaseId(path)); } + public override void WriteId(TextWriter trapFile) + { + trapFile.Write(Semmle.Extraction.Entities.File.PathAsDatabaseId(path)); + } + + public override bool Equals(object obj) + { + return GetType() == obj.GetType() && path == ((File)obj).path; + } + + public override int GetHashCode() => 11 * path.GetHashCode(); + public override IEnumerable Contents { get @@ -31,9 +43,7 @@ namespace Semmle.Extraction.CIL.Entities } } - public override Id IdSuffix => suffix; - - static readonly Id suffix = new StringId(";sourcefile"); + public override string IdSuffix => ";sourcefile"; } public class PdbSourceFile : File diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/Folder.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/Folder.cs index 5a7f7bd83bb..1e7a594c593 100644 --- a/csharp/extractor/Semmle.Extraction.CIL/Entities/Folder.cs +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/Folder.cs @@ -7,17 +7,22 @@ namespace Semmle.Extraction.CIL.Entities { } - public class Folder : LabelledEntity, IFolder + public sealed class Folder : LabelledEntity, IFolder { readonly string path; public Folder(Context cx, string path) : base(cx) { this.path = path; - ShortId = new StringId(Semmle.Extraction.Entities.File.PathAsDatabaseId(path)); } - static readonly Id suffix = new StringId(";folder"); + + public override void WriteId(TextWriter trapFile) + { + trapFile.Write(Semmle.Extraction.Entities.File.PathAsDatabaseId(path)); + } + + public override string IdSuffix => ";folder"; public override IEnumerable Contents { @@ -37,6 +42,11 @@ namespace Semmle.Extraction.CIL.Entities } } - public override Id IdSuffix => suffix; + public override bool Equals(object obj) + { + return obj is Folder folder && path == folder.path; + } + + public override int GetHashCode() => path.GetHashCode(); } } diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/LocalVariable.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/LocalVariable.cs index 4e3fbf43662..42760c2362c 100644 --- a/csharp/extractor/Semmle.Extraction.CIL/Entities/LocalVariable.cs +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/LocalVariable.cs @@ -1,4 +1,5 @@ using System.Collections.Generic; +using System.IO; namespace Semmle.Extraction.CIL.Entities { @@ -17,12 +18,16 @@ namespace Semmle.Extraction.CIL.Entities method = m; index = i; type = t; - ShortId = CIL.Id.Create(method.Label) + underscore + index; } - static readonly Id underscore = CIL.Id.Create("_"); - static readonly Id suffix = CIL.Id.Create(";cil-local"); - public override Id IdSuffix => suffix; + public override void WriteId(TextWriter trapFile) + { + trapFile.WriteSubId(method); + trapFile.Write('_'); + trapFile.Write(index); + } + + public override string IdSuffix => ";cil-local"; public override IEnumerable Contents { diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/Method.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/Method.cs index 90311af8762..a399f7ecfe6 100644 --- a/csharp/extractor/Semmle.Extraction.CIL/Entities/Method.cs +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/Method.cs @@ -7,6 +7,7 @@ using System.Reflection; using System.Linq; using System.Reflection.Metadata.Ecma335; using System.IO; +using Semmle.Util; namespace Semmle.Extraction.CIL.Entities { @@ -42,34 +43,40 @@ namespace Semmle.Extraction.CIL.Entities public virtual IList LocalVariables => throw new NotImplementedException(); public IList Parameters { get; private set; } - static readonly Id tick = CIL.Id.Create("`"); - static readonly Id space = CIL.Id.Create(" "); - static readonly Id dot = CIL.Id.Create("."); - static readonly Id open = CIL.Id.Create("("); - static readonly Id close = CIL.Id.Create(")"); + public override void WriteId(TextWriter trapFile) => MakeMethodId(trapFile, DeclaringType, NameLabel); - internal protected Id MakeMethodId(Type parent, Id methodName) + public abstract string NameLabel { get; } + + internal protected void MakeMethodId(TextWriter trapFile, Type parent, string methodName) { - var id = signature.ReturnType.MakeId(this) + space + parent.ShortId + dot + methodName; + signature.ReturnType.WriteId(trapFile, this); + trapFile.Write(' '); + parent.WriteId(trapFile); + trapFile.Write('.'); + trapFile.Write(methodName); if (signature.GenericParameterCount > 0) { - id += tick + signature.GenericParameterCount; + trapFile.Write('`'); + trapFile.Write(signature.GenericParameterCount); } - - id += open + CIL.Id.CommaSeparatedList(signature.ParameterTypes.Select(p => p.MakeId(this))) + close; - return id; + trapFile.Write('('); + int index = 0; + foreach (var param in signature.ParameterTypes) + { + trapFile.WriteSeparator(",", index++); + param.WriteId(trapFile, this); + } + trapFile.Write(')'); } protected MethodTypeParameter[] genericParams; protected Type declaringType; protected GenericContext gc; protected MethodSignature signature; - protected Id name; + protected string name; - static readonly StringId methodSuffix = new StringId(";cil-method"); - - public override Id IdSuffix => methodSuffix; + public override string IdSuffix => ";cil-method"; protected void PopulateParameters(IEnumerable parameterTypes) { @@ -134,7 +141,7 @@ namespace Semmle.Extraction.CIL.Entities /// /// A definition method - a method defined in the current assembly. /// - class DefinitionMethod : Method, IMember + sealed class DefinitionMethod : Method, IMember { readonly Handle handle; readonly MethodDefinition md; @@ -151,22 +158,30 @@ namespace Semmle.Extraction.CIL.Entities md = cx.mdReader.GetMethodDefinition(handle); this.gc = gc; this.handle = handle; - name = cx.GetId(md.Name); + name = cx.GetString(md.Name); declaringType = (Type)cx.CreateGeneric(this, md.GetDeclaringType()); signature = md.DecodeSignature(new SignatureDecoder(), this); - ShortId = MakeMethodId(declaringType, name); methodDebugInformation = cx.GetMethodDebugInformation(handle); } + public override bool Equals(object obj) + { + return obj is DefinitionMethod method && handle.Equals(method.handle); + } + + public override int GetHashCode() => handle.GetHashCode(); + public override bool IsStatic => !signature.Header.IsInstance; public override Type DeclaringType => declaringType; public override string Name => cx.ShortName(md.Name); + public override string NameLabel => name; + /// /// Holds if this method has bytecode. /// @@ -382,8 +397,9 @@ namespace Semmle.Extraction.CIL.Entities /// /// This is a late-bound reference to a method. /// - class MemberReferenceMethod : Method + sealed class MemberReferenceMethod : Method { + readonly MemberReferenceHandle handle; readonly MemberReference mr; readonly Type declType; readonly GenericContext parent; @@ -391,6 +407,7 @@ namespace Semmle.Extraction.CIL.Entities public MemberReferenceMethod(GenericContext gc, MemberReferenceHandle handle) : base(gc) { + this.handle = handle; this.gc = gc; mr = cx.mdReader.GetMemberReference(handle); @@ -399,19 +416,31 @@ namespace Semmle.Extraction.CIL.Entities parent = (GenericContext)cx.CreateGeneric(gc, mr.Parent); var parentMethod = parent as Method; - var nameLabel = cx.GetId(mr.Name); + nameLabel = cx.GetString(mr.Name); declType = parentMethod == null ? parent as Type : parentMethod.DeclaringType; if (declType is null) throw new InternalError("Parent context of method is not a type"); - ShortId = MakeMethodId(declType, nameLabel); - var typeSourceDeclaration = declType.SourceDeclaration; sourceDeclaration = typeSourceDeclaration == declType ? (Method)this : typeSourceDeclaration.LookupMethod(mr.Name, mr.Signature); } + private readonly string nameLabel; + + public override string NameLabel => nameLabel; + + public override bool Equals(object obj) + { + return obj is MemberReferenceMethod method && handle.Equals(method.handle); + } + + public override int GetHashCode() + { + return handle.GetHashCode(); + } + public override Method SourceDeclaration => sourceDeclaration; public override bool IsStatic => !signature.Header.IsInstance; @@ -451,26 +480,46 @@ namespace Semmle.Extraction.CIL.Entities /// /// A constructed method. /// - class MethodSpecificationMethod : Method + sealed class MethodSpecificationMethod : Method { + readonly MethodSpecificationHandle handle; readonly MethodSpecification ms; readonly Method unboundMethod; readonly ImmutableArray typeParams; public MethodSpecificationMethod(GenericContext gc, MethodSpecificationHandle handle) : base(gc) { + this.handle = handle; ms = cx.mdReader.GetMethodSpecification(handle); - typeParams = ms.DecodeSignature(cx.TypeSignatureDecoder, gc); - unboundMethod = (Method)cx.CreateGeneric(gc, ms.Method); declaringType = unboundMethod.DeclaringType; - - ShortId = unboundMethod.ShortId + openAngle + CIL.Id.CommaSeparatedList(typeParams.Select(p => p.ShortId)) + closeAngle; } - static readonly Id openAngle = CIL.Id.Create("<"); - static readonly Id closeAngle = CIL.Id.Create(">"); + public override void WriteId(TextWriter trapFile) + { + unboundMethod.WriteId(trapFile); + trapFile.Write('<'); + int index = 0; + foreach(var param in typeParams) + { + trapFile.WriteSeparator(",", index++); + trapFile.WriteSubId(param); + } + trapFile.Write('>'); + } + + public override string NameLabel => throw new NotImplementedException(); + + public override bool Equals(object obj) + { + return obj is MethodSpecificationMethod method && handle.Equals(method.handle) && typeParams.SequenceEqual(method.typeParams); + } + + public override int GetHashCode() + { + return handle.GetHashCode() * 11 + typeParams.SequenceHash(); + } public override Method SourceDeclaration => unboundMethod; diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/Namespace.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/Namespace.cs index e34d68554e7..6a7943c8bd1 100644 --- a/csharp/extractor/Semmle.Extraction.CIL/Entities/Namespace.cs +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/Namespace.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.IO; using Semmle.Extraction.Entities; namespace Semmle.Extraction.CIL.Entities @@ -14,28 +15,43 @@ namespace Semmle.Extraction.CIL.Entities /// /// A namespace. /// - public class Namespace : TypeContainer, INamespace + public sealed class Namespace : TypeContainer, INamespace { public Namespace ParentNamespace; - public readonly StringId Name; + public readonly string Name; public bool IsGlobalNamespace => ParentNamespace == null; - static readonly Id suffix = CIL.Id.Create(";namespace"); + public override string IdSuffix => ";namespace"; - public Id CreateId + + public override void WriteId(TextWriter trapFile) { - get + if (ParentNamespace != null && !ParentNamespace.IsGlobalNamespace) { - if (ParentNamespace != null && !ParentNamespace.IsGlobalNamespace) - { - return ParentNamespace.ShortId + cx.Dot + Name; - } - return Name; + ParentNamespace.WriteId(trapFile); + trapFile.Write('.'); } + trapFile.Write(Name); } - public override Id IdSuffix => suffix; + public override bool Equals(object obj) + { + if (obj is Namespace ns && Name == ns.Name) + { + if (ParentNamespace is null) + return ns.ParentNamespace is null; + if (!(ns.ParentNamespace is null)) + return ParentNamespace.Equals(ns.ParentNamespace); + } + return false; + } + + public override int GetHashCode() + { + int h = ParentNamespace is null ? 19 : ParentNamespace.GetHashCode(); + return 13 * h + Name.GetHashCode(); + } public override IEnumerable TypeParameters => throw new NotImplementedException(); @@ -54,22 +70,21 @@ namespace Semmle.Extraction.CIL.Entities return i == -1 ? cx.GlobalNamespace : cx.Populate(new Namespace(cx, fqn.Substring(0, i))); } - public Namespace(Context cx, string fqn) : this(cx, cx.GetId(parseNamespaceName(fqn)), createParentNamespace(cx, fqn)) + public Namespace(Context cx, string fqn) : this(cx, parseNamespaceName(fqn), createParentNamespace(cx, fqn)) { } - public Namespace(Context cx, StringId name, Namespace parent) : base(cx) + public Namespace(Context cx, string name, Namespace parent) : base(cx) { Name = name; ParentNamespace = parent; - ShortId = CreateId; } public override IEnumerable Contents { get { - yield return Tuples.namespaces(this, Name.Value); + yield return Tuples.namespaces(this, Name); if (!IsGlobalNamespace) yield return Tuples.parent_namespace(this, ParentNamespace); } diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/Parameter.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/Parameter.cs index b008ccc2510..9510ee40b57 100644 --- a/csharp/extractor/Semmle.Extraction.CIL/Entities/Parameter.cs +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/Parameter.cs @@ -1,4 +1,5 @@ using System.Collections.Generic; +using System.IO; namespace Semmle.Extraction.CIL.Entities { @@ -12,7 +13,7 @@ namespace Semmle.Extraction.CIL.Entities /// /// A parameter entity. /// - class Parameter : LabelledEntity, IParameter + sealed class Parameter : LabelledEntity, IParameter { readonly Method method; readonly int index; @@ -23,14 +24,26 @@ namespace Semmle.Extraction.CIL.Entities method = m; index = i; type = t; - ShortId = openCurly + method.Label.Value + closeCurly + index; } - static readonly Id parameterSuffix = CIL.Id.Create(";cil-parameter"); - static readonly Id openCurly = CIL.Id.Create("{#"); - static readonly Id closeCurly = CIL.Id.Create("}_"); + public override void WriteId(TextWriter trapFile) + { + trapFile.WriteSubId(method); + trapFile.Write('_'); + trapFile.Write(index); + } - public override Id IdSuffix => parameterSuffix; + public override bool Equals(object obj) + { + return obj is Parameter param && method.Equals(param.method) && index == param.index; + } + + public override int GetHashCode() + { + return 23 * method.GetHashCode() + index; + } + + public override string IdSuffix => ";cil-parameter"; public override IEnumerable Contents { diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/Property.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/Property.cs index 501031ccd95..a7d25dce750 100644 --- a/csharp/extractor/Semmle.Extraction.CIL/Entities/Property.cs +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/Property.cs @@ -2,6 +2,7 @@ using System.Reflection.Metadata; using System.Linq; using System.Reflection.Metadata.Ecma335; +using System.IO; namespace Semmle.Extraction.CIL.Entities { @@ -15,25 +16,44 @@ namespace Semmle.Extraction.CIL.Entities /// /// A property. /// - class Property : LabelledEntity, IProperty + sealed class Property : LabelledEntity, IProperty { readonly Handle handle; readonly Type type; readonly PropertyDefinition pd; - static readonly Id suffix = CIL.Id.Create(";cil-property"); + public override string IdSuffix => ";cil-property"; + readonly GenericContext gc; public Property(GenericContext gc, Type type, PropertyDefinitionHandle handle) : base(gc.cx) { + this.gc = gc; this.handle = handle; pd = cx.mdReader.GetPropertyDefinition(handle); this.type = type; - - var id = type.ShortId + gc.cx.Dot + cx.ShortName(pd.Name); - var signature = pd.DecodeSignature(new SignatureDecoder(), gc); - id += "(" + CIL.Id.CommaSeparatedList(signature.ParameterTypes.Select(p => p.MakeId(gc))) + ")"; - ShortId = id; } + public override void WriteId(TextWriter trapFile) + { + trapFile.WriteSubId(type); + trapFile.Write('.'); + trapFile.Write(cx.GetString(pd.Name)); + trapFile.Write("("); + int index=0; + var signature = pd.DecodeSignature(new SignatureDecoder(), gc); + foreach (var param in signature.ParameterTypes) + { + trapFile.WriteSeparator(",", index++); + param.WriteId(trapFile, gc); + } + trapFile.Write(")"); + } + public override bool Equals(object obj) + { + return obj is Property property && Equals(handle, property.handle); + } + + public override int GetHashCode() => handle.GetHashCode(); + public override IEnumerable Contents { get @@ -62,7 +82,5 @@ namespace Semmle.Extraction.CIL.Entities yield return c; } } - - public override Id IdSuffix => suffix; } } diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/SourceLocation.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/SourceLocation.cs index 88592789099..ade644fbb1c 100644 --- a/csharp/extractor/Semmle.Extraction.CIL/Entities/SourceLocation.cs +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/SourceLocation.cs @@ -1,4 +1,5 @@ using System.Collections.Generic; +using System.IO; using Semmle.Extraction.PDB; namespace Semmle.Extraction.CIL.Entities @@ -16,12 +17,27 @@ namespace Semmle.Extraction.CIL.Entities { this.location = location; file = cx.CreateSourceFile(location.File); - - ShortId = file.ShortId + separator + new IntId(location.StartLine) + separator + new IntId(location.StartColumn) + separator + new IntId(location.EndLine) + separator + new IntId(location.EndColumn); } - static readonly Id suffix = new StringId(";sourcelocation"); - static readonly Id separator = new StringId(","); + public override void WriteId(TextWriter trapFile) + { + file.WriteId(trapFile); + trapFile.Write(','); + trapFile.Write(location.StartLine); + trapFile.Write(','); + trapFile.Write(location.StartColumn); + trapFile.Write(','); + trapFile.Write(location.EndLine); + trapFile.Write(','); + trapFile.Write(location.EndColumn); + } + + public override bool Equals(object obj) + { + return obj is PdbSourceLocation l && location.Equals(l.location); + } + + public override int GetHashCode() => location.GetHashCode(); public override IEnumerable Contents { @@ -32,6 +48,6 @@ namespace Semmle.Extraction.CIL.Entities } } - public override Id IdSuffix => suffix; + public override string IdSuffix => ";sourcelocation"; } } diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/Type.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/Type.cs index 375b2a5cbb8..5766395337b 100644 --- a/csharp/extractor/Semmle.Extraction.CIL/Entities/Type.cs +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/Type.cs @@ -55,13 +55,7 @@ namespace Semmle.Extraction.CIL.Entities public virtual Label Label { get; set; } - public virtual IId Id { get { return ShortId + IdSuffix; } } - - public void WriteId(System.IO.TextWriter trapFile) - { - // TODO: Specialise this - trapFile.WriteIId(Id); - } + public abstract void WriteId(TextWriter trapFile); /// /// For debugging purposes. @@ -70,11 +64,13 @@ namespace Semmle.Extraction.CIL.Entities public void WriteQuotedId(TextWriter trapFile) { + trapFile.Write("@\""); WriteId(trapFile); + trapFile.Write(IdSuffix); + trapFile.Write('\"'); } - public Id ShortId { get; set; } - public abstract Id IdSuffix { get; } + public abstract string IdSuffix { get; } Location IEntity.ReportingLocation => throw new NotImplementedException(); @@ -84,7 +80,11 @@ namespace Semmle.Extraction.CIL.Entities public override string ToString() { - return Id.ToString(); + using (var writer = new StringWriter()) + { + WriteQuotedId(writer); + return writer.ToString(); + } } TrapStackBehaviour IEntity.TrapStackBehaviour => TrapStackBehaviour.NoLabel; @@ -95,8 +95,9 @@ namespace Semmle.Extraction.CIL.Entities /// public abstract class Type : TypeContainer, IMember, IType { - static readonly Id suffix = CIL.Id.Create(";cil-type"); - public override Id IdSuffix => suffix; + public override string IdSuffix => ";cil-type"; + + public sealed override void WriteId(TextWriter trapFile) => WriteId(trapFile, false); /// /// Find the method in this type matching the name and signature. @@ -137,7 +138,7 @@ namespace Semmle.Extraction.CIL.Entities /// /// Gets the assembly identifier of this type. /// - public abstract Id AssemblyPrefix { get; } + public abstract void WriteAssemblyPrefix(TextWriter trapFile); /// /// Gets the ID part to be used in a method. @@ -147,11 +148,14 @@ namespace Semmle.Extraction.CIL.Entities /// (This is to avoid infinite recursion generating a method ID that returns a /// type parameter.) /// - public abstract Id MakeId(bool inContext); + public abstract void WriteId(TextWriter trapFile, bool inContext); - public Id GetId(bool inContext) + public void GetId(TextWriter trapFile, bool inContext) { - return inContext ? MakeId(true) : ShortId; + if (inContext) + WriteId(trapFile, true); + else + WriteId(trapFile); } protected Type(Context cx) : base(cx) { } @@ -167,7 +171,7 @@ namespace Semmle.Extraction.CIL.Entities { get { - yield return Tuples.cil_type(this, Name.Value, Kind, Parent, SourceDeclaration); + yield return Tuples.cil_type(this, Name, Kind, Parent, SourceDeclaration); } } @@ -176,7 +180,7 @@ namespace Semmle.Extraction.CIL.Entities /// public virtual bool IsVisible => true; - public abstract Id Name { get; } + public abstract string Name { get; } public abstract Namespace Namespace { get; } @@ -223,42 +227,103 @@ namespace Semmle.Extraction.CIL.Entities public virtual Type SourceDeclaration => this; - protected static readonly Id builtin = CIL.Id.Create("builtin:"); - - public Id PrimitiveTypeId => builtin + Name; - - public bool IsPrimitiveType + public void PrimitiveTypeId(TextWriter trapFile) { - get + trapFile.Write("builtin:"); + trapFile.Write(Name); + } + + /// + /// Gets the primitive type corresponding to this type, if possible. + /// + /// Extraction context. + /// The resulting primitive type. + /// True if this type is a primitive type. + public bool TryGetPrimitiveType(Context cx, out PrimitiveType t) + { + if (TryGetPrimitiveTypeCode(out var code)) { - if (ContainingType == null && Namespace.ShortId == cx.SystemNamespace.ShortId) - { - switch (Name.Value) - { - case "Boolean": - case "Object": - case "Byte": - case "SByte": - case "Int16": - case "UInt16": - case "Int32": - case "UInt32": - case "Int64": - case "UInt64": - case "Single": - case "Double": - case "String": - case "Void": - case "IntPtr": - case "UIntPtr": - case "Char": - case "TypedReference": - return true; - } - } + t = cx.Create(code); + return true; + } + else + { + t = null; return false; } } + + public bool TryGetPrimitiveTypeCode(out PrimitiveTypeCode code) + { + if (ContainingType == null && Namespace.Name == cx.SystemNamespace.Name) + { + switch (Name) + { + case "Boolean": + code = PrimitiveTypeCode.Boolean; + return true; + case "Object": + code = PrimitiveTypeCode.Object; + return true; + case "Byte": + code = PrimitiveTypeCode.Byte; + return true; + case "SByte": + code = PrimitiveTypeCode.SByte; + return true; + case "Int16": + code = PrimitiveTypeCode.Int16; + return true; + case "UInt16": + code = PrimitiveTypeCode.UInt16; + return true; + case "Int32": + code = PrimitiveTypeCode.Int32; + return true; + case "UInt32": + code = PrimitiveTypeCode.UInt32; + return true; + case "Int64": + code = PrimitiveTypeCode.Int64; + return true; + case "UInt64": + code = PrimitiveTypeCode.UInt64; + return true; + case "Single": + code = PrimitiveTypeCode.Single; + return true; + case "Double": + code = PrimitiveTypeCode.Double; + return true; + case "String": + code = PrimitiveTypeCode.String; + return true; + case "Void": + code = PrimitiveTypeCode.Void; + return true; + case "IntPtr": + code = PrimitiveTypeCode.IntPtr; + return true; + case "UIntPtr": + code = PrimitiveTypeCode.UIntPtr; + return true; + case "Char": + code = PrimitiveTypeCode.Char; + return true; + case "TypedReference": + code = PrimitiveTypeCode.TypedReference; + return true; + } + } + + code = default(PrimitiveTypeCode); + return false; + } + + public bool IsPrimitiveType => TryGetPrimitiveTypeCode(out _); + + public static Type DecodeType(GenericContext gc, TypeSpecificationHandle handle) => + gc.cx.mdReader.GetTypeSpecification(handle).DecodeSignature(gc.cx.TypeSignatureDecoder, gc); } /// @@ -278,44 +343,54 @@ namespace Semmle.Extraction.CIL.Entities td.GetDeclaringType().IsNil ? null : (Type)cx.Create(td.GetDeclaringType()); - ShortId = MakeId(false); - // Lazy because should happen during population. typeParams = new Lazy>(MakeTypeParameters); } - public override Id MakeId(bool inContext) + public override bool Equals(object obj) { - if (IsPrimitiveType) return PrimitiveTypeId; + return obj is TypeDefinitionType t && handle.Equals(t.handle); + } - var name = cx.GetId(td.Name); + public override int GetHashCode() => handle.GetHashCode(); + + public override void WriteId(TextWriter trapFile, bool inContext) + { + if (IsPrimitiveType) + { + PrimitiveTypeId(trapFile); + return; + } + + var name = cx.GetString(td.Name); - Id l; if (ContainingType != null) { - l = ContainingType.GetId(inContext) + cx.Dot; + ContainingType.GetId(trapFile, inContext); + trapFile.Write('.'); } else { - l = AssemblyPrefix; + WriteAssemblyPrefix(trapFile); var ns = Namespace; if (!ns.IsGlobalNamespace) { - l = l + ns.ShortId + cx.Dot; + ns.WriteId(trapFile); + trapFile.Write('.'); } } - return l + name; + trapFile.Write(name); } - public override Id Name + public override string Name { get { - var name = cx.GetId(td.Name); - var tick = name.Value.IndexOf('`'); - return tick == -1 ? name : cx.GetId(name.Value.Substring(0, tick)); + var name = cx.GetString(td.Name); + var tick = name.IndexOf('`'); + return tick == -1 ? name : name.Substring(0, tick); } } @@ -344,13 +419,15 @@ namespace Semmle.Extraction.CIL.Entities return cx.Populate(new ConstructedType(cx, this, typeArguments)); } - public override Id AssemblyPrefix + public override void WriteAssemblyPrefix(TextWriter trapFile) { - get - { - var ct = ContainingType; - return ct != null ? ct.AssemblyPrefix : IsPrimitiveType ? builtin : cx.AssemblyPrefix; - } + var ct = ContainingType; + if (ct is null) + cx.WriteAssemblyPrefix(trapFile); + else if (IsPrimitiveType) + trapFile.Write("builtin:"); + else + ct.WriteAssemblyPrefix(trapFile); } IEnumerable MakeTypeParameters() @@ -488,20 +565,31 @@ namespace Semmle.Extraction.CIL.Entities /// public sealed class TypeReferenceType : Type { + readonly TypeReferenceHandle handle; readonly TypeReference tr; readonly Lazy typeParams; - public TypeReferenceType(Context cx, TypeReferenceHandle handle) : this(cx, cx.mdReader.GetTypeReference(handle)) + public TypeReferenceType(Context cx, TypeReferenceHandle handle) : this(cx, handle, cx.mdReader.GetTypeReference(handle)) { - ShortId = MakeId(false); typeParams = new Lazy(MakeTypeParameters); } - public TypeReferenceType(Context cx, TypeReference tr) : base(cx) + public TypeReferenceType(Context cx, TypeReferenceHandle handle, TypeReference tr) : base(cx) { + this.handle = handle; this.tr = tr; } + public override bool Equals(object obj) + { + return obj is TypeReferenceType t && handle.Equals(t.handle); + } + + public override int GetHashCode() + { + return handle.GetHashCode(); + } + TypeTypeParameter[] MakeTypeParameters() { var newTypeParams = new TypeTypeParameter[ThisTypeParameters]; @@ -524,13 +612,13 @@ namespace Semmle.Extraction.CIL.Entities } } - public override Id Name + public override string Name { get { - var name = cx.GetId(tr.Name); - var tick = name.Value.IndexOf('`'); - return tick == -1 ? name : cx.GetId(name.Value.Substring(0, tick)); + var name = cx.GetString(tr.Name); + var tick = name.IndexOf('`'); + return tick == -1 ? name : name.Substring(0, tick); } } @@ -541,9 +629,9 @@ namespace Semmle.Extraction.CIL.Entities get { // Parse the name - var name = cx.GetId(tr.Name); - var tick = name.Value.IndexOf('`'); - return tick == -1 ? 0 : int.Parse(name.Value.Substring(tick + 1)); + var name = cx.GetString(tr.Name); + var tick = name.IndexOf('`'); + return tick == -1 ? 0 : int.Parse(name.Substring(tick + 1)); } } @@ -568,20 +656,23 @@ namespace Semmle.Extraction.CIL.Entities public override CilTypeKind Kind => CilTypeKind.ValueOrRefType; - public override Id AssemblyPrefix + public override void WriteAssemblyPrefix(TextWriter trapFile) { - get + switch (tr.ResolutionScope.Kind) { - switch (tr.ResolutionScope.Kind) - { - case HandleKind.TypeReference: - return ContainingType.AssemblyPrefix; - case HandleKind.AssemblyReference: - var assemblyDef = cx.mdReader.GetAssemblyReference((AssemblyReferenceHandle)tr.ResolutionScope); - return cx.GetId(assemblyDef.Name) + "_" + cx.GetId(assemblyDef.Version.ToString()) + "::"; - default: - return cx.AssemblyPrefix; - } + case HandleKind.TypeReference: + ContainingType.WriteAssemblyPrefix(trapFile); + break; + case HandleKind.AssemblyReference: + var assemblyDef = cx.mdReader.GetAssemblyReference((AssemblyReferenceHandle)tr.ResolutionScope); + trapFile.Write(cx.GetString(assemblyDef.Name)); + trapFile.Write('_'); + trapFile.Write(assemblyDef.Version.ToString()); + trapFile.Write("::"); + break; + default: + cx.WriteAssemblyPrefix(trapFile); + break; } } @@ -589,30 +680,34 @@ namespace Semmle.Extraction.CIL.Entities public override IEnumerable MethodParameters => throw new InternalError("This type does not have method parameters"); - public override Id MakeId(bool inContext) + public override void WriteId(TextWriter trapFile, bool inContext) { - if (IsPrimitiveType) return PrimitiveTypeId; + if (IsPrimitiveType) + { + PrimitiveTypeId(trapFile); + return; + } var ct = ContainingType; - Id l = null; if (ct != null) { - l = ContainingType.GetId(inContext); + ContainingType.GetId(trapFile, inContext); } else { if (tr.ResolutionScope.Kind == HandleKind.AssemblyReference) { - l = AssemblyPrefix; + WriteAssemblyPrefix(trapFile); } if (!Namespace.IsGlobalNamespace) { - l += Namespace.ShortId; + Namespace.WriteId(trapFile); } } - return l + cx.Dot + cx.GetId(tr.Name); + trapFile.Write('.'); + trapFile.Write(cx.GetString(tr.Name)); } public override Type Construct(IEnumerable typeArguments) @@ -679,14 +774,31 @@ namespace Semmle.Extraction.CIL.Entities containingType = unboundType.ContainingType.Construct(typeArguments.Take(parentParams)); thisTypeArguments = typeArguments.Skip(parentParams).ToArray(); } + } - ShortId = MakeId(false); + public override bool Equals(object obj) + { + if(obj is ConstructedType t && Equals(unboundGenericType, t.unboundGenericType) && Equals(containingType, t.containingType)) + { + if (thisTypeArguments is null) return t.thisTypeArguments is null; + if (!(t.thisTypeArguments is null)) return thisTypeArguments.SequenceEqual(t.thisTypeArguments); + } + return false; + } + + public override int GetHashCode() + { + int h = unboundGenericType.GetHashCode(); + h = 13 * h + (containingType is null ? 0 : containingType.GetHashCode()); + if (!(thisTypeArguments is null)) + h = h * 13 + thisTypeArguments.SequenceHash(); + return h; } readonly Type containingType; public override Type ContainingType => containingType; - public override Id Name => unboundGenericType.Name; + public override string Name => unboundGenericType.Name; public override Namespace Namespace => unboundGenericType.Namespace; @@ -699,43 +811,39 @@ namespace Semmle.Extraction.CIL.Entities throw new NotImplementedException(); } - public override Id MakeId(bool inContext) + public override void WriteId(TextWriter trapFile, bool inContext) { - Id l; if (ContainingType != null) { - l = ContainingType.GetId(inContext) + cx.Dot; + ContainingType.GetId(trapFile, inContext); + trapFile.Write('.'); } else { - l = AssemblyPrefix; + WriteAssemblyPrefix(trapFile); if (!Namespace.IsGlobalNamespace) { - l += Namespace.ShortId + cx.Dot; + Namespace.WriteId(trapFile); + trapFile.Write('.'); } } - l += unboundGenericType.Name; + trapFile.Write(unboundGenericType.Name); if (thisTypeArguments != null && thisTypeArguments.Any()) { - l += open; - bool first = true; + trapFile.Write('<'); + int index = 0; foreach (var t in thisTypeArguments) { - if (first) first = false; else l += comma; - l += t.ShortId; + trapFile.WriteSeparator(",", index++); + t.WriteId(trapFile); } - l += close; + trapFile.Write('>'); } - return l; } - static readonly StringId open = new StringId("<"); - static readonly StringId close = new StringId(">"); - static readonly StringId comma = new StringId(","); - - public override Id AssemblyPrefix => unboundGenericType.AssemblyPrefix; + public override void WriteAssemblyPrefix(TextWriter trapFile) => unboundGenericType.WriteAssemblyPrefix(trapFile); public override IEnumerable TypeParameters => GenericArguments; @@ -748,15 +856,25 @@ namespace Semmle.Extraction.CIL.Entities public PrimitiveType(Context cx, PrimitiveTypeCode tc) : base(cx) { typeCode = tc; - ShortId = MakeId(false); } - public override Id MakeId(bool inContext) + public override bool Equals(object obj) { - return builtin + Name; + return obj is PrimitiveType pt && typeCode == pt.typeCode; } - public override Id Name => typeCode.Id(); + public override int GetHashCode() + { + return 1337 * (int)typeCode; + } + + public override void WriteId(TextWriter trapFile, bool inContext) + { + trapFile.Write("builtin:"); + trapFile.Write(Name); + } + + public override string Name => typeCode.Id(); public override Namespace Namespace => cx.SystemNamespace; @@ -766,9 +884,7 @@ namespace Semmle.Extraction.CIL.Entities public override CilTypeKind Kind => CilTypeKind.ValueOrRefType; - static readonly Id empty = new StringId(""); - - public override Id AssemblyPrefix => empty; + public override void WriteAssemblyPrefix(TextWriter trapFile) { } public override IEnumerable TypeParameters => throw new NotImplementedException(); @@ -789,22 +905,33 @@ namespace Semmle.Extraction.CIL.Entities { rank = shape.Rank; elementType = element; - ShortId = MakeId(false); } public ArrayType(Context cx, Type element) : base(cx) { rank = 1; elementType = element; - ShortId = MakeId(false); } - public override Id MakeId(bool inContext) => elementType.GetId(inContext) + openBracket + rank + closeBracket; + public override bool Equals(object obj) + { + return obj is ArrayType array && elementType.Equals(array.elementType) && rank == array.rank; + } - static readonly StringId openBracket = new StringId("["); - static readonly StringId closeBracket = new StringId("]"); + public override int GetHashCode() + { + return elementType.GetHashCode() * 5 + rank; + } - public override Id Name => elementType.Name + openBracket + closeBracket; + public override void WriteId(TextWriter trapFile, bool inContext) + { + elementType.GetId(trapFile, inContext); + trapFile.Write('['); + trapFile.Write(rank); + trapFile.Write(']'); + } + + public override string Name => elementType.Name + "[]"; public override Namespace Namespace => cx.SystemNamespace; @@ -829,13 +956,7 @@ namespace Semmle.Extraction.CIL.Entities } } - public override Id AssemblyPrefix - { - get - { - return elementType.AssemblyPrefix; - } - } + public override void WriteAssemblyPrefix(TextWriter trapFile) => elementType.WriteAssemblyPrefix(trapFile); public override IEnumerable GenericArguments => elementType.GenericArguments; @@ -865,7 +986,7 @@ namespace Semmle.Extraction.CIL.Entities public override CilTypeKind Kind => CilTypeKind.TypeParameter; - public override Id AssemblyPrefix => throw new NotImplementedException(); + public override void WriteAssemblyPrefix(TextWriter trapFile) => throw new NotImplementedException(); public override Type Construct(IEnumerable typeArguments) => throw new InternalError("Attempt to construct a type parameter"); @@ -901,17 +1022,32 @@ namespace Semmle.Extraction.CIL.Entities readonly Method method; readonly int index; - public override Id MakeId(bool inContext) => inContext && method == gc ? Name : method.ShortId + Name; + public override void WriteId(TextWriter trapFile, bool inContext) + { + if (!(inContext && method == gc)) + { + trapFile.WriteSubId(method); + } + trapFile.Write("!"); + trapFile.Write(index); + } - static readonly Id excl = new StringId("!"); - - public override Id Name => excl + index.ToString(); + public override string Name => "!" + index; public MethodTypeParameter(GenericContext gc, Method m, int index) : base(gc) { method = m; this.index = index; - ShortId = MakeId(false); + } + + public override bool Equals(object obj) + { + return obj is MethodTypeParameter tp && method.Equals(tp.method) && index == tp.index; + } + + public override int GetHashCode() + { + return method.GetHashCode() * 29 + index; } public override TypeContainer Parent => method; @@ -924,7 +1060,7 @@ namespace Semmle.Extraction.CIL.Entities { get { - yield return Tuples.cil_type(this, Name.Value, Kind, method, SourceDeclaration); + yield return Tuples.cil_type(this, Name, Kind, method, SourceDeclaration); yield return Tuples.cil_type_parameter(method, index, this); } } @@ -940,15 +1076,27 @@ namespace Semmle.Extraction.CIL.Entities { index = i; type = t; - ShortId = t.ShortId + Name; } - public override Id MakeId(bool inContext) => type.MakeId(inContext) + Name; + public override bool Equals(object obj) + { + return obj is TypeTypeParameter tp && type.Equals(tp.type) && index == tp.index; + } + + public override int GetHashCode() + { + return type.GetHashCode() * 13 + index; + } + + public override void WriteId(TextWriter trapFile, bool inContext) + { + type.WriteId(trapFile, inContext); + trapFile.Write('!'); + trapFile.Write(index); + } public override TypeContainer Parent => type ?? gc as TypeContainer; - - static readonly Id excl = new StringId("!"); - public override Id Name => excl + index.ToString(); + public override string Name => "!" + index; public override IEnumerable TypeParameters => Enumerable.Empty(); @@ -958,7 +1106,7 @@ namespace Semmle.Extraction.CIL.Entities { get { - yield return Tuples.cil_type(this, Name.Value, Kind, type, SourceDeclaration); + yield return Tuples.cil_type(this, Name, Kind, type, SourceDeclaration); yield return Tuples.cil_type_parameter(type, index, this); } } @@ -975,14 +1123,26 @@ namespace Semmle.Extraction.CIL.Entities public PointerType(Context cx, Type pointee) : base(cx) { this.pointee = pointee; - ShortId = MakeId(false); } - public override Id MakeId(bool inContext) => pointee.MakeId(inContext) + star; + public override bool Equals(object obj) + { + return obj is PointerType pt && pointee.Equals(pt.pointee); + } - static readonly StringId star = new StringId("*"); - public override Id Name => pointee.Name + star; + public override int GetHashCode() + { + return pointee.GetHashCode() * 29; + } + + public override void WriteId(TextWriter trapFile, bool inContext) + { + pointee.WriteId(trapFile, inContext); + trapFile.Write('*'); + } + + public override string Name => pointee.Name + "*"; public override Namespace Namespace => pointee.Namespace; @@ -994,7 +1154,7 @@ namespace Semmle.Extraction.CIL.Entities public override CilTypeKind Kind => CilTypeKind.Pointer; - public override Id AssemblyPrefix => pointee.AssemblyPrefix; + public override void WriteAssemblyPrefix(TextWriter trapFile) => pointee.WriteAssemblyPrefix(trapFile); public override IEnumerable TypeParameters => throw new NotImplementedException(); @@ -1016,14 +1176,13 @@ namespace Semmle.Extraction.CIL.Entities { public ErrorType(Context cx) : base(cx) { - ShortId = MakeId(false); } - public override Id MakeId(bool inContext) => CIL.Id.Create(""); + public override void WriteId(TextWriter trapFile, bool inContext) => trapFile.Write(""); public override CilTypeKind Kind => CilTypeKind.ValueOrRefType; - public override Id Name => new StringId("!error"); + public override string Name => "!error"; public override Namespace Namespace => cx.GlobalNamespace; @@ -1031,7 +1190,7 @@ namespace Semmle.Extraction.CIL.Entities public override int ThisTypeParameters => 0; - public override Id AssemblyPrefix => throw new NotImplementedException(); + public override void WriteAssemblyPrefix(TextWriter trapFile) => throw new NotImplementedException(); public override IEnumerable TypeParameters => throw new NotImplementedException(); @@ -1040,52 +1199,9 @@ namespace Semmle.Extraction.CIL.Entities public override Type Construct(IEnumerable typeArguments) => throw new NotImplementedException(); } - public sealed class TypeSpecificationType : Type - { - readonly TypeSpecification ts; - readonly Type decodedType; - - public TypeSpecificationType(GenericContext gc, TypeSpecificationHandle handle) : base(gc.cx) - { - ts = cx.mdReader.GetTypeSpecification(handle); - decodedType = ts.DecodeSignature(cx.TypeSignatureDecoder, gc); - ShortId = decodedType.ShortId; - } - - public override Id MakeId(bool inContext) => decodedType.MakeId(inContext); - - public override IEnumerable Contents - { - get - { - yield return decodedType; - } - } - - public override Id AssemblyPrefix => throw new NotImplementedException(); - - public override CilTypeKind Kind => throw new NotImplementedException(); - - public override Id Name => decodedType.Name; - - public override Namespace Namespace => throw new NotImplementedException(); - - public override Type ContainingType => decodedType.ContainingType; - - public override int ThisTypeParameters => throw new NotImplementedException(); - - public override IEnumerable TypeParameters => decodedType.TypeParameters; - - public override IEnumerable MethodParameters => throw new NotImplementedException(); - - public override Type Construct(IEnumerable typeArguments) => throw new NotImplementedException(); - - public override Type SourceDeclaration => decodedType.SourceDeclaration; - } - interface ITypeSignature { - Id MakeId(GenericContext gc); + void WriteId(TextWriter trapFile, GenericContext gc); } public class SignatureDecoder : ISignatureTypeProvider @@ -1094,19 +1210,32 @@ namespace Semmle.Extraction.CIL.Entities { public ITypeSignature elementType; public ArrayShape shape; - public Id MakeId(GenericContext gc) => elementType.MakeId(gc) + "[]"; // Make these static + public void WriteId(TextWriter trapFile, GenericContext gc) + { + elementType.WriteId(trapFile, gc); + trapFile.Write("[]"); + } } struct ByRef : ITypeSignature { public ITypeSignature elementType; - public Id MakeId(GenericContext gc) => "ref " + elementType.MakeId(gc); + + public void WriteId(TextWriter trapFile, GenericContext gc) + { + trapFile.Write("ref "); + elementType.WriteId(trapFile, gc); + } } struct FnPtr : ITypeSignature { public MethodSignature signature; - public Id MakeId(GenericContext gc) => Id.Create(""); // !! + + public void WriteId(TextWriter trapFile, GenericContext gc) + { + trapFile.Write(""); + } } ITypeSignature IConstructedTypeProvider.GetArrayType(ITypeSignature elementType, ArrayShape shape) => @@ -1122,34 +1251,49 @@ namespace Semmle.Extraction.CIL.Entities { public ITypeSignature genericType; public ImmutableArray typeArguments; - public Id MakeId(GenericContext gc) => - genericType.MakeId(gc) + "<" + Id.CommaSeparatedList(typeArguments.Select(arg => arg.MakeId(gc))) + ">"; + + public void WriteId(TextWriter trapFile, GenericContext gc) + { + genericType.WriteId(trapFile, gc); + trapFile.Write('<'); + int index = 0; + foreach(var arg in typeArguments) + { + trapFile.WriteSeparator(",", index++); + arg.WriteId(trapFile, gc); + } + trapFile.Write('>'); + } } ITypeSignature IConstructedTypeProvider.GetGenericInstantiation(ITypeSignature genericType, ImmutableArray typeArguments) => new Instantiation { genericType = genericType, typeArguments = typeArguments }; - static readonly Id open = Id.Create("{"); - static readonly Id close = Id.Create("}"); - class GenericMethodParameter : ITypeSignature { public object innerGc; public int index; - static readonly Id excl = Id.Create("M!"); - public Id MakeId(GenericContext outerGc) + + public void WriteId(TextWriter trapFile, GenericContext outerGc) { if (!ReferenceEquals(innerGc, outerGc) && innerGc is Method method) - return open + method.Label.Value + close + excl + index; - return excl + index; + { + trapFile.WriteSubId(method); + } + trapFile.Write("M!"); + trapFile.Write(index); } } class GenericTypeParameter : ITypeSignature { public int index; - static readonly Id excl = Id.Create("T!"); - public Id MakeId(GenericContext gc) => excl + index; + + public void WriteId(TextWriter trapFile, GenericContext gc) + { + trapFile.Write("T!"); + trapFile.Write(index); + } } ITypeSignature ISignatureTypeProvider.GetGenericMethodParameter(object genericContext, int index) => @@ -1164,7 +1308,10 @@ namespace Semmle.Extraction.CIL.Entities public ITypeSignature unmodifiedType; public bool isRequired; - public Id MakeId(GenericContext gc) => unmodifiedType.MakeId(gc); + public void WriteId(TextWriter trapFile, GenericContext gc) + { + unmodifiedType.WriteId(trapFile, gc); + } } ITypeSignature ISignatureTypeProvider.GetModifiedType(ITypeSignature modifier, ITypeSignature unmodifiedType, bool isRequired) @@ -1175,7 +1322,12 @@ namespace Semmle.Extraction.CIL.Entities class Pinned : ITypeSignature { public ITypeSignature elementType; - public Id MakeId(GenericContext gc) => "pinned " + elementType.MakeId(gc); + + public void WriteId(TextWriter trapFile, GenericContext gc) + { + trapFile.Write("pinned "); + elementType.WriteId(trapFile, gc); + } } ITypeSignature ISignatureTypeProvider.GetPinnedType(ITypeSignature elementType) @@ -1186,7 +1338,12 @@ namespace Semmle.Extraction.CIL.Entities class PointerType : ITypeSignature { public ITypeSignature elementType; - public Id MakeId(GenericContext gc) => elementType.MakeId(gc) + "*"; + + public void WriteId(TextWriter trapFile, GenericContext gc) + { + elementType.WriteId(trapFile, gc); + trapFile.Write('*'); + } } ITypeSignature IConstructedTypeProvider.GetPointerType(ITypeSignature elementType) @@ -1197,7 +1354,11 @@ namespace Semmle.Extraction.CIL.Entities class Primitive : ITypeSignature { public PrimitiveTypeCode typeCode; - public Id MakeId(GenericContext gc) => typeCode.Id(); + + public void WriteId(TextWriter trapFile, GenericContext gc) + { + trapFile.Write(typeCode.Id()); + } } ITypeSignature ISimpleTypeProvider.GetPrimitiveType(PrimitiveTypeCode typeCode) @@ -1208,7 +1369,11 @@ namespace Semmle.Extraction.CIL.Entities class SzArrayType : ITypeSignature { public ITypeSignature elementType; - public Id MakeId(GenericContext gc) => elementType.MakeId(gc) + "[]"; + public void WriteId(TextWriter trapFile, GenericContext gc) + { + elementType.WriteId(trapFile, gc); + trapFile.Write("[]"); + } } ITypeSignature ISZArrayTypeProvider.GetSZArrayType(ITypeSignature elementType) @@ -1221,10 +1386,10 @@ namespace Semmle.Extraction.CIL.Entities public TypeDefinitionHandle handle; public byte rawTypeKind; Type type; - public Id MakeId(GenericContext gc) + public void WriteId(TextWriter trapFile, GenericContext gc) { type = (Type)gc.cx.Create(handle); - return type.ShortId; + type.WriteId(trapFile); } } @@ -1238,10 +1403,10 @@ namespace Semmle.Extraction.CIL.Entities public TypeReferenceHandle handle; public byte rawTypeKind; // struct/class (not used) Type type; - public Id MakeId(GenericContext gc) + public void WriteId(TextWriter trapFile, GenericContext gc) { type = (Type)gc.cx.Create(handle); - return type.ShortId; + type.WriteId(trapFile); } } @@ -1298,8 +1463,7 @@ namespace Semmle.Extraction.CIL.Entities Type IConstructedTypeProvider.GetPointerType(Type elementType) => cx.Populate(new PointerType(cx, elementType)); - Type ISimpleTypeProvider.GetPrimitiveType(PrimitiveTypeCode typeCode) => - cx.Populate(new PrimitiveType(cx, typeCode)); + Type ISimpleTypeProvider.GetPrimitiveType(PrimitiveTypeCode typeCode) => cx.Create(typeCode); Type ISZArrayTypeProvider.GetSZArrayType(Type elementType) => cx.Populate(new ArrayType(cx, elementType)); diff --git a/csharp/extractor/Semmle.Extraction.CIL/ExtractionProduct.cs b/csharp/extractor/Semmle.Extraction.CIL/ExtractionProduct.cs index 1f660e3ee83..fd2e169e4af 100644 --- a/csharp/extractor/Semmle.Extraction.CIL/ExtractionProduct.cs +++ b/csharp/extractor/Semmle.Extraction.CIL/ExtractionProduct.cs @@ -58,8 +58,6 @@ namespace Semmle.Extraction.CIL public Microsoft.CodeAnalysis.Location ReportingLocation => throw new NotImplementedException(); - public virtual IId Id => FreshId.Instance; - public virtual void Extract(Context cx2) { cx2.Extract(this); @@ -86,18 +84,16 @@ namespace Semmle.Extraction.CIL public Label Label { get; set; } public Microsoft.CodeAnalysis.Location ReportingLocation => throw new NotImplementedException(); - public Id ShortId { get; set; } - public abstract Id IdSuffix { get; } - public IId Id => ShortId + IdSuffix; + public abstract void WriteId(System.IO.TextWriter trapFile); - public void WriteId(System.IO.TextWriter trapFile) - { - trapFile.WriteIId(Id); - } + public abstract string IdSuffix { get; } public void WriteQuotedId(TextWriter trapFile) { + trapFile.Write("@\""); WriteId(trapFile); + trapFile.Write(IdSuffix); + trapFile.Write('\"'); } public void Extract(Context cx2) @@ -112,7 +108,14 @@ namespace Semmle.Extraction.CIL this.cx = cx; } - public override string ToString() => Id.ToString(); + public override string ToString() + { + using (var writer = new StringWriter()) + { + WriteQuotedId(writer); + return writer.ToString(); + } + } TrapStackBehaviour IEntity.TrapStackBehaviour => TrapStackBehaviour.NoLabel; } @@ -122,8 +125,6 @@ namespace Semmle.Extraction.CIL /// public interface ILabelledEntity : IExtractedEntity { - Id ShortId { get; set; } - Id IdSuffix { get; } } /// diff --git a/csharp/extractor/Semmle.Extraction.CIL/Factories.cs b/csharp/extractor/Semmle.Extraction.CIL/Factories.cs index 9448f789131..9b4de5b5d81 100644 --- a/csharp/extractor/Semmle.Extraction.CIL/Factories.cs +++ b/csharp/extractor/Semmle.Extraction.CIL/Factories.cs @@ -1,5 +1,7 @@ -using System; +using Semmle.Extraction.CIL.Entities; +using System; using System.Collections.Generic; +using System.IO; using System.Reflection.Metadata; namespace Semmle.Extraction.CIL @@ -9,38 +11,78 @@ namespace Semmle.Extraction.CIL /// public partial class Context { - readonly Dictionary ids = new Dictionary(); + readonly Dictionary ids = new Dictionary(); public T Populate(T e) where T : ILabelledEntity { - Id id = e.ShortId; + if(e.Label.Valid) + { + return e; // Already populated + } - if (ids.TryGetValue(id, out var existing)) + if (ids.TryGetValue(e, out var existing)) { // It exists already - e.Label = existing.Item1; - e.ShortId = existing.Item2; // Reuse ID for efficiency + e.Label = existing; } else { e.Label = cx.GetNewLabel(); cx.DefineLabel(e, cx.TrapWriter.Writer); - ids.Add(id, (e.Label, id)); + ids.Add(e, e.Label); cx.PopulateLater(() => { foreach (var c in e.Contents) c.Extract(this); }); +#if DEBUG_LABELS + using (var writer = new StringWriter()) + { + e.WriteId(writer); + var id = writer.ToString(); + + if (debugLabels.TryGetValue(id, out ILabelledEntity previousEntity)) + { + cx.Extractor.Message(new Message("Duplicate trap ID", id, null, severity: Util.Logging.Severity.Warning)); + } + else + { + debugLabels.Add(id, e); + } + } +#endif } return e; } +#if DEBUG_LABELS + Dictionary debugLabels = new Dictionary(); +#endif + public IExtractedEntity Create(Handle h) { var entity = CreateGeneric(defaultGenericContext, h); return entity; } + // Lazily cache primitive types. + PrimitiveType[] primitiveTypes = new PrimitiveType[(int)PrimitiveTypeCode.Object + 1]; + + public PrimitiveType Create(PrimitiveTypeCode code) + { + PrimitiveType e = primitiveTypes[(int)code]; + + if(e is null) + { + e = new PrimitiveType(this, code); + e.Label = cx.GetNewLabel(); + cx.DefineLabel(e, cx.TrapWriter.Writer); + primitiveTypes[(int)code] = e; + } + + return e; + } + /// /// Creates an entity from a Handle in a GenericContext. /// The type of the returned entity depends on the type of the handle. @@ -63,25 +105,27 @@ namespace Semmle.Extraction.CIL switch (handle.Kind) { case HandleKind.MethodDefinition: - entity = new Entities.DefinitionMethod(gc, (MethodDefinitionHandle)handle); + entity = new DefinitionMethod(gc, (MethodDefinitionHandle)handle); break; case HandleKind.MemberReference: entity = Create(gc, (MemberReferenceHandle)handle); break; case HandleKind.MethodSpecification: - entity = new Entities.MethodSpecificationMethod(gc, (MethodSpecificationHandle)handle); + entity = new MethodSpecificationMethod(gc, (MethodSpecificationHandle)handle); break; case HandleKind.FieldDefinition: - entity = new Entities.DefinitionField(gc, (FieldDefinitionHandle)handle); + entity = new DefinitionField(gc, (FieldDefinitionHandle)handle); break; case HandleKind.TypeReference: - entity = new Entities.TypeReferenceType(this, (TypeReferenceHandle)handle); + var tr = new TypeReferenceType(this, (TypeReferenceHandle)handle); + if (tr.TryGetPrimitiveType(gc.cx, out var pt)) + return pt; + entity = tr; break; case HandleKind.TypeSpecification: - entity = new Entities.TypeSpecificationType(gc, (TypeSpecificationHandle)handle); - break; + return Entities.Type.DecodeType(gc, (TypeSpecificationHandle)handle); case HandleKind.TypeDefinition: - entity = new Entities.TypeDefinitionType(this, (TypeDefinitionHandle)handle); + entity = new TypeDefinitionType(this, (TypeDefinitionHandle)handle); break; default: throw new InternalError("Unhandled handle kind " + handle.Kind); @@ -97,123 +141,90 @@ namespace Semmle.Extraction.CIL switch (mr.GetKind()) { case MemberReferenceKind.Method: - return new Entities.MemberReferenceMethod(gc, handle); + return new MemberReferenceMethod(gc, handle); case MemberReferenceKind.Field: - return new Entities.MemberReferenceField(gc, handle); + return new MemberReferenceField(gc, handle); default: throw new InternalError("Unhandled member reference handle"); } } - #region Strings - readonly Dictionary stringHandleIds = new Dictionary(); - readonly Dictionary stringIds = new Dictionary(); - /// - /// Return an ID containing the given string. + /// Gets the string for a string handle. /// /// The string handle. - /// An ID. - public StringId GetId(StringHandle h) - { - StringId result; - if (!stringHandleIds.TryGetValue(h, out result)) - { - result = new StringId(mdReader.GetString(h)); - stringHandleIds.Add(h, result); - } - return result; - } + /// The string. + public string GetString(StringHandle h) => mdReader.GetString(h); - public readonly StringId Dot = new StringId("."); +#region Namespaces - /// - /// Gets an ID containing the given string. - /// Caches existing IDs for more compact storage. - /// - /// The string. - /// An ID containing the string. - public StringId GetId(string str) - { - StringId result; - if (!stringIds.TryGetValue(str, out result)) - { - result = new StringId(str); - stringIds.Add(str, result); - } - return result; - } - #endregion + readonly CachedFunction namespaceFactory; - #region Namespaces + public Namespace CreateNamespace(StringHandle fqn) => namespaceFactory[fqn]; - readonly CachedFunction namespaceFactory; - - public Entities.Namespace CreateNamespace(StringHandle fqn) => namespaceFactory[fqn]; - - readonly Lazy globalNamespace, systemNamespace; + readonly Lazy globalNamespace, systemNamespace; /// /// The entity representing the global namespace. /// - public Entities.Namespace GlobalNamespace => globalNamespace.Value; + public Namespace GlobalNamespace => globalNamespace.Value; /// /// The entity representing the System namespace. /// - public Entities.Namespace SystemNamespace => systemNamespace.Value; + public Namespace SystemNamespace => systemNamespace.Value; /// /// Creates a namespace from a fully-qualified name. /// /// The fully-qualified namespace name. /// The namespace entity. - Entities.Namespace CreateNamespace(string fqn) => Populate(new Entities.Namespace(this, fqn)); + Namespace CreateNamespace(string fqn) => Populate(new Namespace(this, fqn)); - readonly CachedFunction namespaceDefinitionFactory; + readonly CachedFunction namespaceDefinitionFactory; /// /// Creates a namespace from a namespace handle. /// /// The handle of the namespace. /// The namespace entity. - public Entities.Namespace Create(NamespaceDefinitionHandle handle) => namespaceDefinitionFactory[handle]; + public Namespace Create(NamespaceDefinitionHandle handle) => namespaceDefinitionFactory[handle]; - Entities.Namespace CreateNamespace(NamespaceDefinitionHandle handle) + Namespace CreateNamespace(NamespaceDefinitionHandle handle) { if (handle.IsNil) return GlobalNamespace; NamespaceDefinition nd = mdReader.GetNamespaceDefinition(handle); - return Populate(new Entities.Namespace(this, GetId(nd.Name), Create(nd.Parent))); + return Populate(new Namespace(this, GetString(nd.Name), Create(nd.Parent))); } - #endregion +#endregion - #region Locations - readonly CachedFunction sourceFiles; - readonly CachedFunction folders; - readonly CachedFunction sourceLocations; +#region Locations + readonly CachedFunction sourceFiles; + readonly CachedFunction folders; + readonly CachedFunction sourceLocations; /// /// Creates a source file entity from a PDB source file. /// /// The PDB source file. /// A source file entity. - public Entities.PdbSourceFile CreateSourceFile(PDB.ISourceFile file) => sourceFiles[file]; + public PdbSourceFile CreateSourceFile(PDB.ISourceFile file) => sourceFiles[file]; /// /// Creates a folder entitiy with the given path. /// /// The path of the folder. /// A folder entity. - public Entities.Folder CreateFolder(string path) => folders[path]; + public Folder CreateFolder(string path) => folders[path]; /// /// Creates a source location. /// /// The source location from PDB. /// A source location entity. - public Entities.PdbSourceLocation CreateSourceLocation(PDB.Location loc) => sourceLocations[loc]; + public PdbSourceLocation CreateSourceLocation(PDB.Location loc) => sourceLocations[loc]; - #endregion +#endregion readonly CachedFunction genericHandleFactory; diff --git a/csharp/extractor/Semmle.Extraction.CIL/Id.cs b/csharp/extractor/Semmle.Extraction.CIL/Id.cs index 121043bdf32..8d9a78f76d3 100644 --- a/csharp/extractor/Semmle.Extraction.CIL/Id.cs +++ b/csharp/extractor/Semmle.Extraction.CIL/Id.cs @@ -1,202 +1,31 @@ -using System.Collections.Generic; -using System.Reflection.Metadata; +using System.Reflection.Metadata; namespace Semmle.Extraction.CIL { - /// - /// An ID fragment which is designed to be shared, reused - /// and composed using the + operator. - /// - public abstract class Id : IId - { - public void AppendTo(System.IO.TextWriter tb) - { - tb.Write("@\""); - BuildParts(tb); - tb.Write("\""); - } - - public abstract void BuildParts(System.IO.TextWriter tb); - - public static Id operator +(Id l1, Id l2) => Create(l1, l2); - - public static Id operator +(Id l1, string l2) => Create(l1, Create(l2)); - - public static Id operator +(Id l1, int l2) => Create(l1, Create(l2)); - - public static Id operator +(string l1, Id l2) => Create(Create(l1), l2); - - public static Id operator +(int l1, Id l2) => Create(Create(l1), l2); - - public static Id Create(string s) => s == null ? null : new StringId(s); - - public static Id Create(int i) => new IntId(i); - - static readonly Id openCurly = Create("{#"); - static readonly Id closeCurly = Create("}"); - - public static Id Create(Label l) => openCurly + l.Value + closeCurly; - - static readonly Id comma = Id.Create(","); - - public static Id CommaSeparatedList(IEnumerable items) - { - Id result = null; - bool first = true; - foreach (var i in items) - { - if (first) first = false; else result += comma; - result += i; - } - return result; - } - - public static Id Create(Id l1, Id l2) - { - return l1 == null ? l2 : l2 == null ? l1 : new ConsId(l1, l2); - } - - public abstract string Value { get; } - - public override string ToString() => Value; - } - - /// - /// An ID concatenating two other IDs. - /// - public sealed class ConsId : Id - { - readonly Id left, right; - readonly int hash; - - public ConsId(Id l1, Id l2) - { - left = l1; - right = l2; - hash = unchecked(12 + 3 * (left.GetHashCode() + 51 * right.GetHashCode())); - } - - public override void BuildParts(System.IO.TextWriter tb) - { - left.BuildParts(tb); - right.BuildParts(tb); - } - - public override bool Equals(object other) - { - return other is ConsId && Equals((ConsId)other); - } - - public bool Equals(ConsId other) - { - return this == other || - (hash == other.hash && left.Equals(other.left) && right.Equals(other.right)); - } - - public override int GetHashCode() => hash; - - public override string Value => left.Value + right.Value; - - } - - /// - /// A leaf ID storing a string. - /// - public sealed class StringId : Id - { - readonly string value; - public override string Value => value; - - public StringId(string s) - { - value = s; - } - - public override void BuildParts(System.IO.TextWriter tw) - { - tw.Write(value); - } - - public override bool Equals(object obj) - { - return obj is StringId && ((StringId)obj).value == value; - } - - public override int GetHashCode() => Value.GetHashCode() * 31 + 9; - } - - /// - /// A leaf ID storing an integer. - /// - public sealed class IntId : Id - { - readonly int value; - public override string Value => value.ToString(); - - public IntId(int i) - { - value = i; - } - - public override void BuildParts(System.IO.TextWriter tw) - { - tw.Write(value); - } - - public override bool Equals(object obj) - { - return obj is IntId && ((IntId)obj).value == value; - } - - public override int GetHashCode() => unchecked(12 + value * 17); - } - - /// - /// Some predefined IDs. - /// public static class IdUtils { - public static StringId boolId = new StringId("Boolean"); - public static StringId byteId = new StringId("Byte"); - public static StringId charId = new StringId("Char"); - public static StringId doubleId = new StringId("Double"); - public static StringId shortId = new StringId("Int16"); - public static StringId intId = new StringId("Int32"); - public static StringId longId = new StringId("Int64"); - public static StringId intptrId = new StringId("IntPtr"); - public static StringId objectId = new StringId("Object"); - public static StringId sbyteId = new StringId("SByte"); - public static StringId floatId = new StringId("Single"); - public static StringId stringId = new StringId("String"); - public static StringId ushortId = new StringId("UInt16"); - public static StringId uintId = new StringId("UInt32"); - public static StringId ulongId = new StringId("UInt64"); - public static StringId uintptrId = new StringId("UIntPtr"); - public static StringId voidId = new StringId("Void"); - public static StringId typedReferenceId = new StringId("TypedReference"); - - public static StringId Id(this PrimitiveTypeCode typeCode) + public static string Id(this PrimitiveTypeCode typeCode) { switch (typeCode) { - case PrimitiveTypeCode.Boolean: return boolId; - case PrimitiveTypeCode.Byte: return byteId; - case PrimitiveTypeCode.Char: return charId; - case PrimitiveTypeCode.Double: return doubleId; - case PrimitiveTypeCode.Int16: return shortId; - case PrimitiveTypeCode.Int32: return intId; - case PrimitiveTypeCode.Int64: return longId; - case PrimitiveTypeCode.IntPtr: return intptrId; - case PrimitiveTypeCode.Object: return objectId; - case PrimitiveTypeCode.SByte: return sbyteId; - case PrimitiveTypeCode.Single: return floatId; - case PrimitiveTypeCode.String: return stringId; - case PrimitiveTypeCode.UInt16: return ushortId; - case PrimitiveTypeCode.UInt32: return uintId; - case PrimitiveTypeCode.UInt64: return ulongId; - case PrimitiveTypeCode.UIntPtr: return uintptrId; - case PrimitiveTypeCode.Void: return voidId; - case PrimitiveTypeCode.TypedReference: return typedReferenceId; + case PrimitiveTypeCode.Boolean: return "Boolean"; + case PrimitiveTypeCode.Byte: return "Byte"; + case PrimitiveTypeCode.Char: return "Char"; + case PrimitiveTypeCode.Double: return "Double"; + case PrimitiveTypeCode.Int16: return "Int16"; + case PrimitiveTypeCode.Int32: return "Int32"; + case PrimitiveTypeCode.Int64: return "Int64"; + case PrimitiveTypeCode.IntPtr: return "IntPtr"; + case PrimitiveTypeCode.Object: return "Object"; + case PrimitiveTypeCode.SByte: return "SByte"; + case PrimitiveTypeCode.Single: return "Single"; + case PrimitiveTypeCode.String: return "String"; + case PrimitiveTypeCode.UInt16: return "UInt16"; + case PrimitiveTypeCode.UInt32: return "UInt32"; + case PrimitiveTypeCode.UInt64: return "UInt64"; + case PrimitiveTypeCode.UIntPtr: return "UIntPtr"; + case PrimitiveTypeCode.Void: return "Void"; + case PrimitiveTypeCode.TypedReference: return "TypedReference"; default: throw new InternalError($"Unhandled type code {typeCode}"); } } diff --git a/csharp/extractor/Semmle.Extraction.CIL/Semmle.Extraction.CIL.csproj b/csharp/extractor/Semmle.Extraction.CIL/Semmle.Extraction.CIL.csproj index e7e2c50e41a..d765a3ba281 100644 --- a/csharp/extractor/Semmle.Extraction.CIL/Semmle.Extraction.CIL.csproj +++ b/csharp/extractor/Semmle.Extraction.CIL/Semmle.Extraction.CIL.csproj @@ -1,4 +1,4 @@ - + netcoreapp2.2 @@ -8,6 +8,10 @@ true + + DEBUG;DEBUG_LABELS + + diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Compilation.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Compilation.cs index f7c69b0d718..b6ff91f5988 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Compilation.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Compilation.cs @@ -8,8 +8,8 @@ namespace Semmle.Extraction.CSharp.Entities { class Compilation : FreshEntity { - string cwd; - string[] args; + private readonly string cwd; + private readonly string[] args; public Compilation(Context cx, string cwd, string[] args) : base(cx) { diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Method.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Method.cs index ceee6efaae9..f7cbc3d38e0 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Method.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Method.cs @@ -235,7 +235,7 @@ namespace Semmle.Extraction.CSharp.Entities if (method.IsVararg) { - tb.WriteSeparator(",", index++); + tb.WriteSeparator(",", index); tb.Write("__arglist"); } diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Modifier.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Modifier.cs index a644ce13b79..4de6318786c 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Modifier.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Modifier.cs @@ -43,7 +43,7 @@ namespace Semmle.Extraction.CSharp.Entities public override void Populate(TextWriter trapFile) { - trapFile.Emit(new Tuple("modifiers", Label, symbol.name)); + trapFile.modifiers(Label, symbol.name); } public static string AccessbilityModifier(Accessibility access) diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/Type.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/Type.cs index 60dc1cc3921..56c1f887a53 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/Type.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/Type.cs @@ -83,9 +83,11 @@ namespace Semmle.Extraction.CSharp.Entities ExtractMetadataHandle(trapFile); ExtractAttributes(); - var tb = new StringWriter(); - symbol.BuildDisplayName(Context, tb); - trapFile.types(this, GetClassType(Context, symbol), tb.ToString()); + using (var tb = new StringWriter()) + { + symbol.BuildDisplayName(Context, tb); + trapFile.types(this, GetClassType(Context, symbol), tb.ToString()); + } // Visit base types var baseTypes = new List(); diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/UsingDirective.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/UsingDirective.cs index ccbf1788f0c..4fdbe7c18ad 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Entities/UsingDirective.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/UsingDirective.cs @@ -16,7 +16,6 @@ namespace Semmle.Extraction.CSharp.Entities public UsingDirective(Context cx, UsingDirectiveSyntax usingDirective, NamespaceDeclaration parent) : base(cx) { - var trapFile = cx.TrapWriter.Writer; Node = usingDirective; Parent = parent; TryPopulate(); diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Tuples.cs b/csharp/extractor/Semmle.Extraction.CSharp/Tuples.cs index 11636f7d544..97305382f65 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Tuples.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Tuples.cs @@ -308,6 +308,12 @@ namespace Semmle.Extraction.CSharp { trapFile.WriteTuple("methods", method, name, declType, retType, originalDefinition); } + + internal static void modifiers(this TextWriter trapFile, Label entity, string modifier) + { + trapFile.BeginTuple("modifiers").Param(entity).Param(modifier).EndTuple(); + } + internal static void mutator_invocation_mode(this TextWriter trapFile, Expression expr, int mode) { trapFile.WriteTuple("mutator_invocation_mode", expr, mode); diff --git a/csharp/extractor/Semmle.Extraction/CommentProcessing.cs b/csharp/extractor/Semmle.Extraction/CommentProcessing.cs index 4cf8794f1aa..4462f112eb1 100644 --- a/csharp/extractor/Semmle.Extraction/CommentProcessing.cs +++ b/csharp/extractor/Semmle.Extraction/CommentProcessing.cs @@ -332,7 +332,7 @@ namespace Semmle.Extraction.CommentProcessing class CommentBlock : ICommentBlock { - private List lines = new List(); + private readonly List lines = new List(); public IEnumerable CommentLines => lines; diff --git a/csharp/extractor/Semmle.Extraction/Context.cs b/csharp/extractor/Semmle.Extraction/Context.cs index 110382e2d14..dec149783bb 100644 --- a/csharp/extractor/Semmle.Extraction/Context.cs +++ b/csharp/extractor/Semmle.Extraction/Context.cs @@ -105,9 +105,11 @@ namespace Semmle.Extraction if (entity.NeedsPopulation) Populate(init as ISymbol, entity); #if DEBUG_LABELS - var id = new StringWriter(); - entity.WriteId(id); - CheckEntityHasUniqueLabel(id.ToString(), entity); + using (var id = new StringWriter()) + { + entity.WriteId(id); + CheckEntityHasUniqueLabel(id.ToString(), entity); + } #endif } @@ -149,9 +151,11 @@ namespace Semmle.Extraction Populate(init as ISymbol, entity); #if DEBUG_LABELS - var id = new StringWriter(); - entity.WriteId(id); - CheckEntityHasUniqueLabel(id.ToString(), entity); + using (var id = new StringWriter()) + { + entity.WriteId(id); + CheckEntityHasUniqueLabel(id.ToString(), entity); + } #endif return entity; diff --git a/csharp/extractor/Semmle.Extraction/Entities/Assembly.cs b/csharp/extractor/Semmle.Extraction/Entities/Assembly.cs index 76f820e0658..fc428b1600e 100644 --- a/csharp/extractor/Semmle.Extraction/Entities/Assembly.cs +++ b/csharp/extractor/Semmle.Extraction/Entities/Assembly.cs @@ -69,7 +69,7 @@ namespace Semmle.Extraction.Entities public override void WriteId(System.IO.TextWriter trapFile) { trapFile.Write(assembly.ToString()); - if (assemblyPath is null) + if (!(assemblyPath is null)) { trapFile.Write("#file:///"); trapFile.Write(assemblyPath.Replace("\\", "/")); diff --git a/csharp/extractor/Semmle.Extraction/Entity.cs b/csharp/extractor/Semmle.Extraction/Entity.cs index 4bac663d0dd..5b20810dc07 100644 --- a/csharp/extractor/Semmle.Extraction/Entity.cs +++ b/csharp/extractor/Semmle.Extraction/Entity.cs @@ -162,11 +162,13 @@ namespace Semmle.Extraction /// The debug string. public static string GetDebugLabel(this IEntity entity) { - var writer = new StringWriter(); - writer.WriteLabel(entity.Label.Value); - writer.Write('='); - entity.WriteQuotedId(writer); - return writer.ToString(); + using (var writer = new StringWriter()) + { + writer.WriteLabel(entity.Label.Value); + writer.Write('='); + entity.WriteQuotedId(writer); + return writer.ToString(); + } } } diff --git a/csharp/extractor/Semmle.Extraction/Id.cs b/csharp/extractor/Semmle.Extraction/Id.cs index 4d68c4a88bc..d7f7c02e843 100644 --- a/csharp/extractor/Semmle.Extraction/Id.cs +++ b/csharp/extractor/Semmle.Extraction/Id.cs @@ -106,7 +106,7 @@ namespace Semmle.Extraction class IdTrapBuilder { - readonly public List Fragments = new List(); + public readonly List Fragments = new List(); public void Append(object arg) { diff --git a/csharp/extractor/Semmle.Extraction/Options.cs b/csharp/extractor/Semmle.Extraction/Options.cs index c1330be011d..a7d186f9e92 100644 --- a/csharp/extractor/Semmle.Extraction/Options.cs +++ b/csharp/extractor/Semmle.Extraction/Options.cs @@ -90,6 +90,9 @@ namespace Semmle.Extraction CIL = !value; Fast = value; return true; + case "brotli": + TrapCompression = value ? TrapWriter.CompressionMode.Brotli : TrapWriter.CompressionMode.Gzip; + return true; default: return false; } diff --git a/csharp/extractor/Semmle.Extraction/Symbol.cs b/csharp/extractor/Semmle.Extraction/Symbol.cs index 1f481be3435..a760093596c 100644 --- a/csharp/extractor/Semmle.Extraction/Symbol.cs +++ b/csharp/extractor/Semmle.Extraction/Symbol.cs @@ -26,9 +26,11 @@ namespace Semmle.Extraction { get { - var trap = new StringWriter(); - Populate(trap); - return trap.ToString(); + using (var trap = new StringWriter()) + { + Populate(trap); + return trap.ToString(); + } } } diff --git a/csharp/extractor/Semmle.Extraction/TrapExtensions.cs b/csharp/extractor/Semmle.Extraction/TrapExtensions.cs index bfe98da0049..30adb7ebbc7 100644 --- a/csharp/extractor/Semmle.Extraction/TrapExtensions.cs +++ b/csharp/extractor/Semmle.Extraction/TrapExtensions.cs @@ -48,6 +48,12 @@ namespace Semmle.Extraction return new NextParam(Writer); } + public NextParam Param(Label label) + { + Writer.WriteLabel(label.Value); + return new NextParam(Writer); + } + public void EndTuple() { Writer.WriteLine(')'); diff --git a/csharp/extractor/Semmle.Extraction/Tuple.cs b/csharp/extractor/Semmle.Extraction/Tuple.cs index acacfe70d36..93dec265696 100644 --- a/csharp/extractor/Semmle.Extraction/Tuple.cs +++ b/csharp/extractor/Semmle.Extraction/Tuple.cs @@ -155,9 +155,11 @@ namespace Semmle.Extraction public override string ToString() { // Only implemented for debugging purposes - var tsb = new StringWriter(); - EmitToTrapBuilder(tsb); - return tsb.ToString(); + using (var writer = new StringWriter()) + { + EmitToTrapBuilder(writer); + return writer.ToString(); + } } } } diff --git a/csharp/extractor/Semmle.Util/IEnumerableExtensions.cs b/csharp/extractor/Semmle.Util/IEnumerableExtensions.cs index 845b1825628..d53fcf99ff4 100644 --- a/csharp/extractor/Semmle.Util/IEnumerableExtensions.cs +++ b/csharp/extractor/Semmle.Util/IEnumerableExtensions.cs @@ -86,5 +86,19 @@ namespace Semmle.Util { items.ForEach(_ => { }); } + + /// + /// Computes a hash of a sequence. + /// + /// The type of the item. + /// The list of items to hash. + /// The hash code. + public static int SequenceHash(this IEnumerable items) + { + int h = 0; + foreach (var i in items) + h = h * 7 + i.GetHashCode(); + return h; + } } }