diff --git a/csharp/extractor/Semmle.Extraction.CIL.Driver/Program.cs b/csharp/extractor/Semmle.Extraction.CIL.Driver/Program.cs index 53542c970e9..940cd647ed8 100644 --- a/csharp/extractor/Semmle.Extraction.CIL.Driver/Program.cs +++ b/csharp/extractor/Semmle.Extraction.CIL.Driver/Program.cs @@ -24,7 +24,7 @@ namespace Semmle.Extraction.CIL.Driver { var sw = new Stopwatch(); sw.Start(); - Entities.Assembly.ExtractCIL(layout, assemblyPath, logger, nocache, extractPdbs, trapCompression, out _, out _); + Analyser.ExtractCIL(layout, assemblyPath, logger, nocache, extractPdbs, trapCompression, out _, out _); sw.Stop(); logger.Log(Severity.Info, " {0} ({1})", assemblyPath, sw.Elapsed); } diff --git a/csharp/extractor/Semmle.Extraction.CIL/Analyser.cs b/csharp/extractor/Semmle.Extraction.CIL/Analyser.cs new file mode 100644 index 00000000000..dee801cc171 --- /dev/null +++ b/csharp/extractor/Semmle.Extraction.CIL/Analyser.cs @@ -0,0 +1,53 @@ +using Semmle.Util.Logging; +using System; +using Semmle.Util; +using Semmle.Extraction.CIL.Entities; + +namespace Semmle.Extraction.CIL +{ + public static class Analyser + { + private static void ExtractCIL(Extractor extractor, TrapWriter trapWriter, bool extractPdbs) + { + using var cilContext = new Context(extractor, trapWriter, extractor.OutputPath, extractPdbs); + cilContext.Populate(new Assembly(cilContext)); + cilContext.PopulateAll(); + } + + /// + /// Main entry point to the CIL extractor. + /// Call this to extract a given assembly. + /// + /// The trap layout. + /// The full path of the assembly to extract. + /// The logger. + /// True to overwrite existing trap file. + /// Whether to extract PDBs. + /// The path of the trap file. + /// Whether the file was extracted (false=cached). + public static void ExtractCIL(Layout layout, string assemblyPath, ILogger logger, bool nocache, bool extractPdbs, TrapWriter.CompressionMode trapCompression, out string trapFile, out bool extracted) + { + trapFile = ""; + extracted = false; + try + { + var canonicalPathCache = CanonicalPathCache.Create(logger, 1000); + var pathTransformer = new PathTransformer(canonicalPathCache); + var extractor = new Extractor(false, assemblyPath, logger, pathTransformer); + var transformedAssemblyPath = pathTransformer.Transform(assemblyPath); + var project = layout.LookupProjectOrDefault(transformedAssemblyPath); + using var trapWriter = project.CreateTrapWriter(logger, transformedAssemblyPath.WithSuffix(".cil"), trapCompression, discardDuplicates: true); + trapFile = trapWriter.TrapFile; + if (nocache || !System.IO.File.Exists(trapFile)) + { + ExtractCIL(extractor, trapWriter, extractPdbs); + extracted = true; + } + } + catch (Exception ex) // lgtm[cs/catch-of-all-exceptions] + { + logger.Log(Severity.Error, string.Format("Exception extracting {0}: {1}", assemblyPath, ex)); + } + } + } +} diff --git a/csharp/extractor/Semmle.Extraction.CIL/Context.Factories.cs b/csharp/extractor/Semmle.Extraction.CIL/Context.Factories.cs index 1017b2f1a19..65c72487b65 100644 --- a/csharp/extractor/Semmle.Extraction.CIL/Context.Factories.cs +++ b/csharp/extractor/Semmle.Extraction.CIL/Context.Factories.cs @@ -9,11 +9,11 @@ namespace Semmle.Extraction.CIL /// /// Provides methods for creating and caching various entities. /// - public sealed partial class Context + internal sealed partial class Context { private readonly Dictionary ids = new Dictionary(); - public T Populate(T e) where T : IExtractedEntity + internal T Populate(T e) where T : IExtractedEntity { if (e.Label.Valid) { @@ -28,7 +28,7 @@ namespace Semmle.Extraction.CIL else { e.Label = GetNewLabel(); - DefineLabel(e, TrapWriter.Writer, Extractor); + DefineLabel(e); ids.Add(e, e.Label); PopulateLater(() => { @@ -76,7 +76,7 @@ namespace Semmle.Extraction.CIL { Label = GetNewLabel() }; - DefineLabel(e, TrapWriter.Writer, Extractor); + DefineLabel(e); primitiveTypes[(int)code] = e; } diff --git a/csharp/extractor/Semmle.Extraction.CIL/Context.cs b/csharp/extractor/Semmle.Extraction.CIL/Context.cs index a0d6ad1ab6f..acb66dc3284 100644 --- a/csharp/extractor/Semmle.Extraction.CIL/Context.cs +++ b/csharp/extractor/Semmle.Extraction.CIL/Context.cs @@ -10,7 +10,7 @@ namespace Semmle.Extraction.CIL /// Adds additional context that is specific for CIL extraction. /// One context = one DLL/EXE. /// - public sealed partial class Context : Extraction.Context, IDisposable + internal sealed partial class Context : Extraction.Context, IDisposable { private readonly FileStream stream; private Entities.Assembly? assemblyNull; diff --git a/csharp/extractor/Semmle.Extraction.CIL/EmptyContext.cs b/csharp/extractor/Semmle.Extraction.CIL/EmptyContext.cs index 1105bc39cba..d5344877264 100644 --- a/csharp/extractor/Semmle.Extraction.CIL/EmptyContext.cs +++ b/csharp/extractor/Semmle.Extraction.CIL/EmptyContext.cs @@ -5,7 +5,7 @@ namespace Semmle.Extraction.CIL /// /// A generic context which does not contain any type parameters. /// - public class EmptyContext : IGenericContext + internal class EmptyContext : IGenericContext { public EmptyContext(Context cx) { diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/Assembly.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/Assembly.cs index feff55836c0..2191afef125 100644 --- a/csharp/extractor/Semmle.Extraction.CIL/Entities/Assembly.cs +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/Assembly.cs @@ -1,18 +1,15 @@ using System.Reflection; using System.Globalization; using System.Collections.Generic; -using Semmle.Util.Logging; -using System; using Semmle.Extraction.Entities; using System.IO; -using Semmle.Util; namespace Semmle.Extraction.CIL.Entities { /// /// An assembly to extract. /// - public class Assembly : LabelledEntity, ILocation + internal class Assembly : LabelledEntity, ILocation { private readonly File file; private readonly AssemblyName assemblyName; @@ -101,48 +98,5 @@ namespace Semmle.Extraction.CIL.Entities } } } - - private static void ExtractCIL(Extractor extractor, TrapWriter trapWriter, bool extractPdbs) - { - using var cilContext = new Context(extractor, trapWriter, extractor.OutputPath, extractPdbs); - cilContext.Populate(new Assembly(cilContext)); - cilContext.PopulateAll(); - } - - /// - /// Main entry point to the CIL extractor. - /// Call this to extract a given assembly. - /// - /// The trap layout. - /// The full path of the assembly to extract. - /// The logger. - /// True to overwrite existing trap file. - /// Whether to extract PDBs. - /// The path of the trap file. - /// Whether the file was extracted (false=cached). - public static void ExtractCIL(Layout layout, string assemblyPath, ILogger logger, bool nocache, bool extractPdbs, TrapWriter.CompressionMode trapCompression, out string trapFile, out bool extracted) - { - trapFile = ""; - extracted = false; - try - { - var canonicalPathCache = CanonicalPathCache.Create(logger, 1000); - var pathTransformer = new PathTransformer(canonicalPathCache); - var extractor = new Extractor(false, assemblyPath, logger, pathTransformer); - var transformedAssemblyPath = pathTransformer.Transform(assemblyPath); - var project = layout.LookupProjectOrDefault(transformedAssemblyPath); - using var trapWriter = project.CreateTrapWriter(logger, transformedAssemblyPath.WithSuffix(".cil"), trapCompression, discardDuplicates: true); - trapFile = trapWriter.TrapFile; - if (nocache || !System.IO.File.Exists(trapFile)) - { - ExtractCIL(extractor, trapWriter, extractPdbs); - extracted = true; - } - } - catch (Exception ex) // lgtm[cs/catch-of-all-exceptions] - { - logger.Log(Severity.Error, string.Format("Exception extracting {0}: {1}", assemblyPath, ex)); - } - } } } diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/Base/IExtractedEntity.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/Base/IExtractedEntity.cs index 080227a63b9..d94b94df8a8 100644 --- a/csharp/extractor/Semmle.Extraction.CIL/Entities/Base/IExtractedEntity.cs +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/Base/IExtractedEntity.cs @@ -5,7 +5,7 @@ namespace Semmle.Extraction.CIL /// /// A CIL entity which has been extracted. /// - public interface IExtractedEntity : IExtractionProduct, IEntity + internal interface IExtractedEntity : IExtractionProduct, IEntity { /// /// The contents of the entity. diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/Base/IExtractionProduct.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/Base/IExtractionProduct.cs index 6383d7e1b4c..231a311a78c 100644 --- a/csharp/extractor/Semmle.Extraction.CIL/Entities/Base/IExtractionProduct.cs +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/Base/IExtractionProduct.cs @@ -12,7 +12,7 @@ namespace Semmle.Extraction.CIL /// - Enumerate Contents to produce more extraction products /// - Extract these until there is nothing left to extract /// - public interface IExtractionProduct + internal interface IExtractionProduct { /// /// Perform further extraction/population of this item as necessary. diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/Base/IGenericContext.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/Base/IGenericContext.cs index 21c69f7aeb4..65171e63cd0 100644 --- a/csharp/extractor/Semmle.Extraction.CIL/Entities/Base/IGenericContext.cs +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/Base/IGenericContext.cs @@ -6,7 +6,7 @@ namespace Semmle.Extraction.CIL /// When we decode a type/method signature, we need access to /// generic parameters. /// - public interface IGenericContext + internal interface IGenericContext { Context Cx { get; } diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/Base/LabelledEntity.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/Base/LabelledEntity.cs index 80ef03a1821..26f6c00cd0b 100644 --- a/csharp/extractor/Semmle.Extraction.CIL/Entities/Base/LabelledEntity.cs +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/Base/LabelledEntity.cs @@ -8,7 +8,7 @@ namespace Semmle.Extraction.CIL /// An entity that needs to be populated during extraction. /// This assigns a key and optionally extracts its contents. /// - public abstract class LabelledEntity : Extraction.LabelledEntity, IExtractedEntity + internal abstract class LabelledEntity : Extraction.LabelledEntity, IExtractedEntity { // todo: with .NET 5 this can override the base context, and change the return type. public Context Cx => (Context)base.Context; diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/Base/UnlabelledEntity.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/Base/UnlabelledEntity.cs index 3c9924cc98f..42d3824c00c 100644 --- a/csharp/extractor/Semmle.Extraction.CIL/Entities/Base/UnlabelledEntity.cs +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/Base/UnlabelledEntity.cs @@ -7,7 +7,7 @@ namespace Semmle.Extraction.CIL /// An entity that has contents to extract. There is no need to populate /// a key as it's done in the contructor. /// - public abstract class UnlabelledEntity : Extraction.UnlabelledEntity, IExtractedEntity + internal abstract class UnlabelledEntity : Extraction.UnlabelledEntity, IExtractedEntity { // todo: with .NET 5 this can override the base context, and change the return type. public Context Cx => (Context)base.Context; diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/ConstructedType.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/ConstructedType.cs index be2ab5da4d8..6e2fb90beae 100644 --- a/csharp/extractor/Semmle.Extraction.CIL/Entities/ConstructedType.cs +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/ConstructedType.cs @@ -9,7 +9,7 @@ namespace Semmle.Extraction.CIL.Entities /// /// A constructed type. /// - public sealed class ConstructedType : Type + internal sealed class ConstructedType : Type { private readonly Type unboundGenericType; diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/File.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/File.cs index ba447aebc3a..8e440795092 100644 --- a/csharp/extractor/Semmle.Extraction.CIL/Entities/File.cs +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/File.cs @@ -3,7 +3,7 @@ using System.IO; namespace Semmle.Extraction.CIL.Entities { - public class File : LabelledEntity, IFileOrFolder + internal class File : LabelledEntity, IFileOrFolder { protected string OriginalPath { get; } protected PathTransformer.ITransformedPath TransformedPath { get; } diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/Folder.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/Folder.cs index 2294a525e4b..a502a005724 100644 --- a/csharp/extractor/Semmle.Extraction.CIL/Entities/Folder.cs +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/Folder.cs @@ -3,7 +3,7 @@ using System.IO; namespace Semmle.Extraction.CIL.Entities { - public sealed class Folder : LabelledEntity, IFileOrFolder + internal sealed class Folder : LabelledEntity, IFileOrFolder { private readonly PathTransformer.ITransformedPath transformedPath; diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/Namespace.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/Namespace.cs index 5d498c5ac54..6cabce18795 100644 --- a/csharp/extractor/Semmle.Extraction.CIL/Entities/Namespace.cs +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/Namespace.cs @@ -7,7 +7,7 @@ namespace Semmle.Extraction.CIL.Entities /// /// A namespace. /// - public sealed class Namespace : TypeContainer + internal sealed class Namespace : TypeContainer { public Namespace? ParentNamespace { get; } public string Name { get; } diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/PdbSourceFile.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/PdbSourceFile.cs index 5dd5dfa37e6..97e9948f6a7 100644 --- a/csharp/extractor/Semmle.Extraction.CIL/Entities/PdbSourceFile.cs +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/PdbSourceFile.cs @@ -2,7 +2,7 @@ using System.Collections.Generic; namespace Semmle.Extraction.CIL.Entities { - public class PdbSourceFile : File + internal class PdbSourceFile : File { private readonly PDB.ISourceFile file; diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/PrimitiveType.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/PrimitiveType.cs index 661ae0f535b..d807c5778de 100644 --- a/csharp/extractor/Semmle.Extraction.CIL/Entities/PrimitiveType.cs +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/PrimitiveType.cs @@ -5,7 +5,7 @@ using System.IO; namespace Semmle.Extraction.CIL.Entities { - public sealed class PrimitiveType : Type + internal sealed class PrimitiveType : Type { private readonly PrimitiveTypeCode typeCode; public PrimitiveType(Context cx, PrimitiveTypeCode tc) : base(cx) diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/SourceLocation.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/SourceLocation.cs index 318cac14930..ff67844121a 100644 --- a/csharp/extractor/Semmle.Extraction.CIL/Entities/SourceLocation.cs +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/SourceLocation.cs @@ -4,7 +4,7 @@ using Semmle.Extraction.PDB; namespace Semmle.Extraction.CIL.Entities { - public sealed class PdbSourceLocation : LabelledEntity, ILocation + internal sealed class PdbSourceLocation : LabelledEntity, ILocation { private readonly Location location; private readonly PdbSourceFile file; diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/Type.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/Type.cs index ad5a6ababaa..9b452777031 100644 --- a/csharp/extractor/Semmle.Extraction.CIL/Entities/Type.cs +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/Type.cs @@ -9,7 +9,7 @@ namespace Semmle.Extraction.CIL.Entities /// /// A type. /// - public abstract class Type : TypeContainer, IMember + internal abstract class Type : TypeContainer, IMember { internal const string AssemblyTypeNameSeparator = "::"; internal const string PrimitiveTypePrefix = "builtin" + AssemblyTypeNameSeparator + "System."; diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/TypeContainer.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/TypeContainer.cs index 106b6fe83be..e201e065255 100644 --- a/csharp/extractor/Semmle.Extraction.CIL/Entities/TypeContainer.cs +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/TypeContainer.cs @@ -6,7 +6,7 @@ namespace Semmle.Extraction.CIL.Entities /// /// Base class for all type containers (namespaces, types, methods). /// - public abstract class TypeContainer : LabelledEntity, IGenericContext + internal abstract class TypeContainer : LabelledEntity, IGenericContext { protected TypeContainer(Context cx) : base(cx) { diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/TypeDefinitionType.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/TypeDefinitionType.cs index 461141ae719..3f9df14a891 100644 --- a/csharp/extractor/Semmle.Extraction.CIL/Entities/TypeDefinitionType.cs +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/TypeDefinitionType.cs @@ -11,7 +11,7 @@ namespace Semmle.Extraction.CIL.Entities /// /// A type defined in the current assembly. /// - public sealed class TypeDefinitionType : Type + internal sealed class TypeDefinitionType : Type { private readonly TypeDefinitionHandle handle; private readonly TypeDefinition td; diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/TypeReferenceType.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/TypeReferenceType.cs index 49a15965dd0..51677bf0bb9 100644 --- a/csharp/extractor/Semmle.Extraction.CIL/Entities/TypeReferenceType.cs +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/TypeReferenceType.cs @@ -9,7 +9,7 @@ namespace Semmle.Extraction.CIL.Entities /// /// A type reference, to a type in a referenced assembly. /// - public sealed class TypeReferenceType : Type + internal sealed class TypeReferenceType : Type { private readonly TypeReferenceHandle handle; private readonly TypeReference tr; diff --git a/csharp/extractor/Semmle.Extraction.CIL/Entities/TypeSignatureDecoder.cs b/csharp/extractor/Semmle.Extraction.CIL/Entities/TypeSignatureDecoder.cs index 4417becc58d..d1022d58f0c 100644 --- a/csharp/extractor/Semmle.Extraction.CIL/Entities/TypeSignatureDecoder.cs +++ b/csharp/extractor/Semmle.Extraction.CIL/Entities/TypeSignatureDecoder.cs @@ -8,7 +8,7 @@ namespace Semmle.Extraction.CIL.Entities /// /// Decodes a type signature and produces a Type, for use by DecodeSignature() and friends. /// - public class TypeSignatureDecoder : ISignatureTypeProvider + internal class TypeSignatureDecoder : ISignatureTypeProvider { private readonly Context cx; diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Analyser.cs b/csharp/extractor/Semmle.Extraction.CSharp/Analyser.cs index cf4f4e0ebf0..5ef8bd24d4e 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Analyser.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Analyser.cs @@ -310,7 +310,7 @@ namespace Semmle.Extraction.CSharp { var stopwatch = new Stopwatch(); stopwatch.Start(); - CIL.Entities.Assembly.ExtractCIL(layout, r.FilePath, Logger, !options.Cache, options.PDB, options.TrapCompression, out var trapFile, out var extracted); + CIL.Analyser.ExtractCIL(layout, r.FilePath, Logger, !options.Cache, options.PDB, options.TrapCompression, out var trapFile, out var extracted); stopwatch.Stop(); ReportProgress(r.FilePath, trapFile, stopwatch.Elapsed, extracted ? AnalysisAction.Extracted : AnalysisAction.UpToDate); } diff --git a/csharp/extractor/Semmle.Extraction.CSharp/CachedEntityFactory.cs b/csharp/extractor/Semmle.Extraction.CSharp/CachedEntityFactory.cs index 648d90be67f..2673ed0d7ef 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/CachedEntityFactory.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/CachedEntityFactory.cs @@ -1,11 +1,9 @@ -using Microsoft.CodeAnalysis; - namespace Semmle.Extraction.CSharp { /// /// A factory for creating cached entities. /// - public abstract class CachedEntityFactory + internal abstract class CachedEntityFactory : Extraction.CachedEntityFactory where TEntity : CachedEntity { /// diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Context.cs b/csharp/extractor/Semmle.Extraction.CSharp/Context.cs index 181795fbe91..314706c1e47 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Context.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Context.cs @@ -2,6 +2,7 @@ using Microsoft.CodeAnalysis; using System; using System.Diagnostics.CodeAnalysis; using Semmle.Extraction.Entities; +using System.Collections.Generic; namespace Semmle.Extraction.CSharp { @@ -9,7 +10,7 @@ namespace Semmle.Extraction.CSharp /// State that needs to be available throughout the extraction process. /// There is one Context object per trap output file. /// - public class Context : Extraction.Context + internal class Context : Extraction.Context { /// /// The program database provided by Roslyn. @@ -50,7 +51,7 @@ namespace Semmle.Extraction.CSharp public bool IsAssemblyScope => scope is AssemblyScope; - public SyntaxTree SourceTree => scope is SourceScope sc ? sc.SourceTree : null; + private SyntaxTree SourceTree => scope is SourceScope sc ? sc.SourceTree : null; /// /// Whether the given symbol needs to be defined in this context. @@ -116,5 +117,28 @@ namespace Semmle.Extraction.CSharp loc = null; return false; } + + private readonly HashSet