From d725bd91690eec1edeac10770855e03210e9fb65 Mon Sep 17 00:00:00 2001 From: Tamas Vajk Date: Fri, 15 Sep 2023 10:52:30 +0200 Subject: [PATCH 1/3] C#: Generate source file with implicit usings in Standalone --- .../DependencyManager.cs | 130 ++++++++++++++---- .../FileContent.cs | 56 +++++++- .../Git.cs | 61 -------- .../Semmle.Util/EnvironmentVariables.cs | 2 + 4 files changed, 158 insertions(+), 91 deletions(-) delete mode 100644 csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/Git.cs diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs index 5d026722134..e8d9b7760c7 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs @@ -30,9 +30,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching private readonly DotNet dotnet; private readonly FileContent fileContent; private readonly TemporaryDirectory packageDirectory; - private TemporaryDirectory? razorWorkingDirectory; - private readonly Git git; - + private readonly TemporaryDirectory tempWorkingDirectory; /// /// Performs C# dependency fetching. @@ -60,20 +58,21 @@ namespace Semmle.Extraction.CSharp.DependencyFetching this.progressMonitor.FindingFiles(srcDir); packageDirectory = new TemporaryDirectory(ComputeTempDirectory(sourceDir.FullName)); - var allFiles = GetAllFiles().ToList(); - var smallFiles = allFiles.SelectSmallFiles(progressMonitor).SelectFileNames(); - this.fileContent = new FileContent(progressMonitor, smallFiles); - this.allSources = allFiles.SelectFileNamesByExtension(".cs").ToList(); - var allProjects = allFiles.SelectFileNamesByExtension(".csproj"); + tempWorkingDirectory = new TemporaryDirectory(GetTemporaryWorkingDirectory()); + + var allFiles = GetAllFiles(); + var binaryFileExtensions = new HashSet(new[] { ".dll", ".exe" }); // TODO: add more binary file extensions. + var allNonBinaryFiles = allFiles.Where(f => !binaryFileExtensions.Contains(f.Extension.ToLowerInvariant())).ToList(); + var smallNonBinaryFiles = allNonBinaryFiles.SelectSmallFiles(progressMonitor).SelectFileNames(); + this.fileContent = new FileContent(progressMonitor, smallNonBinaryFiles); + this.allSources = allNonBinaryFiles.SelectFileNamesByExtension(".cs").ToList(); + var allProjects = allNonBinaryFiles.SelectFileNamesByExtension(".csproj"); var solutions = options.SolutionFile is not null ? new[] { options.SolutionFile } - : allFiles.SelectFileNamesByExtension(".sln"); - - // If DLL reference paths are specified on the command-line, use those to discover - // assemblies. Otherwise (the default), query the git CLI to determine which DLL files - // are tracked as part of the repository. - this.git = new Git(this.progressMonitor); - var dllDirNames = options.DllDirs.Count == 0 ? this.git.ListFiles("*.dll") : options.DllDirs.Select(Path.GetFullPath).ToList(); + : allNonBinaryFiles.SelectFileNamesByExtension(".sln"); + var dllDirNames = options.DllDirs.Count == 0 + ? allFiles.SelectFileNamesByExtension(".dll").ToList() + : options.DllDirs.Select(Path.GetFullPath).ToList(); // Find DLLs in the .Net / Asp.Net Framework if (options.ScanNetFrameworkDlls) @@ -106,7 +105,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching var restoredProjects = RestoreSolutions(solutions); var projects = allProjects.Except(restoredProjects); RestoreProjects(projects); - DownloadMissingPackages(allFiles); + DownloadMissingPackages(allNonBinaryFiles); } assemblyCache = new AssemblyCache(dllDirNames, progressMonitor); @@ -134,9 +133,11 @@ namespace Semmle.Extraction.CSharp.DependencyFetching if (bool.TryParse(webViewExtractionOption, out var shouldExtractWebViews) && shouldExtractWebViews) { - GenerateSourceFilesFromWebViews(allFiles); + GenerateSourceFilesFromWebViews(allNonBinaryFiles); } + GenerateSourceFileFromImplicitUsings(); + progressMonitor.Summary( AllSourceFiles.Count(), ProjectSourceFiles.Count(), @@ -149,6 +150,46 @@ namespace Semmle.Extraction.CSharp.DependencyFetching DateTime.Now - startTime); } + private void GenerateSourceFileFromImplicitUsings() + { + var usings = new HashSet(); + if (!fileContent.UseImplicitUsings) + { + return; + } + + // Hardcoded values from https://learn.microsoft.com/en-us/dotnet/core/project-sdk/overview#implicit-using-directives + usings.UnionWith(new[] { "System", "System.Collections.Generic", "System.IO", "System.Linq", "System.Net.Http", "System.Threading", + "System.Threading.Tasks" }); + + if (fileContent.UseAspNetDlls) + { + usings.UnionWith(new[] { "System.Net.Http.Json", "Microsoft.AspNetCore.Builder", "Microsoft.AspNetCore.Hosting", + "Microsoft.AspNetCore.Http", "Microsoft.AspNetCore.Routing", "Microsoft.Extensions.Configuration", + "Microsoft.Extensions.DependencyInjection", "Microsoft.Extensions.Hosting", "Microsoft.Extensions.Logging" }); + } + + usings.UnionWith(fileContent.CustomImplicitUsings); + + if (usings.Count > 0) + { + var tempDir = GetTemporaryWorkingDirectory("implicitUsings"); + var path = Path.Combine(tempDir, "GlobalUsings.g.cs"); + using (var writer = new StreamWriter(path)) + { + writer.WriteLine("// "); + writer.WriteLine(""); + + foreach (var u in usings.OrderBy(u => u)) + { + writer.WriteLine($"global using global::{u};"); + } + } + + this.allSources.Add(path); + } + } + private void GenerateSourceFilesFromWebViews(List allFiles) { progressMonitor.LogInfo($"Generating source files from cshtml and razor files."); @@ -165,8 +206,8 @@ namespace Semmle.Extraction.CSharp.DependencyFetching try { var razor = new Razor(sdk, dotnet, progressMonitor); - razorWorkingDirectory = new TemporaryDirectory(ComputeTempDirectory(sourceDir.FullName, "razor")); - var generatedFiles = razor.GenerateFiles(views, usedReferences.Keys, razorWorkingDirectory.ToString()); + var targetDir = GetTemporaryWorkingDirectory("razor"); + var generatedFiles = razor.GenerateFiles(views, usedReferences.Keys, targetDir); this.allSources.AddRange(generatedFiles); } catch (Exception ex) @@ -180,16 +221,25 @@ namespace Semmle.Extraction.CSharp.DependencyFetching public DependencyManager(string srcDir) : this(srcDir, DependencyOptions.Default, new ConsoleLogger(Verbosity.Info)) { } - private IEnumerable GetAllFiles() => - sourceDir.GetFiles("*.*", new EnumerationOptions { RecurseSubdirectories = true }) - .Where(d => d.Extension != ".dll" && !options.ExcludesFile(d.FullName)); + private IEnumerable GetAllFiles() + { + var files = sourceDir.GetFiles("*.*", new EnumerationOptions { RecurseSubdirectories = true }) + .Where(d => !options.ExcludesFile(d.FullName)); + + if (options.DotNetPath != null) + { + files = files.Where(f => !f.FullName.StartsWith(options.DotNetPath, StringComparison.OrdinalIgnoreCase)); + } + + return files; + } /// /// Computes a unique temp directory for the packages associated /// with this source tree. Use a SHA1 of the directory name. /// /// The full path of the temp directory. - private static string ComputeTempDirectory(string srcDir, string subfolderName = "packages") + private static string ComputeTempDirectory(string srcDir) { var bytes = Encoding.Unicode.GetBytes(srcDir); var sha = SHA1.HashData(bytes); @@ -197,7 +247,35 @@ namespace Semmle.Extraction.CSharp.DependencyFetching foreach (var b in sha.Take(8)) sb.AppendFormat("{0:x2}", b); - return Path.Combine(Path.GetTempPath(), "GitHub", subfolderName, sb.ToString()); + return Path.Combine(Path.GetTempPath(), "GitHub", "packages", sb.ToString()); + } + + private static string GetTemporaryWorkingDirectory() + { + var tempFolder = EnvironmentVariables.GetScratchDirectory(); + + if (string.IsNullOrEmpty(tempFolder)) + { + var tempPath = Path.GetTempPath(); + var name = Guid.NewGuid().ToString("N").ToUpper(); + tempFolder = Path.Combine(tempPath, "GitHub", name); + } + + return tempFolder; + } + + /// + /// Creates a temporary directory with the given subfolder name. + /// The created directory might be inside the repo folder, and it is deleted when the object is disposed. + /// + /// + /// + private string GetTemporaryWorkingDirectory(string subfolder) + { + var temp = Path.Combine(tempWorkingDirectory.ToString(), subfolder); + Directory.CreateDirectory(temp); + + return temp; } /// @@ -424,7 +502,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching foreach (var package in notYetDownloadedPackages) { progressMonitor.NugetInstall(package); - using var tempDir = new TemporaryDirectory(ComputeTempDirectory(package)); + using var tempDir = new TemporaryDirectory(GetTemporaryWorkingDirectory(package)); var success = dotnet.New(tempDir.DirInfo.FullName); if (!success) { @@ -467,7 +545,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching public void Dispose() { packageDirectory?.Dispose(); - razorWorkingDirectory?.Dispose(); + tempWorkingDirectory?.Dispose(); } } } diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FileContent.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FileContent.cs index 35bdce4750d..1dd0ad42318 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FileContent.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FileContent.cs @@ -19,6 +19,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching private readonly IUnsafeFileReader unsafeFileReader; private readonly IEnumerable files; private readonly HashSet allPackages = new HashSet(); + private readonly HashSet implicitUsingNamespaces = new HashSet(); private readonly Initializer initialize; public HashSet AllPackages @@ -48,6 +49,26 @@ namespace Semmle.Extraction.CSharp.DependencyFetching } } + private bool useImplicitUsings = false; + + public bool UseImplicitUsings + { + get + { + initialize.Run(); + return useImplicitUsings; + } + } + + public HashSet CustomImplicitUsings + { + get + { + initialize.Run(); + return implicitUsingNamespaces; + } + } + internal FileContent(ProgressMonitor progressMonitor, IEnumerable files, IUnsafeFileReader unsafeFileReader) @@ -62,7 +83,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching public FileContent(ProgressMonitor progressMonitor, IEnumerable files) : this(progressMonitor, files, new UnsafeFileReader()) { } - private static string GetGroup(ReadOnlySpan input, ValueMatch valueMatch, string groupPrefix) + private static string GetGroup(ReadOnlySpan input, ValueMatch valueMatch, string groupPrefix, bool toLower) { var match = input.Slice(valueMatch.Index, valueMatch.Length); var includeIndex = match.IndexOf(groupPrefix, StringComparison.InvariantCultureIgnoreCase); @@ -76,7 +97,14 @@ namespace Semmle.Extraction.CSharp.DependencyFetching var quoteIndex1 = match.IndexOf("\""); var quoteIndex2 = match.Slice(quoteIndex1 + 1).IndexOf("\""); - return match.Slice(quoteIndex1 + 1, quoteIndex2).ToString().ToLowerInvariant(); + var result = match.Slice(quoteIndex1 + 1, quoteIndex2).ToString(); + + if (toLower) + { + result = result.ToLowerInvariant(); + } + + return result; } private static bool IsGroupMatch(ReadOnlySpan line, Regex regex, string groupPrefix, string value) @@ -84,7 +112,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching foreach (var valueMatch in regex.EnumerateMatches(line)) { // We can't get the group from the ValueMatch, so doing it manually: - if (GetGroup(line, valueMatch, groupPrefix) == value.ToLowerInvariant()) + if (GetGroup(line, valueMatch, groupPrefix, toLower: true) == value.ToLowerInvariant()) { return true; } @@ -105,7 +133,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching foreach (var valueMatch in PackageReference().EnumerateMatches(line)) { // We can't get the group from the ValueMatch, so doing it manually: - var packageName = GetGroup(line, valueMatch, "Include"); + var packageName = GetGroup(line, valueMatch, "Include", toLower: true); if (!string.IsNullOrEmpty(packageName)) { allPackages.Add(packageName); @@ -119,6 +147,23 @@ namespace Semmle.Extraction.CSharp.DependencyFetching IsGroupMatch(line, ProjectSdk(), "Sdk", "Microsoft.NET.Sdk.Web") || IsGroupMatch(line, FrameworkReference(), "Include", "Microsoft.AspNetCore.App"); } + + // Determine if implicit usings are used. + if (!useImplicitUsings) + { + useImplicitUsings = line.Contains("enable".AsSpan(), StringComparison.Ordinal) || + line.Contains("true".AsSpan(), StringComparison.Ordinal); + } + + // Find all custom implicit usings. + foreach (var valueMatch in CustomImplicitUsingDeclarations().EnumerateMatches(line)) + { + var ns = GetGroup(line, valueMatch, "Include", toLower: false); + if (!string.IsNullOrEmpty(ns)) + { + implicitUsingNamespaces.Add(ns); + } + } } } catch (Exception ex) @@ -136,6 +181,9 @@ namespace Semmle.Extraction.CSharp.DependencyFetching [GeneratedRegex("<(.*\\s)?Project.*\\sSdk=\"(.*?)\".*/?>", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline)] private static partial Regex ProjectSdk(); + + [GeneratedRegex("", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline)] + private static partial Regex CustomImplicitUsingDeclarations(); } internal interface IUnsafeFileReader diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/Git.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/Git.cs deleted file mode 100644 index d8baf49483b..00000000000 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/Git.cs +++ /dev/null @@ -1,61 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Diagnostics; - -namespace Semmle.Extraction.CSharp.DependencyFetching -{ - /// - /// Utilities for querying information from the git CLI. - /// - internal class Git - { - private readonly ProgressMonitor progressMonitor; - private const string git = "git"; - - public Git(ProgressMonitor progressMonitor) - { - this.progressMonitor = progressMonitor; - } - - /// - /// Lists all files matching which are tracked in the - /// current git repository. - /// - /// The file pattern. - /// A list of all tracked files which match . - /// - public List ListFiles(string pattern) - { - var results = new List(); - var args = string.Join(' ', "ls-files", $"\"{pattern}\""); - - progressMonitor.RunningProcess($"{git} {args}"); - var pi = new ProcessStartInfo(git, args) - { - UseShellExecute = false, - RedirectStandardOutput = true - }; - - using var p = new Process() { StartInfo = pi }; - p.OutputDataReceived += new DataReceivedEventHandler((sender, e) => - { - if (!string.IsNullOrWhiteSpace(e.Data)) - { - results.Add(e.Data); - } - }); - p.Start(); - p.BeginOutputReadLine(); - p.WaitForExit(); - - if (p.ExitCode != 0) - { - progressMonitor.CommandFailed(git, args, p.ExitCode); - throw new Exception($"{git} {args} failed"); - } - - return results; - } - - } -} diff --git a/csharp/extractor/Semmle.Util/EnvironmentVariables.cs b/csharp/extractor/Semmle.Util/EnvironmentVariables.cs index 9dcccf6d878..e6735e8741a 100644 --- a/csharp/extractor/Semmle.Util/EnvironmentVariables.cs +++ b/csharp/extractor/Semmle.Util/EnvironmentVariables.cs @@ -7,6 +7,8 @@ namespace Semmle.Util public static string? GetExtractorOption(string name) => Environment.GetEnvironmentVariable($"CODEQL_EXTRACTOR_CSHARP_OPTION_{name.ToUpper()}"); + public static string? GetScratchDirectory() => Environment.GetEnvironmentVariable("CODEQL_EXTRACTOR_CSHARP_SCRATCH_DIR"); + public static int GetDefaultNumberOfThreads() { if (!int.TryParse(Environment.GetEnvironmentVariable("CODEQL_THREADS"), out var threads) || threads == -1) From c34fef1eb64fbb9b63fa6e6f328bf0386e6ddc64 Mon Sep 17 00:00:00 2001 From: Tamas Vajk Date: Fri, 15 Sep 2023 13:35:25 +0200 Subject: [PATCH 2/3] Adjust integration tests after path changes and generating file with global usings --- .../all-platforms/cshtml_standalone/Files.expected | 1 + .../all-platforms/cshtml_standalone/Files.ql | 11 +++++++---- .../all-platforms/standalone/Files.expected | 1 + 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/csharp/ql/integration-tests/all-platforms/cshtml_standalone/Files.expected b/csharp/ql/integration-tests/all-platforms/cshtml_standalone/Files.expected index 853daba72d3..ee3b14ffc8f 100644 --- a/csharp/ql/integration-tests/all-platforms/cshtml_standalone/Files.expected +++ b/csharp/ql/integration-tests/all-platforms/cshtml_standalone/Files.expected @@ -1,3 +1,4 @@ | Program.cs | | Views/Home/Index.cshtml | | _semmle_code_target_codeql_csharp_integration_tests_ql_csharp_ql_integration_tests_all_platforms_cshtml_standalone_Views_Home_Index_cshtml.g.cs | +| test-db/working/implicitUsings/GlobalUsings.g.cs | diff --git a/csharp/ql/integration-tests/all-platforms/cshtml_standalone/Files.ql b/csharp/ql/integration-tests/all-platforms/cshtml_standalone/Files.ql index db5c1b8c8d0..2d983b86b7c 100644 --- a/csharp/ql/integration-tests/all-platforms/cshtml_standalone/Files.ql +++ b/csharp/ql/integration-tests/all-platforms/cshtml_standalone/Files.ql @@ -1,14 +1,17 @@ import csharp private string getPath(File f) { - result = f.getRelativePath() + result = f.getRelativePath() and + not exists( + result + .indexOf("_semmle_code_target_codeql_csharp_integration_tests_ql_csharp_ql_integration_tests_all_platforms_cshtml_standalone_") + ) or - not exists(f.getRelativePath()) and exists(int index | index = - f.getBaseName() + f.getRelativePath() .indexOf("_semmle_code_target_codeql_csharp_integration_tests_ql_csharp_ql_integration_tests_all_platforms_cshtml_standalone_") and - result = f.getBaseName().substring(index, f.getBaseName().length()) + result = f.getRelativePath().substring(index, f.getRelativePath().length()) ) } diff --git a/csharp/ql/integration-tests/all-platforms/standalone/Files.expected b/csharp/ql/integration-tests/all-platforms/standalone/Files.expected index 2085aa342e7..50089b3c59c 100644 --- a/csharp/ql/integration-tests/all-platforms/standalone/Files.expected +++ b/csharp/ql/integration-tests/all-platforms/standalone/Files.expected @@ -1 +1,2 @@ | Program.cs:0:0:0:0 | Program.cs | +| test-db/working/implicitUsings/GlobalUsings.g.cs:0:0:0:0 | test-db/working/implicitUsings/GlobalUsings.g.cs | From fa814a5276cadcf10bb14c998dc1c357218ce2df Mon Sep 17 00:00:00 2001 From: Tamas Vajk Date: Mon, 18 Sep 2023 12:47:50 +0200 Subject: [PATCH 3/3] Add test cases for implicit using parsing --- .../Semmle.Extraction.Tests/FileContent.cs | 55 +++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/csharp/extractor/Semmle.Extraction.Tests/FileContent.cs b/csharp/extractor/Semmle.Extraction.Tests/FileContent.cs index 00a7238103e..e6915900e2b 100644 --- a/csharp/extractor/Semmle.Extraction.Tests/FileContent.cs +++ b/csharp/extractor/Semmle.Extraction.Tests/FileContent.cs @@ -90,5 +90,60 @@ namespace Semmle.Extraction.Tests Assert.Contains("Microsoft.CodeAnalysis.NetAnalyzers".ToLowerInvariant(), allPackages); Assert.Contains("StyleCop.Analyzers".ToLowerInvariant(), allPackages); } + + private static void ImplicitUsingsTest(string line, bool expected) + { + // Setup + var lines = new List() + { + line + }; + var fileContent = new TestFileContent(lines); + + // Execute + var useImplicitUsings = fileContent.UseImplicitUsings; + + // Verify + Assert.Equal(expected, useImplicitUsings); + } + + [Fact] + public void TestFileContent_ImplicitUsings0() + { + ImplicitUsingsTest("false", false); + } + + [Fact] + public void TestFileContent_ImplicitUsings1() + { + ImplicitUsingsTest("true", true); + } + + [Fact] + public void TestFileContent_ImplicitUsings2() + { + ImplicitUsingsTest("enable", true); + } + + [Fact] + public void TestFileContent_ImplicitUsingsAdditional() + { + // Setup + var lines = new List() + { + "", + "", + "", + }; + var fileContent = new TestFileContent(lines); + + // Execute + var customImplicitUsings = fileContent.CustomImplicitUsings; + + // Verify + Assert.Equal(2, customImplicitUsings.Count); + Assert.Contains("Ns0.Ns1", customImplicitUsings); + Assert.Contains("Ns2", customImplicitUsings); + } } }