github
/
codeql
зеркало из https://github.com/github/codeql.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Merge pull request #14228 from tamasvajk/standalone-implicit-usings 

C#: Generate source file with implicit usings in Standalone
This commit is contained in:
Tamás Vajk 2023-09-18 13:26:09 +02:00 коммит произвёл GitHub
Родитель bd31e1004a fa814a5276
Коммит c4d7302f9e
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
8 изменённых файлов: 222 добавлений и 95 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

130
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs
Просмотреть файл

@ -30,9 +30,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
private readonly DotNet dotnet;
private readonly FileContent fileContent;
private readonly TemporaryDirectory packageDirectory;
private TemporaryDirectory? razorWorkingDirectory;
private readonly Git git;
private readonly TemporaryDirectory tempWorkingDirectory;
/// <summary>
/// Performs C# dependency fetching.
@ -60,20 +58,21 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
this.progressMonitor.FindingFiles(srcDir);
packageDirectory = new TemporaryDirectory(ComputeTempDirectory(sourceDir.FullName));
var allFiles = GetAllFiles().ToList();
var smallFiles = allFiles.SelectSmallFiles(progressMonitor).SelectFileNames();
this.fileContent = new FileContent(progressMonitor, smallFiles);
this.allSources = allFiles.SelectFileNamesByExtension(".cs").ToList();
var allProjects = allFiles.SelectFileNamesByExtension(".csproj");
tempWorkingDirectory = new TemporaryDirectory(GetTemporaryWorkingDirectory());
var allFiles = GetAllFiles();
var binaryFileExtensions = new HashSet<string>(new[] { ".dll", ".exe" }); // TODO: add more binary file extensions.
var allNonBinaryFiles = allFiles.Where(f => !binaryFileExtensions.Contains(f.Extension.ToLowerInvariant())).ToList();
var smallNonBinaryFiles = allNonBinaryFiles.SelectSmallFiles(progressMonitor).SelectFileNames();
this.fileContent = new FileContent(progressMonitor, smallNonBinaryFiles);
this.allSources = allNonBinaryFiles.SelectFileNamesByExtension(".cs").ToList();
var allProjects = allNonBinaryFiles.SelectFileNamesByExtension(".csproj");
var solutions = options.SolutionFile is not null
? new[] { options.SolutionFile }
: allFiles.SelectFileNamesByExtension(".sln");
// If DLL reference paths are specified on the command-line, use those to discover
// assemblies. Otherwise (the default), query the git CLI to determine which DLL files
// are tracked as part of the repository.
this.git = new Git(this.progressMonitor);
var dllDirNames = options.DllDirs.Count == 0 ? this.git.ListFiles("*.dll") : options.DllDirs.Select(Path.GetFullPath).ToList();
: allNonBinaryFiles.SelectFileNamesByExtension(".sln");
var dllDirNames = options.DllDirs.Count == 0
? allFiles.SelectFileNamesByExtension(".dll").ToList()
: options.DllDirs.Select(Path.GetFullPath).ToList();
// Find DLLs in the .Net / Asp.Net Framework
if (options.ScanNetFrameworkDlls)
@ -106,7 +105,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
var restoredProjects = RestoreSolutions(solutions);
var projects = allProjects.Except(restoredProjects);
RestoreProjects(projects);
DownloadMissingPackages(allFiles);
DownloadMissingPackages(allNonBinaryFiles);
}
assemblyCache = new AssemblyCache(dllDirNames, progressMonitor);
@ -134,9 +133,11 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
if (bool.TryParse(webViewExtractionOption, out var shouldExtractWebViews) &&
shouldExtractWebViews)
{
GenerateSourceFilesFromWebViews(allFiles);
GenerateSourceFilesFromWebViews(allNonBinaryFiles);
}
GenerateSourceFileFromImplicitUsings();
progressMonitor.Summary(
AllSourceFiles.Count(),
ProjectSourceFiles.Count(),
@ -149,6 +150,46 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
DateTime.Now - startTime);
}
private void GenerateSourceFileFromImplicitUsings()
{
var usings = new HashSet<string>();
if (!fileContent.UseImplicitUsings)
{
return;
}
// Hardcoded values from https://learn.microsoft.com/en-us/dotnet/core/project-sdk/overview#implicit-using-directives
usings.UnionWith(new[] { "System", "System.Collections.Generic", "System.IO", "System.Linq", "System.Net.Http", "System.Threading",
"System.Threading.Tasks" });
if (fileContent.UseAspNetDlls)
{
usings.UnionWith(new[] { "System.Net.Http.Json", "Microsoft.AspNetCore.Builder", "Microsoft.AspNetCore.Hosting",
"Microsoft.AspNetCore.Http", "Microsoft.AspNetCore.Routing", "Microsoft.Extensions.Configuration",
"Microsoft.Extensions.DependencyInjection", "Microsoft.Extensions.Hosting", "Microsoft.Extensions.Logging" });
}
usings.UnionWith(fileContent.CustomImplicitUsings);
if (usings.Count > 0)
{
var tempDir = GetTemporaryWorkingDirectory("implicitUsings");
var path = Path.Combine(tempDir, "GlobalUsings.g.cs");
using (var writer = new StreamWriter(path))
{
writer.WriteLine("// <auto-generated/>");
writer.WriteLine("");
foreach (var u in usings.OrderBy(u => u))
{
writer.WriteLine($"global using global::{u};");
}
}
this.allSources.Add(path);
}
}
private void GenerateSourceFilesFromWebViews(List<FileInfo> allFiles)
{
progressMonitor.LogInfo($"Generating source files from cshtml and razor files.");
@ -165,8 +206,8 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
try
{
var razor = new Razor(sdk, dotnet, progressMonitor);
razorWorkingDirectory = new TemporaryDirectory(ComputeTempDirectory(sourceDir.FullName, "razor"));
var generatedFiles = razor.GenerateFiles(views, usedReferences.Keys, razorWorkingDirectory.ToString());
var targetDir = GetTemporaryWorkingDirectory("razor");
var generatedFiles = razor.GenerateFiles(views, usedReferences.Keys, targetDir);
this.allSources.AddRange(generatedFiles);
}
catch (Exception ex)
@ -180,16 +221,25 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
public DependencyManager(string srcDir) : this(srcDir, DependencyOptions.Default, new ConsoleLogger(Verbosity.Info)) { }
private IEnumerable<FileInfo> GetAllFiles() =>
sourceDir.GetFiles("*.*", new EnumerationOptions { RecurseSubdirectories = true })
.Where(d => d.Extension != ".dll" && !options.ExcludesFile(d.FullName));
private IEnumerable<FileInfo> GetAllFiles()
{
var files = sourceDir.GetFiles("*.*", new EnumerationOptions { RecurseSubdirectories = true })
.Where(d => !options.ExcludesFile(d.FullName));
if (options.DotNetPath != null)
{
files = files.Where(f => !f.FullName.StartsWith(options.DotNetPath, StringComparison.OrdinalIgnoreCase));
}
return files;
}
/// <summary>
/// Computes a unique temp directory for the packages associated
/// with this source tree. Use a SHA1 of the directory name.
/// </summary>
/// <returns>The full path of the temp directory.</returns>
private static string ComputeTempDirectory(string srcDir, string subfolderName = "packages")
private static string ComputeTempDirectory(string srcDir)
{
var bytes = Encoding.Unicode.GetBytes(srcDir);
var sha = SHA1.HashData(bytes);
@ -197,7 +247,35 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
foreach (var b in sha.Take(8))
sb.AppendFormat("{0:x2}", b);
return Path.Combine(Path.GetTempPath(), "GitHub", subfolderName, sb.ToString());
return Path.Combine(Path.GetTempPath(), "GitHub", "packages", sb.ToString());
}
private static string GetTemporaryWorkingDirectory()
{
var tempFolder = EnvironmentVariables.GetScratchDirectory();
if (string.IsNullOrEmpty(tempFolder))
{
var tempPath = Path.GetTempPath();
var name = Guid.NewGuid().ToString("N").ToUpper();
tempFolder = Path.Combine(tempPath, "GitHub", name);
}
return tempFolder;
}
/// <summary>
/// Creates a temporary directory with the given subfolder name.
/// The created directory might be inside the repo folder, and it is deleted when the object is disposed.
/// </summary>
/// <param name="subfolder"></param>
/// <returns></returns>
private string GetTemporaryWorkingDirectory(string subfolder)
{
var temp = Path.Combine(tempWorkingDirectory.ToString(), subfolder);
Directory.CreateDirectory(temp);
return temp;
}
/// <summary>
@ -424,7 +502,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
foreach (var package in notYetDownloadedPackages)
{
progressMonitor.NugetInstall(package);
using var tempDir = new TemporaryDirectory(ComputeTempDirectory(package));
using var tempDir = new TemporaryDirectory(GetTemporaryWorkingDirectory(package));
var success = dotnet.New(tempDir.DirInfo.FullName);
if (!success)
{
@ -467,7 +545,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
public void Dispose()
{
packageDirectory?.Dispose();
razorWorkingDirectory?.Dispose();
tempWorkingDirectory?.Dispose();
}
}
}

56
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FileContent.cs
Просмотреть файл

@ -19,6 +19,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
private readonly IUnsafeFileReader unsafeFileReader;
private readonly IEnumerable<string> files;
private readonly HashSet<string> allPackages = new HashSet<string>();
private readonly HashSet<string> implicitUsingNamespaces = new HashSet<string>();
private readonly Initializer initialize;
public HashSet<string> AllPackages
@ -48,6 +49,26 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
}
}
private bool useImplicitUsings = false;
public bool UseImplicitUsings
{
get
{
initialize.Run();
return useImplicitUsings;
}
}
public HashSet<string> CustomImplicitUsings
{
get
{
initialize.Run();
return implicitUsingNamespaces;
}
}
internal FileContent(ProgressMonitor progressMonitor,
IEnumerable<string> files,
IUnsafeFileReader unsafeFileReader)
@ -62,7 +83,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
public FileContent(ProgressMonitor progressMonitor, IEnumerable<string> files) : this(progressMonitor, files, new UnsafeFileReader())
{ }
private static string GetGroup(ReadOnlySpan<char> input, ValueMatch valueMatch, string groupPrefix)
private static string GetGroup(ReadOnlySpan<char> input, ValueMatch valueMatch, string groupPrefix, bool toLower)
{
var match = input.Slice(valueMatch.Index, valueMatch.Length);
var includeIndex = match.IndexOf(groupPrefix, StringComparison.InvariantCultureIgnoreCase);
@ -76,7 +97,14 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
var quoteIndex1 = match.IndexOf("\"");
var quoteIndex2 = match.Slice(quoteIndex1 + 1).IndexOf("\"");
return match.Slice(quoteIndex1 + 1, quoteIndex2).ToString().ToLowerInvariant();
var result = match.Slice(quoteIndex1 + 1, quoteIndex2).ToString();
if (toLower)
{
result = result.ToLowerInvariant();
}
return result;
}
private static bool IsGroupMatch(ReadOnlySpan<char> line, Regex regex, string groupPrefix, string value)
@ -84,7 +112,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
foreach (var valueMatch in regex.EnumerateMatches(line))
{
// We can't get the group from the ValueMatch, so doing it manually:
if (GetGroup(line, valueMatch, groupPrefix) == value.ToLowerInvariant())
if (GetGroup(line, valueMatch, groupPrefix, toLower: true) == value.ToLowerInvariant())
{
return true;
}
@ -105,7 +133,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
foreach (var valueMatch in PackageReference().EnumerateMatches(line))
{
// We can't get the group from the ValueMatch, so doing it manually:
var packageName = GetGroup(line, valueMatch, "Include");
var packageName = GetGroup(line, valueMatch, "Include", toLower: true);
if (!string.IsNullOrEmpty(packageName))
{
allPackages.Add(packageName);
@ -119,6 +147,23 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
IsGroupMatch(line, ProjectSdk(), "Sdk", "Microsoft.NET.Sdk.Web") ||
IsGroupMatch(line, FrameworkReference(), "Include", "Microsoft.AspNetCore.App");
}
// Determine if implicit usings are used.
if (!useImplicitUsings)
{
useImplicitUsings = line.Contains("<ImplicitUsings>enable</ImplicitUsings>".AsSpan(), StringComparison.Ordinal) ||
line.Contains("<ImplicitUsings>true</ImplicitUsings>".AsSpan(), StringComparison.Ordinal);
}
// Find all custom implicit usings.
foreach (var valueMatch in CustomImplicitUsingDeclarations().EnumerateMatches(line))
{
var ns = GetGroup(line, valueMatch, "Include", toLower: false);
if (!string.IsNullOrEmpty(ns))
{
implicitUsingNamespaces.Add(ns);
}
}
}
}
catch (Exception ex)
@ -136,6 +181,9 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
[GeneratedRegex("<(.*\\s)?Project.*\\sSdk=\"(.*?)\".*/?>", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline)]
private static partial Regex ProjectSdk();
[GeneratedRegex("<Using.*\\sInclude=\"(.*?)\".*/?>", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline)]
private static partial Regex CustomImplicitUsingDeclarations();
}
internal interface IUnsafeFileReader

61
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/Git.cs
Просмотреть файл

@ -1,61 +0,0 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
namespace Semmle.Extraction.CSharp.DependencyFetching
{
/// <summary>
/// Utilities for querying information from the git CLI.
/// </summary>
internal class Git
{
private readonly ProgressMonitor progressMonitor;
private const string git = "git";
public Git(ProgressMonitor progressMonitor)
{
this.progressMonitor = progressMonitor;
}
/// <summary>
/// Lists all files matching <paramref name="pattern"/> which are tracked in the
/// current git repository.
/// </summary>
/// <param name="pattern">The file pattern.</param>
/// <returns>A list of all tracked files which match <paramref name="pattern"/>.</returns>
/// <exception cref="Exception"></exception>
public List<string> ListFiles(string pattern)
{
var results = new List<string>();
var args = string.Join(' ', "ls-files", $"\"{pattern}\"");
progressMonitor.RunningProcess($"{git} {args}");
var pi = new ProcessStartInfo(git, args)
{
UseShellExecute = false,
RedirectStandardOutput = true
};
using var p = new Process() { StartInfo = pi };
p.OutputDataReceived += new DataReceivedEventHandler((sender, e) =>
{
if (!string.IsNullOrWhiteSpace(e.Data))
{
results.Add(e.Data);
}
});
p.Start();
p.BeginOutputReadLine();
p.WaitForExit();
if (p.ExitCode != 0)
{
progressMonitor.CommandFailed(git, args, p.ExitCode);
throw new Exception($"{git} {args} failed");
}
return results;
}
}
}

55
csharp/extractor/Semmle.Extraction.Tests/FileContent.cs
Просмотреть файл

@ -90,5 +90,60 @@ namespace Semmle.Extraction.Tests
Assert.Contains("Microsoft.CodeAnalysis.NetAnalyzers".ToLowerInvariant(), allPackages);
Assert.Contains("StyleCop.Analyzers".ToLowerInvariant(), allPackages);
}
private static void ImplicitUsingsTest(string line, bool expected)
{
// Setup
var lines = new List<string>()
{
line
};
var fileContent = new TestFileContent(lines);
// Execute
var useImplicitUsings = fileContent.UseImplicitUsings;
// Verify
Assert.Equal(expected, useImplicitUsings);
}
[Fact]
public void TestFileContent_ImplicitUsings0()
{
ImplicitUsingsTest("<ImplicitUsings>false</ImplicitUsings>", false);
}
[Fact]
public void TestFileContent_ImplicitUsings1()
{
ImplicitUsingsTest("<ImplicitUsings>true</ImplicitUsings>", true);
}
[Fact]
public void TestFileContent_ImplicitUsings2()
{
ImplicitUsingsTest("<ImplicitUsings>enable</ImplicitUsings>", true);
}
[Fact]
public void TestFileContent_ImplicitUsingsAdditional()
{
// Setup
var lines = new List<string>()
{
"<Using Include=\"Ns0.Ns1\" />",
"<Using Include=\"Ns2\" />",
"<Using Remove=\"Ns3\" />",
};
var fileContent = new TestFileContent(lines);
// Execute
var customImplicitUsings = fileContent.CustomImplicitUsings;
// Verify
Assert.Equal(2, customImplicitUsings.Count);
Assert.Contains("Ns0.Ns1", customImplicitUsings);
Assert.Contains("Ns2", customImplicitUsings);
}
}
}

2
csharp/extractor/Semmle.Util/EnvironmentVariables.cs
Просмотреть файл

@ -7,6 +7,8 @@ namespace Semmle.Util
public static string? GetExtractorOption(string name) =>
Environment.GetEnvironmentVariable($"CODEQL_EXTRACTOR_CSHARP_OPTION_{name.ToUpper()}");
public static string? GetScratchDirectory() => Environment.GetEnvironmentVariable("CODEQL_EXTRACTOR_CSHARP_SCRATCH_DIR");
public static int GetDefaultNumberOfThreads()
{
if (!int.TryParse(Environment.GetEnvironmentVariable("CODEQL_THREADS"), out var threads) || threads == -1)

1
csharp/ql/integration-tests/all-platforms/cshtml_standalone/Files.expected
Просмотреть файл

@ -1,3 +1,4 @@
| Program.cs |
| Views/Home/Index.cshtml |
| _semmle_code_target_codeql_csharp_integration_tests_ql_csharp_ql_integration_tests_all_platforms_cshtml_standalone_Views_Home_Index_cshtml.g.cs |
| test-db/working/implicitUsings/GlobalUsings.g.cs |

11
csharp/ql/integration-tests/all-platforms/cshtml_standalone/Files.ql
Просмотреть файл

@ -1,14 +1,17 @@
import csharp
private string getPath(File f) {
result = f.getRelativePath()
result = f.getRelativePath() and
not exists(
result
.indexOf("_semmle_code_target_codeql_csharp_integration_tests_ql_csharp_ql_integration_tests_all_platforms_cshtml_standalone_")
)
or
not exists(f.getRelativePath()) and
exists(int index |
index =
f.getBaseName()
f.getRelativePath()
.indexOf("_semmle_code_target_codeql_csharp_integration_tests_ql_csharp_ql_integration_tests_all_platforms_cshtml_standalone_") and
result = f.getBaseName().substring(index, f.getBaseName().length())
result = f.getRelativePath().substring(index, f.getRelativePath().length())
)
}

1
csharp/ql/integration-tests/all-platforms/standalone/Files.expected
Просмотреть файл

@ -1 +1,2 @@
| Program.cs:0:0:0:0 | Program.cs |
| test-db/working/implicitUsings/GlobalUsings.g.cs:0:0:0:0 | test-db/working/implicitUsings/GlobalUsings.g.cs |