зеркало из https://github.com/microsoft/BuildXL.git
Merged PR 642321: Remove Mimic and MimicGenerator
These haven't been used in years and won't likely be revived. Deleting to clean up codebase. They can be retrieved from version history if necessary.
This commit is contained in:
Michael Pysson
Родитель
38688fac46
Коммит
e615dc34e5
|
|
@ -1,148 +0,0 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT License.
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using BuildXL.ToolSupport;
|
||||
|
||||
namespace Tool.Mimic
|
||||
{
|
||||
internal sealed class Args : CommandLineUtilities
|
||||
{
|
||||
public List<ReadFile> ReadFiles = new List<ReadFile>();
|
||||
public string ObservedAccesses = null;
|
||||
public string ObservedAccessesRoot = null;
|
||||
public List<WriteFile> WriteFiles = new List<WriteFile>();
|
||||
public List<EnumerateDirectory> EnumerateDirectories = new List<EnumerateDirectory>();
|
||||
public List<ProbeAbsentFile> ProbeAbsentFiles = new List<ProbeAbsentFile>();
|
||||
public TimeSpan ProcessDuration = new TimeSpan(0);
|
||||
public bool AllowMissingInputs = false;
|
||||
public bool CreateOutputDirectories = false;
|
||||
public bool IgnoreFilesOverlappingDirectories = false;
|
||||
public double IOScaleFactor = 1;
|
||||
public double RuntimeScaleFactor = 1;
|
||||
public bool Spin = true;
|
||||
|
||||
private static readonly string[] s_helpStrings = new[] { "?", "help" };
|
||||
|
||||
public Args(string[] args)
|
||||
: base(args)
|
||||
{
|
||||
foreach (Option opt in Options)
|
||||
{
|
||||
if (opt.Name.Equals("DurationMS", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
ProcessDuration = TimeSpan.FromMilliseconds(ParseInt32Option(opt, 0, int.MaxValue));
|
||||
}
|
||||
else if (opt.Name.Equals("Input", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
ReadFiles.Add(new ReadFile(ParsePathOption(opt)));
|
||||
}
|
||||
else if (opt.Name.Equals("observedAccesses", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
ObservedAccesses = ParsePathOption(opt);
|
||||
}
|
||||
else if (opt.Name.Equals("observedAccessesRoot", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
ObservedAccessesRoot = ParsePathOption(opt);
|
||||
}
|
||||
else if (opt.Name.Equals("Output", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
string[] split = opt.Value.Split('|');
|
||||
|
||||
if (split.Length != 3)
|
||||
{
|
||||
throw Error("Output option must be of form: /Output:<path>|<lengthInBytes>|<repeatingContent>");
|
||||
}
|
||||
|
||||
string path = Path.GetFullPath(TrimPathQuotation(split[0]));
|
||||
long length;
|
||||
|
||||
if (!long.TryParse(split[1], out length))
|
||||
{
|
||||
throw Error("Could not parse '{0}' to a long for /Output option file length.", split[1]);
|
||||
}
|
||||
|
||||
var mimicSalt = Environment.GetEnvironmentVariable("mimicSalt");
|
||||
|
||||
WriteFiles.Add(new WriteFile(path, length, split[2] + (!string.IsNullOrWhiteSpace(mimicSalt) ? mimicSalt : string.Empty)));
|
||||
}
|
||||
else if (opt.Name.StartsWith("AllowMissingInputs", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
AllowMissingInputs = ParseBooleanOption(opt);
|
||||
}
|
||||
else if (opt.Name.StartsWith("CreateOutputDirectories", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
CreateOutputDirectories = ParseBooleanOption(opt);
|
||||
}
|
||||
else if (opt.Name.StartsWith("IgnoreFilesOverlappingDirectories", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
IgnoreFilesOverlappingDirectories = ParseBooleanOption(opt);
|
||||
}
|
||||
else if (opt.Name.Equals("IOScaleFactor", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
IOScaleFactor = ParseDoubleOption(opt, 0, double.MaxValue);
|
||||
}
|
||||
else if (opt.Name.Equals("RuntimeScaleFactor", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
RuntimeScaleFactor = ParseDoubleOption(opt, 0, double.MaxValue);
|
||||
}
|
||||
else if (opt.Name.StartsWith("Spin", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
Spin = ParseBooleanOption(opt);
|
||||
}
|
||||
else if (s_helpStrings.Any(s => opt.Name.Equals(s, StringComparison.OrdinalIgnoreCase)))
|
||||
{
|
||||
WriteHelp();
|
||||
}
|
||||
}
|
||||
|
||||
if (ProcessDuration.Ticks == 0)
|
||||
{
|
||||
throw Error("DurationMS parameter is required");
|
||||
}
|
||||
}
|
||||
|
||||
private static void WriteHelp()
|
||||
{
|
||||
HelpWriter writer = new HelpWriter();
|
||||
writer.WriteBanner("Mimic.exe - Mimics a process performing reads and writes. After reads and writes the process will spin until the requested duration is reached.");
|
||||
|
||||
writer.WriteOption("DurationMs", "Required. Duration in milliseconds the process should run.");
|
||||
writer.WriteOption("Input", "A file that will be read sequentially to the end");
|
||||
writer.WriteOption("Output", "A file that will be written sequentially for a specified number of bytes");
|
||||
writer.WriteOption("AllowMissingInputs", "Performs a file existence check and no-ops reading an input if files are missing. Disabled by default.");
|
||||
writer.WriteOption("CreateOutputDirectories", "Creates directories for output files if they don't already exist. Disabled by default.");
|
||||
writer.WriteOption("IgnoreFilesOverlappingDirectories", "Skips attempting to create a file at a path if a directory already exists there.");
|
||||
writer.WriteOption("RuntimeScaleFactor", "Scales DurationMs.");
|
||||
writer.WriteOption("IOScaleFactor", "Scales IO by the factor. If smaller than 1, only part of inputs will be read. If larger, inputs will be read multiple times");
|
||||
writer.WriteOption("Spin", "True if the process should spin for its runtime. If false the process will sleep. Defaults to true");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Removes surrounding single or double quotes from a path.
|
||||
/// We need specific logic for Mimic here so this method hides the implementation in the base class.
|
||||
/// No need to put mimic specific logic to the 'BuildXL.ToolSupport'.
|
||||
/// </summary>
|
||||
public new static string TrimPathQuotation(string path)
|
||||
{
|
||||
foreach (char c in new char[] { '"', '\'' })
|
||||
{
|
||||
if (path.Length > 2 && path[0] == c && path[path.Length - 1] == c)
|
||||
{
|
||||
path = path.Substring(1, path.Length - 2);
|
||||
}
|
||||
else if (path.Length > 2 && path[0] == c)
|
||||
{
|
||||
// Consider DS specs for Mimic. The quotes surround the whole arg including 'repeatingContent' and 'filelength'.
|
||||
// example: "\"C:\\find build.disco|685|9011d941953457ac27515fdcce3029187d376cae\""
|
||||
path = path.Substring(1, path.Length - 1);
|
||||
}
|
||||
}
|
||||
|
||||
return path;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,33 +0,0 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT License.
|
||||
|
||||
using System;
|
||||
using System.IO;
|
||||
|
||||
namespace Tool.Mimic
|
||||
{
|
||||
/// <summary>
|
||||
/// Enumerates a directory
|
||||
/// </summary>
|
||||
internal sealed class EnumerateDirectory
|
||||
{
|
||||
internal readonly string Path;
|
||||
|
||||
internal EnumerateDirectory(string path)
|
||||
{
|
||||
Path = path;
|
||||
}
|
||||
|
||||
internal void Enumerate()
|
||||
{
|
||||
string members = "N/A";
|
||||
if (Directory.Exists(Path))
|
||||
{
|
||||
members = string.Join(",", Directory.GetFiles(Path, "*", SearchOption.TopDirectoryOnly));
|
||||
}
|
||||
|
||||
Console.WriteLine("Enumerate: {0}. Members: {1}", Path, members);
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,27 +0,0 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT License.
|
||||
|
||||
using System;
|
||||
using System.IO;
|
||||
|
||||
namespace Tool.Mimic
|
||||
{
|
||||
/// <summary>
|
||||
/// Enumerates a directory
|
||||
/// </summary>
|
||||
internal sealed class ProbeAbsentFile
|
||||
{
|
||||
internal readonly string Path;
|
||||
|
||||
internal ProbeAbsentFile(string path)
|
||||
{
|
||||
Path = path;
|
||||
}
|
||||
|
||||
internal void Probe()
|
||||
{
|
||||
Console.WriteLine("Probe: {0}. Exists: {1}", Path, File.Exists(Path));
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,113 +0,0 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT License.
|
||||
|
||||
using System;
|
||||
using System.Diagnostics;
|
||||
using System.IO;
|
||||
using System.Threading;
|
||||
|
||||
namespace Tool.Mimic
|
||||
{
|
||||
/// <summary>
|
||||
/// Mimics a process
|
||||
/// </summary>
|
||||
public static class Program
|
||||
{
|
||||
public static void Main(string[] arguments)
|
||||
{
|
||||
Args args = new Args(arguments);
|
||||
try
|
||||
{
|
||||
// File accesses can be defined in 2 forms: Either the actual observed accesses are serialized to a file
|
||||
// or they can be specified on the command line. The external file takes presidence.
|
||||
if (args.ObservedAccesses != null)
|
||||
{
|
||||
args.ReadFiles.Clear();
|
||||
args.ProbeAbsentFiles.Clear();
|
||||
args.EnumerateDirectories.Clear();
|
||||
|
||||
using (StreamReader reader = new StreamReader(args.ObservedAccesses))
|
||||
{
|
||||
const string ObservedAccessesVersion = "1";
|
||||
string version = reader.ReadLine();
|
||||
if (version != ObservedAccessesVersion)
|
||||
{
|
||||
Console.Error.WriteLine("ObservedAccesses version doesn't match expected version: " + ObservedAccessesVersion);
|
||||
Environment.Exit(-1);
|
||||
}
|
||||
|
||||
if (args.ObservedAccessesRoot == null)
|
||||
{
|
||||
Console.Error.WriteLine("/observedAccessesRoot is required when /observedAccesses is specified");
|
||||
Environment.Exit(-1);
|
||||
}
|
||||
|
||||
while (!reader.EndOfStream)
|
||||
{
|
||||
string path = reader.ReadLine();
|
||||
path = Path.Combine(args.ObservedAccessesRoot, path);
|
||||
switch (reader.ReadLine())
|
||||
{
|
||||
case "A":
|
||||
args.ProbeAbsentFiles.Add(new ProbeAbsentFile(path));
|
||||
break;
|
||||
case "D":
|
||||
args.EnumerateDirectories.Add(new EnumerateDirectory(path));
|
||||
break;
|
||||
case "F":
|
||||
args.ReadFiles.Add(new ReadFile(path));
|
||||
break;
|
||||
default:
|
||||
Console.Error.WriteLine("ObservedAccesses file is not the correct format. " + args.ObservedAccesses);
|
||||
Environment.Exit(-1);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
foreach (ReadFile read in args.ReadFiles)
|
||||
{
|
||||
read.Read(args.AllowMissingInputs, args.IgnoreFilesOverlappingDirectories);
|
||||
}
|
||||
|
||||
foreach (WriteFile write in args.WriteFiles)
|
||||
{
|
||||
write.Write(args.CreateOutputDirectories, args.IgnoreFilesOverlappingDirectories, args.IOScaleFactor);
|
||||
}
|
||||
|
||||
foreach (EnumerateDirectory enumerate in args.EnumerateDirectories)
|
||||
{
|
||||
enumerate.Enumerate();
|
||||
}
|
||||
|
||||
foreach (ProbeAbsentFile probe in args.ProbeAbsentFiles)
|
||||
{
|
||||
probe.Probe();
|
||||
}
|
||||
|
||||
// Spin for remainder of duration
|
||||
DateTime processStartTime = Process.GetCurrentProcess().StartTime.ToUniversalTime();
|
||||
TimeSpan duration = TimeSpan.FromMilliseconds(args.ProcessDuration.TotalMilliseconds * args.RuntimeScaleFactor);
|
||||
while (DateTime.UtcNow - processStartTime < duration)
|
||||
{
|
||||
if (args.Spin)
|
||||
{
|
||||
Thread.SpinWait(10);
|
||||
}
|
||||
else
|
||||
{
|
||||
Thread.Sleep(10);
|
||||
}
|
||||
}
|
||||
|
||||
Environment.Exit(0);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Console.Error.Write(ex);
|
||||
Environment.Exit(1);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,44 +0,0 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT License.
|
||||
|
||||
using System;
|
||||
using System.IO;
|
||||
|
||||
namespace Tool.Mimic
|
||||
{
|
||||
/// <summary>
|
||||
/// Reades a file
|
||||
/// </summary>
|
||||
internal sealed class ReadFile
|
||||
{
|
||||
internal readonly string Path;
|
||||
|
||||
internal ReadFile(string path)
|
||||
{
|
||||
Path = path;
|
||||
}
|
||||
|
||||
internal void Read(bool ignoreIfMissing, bool ignoreFilesOverlappingDirectories)
|
||||
{
|
||||
if (ignoreIfMissing && !File.Exists(Path))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if (ignoreFilesOverlappingDirectories && Directory.Exists(Path))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
using (StreamReader reader = new StreamReader(Path))
|
||||
{
|
||||
while (!reader.EndOfStream)
|
||||
{
|
||||
reader.ReadLine();
|
||||
}
|
||||
}
|
||||
|
||||
Console.WriteLine("Read File: {0}.", Path);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,15 +0,0 @@
|
|||
// Copyright (c) Microsoft. All rights reserved.
|
||||
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
|
||||
|
||||
namespace Mimic {
|
||||
@@public
|
||||
export const exe = BuildXLSdk.executable({
|
||||
assemblyName: "Mimic",
|
||||
rootNamespace: "Tool.Mimic",
|
||||
skipDocumentationGeneration: true,
|
||||
sources: globR(d`.`, "*.cs"),
|
||||
references: [
|
||||
importFrom("BuildXL.Utilities").ToolSupport.dll,
|
||||
],
|
||||
});
|
||||
}
|
||||
|
|
@ -1,70 +0,0 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT License.
|
||||
|
||||
using System;
|
||||
using System.Diagnostics.CodeAnalysis;
|
||||
using System.IO;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
|
||||
namespace Tool.Mimic
|
||||
{
|
||||
/// <summary>
|
||||
/// Writes a file
|
||||
/// </summary>
|
||||
internal sealed class WriteFile
|
||||
{
|
||||
internal readonly string Path;
|
||||
internal readonly long LengthInBytes;
|
||||
internal readonly string RepeatingContent;
|
||||
|
||||
internal WriteFile(string path, long lengthInBytes, string repeatingContent)
|
||||
{
|
||||
Path = path;
|
||||
LengthInBytes = lengthInBytes;
|
||||
RepeatingContent = repeatingContent;
|
||||
}
|
||||
|
||||
[SuppressMessage("Microsoft.Usage", "CA2202:Do not dispose objects multiple times")]
|
||||
internal void Write(bool createDirectory, bool ignoreFilesOverlappingDirectories, double scaleFactor)
|
||||
{
|
||||
if (createDirectory)
|
||||
{
|
||||
Directory.CreateDirectory(System.IO.Path.GetDirectoryName(Path));
|
||||
}
|
||||
|
||||
if (ignoreFilesOverlappingDirectories && Directory.Exists(Path))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
#pragma warning disable CA5351 // Do not use insecure cryptographic algorithm MD5.
|
||||
using (var md5 = MD5.Create())
|
||||
#pragma warning restore CA5351 // Do not use insecure cryptographic algorithm MD5.
|
||||
{
|
||||
byte[] hash = md5.ComputeHash(Encoding.UTF8.GetBytes(RepeatingContent ?? string.Empty));
|
||||
int seedFromHash = BitConverter.ToInt32(hash, 0);
|
||||
|
||||
Random rng = new Random(seedFromHash);
|
||||
byte[] bytes = new byte[8192];
|
||||
|
||||
long remainingScaledFileSize = (long)(LengthInBytes * scaleFactor);
|
||||
|
||||
using (FileStream s = new FileStream(Path, FileMode.Create, FileAccess.Write))
|
||||
{
|
||||
using (BinaryWriter writer = new BinaryWriter(s, Encoding.ASCII, leaveOpen: true))
|
||||
{
|
||||
while (remainingScaledFileSize > 0)
|
||||
{
|
||||
rng.NextBytes(bytes);
|
||||
writer.Write(bytes, 0, (int)Math.Min(bytes.Length, remainingScaledFileSize));
|
||||
remainingScaledFileSize -= bytes.Length;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Console.WriteLine("Write File: {0}", Path);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,148 +0,0 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT License.
|
||||
|
||||
using System;
|
||||
using System.Linq;
|
||||
using BuildXL.ToolSupport;
|
||||
|
||||
namespace Tool.MimicGenerator
|
||||
{
|
||||
/// <summary>
|
||||
/// Command line arguments for MimicGenerator
|
||||
/// </summary>
|
||||
internal sealed class Args : CommandLineUtilities
|
||||
{
|
||||
/// <summary>
|
||||
/// Whether help text was displayed. When true the application should exit without using the Args
|
||||
/// </summary>
|
||||
public bool HelpDisplayed = false;
|
||||
|
||||
/// <summary>
|
||||
/// Full path of JsonGraph input
|
||||
/// </summary>
|
||||
public string JsonGraph;
|
||||
|
||||
/// <summary>
|
||||
/// Full path of the ObservedInputs file
|
||||
/// </summary>
|
||||
public string ObservedInputs;
|
||||
|
||||
/// <summary>
|
||||
/// Full path to destination to write files
|
||||
/// </summary>
|
||||
public string Dest;
|
||||
|
||||
/// <summary>
|
||||
/// True if input files should be written
|
||||
/// </summary>
|
||||
public bool WriteInputs = false;
|
||||
|
||||
/// <summary>
|
||||
/// True if response file pips from source build should be excluded from created mimic build
|
||||
/// </summary>
|
||||
public bool IgnoreResponseFiles = false;
|
||||
|
||||
/// <summary>
|
||||
/// Scale factor for input files
|
||||
/// </summary>
|
||||
public double InputScaleFactor = 1;
|
||||
|
||||
/// <summary>
|
||||
/// Number of times to duplicate the graph
|
||||
/// </summary>
|
||||
public int DuplicateGraph = 0;
|
||||
|
||||
/// <summary>
|
||||
/// Maximum number of pips to include in a spec file before breaking it into a second spec file
|
||||
/// </summary>
|
||||
public int MaxPipsPerSpec = int.MaxValue;
|
||||
|
||||
/// <summary>
|
||||
/// Which language will be used to generate specs, module, and config files
|
||||
/// </summary>
|
||||
public Language Language;
|
||||
|
||||
private static readonly string[] s_helpStrings = new[] { "?", "help" };
|
||||
|
||||
/// <inhertdoc/>
|
||||
public Args(string[] args)
|
||||
: base(args)
|
||||
{
|
||||
foreach (Option opt in Options)
|
||||
{
|
||||
if (opt.Name.Equals("JsonGraph", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
JsonGraph = ParsePathOption(opt);
|
||||
}
|
||||
|
||||
if (opt.Name.Equals("ObservedInputs", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
ObservedInputs = ParsePathOption(opt);
|
||||
}
|
||||
else if (opt.Name.Equals("Dest", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
Dest = ParsePathOption(opt);
|
||||
}
|
||||
else if (opt.Name.Equals("WriteInputs", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
WriteInputs = ParseBooleanOption(opt);
|
||||
}
|
||||
else if (opt.Name.Equals("InputScaleFactor", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
InputScaleFactor = ParseDoubleOption(opt, 0, double.MaxValue);
|
||||
}
|
||||
else if (opt.Name.Equals("IgnoreResponseFiles", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
IgnoreResponseFiles = ParseBooleanOption(opt);
|
||||
}
|
||||
else if (opt.Name.Equals("Language", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
Language = ParseEnumOption<Language>(opt);
|
||||
}
|
||||
else if (opt.Name.Equals("DuplicateGraph", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
DuplicateGraph = ParseInt32Option(opt, 0, int.MaxValue);
|
||||
}
|
||||
else if (opt.Name.Equals("MaxPipsPerSpec", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
MaxPipsPerSpec = ParseInt32Option(opt, 0, int.MaxValue);
|
||||
}
|
||||
else if (s_helpStrings.Any(s => opt.Name.Equals(s, StringComparison.OrdinalIgnoreCase)))
|
||||
{
|
||||
WriteHelp();
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(JsonGraph))
|
||||
{
|
||||
throw Error("JsonGraph parameter is required");
|
||||
}
|
||||
else if (string.IsNullOrWhiteSpace(Dest))
|
||||
{
|
||||
throw Error("Dest parameter is required");
|
||||
}
|
||||
else if (Language == Language.None)
|
||||
{
|
||||
throw Error("Language parameter is required");
|
||||
}
|
||||
}
|
||||
|
||||
private void WriteHelp()
|
||||
{
|
||||
HelpWriter writer = new HelpWriter();
|
||||
writer.WriteBanner("MimicGenerator.exe - Creates spec files to perform mimiced build.");
|
||||
|
||||
writer.WriteOption("JsonGraph", "Required. Path to the json graph.");
|
||||
writer.WriteOption("ObservedInputs", "Optional. Dump of observed file accesses. When specified, pips will mimic these accesses.");
|
||||
writer.WriteOption("Dest", "Required. Destination for generated specs.");
|
||||
writer.WriteOption("WriteInputs", "Whether input files should be written in addition to BuildXL files. Defaults to false.");
|
||||
writer.WriteOption("InputScaleFactor", "Scale factor to increase or decrease the size of input files. Defaults to 1.");
|
||||
writer.WriteOption("IgnoreResponseFiles", "Whether response files from the input build should be ignored in mimiced build. Defaults to false.");
|
||||
writer.WriteOption("Language", "Language of files to write. Available options: 'DScript'");
|
||||
writer.WriteOption("DuplicateGraph", "Duplicates the graph the number of times specified");
|
||||
|
||||
HelpDisplayed = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,223 +0,0 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT License.
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics.CodeAnalysis;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace Tool.MimicGenerator
|
||||
{
|
||||
/// <summary>
|
||||
/// Analyzes json build graphs to pull out various statistics and make comparisons. This created to pull various stats
|
||||
/// into a csv for excel.
|
||||
/// </summary>
|
||||
public static class BuildAnalyzer
|
||||
{
|
||||
private const string PhoneSpecDirRootMarker = "\\wm";
|
||||
private const string ConvertedSpecName = "sources.ds";
|
||||
|
||||
private struct ProjectStats
|
||||
{
|
||||
public int ProcessCount;
|
||||
public int CopyFileCount;
|
||||
public int WriteFileCount;
|
||||
public int CppFileCount;
|
||||
public int CsFileCount;
|
||||
public int OutputFileCount;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Compares a wrapper spec and native spec build
|
||||
/// </summary>
|
||||
[SuppressMessage("Microsoft.Naming", "CA2204")]
|
||||
public static void CompareWrapperAndConvPhoneBuilds()
|
||||
{
|
||||
// TODO:= Set these paths if you intend on using this
|
||||
var wrapperTask = ReadGraph(@"D:\mimic\graphs\\DBuild.M.x86fre.json");
|
||||
var convTask = ReadGraph(@"D:\mimic\graphs\PhonePartialNativeSpecX86fre.json");
|
||||
|
||||
BuildGraph wrapperGraph = wrapperTask.Result;
|
||||
BuildGraph convGraph = convTask.Result;
|
||||
|
||||
Dictionary<string, Tuple<List<Pip>, List<Pip>>> pipsBySpec = new Dictionary<string, Tuple<List<Pip>, List<Pip>>>();
|
||||
HashSet<string> convertedPaths = new HashSet<string>();
|
||||
|
||||
foreach (var pip in wrapperGraph.Pips.Values)
|
||||
{
|
||||
string relativeSpec = PrepareCollection(pip, pipsBySpec);
|
||||
if (relativeSpec == null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
pipsBySpec[relativeSpec].Item1.Add(pip);
|
||||
}
|
||||
|
||||
foreach (var pip in convGraph.Pips.Values)
|
||||
{
|
||||
string relativeSpec = PrepareCollection(pip, pipsBySpec);
|
||||
|
||||
if (pip.Spec.EndsWith(ConvertedSpecName, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
convertedPaths.Add(relativeSpec);
|
||||
}
|
||||
|
||||
if (relativeSpec == null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
pipsBySpec[relativeSpec].Item2.Add(pip);
|
||||
}
|
||||
|
||||
Console.WriteLine("WrapperProcesses,WrapperCopyFile,WrapperWriteFile,WrapperOutputCount,WasConverted,ConvProcesses,ConvCopyFile,ConvWriteFile,CppCount,CsCount,ConvPrediction,SpecDir");
|
||||
foreach (var item in pipsBySpec)
|
||||
{
|
||||
ProjectStats wrapperStats = GetStats(item.Value.Item1, wrapperGraph);
|
||||
ProjectStats convStats = GetStats(item.Value.Item2, convGraph);
|
||||
|
||||
int convPrediction = 0;
|
||||
if (wrapperStats.CppFileCount > 0)
|
||||
{
|
||||
convPrediction = wrapperStats.CppFileCount + 2;
|
||||
}
|
||||
else
|
||||
{
|
||||
convPrediction = wrapperStats.ProcessCount;
|
||||
}
|
||||
|
||||
bool wasConverted = convertedPaths.Contains(item.Key);
|
||||
|
||||
Console.WriteLine("{0},{1},{2},{3},{4},{5},{6},{7},{8},{9},{10},{11}", wrapperStats.ProcessCount, wrapperStats.CopyFileCount, wrapperStats.WriteFileCount, wrapperStats.OutputFileCount,
|
||||
wasConverted,
|
||||
convStats.ProcessCount, convStats.CopyFileCount, convStats.WriteFileCount,
|
||||
wrapperStats.CppFileCount, wrapperStats.CsFileCount,
|
||||
convPrediction,
|
||||
item.Key);
|
||||
}
|
||||
}
|
||||
|
||||
private static Task<BuildGraph> ReadGraph(string fullPath)
|
||||
{
|
||||
return Task.Factory.StartNew(() =>
|
||||
{
|
||||
GraphReader reader = new GraphReader(fullPath, null);
|
||||
return reader.ReadGraph();
|
||||
});
|
||||
}
|
||||
|
||||
private static string PrepareCollection(Pip pip, Dictionary<string, Tuple<List<Pip>, List<Pip>>> pipsBySpec)
|
||||
{
|
||||
int start = pip.Spec.IndexOf(PhoneSpecDirRootMarker, StringComparison.OrdinalIgnoreCase);
|
||||
|
||||
if (start == -1)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
string directory = Path.GetDirectoryName(pip.Spec);
|
||||
|
||||
string relativeSpec = directory.Substring(start, directory.Length - start);
|
||||
Tuple<List<Pip>, List<Pip>> tuple;
|
||||
if (!pipsBySpec.ContainsKey(relativeSpec))
|
||||
{
|
||||
tuple = new Tuple<List<Pip>, List<Pip>>(new List<Pip>(), new List<Pip>());
|
||||
pipsBySpec.Add(relativeSpec, tuple);
|
||||
}
|
||||
|
||||
return relativeSpec;
|
||||
}
|
||||
|
||||
private static ProjectStats GetStats(IEnumerable<Pip> pips, BuildGraph graph)
|
||||
{
|
||||
ProjectStats stats = default(ProjectStats);
|
||||
HashSet<string> inputFiles = new HashSet<string>();
|
||||
HashSet<string> outputFiles = new HashSet<string>();
|
||||
|
||||
foreach (Pip pip in pips)
|
||||
{
|
||||
Process process = pip as Process;
|
||||
|
||||
if (process != null)
|
||||
{
|
||||
stats.ProcessCount++;
|
||||
foreach (int consumes in process.Consumes)
|
||||
{
|
||||
File f;
|
||||
if (graph.Files.TryGetValue(consumes, out f))
|
||||
{
|
||||
inputFiles.Add(f.Location);
|
||||
}
|
||||
}
|
||||
|
||||
foreach (int produces in process.Produces)
|
||||
{
|
||||
File f;
|
||||
if (graph.Files.TryGetValue(produces, out f))
|
||||
{
|
||||
outputFiles.Add(f.Location);
|
||||
}
|
||||
}
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
CopyFile copyFile = pip as CopyFile;
|
||||
if (copyFile != null)
|
||||
{
|
||||
stats.CopyFileCount++;
|
||||
continue;
|
||||
}
|
||||
|
||||
WriteFile writeFile = pip as WriteFile;
|
||||
if (writeFile != null)
|
||||
{
|
||||
stats.WriteFileCount++;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
stats.CppFileCount = inputFiles.Where(f => f.EndsWith(".cpp", StringComparison.OrdinalIgnoreCase)).Count();
|
||||
stats.CsFileCount = inputFiles.Where(f => f.EndsWith(".cs", StringComparison.OrdinalIgnoreCase)).Count();
|
||||
stats.OutputFileCount = outputFiles.Count;
|
||||
|
||||
return stats;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// For debugging
|
||||
/// </summary>
|
||||
[SuppressMessage("Microsoft.Performance", "CA1811")]
|
||||
private static string DescribeProcess(Process p, BuildGraph graph)
|
||||
{
|
||||
StringBuilder desc = new StringBuilder();
|
||||
desc.AppendLine("Consumes:");
|
||||
foreach (var file in p.Consumes)
|
||||
{
|
||||
File f;
|
||||
if (graph.Files.TryGetValue(file, out f))
|
||||
{
|
||||
desc.Append(" ");
|
||||
desc.AppendLine(f.Location);
|
||||
}
|
||||
}
|
||||
|
||||
desc.AppendLine("Produces:");
|
||||
foreach (var file in p.Produces)
|
||||
{
|
||||
File f;
|
||||
if (graph.Files.TryGetValue(file, out f))
|
||||
{
|
||||
desc.Append(" ");
|
||||
desc.AppendLine(f.Location);
|
||||
}
|
||||
}
|
||||
|
||||
return desc.ToString();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,459 +0,0 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT License.
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Globalization;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using BuildXL.Utilities.Collections;
|
||||
|
||||
namespace Tool.MimicGenerator
|
||||
{
|
||||
/// <summary>
|
||||
/// A build graph read in from a json file
|
||||
/// </summary>
|
||||
public sealed class BuildGraph
|
||||
{
|
||||
/// <summary>
|
||||
/// Directory id to directory
|
||||
/// </summary>
|
||||
public readonly ConcurrentBigMap<int, Dir> Directories = new ConcurrentBigMap<int, Dir>();
|
||||
|
||||
/// <summary>
|
||||
/// File id to file. Do not add directly add files to this collection. Instead, use the <see cref="TryAddFile(int, File)"/> method.
|
||||
/// </summary>
|
||||
public readonly ConcurrentBigMap<int, File> Files = new ConcurrentBigMap<int, File>();
|
||||
|
||||
/// <summary>
|
||||
/// string to file id. This doesn't take rewrites into account, so the FileId returned is arbitrary.
|
||||
/// That approximation is generally ok for the sake of duplicating dependencies
|
||||
/// </summary>
|
||||
public readonly ConcurrentBigMap<string, int> FilesByPath = new ConcurrentBigMap<string, int>();
|
||||
|
||||
/// <summary>
|
||||
/// PipId to pip
|
||||
/// </summary>
|
||||
public readonly ConcurrentBigMap<int, Pip> Pips = new ConcurrentBigMap<int, Pip>();
|
||||
|
||||
/// <summary>
|
||||
/// SemiStableHash to PipId
|
||||
/// </summary>
|
||||
public readonly ConcurrentBigMap<string, Pip> SemiStableHashes = new ConcurrentBigMap<string, Pip>();
|
||||
|
||||
/// <summary>
|
||||
/// FileId to the producing pip
|
||||
/// </summary>
|
||||
public readonly ConcurrentBigMap<int, int> OutputArtifactToProducingPip = new ConcurrentBigMap<int, int>();
|
||||
|
||||
/// <summary>
|
||||
/// Observed accesses for process pips
|
||||
/// </summary>
|
||||
public readonly ConcurrentBigMap<int, ObservedAccess[]> ObservedAccesses = new ConcurrentBigMap<int, ObservedAccess[]>();
|
||||
|
||||
/// <summary>
|
||||
/// The statistics for the output files
|
||||
/// </summary>
|
||||
public readonly Stats OutputFileStats = new Stats("OutputFileSize");
|
||||
|
||||
/// <summary>
|
||||
/// The statistics for the output files
|
||||
/// </summary>
|
||||
public readonly Stats SourceFileStats = new Stats("SourceFileSize");
|
||||
|
||||
/// <summary>
|
||||
/// The statistics for the output files
|
||||
/// </summary>
|
||||
public readonly Stats PipDurationStats = new Stats("PipDuration");
|
||||
|
||||
/// <summary>
|
||||
/// The statistics for process pips
|
||||
/// </summary>
|
||||
public readonly Stats ProcessPipStats = new Stats("ProcessPips");
|
||||
|
||||
/// <summary>
|
||||
/// The total time of the build
|
||||
/// </summary>
|
||||
public Interval BuildInterval = new Interval("BuildInterval");
|
||||
|
||||
/// <summary>
|
||||
/// Registers an output file and its producing pip
|
||||
/// </summary>
|
||||
public void AddOutputArtifact(int outputId, int producerId, bool isDirectory = false)
|
||||
{
|
||||
if (!OutputArtifactToProducingPip.TryAdd(outputId, producerId))
|
||||
{
|
||||
throw new MimicGeneratorException("Encountered duplicate output artifact. OutputId:{0}. ProducerId:{1}", outputId, producerId);
|
||||
}
|
||||
|
||||
if (isDirectory)
|
||||
{
|
||||
Dir dir;
|
||||
if (Directories.TryGetValue(outputId, out dir))
|
||||
{
|
||||
dir.ProducerId = producerId;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
File file;
|
||||
if (Files.TryGetValue(outputId, out file))
|
||||
{
|
||||
file.IsOutputFile = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public bool TryAddPip(Pip p)
|
||||
{
|
||||
bool added = Pips.TryAdd(p.PipId, p);
|
||||
if (added)
|
||||
{
|
||||
SemiStableHashes.TryAdd(p.StableId, p);
|
||||
}
|
||||
|
||||
m_maxPipId = Math.Max(m_maxPipId, p.PipId);
|
||||
|
||||
return added;
|
||||
}
|
||||
|
||||
public int AddWithNewPipId(Pip p, int originalPip)
|
||||
{
|
||||
p.OriginalPipId = originalPip;
|
||||
p.PipId = ++m_maxPipId;
|
||||
if (!Pips.TryAdd(p.PipId, p))
|
||||
{
|
||||
throw new MimicGeneratorException("Failed to add new pip");
|
||||
}
|
||||
|
||||
return p.PipId;
|
||||
}
|
||||
|
||||
private int m_maxPipId;
|
||||
|
||||
public bool TryAddFile(int id, File file)
|
||||
{
|
||||
m_maxFile = Math.Max(m_maxFile, id);
|
||||
bool result = Files.TryAdd(id, file);
|
||||
if (result)
|
||||
{
|
||||
FilesByPath[file.Location] = id;
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
public int DuplicateFile(File file, string newRoot)
|
||||
{
|
||||
string newLocation = DuplicatePath(file.Location, newRoot);
|
||||
int existing;
|
||||
if (FilesByPath.TryGetValue(newLocation, out existing))
|
||||
{
|
||||
return existing;
|
||||
}
|
||||
else
|
||||
{
|
||||
File newFile = new File(newLocation) { Hash = Guid.NewGuid().ToString(), IsOutputFile = file.IsOutputFile };
|
||||
int newFileId = ++m_maxFile;
|
||||
Files[newFileId] = newFile;
|
||||
FilesByPath.Add(newLocation, newFileId);
|
||||
newFile.SetUnscaledLength(file.GetScaledLengthInBytes(1.0));
|
||||
return newFileId;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Duplicates a path by injecting a new root within the path.
|
||||
///
|
||||
/// path: d:\foo\bar.txt
|
||||
/// newRoot: 13
|
||||
/// result: d:\mim13\foo\bar.txt
|
||||
/// </summary>
|
||||
public static string DuplicatePath(string path, string newRoot)
|
||||
{
|
||||
int drive = path.IndexOf(Path.DirectorySeparatorChar);
|
||||
int split = drive;
|
||||
if (path.StartsWith("mim", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
split = path.IndexOf(Path.DirectorySeparatorChar, split);
|
||||
}
|
||||
|
||||
string newLocation = string.Format(CultureInfo.InvariantCulture, @"{0}\mim{1}{2}",
|
||||
path.Substring(0, drive),
|
||||
newRoot,
|
||||
path.Substring(split, path.Length - split));
|
||||
return newLocation;
|
||||
}
|
||||
|
||||
private int m_maxFile;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// File in the build graph
|
||||
/// </summary>
|
||||
public sealed class File
|
||||
{
|
||||
private const int DefaultFileLength = 14000;
|
||||
private const int UnsetFileLength = -1;
|
||||
|
||||
/// <summary>
|
||||
/// Location of the file
|
||||
/// </summary>
|
||||
public readonly string Location;
|
||||
|
||||
/// <summary>
|
||||
/// Indicates if the file is an output file
|
||||
/// </summary>
|
||||
public bool IsOutputFile;
|
||||
|
||||
public void SetUnscaledLength(int length)
|
||||
{
|
||||
// Size of -1 is in the json graph when the size could not be computed. Fall back on the default
|
||||
// in that case.
|
||||
if (length > -1)
|
||||
{
|
||||
m_lengthInBytes = length;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Returns true if the
|
||||
/// </summary>
|
||||
public bool WasLengthSet
|
||||
{
|
||||
get
|
||||
{
|
||||
return m_lengthInBytes != UnsetFileLength;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Get the scaled length
|
||||
/// </summary>
|
||||
public int GetScaledLengthInBytes(double scaleFactor)
|
||||
{
|
||||
if (WasLengthSet)
|
||||
{
|
||||
return (int)(m_lengthInBytes * scaleFactor);
|
||||
}
|
||||
else
|
||||
{
|
||||
return (int)(DefaultFileLength * scaleFactor);
|
||||
}
|
||||
}
|
||||
|
||||
private int m_lengthInBytes = UnsetFileLength;
|
||||
|
||||
/// <summary>
|
||||
/// Hash of file
|
||||
/// </summary>
|
||||
public string Hash
|
||||
{
|
||||
get
|
||||
{
|
||||
if (m_hash == null)
|
||||
{
|
||||
// Hash the filename if no hash was provided
|
||||
#pragma warning disable CA5351 // Do not use insecure cryptographic algorithm MD5.
|
||||
using (var md5 = MD5.Create())
|
||||
#pragma warning restore CA5351 // Do not use insecure cryptographic algorithm MD5.
|
||||
{
|
||||
m_hash = string.Join(string.Empty, md5.ComputeHash(Encoding.UTF8.GetBytes(Location)).Select(b => b.ToString("X2", CultureInfo.InvariantCulture)));
|
||||
}
|
||||
}
|
||||
|
||||
return m_hash;
|
||||
}
|
||||
|
||||
set
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
m_hash = value;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private string m_hash;
|
||||
|
||||
public File(string path)
|
||||
{
|
||||
Location = path;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Statistics for a set of values
|
||||
/// </summary>
|
||||
public sealed class Interval
|
||||
{
|
||||
private long m_min = long.MaxValue;
|
||||
private long m_max;
|
||||
private string m_displayPrefix;
|
||||
|
||||
/// <summary>
|
||||
/// The min of the values
|
||||
/// </summary>
|
||||
public long Min
|
||||
{
|
||||
get { return m_min; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The max of the values
|
||||
/// </summary>
|
||||
public long Max
|
||||
{
|
||||
get { return m_max; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The total sum of all added values
|
||||
/// </summary>
|
||||
public long Total
|
||||
{
|
||||
get { return m_max - m_min; }
|
||||
}
|
||||
|
||||
public Interval(string displayPrefix)
|
||||
{
|
||||
m_displayPrefix = displayPrefix;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Adds a value to the interval
|
||||
/// </summary>
|
||||
public void Add(long value)
|
||||
{
|
||||
m_min = Math.Min(m_min, value);
|
||||
m_max = Math.Max(m_max, value);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Writes the stats
|
||||
/// </summary>
|
||||
public void Write(TextWriter writer)
|
||||
{
|
||||
writer.WriteLine("{0}.Min={1}", m_displayPrefix, Min);
|
||||
writer.WriteLine("{0}.Max={1}", m_displayPrefix, Max);
|
||||
writer.WriteLine("{0}.Total={1}", m_displayPrefix, Total);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Statistics for a set of values
|
||||
/// </summary>
|
||||
public sealed class Stats
|
||||
{
|
||||
private long m_min = long.MaxValue;
|
||||
private long m_total;
|
||||
private long m_max;
|
||||
private long m_count;
|
||||
private string m_displayPrefix;
|
||||
|
||||
/// <summary>
|
||||
/// The min of the values
|
||||
/// </summary>
|
||||
public long Min
|
||||
{
|
||||
get { return m_min; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The max of the values
|
||||
/// </summary>
|
||||
public long Max
|
||||
{
|
||||
get { return m_max; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The count of values added
|
||||
/// </summary>
|
||||
public long Count
|
||||
{
|
||||
get { return m_count; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The total sum of all added values
|
||||
/// </summary>
|
||||
public long Total
|
||||
{
|
||||
get { return m_total; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The total sum of all added values
|
||||
/// </summary>
|
||||
public long Average
|
||||
{
|
||||
get
|
||||
{
|
||||
if (m_count == 0)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
return m_total / m_count;
|
||||
}
|
||||
}
|
||||
|
||||
public Stats(string displayPrefix)
|
||||
{
|
||||
m_displayPrefix = displayPrefix;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Adds a value to the stats
|
||||
/// </summary>
|
||||
public void Add(long value)
|
||||
{
|
||||
m_min = Math.Min(m_min, value);
|
||||
m_max = Math.Max(m_max, value);
|
||||
m_count++;
|
||||
m_total += value;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Writes the stats
|
||||
/// </summary>
|
||||
public void Write(TextWriter writer)
|
||||
{
|
||||
writer.WriteLine("{0}.Min={1}", m_displayPrefix, Min);
|
||||
writer.WriteLine("{0}.Max={1}", m_displayPrefix, Max);
|
||||
writer.WriteLine("{0}.Count={1}", m_displayPrefix, Count);
|
||||
writer.WriteLine("{0}.Total={1}", m_displayPrefix, Total);
|
||||
writer.WriteLine("{0}.Average={1}", m_displayPrefix, Average);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Directory in the build graph
|
||||
/// </summary>
|
||||
public sealed class Dir
|
||||
{
|
||||
/// <summary>
|
||||
/// Location of the directory
|
||||
/// </summary>
|
||||
public readonly string Location;
|
||||
|
||||
/// <summary>
|
||||
/// Files in the directory
|
||||
/// </summary>
|
||||
public readonly List<int> Contents;
|
||||
|
||||
/// <summary>
|
||||
/// The id of corresponding output pip: 0 if there is none
|
||||
/// </summary>
|
||||
public int ProducerId;
|
||||
|
||||
public Dir(string path, List<int> contents)
|
||||
{
|
||||
Location = path;
|
||||
Contents = contents;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,537 +0,0 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT License.
|
||||
|
||||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics.ContractsLight;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Reflection;
|
||||
using System.Text;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using BuildXL.Utilities;
|
||||
using BuildXL.Utilities.Collections;
|
||||
|
||||
namespace Tool.MimicGenerator
|
||||
{
|
||||
/// <summary>
|
||||
/// Writes out inputs and build files for a mimic'ed build
|
||||
/// </summary>
|
||||
public sealed class BuildWriter
|
||||
{
|
||||
private const string ResponseFileName = "args.rsp";
|
||||
|
||||
private readonly string m_outputDirectory;
|
||||
private readonly bool m_writeInputFiles;
|
||||
private readonly BuildGraph m_buildGraph;
|
||||
private ConcurrentBigSet<int> m_sourceFilesWritten = new ConcurrentBigSet<int>();
|
||||
private readonly ILookup<string, Pip> m_specFileToPipsLookup;
|
||||
|
||||
// Counter to give some progress while running
|
||||
private int m_processesEncountered;
|
||||
private int m_inputsWritten;
|
||||
private int m_inputsWithDefaultSize;
|
||||
|
||||
private double m_inputScaleFactor;
|
||||
private bool m_ignoreResponseFiles;
|
||||
|
||||
private readonly Language m_language;
|
||||
|
||||
private ConcurrentBag<Tuple<string, string, int>> m_inputsToWrite = new ConcurrentBag<Tuple<string, string, int>>();
|
||||
|
||||
public BuildWriter(string outputDirectory, bool writeInputFiles, double inputScaleFactor, BuildGraph buildGraph, bool ignoreResponseFiles, Language language)
|
||||
{
|
||||
m_outputDirectory = outputDirectory;
|
||||
m_writeInputFiles = writeInputFiles;
|
||||
m_inputScaleFactor = inputScaleFactor;
|
||||
m_buildGraph = buildGraph;
|
||||
m_specFileToPipsLookup = buildGraph.Pips.Values.ToLookup(pip => pip.Spec, StringComparer.Ordinal);
|
||||
m_ignoreResponseFiles = ignoreResponseFiles;
|
||||
m_language = language;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Writes out build files
|
||||
/// </summary>
|
||||
public bool WriteBuildFiles()
|
||||
{
|
||||
bool success = true;
|
||||
Console.WriteLine("Writing build files and inputs");
|
||||
|
||||
Directory.CreateDirectory(m_outputDirectory);
|
||||
|
||||
using (var textWriter = new StreamWriter(Path.Combine(m_outputDirectory, "stats.txt")))
|
||||
{
|
||||
m_buildGraph.OutputFileStats.Write(textWriter);
|
||||
m_buildGraph.SourceFileStats.Write(textWriter);
|
||||
m_buildGraph.PipDurationStats.Write(textWriter);
|
||||
m_buildGraph.BuildInterval.Write(textWriter);
|
||||
m_buildGraph.ProcessPipStats.Write(textWriter);
|
||||
}
|
||||
|
||||
// Write out the cache config file
|
||||
System.IO.File.WriteAllText(
|
||||
Path.Combine(m_outputDirectory, "cacheConfig.json"),
|
||||
GetEmbeddedResourceFile("Tool.MimicGenerator.Content.CacheConfig.json"));
|
||||
WriteBuildScript();
|
||||
|
||||
var provider = new LanguageProvider(m_language);
|
||||
|
||||
using (var configWriter = provider.CreateConfigWriter(Path.Combine(m_outputDirectory, "mimic")))
|
||||
{
|
||||
using (var moduleWriter = provider.CreateModuleWriter(m_outputDirectory, "MimicModule", new string[] { "{EngineLayout.DefaultMounts.DeploymentRootPath.Path.Combine('BuildXL.Transformers.Runners.dll')}" }))
|
||||
{
|
||||
configWriter.AddModule(moduleWriter);
|
||||
|
||||
// Write each pip into its spec file
|
||||
using (Timer updateTimer = new Timer(ReportProgress, null, TimeSpan.FromSeconds(30), TimeSpan.FromSeconds(30)))
|
||||
{
|
||||
Parallel.ForEach(m_specFileToPipsLookup, specPips =>
|
||||
{
|
||||
using (var writer = provider.CreateSpecWriter(RemapPath(specPips.Key)))
|
||||
{
|
||||
List<Process> allProcesses = new List<Process>();
|
||||
foreach (var p in specPips)
|
||||
{
|
||||
Interlocked.Increment(ref m_processesEncountered);
|
||||
|
||||
lock (moduleWriter)
|
||||
{
|
||||
int lengthToStrip = m_outputDirectory.Length;
|
||||
if (!(m_outputDirectory.EndsWith(@"/", StringComparison.Ordinal) || m_outputDirectory.EndsWith(@"\", StringComparison.Ordinal)))
|
||||
{
|
||||
lengthToStrip++;
|
||||
}
|
||||
|
||||
string specPath = writer.AbsolutePath.Remove(0, lengthToStrip);
|
||||
moduleWriter.AddSpec(specPath, writer);
|
||||
}
|
||||
|
||||
Process process = p as Process;
|
||||
if (process != null)
|
||||
{
|
||||
allProcesses.Add(process);
|
||||
}
|
||||
|
||||
CopyFile copyFile = p as CopyFile;
|
||||
if (copyFile != null)
|
||||
{
|
||||
bool isDirectory;
|
||||
bool isResponseFile;
|
||||
writer.AddCopyFile(
|
||||
GetProcessOutputName(p.PipId),
|
||||
GetInputExpression(copyFile.Source, writer, configWriter, out isDirectory, out isResponseFile),
|
||||
GetOutputExpression(copyFile.Destination, configWriter));
|
||||
if (isDirectory)
|
||||
{
|
||||
throw new MimicGeneratorException("CopyFile shouldn't produce a directory");
|
||||
}
|
||||
}
|
||||
|
||||
WriteFile writeFile = p as WriteFile;
|
||||
if (writeFile != null)
|
||||
{
|
||||
bool ignore = false;
|
||||
if (m_ignoreResponseFiles)
|
||||
{
|
||||
File f;
|
||||
if (m_buildGraph.Files.TryGetValue(writeFile.Destination, out f))
|
||||
{
|
||||
if (f.Location.EndsWith(ResponseFileName, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
ignore = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!ignore)
|
||||
{
|
||||
writer.AddWriteFile(GetProcessOutputName(p.PipId), GetOutputExpression(writeFile.Destination, configWriter));
|
||||
}
|
||||
}
|
||||
|
||||
SealDirectory sealDirectory = p as SealDirectory;
|
||||
if (sealDirectory != null)
|
||||
{
|
||||
HandleSealDirectory(sealDirectory, writer, configWriter);
|
||||
}
|
||||
}
|
||||
|
||||
HandleProcesses(writer, configWriter, allProcesses);
|
||||
}
|
||||
});
|
||||
|
||||
success &= WriteQueuedInputFiles();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Console.WriteLine("Write {0} input files", m_inputsWritten);
|
||||
Console.WriteLine("{0} Input files were using the default size", m_inputsWithDefaultSize);
|
||||
|
||||
return success;
|
||||
}
|
||||
|
||||
public void WriteBuildScript()
|
||||
{
|
||||
using (StreamWriter writer = new StreamWriter(Path.Combine(m_outputDirectory, "MimicBuild.bat")))
|
||||
{
|
||||
writer.WriteLine("REM This file sets up the environment to be able to perform mimic builds");
|
||||
writer.WriteLine(string.Empty);
|
||||
writer.WriteLine("echo Update the '[UPDATE]' areas with the root of a fully built BuildXL repo and uncomment them");
|
||||
writer.WriteLine(string.Empty);
|
||||
writer.WriteLine("SET BUILDXL_DISABLE_DECLARE_BEFORE_USE_CHECK=1");
|
||||
writer.WriteLine(@"REM SET ScriptSdk=[UPDATE]\Public\Script\Sdk");
|
||||
writer.WriteLine($@"REM [UPDATE]\Out\Bin\release\net461\{Branding.ProductExecutableName} /c:config.dsc /cacheconfigfilepath:cacheConfig.json /p:deploymentroot=[UPDATE]\out\bin\release /logstats /viewer:show");
|
||||
}
|
||||
}
|
||||
|
||||
public void HandleSealDirectory(SealDirectory sealDirectory, SpecWriter writer, ConfigWriter configWriter)
|
||||
{
|
||||
List<string> remappedContents = new List<string>();
|
||||
Dir dir = m_buildGraph.Directories[sealDirectory.Directory];
|
||||
foreach (int content in dir.Contents)
|
||||
{
|
||||
int producingProcess;
|
||||
if (m_buildGraph.OutputArtifactToProducingPip.TryGetValue(content, out producingProcess))
|
||||
{
|
||||
var p = m_buildGraph.Pips[producingProcess];
|
||||
remappedContents.Add(writer.GetProcessInputName(GetProcessOutputName(producingProcess), p.Spec,
|
||||
content));
|
||||
}
|
||||
else
|
||||
{
|
||||
remappedContents.Add(configWriter.ToRelativePathExpression(m_buildGraph.Files[content].Location));
|
||||
QueueWritingInputFileIfNecessary(content);
|
||||
}
|
||||
}
|
||||
|
||||
writer.AddSealDirectory(GetSealOutputName(sealDirectory.Directory), configWriter.ToRelativePathExpression(dir.Location), remappedContents);
|
||||
}
|
||||
|
||||
private void HandleProcesses(SpecWriter writer, ConfigWriter configWriter, List<Process> processes)
|
||||
{
|
||||
Tuple<ProcessStats, Process>[] stats = new Tuple<ProcessStats, Process>[processes.Count];
|
||||
|
||||
for (int i = 0; i < processes.Count; i++)
|
||||
{
|
||||
stats[i] = new Tuple<ProcessStats, Process>(GetProcessStats(processes[i]), processes[i]);
|
||||
}
|
||||
|
||||
var sortedStats = stats.OrderByDescending(s => s.Item1.ExecutionTimeMs);
|
||||
|
||||
// Mark the longest process in the spec so it may be used for scaling in the transformer if applicable
|
||||
bool isLongest = true;
|
||||
foreach (var item in sortedStats)
|
||||
{
|
||||
WriteMimicInvocation(item.Item2, writer, configWriter, isLongest);
|
||||
isLongest = false;
|
||||
}
|
||||
}
|
||||
|
||||
private void WriteMimicInvocation(Process process, SpecWriter writer, ConfigWriter configWriter, bool isLongestProcess)
|
||||
{
|
||||
// Declared inputs & seal directory dependencies
|
||||
List<string> directoryDependencies = new List<string>();
|
||||
List<string> fileDependencies = new List<string>();
|
||||
foreach (int consumes in process.Consumes)
|
||||
{
|
||||
bool wasDirectory;
|
||||
bool isResponseFile;
|
||||
string expression = GetInputExpression(consumes, writer, configWriter, out wasDirectory, out isResponseFile);
|
||||
|
||||
if (wasDirectory)
|
||||
{
|
||||
directoryDependencies.Add(expression);
|
||||
}
|
||||
else if (!m_ignoreResponseFiles || !isResponseFile)
|
||||
{
|
||||
fileDependencies.Add(expression);
|
||||
}
|
||||
}
|
||||
|
||||
// Declared outputs
|
||||
List<SpecWriter.MimicFileOutput> mimicOutputs = new List<SpecWriter.MimicFileOutput>();
|
||||
foreach (var produces in process.Produces)
|
||||
{
|
||||
File file = m_buildGraph.Files[produces];
|
||||
SpecWriter.MimicFileOutput mimicOutput = new SpecWriter.MimicFileOutput()
|
||||
{
|
||||
Path = GetOutputExpression(produces, configWriter),
|
||||
LengthInBytes = file.GetScaledLengthInBytes(m_inputScaleFactor),
|
||||
RepeatingContent = file.Hash,
|
||||
FileId = produces,
|
||||
};
|
||||
|
||||
mimicOutputs.Add(mimicOutput);
|
||||
}
|
||||
|
||||
// Observed accesses (if applicable)
|
||||
ObservedAccess[] accesses = null;
|
||||
int lookupPip = -1;
|
||||
if (m_buildGraph.ObservedAccesses.Count > 0)
|
||||
{
|
||||
lookupPip = process.OriginalPipId ?? process.PipId;
|
||||
if (!m_buildGraph.ObservedAccesses.TryGetValue(lookupPip, out accesses))
|
||||
{
|
||||
Console.WriteLine("Warning: No observed accesses recorded for Pip Id: {0}", lookupPip);
|
||||
}
|
||||
}
|
||||
|
||||
string relativeObservedAccessPath = null;
|
||||
if (accesses != null)
|
||||
{
|
||||
string observedAccessesPath = writer.WriteObservedAccessesFile(accesses, lookupPip);
|
||||
relativeObservedAccessPath = observedAccessesPath.Replace(writer.AbsolutePath, string.Empty).TrimStart('\\');
|
||||
}
|
||||
|
||||
writer.AddMimicInvocation(
|
||||
GetProcessOutputName(process.PipId),
|
||||
directoryDependencies,
|
||||
fileDependencies,
|
||||
mimicOutputs,
|
||||
relativeObservedAccessPath,
|
||||
process.Semaphores,
|
||||
Math.Max(1, process.ProcessWallTimeMs),
|
||||
isLongestProcess);
|
||||
}
|
||||
|
||||
private struct ProcessStats
|
||||
{
|
||||
public int ExecutionTimeMs;
|
||||
public int CppFiles;
|
||||
public int CsFiles;
|
||||
}
|
||||
|
||||
private ProcessStats GetProcessStats(Process p)
|
||||
{
|
||||
ProcessStats stats = new ProcessStats()
|
||||
{
|
||||
ExecutionTimeMs = p.ProcessWallTimeMs,
|
||||
};
|
||||
|
||||
foreach (int file in p.Consumes)
|
||||
{
|
||||
File f;
|
||||
if (m_buildGraph.Files.TryGetValue(file, out f))
|
||||
{
|
||||
if (f.Location.EndsWith(".cs", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
stats.CsFiles++;
|
||||
}
|
||||
else if (f.Location.EndsWith(".cpp", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
stats.CppFiles++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return stats;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the expression for an input
|
||||
/// </summary>
|
||||
/// <param name="depId">Id of the input artifact</param>
|
||||
/// <param name="specWriter">
|
||||
/// SpecWriter to receive expression string according to the language and add ImportStatement to
|
||||
/// the Spec if the language is DScript.
|
||||
/// </param>
|
||||
/// <param name="configWriter">ConfigWriter to add a new mount if necessary</param>
|
||||
/// <param name="isDirectory">returns whether the input was a directory</param>
|
||||
/// <param name="isResponseFile">returns whether the input was a response file</param>
|
||||
/// <returns>expression for the input dependency</returns>
|
||||
private string GetInputExpression(int depId, SpecWriter specWriter, ConfigWriter configWriter,
|
||||
out bool isDirectory, out bool isResponseFile)
|
||||
{
|
||||
Dir dir;
|
||||
int producingProcess;
|
||||
File f;
|
||||
isResponseFile = false;
|
||||
if (m_buildGraph.Directories.TryGetValue(depId, out dir))
|
||||
{
|
||||
isDirectory = true;
|
||||
var producingSpecPath = m_buildGraph.Pips[dir.ProducerId].Spec;
|
||||
return specWriter.GetSealCopyWriteInputName(GetSealOutputName(depId), producingSpecPath);
|
||||
}
|
||||
|
||||
if (m_buildGraph.OutputArtifactToProducingPip.TryGetValue(depId, out producingProcess))
|
||||
{
|
||||
isDirectory = false;
|
||||
Pip p = m_buildGraph.Pips[producingProcess];
|
||||
|
||||
if (p is Process)
|
||||
{
|
||||
return specWriter.GetProcessInputName(GetProcessOutputName(producingProcess), p.Spec, depId);
|
||||
}
|
||||
|
||||
if (p is CopyFile || p is WriteFile)
|
||||
{
|
||||
if (m_buildGraph.Files.TryGetValue(depId, out f))
|
||||
{
|
||||
isResponseFile = f.Location.EndsWith(ResponseFileName, StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
return specWriter.GetSealCopyWriteInputName(GetProcessOutputName(producingProcess), p.Spec);
|
||||
}
|
||||
|
||||
throw new MimicGeneratorException("Error. Pip isn't of known type");
|
||||
}
|
||||
|
||||
if (m_buildGraph.Files.TryGetValue(depId, out f))
|
||||
{
|
||||
isDirectory = false;
|
||||
QueueWritingInputFileIfNecessary(depId);
|
||||
return configWriter.ToRelativePathExpression(f.Location);
|
||||
}
|
||||
|
||||
throw new MimicGeneratorException("Could not find a producer for dependency:{0}", depId);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets an expression for an output file
|
||||
/// </summary>
|
||||
private string GetOutputExpression(int fileId, ConfigWriter configWriter)
|
||||
{
|
||||
return configWriter.ToRelativePathExpression(m_buildGraph.Files[fileId].Location);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Writes an input file if necessary
|
||||
/// </summary>
|
||||
private void QueueWritingInputFileIfNecessary(int fileId)
|
||||
{
|
||||
if (!m_writeInputFiles || !m_sourceFilesWritten.Add(fileId))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
File file = m_buildGraph.Files[fileId];
|
||||
string originalLocation = file.Location;
|
||||
string remappedPath = RemapPath(originalLocation);
|
||||
|
||||
// Avoid overwriting spec files when they are registered as inputs
|
||||
if (!m_specFileToPipsLookup.Contains(originalLocation))
|
||||
{
|
||||
try
|
||||
{
|
||||
if (!file.WasLengthSet)
|
||||
{
|
||||
Interlocked.Increment(ref m_inputsWithDefaultSize);
|
||||
}
|
||||
|
||||
m_inputsToWrite.Add(new Tuple<string, string, int>(remappedPath, file.Hash, file.GetScaledLengthInBytes(m_inputScaleFactor)));
|
||||
}
|
||||
#pragma warning disable ERP022 // Unobserved exception in generic exception handler
|
||||
catch
|
||||
{
|
||||
// Some of the registered inputs are directories instead of actual files. As luck has it the directories
|
||||
// are generally created before the the file is attempted to be created. So we can just skip it and move along.
|
||||
Console.WriteLine("Warning: Could not write input file: " + remappedPath);
|
||||
}
|
||||
#pragma warning restore ERP022 // Unobserved exception in generic exception handler
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Writes out the queued input files
|
||||
/// </summary>
|
||||
private bool WriteQueuedInputFiles()
|
||||
{
|
||||
// First we need to strip out any "files" that are actually directories. Otherwise if we first create a path
|
||||
// as a file and later try to create a directory at the same path, the directory creation will fail
|
||||
|
||||
// We need to be able to skip creating any inputs that are actually directories, otherwise if we create a file
|
||||
// at a path and later realize it is a directory, the directory creation will fail.
|
||||
// To do this we create a set with all paths that must be directories.
|
||||
HashSet<string> directoryPaths = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
StringBuilder builder = new StringBuilder();
|
||||
foreach (var tuple in m_inputsToWrite)
|
||||
{
|
||||
builder.Clear();
|
||||
|
||||
string[] split = tuple.Item1.Split(Path.DirectorySeparatorChar);
|
||||
|
||||
for (int i = 0; i < split.Length - 1; i++)
|
||||
{
|
||||
if (i > 0)
|
||||
{
|
||||
builder.Append(Path.DirectorySeparatorChar);
|
||||
}
|
||||
|
||||
builder.Append(split[i]);
|
||||
directoryPaths.Add(builder.ToString());
|
||||
}
|
||||
}
|
||||
|
||||
bool success = true;
|
||||
Parallel.ForEach(m_inputsToWrite, (tuple) =>
|
||||
{
|
||||
// Now skip any "file" that is actually a directory
|
||||
if (directoryPaths.Contains(tuple.Item1))
|
||||
{
|
||||
Console.WriteLine("Warning: skipping declared input '{0}' because it is actually a directory", tuple.Item1);
|
||||
return;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
using (SourceWriter writer = new SourceWriter(tuple.Item1, tuple.Item2, tuple.Item3))
|
||||
{
|
||||
Interlocked.Increment(ref m_inputsWritten);
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
success = false;
|
||||
Console.WriteLine("Error: Could not write input file: {0}. Message:{1} ", tuple.Item1, ex.GetLogEventMessage());
|
||||
}
|
||||
});
|
||||
|
||||
return success;
|
||||
}
|
||||
|
||||
internal static string GetProcessOutputName(int pipId)
|
||||
{
|
||||
return "Mimic" + pipId;
|
||||
}
|
||||
|
||||
private static string GetSealOutputName(int directoryNumber)
|
||||
{
|
||||
return "SealDirectory" + directoryNumber;
|
||||
}
|
||||
|
||||
private string RemapPath(string path)
|
||||
{
|
||||
return Path.Combine(m_outputDirectory, path[0].ToString(), path.Remove(0, 3));
|
||||
}
|
||||
|
||||
private void ReportProgress(object state)
|
||||
{
|
||||
Console.WriteLine("Progress: {0}/{1} pips. {2}/{3} inputs written", Volatile.Read(ref m_processesEncountered), m_buildGraph.Pips.Count,
|
||||
Volatile.Read(ref m_inputsWritten), m_inputsToWrite.Count);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Helper to get the string content of a resource file from the current assembly.
|
||||
/// </summary>
|
||||
/// <remarks>This unfortunately cannot be in a shared location like 'AssemblyHelpers' because on .Net Core it ignores the assembly and always tries to extract the resources from the running assembly. Even though GetManifestResourceNames() does respect it.</remarks>
|
||||
private static string GetEmbeddedResourceFile(string resourceKey)
|
||||
{
|
||||
var callingAssembly = typeof(BuildWriter).GetTypeInfo().Assembly;
|
||||
var stream = callingAssembly.GetManifestResourceStream(resourceKey);
|
||||
if (stream == null)
|
||||
{
|
||||
Contract.Assert(false, $"Expected embedded resource key '{resourceKey}' not found in assembly {callingAssembly.FullName}. Valid resource names are: {string.Join(",", callingAssembly.GetManifestResourceNames())}");
|
||||
return null;
|
||||
}
|
||||
|
||||
using (var sr = new StreamReader(stream))
|
||||
{
|
||||
return sr.ReadToEnd();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,89 +0,0 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT License.
|
||||
|
||||
using System;
|
||||
using System.Diagnostics.CodeAnalysis;
|
||||
using System.IO;
|
||||
|
||||
namespace Tool.MimicGenerator
|
||||
{
|
||||
/// <summary>
|
||||
/// Base class for writing a file
|
||||
/// </summary>
|
||||
public abstract class BuildXLFileWriter : IDisposable
|
||||
{
|
||||
/// <summary>
|
||||
/// Absolute path of the underlying file
|
||||
/// </summary>
|
||||
public readonly string AbsolutePath;
|
||||
|
||||
/// <summary>
|
||||
/// StreamWriter
|
||||
/// </summary>
|
||||
public StreamWriter Writer
|
||||
{
|
||||
get
|
||||
{
|
||||
if (m_writer == null)
|
||||
{
|
||||
if (!m_wasInitialized)
|
||||
{
|
||||
m_writer = new StreamWriter(AbsolutePath, append: false);
|
||||
WriteStart();
|
||||
m_wasInitialized = true;
|
||||
}
|
||||
else
|
||||
{
|
||||
m_writer = new StreamWriter(AbsolutePath, append: true);
|
||||
}
|
||||
}
|
||||
|
||||
return m_writer;
|
||||
}
|
||||
}
|
||||
|
||||
private StreamWriter m_writer;
|
||||
|
||||
private bool m_wasInitialized = false;
|
||||
|
||||
/// <summary>
|
||||
/// Constructor
|
||||
/// </summary>
|
||||
/// <param name="absolutePath">Absolute path to the output file</param>
|
||||
protected BuildXLFileWriter(string absolutePath)
|
||||
{
|
||||
AbsolutePath = absolutePath;
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(absolutePath));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Closes the StreamWriter so there aren't a bazillion handles open at once. Future writes are still allowed
|
||||
/// and will cause a new underlying StreamWriter to be created.
|
||||
/// </summary>
|
||||
public void SoftClose()
|
||||
{
|
||||
m_writer.Dispose();
|
||||
m_writer = null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Writes the start of the file
|
||||
/// </summary>
|
||||
protected abstract void WriteStart();
|
||||
|
||||
/// <summary>
|
||||
/// Writes the end of the file
|
||||
/// </summary>
|
||||
protected abstract void WriteEnd();
|
||||
|
||||
/// <inheritdoc/>
|
||||
[SuppressMessage("Microsoft.Usage", "CA1816")]
|
||||
[SuppressMessage("Microsoft.Design", "CA1063")]
|
||||
public void Dispose()
|
||||
{
|
||||
WriteEnd();
|
||||
m_writer.Dispose();
|
||||
m_writer = null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,31 +0,0 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT License.
|
||||
|
||||
using System.Diagnostics.CodeAnalysis;
|
||||
|
||||
namespace Tool.MimicGenerator
|
||||
{
|
||||
public abstract class ConfigWriter : BuildXLFileWriter
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets the allowed environment variables
|
||||
/// </summary>
|
||||
[SuppressMessage("Microsoft.Security", "CA2105:ArrayFieldsShouldNotBeReadOnly")]
|
||||
public static readonly string[] AllowedEnvironmentVariables = new string[]
|
||||
{
|
||||
"RuntimeScaleFactor",
|
||||
"IoScaleFactor",
|
||||
"PredictNativeSpecs",
|
||||
};
|
||||
|
||||
protected ConfigWriter(string absolutePath)
|
||||
: base(absolutePath) { }
|
||||
|
||||
[SuppressMessage("Microsoft.Design", "CA1011:ConsiderPassingBaseTypesAsParameters")]
|
||||
public abstract void AddModule(ModuleWriter moduleWriter);
|
||||
|
||||
public abstract void WriteMount(string mountName, string mountAbsolutePath);
|
||||
|
||||
public abstract string ToRelativePathExpression(string path);
|
||||
}
|
||||
}
|
||||
|
|
@ -1,8 +0,0 @@
|
|||
{
|
||||
"Assembly":"BuildXL.Cache.MemoizationStoreAdapter",
|
||||
"Type":"BuildXL.Cache.MemoizationStoreAdapter.MemoizationStoreCacheFactory",
|
||||
"MaxCacheSizeInMB":512000,
|
||||
"CacheId":"LocalCache",
|
||||
"CacheRootPath":"[BuildXLSelectedRootPath]",
|
||||
"CacheLogPath":"[BuildXLSelectedLogPath]"
|
||||
}
|
||||
|
|
@ -1,134 +0,0 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT License.
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Globalization;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
|
||||
namespace Tool.MimicGenerator
|
||||
{
|
||||
/// <summary>
|
||||
/// Takes an existing <see cref="BuildGraph"/> and synthetically increases its size
|
||||
/// </remarks>
|
||||
public sealed class GraphMultiplier
|
||||
{
|
||||
private readonly BuildGraph m_graph;
|
||||
|
||||
/// <summary>
|
||||
/// Constructor
|
||||
/// </summary>
|
||||
private GraphMultiplier(BuildGraph sourceGraph)
|
||||
{
|
||||
m_graph = sourceGraph;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Mutates the graph by duplicating the graph into n additional parallelizable graphs.
|
||||
/// </summary>
|
||||
public static void DuplicateAsParallelGraphs(BuildGraph graph, int duplicationFactor, int maxPipsPerSpec)
|
||||
{
|
||||
GraphMultiplier multiplier = new GraphMultiplier(graph);
|
||||
multiplier.Duplicate(duplicationFactor, maxPipsPerSpec);
|
||||
}
|
||||
|
||||
private void Duplicate(int multiplicationFactor, int maxPipsPerSpec)
|
||||
{
|
||||
Console.WriteLine("Duplicating Graph {0} times", multiplicationFactor);
|
||||
|
||||
// Take a snapshot of the pips currently in the graph before any multiplication is applied.
|
||||
Process[] originalProcesses = m_graph.Pips.Values.OfType<Process>().ToArray();
|
||||
WriteFile[] originalWriteFiles = m_graph.Pips.Values.OfType<WriteFile>().ToArray();
|
||||
|
||||
// The basic strategy here is to clone every pip in a parallelizeable way. Each pip gets n copies with same
|
||||
// source inputs, but different outputs. Dependencies within the original set are translated to the new set
|
||||
for (int i = 0; i < multiplicationFactor; i++)
|
||||
{
|
||||
Console.WriteLine("Duplicating Graph iteration {0}", i);
|
||||
string newRoot = i.ToString(CultureInfo.InvariantCulture);
|
||||
for (int j = 0; j < originalProcesses.Length; j++)
|
||||
{
|
||||
Process p = originalProcesses[j];
|
||||
List<int> clonedConsumes = new List<int>(p.Consumes.Count);
|
||||
List<int> clonedProduces = new List<int>(p.Produces.Count);
|
||||
foreach (int consume in p.Consumes)
|
||||
{
|
||||
int producingPip;
|
||||
if (m_graph.OutputArtifactToProducingPip.TryGetValue(consume, out producingPip))
|
||||
{
|
||||
Process process = m_graph.Pips[producingPip] as Process;
|
||||
WriteFile writeFile = m_graph.Pips[producingPip] as WriteFile;
|
||||
if (process != null || writeFile != null)
|
||||
{
|
||||
// This is an output file created by a pip that will also be cloned. We need to translate it to the new path
|
||||
File f;
|
||||
if (!m_graph.Files.TryGetValue(consume, out f))
|
||||
{
|
||||
throw new MimicGeneratorException("Failed to find referenced file with id: {0}", consume);
|
||||
}
|
||||
|
||||
clonedConsumes.Add(m_graph.DuplicateFile(f, newRoot));
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
// If the path isn't translated based on cloning, just consume it directly.
|
||||
clonedConsumes.Add(consume);
|
||||
}
|
||||
|
||||
foreach (int produce in p.Produces)
|
||||
{
|
||||
File f;
|
||||
if (!m_graph.Files.TryGetValue(produce, out f))
|
||||
{
|
||||
throw new MimicGeneratorException("Failed to find referenced file with id: {0}", produce);
|
||||
}
|
||||
|
||||
clonedProduces.Add(m_graph.DuplicateFile(f, newRoot));
|
||||
}
|
||||
|
||||
Process cloned = new Process(0, string.Empty, BuildGraph.DuplicatePath(GetSpecName(p.Spec, j, maxPipsPerSpec), newRoot), clonedProduces, clonedConsumes, p.Semaphores);
|
||||
int newPipId = m_graph.AddWithNewPipId(cloned, p.PipId);
|
||||
|
||||
// Need to register all of the outputs of the cloned pip so consumers reference it as an output
|
||||
// rather than a source file when specs are generated
|
||||
foreach (var file in clonedProduces)
|
||||
{
|
||||
m_graph.OutputArtifactToProducingPip.TryAdd(file, newPipId);
|
||||
}
|
||||
}
|
||||
|
||||
for (int j = 0; j < originalWriteFiles.Length; j++)
|
||||
{
|
||||
WriteFile wf = originalWriteFiles[j];
|
||||
File f;
|
||||
if (!m_graph.Files.TryGetValue(wf.Destination, out f))
|
||||
{
|
||||
throw new MimicGeneratorException("Failed to find referenced file with id: {0}", wf.Destination);
|
||||
}
|
||||
|
||||
int clonedDestination = m_graph.DuplicateFile(f, newRoot);
|
||||
WriteFile cloned = new WriteFile(0, string.Empty, BuildGraph.DuplicatePath(GetSpecName(wf.Spec, j, maxPipsPerSpec), newRoot), clonedDestination);
|
||||
int newPipId = m_graph.AddWithNewPipId(cloned, wf.PipId);
|
||||
m_graph.OutputArtifactToProducingPip.TryAdd(clonedDestination, newPipId);
|
||||
}
|
||||
}
|
||||
|
||||
// TODO: Mirror CopyFile pips
|
||||
}
|
||||
|
||||
private static string GetSpecName(string specName, int processPipCount, int maxPipsPerSpec)
|
||||
{
|
||||
int clone = processPipCount / maxPipsPerSpec;
|
||||
if (clone > 0)
|
||||
{
|
||||
string specWithoutExtension = Path.GetFileNameWithoutExtension(specName) + clone;
|
||||
string newPath = Path.ChangeExtension(Path.Combine(Path.GetDirectoryName(specName), specWithoutExtension), Path.GetExtension(specWithoutExtension));
|
||||
return newPath;
|
||||
}
|
||||
|
||||
return specName;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,705 +0,0 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT License.
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics.CodeAnalysis;
|
||||
using System.IO;
|
||||
using Newtonsoft.Json;
|
||||
|
||||
namespace Tool.MimicGenerator
|
||||
{
|
||||
/// <summary>
|
||||
/// Reads in a json build graph and writes out build specs and optionally dummy input files to be able to mimic
|
||||
/// a build.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// This is currently working off the json graph. It'd be better to make it consume the SQL database instead so there
|
||||
/// are fewer separate components parsing the json directly. It would also benefit from additional data available in the database.
|
||||
///
|
||||
/// This version has a number of shortcuts which make the build that runs less accurate:
|
||||
/// * All input and output file sizes are hardcoded 10 bytes
|
||||
/// * All process runtimes are hardcoded to some small amount
|
||||
/// * Using preducted intputs & outputs rather than actual
|
||||
/// * Launched processes are not consuming seal directory inputs
|
||||
/// * Copy file pips are not represented
|
||||
/// * Write file pips are not represented
|
||||
/// </remarks>
|
||||
public sealed class GraphReader
|
||||
{
|
||||
private readonly string m_graphPath;
|
||||
private readonly string m_observedInputsPath;
|
||||
|
||||
private BuildGraph m_graph = new BuildGraph();
|
||||
|
||||
/// <summary>
|
||||
/// Constructor
|
||||
/// </summary>
|
||||
public GraphReader(string graphPath, string observedInputsPath)
|
||||
{
|
||||
m_graphPath = graphPath;
|
||||
m_observedInputsPath = observedInputsPath;
|
||||
}
|
||||
|
||||
public BuildGraph ReadGraph()
|
||||
{
|
||||
ReadJson();
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(m_observedInputsPath))
|
||||
{
|
||||
ReadObservedInputs();
|
||||
}
|
||||
|
||||
return m_graph;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Parses the json file
|
||||
/// </summary>
|
||||
private void ReadJson()
|
||||
{
|
||||
Console.WriteLine("Parsing JSON graph");
|
||||
|
||||
using (TextReader tr = new StreamReader(m_graphPath))
|
||||
{
|
||||
JsonTextReader reader = new JsonTextReader(tr);
|
||||
|
||||
// Read into the file
|
||||
reader.Read();
|
||||
ExpectTokenType(reader, JsonToken.StartObject);
|
||||
reader.Read();
|
||||
ExpectTokenType(reader, JsonToken.PropertyName);
|
||||
|
||||
while (reader.TokenType != JsonToken.EndObject)
|
||||
{
|
||||
ExpectTokenType(reader, JsonToken.PropertyName);
|
||||
|
||||
if (CurrentValueMatches(reader, "description"))
|
||||
{
|
||||
reader.Read();
|
||||
reader.Read();
|
||||
ExpectTokenType(reader, JsonToken.PropertyName);
|
||||
}
|
||||
else if (CurrentValueMatches(reader, "dateUtc"))
|
||||
{
|
||||
reader.Read();
|
||||
reader.Read();
|
||||
ExpectTokenType(reader, JsonToken.PropertyName);
|
||||
}
|
||||
else if (CurrentValueMatches(reader, "version"))
|
||||
{
|
||||
reader.Read();
|
||||
reader.Read();
|
||||
ExpectTokenType(reader, JsonToken.PropertyName);
|
||||
}
|
||||
else if (CurrentValueMatches(reader, "artifacts"))
|
||||
{
|
||||
reader.Read();
|
||||
ExpectTokenType(reader, JsonToken.StartArray);
|
||||
ReadArtifactArray(reader);
|
||||
ExpectTokenType(reader, JsonToken.PropertyName);
|
||||
}
|
||||
else if (CurrentValueMatches(reader, "filedetails"))
|
||||
{
|
||||
reader.Read();
|
||||
ExpectTokenType(reader, JsonToken.StartArray);
|
||||
ReadFileDetailsArray(reader);
|
||||
}
|
||||
else if (CurrentValueMatches(reader, "graph"))
|
||||
{
|
||||
reader.Read();
|
||||
ExpectTokenType(reader, JsonToken.StartArray);
|
||||
ReadGraphArray(reader);
|
||||
ExpectTokenType(reader, JsonToken.PropertyName);
|
||||
}
|
||||
else if (CurrentValueMatches(reader, "execution"))
|
||||
{
|
||||
reader.Read();
|
||||
ReadExecutionArray(reader);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void ReadFileDetailsArray(JsonTextReader reader)
|
||||
{
|
||||
ExpectTokenType(reader, JsonToken.StartArray);
|
||||
reader.Read();
|
||||
while (reader.TokenType != JsonToken.EndArray)
|
||||
{
|
||||
ExpectTokenType(reader, JsonToken.StartObject);
|
||||
reader.Read();
|
||||
|
||||
int fileId = -1;
|
||||
int length = -1;
|
||||
string hash = string.Empty;
|
||||
|
||||
while (reader.TokenType != JsonToken.EndObject)
|
||||
{
|
||||
if (CurrentValueMatches(reader, "file"))
|
||||
{
|
||||
fileId = reader.ReadAsInt32().Value;
|
||||
reader.Read();
|
||||
}
|
||||
else if (CurrentValueMatches(reader, "length"))
|
||||
{
|
||||
length = reader.ReadAsInt32().Value;
|
||||
reader.Read();
|
||||
}
|
||||
else if (CurrentValueMatches(reader, "hash"))
|
||||
{
|
||||
hash = reader.ReadAsString();
|
||||
reader.Read();
|
||||
}
|
||||
else
|
||||
{
|
||||
reader.Read();
|
||||
}
|
||||
}
|
||||
|
||||
ExpectTokenType(reader, JsonToken.EndObject);
|
||||
reader.Read();
|
||||
|
||||
if (fileId != -1)
|
||||
{
|
||||
File file;
|
||||
if (m_graph.Files.TryGetValue(fileId, out file))
|
||||
{
|
||||
file.SetUnscaledLength(length);
|
||||
file.Hash = hash;
|
||||
if (file.IsOutputFile)
|
||||
{
|
||||
m_graph.OutputFileStats.Add(length);
|
||||
}
|
||||
else
|
||||
{
|
||||
m_graph.SourceFileStats.Add(length);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
ExpectTokenType(reader, JsonToken.EndArray);
|
||||
reader.Read();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reads the execution section
|
||||
/// </summary>
|
||||
private void ReadExecutionArray(JsonTextReader reader)
|
||||
{
|
||||
ExpectTokenType(reader, JsonToken.StartArray);
|
||||
reader.Read();
|
||||
while (reader.TokenType != JsonToken.EndArray)
|
||||
{
|
||||
Tuple<int, TimeSpan> pipAndRuntime = ReadExecutionItem(reader);
|
||||
ExpectTokenType(reader, JsonToken.EndObject);
|
||||
reader.Read();
|
||||
Pip p;
|
||||
if (m_graph.Pips.TryGetValue(pipAndRuntime.Item1, out p))
|
||||
{
|
||||
Process process = p as Process;
|
||||
if (process != null)
|
||||
{
|
||||
process.ProcessWallTimeMs = (int)pipAndRuntime.Item2.TotalMilliseconds;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
ExpectTokenType(reader, JsonToken.EndArray);
|
||||
reader.Read();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reads an item in the execution array
|
||||
/// </summary>
|
||||
/// <returns>Tuple of pipId and wallTimeMs</returns>
|
||||
private Tuple<int, TimeSpan> ReadExecutionItem(JsonTextReader reader)
|
||||
{
|
||||
int pipId = -1;
|
||||
TimeSpan executionTime = TimeSpan.FromTicks(0);
|
||||
|
||||
ExpectTokenType(reader, JsonToken.StartObject);
|
||||
reader.Read();
|
||||
|
||||
while (reader.TokenType != JsonToken.EndObject)
|
||||
{
|
||||
if (CurrentValueMatches(reader, "processWallTime"))
|
||||
{
|
||||
// wall time is in ticks which will overflow an int32 in about an hour of runtime
|
||||
string wallTime = reader.ReadAsString();
|
||||
long longWallTime;
|
||||
if (!long.TryParse(wallTime, out longWallTime))
|
||||
{
|
||||
throw new MimicGeneratorException("Failed to parse processWallTime to numeric for PipId:{0} at line: {1}", pipId, reader.LineNumber);
|
||||
}
|
||||
|
||||
executionTime = TimeSpan.FromTicks(longWallTime);
|
||||
m_graph.PipDurationStats.Add(longWallTime);
|
||||
}
|
||||
else if (CurrentValueMatches(reader, "pipId"))
|
||||
{
|
||||
pipId = reader.ReadAsInt32().Value;
|
||||
}
|
||||
else if (CurrentValueMatches(reader, "io"))
|
||||
{
|
||||
while (reader.TokenType != JsonToken.EndObject)
|
||||
{
|
||||
reader.Read();
|
||||
}
|
||||
}
|
||||
else if (CurrentValueMatches(reader, "startTime") || CurrentValueMatches(reader, "endTime"))
|
||||
{
|
||||
long startOrEndTime = ReadAsLong(reader);
|
||||
m_graph.BuildInterval.Add(startOrEndTime);
|
||||
}
|
||||
|
||||
reader.Read();
|
||||
}
|
||||
|
||||
return new Tuple<int, TimeSpan>(pipId, executionTime);
|
||||
}
|
||||
|
||||
private static long ReadAsLong(JsonTextReader reader)
|
||||
{
|
||||
string value = reader.ReadAsString();
|
||||
long integralValue;
|
||||
if (!long.TryParse(value, out integralValue))
|
||||
{
|
||||
throw new MimicGeneratorException("Failed to parse Int64 for at line: {0}, column: {1}", reader.LineNumber, reader.LinePosition);
|
||||
}
|
||||
|
||||
return integralValue;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reads the graph array
|
||||
/// </summary>
|
||||
private void ReadGraphArray(JsonTextReader reader)
|
||||
{
|
||||
ExpectTokenType(reader, JsonToken.StartArray);
|
||||
reader.Read();
|
||||
|
||||
while (reader.TokenType == JsonToken.StartObject)
|
||||
{
|
||||
ReadPip(reader);
|
||||
}
|
||||
|
||||
ExpectTokenType(reader, JsonToken.EndArray);
|
||||
reader.Read();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reads a pip from within the graph array
|
||||
/// </summary>
|
||||
[SuppressMessage("Microsoft.Naming", "CA2204")]
|
||||
private void ReadPip(JsonTextReader reader)
|
||||
{
|
||||
ExpectTokenType(reader, JsonToken.StartObject);
|
||||
reader.Read();
|
||||
|
||||
int pipId = -1;
|
||||
string stableId = null;
|
||||
string spec = null;
|
||||
List<int> produces = new List<int>();
|
||||
List<int> consumes = new List<int>();
|
||||
List<SemaphoreInfo> semaphores = new List<SemaphoreInfo>();
|
||||
string pipType = null;
|
||||
|
||||
while (reader.TokenType != JsonToken.EndObject)
|
||||
{
|
||||
if (CurrentValueMatches(reader, "pipId"))
|
||||
{
|
||||
pipId = reader.ReadAsInt32().Value;
|
||||
}
|
||||
|
||||
if (CurrentValueMatches(reader, "stableId"))
|
||||
{
|
||||
stableId = reader.ReadAsString();
|
||||
}
|
||||
else if (CurrentValueMatches(reader, "provenance"))
|
||||
{
|
||||
reader.Read();
|
||||
ExpectTokenType(reader, JsonToken.StartObject);
|
||||
reader.Read();
|
||||
|
||||
while (reader.TokenType != JsonToken.EndObject)
|
||||
{
|
||||
if (CurrentValueMatches(reader, "spec"))
|
||||
{
|
||||
spec = reader.ReadAsString();
|
||||
}
|
||||
|
||||
reader.Read();
|
||||
}
|
||||
|
||||
ExpectTokenType(reader, JsonToken.EndObject);
|
||||
}
|
||||
else if (CurrentValueMatches(reader, "type"))
|
||||
{
|
||||
pipType = reader.ReadAsString();
|
||||
}
|
||||
else if (CurrentValueMatches(reader, "consumes"))
|
||||
{
|
||||
consumes = ReadIntArray(reader);
|
||||
}
|
||||
else if (CurrentValueMatches(reader, "produces"))
|
||||
{
|
||||
produces = ReadIntArray(reader);
|
||||
}
|
||||
else if (CurrentValueMatches(reader, "semaphores"))
|
||||
{
|
||||
semaphores = ReadSemaphoreArray(reader);
|
||||
}
|
||||
else
|
||||
{
|
||||
// continue
|
||||
}
|
||||
|
||||
reader.Read();
|
||||
}
|
||||
|
||||
ExpectTokenType(reader, JsonToken.EndObject);
|
||||
reader.Read();
|
||||
|
||||
if (pipType == "Process")
|
||||
{
|
||||
if (!m_graph.TryAddPip(new Process(pipId, stableId, spec, produces, consumes, semaphores)))
|
||||
{
|
||||
throw new MimicGeneratorException("Encountered duplicate Process. PipId:{0}. Line:{1}", pipId, reader.LineNumber);
|
||||
}
|
||||
|
||||
foreach (int produced in produces)
|
||||
{
|
||||
m_graph.AddOutputArtifact(produced, pipId);
|
||||
}
|
||||
|
||||
m_graph.ProcessPipStats.Add(1);
|
||||
}
|
||||
else if (pipType == "CopyFile")
|
||||
{
|
||||
if (consumes.Count != 1)
|
||||
{
|
||||
throw new MimicGeneratorException("Encountered malformed CopyFile. PipId:{0}. Line:{1}", pipId, reader.LineNumber);
|
||||
}
|
||||
|
||||
if (produces.Count != 1)
|
||||
{
|
||||
Console.WriteLine("Warning. CopyFile pip with ID:{0} at line:{1} had no consumers of its destination. It will be skipped", pipId, reader.LineNumber);
|
||||
}
|
||||
else
|
||||
{
|
||||
if (!m_graph.TryAddPip(new CopyFile(pipId, stableId, spec, consumes[0], produces[0])))
|
||||
{
|
||||
throw new MimicGeneratorException("Encountered duplicate CopyFile. PipId:{0}. Line{1}", pipId, reader.LineNumber);
|
||||
}
|
||||
|
||||
m_graph.AddOutputArtifact(produces[0], pipId);
|
||||
}
|
||||
}
|
||||
else if (pipType == "WriteFile")
|
||||
{
|
||||
if (consumes.Count != 0)
|
||||
{
|
||||
throw new MimicGeneratorException("Encountered malformed WriteFile. PipId:{0}. Line:{1}", pipId, reader.LineNumber);
|
||||
}
|
||||
|
||||
if (produces.Count != 1)
|
||||
{
|
||||
Console.WriteLine("Warning. WriteFile with ID:{0} at line:{1} had no consumers of its destination. It will be skipped", pipId, reader.LineNumber);
|
||||
}
|
||||
else
|
||||
{
|
||||
if (!m_graph.TryAddPip(new WriteFile(pipId, stableId, spec, produces[0])))
|
||||
{
|
||||
throw new MimicGeneratorException("Encountered duplicate WriteFile. PipId:{0}. Line{1}", pipId, reader.LineNumber);
|
||||
}
|
||||
|
||||
m_graph.AddOutputArtifact(produces[0], pipId);
|
||||
}
|
||||
}
|
||||
else if (pipType == "SealDirectory")
|
||||
{
|
||||
if (produces.Count != 1)
|
||||
{
|
||||
Console.WriteLine("Warning. SealDirectory with ID:{0} at line:{1} had no consumers of its destination. It will be skipped", pipId, reader.LineNumber);
|
||||
}
|
||||
else
|
||||
{
|
||||
if (!m_graph.TryAddPip(new SealDirectory(pipId, stableId, spec, produces[0])))
|
||||
{
|
||||
throw new MimicGeneratorException("Encountered duplicate SealDirectory. PipId:{0}. Line:{1}", pipId, reader.LineNumber);
|
||||
}
|
||||
|
||||
m_graph.AddOutputArtifact(produces[0], pipId, isDirectory: true);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void ReadArtifactArray(JsonTextReader reader)
|
||||
{
|
||||
ExpectTokenType(reader, JsonToken.StartArray);
|
||||
reader.Read();
|
||||
|
||||
while (reader.TokenType == JsonToken.StartObject)
|
||||
{
|
||||
ReadArtifact(reader);
|
||||
}
|
||||
|
||||
ExpectTokenType(reader, JsonToken.EndArray);
|
||||
reader.Read();
|
||||
}
|
||||
|
||||
private void ReadArtifact(JsonTextReader reader)
|
||||
{
|
||||
ExpectTokenType(reader, JsonToken.StartObject);
|
||||
reader.Read();
|
||||
|
||||
int id = -1;
|
||||
string file = null;
|
||||
string directory = null;
|
||||
List<int> contents = null;
|
||||
|
||||
while (reader.TokenType != JsonToken.EndObject)
|
||||
{
|
||||
if (CurrentValueMatches(reader, "id"))
|
||||
{
|
||||
id = reader.ReadAsInt32().Value;
|
||||
}
|
||||
else if (CurrentValueMatches(reader, "file"))
|
||||
{
|
||||
file = reader.ReadAsString();
|
||||
}
|
||||
else if (CurrentValueMatches(reader, "directory"))
|
||||
{
|
||||
directory = reader.ReadAsString();
|
||||
}
|
||||
else if (CurrentValueMatches(reader, "contents"))
|
||||
{
|
||||
contents = ReadIntArray(reader);
|
||||
}
|
||||
else
|
||||
{
|
||||
throw new MimicGeneratorException("Unrecognized property: {0}", reader.Value.ToString());
|
||||
}
|
||||
|
||||
reader.Read();
|
||||
}
|
||||
|
||||
if (file != null)
|
||||
{
|
||||
if (!m_graph.TryAddFile(id, new File(file)))
|
||||
{
|
||||
throw new MimicGeneratorException("File artifact registered twice: " + file);
|
||||
}
|
||||
}
|
||||
else if (directory != null)
|
||||
{
|
||||
if (!m_graph.Directories.TryAdd(id, new Dir(directory, contents)))
|
||||
{
|
||||
throw new MimicGeneratorException("Directory registered twice: " + directory);
|
||||
}
|
||||
}
|
||||
|
||||
ExpectTokenType(reader, JsonToken.EndObject);
|
||||
reader.Read();
|
||||
}
|
||||
|
||||
private static List<int> ReadIntArray(JsonTextReader reader)
|
||||
{
|
||||
List<int> list = new List<int>();
|
||||
|
||||
reader.Read();
|
||||
ExpectTokenType(reader, JsonToken.StartArray);
|
||||
|
||||
while (reader.TokenType != JsonToken.EndArray)
|
||||
{
|
||||
var value = reader.ReadAsInt32();
|
||||
if (value.HasValue)
|
||||
{
|
||||
list.Add(value.Value);
|
||||
}
|
||||
else
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
ExpectTokenType(reader, JsonToken.EndArray);
|
||||
|
||||
return list;
|
||||
}
|
||||
|
||||
private static List<SemaphoreInfo> ReadSemaphoreArray(JsonTextReader reader)
|
||||
{
|
||||
List<SemaphoreInfo> list = new List<SemaphoreInfo>();
|
||||
|
||||
reader.Read();
|
||||
ExpectTokenType(reader, JsonToken.StartArray);
|
||||
reader.Read();
|
||||
|
||||
while (reader.TokenType != JsonToken.EndArray)
|
||||
{
|
||||
list.Add(ReadSemaphore(reader));
|
||||
reader.Read();
|
||||
}
|
||||
|
||||
ExpectTokenType(reader, JsonToken.EndArray);
|
||||
|
||||
return list;
|
||||
}
|
||||
|
||||
private static SemaphoreInfo ReadSemaphore(JsonTextReader reader)
|
||||
{
|
||||
ExpectTokenType(reader, JsonToken.StartObject);
|
||||
SemaphoreInfo semaphore = new SemaphoreInfo();
|
||||
reader.Read();
|
||||
|
||||
while (reader.TokenType != JsonToken.EndObject)
|
||||
{
|
||||
if (CurrentValueMatches(reader, "name"))
|
||||
{
|
||||
semaphore.Name = reader.ReadAsString();
|
||||
}
|
||||
else if (CurrentValueMatches(reader, "value"))
|
||||
{
|
||||
semaphore.Value = reader.ReadAsInt32().Value;
|
||||
}
|
||||
else if (CurrentValueMatches(reader, "limit"))
|
||||
{
|
||||
semaphore.Limit = reader.ReadAsInt32().Value;
|
||||
}
|
||||
else
|
||||
{
|
||||
throw new MimicGeneratorException("Unrecognized property: {0}", reader.Value.ToString());
|
||||
}
|
||||
|
||||
reader.Read();
|
||||
}
|
||||
|
||||
ExpectTokenType(reader, JsonToken.EndObject);
|
||||
|
||||
return semaphore;
|
||||
}
|
||||
|
||||
private static void ExpectTokenType(JsonTextReader reader, JsonToken tokenType)
|
||||
{
|
||||
if (reader.TokenType != tokenType)
|
||||
{
|
||||
throw new MimicGeneratorException("Expected token type:{0}. Instead encountered:{1}. At line:{2}", tokenType.ToString(), reader.TokenType, reader.LineNumber);
|
||||
}
|
||||
}
|
||||
|
||||
private static bool CurrentValueMatches(JsonTextReader reader, string value)
|
||||
{
|
||||
return reader.Value != null && reader.Value.ToString().Equals(value);
|
||||
}
|
||||
|
||||
private void ReadObservedInputs()
|
||||
{
|
||||
Console.WriteLine("Parsing observed inputs");
|
||||
|
||||
using (StreamReader sr = new StreamReader(m_observedInputsPath))
|
||||
{
|
||||
string semiStableId;
|
||||
string path;
|
||||
string type;
|
||||
string contentHash;
|
||||
while (!sr.EndOfStream)
|
||||
{
|
||||
string pipLine = sr.ReadLine();
|
||||
string[] split = pipLine.Split();
|
||||
if (split.Length > 0)
|
||||
{
|
||||
semiStableId = split[0].TrimEnd(',').Replace(BuildXL.Pips.Operations.Pip.SemiStableHashPrefix, string.Empty);
|
||||
if (string.IsNullOrWhiteSpace(semiStableId))
|
||||
{
|
||||
throw new MimicGeneratorException("Could not parse SemiStableId: {0}", pipLine);
|
||||
}
|
||||
|
||||
int observedInputRecords;
|
||||
string[] recordCountSplit = sr.ReadLine().Split(':');
|
||||
if (recordCountSplit.Length != 2)
|
||||
{
|
||||
throw new MimicGeneratorException("Unexpected format for ObservedInputHashesByPath line: {0}", string.Join(":", recordCountSplit));
|
||||
}
|
||||
|
||||
if (!int.TryParse(recordCountSplit[1], out observedInputRecords))
|
||||
{
|
||||
throw new MimicGeneratorException("Unexpected format for ObservedInputHashesByPath line: {0}", string.Join(":", recordCountSplit));
|
||||
}
|
||||
|
||||
// Look up the pip by semiStableId
|
||||
Pip p;
|
||||
if (!m_graph.SemiStableHashes.TryGetValue(semiStableId, out p))
|
||||
{
|
||||
throw new MimicGeneratorException("Failed to find matching pip for semiStableId: {0}", semiStableId);
|
||||
}
|
||||
|
||||
ObservedAccess[] accesses = new ObservedAccess[observedInputRecords];
|
||||
|
||||
for (int i = 0; i < observedInputRecords; i++)
|
||||
{
|
||||
path = ExtractKVP(sr.ReadLine(), "Path").Value;
|
||||
contentHash = ExtractKVP(sr.ReadLine(), "ContentHash").Value;
|
||||
type = ExtractKVP(sr.ReadLine(), "Type").Value;
|
||||
|
||||
ObservedAccess access = new ObservedAccess()
|
||||
{
|
||||
ObservedAccessType = ParseObservedAccess(type),
|
||||
Path = path,
|
||||
ContentHash = contentHash,
|
||||
};
|
||||
|
||||
accesses[i] = access;
|
||||
}
|
||||
|
||||
if (!m_graph.ObservedAccesses.TryAdd(p.PipId, accesses))
|
||||
{
|
||||
throw new MimicGeneratorException("Duplicate observed access for pip: {0}", semiStableId);
|
||||
}
|
||||
|
||||
sr.ReadLine();
|
||||
}
|
||||
else
|
||||
{
|
||||
throw new MimicGeneratorException("Incorrect format for ObservedInputs line: ", pipLine);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static KeyValuePair<string, string> ExtractKVP(string line, string expectedKeyName)
|
||||
{
|
||||
string[] split = line.Split('=');
|
||||
if (split.Length != 2)
|
||||
{
|
||||
throw new MimicGeneratorException("Line had unexpected format: {0}", line);
|
||||
}
|
||||
|
||||
var result = new KeyValuePair<string, string>(split[0].Trim(' '), split[1].Trim(' '));
|
||||
|
||||
if (result.Key != expectedKeyName)
|
||||
{
|
||||
throw new MimicGeneratorException("Line had unexpected format. Expected key '{0} in line: {1}", expectedKeyName, line);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
private static ObservedAccessType ParseObservedAccess(string type)
|
||||
{
|
||||
switch (type)
|
||||
{
|
||||
case "DirectoryEnumeration":
|
||||
return ObservedAccessType.DirectoryEnumeration;
|
||||
case "AbsentPathProbe":
|
||||
return ObservedAccessType.AbsentPathProbe;
|
||||
case "FileContentRead":
|
||||
return ObservedAccessType.FileContentRead;
|
||||
default:
|
||||
throw new MimicGeneratorException("Unknown observed access type: {0}", type);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,65 +0,0 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT License.
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using BuildXL.FrontEnd.Script.Constants;
|
||||
using Tool.MimicGenerator.LanguageWriters;
|
||||
|
||||
namespace Tool.MimicGenerator
|
||||
{
|
||||
public enum Language : byte
|
||||
{
|
||||
None = 0,
|
||||
DScript,
|
||||
}
|
||||
|
||||
public sealed class LanguageProvider
|
||||
{
|
||||
private readonly Language m_language;
|
||||
|
||||
public LanguageProvider(Language language)
|
||||
{
|
||||
m_language = language;
|
||||
}
|
||||
|
||||
public ConfigWriter CreateConfigWriter(string absolutePath)
|
||||
{
|
||||
switch (m_language)
|
||||
{
|
||||
case Language.DScript:
|
||||
return new DScriptConfigWriter(absolutePath);
|
||||
default:
|
||||
throw new NotImplementedException(m_language.ToString());
|
||||
}
|
||||
}
|
||||
|
||||
public ModuleWriter CreateModuleWriter(string absolutePath, string identity, IEnumerable<string> logicAssemblies)
|
||||
{
|
||||
switch (m_language)
|
||||
{
|
||||
case Language.DScript:
|
||||
return new DScriptPackageWriter(absolutePath, identity, logicAssemblies);
|
||||
default:
|
||||
throw new NotImplementedException(m_language.ToString());
|
||||
}
|
||||
}
|
||||
|
||||
public SpecWriter CreateSpecWriter(string absolutePath)
|
||||
{
|
||||
switch (m_language)
|
||||
{
|
||||
case Language.DScript:
|
||||
if (!ExtensionUtilities.IsLegacyFileExtension(Path.GetExtension(absolutePath)))
|
||||
{
|
||||
absolutePath = Path.ChangeExtension(absolutePath, Names.DotDscExtension);
|
||||
}
|
||||
|
||||
return new DScriptSpecWriter(absolutePath);
|
||||
default:
|
||||
throw new NotImplementedException(m_language.ToString());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,78 +0,0 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT License.
|
||||
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using BuildXL.FrontEnd.Script.Constants;
|
||||
|
||||
namespace Tool.MimicGenerator.LanguageWriters
|
||||
{
|
||||
public sealed class DScriptConfigWriter : ConfigWriter
|
||||
{
|
||||
private readonly IList<string> m_importedModules;
|
||||
private readonly IDictionary<string, string> m_mountPoints;
|
||||
|
||||
public DScriptConfigWriter(string absolutePath)
|
||||
: base(Path.GetDirectoryName(absolutePath) + Path.DirectorySeparatorChar + Names.ConfigDsc)
|
||||
{
|
||||
m_importedModules = new List<string>();
|
||||
m_mountPoints = new Dictionary<string, string>();
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public override void AddModule(ModuleWriter moduleWriter)
|
||||
{
|
||||
m_importedModules.Add(DScriptWriterUtils.ToRelativePath(moduleWriter.AbsolutePath, AbsolutePath));
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public override void WriteMount(string mountName, string mountAbsolutePath)
|
||||
{
|
||||
m_mountPoints[mountName] = mountAbsolutePath;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public override string ToRelativePathExpression(string path)
|
||||
{
|
||||
string drive = char.ToUpperInvariant(path[0]).ToString();
|
||||
WriteMount(drive, drive);
|
||||
return DScriptWriterUtils.EncloseInQuotes(DScriptWriterUtils.GetPathFromExpression(path), "'");
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override void WriteStart()
|
||||
{
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override void WriteEnd()
|
||||
{
|
||||
var environmentVariablesString = string.Join(", ", AllowedEnvironmentVariables.Select(DScriptWriterUtils.EncloseInDoubleQuotes));
|
||||
Writer.WriteLine(
|
||||
@"config({
|
||||
projects: [],
|
||||
modules: [f`package.dsc`],
|
||||
resolvers: Environment.getPathValues(""ScriptSdk"", "";"").map(path => <SourceResolver>{
|
||||
kind: ""SourceResolver"",
|
||||
root: d`${path}`
|
||||
}),");
|
||||
Writer.WriteLine(" allowedEnvironmentVariables: [{0}],", environmentVariablesString);
|
||||
Writer.WriteLine(" mounts: [");
|
||||
foreach (var mountPoint in m_mountPoints)
|
||||
{
|
||||
Writer.WriteLine(" {");
|
||||
Writer.WriteLine(" name: PathAtom.create(\"{0}\"),", mountPoint.Key);
|
||||
Writer.WriteLine(" path: {0}, ", DScriptWriterUtils.ToPath(mountPoint.Value, PathType.Path));
|
||||
Writer.WriteLine(" trackSourceFileChanges: true, ");
|
||||
Writer.WriteLine(" isReadable: true, ");
|
||||
Writer.WriteLine(" isWritable: true, ");
|
||||
Writer.WriteLine(" isSystem: false ");
|
||||
Writer.WriteLine(" },");
|
||||
}
|
||||
|
||||
Writer.WriteLine(" ]");
|
||||
Writer.WriteLine(@"});");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,67 +0,0 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT License.
|
||||
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using BuildXL.FrontEnd.Script.Constants;
|
||||
|
||||
#pragma warning disable SA1649 // File name must match first type name
|
||||
|
||||
namespace Tool.MimicGenerator.LanguageWriters
|
||||
{
|
||||
internal readonly struct ImportSpec
|
||||
{
|
||||
public readonly DScriptSpecWriter SpecWriter;
|
||||
public readonly string VarName;
|
||||
|
||||
public ImportSpec(DScriptSpecWriter specWriter, string importVarName)
|
||||
{
|
||||
SpecWriter = specWriter;
|
||||
VarName = importVarName;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Writes a module file for DScript.
|
||||
/// </summary>
|
||||
public sealed class DScriptPackageWriter : ModuleWriter
|
||||
{
|
||||
private readonly Dictionary<string, ImportSpec> m_specs = new Dictionary<string, ImportSpec>();
|
||||
private int m_importCounter = 0;
|
||||
|
||||
public DScriptPackageWriter(string absolutePath, string identity, IEnumerable<string> logicAssemblies)
|
||||
: base(Path.Combine(absolutePath, Names.PackageDsc), identity, logicAssemblies)
|
||||
{
|
||||
using (StreamWriter sw = new StreamWriter(Path.Combine(absolutePath, Names.PackageConfigDsc)))
|
||||
{
|
||||
sw.WriteLine("module({ name: \"Mimic\" });");
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override void WriteStart()
|
||||
{
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override void WriteEnd()
|
||||
{
|
||||
// Do nothing.
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public override void AddSpec(string specRelativePath, SpecWriter specWriter)
|
||||
{
|
||||
var dsSpecWriter = specWriter as DScriptSpecWriter;
|
||||
|
||||
if (dsSpecWriter != null && !m_specs.ContainsKey(specRelativePath))
|
||||
{
|
||||
var importSpec = new ImportSpec(dsSpecWriter, "P" + m_importCounter++);
|
||||
m_specs[specRelativePath] = importSpec;
|
||||
string normalizedPath = DScriptWriterUtils.NormalizePath(specRelativePath);
|
||||
string name = normalizedPath.Replace("/", "__").Replace(".", "__").Replace("-", "_");
|
||||
Writer.WriteLine("import * as Mimic{1} from '/{0}';", normalizedPath, name);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,244 +0,0 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT License.
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics.CodeAnalysis;
|
||||
using System.Globalization;
|
||||
using System.IO;
|
||||
using System.Text;
|
||||
|
||||
namespace Tool.MimicGenerator.LanguageWriters
|
||||
{
|
||||
/// <summary>
|
||||
/// Writes a spec for DScript.
|
||||
/// </summary>
|
||||
public sealed class DScriptSpecWriter : SpecWriter
|
||||
{
|
||||
// key: specPath of pip value: variable name
|
||||
private readonly Dictionary<string, string> m_imports;
|
||||
private readonly List<string> m_runners;
|
||||
private int m_counter;
|
||||
private readonly MemoryStream m_memoryStream;
|
||||
private readonly StreamWriter m_writer;
|
||||
|
||||
/// <summary>
|
||||
/// Constructor
|
||||
/// </summary>
|
||||
public DScriptSpecWriter(string absolutePath)
|
||||
: base(absolutePath)
|
||||
{
|
||||
m_imports = new Dictionary<string, string>();
|
||||
m_runners = new List<string>();
|
||||
m_memoryStream = new MemoryStream();
|
||||
m_writer = new StreamWriter(m_memoryStream);
|
||||
}
|
||||
|
||||
/// <nodoc/>
|
||||
public IEnumerable<string> Runners
|
||||
{
|
||||
get { return m_runners; }
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override void WriteStart()
|
||||
{
|
||||
Writer.WriteLine("import * as Mimic from \"Tools.Mimic\";");
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override void WriteEnd()
|
||||
{
|
||||
Writer.WriteLine();
|
||||
m_writer.Flush();
|
||||
var contents = Encoding.UTF8.GetString(m_memoryStream.GetBuffer(), 0, (int)m_memoryStream.Length);
|
||||
m_writer.Dispose();
|
||||
|
||||
Writer.WriteLine(contents);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public override void AddSealDirectory(string valueName, string relativeDir, IEnumerable<string> relativePaths)
|
||||
{
|
||||
valueName = ToSealCopyWriteVarName(valueName);
|
||||
m_runners.Add(valueName);
|
||||
|
||||
m_writer.WriteLine("export const {0} = Transformer.sealPartialDirectory(", valueName);
|
||||
m_writer.WriteLine(" {0},", IfStringThenToPath(relativeDir, PathType.Directory));
|
||||
m_writer.WriteLine(" [");
|
||||
foreach (var relativePath in relativePaths)
|
||||
{
|
||||
m_writer.WriteLine(" {0}, ", IfStringThenToPath(relativePath, PathType.File));
|
||||
}
|
||||
|
||||
m_writer.WriteLine(" ]);");
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public override void AddMimicInvocation(
|
||||
string outputValue,
|
||||
IEnumerable<string> sealDirectoryInputs,
|
||||
IEnumerable<string> pathInputs,
|
||||
IEnumerable<MimicFileOutput> outputs,
|
||||
string observedAccessesPath,
|
||||
IEnumerable<SemaphoreInfo> semaphores,
|
||||
int runTimeInMs,
|
||||
bool isLongestProcess = false)
|
||||
{
|
||||
string mimicName = ToMimicRunnerVarName(outputValue);
|
||||
m_runners.Add(mimicName);
|
||||
|
||||
m_writer.WriteLine("export const {0} = Mimic.evaluate({{", mimicName);
|
||||
m_writer.WriteLine(" processRunningTime: {0},", runTimeInMs);
|
||||
m_writer.WriteLine(" isLongestProcess: {0},", isLongestProcess.ToString().ToLower(CultureInfo.CurrentCulture));
|
||||
if (observedAccessesPath != null)
|
||||
{
|
||||
m_writer.WriteLine(" observedAccesses: {0},", DScriptWriterUtils.ToPath(observedAccessesPath, PathType.Path));
|
||||
m_writer.WriteLine(" observedAccessesRoot: {0},", "p`/RootMarker.dummy`.parent");
|
||||
}
|
||||
|
||||
m_writer.WriteLine(" sealDirectoryInputs: [");
|
||||
foreach (var sealDirectory in sealDirectoryInputs)
|
||||
{
|
||||
m_writer.WriteLine(" {0},", IfStringThenToPath(sealDirectory, PathType.Directory));
|
||||
}
|
||||
|
||||
m_writer.WriteLine(" ],");
|
||||
|
||||
m_writer.WriteLine(" fileInputs: [");
|
||||
foreach (var pathInput in pathInputs)
|
||||
{
|
||||
m_writer.WriteLine(" {0},", IfStringThenToPath(pathInput, PathType.File));
|
||||
}
|
||||
|
||||
m_writer.WriteLine(" ],");
|
||||
|
||||
m_writer.WriteLine(" fileOutputs: [");
|
||||
|
||||
foreach (var output in outputs)
|
||||
{
|
||||
m_writer.WriteLine(" {");
|
||||
m_writer.WriteLine(" path: {0},", IfStringThenToPath(output.Path, PathType.Path));
|
||||
m_writer.WriteLine(" repeatingContent: \"{0}\",", output.RepeatingContent);
|
||||
m_writer.WriteLine(" lengthInBytes: {0},", output.LengthInBytes);
|
||||
m_writer.WriteLine(" fileId: {0}", output.FileId);
|
||||
m_writer.WriteLine(" }, ");
|
||||
}
|
||||
|
||||
m_writer.WriteLine(" ],");
|
||||
|
||||
m_writer.WriteLine(" semaphores: [");
|
||||
|
||||
foreach (var semaphore in semaphores)
|
||||
{
|
||||
m_writer.WriteLine(" {");
|
||||
m_writer.WriteLine(" name: {0},", semaphore.Name);
|
||||
m_writer.WriteLine(" value: \"{0}\",", semaphore.Value);
|
||||
m_writer.WriteLine(" limit: {0},", semaphore.Limit);
|
||||
m_writer.WriteLine(" }, ");
|
||||
}
|
||||
|
||||
m_writer.WriteLine(" ],");
|
||||
|
||||
// TODO: need to plumb this stuff through to DScript
|
||||
m_writer.WriteLine("});");
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public override void AddWriteFile(string valueName, string relativeDestination)
|
||||
{
|
||||
valueName = ToSealCopyWriteVarName(valueName);
|
||||
m_runners.Add(valueName);
|
||||
m_writer.WriteLine(
|
||||
"export const {0} = Transformer.writeFile({1}, {2});",
|
||||
valueName,
|
||||
IfStringThenToPath(relativeDestination, PathType.Path),
|
||||
"\"Dummy Write file\"");
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public override void AddCopyFile(string valueName, string relativeSource, string relativeDestination)
|
||||
{
|
||||
valueName = ToSealCopyWriteVarName(valueName);
|
||||
m_runners.Add(valueName);
|
||||
m_writer.WriteLine(
|
||||
"export const {0} = Transformer.copyFile({1}, {2});",
|
||||
valueName,
|
||||
IfStringThenToPath(relativeSource, PathType.File),
|
||||
IfStringThenToPath(relativeDestination, PathType.Path));
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public override string GetProcessInputName(string variableName, string specPath, int depId)
|
||||
{
|
||||
variableName = ToMimicRunnerVarName(variableName);
|
||||
var importVariable = GetImportVariable(specPath);
|
||||
string lhs = string.IsNullOrEmpty(importVariable) ? variableName : importVariable + "." + variableName;
|
||||
return string.Format(CultureInfo.InvariantCulture, "{0}.producedFiles.get({1})", lhs, depId);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public override string GetSealCopyWriteInputName(string variableName, string specPath)
|
||||
{
|
||||
variableName = ToSealCopyWriteVarName(variableName);
|
||||
var importVariable = GetImportVariable(specPath);
|
||||
return string.IsNullOrEmpty(importVariable)
|
||||
? string.Format(CultureInfo.InvariantCulture, "{0}", variableName)
|
||||
: string.Format(CultureInfo.InvariantCulture, "{0}.{1}", importVariable, variableName);
|
||||
}
|
||||
|
||||
[SuppressMessage("Microsoft.Design", "CA1063")]
|
||||
public new void Dispose()
|
||||
{
|
||||
base.Dispose();
|
||||
m_memoryStream.Dispose();
|
||||
m_writer.Dispose();
|
||||
}
|
||||
|
||||
private string GetImportVariable(string specPath)
|
||||
{
|
||||
specPath = DScriptWriterUtils.GetPathFromExpression(specPath);
|
||||
|
||||
// Check whether producer of variableName is the same file or not.
|
||||
if (DScriptWriterUtils.NormalizePath(AbsolutePath).EndsWith(specPath, StringComparison.Ordinal))
|
||||
{
|
||||
return string.Empty;
|
||||
}
|
||||
|
||||
specPath = DScriptWriterUtils.RemoveFileEnding(specPath);
|
||||
|
||||
string variableName;
|
||||
if (m_imports.TryGetValue(specPath, out variableName))
|
||||
{
|
||||
// return null;
|
||||
return variableName;
|
||||
}
|
||||
|
||||
variableName = "P" + m_counter++;
|
||||
m_imports.Add(specPath, variableName);
|
||||
Writer.WriteLine("import * as {1} from '{0}.dsc';", DScriptWriterUtils.NormalizePath(specPath), variableName);
|
||||
return variableName;
|
||||
}
|
||||
|
||||
private static string ToSealCopyWriteVarName(string name)
|
||||
{
|
||||
return "sealCopyWrite" + name;
|
||||
}
|
||||
|
||||
private static string ToMimicRunnerVarName(string name)
|
||||
{
|
||||
return "mimicRunner" + name;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// If given 'expr' is quoted (<see cref="IsStringExpression"/>), the expression is converted
|
||||
/// to path (<see cref="ToPath"/>); otherwise, the expression is returned.
|
||||
/// </summary>
|
||||
private static string IfStringThenToPath(string expression, PathType type)
|
||||
{
|
||||
return DScriptWriterUtils.IsStringExpression(expression)
|
||||
? DScriptWriterUtils.ToPath(expression, type)
|
||||
: expression;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,153 +0,0 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT License.
|
||||
|
||||
using System;
|
||||
using System.Diagnostics.ContractsLight;
|
||||
using System.IO;
|
||||
|
||||
namespace Tool.MimicGenerator.LanguageWriters
|
||||
{
|
||||
internal enum PathType
|
||||
{
|
||||
Path,
|
||||
Directory,
|
||||
File,
|
||||
}
|
||||
|
||||
internal static class DScriptWriterUtils
|
||||
{
|
||||
/// <summary>
|
||||
/// It is a string expression if it is enclosed in single or double quotes.
|
||||
/// </summary>
|
||||
internal static bool IsStringExpression(string s)
|
||||
{
|
||||
if (s == null)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
s = s.Trim();
|
||||
return (s.StartsWith("'", StringComparison.Ordinal) && s.EndsWith("'", StringComparison.Ordinal)) ||
|
||||
(s.StartsWith("\"", StringComparison.Ordinal) && s.EndsWith("\"", StringComparison.Ordinal));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The DScript syntax for paths is p``, d``, f``, so if 's' is a string expression
|
||||
/// (i.e., quoted), it first unquotes it, then, in any case, encloses it in _` and `.
|
||||
/// </summary>
|
||||
internal static string ToPath(string s, PathType type)
|
||||
{
|
||||
Contract.Requires(s != null);
|
||||
|
||||
bool isStringExpr = IsStringExpression(s);
|
||||
string unquoted = isStringExpr ? s.Substring(1, s.Length - 2) : s;
|
||||
string escaped = unquoted.Replace("'", "\'").Replace("`", "\\`");
|
||||
|
||||
// Switch the path to a unix style
|
||||
escaped = escaped.Replace('\\', '/');
|
||||
|
||||
string prefix = string.Empty;
|
||||
switch (type)
|
||||
{
|
||||
case PathType.Directory:
|
||||
prefix = "d`";
|
||||
break;
|
||||
case PathType.File:
|
||||
prefix = "f`";
|
||||
break;
|
||||
case PathType.Path:
|
||||
prefix = "p`";
|
||||
break;
|
||||
}
|
||||
|
||||
return prefix + escaped + "`";
|
||||
}
|
||||
|
||||
/// <nodoc/>
|
||||
internal static string EncloseInQuotes(string s, string quoteMark)
|
||||
{
|
||||
return quoteMark + s + quoteMark;
|
||||
}
|
||||
|
||||
/// <nodoc/>
|
||||
internal static string EncloseInDoubleQuotes(string s)
|
||||
{
|
||||
return EncloseInQuotes(s, "\"");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Takes 2 path, 'target' and 'from', and returns a string representing the relative path
|
||||
/// from 'from' to 'target'. If no common prefix is found, no relative path can be constructed,
|
||||
/// so 'target' is returned instead.
|
||||
/// </summary>
|
||||
internal static string ToRelativePath(string target, string from)
|
||||
{
|
||||
Contract.Requires(target != null);
|
||||
Contract.Requires(from != null);
|
||||
|
||||
string[] targetDirSegments = NormalizePath(Path.GetDirectoryName(target)).Split('/');
|
||||
string[] fromDirSegments = NormalizePath(Path.GetDirectoryName(from)).Split('/');
|
||||
|
||||
// find the common prefix
|
||||
int cnt = 0;
|
||||
while (cnt < targetDirSegments.Length && cnt < fromDirSegments.Length && targetDirSegments[cnt].Equals(fromDirSegments[cnt]))
|
||||
{
|
||||
cnt++;
|
||||
}
|
||||
|
||||
// no common prefix found -> return target
|
||||
if (cnt == 0)
|
||||
{
|
||||
return target;
|
||||
}
|
||||
|
||||
// prefix found ->
|
||||
// 1. navigate up the remainder fo 'fromDirSegments' (using '..')
|
||||
int relPathCnt = 0;
|
||||
string[] relPath = new string[(fromDirSegments.Length - cnt) + (targetDirSegments.Length - cnt) + 1];
|
||||
for (int i = cnt; i < fromDirSegments.Length; i++)
|
||||
{
|
||||
relPath[relPathCnt++] = "..";
|
||||
}
|
||||
|
||||
// 2. then down the remainder of 'targetDirSegments' (using those exact segements)
|
||||
for (int i = cnt; i < targetDirSegments.Length; i++)
|
||||
{
|
||||
relPath[relPathCnt++] = targetDirSegments[i];
|
||||
}
|
||||
|
||||
// 3. finally add the target file name
|
||||
relPath[relPathCnt] = Path.GetFileName(target);
|
||||
return string.Join("/", relPath);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Return relative path (to the package location) instead of Path.Combine expression
|
||||
/// </summary>
|
||||
internal static string GetPathFromExpression(string path)
|
||||
{
|
||||
Contract.Requires(path != null);
|
||||
Contract.Requires(path.Length >= 3);
|
||||
|
||||
var driveLetter = char.ToUpperInvariant(path[0]);
|
||||
path = NormalizePath(path).Replace("'", @"\'");
|
||||
return "/" + driveLetter + "/" + path.Remove(0, 3);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Returns full file path without the file extension.
|
||||
/// </summary>
|
||||
internal static string RemoveFileEnding(string path)
|
||||
{
|
||||
return Path.GetDirectoryName(path) + Path.DirectorySeparatorChar + Path.GetFileNameWithoutExtension(path);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Replaces backslashes (\) with forwardslashes (/).
|
||||
/// </summary>
|
||||
internal static string NormalizePath(string path)
|
||||
{
|
||||
return path.Replace('\\', '/');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,19 +0,0 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT License.
|
||||
|
||||
using System;
|
||||
using System.Diagnostics.CodeAnalysis;
|
||||
using System.Globalization;
|
||||
|
||||
namespace Tool.MimicGenerator
|
||||
{
|
||||
[SuppressMessage("Microsoft.Design", "CA1032:ImplementStandardExceptionConstructors")]
|
||||
[SuppressMessage("Microsoft.Usage", "CA2237:MarkISerializableTypesWithSerializable")]
|
||||
public sealed class MimicGeneratorException : Exception
|
||||
{
|
||||
public MimicGeneratorException(string format, params object[] args)
|
||||
: base(string.Format(CultureInfo.InvariantCulture, format, args))
|
||||
{
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,25 +0,0 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT License.
|
||||
|
||||
using System.Collections.Generic;
|
||||
|
||||
namespace Tool.MimicGenerator
|
||||
{
|
||||
/// <summary>
|
||||
/// Writes a module file
|
||||
/// </summary>
|
||||
public abstract class ModuleWriter : BuildXLFileWriter
|
||||
{
|
||||
protected readonly string Identity;
|
||||
protected readonly IEnumerable<string> LogicAssemblies;
|
||||
|
||||
protected ModuleWriter(string absolutePath, string identity, IEnumerable<string> logicAssemblies)
|
||||
: base(absolutePath)
|
||||
{
|
||||
Identity = identity;
|
||||
LogicAssemblies = logicAssemblies;
|
||||
}
|
||||
|
||||
public abstract void AddSpec(string specRelativePath, SpecWriter specWriter);
|
||||
}
|
||||
}
|
||||
|
|
@ -1,112 +0,0 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT License.
|
||||
|
||||
using System.Collections.Generic;
|
||||
|
||||
namespace Tool.MimicGenerator
|
||||
{
|
||||
public abstract class Pip
|
||||
{
|
||||
public int? OriginalPipId;
|
||||
public int PipId;
|
||||
public readonly string Spec;
|
||||
public readonly string StableId;
|
||||
|
||||
protected Pip(int pipId, string stableId, string spec)
|
||||
{
|
||||
PipId = pipId;
|
||||
StableId = stableId;
|
||||
Spec = spec;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Data for a declared semaphore
|
||||
/// </summary>
|
||||
public sealed class SemaphoreInfo
|
||||
{
|
||||
/// <summary>
|
||||
/// The resource name
|
||||
/// </summary>
|
||||
public string Name;
|
||||
|
||||
/// <summary>
|
||||
/// The semaphore value
|
||||
/// </summary>
|
||||
public int Value;
|
||||
|
||||
/// <summary>
|
||||
/// The maximum value
|
||||
/// </summary>
|
||||
public int Limit;
|
||||
}
|
||||
|
||||
public sealed class Process : Pip
|
||||
{
|
||||
public readonly List<int> Produces;
|
||||
public readonly List<int> Consumes;
|
||||
public readonly List<SemaphoreInfo> Semaphores;
|
||||
|
||||
// TODO: Using a default wall time of 10 seconds. This should go away once execution time is required
|
||||
public int ProcessWallTimeMs = 10000;
|
||||
|
||||
public Process(int pipId, string stableId, string spec, List<int> produces, List<int> consumes, List<SemaphoreInfo> semaphores)
|
||||
: base(pipId, stableId, spec)
|
||||
{
|
||||
Produces = produces;
|
||||
Consumes = consumes;
|
||||
Semaphores = semaphores;
|
||||
}
|
||||
}
|
||||
|
||||
public sealed class WriteFile : Pip
|
||||
{
|
||||
public readonly int Destination;
|
||||
|
||||
public WriteFile(int pipId, string stableId, string spec, int destination)
|
||||
: base(pipId, stableId, spec)
|
||||
{
|
||||
Destination = destination;
|
||||
}
|
||||
}
|
||||
|
||||
public sealed class CopyFile : Pip
|
||||
{
|
||||
public readonly int Source;
|
||||
public readonly int Destination;
|
||||
|
||||
public CopyFile(int pipId, string stableId, string spec, int source, int destination)
|
||||
: base(pipId, stableId, spec)
|
||||
{
|
||||
Source = source;
|
||||
Destination = destination;
|
||||
}
|
||||
}
|
||||
|
||||
public sealed class SealDirectory : Pip
|
||||
{
|
||||
public readonly int Directory;
|
||||
|
||||
public SealDirectory(int pipId, string stableId, string spec, int directory)
|
||||
: base(pipId, stableId, spec)
|
||||
{
|
||||
Directory = directory;
|
||||
}
|
||||
}
|
||||
|
||||
public sealed class ObservedAccess
|
||||
{
|
||||
public ObservedAccessType ObservedAccessType { get; set; }
|
||||
|
||||
public string Path { get; set; }
|
||||
|
||||
public string ContentHash { get; set; }
|
||||
}
|
||||
|
||||
public enum ObservedAccessType
|
||||
{
|
||||
DirectoryEnumeration,
|
||||
AbsentPathProbe,
|
||||
FileContentRead,
|
||||
}
|
||||
}
|
||||
|
|
@ -1,72 +0,0 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT License.
|
||||
|
||||
using System;
|
||||
using System.Diagnostics;
|
||||
using System.Diagnostics.CodeAnalysis;
|
||||
|
||||
namespace Tool.MimicGenerator
|
||||
{
|
||||
/// <summary>
|
||||
/// Generates spec and input files to mimic a build
|
||||
/// </summary>
|
||||
public static class Program
|
||||
{
|
||||
/// <summary>
|
||||
/// Program main
|
||||
/// </summary>
|
||||
[SuppressMessage("Microsoft.Naming", "CA2204")]
|
||||
public static void Main(string[] arguments)
|
||||
{
|
||||
string debug = Environment.GetEnvironmentVariable("MimicGeneratorDebugOnStart");
|
||||
if (debug != null && debug != "0")
|
||||
{
|
||||
Debugger.Launch();
|
||||
}
|
||||
|
||||
Stopwatch sw = Stopwatch.StartNew();
|
||||
Args args = new Args(arguments);
|
||||
if (args.HelpDisplayed)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
GraphReader reader = new GraphReader(args.JsonGraph, args.ObservedInputs);
|
||||
BuildGraph graph = reader.ReadGraph();
|
||||
|
||||
if (args.DuplicateGraph > 1)
|
||||
{
|
||||
GraphMultiplier.DuplicateAsParallelGraphs(graph, args.DuplicateGraph, args.MaxPipsPerSpec);
|
||||
}
|
||||
|
||||
BuildWriter writer = new BuildWriter(args.Dest, args.WriteInputs, args.InputScaleFactor, graph, args.IgnoreResponseFiles, args.Language);
|
||||
if (!writer.WriteBuildFiles())
|
||||
{
|
||||
ExitError(sw);
|
||||
}
|
||||
|
||||
Console.WriteLine("MimicGenerator completed successfully in {0} seconds.", sw.Elapsed.TotalSeconds);
|
||||
Environment.Exit(0);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
ExitError(sw, ex);
|
||||
}
|
||||
}
|
||||
|
||||
[SuppressMessage("Microsoft.Naming", "CA2204:Literals should be spelled correctly",
|
||||
MessageId = "MimicGenerator")]
|
||||
private static void ExitError(Stopwatch sw, Exception ex = null)
|
||||
{
|
||||
Console.WriteLine("MimicGenerator failed {0} seconds.", sw.Elapsed.TotalSeconds);
|
||||
if (ex != null)
|
||||
{
|
||||
Console.Error.Write(ex);
|
||||
}
|
||||
|
||||
Environment.Exit(1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,39 +0,0 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT License.
|
||||
|
||||
namespace Tool.MimicGenerator
|
||||
{
|
||||
/// <summary>
|
||||
/// Writes a source file
|
||||
/// </summary>
|
||||
public class SourceWriter : BuildXLFileWriter
|
||||
{
|
||||
/// <summary>
|
||||
/// Writes a source file
|
||||
/// </summary>
|
||||
/// <param name="absolutePath">absolute path of the file</param>
|
||||
/// <param name="repeatingContent">content to be written to file until lengthInBytes is reached</param>
|
||||
/// <param name="lengthInBytes">Approximate length of file. Actual written file may be off by a factor of
|
||||
/// however long the releatingContent parameter is.</param>
|
||||
public SourceWriter(string absolutePath, string repeatingContent, long lengthInBytes)
|
||||
: base(absolutePath)
|
||||
{
|
||||
while (Writer.BaseStream.Position < lengthInBytes)
|
||||
{
|
||||
Writer.Write(repeatingContent);
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
protected override void WriteStart()
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
protected override void WriteEnd()
|
||||
{
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,132 +0,0 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT License.
|
||||
|
||||
using System.Collections.Generic;
|
||||
using System.Globalization;
|
||||
using System.IO;
|
||||
|
||||
namespace Tool.MimicGenerator
|
||||
{
|
||||
/// <summary>
|
||||
/// Writes a spec
|
||||
/// </summary>
|
||||
public abstract class SpecWriter : BuildXLFileWriter
|
||||
{
|
||||
/// <summary>
|
||||
/// Constructor
|
||||
/// </summary>
|
||||
protected SpecWriter(string absolutePath)
|
||||
: base(absolutePath) { }
|
||||
|
||||
/// <summary>
|
||||
/// Adds a SealPartialDirectory runner
|
||||
/// </summary>
|
||||
/// <param name="valueName">Name of the resulting value</param>
|
||||
/// <param name="relativeDir">the relative path to the directory root</param>
|
||||
/// <param name="relativePaths">the relative paths of the files within the directory</param>
|
||||
public abstract void AddSealDirectory(string valueName, string relativeDir, IEnumerable<string> relativePaths);
|
||||
|
||||
/// <summary>
|
||||
/// Data for a Mimic.exe output file
|
||||
/// </summary>
|
||||
public sealed class MimicFileOutput
|
||||
{
|
||||
/// <summary>
|
||||
/// Path of the file
|
||||
/// </summary>
|
||||
public string Path;
|
||||
|
||||
/// <summary>
|
||||
/// File content to be repeated up to LengthInBytes
|
||||
/// </summary>
|
||||
public string RepeatingContent;
|
||||
|
||||
/// <summary>
|
||||
/// Length of file that should be created
|
||||
/// </summary>
|
||||
public int LengthInBytes;
|
||||
|
||||
/// <summary>
|
||||
/// The ID of the file
|
||||
/// </summary>
|
||||
public int FileId;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Adds a Mimic processes
|
||||
/// </summary>
|
||||
/// <param name="outputValue">Name of the output value</param>
|
||||
/// <param name="sealDirectoryInputs">SealDirectory inputs. Should already be in the form of a expression</param>
|
||||
/// <param name="pathInputs">inputs</param>
|
||||
/// <param name="outputs">outputs</param>
|
||||
/// <param name="observedAccessesPath">Path to file containing observed file accesses</param>
|
||||
/// <param name="semaphores">semaphores</param>
|
||||
/// <param name="runTimeInMs">diration for the mimic invocation</param>
|
||||
/// <param name="isLongestProcess">true if this is the longest running process in a spec</param>
|
||||
public abstract void AddMimicInvocation(
|
||||
string outputValue,
|
||||
IEnumerable<string> sealDirectoryInputs,
|
||||
IEnumerable<string> pathInputs,
|
||||
IEnumerable<MimicFileOutput> outputs,
|
||||
string observedAccessesPath,
|
||||
IEnumerable<SemaphoreInfo> semaphores,
|
||||
int runTimeInMs,
|
||||
bool isLongestProcess = false);
|
||||
|
||||
public abstract void AddWriteFile(string valueName, string relativeDestination);
|
||||
|
||||
public abstract void AddCopyFile(string valueName, string relativeSource, string relativeDestination);
|
||||
|
||||
public abstract string GetProcessInputName(string variableName, string specPath, int depId);
|
||||
|
||||
public abstract string GetSealCopyWriteInputName(string variableName, string specPath);
|
||||
|
||||
/// <summary>
|
||||
/// Writes the observedAccesses to a file next to the generated spec. This file may be consumed by the Mimic.exe
|
||||
/// pip to more faithfully reply file accesses
|
||||
/// </summary>
|
||||
/// <returns>Absolute path to the ObservedAccesses file</returns>
|
||||
public string WriteObservedAccessesFile(ObservedAccess[] accesses, int lookupId)
|
||||
{
|
||||
if (lookupId == -1 || accesses == null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
// Write the observed accesses in a file next to the
|
||||
string destinationDir = Path.Combine(Path.GetDirectoryName(AbsolutePath), "observedAccesses");
|
||||
Directory.CreateDirectory(destinationDir);
|
||||
string destination = Path.Combine(destinationDir, "Pip_" + lookupId.ToString(CultureInfo.InvariantCulture) + ".txt");
|
||||
|
||||
using (StreamWriter writer = new StreamWriter(destination))
|
||||
{
|
||||
writer.WriteLine(ObservedAccessesVersion);
|
||||
|
||||
foreach (var access in accesses)
|
||||
{
|
||||
// Observed inputs are full paths like c:\foo\bar. Strip out the ':' to make them relative paths to
|
||||
// where input files are generated.
|
||||
writer.WriteLine(access.Path.Replace(":", string.Empty));
|
||||
switch (access.ObservedAccessType)
|
||||
{
|
||||
case ObservedAccessType.AbsentPathProbe:
|
||||
writer.WriteLine("A");
|
||||
break;
|
||||
case ObservedAccessType.DirectoryEnumeration:
|
||||
writer.WriteLine("D");
|
||||
break;
|
||||
case ObservedAccessType.FileContentRead:
|
||||
writer.WriteLine("F");
|
||||
break;
|
||||
default:
|
||||
throw new MimicGeneratorException("Unknown observed access type {0}", access.ObservedAccessType);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return destination;
|
||||
}
|
||||
|
||||
private const int ObservedAccessesVersion = 1;
|
||||
}
|
||||
}
|
||||
|
|
@ -1,31 +0,0 @@
|
|||
// Copyright (c) Microsoft. All rights reserved.
|
||||
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
|
||||
|
||||
namespace MimicGenerator {
|
||||
@@public
|
||||
export const exe = BuildXLSdk.executable({
|
||||
assemblyName: "MimicGenerator",
|
||||
rootNamespace: "Tool.MimicGenerator",
|
||||
skipDocumentationGeneration: true,
|
||||
sources: globR(d`.`, "*.cs"),
|
||||
references: [
|
||||
...addIf(BuildXLSdk.isFullFramework,
|
||||
NetFx.System.IO.dll
|
||||
),
|
||||
importFrom("BuildXL.Pips").dll,
|
||||
importFrom("BuildXL.Utilities").dll,
|
||||
importFrom("BuildXL.Utilities").Branding.dll,
|
||||
importFrom("BuildXL.Utilities").ToolSupport.dll,
|
||||
importFrom("BuildXL.Utilities").Script.Constants.dll,
|
||||
importFrom("BuildXL.Utilities").Collections.dll,
|
||||
importFrom("Newtonsoft.Json").pkg,
|
||||
],
|
||||
embeddedResources: [
|
||||
{
|
||||
linkedContent: [
|
||||
f`Content/CacheConfig.json`
|
||||
]
|
||||
}
|
||||
],
|
||||
});
|
||||
}
|
||||
|
|
@ -1,8 +0,0 @@
|
|||
// Copyright (c) Microsoft. All rights reserved.
|
||||
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
|
||||
|
||||
import * as BuildXLSdk from "Sdk.BuildXL";
|
||||
import { NetFx } from "Sdk.BuildXL";
|
||||
|
||||
export {BuildXLSdk, NetFx};
|
||||
export declare const qualifier: BuildXLSdk.FullFrameworkQualifier;
|
||||
|
|
@ -1,6 +0,0 @@
|
|||
// Copyright (c) Microsoft. All rights reserved.
|
||||
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
|
||||
|
||||
module({
|
||||
name: "BuildXL.PrivateTools"
|
||||
});
|
||||
Загрузка…
Ссылка в новой задаче