microsoft
/
BuildXL
зеркало из https://github.com/microsoft/BuildXL.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Use jsondiffpatch.net for cache miss analysis (#207) 

* Switch cache miss to use jsondiffpatch.net

* Rip out old diffing classes

* Remove unnecesary line in test
This commit is contained in:
Annie Fu 2019-04-26 16:35:29 -07:00 коммит произвёл dannyvv
Родитель 628f28b76e
Коммит d72a42a6cd
8 изменённых файлов: 80 добавлений и 872 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

1
Public/Src/Engine/Cache/BuildXL.Cache.dsc
Просмотреть файл

@ -35,6 +35,7 @@ namespace Cache {
importFrom("BuildXL.Utilities").Storage.dll,
importFrom("BuildXL.Utilities").Collections.dll,
importFrom("BuildXL.Utilities").Configuration.dll,
importFrom("JsonDiffPatch.Net").pkg,
importFrom("Newtonsoft.Json").pkg,
],
});

288
Public/Src/Engine/Cache/Serialization/ChangeList.cs
Просмотреть файл

@ -1,288 +0,0 @@
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
using System.Collections.Generic;
using BuildXL.Utilities;
namespace BuildXL.Engine.Cache.Serialization
{
/// <summary>
/// A class used to diff two ordered lists.
/// A "diff" is considered a set of add or remove operations applied to
/// the first list to transform it the second list.
/// </summary>
public class ChangeList<T>
{
/// <summary>
/// Represents the type of change operations that can be
/// used by this class when diffing two lists.
/// </summary>
public enum ChangeType
{
/// <summary>
/// Default.
/// </summary>
None,
/// <summary>
/// The element was removed.
/// </summary>
Removed,
/// <summary>
/// The element was added.
/// </summary>
Added
}
/// <summary>
/// Represents an element in a <see cref="ChangeList{T}"/>.
/// </summary>
public struct ChangeListValue
{
/// <summary>
/// Depending on the <see cref="ChangeType"/>, a value from either <see cref="OldList"/> or <see cref="NewList"/>.
/// </summary>
public T Value;
/// <summary>
/// The type of change that was applied to the element.
/// </summary>
public ChangeType ChangeType;
/// <summary>
/// Constructor.
/// </summary>
public ChangeListValue(T value, ChangeType changeType)
{
Value = value;
ChangeType = changeType;
}
/// <inheritdoc />
public override string ToString()
{
var changeSymbol = ChangeType == ChangeType.Removed ? "-" : "+";
return string.Format("\t{0} {1}", changeSymbol, Value.ToString());
}
}
/// <summary>
/// When computing <see cref="Changes"/>, the
/// original list that was transformed to <see cref="NewList"/>.
/// </summary>
public IReadOnlyList<T> OldList { get; private set; }
/// <summary>
/// When computing <see cref="Changes"/>, the
/// resulting list that was transformed from <see cref="OldList"/>.
/// </summary>
public IReadOnlyList<T> NewList { get; private set; }
/// <summary>
/// A list of changes that could have been made to transform
/// <see cref="OldList"/> to <see cref="NewList"/>.
/// This is the underlying list for the <see cref="ChangeList{T}"/>.
/// </summary>
private List<ChangeListValue> Changes { get; set; }
/// <summary>
/// The number of changes in the change list.
/// </summary>
public int Count => Changes.Count;
/// <summary>
/// Constructor.
/// </summary>
public ChangeList(IReadOnlyList<T> oldList, IReadOnlyList<T> newList)
{
OldList = oldList;
NewList = newList;
Changes = new List<ChangeListValue>();
DiffLists();
}
/// <summary>
/// Adds an element that was removed from
/// <see cref="OldList"/> to the <see cref="ChangeList{T}"/>.
/// </summary>
/// <param name="element"></param>
private void AddRemoved(T element)
{
Changes.Add(new ChangeListValue
{
Value = element,
ChangeType = ChangeType.Removed
});
}
/// <summary>
/// Adds an element that was added to
/// <see cref="NewList"/> to the <see cref="ChangeList{T}"/>.
/// </summary>
/// <param name="element"></param>
private void AddAdded(T element)
{
Changes.Add(new ChangeListValue
{
Value = element,
ChangeType = ChangeType.Added
});
}
/// <summary>
/// Computes the longest common subsequence of two lists.
/// A subsequence is a sequence that appears in the same relative order,
/// but not necessarily contiguously.
/// </summary>
/// <note>
/// This function implements the dynamic programming solution to the longest common
/// subsequence problem. Since the problem definition and algorithm are well documented
/// publicly, full details are not provided here.
/// </note>
/// <returns>
/// The longest common subsequence represented by their indices in <see cref="OldList"/>
/// (to prevent copying the elements).
/// </returns>
private List<int> ComputeLongestCommonSubsequence()
{
// Produces a memoization matrix of size [old list size + 1][new list size + 1].
// The index [r, c] represents the size of the longest common subsequence between
// the sub-lists OldList[0, r - 1] and NewList[0, c - 1].
var lcs_Matrix = new int[OldList.Count + 1][];
for (int i = 0; i < OldList.Count + 1; ++i)
{
lcs_Matrix[i] = new int[NewList.Count + 1];
}
for (int r = 1; r < OldList.Count + 1; ++r)
{
for (int c = 1; c < NewList.Count + 1; ++c)
{
if (OldList[r - 1].Equals(NewList[c - 1]))
{
lcs_Matrix[r][c] = lcs_Matrix[r - 1][c - 1] + 1;
}
else
{
lcs_Matrix[r][c] = System.Math.Max(lcs_Matrix[r - 1][c], lcs_Matrix[r][c - 1]);
}
}
}
// Traversing the matrix according to the algorithm produces the longest common subsequence
// in reverse order. Add to the front of a linked list to reverse the order back.
var longestCommonSubsequence = new LinkedList<int>();
for (int r = OldList.Count, c = NewList.Count; r != 0 && c != 0;)
{
if (OldList[r - 1].Equals(NewList[c - 1]))
{
longestCommonSubsequence.AddFirst(r - 1);
r--;
c--;
}
else
{
if (lcs_Matrix[r - 1][c] > lcs_Matrix[r][c - 1])
{
r--;
}
else
{
c--;
}
}
}
return new List<int>(longestCommonSubsequence);
}
/// <summary>
/// Diffs two ordered (but not necessarily sorted) lists.
/// </summary>
private void DiffLists()
{
// The longest common subsequence is the maximum amount of elements shared
// between both lists. Since shared elements are not included in the diff,
// maximizing the number of shared elements minimizes the amount of elements diffed.
var lcs = ComputeLongestCommonSubsequence();
int oldIdx = 0;
int newIdx = 0;
for (int i = 0; i < lcs.Count; ++i, ++oldIdx, ++newIdx)
{
// To prevent copying the elements over to the subsequence,
// subsequence elements are stored as their corresponding indices
// in OldList
var commonElement = OldList[lcs[i]];
// Any element that is in the old list, but not the longest common subsequence
// must have been removed to get to the new list
while (!OldList[oldIdx].Equals(commonElement))
{
AddRemoved(OldList[oldIdx]);
oldIdx++;
}
// Any element that is the new list, but not the longest common subsequence
// must have been added to get to the new list
while (!NewList[newIdx].Equals(commonElement))
{
AddAdded(NewList[newIdx]);
newIdx++;
}
}
// Handle trailing elements after the last element in the longest common subsequence
while (oldIdx < OldList.Count)
{
AddRemoved(OldList[oldIdx]);
++oldIdx;
}
while (newIdx < NewList.Count)
{
AddAdded(NewList[newIdx]);
++newIdx;
}
}
/// <summary>
/// Index into the change list.
/// </summary>
public ChangeListValue this[int i]
{
get
{
return Changes[i];
}
}
/// <inheritdoc />
public override string ToString() => ToString(string.Empty);
/// <summary>
/// Converts a <see cref="ChangeList{T}"/> to a string.
/// </summary>
/// <param name="prefix">
/// A prefix string to append to each value.
/// </param>
public string ToString(string prefix)
{
using (var pool = Pools.GetStringBuilder())
{
var sb = pool.Instance;
foreach (var change in Changes)
{
sb.AppendLine(prefix + change.ToString());
}
return sb.ToString();
}
}
}
}

303
Public/Src/Engine/Cache/Serialization/JsonTree.cs
Просмотреть файл

@ -5,7 +5,9 @@ using System;
using System.Collections.Generic;
using System.IO;
using BuildXL.Utilities;
using JsonDiffPatchDotNet;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
namespace BuildXL.Engine.Cache.Serialization
{
@ -115,8 +117,19 @@ namespace BuildXL.Engine.Cache.Serialization
/// <summary>
/// Reads a valid JSON object into a tree.
/// </summary>
public class JsonTree
public static class JsonTree
{
private static JsonDiffPatch s_jdp = null;
static JsonTree()
{
var diffOptions = new Options();
diffOptions.ArrayDiff = ArrayDiffMode.Efficient;
diffOptions.TextDiff = TextDiffMode.Simple;
s_jdp = new JsonDiffPatch(diffOptions);
}
/// <summary>
/// Given a valid JSON object, builds a tree of <see cref="JsonNode"/>s.
/// </summary>
@ -151,6 +164,9 @@ namespace BuildXL.Engine.Cache.Serialization
case JsonToken.EndObject:
currentNode = parentStack.Pop();
break;
// Arrays are represented by either nodes having multiple values, or multiple children
// This will remove unnecessary "nameless" nested arrays that don't provide meaningful information
// These two strings deserialize the same: { "key" : [value] }, { "key" : ["value"] }
case JsonToken.StartArray:
case JsonToken.EndArray:
default:
@ -223,110 +239,21 @@ namespace BuildXL.Engine.Cache.Serialization
}
/// <summary>
/// Collects all the nodes of a tree that have values.
/// </summary>
public static List<JsonNode> CollectValueNodes(JsonNode root)
{
var nodeStack = new Stack<JsonNode>();
var valueNodes = new List<JsonNode>();
nodeStack.Push(root);
JsonNode currentNode;
while (nodeStack.Count != 0)
{
currentNode = nodeStack.Pop();
// Value node
if (currentNode.Values.Count > 0)
{
valueNodes.Add(currentNode);
}
foreach (var child in currentNode.Children)
{
nodeStack.Push(child);
}
}
return valueNodes;
}
/// <summary>
/// Diffs two <see cref="JsonNode"/> trees.
/// </summary>
/// <returns>
/// A string representation of the diff.
/// A JSON string representation of the diff.
/// </returns>
public static string PrintTreeDiff(JsonNode rootA, JsonNode rootB)
{
var changeList = DiffTrees(rootA, rootB);
return PrintTreeChangeList(changeList);
}
var jsonA = Serialize(rootA);
var jsonB = Serialize(rootB);
/// <summary>
/// Formats and prints the resulting <see cref="ChangeList{T}"/> from a <see cref="DiffTrees(JsonNode, JsonNode)"/>.
/// </summary>
/// <returns>
/// A string representation of the change list.
/// </returns>
public static string PrintTreeChangeList(ChangeList<JsonNode> changeList)
{
var treeToPrint = new PrintNode();
for (int i = 0; i < changeList.Count; ++i)
{
var change = changeList[i];
var node = change.Value;
var path = new LinkedList<string>();
// Use "it.Parent != null" instead of "it != null" to
// ignore printing the root parent node that matches the opening bracket
// for all JSON objects
for (var it = node; it.Parent != null; it = it.Parent)
{
path.AddFirst(it.Name);
}
var diff = s_jdp.Diff(JToken.Parse(jsonA), JToken.Parse(jsonB));
var printNode = treeToPrint;
// Build a tree of the change list values, placing nodes based off
// their positions in the old or new tree
foreach (var pathAtom in path)
{
if (!printNode.Children.ContainsKey(pathAtom))
{
printNode.Children.Add(pathAtom, new PrintNode());
}
printNode = printNode.Children[pathAtom];
}
printNode.ChangedNodes.Add(change);
}
return treeToPrint.ToString();
}
/// <summary>
/// Diffs two trees.
/// </summary>
/// <param name="rootA">
/// The root of the original tree.
/// </param>
/// <param name="rootB">
/// The root of the transformed tree.
/// </param>
/// <returns>
/// A <see cref="ChangeList{JsonNode}"/> containing
/// the leaf nodes that differ.
/// </returns>
public static ChangeList<JsonNode> DiffTrees(JsonNode rootA, JsonNode rootB)
{
var leavesA = CollectValueNodes(rootA);
var leavesB = CollectValueNodes(rootB);
var changeList = new ChangeList<JsonNode>(leavesA, leavesB);
return changeList;
return diff == null ? string.Empty : diff.ToString();
}
/// <summary>
@ -400,50 +327,6 @@ namespace BuildXL.Engine.Cache.Serialization
root.Parent = newParent;
}
/// <summary>
/// Prints the leaves in a tree and the paths to the leaves. Internal for testing.
/// </summary>
internal static string PrintLeaves(JsonNode root)
{
using (var sbPool = Pools.GetStringBuilder())
{
var sb = sbPool.Instance;
var nodeStack = new Stack<JsonNode>();
var nameStack = new Stack<string>();
nodeStack.Push(root);
nameStack.Push(root.Name);
JsonNode currentNode;
string currentName;
while (nodeStack.Count != 0)
{
currentNode = nodeStack.Pop();
currentName = nameStack.Pop();
// Leaf node
if (currentNode.Children.Count == 0)
{
sb.AppendLine();
sb.AppendFormat("{0}:", currentName);
foreach (var value in currentNode.Values)
{
sb.AppendFormat("\"{0}\",", value);
}
}
foreach (var child in currentNode.Children)
{
nodeStack.Push(child);
nameStack.Push(currentName + ":" + child.Name);
}
}
return sb.ToString();
}
}
/// <summary>
/// Reads a JSON string into a new, formatted JSON string.
@ -501,147 +384,5 @@ namespace BuildXL.Engine.Cache.Serialization
}
}
}
/// <summary>
/// Helper class for printing the result of <see cref="DiffTrees(JsonNode, JsonNode)"/>
/// in a tree format, organizing <see cref="ChangeList{T}"/> values by their position in the original trees.
/// Each <see cref="PrintNode"/> represents a position in a tree. That position may have existed in the original tree,
/// the resulting tree, or both.
/// </summary>
private class PrintNode
{
/// <summary>
/// The <see cref="ChangeList{T}.ChangeListValue"/>s that represent nodes removed or added at this particular position in the tree.
/// </summary>
public readonly List<ChangeList<JsonNode>.ChangeListValue> ChangedNodes;
/// <summary>
/// The children of this node. The name of a child node is encapsulated in the key.
/// This means that the root of a tree will be nameless.
/// </summary>
public readonly Dictionary<string, PrintNode> Children;
/// <summary>
/// Constructor.
/// </summary>
public PrintNode()
{
ChangedNodes = new List<ChangeList<JsonNode>.ChangeListValue>();
Children = new Dictionary<string, PrintNode>();
}
/// <summary>
/// Helper struct for holding state used while recursively printing tree.
/// </summary>
private struct RecursionState
{
/// <summary>
/// The node being printed.
/// </summary>
public readonly PrintNode PrintNode;
/// <summary>
/// How many level deeps in the tree the node is, where 0 is top-level node
/// that gets printed.
/// </summary>
public readonly int Level;
/// <summary>
/// The name of the node being printed.
/// </summary>
public readonly string Name;
/// <summary>
/// Constructor.
/// </summary>
public RecursionState(PrintNode printNode, int level, string name)
{
PrintNode = printNode;
Level = level;
Name = name;
}
}
/// <summary>
/// Returns the tree of changes encapusulated by this <see cref="PrintNode"/> as an indented, formatted string.
/// </summary>
public override string ToString()
{
using (var sbPool = Pools.GetStringBuilder())
{
var sb = sbPool.Instance;
var stack = new Stack<RecursionState>();
stack.Push(new RecursionState(this, -1, null));
while (stack.Count != 0)
{
var curr = stack.Pop();
var indentPrefix = curr.Level > 0 ? new string('\t', curr.Level) : string.Empty;
if (curr.Name != null)
{
sb.AppendLine(indentPrefix + curr.Name);
}
// Each PrintNode represents one position in a tree. The values of a PrintNode represent nodes that were added or removed at that position.
// The max number of values at a position is 2, when a node is removed from one position and a new one is added at the same position.
// This is equivalent to modifying the state of that node.
switch (curr.PrintNode.ChangedNodes.Count)
{
// A node exists in one tree that does not exist in the other
// In this case, print all the values of the node as either added or removed
case 1:
var changeListValue = curr.PrintNode.ChangedNodes[0];
sb.Append(indentPrefix + changeListValue.ToString());
break;
// A node exists in both trees, but with different values (equivalent to modifying the node)
// Consolidate the print out by diffing just the values
case 2:
ChangeList<JsonNode>.ChangeListValue removed, added;
if (curr.PrintNode.ChangedNodes[0].ChangeType == ChangeList<JsonNode>.ChangeType.Removed)
{
removed = curr.PrintNode.ChangedNodes[0];
added = curr.PrintNode.ChangedNodes[1];
}
else
{
removed = curr.PrintNode.ChangedNodes[1];
added = curr.PrintNode.ChangedNodes[0];
}
// Order of removed vs added node is not guaranteed in the change list,
// but when diffing the nodes' values, the removed node represents a node from the old tree,
// so it should go first and represent the old list to get the correct diff.
var changeList = new ChangeList<string>(removed.Value.Values, added.Value.Values);
if (changeList.Count == 0)
{
// There was no difference in values between the removed node and added node,
// this means that the node simply moved positions.
// In this case, print the full list of values for both the removed and added node
sb.Append(indentPrefix + removed.ToString());
sb.Append(indentPrefix + added.ToString());
}
else
{
// Otherwise, rely on the normal diff
sb.Append(changeList.ToString(indentPrefix));
}
break;
default:
break;
}
foreach (var child in curr.PrintNode.Children)
{
stack.Push(new RecursionState(child.Value, curr.Level + 1, child.Key));
}
}
return sb.ToString();
}
}
}
}
}

18
Public/Src/Engine/Scheduler/Tracing/CacheMissAnalysisUtilities.cs
Просмотреть файл

@ -157,10 +157,20 @@ namespace BuildXL.Scheduler.Tracing
if (oldPipSession.FormattedSemiStableHash != newPipSession.FormattedSemiStableHash)
{
WriteLine(RepeatedStrings.FormattedSemiStableHashChanged, writer);
// Make a trivial change list so the print looks like the rest of the diff
var changeList = new ChangeList<string>(new string[] { oldPipSession.FormattedSemiStableHash }, new string[] { newPipSession.FormattedSemiStableHash });
WriteLine(changeList.ToString(), writer);
// Make trivial json so the print looks like the rest of the diff
var oldNode = new JsonNode
{
Name = RepeatedStrings.FormattedSemiStableHashChanged
};
oldNode.Values.Add(oldPipSession.FormattedSemiStableHash);
var newNode = new JsonNode
{
Name = RepeatedStrings.FormattedSemiStableHashChanged
};
newNode.Values.Add(newPipSession.FormattedSemiStableHash);
WriteLine(JsonTree.PrintTreeDiff(oldNode, newNode), writer);
}
// Diff based off the actual fingerprints instead of the PipCacheMissType

142
Public/Src/Engine/UnitTests/Cache/ChangeListTests.cs
Просмотреть файл

@ -1,142 +0,0 @@
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
using Xunit;
using BuildXL.Engine.Cache.Serialization;
using System.Collections.Generic;
using Test.BuildXL.TestUtilities.Xunit;
namespace Test.BuildXL.Engine.Cache
{
public class ChangeListTests
{
public ChangeListTests()
{
}
private struct TestPair
{
public List<int> OldList;
public List<int> NewList;
}
[Fact]
public void Tests()
{
var testPairs = new TestPair[]
{
new TestPair
{
OldList = new List<int> { 6, 2, 4, 0 },
NewList = new List<int> { 0, 2, 3, 4, 1 }
},
new TestPair
{
OldList = new List<int> { 1, 2, 3, 4, 1 },
NewList = new List<int> { 3, 4, 1, 2, 1, 3 }
},
new TestPair
{
OldList = new List<int> { 9, 8, 7 },
NewList = new List<int> { 9, 1, 2, 6 }
},
new TestPair
{
OldList = new List<int> { 3, 9, 8, 3, 9, 7, 9, 7, 0 },
NewList = new List<int> { 3, 3, 9, 9, 9, 1, 7, 2, 0, 6 }
},
new TestPair
{
OldList = new List<int> { 1, 2 },
NewList = new List<int> { 3, 4, 5 }
}
};
foreach (var test in testPairs)
{
var changeList = new ChangeList<int>(test.OldList, test.NewList);
AssertValidChange(test.OldList, test.NewList, changeList);
}
}
[Fact]
public void NoChangeTests()
{
var emptyList = new List<int>();
var changeList = new ChangeList<int>(emptyList, emptyList);
XAssert.AreEqual(0, changeList.Count);
var list = new List<int> { 6, 2, 4, 0 };
changeList = new ChangeList<int>(list, list);
XAssert.AreEqual(0, changeList.Count);
}
/// <summary>
/// Checks that when the changes in change list are applied to
/// old list, the resulting list has the same values as new list.
/// </summary>
private void AssertValidChange<T>(List<T> oldList, List<T> newList, ChangeList<T> changeList)
{
var transformedOld = ListToDictionary<T>(oldList);
for (int i = 0; i < changeList.Count; ++i)
{
var change = changeList[i];
switch (change.ChangeType)
{
case ChangeList<T>.ChangeType.Removed:
if (transformedOld[change.Value] == 1)
{
transformedOld.Remove(change.Value);
}
else
{
transformedOld[change.Value]--;
}
break;
case ChangeList<T>.ChangeType.Added:
if (transformedOld.ContainsKey(change.Value))
{
transformedOld[change.Value]++;
}
else
{
transformedOld[change.Value] = 1;
}
break;
}
}
var transformedNew = ListToDictionary(newList);
XAssert.AreEqual(transformedOld.Count, transformedNew.Count);
foreach (var valueCount in transformedNew)
{
XAssert.AreEqual(valueCount.Value, transformedOld[valueCount.Key]);
}
}
/// <summary>
/// Transforms a list to a dictionary with a count of the number of times
/// each value appears.
/// </summary>
private Dictionary<T, int> ListToDictionary<T>(List<T> list)
{
var listDictionary = new Dictionary<T, int>();
foreach (var value in list)
{
if (listDictionary.ContainsKey(value))
{
listDictionary[value]++;
}
else
{
listDictionary[value] = 1;
}
}
return listDictionary;
}
}
}

182
Public/Src/Engine/UnitTests/Cache/JsonTreeTests.cs
Просмотреть файл

@ -13,38 +13,22 @@ namespace Test.BuildXL.Engine.Cache
{
}
[Fact]
public void MoveObjectsInList()
{
string jsonA = "{\"Dependencies\":[{\"a\":\"valueA\"},{\"b\":\"valueB\"}]}";
string jsonB = "{\"Dependencies\":[{\"b\":\"valueB\"},{\"a\":\"valueA\"}]}";
var treeA = JsonTree.Deserialize(jsonA);
var treeB = JsonTree.Deserialize(jsonB);
var changeList = JsonTree.DiffTrees(treeA, treeB);
// The change list must detect either "a" or "b" as having moved positions
// the current algorithm detects happens to detect "a"
AssertRemoved("a", changeList);
AssertAdded("a", changeList);
var printDiff = JsonTree.PrintTreeChangeList(changeList);
XAssert.IsTrue(printDiff.Contains("a") && printDiff.Contains("valueA"));
XAssert.IsFalse(printDiff.Contains("b") || printDiff.Contains("valueB"));
}
[Fact]
public void BasicTest()
{
string jsonA = "{\"Object\":[{\"WeakFingerprint\":\"097ED1ED5703816B8F286410076E658260D029BC\"},{\"StrongFingerprint\":\"C926945B5824E1CC7C512D66FB3B8FE869B71936\"}]}";
string jsonB = "{\"Object\":[{\"WeakFingerprint\":\"097ED1ED5703816B8F286410076E658260D029BC\"},{\"StrongFingerprint\":\"DefinitelyNotTheSameFingerprint\"}]}";
string jsonA = "{\"Dependencies\":[{\"a\":\"valueA\"},{\"b\":\"valueB\"}]}";
var treeA = JsonTree.Deserialize(jsonA);
var treeB = JsonTree.Deserialize(jsonB);
string deserialized = JsonTree.Serialize(treeA);
var changeList = JsonTree.DiffTrees(treeA, treeB);
AssertUnchanged("WeakFingerprint", changeList);
AssertChanged("StrongFingerprint", changeList);
XAssert.AreEqual(JsonTree.PrettyPrintJson(jsonA), deserialized);
string jsonB = "{\"Object\":[{\"WeakFingerprint\":\"097ED1ED5703816B8F286410076E658260D029BC\"},{\"StrongFingerprint\":\"C926945B5824E1CC7C512D66FB3B8FE869B71936\"}]}";
var treeB = JsonTree.Deserialize(jsonB);
string deserializedB = JsonTree.Serialize(treeB);
XAssert.AreEqual(JsonTree.PrettyPrintJson(jsonB), deserializedB);
}
[Fact]
@ -52,147 +36,49 @@ namespace Test.BuildXL.Engine.Cache
{
string jsonA = "{\"Object\":[{\"PathSet\":\"VSO0:890000000000000000000000000000000000000000000000000000000000000000\"}," +
"{\"ObservedInputs\":[{\"P\":\"\"},{\"P\":\"\"},{\"P\":\"\"},{\"P\":\"\"},{\"P\":\"\"},{\"A\":\"\"},{\"P\":\"\"},{\"E\":\"VSO0:4D939FB1E1CE7586909F84F4FEFB0F385B31DD586FF97FC14874BCDB4B2A801400\"}]}]}";
string jsonB = "{\"Object\":[{\"PathSet\":\"VSO0:890000000000000000000000000000000000000000000000000000000000000000\"}," +
"{\"ObservedInputs\":[{\"P\":\"\"},{\"P\":\"\"},{\"P\":\"CHANGEDVALUE\"},{\"P\":\"\"},{\"P\":\"\"},{\"P\":\"\"},{\"E\":\"VSO0:4D939FB1E1CE7586909F84F4FEFB0F385B31DD586FF97FC14874BCDB4B2A801400\"}]}]}";
var treeA = JsonTree.Deserialize(jsonA);
var treeB = JsonTree.Deserialize(jsonB);
string deserialized = JsonTree.Serialize(treeA);
var changeList = JsonTree.DiffTrees(treeA, treeB);
AssertChanged("P", changeList);
AssertAdded("P", changeList);
AssertRemoved("A", changeList);
XAssert.AreEqual(JsonTree.PrettyPrintJson(jsonA), deserialized);
}
[Fact]
public void NestedNamelessArraysTest()
{
var jsonA = "{\"key\":[\"value\", [\"nestedValue\"]]}";
var treeA = JsonTree.Deserialize(jsonA);
var keyNode = JsonTree.FindNodeByName(treeA, "key");
XAssert.IsTrue(keyNode.Values.Contains("value") && keyNode.Values.Contains("nestedValue"));
string deserialized = JsonTree.Serialize(treeA);
XAssert.IsTrue(deserialized.Contains("value") && deserialized.Contains("nestedValue"));
XAssert.AreNotEqual(JsonTree.PrettyPrintJson(jsonA), deserialized);
}
[Fact]
public void LargeNestedObjectTest()
{
string jsonA = "{\"Object\":[{\"ExecutionAndFingerprintOptionsHash\":\"224F5F8E17D4590F2CD7AFAB82AA672D2E0C86E8\"}," +
string jsonA = "{\"Object\":[\"10?B:\\\\out\\\\objects\\\\d\\\\z\\\\bqkrr8zzp30wta21jx01qhl915zwxn\\\\xunit-out\\\\FingerprintS1F4BDE58\\\\2\\\\obj\\\\readonly\\\\obj_0???None?\"," +
"\"5?B:\\\\out\\\\objects\\\\d\\\\z\\\\bqkrr8zzp30wta21jx01qhl915zwxn\\\\xunit-out\\\\FingerprintS1F4BDE58\\\\2\\\\obj\\\\obj_1???None?\",{\"ExecutionAndFingerprintOptionsHash\":\"224F5F8E17D4590F2CD7AFAB82AA672D2E0C86E8\"}," +
"{\"ContentHashAlgorithmName\":\"Vso0\"},{\"PipType\":\"80\"}," +
"{\"Executable\":[{\"B:/out/objects/d/z/bqkrr8zzp30wta21jx01qhl915zwxn/Test.BuildXL.FingerprintStore-test-deployment/TestProcess/Test.BuildXL.Executables.TestProcess.exe\":\"VSO0:17426AF3467E448CD8E1146EEE9CFC27107690D39B8091E2882AB164B57ED90400\"}]}," +
"{\"Executable\":{\"B:/out/objects/d/z/bqkrr8zzp30wta21jx01qhl915zwxn/Test.BuildXL.FingerprintStore-test-deployment/TestProcess/Test.BuildXL.Executables.TestProcess.exe\":\"VSO0:17426AF3467E448CD8E1146EEE9CFC27107690D39B8091E2882AB164B57ED90400\"}}," +
"{\"WorkingDirectory\":\"B:/out/objects/d/z/bqkrr8zzp30wta21jx01qhl915zwxn/xunit-out/FingerprintS1F4BDE58/2/obj/z/7/69c8nz08y0ehxg5vn8ex8g7g59at90/Test.BuildXL.Executables.TestProcess\"}," +
"{\"StandardError\":\"{Invalid}\"},{\"StandardOutput\":\"{Invalid}\"}," +
"{\"Dependencies\":[{\"B:/out/objects/d/z/bqkrr8zzp30wta21jx01qhl915zwxn/Test.BuildXL.FingerprintStore-test-deployment/TestProcess/Test.BuildXL.Executables.TestProcess.exe\":\"VSO0:17426AF3467E448CD8E1146EEE9CFC27107690D39B8091E2882AB164B57ED90400\"}]}," +
"{\"Dependencies\":{\"B:/out/objects/d/z/bqkrr8zzp30wta21jx01qhl915zwxn/Test.BuildXL.FingerprintStore-test-deployment/TestProcess/Test.BuildXL.Executables.TestProcess.exe\":\"VSO0:17426AF3467E448CD8E1146EEE9CFC27107690D39B8091E2882AB164B57ED90400\"}}," +
"{\"DirectoryDependencies\":[]},{\"Outputs\":[{\"Path\":\"B:/out/objects/d/z/bqkrr8zzp30wta21jx01qhl915zwxn/xunit-out/ FingerprintS1F4BDE58 / 2 / obj / obj_1\"},{\"Attributes\":\"0\"}]},{\"DirectoryOutputs\":[]}," +
"{\"UntrackedPaths\":[]},{\"UntrackedScopes\":[\"C:/WINDOWS\",\"C:/Users/userName/AppData/Local/Microsoft/Windows/INetCache\",\"C:/Users/userName/AppData/Local/Microsoft/Windows/History\"]}," +
"{\"HasUntrackedChildProcesses\":\"0\"},{\"PipData\":\"Arguments\"},[{\"Escaping\":\"2\"},{\"Separator\":\" \"},{\"Fragments\":\"1\"},[{\"PipData\":\"NestedFragment\"},[{\"Escaping\":\"2\"},{\"Separator\":\" \"},{\"Fragments\":\"2\"}," +
"[\"10?B:\\\\out\\\\objects\\\\d\\\\z\\\\bqkrr8zzp30wta21jx01qhl915zwxn\\\\xunit-out\\\\FingerprintS1F4BDE58\\\\2\\\\obj\\\\readonly\\\\obj_0???None?\"," +
"\"5?B:\\\\out\\\\objects\\\\d\\\\z\\\\bqkrr8zzp30wta21jx01qhl915zwxn\\\\xunit-out\\\\FingerprintS1F4BDE58\\\\2\\\\obj\\\\obj_1???None?\"]]]]," +
"{\"HasUntrackedChildProcesses\":\"0\"},{\"PipData\":\"Arguments\"},{\"Escaping\":\"2\"},{\"Separator\":\" \"},{\"Fragments\":\"1\"},{\"PipData\":\"NestedFragment\"},{\"Escaping\":\"2\"},{\"Separator\":\" \"},{\"Fragments\":\"2\"}," +
"{\"Environment\":[]},{\"WarningTimeout\":\"-1\"},{\"WarningRegex.Pattern\":\"^\\\\s*((((((\\\\d+>)?[a-zA-Z]?:[^:]*)|([^:]*))):)|())(()|([^:]*? ))warning( \\\\s*([^: ]*))?\\\\s*:.*$\"},{\"WarningRegex.Options\":\"1\"}," +
"{\"ErrorRegex.Pattern\":\".*\"},{\"ErrorRegex.Options\":\"1\"},{\"SuccessExitCodes\":[]}]}";
string jsonB = "{\"Object\":[{\"ExecutionAndFingerprintOptionsHash\":\"224F5F8E17D4590F2CD7AFAB82AA672D2E0C86E8\"}," +
"{\"ContentHashAlgorithmName\":\"Vso0\"},{\"PipType\":\"80\"}," +
"{\"Executable\":[{\"B:/out/objects/d/z/bqkrr8zzp30wta21jx01qhl915zwxn/Test.BuildXL.FingerprintStore-test-deployment/TestProcess/Test.BuildXL.Executables.TestProcess.exe\":\"VSO0:17426AF3467E448CD8E1146EEE9CFC27107690D39B8091E2882AB164B57ED90400\"}]}," +
"{\"WorkingDirectory\":\"B:/out/objects/d/z/bqkrr8zzp30wta21jx01qhl915zwxn/xunit-out/FingerprintS1F4BDE58/2/obj/z/7/69c8nz08y0ehxg5vn8ex8g7g59at90/Test.BuildXL.Executables.TestProcess\"}," +
"{\"StandardOutput\":\"CHANGED\"}," +
"{\"Dependencies\":[{\"B:/out/objects/d/z/bqkrr8zzp30wta21jx01qhl915zwxn/Test.BuildXL.FingerprintStore-test-deployment/TestProcess/Test.BuildXL.Executables.TestProcess.exe\":\"VSO0:17426AF3467E448CD8E1146EEE9CFC27107690D39B8091E2882AB164B57ED90400\"}]}," +
"{\"DirectoryDependencies\":[]},{\"Outputs\":[{\"Path\":\"B:/out/objects/d/z/bqkrr8zzp30wta21jx01qhl915zwxn/xunit-out/ FingerprintS1F4BDE58 / 2 / obj / obj_1\"},{\"Attributes\":\"0\"}]},{\"DirectoryOutputs\":[]}," +
"{\"UntrackedPaths\":[]},{\"UntrackedScopes\":[\"C:/WINDOWS\",\"C:/Users/userName/AppData/Local/Microsoft/Windows/INetCache\",\"C:/Users/userName/AppData/Local/Microsoft/Windows/CHANGED\"]}," +
"{\"HasUntrackedChildProcesses\":\"0\"},{\"Timeout\":\"-1\"},{\"PipData\":\"Arguments\"},[{\"Escaping\":\"2\"},{\"Separator\":\" \"},{\"Fragments\":\"1\"},[{\"PipData\":\"NestedFragment\"},[{\"Escaping\":\"2\"},{\"Separator\":\" \"},{\"Fragments\":\"2\"}," +
"[\"10?B:\\\\out\\\\objects\\\\d\\\\z\\\\bqkrr8zzp30wta21jx01qhl915zwxn\\\\xunit-out\\\\FingerprintS1F4BDE58\\\\2\\\\obj\\\\readonly\\\\obj_0???None?\"," +
"\"5?B:\\\\out\\\\objects\\\\d\\\\z\\\\bqkrr8zzp30wta21jx01qhl915zwxn\\\\xunit-out\\\\FingerprintS1F4BDE58\\\\2\\\\obj\\\\obj_1???None?\"]]]]," +
"{\"Environment\":[]},{\"WarningTimeout\":\"-1\"},{\"AddedTimeout\":\"-1\"},{\"WarningRegex.Pattern\":\"^\\\\s*((((((\\\\d+>)?[a-zA-Z]?:[^:]*)|([^:]*))):)|())(()|([^:]*? ))warning( \\\\s*([^: ]*))?\\\\s*:.*$\"},{\"WarningRegex.Options\":\"1\"}," +
"{\"ErrorRegex.Pattern\":\".*\"},{\"ErrorRegex.Options\":\"1\"},{\"SuccessExitCodes\":[]}]}";
var treeA = JsonTree.Deserialize(jsonA);
var treeB = JsonTree.Deserialize(jsonB);
string deserialized = JsonTree.Serialize(treeA);
var changeList = JsonTree.DiffTrees(treeA, treeB);
AssertUnchanged("Outputs", changeList);
AssertChanged("StandardOutput", changeList);
AssertChanged("UntrackedScopes", changeList);
AssertRemoved("StandardError", changeList);
AssertAdded("AddedTimeout", changeList);
}
private void AssertRemoved(string propertyName, ChangeList<JsonNode> changeList)
{
bool removedFound = false;
for (int i = 0; i < changeList.Count; ++i)
{
var change = changeList[i];
if (change.Value.Name == propertyName
&& change.ChangeType == ChangeList<JsonNode>.ChangeType.Removed)
{
removedFound = true;
break;
}
}
XAssert.IsTrue(removedFound);
}
private void AssertAdded(string propertyName, ChangeList<JsonNode> changeList)
{
bool addedFound = false;
for (int i = 0; i < changeList.Count; ++i)
{
var change = changeList[i];
if (change.Value.Name == propertyName
&& change.ChangeType == ChangeList<JsonNode>.ChangeType.Added)
{
addedFound = true;
break;
}
}
XAssert.IsTrue(addedFound);
}
private void AssertUnchanged(string propertyName, ChangeList<JsonNode> changeList)
{
JsonNode removed = null;
JsonNode added = null;
for (int i = 0; i < changeList.Count; ++i)
{
var change = changeList[i];
if (change.Value.Name != propertyName)
{
continue;
}
switch (change.ChangeType)
{
case ChangeList<JsonNode>.ChangeType.Removed:
removed = change.Value;
break;
case ChangeList<JsonNode>.ChangeType.Added:
added = change.Value;
break;
}
}
XAssert.AreEqual(null, removed);
XAssert.AreEqual(null, added);
}
/// <summary>
/// To be "changed", a node must have the same name, but different values.
/// </summary>
private void AssertChanged(string propertyName, ChangeList<JsonNode> changeList)
{
JsonNode removed = null;
JsonNode added = null;
for (int i = 0; i < changeList.Count; ++i)
{
var change = changeList[i];
if (change.Value.Name != propertyName)
{
continue;
}
switch (change.ChangeType)
{
case ChangeList<JsonNode>.ChangeType.Removed:
removed = change.Value;
break;
case ChangeList<JsonNode>.ChangeType.Added:
added = change.Value;
break;
}
}
XAssert.AreNotEqual(null, removed);
XAssert.AreNotEqual(null, added);
XAssert.AreNotEqual(removed, added);
XAssert.AreEqual(JsonTree.PrettyPrintJson(jsonA), deserialized);
}
}
}

12
Public/Src/Tools/UnitTests/Analyzers/FingerprintStoreAnalyzerTests.cs
Просмотреть файл

@ -174,7 +174,7 @@ namespace Test.Tool.Analyzers
var entry = new BuildXLConfiguration.Mutable.FileAccessWhitelistEntry()
{
Value = "testValue",
PathFragment = ArtifactToPrint(whitelistFile),
PathFragment = ArtifactToString(whitelistFile),
};
Configuration.FileAccessWhiteList.Add(entry);
@ -276,8 +276,8 @@ namespace Test.Tool.Analyzers
pipA,
PipCacheMissType.MissForDescriptorsDueToWeakFingerprints,
ArtifactToPrint(srcA));
// Analyze downstream pips
allPipsResult.AssertPipMiss(
// Analyze downstream pips, runtime cache miss does not have an all pips option
allPipsResult.AssertAnalyzerPipMiss(
pipA,
PipCacheMissType.MissForDescriptorsDueToWeakFingerprints,
ArtifactToPrint(srcB));
@ -688,8 +688,6 @@ namespace Test.Tool.Analyzers
result.AssertPipMiss(
pip,
PipCacheMissType.MissForDescriptorsDueToStrongFingerprints,
"-",
"+",
"UnsafeOptions");
}
@ -773,7 +771,7 @@ namespace Test.Tool.Analyzers
// Ensure that the analyzer falls-back on the execution-time store when the cache-lookup store is missing entries
// srcB from pipB's dependencies will show up in the analysis because of earlier manipulation of the execution-time store
var incorrectOut = RunAnalyzer(build1, build2).AssertPipMiss(
var incorrectOut = RunAnalyzer(build1, build2).AssertAnalyzerPipMiss(
pipA,
PipCacheMissType.MissForDescriptorsDueToStrongFingerprints,
ArtifactToPrint(srcB)).FileOutput;
@ -787,7 +785,7 @@ namespace Test.Tool.Analyzers
/// </summary>
private string ArtifactToPrint(FileOrDirectoryArtifact artifact)
{
return Expander.ExpandPath(Context.PathTable, artifact.Path).ToLowerInvariant();
return Expander.ExpandPath(Context.PathTable, artifact.Path).ToLowerInvariant().Replace(@"\", @"\\");
}
public void AssertCacheMissEventLogged(params string[] requiredMessages)

2
config.dsc
Просмотреть файл

@ -137,6 +137,8 @@ config({
{ id: "RocksDbSharp", version: "5.8.0-b20181023.3", alias: "RocksDbSharpSigned" },
{ id: "RocksDbNative", version: "5.14.3-b20181023.3" },
{ id: "JsonDiffPatch.Net", version: "2.1.0" },
// Event hubs
{ id: "Microsoft.Azure.Amqp", version: "2.3.5" },
{ id: "Microsoft.Azure.EventHubs", version: "2.1.0",