482 строки
22 KiB
C#
482 строки
22 KiB
C#
using Microsoft.VisualStudio.TestTools.UnitTesting;
|
|
using Newtonsoft.Json;
|
|
using System;
|
|
using System.Diagnostics;
|
|
using System.IO;
|
|
using System.Linq;
|
|
|
|
namespace BlackBox
|
|
{
|
|
[TestClass]
|
|
public class Tests
|
|
{
|
|
[TestMethod]
|
|
public void TestPrg()
|
|
{
|
|
// 2 tests for integer interval and 2 tests for unit interval.
|
|
var prgTests = new ITestConfiguration[4];
|
|
for (uint i = 0; i < prgTests.Length; i++)
|
|
{
|
|
prgTests[i] = new PrgTestConfiguration
|
|
{
|
|
Seed = i,
|
|
Iterations = 1000,
|
|
UniformInterval = i < 2 ? new Tuple<uint, uint>(i * 10, (i + 10) * 10) : null,
|
|
};
|
|
}
|
|
Run(outputFilePatternExpected, outputFilePatternActual, outputJsonConfigFile, prgTests);
|
|
|
|
for (uint i = 0; i < 2; i++)
|
|
{
|
|
// integer content so should be exact match
|
|
CompareExactMatch(FormatPath(outputFilePatternExpected, i), FormatPath(outputFilePatternActual, i));
|
|
}
|
|
|
|
for (uint i = 2; i < 4; i++)
|
|
{
|
|
float[] expected = File.ReadAllLines(FormatPath(outputFilePatternExpected, i)).Select(l => Convert.ToSingle(l)).ToArray();
|
|
float[] actual = File.ReadAllLines(FormatPath(outputFilePatternActual, i)).Select(l => Convert.ToSingle(l)).ToArray();
|
|
|
|
Assert.AreEqual(expected.Length, actual.Length);
|
|
for (int j = 0; j < expected.Length; j++)
|
|
{
|
|
// allow slightly different float precision
|
|
Assert.IsTrue(Math.Abs(expected[j] - actual[j]) < PrecisionOffset);
|
|
}
|
|
}
|
|
}
|
|
|
|
[TestMethod]
|
|
public void TestHash()
|
|
{
|
|
var chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789abcdefghijklmnopqrstuvwxyz";
|
|
|
|
var rand = new Random();
|
|
|
|
var hashTests = new ITestConfiguration[5];
|
|
for (uint i = 0; i < hashTests.Length - 1; i++)
|
|
{
|
|
int numValues = rand.Next(10, 20);
|
|
var values = new string[numValues];
|
|
|
|
for (int v = 0; v < numValues; v++)
|
|
{
|
|
if (rand.NextDouble() < 0.5)
|
|
{
|
|
// generate string of numbers only
|
|
values[v] = rand.Next(1000000).ToString("000000");
|
|
}
|
|
else
|
|
{
|
|
// generate random string
|
|
int length = rand.Next(10, 20); // random length (in characters)
|
|
|
|
values[v] = new string(Enumerable
|
|
.Repeat(chars, length)
|
|
.Select(s => s[rand.Next(s.Length)])
|
|
.ToArray());
|
|
}
|
|
}
|
|
|
|
hashTests[i] = new HashTestConfiguration
|
|
{
|
|
Values = values.ToList()
|
|
};
|
|
}
|
|
|
|
hashTests[hashTests.Length - 1] = new HashTestConfiguration
|
|
{
|
|
Values = new[] { "是是asda阿斯顿", " ", " ", "\t\r\n", "\r", "hô ooaf à ị ẻ ể \r \t \n \\ồ" }.ToList()
|
|
};
|
|
|
|
Run(outputFilePatternExpected, outputFilePatternActual, outputJsonConfigFile, hashTests);
|
|
|
|
for (uint i = 0; i < hashTests.Length; i++)
|
|
{
|
|
// integer content so should be exact match
|
|
CompareExactMatch(FormatPath(outputFilePatternExpected, i), FormatPath(outputFilePatternActual, i));
|
|
}
|
|
}
|
|
|
|
[TestMethod]
|
|
public void TestEpsilonGreedy()
|
|
{
|
|
var epsilonGreedyTests = new EpsilonGreedyTestConfiguration[]
|
|
{
|
|
// No exploration
|
|
new EpsilonGreedyTestConfiguration
|
|
{
|
|
AppId = TestContext.TestName + "NoExplorationFixedActionContext",
|
|
ContextType = ContextType.FixedAction, // test fixed-action context
|
|
Epsilon = 0f,
|
|
NumberOfActions = 20,
|
|
ExperimentalUnitIdList = Enumerable.Range(1, 100).Select(i => i.ToString()).ToList(),
|
|
PolicyConfiguration = new FixedPolicyConfiguration { Action = 10 }
|
|
},
|
|
new EpsilonGreedyTestConfiguration
|
|
{
|
|
AppId = TestContext.TestName + "NoExplorationVariableActionContext",
|
|
ContextType = ContextType.VariableAction, // test variable-action context
|
|
Epsilon = 0f,
|
|
NumberOfActions = 20,
|
|
ExperimentalUnitIdList = Enumerable.Range(1, 100).Select(i => i.ToString()).ToList(),
|
|
PolicyConfiguration = new FixedPolicyConfiguration { Action = 10 }
|
|
},
|
|
|
|
// Regular exploration
|
|
new EpsilonGreedyTestConfiguration
|
|
{
|
|
AppId = TestContext.TestName + "RegularExplorationFixedActionContext",
|
|
ContextType = ContextType.FixedAction,
|
|
Epsilon = 0.2f,
|
|
NumberOfActions = 10,
|
|
ExperimentalUnitIdList = Enumerable.Range(1, 100).Select(i => i.ToString()).ToList(),
|
|
PolicyConfiguration = new FixedPolicyConfiguration { Action = 9 }
|
|
},
|
|
new EpsilonGreedyTestConfiguration
|
|
{
|
|
AppId = TestContext.TestName + "RegularExplorationVariableActionContext",
|
|
ContextType = ContextType.VariableAction,
|
|
Epsilon = 0.2f,
|
|
NumberOfActions = 10,
|
|
ExperimentalUnitIdList = Enumerable.Range(1, 100).Select(i => i.ToString()).ToList(),
|
|
PolicyConfiguration = new FixedPolicyConfiguration { Action = 9 }
|
|
},
|
|
|
|
// Heavy exploration
|
|
new EpsilonGreedyTestConfiguration
|
|
{
|
|
AppId = TestContext.TestName + "HeavyExplorationFixedActionContext",
|
|
ContextType = ContextType.FixedAction,
|
|
Epsilon = 0.9f,
|
|
NumberOfActions = 90, // test many actions
|
|
ExperimentalUnitIdList = Enumerable.Range(1, 100).Select(i => i.ToString()).ToList(),
|
|
PolicyConfiguration = new FixedPolicyConfiguration { Action = 81 }
|
|
},
|
|
new EpsilonGreedyTestConfiguration
|
|
{
|
|
AppId = TestContext.TestName + "HeavyExplorationVariableActionContext",
|
|
ContextType = ContextType.VariableAction,
|
|
Epsilon = 0.9f,
|
|
NumberOfActions = 90,
|
|
ExperimentalUnitIdList = Enumerable.Range(1, 100).Select(i => i.ToString()).ToList(),
|
|
PolicyConfiguration = new FixedPolicyConfiguration { Action = 81 }
|
|
}
|
|
};
|
|
|
|
Run(outputFilePatternExpected, outputFilePatternActual, outputJsonConfigFile, epsilonGreedyTests);
|
|
|
|
for (uint i = 0; i < epsilonGreedyTests.Length; i++)
|
|
{
|
|
CompareExplorationData(FormatPath(outputFilePatternExpected, i), FormatPath(outputFilePatternActual, i));
|
|
}
|
|
}
|
|
|
|
[TestMethod]
|
|
public void TestTauFirst()
|
|
{
|
|
var tauFirstTests = new TauFirstTestConfiguration[]
|
|
{
|
|
// No exploration
|
|
new TauFirstTestConfiguration
|
|
{
|
|
AppId = TestContext.TestName + "NoExplorationFixedActionContext",
|
|
ContextType = ContextType.FixedAction, // test fixed-action context
|
|
Tau = 0,
|
|
NumberOfActions = 20,
|
|
ExperimentalUnitIdList = Enumerable.Range(1, 100).Select(i => i.ToString()).ToList(),
|
|
PolicyConfiguration = new FixedPolicyConfiguration { Action = 10 }
|
|
},
|
|
new TauFirstTestConfiguration
|
|
{
|
|
AppId = TestContext.TestName + "NoExplorationVariableActionContext",
|
|
ContextType = ContextType.VariableAction, // test variable-action context
|
|
Tau = 5,
|
|
NumberOfActions = 20,
|
|
ExperimentalUnitIdList = Enumerable.Range(1, 100).Select(i => i.ToString()).ToList(),
|
|
PolicyConfiguration = new FixedPolicyConfiguration { Action = 10 }
|
|
},
|
|
|
|
// Heavy exploration
|
|
new TauFirstTestConfiguration
|
|
{
|
|
AppId = TestContext.TestName + "HeavyExplorationFixedActionContext",
|
|
ContextType = ContextType.FixedAction,
|
|
Tau = 100,
|
|
NumberOfActions = 10,
|
|
ExperimentalUnitIdList = Enumerable.Range(1, 100).Select(i => i.ToString()).ToList(),
|
|
PolicyConfiguration = new FixedPolicyConfiguration { Action = 9 }
|
|
},
|
|
new TauFirstTestConfiguration
|
|
{
|
|
AppId = TestContext.TestName + "HeavyExplorationVariableActionContext",
|
|
ContextType = ContextType.VariableAction,
|
|
Tau = 100,
|
|
NumberOfActions = 10,
|
|
ExperimentalUnitIdList = Enumerable.Range(1, 100).Select(i => i.ToString()).ToList(),
|
|
PolicyConfiguration = new FixedPolicyConfiguration { Action = 9 }
|
|
}
|
|
};
|
|
|
|
Run(outputFilePatternExpected, outputFilePatternActual, outputJsonConfigFile, tauFirstTests);
|
|
|
|
for (uint i = 0; i < tauFirstTests.Length; i++)
|
|
{
|
|
CompareExplorationData(FormatPath(outputFilePatternExpected, i), FormatPath(outputFilePatternActual, i));
|
|
}
|
|
}
|
|
|
|
[TestMethod]
|
|
public void TestSoftmax()
|
|
{
|
|
var softmaxTests = new SoftmaxTestConfiguration[]
|
|
{
|
|
new SoftmaxTestConfiguration
|
|
{
|
|
AppId = TestContext.TestName + "LowLambdaFixedActionContext",
|
|
ContextType = ContextType.FixedAction, // test fixed-action context
|
|
Lambda = 0.1f,
|
|
NumberOfActions = 20,
|
|
ExperimentalUnitIdList = Enumerable.Range(1, 100).Select(i => i.ToString()).ToList(),
|
|
ScorerConfiguration = new FixedScorerConfiguration { Score = 1 }
|
|
},
|
|
new SoftmaxTestConfiguration
|
|
{
|
|
AppId = TestContext.TestName + "LowLambdaVariableActionContext",
|
|
ContextType = ContextType.VariableAction, // test variable-action context
|
|
Lambda = 0.1f,
|
|
NumberOfActions = 20,
|
|
ExperimentalUnitIdList = Enumerable.Range(1, 100).Select(i => i.ToString()).ToList(),
|
|
ScorerConfiguration = new FixedScorerConfiguration { Score = 5 }
|
|
},
|
|
|
|
new SoftmaxTestConfiguration
|
|
{
|
|
AppId = TestContext.TestName + "HighLambdaFixedActionContext",
|
|
ContextType = ContextType.FixedAction,
|
|
Lambda = 0.9f,
|
|
NumberOfActions = 10,
|
|
ExperimentalUnitIdList = Enumerable.Range(1, 100).Select(i => i.ToString()).ToList(),
|
|
ScorerConfiguration = new IntegerProgressionScorerConfiguration { Start = 1 }
|
|
},
|
|
new SoftmaxTestConfiguration
|
|
{
|
|
AppId = TestContext.TestName + "HighLambdaVariableActionContext",
|
|
ContextType = ContextType.VariableAction,
|
|
Lambda = 0.9f,
|
|
NumberOfActions = 10,
|
|
ExperimentalUnitIdList = Enumerable.Range(1, 100).Select(i => i.ToString()).ToList(),
|
|
ScorerConfiguration = new IntegerProgressionScorerConfiguration { Start = 5 }
|
|
}
|
|
};
|
|
|
|
Run(outputFilePatternExpected, outputFilePatternActual, outputJsonConfigFile, softmaxTests);
|
|
|
|
for (uint i = 0; i < softmaxTests.Length; i++)
|
|
{
|
|
CompareExplorationData(FormatPath(outputFilePatternExpected, i), FormatPath(outputFilePatternActual, i));
|
|
}
|
|
}
|
|
|
|
[TestMethod]
|
|
public void TestGeneric()
|
|
{
|
|
var genericTests = new GenericTestConfiguration[]
|
|
{
|
|
new GenericTestConfiguration
|
|
{
|
|
AppId = TestContext.TestName + "FixedScorerFixedActionContext",
|
|
ContextType = ContextType.FixedAction, // test fixed-action context
|
|
NumberOfActions = 20,
|
|
ExperimentalUnitIdList = Enumerable.Range(1, 100).Select(i => i.ToString()).ToList(),
|
|
ScorerConfiguration = new FixedScorerConfiguration { Score = 1 }
|
|
},
|
|
new GenericTestConfiguration
|
|
{
|
|
AppId = TestContext.TestName + "FixedScorerVariableActionContext",
|
|
ContextType = ContextType.VariableAction, // test variable-action context
|
|
NumberOfActions = 20,
|
|
ExperimentalUnitIdList = Enumerable.Range(1, 100).Select(i => i.ToString()).ToList(),
|
|
ScorerConfiguration = new FixedScorerConfiguration { Score = 5 }
|
|
},
|
|
new GenericTestConfiguration
|
|
{
|
|
AppId = TestContext.TestName + "ProgressScorerFixedActionContext",
|
|
ContextType = ContextType.FixedAction,
|
|
NumberOfActions = 10,
|
|
ExperimentalUnitIdList = Enumerable.Range(1, 100).Select(i => i.ToString()).ToList(),
|
|
ScorerConfiguration = new IntegerProgressionScorerConfiguration { Start = 1 }
|
|
},
|
|
new GenericTestConfiguration
|
|
{
|
|
AppId = TestContext.TestName + "ProgressScorerVariableActionContext",
|
|
ContextType = ContextType.VariableAction,
|
|
NumberOfActions = 10,
|
|
ExperimentalUnitIdList = Enumerable.Range(1, 100).Select(i => i.ToString()).ToList(),
|
|
ScorerConfiguration = new IntegerProgressionScorerConfiguration { Start = 5 }
|
|
}
|
|
};
|
|
|
|
Run(outputFilePatternExpected, outputFilePatternActual, outputJsonConfigFile, genericTests);
|
|
|
|
for (uint i = 0; i < genericTests.Length; i++)
|
|
{
|
|
CompareExplorationData(FormatPath(outputFilePatternExpected, i), FormatPath(outputFilePatternActual, i));
|
|
}
|
|
}
|
|
|
|
// TODO: Bootstrap is not fully supported at the moment
|
|
//[TestMethod]
|
|
//public void TestBootstrap()
|
|
//{
|
|
// var bootstrapTests = new BootstrapTestConfiguration[]
|
|
// {
|
|
// new BootstrapTestConfiguration
|
|
// {
|
|
// AppId = TestContext.TestName + "NoPolicyAgreeListFixedActionContext",
|
|
// ContextType = ContextType.FixedAction, // test fixed-action context
|
|
// NumberOfActions = 20,
|
|
// ExperimentalUnitIdList = Enumerable.Range(1, 100).Select(i => i.ToString()).ToList(),
|
|
// PolicyConfigurations = Enumerable.Range(1, 10).Select(i =>
|
|
// new FixedPolicyConfiguration { Action = (uint)i }).ToArray()
|
|
// },
|
|
// new BootstrapTestConfiguration
|
|
// {
|
|
// AppId = TestContext.TestName + "NoPolicyAgreeListVariableActionContext",
|
|
// ContextType = ContextType.VariableAction, // test variable-action context
|
|
// NumberOfActions = 20,
|
|
// ExperimentalUnitIdList = Enumerable.Range(1, 100).Select(i => i.ToString()).ToList(),
|
|
// PolicyConfigurations = Enumerable.Range(1, 10).Select(i =>
|
|
// new FixedPolicyConfiguration { Action = (uint)i }).ToArray()
|
|
// },
|
|
// new BootstrapTestConfiguration
|
|
// {
|
|
// AppId = TestContext.TestName + "SomePolicyAgreeListVariableActionContext",
|
|
// ContextType = ContextType.VariableAction, // test variable-action context
|
|
// NumberOfActions = 20,
|
|
// ExperimentalUnitIdList = Enumerable.Range(1, 100).Select(i => i.ToString()).ToList(),
|
|
// PolicyConfigurations = Enumerable.Range(1, 10).Select(i =>
|
|
// new FixedPolicyConfiguration { Action = (uint) (i % 7) + 1 }).ToArray()
|
|
// }
|
|
// };
|
|
|
|
// Run(outputFilePatternExpected, outputFilePatternActual, outputJsonConfigFile, bootstrapTests);
|
|
|
|
// for (uint i = 0; i < bootstrapTests.Length; i++)
|
|
// {
|
|
// CompareExplorationData(FormatPath(outputFilePatternExpected, i), FormatPath(outputFilePatternActual, i));
|
|
// }
|
|
//}
|
|
|
|
[TestInitialize]
|
|
public void Initialize()
|
|
{
|
|
if (Directory.Exists(WorkingDir))
|
|
{
|
|
Directory.Delete(WorkingDir, recursive: true);
|
|
}
|
|
Directory.CreateDirectory(WorkingDir);
|
|
|
|
// {0} is test config id since multiple configs can run within one test method.
|
|
outputFilePatternExpected = TestContext.TestName + "_result_{0}_expected.txt";
|
|
outputFilePatternActual = TestContext.TestName + "_result_{0}_actual.txt";
|
|
outputJsonConfigFile = TestContext.TestName + ".json";
|
|
}
|
|
|
|
[TestCleanup]
|
|
public void CleanUp()
|
|
{
|
|
if (Directory.Exists(WorkingDir))
|
|
{
|
|
Directory.Delete(WorkingDir, recursive: true);
|
|
}
|
|
}
|
|
|
|
private void CompareExactMatch(string file1, string file2)
|
|
{
|
|
string[] lines1 = File.ReadAllLines(file1).Select(l => l.Trim()).Where(l => !String.IsNullOrEmpty(l)).ToArray();
|
|
string[] lines2 = File.ReadAllLines(file2).Select(l => l.Trim()).Where(l => !String.IsNullOrEmpty(l)).ToArray();
|
|
|
|
Assert.IsTrue(Enumerable.SequenceEqual(lines1, lines2));
|
|
}
|
|
|
|
private void CompareExplorationData(string file1, string file2)
|
|
{
|
|
var extractExploration = (Func<string, Tuple<uint, string, float, string>>)((line) =>
|
|
{
|
|
string[] barData = line.Split(new char[] { '|' }, StringSplitOptions.RemoveEmptyEntries);
|
|
string[] spaceData = barData[0].Split(new char[] { ' ' }, StringSplitOptions.RemoveEmptyEntries);
|
|
return new Tuple<uint, string, float, string>(
|
|
Convert.ToUInt32(spaceData[0]), // action
|
|
spaceData[1].Trim(), // key
|
|
Convert.ToSingle(spaceData[2]), // prob
|
|
barData[1].Trim()); // context
|
|
});
|
|
|
|
string[] lines1 = File.ReadAllLines(file1).Select(l => l.Trim()).Where(l => !String.IsNullOrEmpty(l)).ToArray();
|
|
string[] lines2 = File.ReadAllLines(file2).Select(l => l.Trim()).Where(l => !String.IsNullOrEmpty(l)).ToArray();
|
|
|
|
Assert.AreEqual(lines1.Length, lines2.Length);
|
|
|
|
for (int i = 0; i < lines1.Length; i++)
|
|
{
|
|
Tuple<uint, string, float, string> data1 = extractExploration(lines1[i]);
|
|
Tuple<uint, string, float, string> data2 = extractExploration(lines2[i]);
|
|
|
|
Assert.AreEqual(data1.Item1, data2.Item1);
|
|
Assert.AreEqual(data1.Item2, data2.Item2);
|
|
Assert.IsTrue(Math.Abs(data1.Item3 - data2.Item3) <= PrecisionOffset);
|
|
Assert.AreEqual(data1.Item4, data2.Item4);
|
|
}
|
|
}
|
|
|
|
private void Run(string outputFilePatternExpected, string outputFilePatternActual, string outputJsonConfigFile, ITestConfiguration[] tests)
|
|
{
|
|
for (uint i = 0; i < tests.Length; i++)
|
|
{
|
|
tests[i].OutputFile = FormatPath(outputFilePatternExpected, i);
|
|
}
|
|
|
|
string jsonConfigFile = Path.Combine(WorkingDir, outputJsonConfigFile);
|
|
File.WriteAllText(jsonConfigFile, JsonConvert.SerializeObject(tests));
|
|
|
|
var psi = new ProcessStartInfo();
|
|
psi.FileName = CppExePath;
|
|
psi.Arguments = jsonConfigFile;
|
|
psi.CreateNoWindow = true;
|
|
psi.WindowStyle = ProcessWindowStyle.Hidden;
|
|
|
|
Process.Start(psi).WaitForExit();
|
|
|
|
for (uint i = 0; i < tests.Length; i++)
|
|
{
|
|
tests[i].OutputFile = FormatPath(outputFilePatternActual, i);
|
|
}
|
|
File.WriteAllText(jsonConfigFile, JsonConvert.SerializeObject(tests));
|
|
|
|
psi.FileName = CsharpExePath;
|
|
Process.Start(psi).WaitForExit();
|
|
}
|
|
|
|
private string FormatPath(string pattern, uint iteration)
|
|
{
|
|
return Path.Combine(WorkingDir, string.Format(pattern, iteration));
|
|
}
|
|
|
|
/// <summary>
|
|
/// Gets or sets the test context which provides
|
|
/// information about and functionality for the current test run.
|
|
/// </summary>
|
|
public TestContext TestContext { get; set; }
|
|
|
|
private string outputFilePatternExpected;
|
|
private string outputFilePatternActual;
|
|
private string outputJsonConfigFile;
|
|
|
|
private readonly string CppExePath = @"..\..\..\explore-cpp\bin\x64\Release\black_box_tests.exe";
|
|
private readonly string CsharpExePath = @"..\..\..\explore-csharp\bin\AnyCPU\Release\BlackBoxTests.exe";
|
|
private readonly string WorkingDir = Path.Combine(Directory.GetCurrentDirectory(), "Test");
|
|
private readonly float PrecisionOffset = 1e-4f;
|
|
}
|
|
}
|