Merged PR 5170: removed ML.NET files as they are now in ML.NET itself

removed ML.NET files as they are now in ML.NET itself
This commit is contained in:
Michael Sharp 2019-09-26 22:29:51 +00:00
Родитель f1a713d9df
Коммит e4649ef6a2
2 изменённых файлов: 0 добавлений и 482 удалений

Просмотреть файл

@ -1,351 +0,0 @@
// --------------------------------------------------------------------------------
// |
// | WARNING:
// | This file was generated; any local changes will be overwritten during
// | future invocations of the generator!
// |
// | Generated by: Plugins/MlNetPluginImpl/CreateCsFile.py
// | Generated on: 2019-07-24 13:29:27.151329
// |
// --------------------------------------------------------------------------------
using Microsoft.ML.Data;
using Microsoft.ML.Runtime;
using Microsoft.Win32.SafeHandles;
using System;
using System.Linq;
using System.Runtime.InteropServices;
using System.Security;
namespace Microsoft.ML.Autogen
{
public static class DateTimeTransformerExtensionClass
{
public class TransformParameter<T>
{
private readonly T _rawValue;
public readonly DataViewSchema.Column? Column;
public T GetValue(DataViewRow row)
{
if (Column.HasValue)
{
var column = row.Schema[Column.Value.Name];
var getter = row.GetGetter<T>(column);
T value = default;
getter(ref value);
return value;
}
else
{
return _rawValue;
}
}
public TransformParameter(T value)
{
_rawValue = value;
Column = null;
}
public TransformParameter(DataViewSchema.Column column)
{
_rawValue = default;
Column = column;
}
}
public static DateTimeTransformerEstimator DateTimeTransformer(this TransformsCatalog catalog, DateTime dateTime, string columnPrefix)
=> DateTimeTransformerEstimator.Create(CatalogUtils.GetEnvironment(catalog), dateTime, columnPrefix);
public static DateTimeTransformerEstimator DateTimeTransformer(this TransformsCatalog catalog, DataViewSchema.Column dateTime, string columnPrefix)
=> DateTimeTransformerEstimator.Create(CatalogUtils.GetEnvironment(catalog), dateTime, columnPrefix);
}
public sealed class DateTimeTransformerEstimator : IEstimator<DateTimeTransformer>
{
private DateTimeTransformerExtensionClass.TransformParameter<DateTime> _dateTime = default;
private string _columnPrefix = default;
private readonly IHost _host;
private static Type GetParameterType(object obj)
{
if (obj.GetType() == typeof(DataViewSchema.Column))
{
return ((DataViewSchema.Column)obj).Type.RawType;
}
else
{
return obj.GetType();
}
}
private static Type GetParameterClassType(object obj)
{
var type = GetParameterType(obj);
return typeof(DateTimeTransformerExtensionClass.TransformParameter<>).MakeGenericType(type);
}
public static DateTimeTransformerEstimator Create(IHostEnvironment env, object dateTime, string columnPrefix)
{
var dateTimeParam = new object[] { dateTime };
var dateTimeType = GetParameterClassType(dateTime);
var dateTimeInstance = Activator.CreateInstance(dateTimeType, dateTimeParam);
var param = new object[] { env, dateTimeInstance, columnPrefix };
var estimator = Activator.CreateInstance(typeof(DateTimeTransformerEstimator), param);
return (DateTimeTransformerEstimator)estimator;
}
public DateTimeTransformerEstimator(IHostEnvironment env, DateTimeTransformerExtensionClass.TransformParameter<DateTime> dateTime, string columnPrefix)
{
_dateTime = dateTime;
_columnPrefix = columnPrefix;
_host = env.Register(nameof(DateTimeTransformerEstimator));
}
public DateTimeTransformer Fit(IDataView input)
{
return new DateTimeTransformer(_host, _dateTime, _columnPrefix);
}
public SchemaShape GetOutputSchema(SchemaShape inputSchema)
{
var columns = inputSchema.ToDictionary(x => x.Name);
// this code returns each member of the struct as its own column
columns[_columnPrefix + "_year"] = new SchemaShape.Column(_columnPrefix + "_year", SchemaShape.Column.VectorKind.Scalar,
ColumnTypeExtensions.PrimitiveTypeFromType(typeof(int)), false, null);
columns[_columnPrefix + "_month"] = new SchemaShape.Column(_columnPrefix + "_month", SchemaShape.Column.VectorKind.Scalar,
ColumnTypeExtensions.PrimitiveTypeFromType(typeof(byte)), false, null);
columns[_columnPrefix + "_day"] = new SchemaShape.Column(_columnPrefix + "_day", SchemaShape.Column.VectorKind.Scalar,
ColumnTypeExtensions.PrimitiveTypeFromType(typeof(byte)), false, null);
columns[_columnPrefix + "_hour"] = new SchemaShape.Column(_columnPrefix + "_hour", SchemaShape.Column.VectorKind.Scalar,
ColumnTypeExtensions.PrimitiveTypeFromType(typeof(byte)), false, null);
columns[_columnPrefix + "_minute"] = new SchemaShape.Column(_columnPrefix + "_minute", SchemaShape.Column.VectorKind.Scalar,
ColumnTypeExtensions.PrimitiveTypeFromType(typeof(byte)), false, null);
columns[_columnPrefix + "_second"] = new SchemaShape.Column(_columnPrefix + "_second", SchemaShape.Column.VectorKind.Scalar,
ColumnTypeExtensions.PrimitiveTypeFromType(typeof(byte)), false, null);
columns[_columnPrefix + "_dayOfWeek"] = new SchemaShape.Column(_columnPrefix + "_dayOfWeek", SchemaShape.Column.VectorKind.Scalar,
ColumnTypeExtensions.PrimitiveTypeFromType(typeof(byte)), false, null);
columns[_columnPrefix + "_dayOfYear"] = new SchemaShape.Column(_columnPrefix + "_dayOfYear", SchemaShape.Column.VectorKind.Scalar,
ColumnTypeExtensions.PrimitiveTypeFromType(typeof(ushort)), false, null);
columns[_columnPrefix + "_quarterOfYear"] = new SchemaShape.Column(_columnPrefix + "_quarterOfYear", SchemaShape.Column.VectorKind.Scalar,
ColumnTypeExtensions.PrimitiveTypeFromType(typeof(byte)), false, null);
columns[_columnPrefix + "_weekOfMonth"] = new SchemaShape.Column(_columnPrefix + "_weekOfMonth", SchemaShape.Column.VectorKind.Scalar,
ColumnTypeExtensions.PrimitiveTypeFromType(typeof(byte)), false, null);
return new SchemaShape(columns.Values);
}
}
public class DateTimeTransformer : RowToRowTransformerBase
{
private DateTimeTransformerExtensionClass.TransformParameter<DateTime> _dateTime = default;
private string _columnPrefix = default;
public DateTimeTransformer(IHost host, DateTimeTransformerExtensionClass.TransformParameter<DateTime> dateTime, string columnPrefix) :
base(host.Register(nameof(DateTimeTransformer)))
{
_dateTime = dateTime;
_columnPrefix = columnPrefix;
}
protected sealed class Mapper : MapperBase, IDisposable
{
private readonly DateTimeTransformer _parent;
private TransformerSafeHandler _cppTransformer;
// Unix time starts in the year 1970, so this is to allow for easier time conversions between datetime and unix time.
private readonly DateTime _cppStartDate;
private long _position;
private TimePoint _currentData;
[DllImport("DateTimeTransformer.dll", EntryPoint = "CreateDateTimeTransformProxy"), SuppressUnmanagedCodeSecurity]
extern static bool CreateDateTimeTransformProxy(long dateTime, TransformerSafeHandler transformer, out IntPtr timePoint);
[DllImport("DateTimeTransformer.dll", EntryPoint = "DeleteDateTimeTransformProxy"), SuppressUnmanagedCodeSecurity]
extern static bool DeleteDateTimeTransformProxy(IntPtr timePoint);
[DllImport("DateTimeTransformer.dll", EntryPoint = "CreateDateTimeTransformer"), SuppressUnmanagedCodeSecurity]
extern static IntPtr CreateCppTransformer();
[DllImport("DateTimeTransformer.dll", EntryPoint = "DeleteDateTimeTransformer"), SuppressUnmanagedCodeSecurity]
extern static bool DeleteCppTransformer(IntPtr transformer);
public Mapper(DateTimeTransformer parent, DataViewSchema inputSchema) :
base(parent.Host.Register(nameof(Mapper)), inputSchema, parent)
{
_parent = parent;
_cppTransformer = new TransformerSafeHandler(CreateCppTransformer());
_cppStartDate = new DateTime(1970, 1, 1);
_position = -1;
}
[StructLayoutAttribute(LayoutKind.Sequential)]
private struct TimePoint
{
public int year;
public byte month;
public byte day;
public byte hour;
public byte minute;
public byte second;
public byte dayOfWeek;
public ushort dayOfYear;
public byte quarterOfYear;
public byte weekOfMonth;
};
internal class TransformerSafeHandler : SafeHandleZeroOrMinusOneIsInvalid
{
public TransformerSafeHandler(IntPtr handle) : base(true)
{
SetHandle(handle);
}
protected override bool ReleaseHandle()
{
DeleteCppTransformer(handle);
return true;
}
}
internal class TimePointSafehandler : SafeHandleZeroOrMinusOneIsInvalid
{
public TimePointSafehandler(IntPtr handle) : base(true)
{
SetHandle(handle);
}
protected override bool ReleaseHandle()
{
DeleteDateTimeTransformProxy(handle);
return true;
}
}
public void Dispose()
{
_cppTransformer.Dispose();
}
protected override DataViewSchema.DetachedColumn[] GetOutputColumnsCore()
{
var year = new DataViewSchema.DetachedColumn(_parent._columnPrefix + "_year",
ColumnTypeExtensions.PrimitiveTypeFromType(typeof(int)));
var month = new DataViewSchema.DetachedColumn(_parent._columnPrefix + "_month",
ColumnTypeExtensions.PrimitiveTypeFromType(typeof(byte)));
var day = new DataViewSchema.DetachedColumn(_parent._columnPrefix + "_day",
ColumnTypeExtensions.PrimitiveTypeFromType(typeof(byte)));
var hour = new DataViewSchema.DetachedColumn(_parent._columnPrefix + "_hour",
ColumnTypeExtensions.PrimitiveTypeFromType(typeof(byte)));
var minute = new DataViewSchema.DetachedColumn(_parent._columnPrefix + "_minute",
ColumnTypeExtensions.PrimitiveTypeFromType(typeof(byte)));
var second = new DataViewSchema.DetachedColumn(_parent._columnPrefix + "_second",
ColumnTypeExtensions.PrimitiveTypeFromType(typeof(byte)));
var dayOfWeek = new DataViewSchema.DetachedColumn(_parent._columnPrefix + "_dayOfWeek",
ColumnTypeExtensions.PrimitiveTypeFromType(typeof(byte)));
var dayOfYear = new DataViewSchema.DetachedColumn(_parent._columnPrefix + "_dayOfYear",
ColumnTypeExtensions.PrimitiveTypeFromType(typeof(ushort)));
var quarterOfYear = new DataViewSchema.DetachedColumn(_parent._columnPrefix + "_quarterOfYear",
ColumnTypeExtensions.PrimitiveTypeFromType(typeof(byte)));
var weekOfMonth = new DataViewSchema.DetachedColumn(_parent._columnPrefix + "_weekOfMonth",
ColumnTypeExtensions.PrimitiveTypeFromType(typeof(byte)));
return new DataViewSchema.DetachedColumn[] { year, month, day, hour, minute, second, dayOfWeek, dayOfYear, quarterOfYear, weekOfMonth };
}
private Delegate MakeGetter<T>(DataViewRow input, int iinfo)
{
ValueGetter<T> result = (ref T dst) =>
{
if (_position != input.Position)
{
var dateTime = _parent._dateTime.GetValue(input);
var success = CreateDateTimeTransformProxy(dateTime.Ticks - _cppStartDate.Ticks, _cppTransformer, out IntPtr timePoint);
using (var timePointSafeHandler = new TimePointSafehandler(timePoint))
{
_currentData = Marshal.PtrToStructure<TimePoint>(timePoint);
}
_position = input.Position;
}
if (iinfo == 0)
dst = (T)Convert.ChangeType(_currentData.year, typeof(T));
else if (iinfo == 1)
dst = (T)Convert.ChangeType(_currentData.month, typeof(T));
else if (iinfo == 2)
dst = (T)Convert.ChangeType(_currentData.day, typeof(T));
else if (iinfo == 3)
dst = (T)Convert.ChangeType(_currentData.hour, typeof(T));
else if (iinfo == 4)
dst = (T)Convert.ChangeType(_currentData.minute, typeof(T));
else if (iinfo == 5)
dst = (T)Convert.ChangeType(_currentData.second, typeof(T));
else if (iinfo == 6)
dst = (T)Convert.ChangeType(_currentData.dayOfWeek, typeof(T));
else if (iinfo == 7)
dst = (T)Convert.ChangeType(_currentData.dayOfYear, typeof(T));
else if (iinfo == 8)
dst = (T)Convert.ChangeType(_currentData.quarterOfYear, typeof(T));
else
dst = (T)Convert.ChangeType(_currentData.weekOfMonth, typeof(T));
};
return result;
}
protected override Delegate MakeGetter(DataViewRow input, int iinfo, Func<int, bool> activeOutput, out Action disposer)
{
disposer = null;
if (iinfo == 0) // Year
return MakeGetter<int>(input, iinfo);
else if (iinfo == 7)
return MakeGetter<ushort>(input, iinfo);
else
return MakeGetter<byte>(input, iinfo);
}
protected override Func<int, bool> GetDependenciesCore(Func<int, bool> activeOutput)
{
var active = new bool[InputSchema.Count];
for (int i = 0; i < InputSchema.Count; i++)
{
if (_parent._dateTime.Column.HasValue && InputSchema[i].Name.Equals(_parent._dateTime.Column.Value.Name))
{
active[i] = true;
}
}
return col => active[col];
}
protected override void SaveModel(ModelSaveContext ctx)
{
throw new NotImplementedException();
}
}
protected override IRowMapper MakeRowMapper(DataViewSchema schema) => new Mapper(this, schema);
protected override void SaveModel(ModelSaveContext ctx)
{
throw new NotImplementedException();
}
}
}

Просмотреть файл

@ -1,131 +0,0 @@
using Microsoft.ML.Data;
using System;
using System.IO;
using Xunit;
namespace Microsoft.ML.Autogen
{
public class DateTimeFeaturizerTests
{
[Fact]
public void CorrectNumberOfColumns()
{
MLContext mlContext = new MLContext(1);
var path = Path.Combine(Environment.CurrentDirectory, "TestData", "dates.txt");
var loader = mlContext.Data.CreateTextLoader(
new[] {
new TextLoader.Column("date", DataKind.DateTime, 0)
}
);
IDataView data = loader.Load(path);
// Build the pipeline, fit, and transform it.
var dateColumn = data.Schema["date"];
var columnPrefix = "DTC";
var pipeline = mlContext.Transforms.DateTimeTransformer(dateColumn, columnPrefix);
var model = pipeline.Fit(data);
var output = model.Transform(data);
var schema = output.Schema;
// Check the schema has 11 columns
Assert.Equal(11, schema.Count);
// Make sure names with prefix and order are correct
Assert.Equal($"{columnPrefix}_year", schema[1].Name);
Assert.Equal($"{columnPrefix}_month", schema[2].Name);
Assert.Equal($"{columnPrefix}_day", schema[3].Name);
Assert.Equal($"{columnPrefix}_hour", schema[4].Name);
Assert.Equal($"{columnPrefix}_minute", schema[5].Name);
Assert.Equal($"{columnPrefix}_second", schema[6].Name);
Assert.Equal($"{columnPrefix}_dayOfWeek", schema[7].Name);
Assert.Equal($"{columnPrefix}_dayOfYear", schema[8].Name);
Assert.Equal($"{columnPrefix}_quarterOfYear", schema[9].Name);
Assert.Equal($"{columnPrefix}_weekOfMonth", schema[10].Name);
// Make sure types are correct
Assert.Equal(typeof(int), schema[1].Type.RawType);
Assert.Equal(typeof(byte), schema[2].Type.RawType);
Assert.Equal(typeof(byte), schema[3].Type.RawType);
Assert.Equal(typeof(byte), schema[4].Type.RawType);
Assert.Equal(typeof(byte), schema[5].Type.RawType);
Assert.Equal(typeof(byte), schema[6].Type.RawType);
Assert.Equal(typeof(byte), schema[7].Type.RawType);
Assert.Equal(typeof(ushort), schema[8].Type.RawType);
Assert.Equal(typeof(byte), schema[9].Type.RawType);
Assert.Equal(typeof(byte), schema[10].Type.RawType);
}
[Fact]
public void CanUseDateFromColumn()
{
MLContext mlContext = new MLContext(1);
var path = Path.Combine(Environment.CurrentDirectory, "TestData", "dates.txt");
var loader = mlContext.Data.CreateTextLoader(
new[] {
new TextLoader.Column("date", DataKind.DateTime, 0)
}
);
IDataView data = loader.Load(path);
// Build the pipeline, fit, and transform it.
var dateColumn = data.Schema["date"];
var pipeline = mlContext.Transforms.DateTimeTransformer(dateColumn, "DTC");
var model = pipeline.Fit(data);
var output = model.Transform(data);
// Get the data from the first row and make sure it matches expected
var row = output.Preview(1).RowView[0].Values;
// Assert the data from the first row is what we expect
Assert.Equal(2019, row[1].Value);
Assert.Equal((byte)6, row[2].Value);
Assert.Equal((byte)29, row[3].Value);
Assert.Equal((byte)0, row[4].Value);
Assert.Equal((byte)1, row[5].Value);
Assert.Equal((byte)0, row[6].Value);
Assert.Equal((byte)6, row[7].Value);
Assert.Equal((ushort)179, row[8].Value);
Assert.Equal((byte)2, row[9].Value);
Assert.Equal((byte)4, row[10].Value);
}
[Fact]
public void CanUseRawDate()
{
MLContext mlContext = new MLContext(1);
var path = Path.Combine(Environment.CurrentDirectory, "TestData", "dates.txt");
var loader = mlContext.Data.CreateTextLoader(
new[] {
new TextLoader.Column("date", DataKind.DateTime, 0)
}
);
IDataView data = loader.Load(path);
// Build the pipeline, fit, and transform it.
var date = DateTime.Parse("2019-06-29 12:01:00 AM");
var pipeline = mlContext.Transforms.DateTimeTransformer(date, "DTC");
var model = pipeline.Fit(data);
var output = model.Transform(data);
// Get the data from the first row and make sure it matches expected
var row = output.Preview(1).RowView[0].Values;
// Assert the data from the first row is what we expect
Assert.Equal(2019, row[1].Value);
Assert.Equal((byte)6, row[2].Value);
Assert.Equal((byte)29, row[3].Value);
Assert.Equal((byte)0, row[4].Value);
Assert.Equal((byte)1, row[5].Value);
Assert.Equal((byte)0, row[6].Value);
Assert.Equal((byte)6, row[7].Value);
Assert.Equal((ushort)179, row[8].Value);
Assert.Equal((byte)2, row[9].Value);
Assert.Equal((byte)4, row[10].Value);
}
}
}