dotnet format/spellchecking (#5988)
This commit is contained in:
Родитель
cc5e6395e0
Коммит
0577957256
|
@ -306,6 +306,10 @@ dotnet_diagnostic.IDE2004.severity = warning
|
|||
dotnet_diagnostic.CA1822.severity = suggestion
|
||||
|
||||
[test/**/*.cs]
|
||||
# IDE0044: Don't force readonly for tests
|
||||
dotnet_diagnostic.IDE0044.severity = none
|
||||
dotnet_style_readonly_field = false
|
||||
|
||||
# MSML_GeneralName: This name should be PascalCased
|
||||
dotnet_diagnostic.MSML_GeneralName.severity = none
|
||||
|
||||
|
@ -322,4 +326,18 @@ dotnet_diagnostic.MSML_ExtendBaseTestClass.severity = none
|
|||
|
||||
# The MSML_RelaxTestNaming suppressor for VSTHRD200 is not active for CodeAnalyzer.Tests, so we disable it altogether.
|
||||
# VSTHRD200: Use "Async" suffix for async methods
|
||||
dotnet_diagnostic.VSTHRD200.severity = none
|
||||
dotnet_diagnostic.VSTHRD200.severity = none
|
||||
|
||||
[docs/**/*.cs]
|
||||
# IDE0073: Dont want license file header in samples
|
||||
dotnet_diagnostic.IDE0073.severity = none
|
||||
file_header_template = unset
|
||||
|
||||
# IDE0044: Don't force readonly for samples
|
||||
dotnet_diagnostic.IDE0044.severity = none
|
||||
dotnet_style_readonly_field = false
|
||||
|
||||
[test/Microsoft.ML.TestFrameworkCommon/Utility/*.cs]
|
||||
# IDE0073: Dont want license file header in code we are using from elsewhere
|
||||
dotnet_diagnostic.IDE0073.severity = none
|
||||
file_header_template = unset
|
|
@ -17,6 +17,7 @@
|
|||
<DebugType>portable</DebugType>
|
||||
<DebugSymbols>true</DebugSymbols>
|
||||
<LangVersion>latest</LangVersion>
|
||||
<EnforceCodeStyleInBuild>true</EnforceCodeStyleInBuild>
|
||||
</PropertyGroup>
|
||||
|
||||
<PropertyGroup>
|
||||
|
|
|
@ -1,10 +1,10 @@
|
|||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Text;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using Microsoft.ML.Data;
|
||||
using System.Text;
|
||||
using Microsoft.ML.AutoML.Samples.DataStructures;
|
||||
using Microsoft.ML.Data;
|
||||
|
||||
namespace Microsoft.ML.AutoML.Samples
|
||||
{
|
||||
|
|
|
@ -2,15 +2,15 @@
|
|||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.IO.Compression;
|
||||
using System.Linq;
|
||||
using System.Net;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.ML;
|
||||
using static Microsoft.ML.DataOperationsCatalog;
|
||||
using System.Linq;
|
||||
using Microsoft.ML.Data;
|
||||
using System.IO.Compression;
|
||||
using System.Threading;
|
||||
using System.Net;
|
||||
using Microsoft.ML.Vision;
|
||||
using static Microsoft.ML.DataOperationsCatalog;
|
||||
|
||||
namespace Samples.Dynamic
|
||||
{
|
||||
|
|
|
@ -18,7 +18,7 @@ namespace Samples.Dynamic
|
|||
// AlienBody members to produce a SuperAlienHero entity with a "Name" member of type
|
||||
// string and a single "Merged" member of type AlienBody, where the merger is just
|
||||
// the addition of the various members of AlienBody.
|
||||
static public void Example()
|
||||
public static void Example()
|
||||
{
|
||||
var mlContext = new MLContext();
|
||||
// Build in-memory data.
|
||||
|
|
|
@ -7,7 +7,7 @@ namespace Samples.Dynamic
|
|||
{
|
||||
class ConvertToGrayScaleInMemory
|
||||
{
|
||||
static public void Example()
|
||||
public static void Example()
|
||||
{
|
||||
var mlContext = new MLContext();
|
||||
// Create an image list.
|
||||
|
|
|
@ -1,8 +1,8 @@
|
|||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using Microsoft.ML;
|
||||
using Microsoft.ML.Transforms.TimeSeries;
|
||||
using System.IO;
|
||||
|
||||
namespace Samples.Dynamic
|
||||
{
|
||||
|
|
|
@ -20,9 +20,9 @@ namespace Microsoft.Data.Analysis
|
|||
/// </summary>
|
||||
public partial class ArrowStringDataFrameColumn : DataFrameColumn, IEnumerable<string>
|
||||
{
|
||||
private IList<ReadOnlyDataFrameBuffer<byte>> _dataBuffers;
|
||||
private IList<ReadOnlyDataFrameBuffer<int>> _offsetsBuffers;
|
||||
private IList<ReadOnlyDataFrameBuffer<byte>> _nullBitMapBuffers;
|
||||
private readonly IList<ReadOnlyDataFrameBuffer<byte>> _dataBuffers;
|
||||
private readonly IList<ReadOnlyDataFrameBuffer<int>> _offsetsBuffers;
|
||||
private readonly IList<ReadOnlyDataFrameBuffer<byte>> _nullBitMapBuffers;
|
||||
|
||||
/// <summary>
|
||||
/// Constructs an empty <see cref="ArrowStringDataFrameColumn"/> with the given <paramref name="name"/>.
|
||||
|
@ -40,7 +40,7 @@ namespace Microsoft.Data.Analysis
|
|||
/// </summary>
|
||||
/// <param name="name">The name of the column.</param>
|
||||
/// <param name="values">The Arrow formatted string values in this column.</param>
|
||||
/// <param name="offsets">The Arrow formatted offets in this column.</param>
|
||||
/// <param name="offsets">The Arrow formatted offsets in this column.</param>
|
||||
/// <param name="nullBits">The Arrow formatted null bits in this column.</param>
|
||||
/// <param name="length">The length of the column.</param>
|
||||
/// <param name="nullCount">The number of <see langword="null" /> values in this column.</param>
|
||||
|
|
|
@ -293,10 +293,10 @@ namespace Microsoft.Data.Analysis
|
|||
|
||||
private class WrappedStreamReaderOrStringReader
|
||||
{
|
||||
private Stream _stream;
|
||||
private long _initialPosition;
|
||||
private Encoding _encoding;
|
||||
private string _csvString;
|
||||
private readonly Stream _stream;
|
||||
private readonly long _initialPosition;
|
||||
private readonly Encoding _encoding;
|
||||
private readonly string _csvString;
|
||||
|
||||
public WrappedStreamReaderOrStringReader(Stream stream, Encoding encoding)
|
||||
{
|
||||
|
|
|
@ -142,7 +142,7 @@ namespace Microsoft.Data.Analysis
|
|||
return ret;
|
||||
}
|
||||
|
||||
private static bool IsAnyNullValueInColumns (IReadOnlyCollection<DataFrameColumn> columns, long index)
|
||||
private static bool IsAnyNullValueInColumns(IReadOnlyCollection<DataFrameColumn> columns, long index)
|
||||
{
|
||||
foreach (var column in columns)
|
||||
{
|
||||
|
@ -176,19 +176,19 @@ namespace Microsoft.Data.Analysis
|
|||
throw new ArgumentNullException(nameof(supplemetaryJoinColumnNames));
|
||||
|
||||
if (retainedJoinColumnNames.Length != supplemetaryJoinColumnNames.Length)
|
||||
throw new ArgumentException(Strings.MismatchedArrayLengths, nameof(retainedJoinColumnNames));
|
||||
|
||||
throw new ArgumentException(Strings.MismatchedArrayLengths, nameof(retainedJoinColumnNames));
|
||||
|
||||
|
||||
HashSet<long> intersection = calculateIntersection ? new HashSet<long>() : null;
|
||||
|
||||
// Get occurrences of values in columns used for join in the retained and supplementary dataframes
|
||||
Dictionary<long, ICollection<long>> occurrences = null;
|
||||
Dictionary<long, long> retainedIndicesReverseMapping = null;
|
||||
|
||||
|
||||
HashSet<long> supplementaryJoinColumnsNullIndices = new HashSet<long>();
|
||||
|
||||
|
||||
for (int colNameIndex = 0; colNameIndex < retainedJoinColumnNames.Length; colNameIndex++)
|
||||
|
||||
for (int colNameIndex = 0; colNameIndex < retainedJoinColumnNames.Length; colNameIndex++)
|
||||
{
|
||||
DataFrameColumn shrinkedRetainedColumn = retainedDataFrame.Columns[retainedJoinColumnNames[colNameIndex]];
|
||||
|
||||
|
@ -211,7 +211,7 @@ namespace Microsoft.Data.Analysis
|
|||
retainedIndicesReverseMapping = newRetainedIndicesReverseMapping;
|
||||
shrinkedRetainedColumn = shrinkedRetainedColumn.Clone(new Int64DataFrameColumn("Indices", shrinkedRetainedIndices));
|
||||
}
|
||||
|
||||
|
||||
DataFrameColumn supplementaryColumn = supplementaryDataFrame.Columns[supplemetaryJoinColumnNames[colNameIndex]];
|
||||
|
||||
//Find occurrenses on current step (join column)
|
||||
|
@ -222,7 +222,7 @@ namespace Microsoft.Data.Analysis
|
|||
newOccurrences = newOccurrences.ToDictionary(kvp => retainedIndicesReverseMapping[kvp.Key], kvp => kvp.Value);
|
||||
|
||||
supplementaryJoinColumnsNullIndices.UnionWith(supplementaryColumnNullIndices);
|
||||
|
||||
|
||||
// shrink join result on current column by previous join columns (if any)
|
||||
// (we have to remove occurrences that doesn't exist in previous columns, because JOIN happens only if ALL left and right columns in JOIN are matched)
|
||||
if (occurrences != null)
|
||||
|
@ -242,7 +242,7 @@ namespace Microsoft.Data.Analysis
|
|||
|
||||
occurrences = newOccurrences;
|
||||
}
|
||||
|
||||
|
||||
retainedRowIndices = new Int64DataFrameColumn("RetainedIndices");
|
||||
supplementaryRowIndices = new Int64DataFrameColumn("SupplementaryIndices");
|
||||
|
||||
|
@ -280,7 +280,7 @@ namespace Microsoft.Data.Analysis
|
|||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
{
|
||||
foreach (long row in supplementaryJoinColumnsNullIndices)
|
||||
{
|
||||
retainedRowIndices.Append(i);
|
||||
|
@ -288,10 +288,10 @@ namespace Microsoft.Data.Analysis
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
return intersection;
|
||||
}
|
||||
|
||||
|
||||
public DataFrame Merge(DataFrame other, string[] leftJoinColumns, string[] rightJoinColumns, string leftSuffix = "_left", string rightSuffix = "_right", JoinAlgorithm joinAlgorithm = JoinAlgorithm.Left)
|
||||
{
|
||||
if (other == null)
|
||||
|
@ -335,7 +335,7 @@ namespace Microsoft.Data.Analysis
|
|||
else if (joinAlgorithm == JoinAlgorithm.FullOuter)
|
||||
{
|
||||
//In full outer join we would like to retain data from both side, so we do it into 2 steps: one first we do LEFT JOIN and then add lost data from the RIGHT side
|
||||
|
||||
|
||||
//Step 1
|
||||
//Do LEFT JOIN
|
||||
isLeftDataFrameRetained = true;
|
||||
|
@ -347,7 +347,7 @@ namespace Microsoft.Data.Analysis
|
|||
var retainedJoinColumns = isLeftDataFrameRetained ? leftJoinColumns : rightJoinColumns;
|
||||
|
||||
var intersection = Merge(retainedDataFrame, supplementaryDataFrame, retainedJoinColumns, supplementaryJoinColumns, out retainedRowIndices, out supplementaryRowIndices, calculateIntersection: true);
|
||||
|
||||
|
||||
//Step 2
|
||||
//Do RIGHT JOIN to retain all data from supplementary DataFrame too (take into account data intersection from the first step to avoid duplicates)
|
||||
for (long i = 0; i < supplementaryDataFrame.Columns.RowCount; i++)
|
||||
|
@ -365,9 +365,9 @@ namespace Microsoft.Data.Analysis
|
|||
}
|
||||
else
|
||||
throw new NotImplementedException(nameof(joinAlgorithm));
|
||||
|
||||
|
||||
DataFrame ret = new DataFrame();
|
||||
|
||||
|
||||
//insert columns from left dataframe (this)
|
||||
for (int i = 0; i < this.Columns.Count; i++)
|
||||
{
|
||||
|
|
|
@ -244,7 +244,7 @@ namespace Microsoft.Data.Analysis
|
|||
ret.Add(i, otherRowIndices);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
|
|
|
@ -13,11 +13,11 @@ namespace Microsoft.Data.Analysis
|
|||
/// </summary>
|
||||
public class DataFrameColumnCollection : Collection<DataFrameColumn>
|
||||
{
|
||||
private Action ColumnsChanged;
|
||||
private readonly Action ColumnsChanged;
|
||||
|
||||
private List<string> _columnNames = new List<string>();
|
||||
private readonly List<string> _columnNames = new List<string>();
|
||||
|
||||
private Dictionary<string, int> _columnNameToIndexDictionary = new Dictionary<string, int>(StringComparer.Ordinal);
|
||||
private readonly Dictionary<string, int> _columnNameToIndexDictionary = new Dictionary<string, int>(StringComparer.Ordinal);
|
||||
|
||||
internal long RowCount { get; set; }
|
||||
|
||||
|
|
|
@ -101,9 +101,9 @@ namespace Microsoft.Data.Analysis
|
|||
|
||||
#endregion
|
||||
|
||||
private int _groupByColumnIndex;
|
||||
private IDictionary<TKey, ICollection<long>> _keyToRowIndicesMap;
|
||||
private DataFrame _dataFrame;
|
||||
private readonly int _groupByColumnIndex;
|
||||
private readonly IDictionary<TKey, ICollection<long>> _keyToRowIndicesMap;
|
||||
private readonly DataFrame _dataFrame;
|
||||
|
||||
public GroupBy(DataFrame dataFrame, int groupByColumnIndex, IDictionary<TKey, ICollection<long>> keyToRowIndices)
|
||||
{
|
||||
|
|
|
@ -20,7 +20,7 @@ namespace Microsoft.Data.Analysis
|
|||
public partial class PrimitiveDataFrameColumn<T> : DataFrameColumn, IEnumerable<T?>
|
||||
where T : unmanaged
|
||||
{
|
||||
private PrimitiveColumnContainer<T> _columnContainer;
|
||||
private readonly PrimitiveColumnContainer<T> _columnContainer;
|
||||
|
||||
internal PrimitiveColumnContainer<T> ColumnContainer => _columnContainer;
|
||||
|
||||
|
|
|
@ -17,7 +17,7 @@ namespace Microsoft.Data.Analysis
|
|||
/// <remarks> Is NOT Arrow compatible </remarks>
|
||||
public partial class StringDataFrameColumn : DataFrameColumn, IEnumerable<string>
|
||||
{
|
||||
private List<List<string>> _stringBuffers = new List<List<string>>(); // To store more than intMax number of strings
|
||||
private readonly List<List<string>> _stringBuffers = new List<List<string>>(); // To store more than intMax number of strings
|
||||
|
||||
public StringDataFrameColumn(string name, long length = 0) : base(name, length, typeof(string))
|
||||
{
|
||||
|
|
|
@ -22,12 +22,12 @@ namespace Microsoft.Data.Analysis
|
|||
|
||||
internal class QuoteDelimitedFieldBuilder
|
||||
{
|
||||
private StringBuilder _field;
|
||||
private readonly StringBuilder _field;
|
||||
private bool _fieldFinished;
|
||||
private int _index;
|
||||
private int _delimiterLength;
|
||||
private Regex _delimiterRegex;
|
||||
private string _spaceChars;
|
||||
private readonly Regex _delimiterRegex;
|
||||
private readonly string _spaceChars;
|
||||
private bool _malformedLine;
|
||||
|
||||
public QuoteDelimitedFieldBuilder(Regex delimiterRegex, string spaceChars)
|
||||
|
@ -148,7 +148,7 @@ namespace Microsoft.Data.Analysis
|
|||
|
||||
private Regex _delimiterWithEndCharsRegex;
|
||||
|
||||
private int[] _whitespaceCodes = new int[] { '\u0009', '\u000B', '\u000C', '\u0020', '\u0085', '\u00A0', '\u1680', '\u2000', '\u2001', '\u2002', '\u2003', '\u2004', '\u2005', '\u2006', '\u2007', '\u2008', '\u2009', '\u200A', '\u200B', '\u2028', '\u2029', '\u3000', '\uFEFF' };
|
||||
private readonly int[] _whitespaceCodes = new int[] { '\u0009', '\u000B', '\u000C', '\u0020', '\u0085', '\u00A0', '\u1680', '\u2000', '\u2001', '\u2002', '\u2003', '\u2004', '\u2005', '\u2006', '\u2007', '\u2008', '\u2009', '\u200A', '\u200B', '\u2028', '\u2029', '\u3000', '\uFEFF' };
|
||||
|
||||
private Regex _beginQuotesRegex;
|
||||
|
||||
|
@ -172,13 +172,13 @@ namespace Microsoft.Data.Analysis
|
|||
|
||||
private string _spaceChars;
|
||||
|
||||
private int _maxLineSize = 10000000;
|
||||
private readonly int _maxLineSize = 10000000;
|
||||
|
||||
private int _maxBufferSize = 10000000;
|
||||
private readonly int _maxBufferSize = 10000000;
|
||||
|
||||
private bool _leaveOpen;
|
||||
private readonly bool _leaveOpen;
|
||||
|
||||
private char[] newLineChars = Environment.NewLine.ToCharArray();
|
||||
private readonly char[] _newLineChars = Environment.NewLine.ToCharArray();
|
||||
|
||||
public string[] CommentTokens
|
||||
{
|
||||
|
@ -419,7 +419,7 @@ namespace Microsoft.Data.Analysis
|
|||
}
|
||||
|
||||
_lineNumber++;
|
||||
return line.TrimEnd(newLineChars);
|
||||
return line.TrimEnd(_newLineChars);
|
||||
}
|
||||
|
||||
public string[] ReadFields()
|
||||
|
@ -470,7 +470,7 @@ namespace Microsoft.Data.Analysis
|
|||
return null;
|
||||
}
|
||||
|
||||
line = line.TrimEnd(newLineChars);
|
||||
line = line.TrimEnd(_newLineChars);
|
||||
if (line.Length < numberOfChars)
|
||||
{
|
||||
return line;
|
||||
|
@ -799,7 +799,7 @@ namespace Microsoft.Data.Analysis
|
|||
endHelper.BuildField(line, index);
|
||||
if (endHelper.MalformedLine)
|
||||
{
|
||||
_errorLine = line.TrimEnd(newLineChars);
|
||||
_errorLine = line.TrimEnd(_newLineChars);
|
||||
_errorLineNumber = currentLineNumber;
|
||||
throw new Exception(string.Format(Strings.CannotParseWithDelimiters, currentLineNumber));
|
||||
}
|
||||
|
@ -817,13 +817,13 @@ namespace Microsoft.Data.Analysis
|
|||
string newLine = ReadNextDataLine();
|
||||
if (newLine == null)
|
||||
{
|
||||
_errorLine = line.TrimEnd(newLineChars);
|
||||
_errorLine = line.TrimEnd(_newLineChars);
|
||||
_errorLineNumber = currentLineNumber;
|
||||
throw new Exception(string.Format(Strings.CannotParseWithDelimiters, currentLineNumber));
|
||||
}
|
||||
if (line.Length + newLine.Length > _maxLineSize)
|
||||
{
|
||||
_errorLine = line.TrimEnd(newLineChars);
|
||||
_errorLine = line.TrimEnd(_newLineChars);
|
||||
_errorLineNumber = currentLineNumber;
|
||||
throw new Exception(string.Format(Strings.LineExceedsMaxLineSize, currentLineNumber));
|
||||
}
|
||||
|
@ -832,7 +832,7 @@ namespace Microsoft.Data.Analysis
|
|||
endHelper.BuildField(line, endOfLine);
|
||||
if (endHelper.MalformedLine)
|
||||
{
|
||||
_errorLine = line.TrimEnd(newLineChars);
|
||||
_errorLine = line.TrimEnd(_newLineChars);
|
||||
_errorLineNumber = currentLineNumber;
|
||||
throw new Exception(string.Format(Strings.CannotParseWithDelimiters, currentLineNumber));
|
||||
}
|
||||
|
@ -862,7 +862,7 @@ namespace Microsoft.Data.Analysis
|
|||
index = delimiterMatch.Index + delimiterMatch.Length;
|
||||
continue;
|
||||
}
|
||||
field = line.Substring(index).TrimEnd(newLineChars);
|
||||
field = line.Substring(index).TrimEnd(_newLineChars);
|
||||
if (_trimWhiteSpace)
|
||||
{
|
||||
field = field.Trim();
|
||||
|
@ -881,7 +881,7 @@ namespace Microsoft.Data.Analysis
|
|||
{
|
||||
return null;
|
||||
}
|
||||
line = line.TrimEnd(newLineChars);
|
||||
line = line.TrimEnd(_newLineChars);
|
||||
StringInfo lineInfo = new StringInfo(line);
|
||||
ValidateFixedWidthLine(lineInfo, _lineNumber - 1);
|
||||
int index = 0;
|
||||
|
@ -897,7 +897,7 @@ namespace Microsoft.Data.Analysis
|
|||
|
||||
private string GetFixedWidthField(StringInfo line, int index, int fieldLength)
|
||||
{
|
||||
string field = (fieldLength > 0) ? line.SubstringByTextElements(index, fieldLength) : ((index < line.LengthInTextElements) ? line.SubstringByTextElements(index).TrimEnd(newLineChars) : string.Empty);
|
||||
string field = (fieldLength > 0) ? line.SubstringByTextElements(index, fieldLength) : ((index < line.LengthInTextElements) ? line.SubstringByTextElements(index).TrimEnd(_newLineChars) : string.Empty);
|
||||
if (_trimWhiteSpace)
|
||||
{
|
||||
return field.Trim();
|
||||
|
@ -1062,7 +1062,7 @@ namespace Microsoft.Data.Analysis
|
|||
{
|
||||
throw new Exception(Strings.EmptyDelimiters);
|
||||
}
|
||||
if (delimiter.IndexOfAny(newLineChars) > -1)
|
||||
if (delimiter.IndexOfAny(_newLineChars) > -1)
|
||||
{
|
||||
throw new Exception(Strings.DelimiterCannotBeNewlineChar);
|
||||
}
|
||||
|
|
|
@ -1,3 +1,7 @@
|
|||
using System.Runtime.CompilerServices;
|
||||
// Licensed to the .NET Foundation under one or more agreements.
|
||||
// The .NET Foundation licenses this file to you under the MIT license.
|
||||
// See the LICENSE file in the project root for more information.
|
||||
|
||||
using System.Runtime.CompilerServices;
|
||||
|
||||
[assembly: InternalsVisibleTo("Microsoft.Extensions.ML.Tests, PublicKey=00240000048000009400000006020000002400005253413100040000010001004b86c4cb78549b34bab61a3b1800e23bfeb5b3ec390074041536a7e3cbd97f5f04cf0f857155a8928eaa29ebfd11cfbbad3ba70efea7bda3226c6a8d370a4cd303f714486b6ebc225985a638471e6ef571cc92a4613c00b8fa65d61ccee0cbe5f36330c9a01f4183559f1bef24cc2917c6d913e3a541333a1d05d9bed22b38cb")]
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
// Foundation under one or more agreements.
|
||||
// Licensed to the .NET Foundation under one or more agreements.
|
||||
// The .NET Foundation licenses this file to you under the MIT license.
|
||||
// See the LICENSE file in the project root for more information.
|
||||
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
// Foundation under one or more agreements.
|
||||
// Licensed to the .NET Foundation under one or more agreements.
|
||||
// The .NET Foundation licenses this file to you under the MIT license.
|
||||
// See the LICENSE file in the project root for more information.
|
||||
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
// Foundation under one or more agreements.
|
||||
// Licensed to the .NET Foundation under one or more agreements.
|
||||
// The .NET Foundation licenses this file to you under the MIT license.
|
||||
// See the LICENSE file in the project root for more information.
|
||||
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
// Foundation under one or more agreements.
|
||||
// Licensed to the .NET Foundation under one or more agreements.
|
||||
// The .NET Foundation licenses this file to you under the MIT license.
|
||||
// See the LICENSE file in the project root for more information.
|
||||
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
// Foundation under one or more agreements.
|
||||
// Licensed to the .NET Foundation under one or more agreements.
|
||||
// The .NET Foundation licenses this file to you under the MIT license.
|
||||
// See the LICENSE file in the project root for more information.
|
||||
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
// Foundation under one or more agreements.
|
||||
// Licensed to the .NET Foundation under one or more agreements.
|
||||
// The .NET Foundation licenses this file to you under the MIT license.
|
||||
// See the LICENSE file in the project root for more information.
|
||||
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
// Foundation under one or more agreements.
|
||||
// Licensed to the .NET Foundation under one or more agreements.
|
||||
// The .NET Foundation licenses this file to you under the MIT license.
|
||||
// See the LICENSE file in the project root for more information.
|
||||
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
// Foundation under one or more agreements.
|
||||
// Licensed to the .NET Foundation under one or more agreements.
|
||||
// The .NET Foundation licenses this file to you under the MIT license.
|
||||
// See the LICENSE file in the project root for more information.
|
||||
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
// Foundation under one or more agreements.
|
||||
// Licensed to the .NET Foundation under one or more agreements.
|
||||
// The .NET Foundation licenses this file to you under the MIT license.
|
||||
// See the LICENSE file in the project root for more information.
|
||||
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
// Foundation under one or more agreements.
|
||||
// Licensed to the .NET Foundation under one or more agreements.
|
||||
// The .NET Foundation licenses this file to you under the MIT license.
|
||||
// See the LICENSE file in the project root for more information.
|
||||
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
// Foundation under one or more agreements.
|
||||
// Licensed to the .NET Foundation under one or more agreements.
|
||||
// The .NET Foundation licenses this file to you under the MIT license.
|
||||
// See the LICENSE file in the project root for more information.
|
||||
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
// Foundation under one or more agreements.
|
||||
// Licensed to the .NET Foundation under one or more agreements.
|
||||
// The .NET Foundation licenses this file to you under the MIT license.
|
||||
// See the LICENSE file in the project root for more information.
|
||||
|
||||
|
|
|
@ -10,18 +10,18 @@ namespace Microsoft.ML.AutoML
|
|||
{
|
||||
public bool IsMaximizing { get; }
|
||||
|
||||
private static RegressionMetric[] _minimizingRegressionMetrics = new RegressionMetric[]
|
||||
private static readonly RegressionMetric[] _minimizingRegressionMetrics = new RegressionMetric[]
|
||||
{
|
||||
RegressionMetric.MeanAbsoluteError,
|
||||
RegressionMetric.MeanSquaredError,
|
||||
RegressionMetric.RootMeanSquaredError
|
||||
};
|
||||
|
||||
private static BinaryClassificationMetric[] _minimizingBinaryMetrics = new BinaryClassificationMetric[]
|
||||
private static readonly BinaryClassificationMetric[] _minimizingBinaryMetrics = new BinaryClassificationMetric[]
|
||||
{
|
||||
};
|
||||
|
||||
private static MulticlassClassificationMetric[] _minimizingMulticlassMetrics = new MulticlassClassificationMetric[]
|
||||
private static readonly MulticlassClassificationMetric[] _minimizingMulticlassMetrics = new MulticlassClassificationMetric[]
|
||||
{
|
||||
MulticlassClassificationMetric.LogLoss,
|
||||
};
|
||||
|
|
|
@ -344,8 +344,8 @@ namespace Microsoft.ML.AutoML
|
|||
// Todo: Remove the reflection below for TreeTreeEnsembleModelParameters methods GetLeaf and GetLeafValue.
|
||||
// Long-term, replace with tree featurizer once it becomes available
|
||||
// Tracking issue -- https://github.com/dotnet/machinelearning-automl/issues/342
|
||||
private static MethodInfo _getLeafMethod = typeof(TreeEnsembleModelParameters).GetMethod("GetLeaf", BindingFlags.NonPublic | BindingFlags.Instance);
|
||||
private static MethodInfo _getLeafValueMethod = typeof(TreeEnsembleModelParameters).GetMethod("GetLeafValue", BindingFlags.NonPublic | BindingFlags.Instance);
|
||||
private static readonly MethodInfo _getLeafMethod = typeof(TreeEnsembleModelParameters).GetMethod("GetLeaf", BindingFlags.NonPublic | BindingFlags.Instance);
|
||||
private static readonly MethodInfo _getLeafValueMethod = typeof(TreeEnsembleModelParameters).GetMethod("GetLeafValue", BindingFlags.NonPublic | BindingFlags.Instance);
|
||||
|
||||
private static int GetLeaf(TreeEnsembleModelParameters model, int treeId, VBuffer<Float> features)
|
||||
{
|
||||
|
|
|
@ -75,7 +75,7 @@ namespace Microsoft.ML.AutoML
|
|||
return options;
|
||||
}
|
||||
|
||||
private static string[] _lightGbmBoosterParamNames = new[] { "L2Regularization", "L1Regularization" };
|
||||
private static readonly string[] _lightGbmBoosterParamNames = new[] { "L2Regularization", "L1Regularization" };
|
||||
private const string LightGbmBoosterPropName = "Booster";
|
||||
|
||||
public static TOptions CreateLightGbmOptions<TOptions, TOutput, TTransformer, TModel>(IEnumerable<SweepableParam> sweepParams, ColumnInformation columnInfo)
|
||||
|
|
|
@ -1,4 +1,8 @@
|
|||
using System;
|
||||
// Licensed to the .NET Foundation under one or more agreements.
|
||||
// The .NET Foundation licenses this file to you under the MIT license.
|
||||
// See the LICENSE file in the project root for more information.
|
||||
|
||||
using System;
|
||||
|
||||
namespace Microsoft.ML.AutoML.Utils
|
||||
{
|
||||
|
|
|
@ -76,7 +76,7 @@ namespace Microsoft.ML.CodeGenerator.CodeGenerator.CSharp
|
|||
IncludeRecommenderPackage = false,
|
||||
StablePackageVersion = _settings.StablePackageVersion,
|
||||
UnstablePackageVersion = _settings.UnstablePackageVersion,
|
||||
OnnxRuntimePackageVersion = _settings.OnnxRuntimePacakgeVersion,
|
||||
OnnxRuntimePackageVersion = _settings.OnnxRuntimePackageVersion,
|
||||
}.TransformText(),
|
||||
Name = $"{_settings.OutputName}.ConsoleApp.csproj",
|
||||
};
|
||||
|
|
|
@ -92,7 +92,7 @@ namespace Microsoft.ML.CodeGenerator.CodeGenerator.CSharp.AzureCodeGenerator
|
|||
IncludeRecommenderPackage = false,
|
||||
StablePackageVersion = _settings.StablePackageVersion,
|
||||
UnstablePackageVersion = _settings.UnstablePackageVersion,
|
||||
OnnxRuntimePackageVersion = _settings.OnnxRuntimePacakgeVersion,
|
||||
OnnxRuntimePackageVersion = _settings.OnnxRuntimePackageVersion,
|
||||
Target = _settings.Target,
|
||||
}.TransformText(),
|
||||
Name = $"{ _settings.OutputName }.Model.csproj",
|
||||
|
|
|
@ -285,7 +285,7 @@ namespace Microsoft.ML.CodeGenerator.CSharp
|
|||
IncludeRecommenderPackage = includeRecommenderPackage,
|
||||
StablePackageVersion = stablePackageVersion,
|
||||
UnstablePackageVersion = unstablePackageVersion,
|
||||
OnnxRuntimePackageVersion = _settings.OnnxRuntimePacakgeVersion,
|
||||
OnnxRuntimePackageVersion = _settings.OnnxRuntimePackageVersion,
|
||||
Target = target,
|
||||
};
|
||||
|
||||
|
@ -336,7 +336,7 @@ namespace Microsoft.ML.CodeGenerator.CSharp
|
|||
IncludeRecommenderPackage = includeRecommenderPackage,
|
||||
StablePackageVersion = stablePackageVersion,
|
||||
UnstablePackageVersion = unstablePackageVersion,
|
||||
OnnxRuntimePackageVersion = _settings.OnnxRuntimePacakgeVersion,
|
||||
OnnxRuntimePackageVersion = _settings.OnnxRuntimePackageVersion,
|
||||
Target = target,
|
||||
};
|
||||
return predictProjectFileContent.TransformText();
|
||||
|
|
|
@ -54,7 +54,7 @@ namespace Microsoft.ML.CodeGenerator.CSharp
|
|||
|
||||
public string UnstablePackageVersion { get; set; }
|
||||
|
||||
public string OnnxRuntimePacakgeVersion { get; set; }
|
||||
public string OnnxRuntimePackageVersion { get; set; }
|
||||
|
||||
public bool IsAzureAttach { get; set; }
|
||||
|
||||
|
|
|
@ -538,7 +538,7 @@ namespace Microsoft.ML.CodeGenerator.CSharp
|
|||
|
||||
internal class OneVersusAll : TrainerGeneratorBase
|
||||
{
|
||||
private PipelineNode _node;
|
||||
private readonly PipelineNode _node;
|
||||
private string[] _binaryTrainerUsings;
|
||||
|
||||
//ClassName of the trainer
|
||||
|
|
|
@ -10,7 +10,7 @@ namespace Microsoft.ML.CommandLine
|
|||
[BestFriend]
|
||||
internal sealed class CmdLexer
|
||||
{
|
||||
private CharCursor _curs;
|
||||
private readonly CharCursor _curs;
|
||||
|
||||
// Whether \ has a special significance.
|
||||
private readonly bool _escapes;
|
||||
|
@ -307,7 +307,7 @@ namespace Microsoft.ML.CommandLine
|
|||
internal sealed class CmdQuoter
|
||||
{
|
||||
private readonly string _str;
|
||||
private StringBuilder _sb;
|
||||
private readonly StringBuilder _sb;
|
||||
private int _ich;
|
||||
|
||||
private CmdQuoter(string str, StringBuilder sb)
|
||||
|
|
|
@ -79,7 +79,7 @@ namespace Microsoft.ML
|
|||
}
|
||||
|
||||
// These are the open entries that may contain streams into our DirTemp.
|
||||
private List<Entry> _open;
|
||||
private readonly List<Entry> _open;
|
||||
|
||||
private bool _disposed;
|
||||
|
||||
|
@ -435,11 +435,11 @@ namespace Microsoft.ML
|
|||
[BestFriend]
|
||||
internal sealed class RepositoryReader : Repository
|
||||
{
|
||||
private ZipArchive _archive;
|
||||
private readonly ZipArchive _archive;
|
||||
|
||||
// Maps from a normalized path to the entry in the _archive. This is needed since
|
||||
// a zip might use / or \ for directory separation.
|
||||
private Dictionary<string, ZipArchiveEntry> _entries;
|
||||
private readonly Dictionary<string, ZipArchiveEntry> _entries;
|
||||
|
||||
public static RepositoryReader Open(Stream stream, IExceptionContext ectx = null, bool useFileSystem = true)
|
||||
{
|
||||
|
|
|
@ -352,7 +352,7 @@ namespace Microsoft.ML.Runtime
|
|||
private readonly MessageSensitivity _sensitivityFlags;
|
||||
|
||||
// This object is used to write to the test log along with the console if the host process is a test environment
|
||||
private TextWriter _testWriter;
|
||||
private readonly TextWriter _testWriter;
|
||||
|
||||
/// <summary>
|
||||
/// Create an ML.NET <see cref="IHostEnvironment"/> for local execution, with console feedback.
|
||||
|
|
|
@ -13,7 +13,7 @@ namespace Microsoft.ML.Internal.Utilities
|
|||
{
|
||||
private Single[] _valuesSng; // distinct values
|
||||
private Double[] _valuesDbl; // distinct values
|
||||
private List<int> _counts; // counts for each value
|
||||
private readonly List<int> _counts; // counts for each value
|
||||
private int[] _path; // current set of pegs
|
||||
|
||||
protected int CountBins { get; private set; } // number of bins
|
||||
|
@ -314,8 +314,8 @@ namespace Microsoft.ML.Internal.Utilities
|
|||
}
|
||||
}
|
||||
|
||||
private Heap<Segment> _segmentHeap; // heap used for dropping initial peg placement
|
||||
private HeapNode.Heap<Peg> _pegHeap; // heap used for selecting the largest energy decrease
|
||||
private readonly Heap<Segment> _segmentHeap; // heap used for dropping initial peg placement
|
||||
private readonly HeapNode.Heap<Peg> _pegHeap; // heap used for selecting the largest energy decrease
|
||||
private int[] _accum; // integral of counts
|
||||
private int[] _path; // current set of pegs
|
||||
private float _meanBinSize;
|
||||
|
|
|
@ -732,7 +732,7 @@ LAfterDigits:
|
|||
|
||||
// Map from base-10 exponent to 64-bit mantissa.
|
||||
// The approximation for 10^n is _mpe10man[n-1] * 2^(_mpe10e2[n-1]-64).
|
||||
private static ulong[] _mpe10Man = new ulong[] {
|
||||
private static readonly ulong[] _mpe10Man = new ulong[] {
|
||||
0xA000000000000000UL, 0xC800000000000000UL, 0xFA00000000000000UL, 0x9C40000000000000UL, 0xC350000000000000UL, /*005*/
|
||||
0xF424000000000000UL, 0x9896800000000000UL, 0xBEBC200000000000UL, 0xEE6B280000000000UL, 0x9502F90000000000UL, /*010*/
|
||||
0xBA43B74000000000UL, 0xE8D4A51000000000UL, 0x9184E72A00000000UL, 0xB5E620F480000000UL, 0xE35FA931A0000000UL, /*015*/
|
||||
|
@ -801,7 +801,7 @@ LAfterDigits:
|
|||
|
||||
// Map from negative base-10 exponent to 64-bit mantissa. Note that the top bit of these is set.
|
||||
// The approximation for 10^-n is _mpne10man[n-1] * 2^(-_mpne10ne2[n-1]-64).
|
||||
private static ulong[] _mpne10Man = new ulong[] {
|
||||
private static readonly ulong[] _mpne10Man = new ulong[] {
|
||||
0xCCCCCCCCCCCCCCCDUL, 0xA3D70A3D70A3D70AUL, 0x83126E978D4FDF3BUL, 0xD1B71758E219652CUL, 0xA7C5AC471B478423UL, /*005*/
|
||||
0x8637BD05AF6C69B6UL, 0xD6BF94D5E57A42BCUL, 0xABCC77118461CEFDUL, 0x89705F4136B4A597UL, 0xDBE6FECEBDEDD5BFUL, /*010*/
|
||||
0xAFEBFF0BCB24AAFFUL, 0x8CBCCC096F5088CCUL, 0xE12E13424BB40E13UL, 0xB424DC35095CD80FUL, 0x901D7CF73AB0ACD9UL, /*015*/
|
||||
|
@ -878,7 +878,7 @@ LAfterDigits:
|
|||
|
||||
// Map from base-10 exponent to base-2 exponent.
|
||||
// The approximation for 10^n is _mpe10man[n-1] * 2^(_mpe10e2[n-1]-64).
|
||||
private static short[] _mpe10e2 = new short[] {
|
||||
private static readonly short[] _mpe10e2 = new short[] {
|
||||
4, 7, 10, 14, 17, 20, 24, 27, 30, 34, 37, 40, 44, 47, 50, 54, 57, 60, 64, 67, /*020*/
|
||||
70, 74, 77, 80, 84, 87, 90, 94, 97, 100, 103, 107, 110, 113, 117, 120, 123, 127, 130, 133, /*040*/
|
||||
137, 140, 143, 147, 150, 153, 157, 160, 163, 167, 170, 173, 177, 180, 183, 187, 190, 193, 196, 200, /*060*/
|
||||
|
|
|
@ -42,7 +42,7 @@ namespace Microsoft.ML.Internal.Utilities
|
|||
return result;
|
||||
}
|
||||
|
||||
private static double _vScale = 2 * Math.Sqrt(2 / Math.E);
|
||||
private static readonly double _vScale = 2 * Math.Sqrt(2 / Math.E);
|
||||
|
||||
/// <summary>
|
||||
/// Returns a number sampled from a zero-mean, unit variance Gaussian
|
||||
|
@ -242,7 +242,7 @@ namespace Microsoft.ML.Internal.Utilities
|
|||
// n*p at which we switch algorithms
|
||||
private const int NPThresh = 10;
|
||||
|
||||
private static double[] _fctab = new double[] {
|
||||
private static readonly double[] _fctab = new double[] {
|
||||
0.08106146679532726, 0.04134069595540929, 0.02767792568499834,
|
||||
0.02079067210376509, 0.01664469118982119, 0.01387612882307075,
|
||||
0.01189670994589177, 0.01041126526197209, 0.009255462182712733,
|
||||
|
|
|
@ -28,7 +28,7 @@ namespace Microsoft.ML.Internal.Utilities
|
|||
private string _line;
|
||||
private int _lineCur;
|
||||
|
||||
private byte[] _buff;
|
||||
private readonly byte[] _buff;
|
||||
private int _buffCur;
|
||||
private int _buffLim;
|
||||
private bool _eof;
|
||||
|
|
|
@ -28,7 +28,7 @@ namespace Microsoft.ML.Internal.CpuMath
|
|||
private readonly int _cbAlign; // The alignment in bytes, a power of two, divisible by sizeof(Float).
|
||||
private int _base; // Where the values start in Items (changes to ensure alignment).
|
||||
|
||||
private object _lock; // Used to make sure only one thread can re-align the values.
|
||||
private readonly object _lock; // Used to make sure only one thread can re-align the values.
|
||||
|
||||
/// <summary>
|
||||
/// Allocate an aligned vector with the given alignment (in bytes).
|
||||
|
|
|
@ -23,7 +23,7 @@ namespace Microsoft.ML.Data
|
|||
/// For example, UInt32 and Key can be mapped to <see langword="uint"/>. This class enforces one-to-one mapping for all
|
||||
/// user-registered types.
|
||||
/// </summary>
|
||||
private static HashSet<Type> _bannedRawTypes = new HashSet<Type>()
|
||||
private static readonly HashSet<Type> _bannedRawTypes = new HashSet<Type>()
|
||||
{
|
||||
typeof(Boolean), typeof(SByte), typeof(Byte),
|
||||
typeof(Int16), typeof(UInt16), typeof(Int32), typeof(UInt32),
|
||||
|
@ -36,17 +36,17 @@ namespace Microsoft.ML.Data
|
|||
/// <summary>
|
||||
/// Mapping from a <see cref="Type"/> plus its <see cref="Attribute"/>s to a <see cref="DataViewType"/>.
|
||||
/// </summary>
|
||||
private static Dictionary<TypeWithAttributes, DataViewType> _rawTypeToDataViewTypeMap = new Dictionary<TypeWithAttributes, DataViewType>();
|
||||
private static readonly Dictionary<TypeWithAttributes, DataViewType> _rawTypeToDataViewTypeMap = new Dictionary<TypeWithAttributes, DataViewType>();
|
||||
|
||||
/// <summary>
|
||||
/// Mapping from a <see cref="DataViewType"/> to a <see cref="Type"/> plus its <see cref="Attribute"/>s.
|
||||
/// </summary>
|
||||
private static Dictionary<DataViewType, TypeWithAttributes> _dataViewTypeToRawTypeMap = new Dictionary<DataViewType, TypeWithAttributes>();
|
||||
private static readonly Dictionary<DataViewType, TypeWithAttributes> _dataViewTypeToRawTypeMap = new Dictionary<DataViewType, TypeWithAttributes>();
|
||||
|
||||
/// <summary>
|
||||
/// The lock that one should acquire if the state of <see cref="DataViewTypeManager"/> will be accessed or modified.
|
||||
/// </summary>
|
||||
private static object _lock = new object();
|
||||
private static readonly object _lock = new object();
|
||||
|
||||
/// <summary>
|
||||
/// Returns the <see cref="DataViewType"/> registered for <paramref name="type"/> and its <paramref name="typeAttributes"/>.
|
||||
|
@ -228,7 +228,7 @@ namespace Microsoft.ML.Data
|
|||
/// a key when using <see cref="TypeWithAttributes"/> as the key type in <see cref="Dictionary{TKey, TValue}"/>. Note that the
|
||||
/// uniqueness is determined by <see cref="Equals(object)"/> and <see cref="GetHashCode"/> below.
|
||||
/// </summary>
|
||||
private DataViewTypeAttribute _associatedAttribute;
|
||||
private readonly DataViewTypeAttribute _associatedAttribute;
|
||||
|
||||
public TypeWithAttributes(Type type, DataViewTypeAttribute attribute)
|
||||
{
|
||||
|
|
|
@ -107,8 +107,8 @@ namespace Microsoft.ML.Data.IO
|
|||
|
||||
private sealed class WritePipe<T> : WritePipe
|
||||
{
|
||||
private ValueGetter<T> _getter;
|
||||
private IValueCodec<T> _codec;
|
||||
private readonly ValueGetter<T> _getter;
|
||||
private readonly IValueCodec<T> _codec;
|
||||
private IValueWriter<T> _writer;
|
||||
private MemoryStream _currentStream;
|
||||
private T _value;
|
||||
|
|
|
@ -313,7 +313,7 @@ namespace Microsoft.ML.Data.IO
|
|||
private sealed class Writer : ValueWriterBase<ReadOnlyMemory<char>>
|
||||
{
|
||||
private StringBuilder _builder;
|
||||
private List<int> _boundaries;
|
||||
private readonly List<int> _boundaries;
|
||||
|
||||
public Writer(TextCodec codec, Stream stream)
|
||||
: base(codec.Factory, stream)
|
||||
|
@ -353,7 +353,7 @@ namespace Microsoft.ML.Data.IO
|
|||
private readonly int _entries;
|
||||
private readonly int[] _boundaries;
|
||||
private int _index;
|
||||
private string _text;
|
||||
private readonly string _text;
|
||||
|
||||
public Reader(TextCodec codec, Stream stream, int items)
|
||||
: base(codec.Factory, stream)
|
||||
|
|
|
@ -25,7 +25,7 @@ namespace Microsoft.ML.Internal.Internallearn
|
|||
|
||||
internal static class UnsafeTypeOpsFactory
|
||||
{
|
||||
private static Dictionary<Type, object> _type2ops;
|
||||
private static readonly Dictionary<Type, object> _type2ops;
|
||||
|
||||
static UnsafeTypeOpsFactory()
|
||||
{
|
||||
|
|
|
@ -249,7 +249,7 @@ namespace Microsoft.ML.Data
|
|||
private readonly TryParseMapper<TResult> _conv;
|
||||
|
||||
// Has length Rows.Count, so indexed by irow.
|
||||
private TResult[] _values;
|
||||
private readonly TResult[] _values;
|
||||
|
||||
public override bool HasNA { get; }
|
||||
|
||||
|
@ -436,7 +436,7 @@ namespace Microsoft.ML.Data
|
|||
}
|
||||
|
||||
// Has length Rows.Count, so indexed by irow.
|
||||
private VectorValue[] _values;
|
||||
private readonly VectorValue[] _values;
|
||||
|
||||
public VectorPipe(RowSet rows, PrimitiveDataViewType type, TryParseMapper<TItem> conv)
|
||||
: base(rows)
|
||||
|
|
|
@ -221,7 +221,7 @@ namespace Microsoft.ML.Data.IO
|
|||
{
|
||||
private readonly ValueGetter<T> _getSrc;
|
||||
private T _src;
|
||||
private string _columnName;
|
||||
private readonly string _columnName;
|
||||
|
||||
public ValueWriter(DataViewRowCursor cursor, PrimitiveDataViewType type, int source, char sep)
|
||||
: base(type, source, sep)
|
||||
|
@ -535,8 +535,8 @@ namespace Microsoft.ML.Data.IO
|
|||
private int _dstPrev;
|
||||
|
||||
// Map from column to starting destination index and slot.
|
||||
private int[] _mpcoldst;
|
||||
private int[] _mpcolslot;
|
||||
private readonly int[] _mpcoldst;
|
||||
private readonly int[] _mpcolslot;
|
||||
|
||||
// "slot" is an index into _mpslotdst and _mpslotichLim. _mpslotdst is the sequence of
|
||||
// destination indices. _mpslotichLim is the sequence of upper bounds on the characters
|
||||
|
|
|
@ -89,7 +89,7 @@ namespace Microsoft.ML.Data
|
|||
private sealed class SubsetActive : DataViewRow
|
||||
{
|
||||
private readonly DataViewRow _row;
|
||||
private Func<int, bool> _pred;
|
||||
private readonly Func<int, bool> _pred;
|
||||
|
||||
public SubsetActive(DataViewRow row, Func<int, bool> pred)
|
||||
{
|
||||
|
|
|
@ -686,7 +686,7 @@ namespace Microsoft.ML.Data
|
|||
internal sealed class StreamingDataView<TRow> : DataViewBase<TRow>
|
||||
where TRow : class
|
||||
{
|
||||
private IEnumerable<TRow> _data;
|
||||
private readonly IEnumerable<TRow> _data;
|
||||
|
||||
public StreamingDataView(IHostEnvironment env, IEnumerable<TRow> data, InternalSchemaDefinition schemaDefn)
|
||||
: base(env, "StreamingDataView", schemaDefn)
|
||||
|
@ -936,7 +936,7 @@ namespace Microsoft.ML.Data
|
|||
if (AnnotationType is VectorDataViewType annotationVectorType)
|
||||
{
|
||||
// VBuffer<T> -> VBuffer<T>
|
||||
// REVIEW: Do we care about accomodating VBuffer<string> -> VBuffer<ReadOnlyMemory<char>>?
|
||||
// REVIEW: Do we care about accommodating VBuffer<string> -> VBuffer<ReadOnlyMemory<char>>?
|
||||
|
||||
Contracts.Assert(typeT.IsGenericType);
|
||||
Contracts.Check(typeof(TDst).IsGenericType);
|
||||
|
|
|
@ -58,7 +58,7 @@ namespace Microsoft.ML.Internal.Internallearn
|
|||
return CmdParser.CombineSettings(settings) + " " + CmdParser.CombineSettings(SplitOnSemis(extraSettings));
|
||||
}
|
||||
|
||||
private static char[] _dontSplitChars = new char[] { ' ', '=', '{', '}', '\t' };
|
||||
private static readonly char[] _dontSplitChars = new char[] { ' ', '=', '{', '}', '\t' };
|
||||
|
||||
// REVIEW: Deprecate this!
|
||||
public static string[] SplitOnSemis(string[] args)
|
||||
|
|
|
@ -427,7 +427,7 @@ namespace Microsoft.ML.EntryPoints
|
|||
|
||||
public TimeSpan RunTime { get; internal set; }
|
||||
|
||||
private static Regex _stageIdRegex = new Regex(@"[a-zA-Z0-9]*", RegexOptions.Compiled);
|
||||
private static readonly Regex _stageIdRegex = new Regex(@"[a-zA-Z0-9]*", RegexOptions.Compiled);
|
||||
private string _stageId;
|
||||
/// <summary>
|
||||
/// An alphanumeric string indicating the stage of a node.
|
||||
|
@ -1060,7 +1060,7 @@ namespace Microsoft.ML.EntryPoints
|
|||
// An EntryPoint variable can be followed with an array or dictionary specifier, which begins
|
||||
// with '[', contains either an integer or alphanumeric string, optionally wrapped in single-quotes,
|
||||
// followed with ']'.
|
||||
private static Regex _variableRegex = new Regex(
|
||||
private static readonly Regex _variableRegex = new Regex(
|
||||
@"\$(?<Name>[a-zA-Z_][a-zA-Z0-9_]*)(\[(((?<NumericAccessor>[0-9]*))|(\'?(?<StringAccessor>[a-zA-Z0-9_]*)\'?))\])?",
|
||||
RegexOptions.Compiled);
|
||||
|
||||
|
@ -1193,7 +1193,7 @@ namespace Microsoft.ML.EntryPoints
|
|||
|
||||
/// <summary>
|
||||
/// Represents the l-value assignable destination of a <see cref="VariableBinding"/>.
|
||||
/// Subclasses exist to express the needed bindinds for subslots
|
||||
/// Subclasses exist to express the needed bindings for subslots
|
||||
/// of a yet-to-be-constructed array or dictionary EntryPoint input parameter
|
||||
/// (for example, "myVar": ["$var1", "$var2"] would yield two <see cref="ArrayIndexParameterBinding"/>: (myVar, 0), (myVar, 1))
|
||||
/// </summary>
|
||||
|
|
|
@ -210,7 +210,7 @@ namespace Microsoft.ML.Data
|
|||
private readonly Counters _counters;
|
||||
private readonly Counters _weightedCounters;
|
||||
|
||||
private VBuffer<ReadOnlyMemory<char>> _slotNames;
|
||||
private readonly VBuffer<ReadOnlyMemory<char>> _slotNames;
|
||||
|
||||
public override CountersBase UnweightedCounters { get { return _counters; } }
|
||||
|
||||
|
|
|
@ -1069,8 +1069,8 @@ namespace Microsoft.ML.Calibrators
|
|||
{
|
||||
private readonly IHost _host;
|
||||
|
||||
private List<float> _cMargins;
|
||||
private List<float> _ncMargins;
|
||||
private readonly List<float> _cMargins;
|
||||
private readonly List<float> _ncMargins;
|
||||
|
||||
public int NumBins;
|
||||
public float BinSize;
|
||||
|
@ -1846,7 +1846,7 @@ namespace Microsoft.ML.Calibrators
|
|||
[BestFriend]
|
||||
internal sealed class PavCalibratorTrainer : CalibratorTrainerBase
|
||||
{
|
||||
// a piece of the piecwise function
|
||||
// a piece of the piecewise function
|
||||
private readonly struct Piece
|
||||
{
|
||||
public readonly float MinX; // end of interval.
|
||||
|
|
|
@ -69,7 +69,7 @@ namespace Microsoft.ML.Calibrators
|
|||
LabelColumn = TrainerUtils.MakeBoolScalarLabel(labelColumn);
|
||||
else
|
||||
env.CheckParam(!calibratorTrainer.NeedsTraining, nameof(labelColumn), "For trained calibrators, " + nameof(labelColumn) + " must be specified.");
|
||||
ScoreColumn = TrainerUtils.MakeR4ScalarColumn(scoreColumn); // Do we fanthom this being named anything else (renaming column)? Complete metadata?
|
||||
ScoreColumn = TrainerUtils.MakeR4ScalarColumn(scoreColumn); // Do we fathom this being named anything else (renaming column)? Complete metadata?
|
||||
|
||||
if (weightColumn != null)
|
||||
WeightColumn = TrainerUtils.MakeR4ScalarWeightColumn(weightColumn);
|
||||
|
@ -239,10 +239,10 @@ namespace Microsoft.ML.Calibrators
|
|||
private sealed class Mapper<TCalibrator> : MapperBase, ISaveAsOnnx
|
||||
where TCalibrator : class, ICalibrator
|
||||
{
|
||||
private TCalibrator _calibrator;
|
||||
private readonly TCalibrator _calibrator;
|
||||
private readonly int _scoreColIndex;
|
||||
private CalibratorTransformer<TCalibrator> _parent;
|
||||
private string _scoreColumnName;
|
||||
private readonly CalibratorTransformer<TCalibrator> _parent;
|
||||
private readonly string _scoreColumnName;
|
||||
|
||||
bool ICanSaveOnnx.CanSaveOnnx(OnnxContext ctx) => _calibrator is ICanSaveOnnx onnxMapper ? onnxMapper.CanSaveOnnx(ctx) : false;
|
||||
|
||||
|
@ -294,7 +294,7 @@ namespace Microsoft.ML.Calibrators
|
|||
builder.Add(setIdCol.Name, setIdType, annotation.GetGetter<uint>(setIdCol));
|
||||
// Now, this next one I'm a little less sure about. It is entirely reasonable for someone to, say,
|
||||
// try to calibrate the result of a regression or ranker training, or something else. But should we
|
||||
// just pass through this class just like that? Having throught through the alternatives I view this
|
||||
// just pass through this class just like that? Having thought through the alternatives I view this
|
||||
// as the least harmful thing we could be doing, but it is something to consider I may be wrong
|
||||
// about if it proves that it ever causes problems to, say, have something identified as a probability
|
||||
// column but be marked as being a regression task, or what have you.
|
||||
|
@ -415,7 +415,7 @@ namespace Microsoft.ML.Calibrators
|
|||
}
|
||||
|
||||
/// <summary>
|
||||
/// The naive binning-based calbirator estimator.
|
||||
/// The naive binning-based calibrator estimator.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// It divides the range of the outputs into equally sized bins. In each bin,
|
||||
|
|
|
@ -299,7 +299,7 @@ namespace Microsoft.ML.Data
|
|||
private readonly BindableMapper _parent;
|
||||
private readonly DataViewSchema _outputSchema;
|
||||
private readonly DataViewSchema _outputGenericSchema;
|
||||
private VBuffer<ReadOnlyMemory<char>> _slotNames;
|
||||
private readonly VBuffer<ReadOnlyMemory<char>> _slotNames;
|
||||
|
||||
public RoleMappedSchema InputRoleMappedSchema { get; }
|
||||
|
||||
|
|
|
@ -90,7 +90,7 @@ namespace Microsoft.ML.Transforms
|
|||
/// <param name="keepHidden">If true will keep hidden columns and false will remove hidden columns. The argument is
|
||||
/// ignored if the Estimator is in "drop mode".</param>
|
||||
/// <param name="ignoreMissing">If false will check for any columns given in <paramref name="keepColumns"/>
|
||||
/// or <paramref name="dropColumns"/> that are missing from the input. If a missing colums exists a
|
||||
/// or <paramref name="dropColumns"/> that are missing from the input. If a missing column exists a
|
||||
/// SchemaMistmatch exception is thrown. If true, the check is not made.</param>
|
||||
internal ColumnSelectingEstimator(IHostEnvironment env, string[] keepColumns,
|
||||
string[] dropColumns, bool keepHidden = Defaults.KeepHidden,
|
||||
|
@ -160,7 +160,7 @@ namespace Microsoft.ML.Transforms
|
|||
internal const string ChooseLoaderSignatureOld = "ChooseColumnsFunction";
|
||||
|
||||
private readonly IHost _host;
|
||||
private string[] _selectedColumns;
|
||||
private readonly string[] _selectedColumns;
|
||||
|
||||
bool ITransformer.IsRowToRowMapper => true;
|
||||
|
||||
|
@ -697,7 +697,7 @@ namespace Microsoft.ML.Transforms
|
|||
var inputCols = ((IRowToRowMapper)this).GetDependencies(columnsNeeded);
|
||||
var inputs = Source.GetRowCursorSet(inputCols, n, rand);
|
||||
|
||||
// Build out the acitve state for the output
|
||||
// Build out the active state for the output
|
||||
var active = Utils.BuildArray(_mapper.OutputSchema.Count, columnsNeeded);
|
||||
_host.AssertNonEmpty(inputs);
|
||||
|
||||
|
|
|
@ -1830,7 +1830,7 @@ namespace Microsoft.ML.Transforms
|
|||
{
|
||||
private readonly bool _fix;
|
||||
private readonly int _numBins;
|
||||
private List<TFloat> _values;
|
||||
private readonly List<TFloat> _values;
|
||||
|
||||
private BinOneColumnFunctionBuilder(IHost host, long lim, bool fix, int numBins, ValueGetter<TFloat> getSrc)
|
||||
: base(host, lim, getSrc)
|
||||
|
@ -1874,7 +1874,7 @@ namespace Microsoft.ML.Transforms
|
|||
{
|
||||
private readonly bool _fix;
|
||||
private readonly int _numBins;
|
||||
private List<TFloat>[] _values;
|
||||
private readonly List<TFloat>[] _values;
|
||||
|
||||
private BinVecColumnFunctionBuilder(IHost host, int cv, long lim, bool fix, int numBins,
|
||||
ValueGetter<VBuffer<TFloat>> getSrc)
|
||||
|
@ -2097,7 +2097,7 @@ namespace Microsoft.ML.Transforms
|
|||
// The difference is how the min/max are used.
|
||||
_minMaxAggregator = new MinMaxDblAggregator(vectorSize);
|
||||
|
||||
// If we aren't centering data dont need the median.
|
||||
// If we aren't centering data don't need the median.
|
||||
_medianAggregators = new MedianDblAggregator[vectorSize];
|
||||
|
||||
for (int i = 0; i < vectorSize; i++)
|
||||
|
|
|
@ -1993,7 +1993,7 @@ namespace Microsoft.ML.Transforms
|
|||
{
|
||||
private readonly bool _fix;
|
||||
private readonly int _numBins;
|
||||
private List<TFloat> _values;
|
||||
private readonly List<TFloat> _values;
|
||||
|
||||
private BinOneColumnFunctionBuilder(IHost host, long lim, bool fix, int numBins, ValueGetter<TFloat> getSrc)
|
||||
: base(host, lim, getSrc)
|
||||
|
@ -2037,7 +2037,7 @@ namespace Microsoft.ML.Transforms
|
|||
{
|
||||
private readonly bool _fix;
|
||||
private readonly int _numBins;
|
||||
private List<TFloat>[] _values;
|
||||
private readonly List<TFloat>[] _values;
|
||||
|
||||
private BinVecColumnFunctionBuilder(IHost host, int cv, long lim, bool fix, int numBins,
|
||||
ValueGetter<VBuffer<TFloat>> getSrc)
|
||||
|
|
|
@ -714,7 +714,7 @@ namespace Microsoft.ML.Transforms
|
|||
|
||||
private sealed class Mapper : OneToOneMapperBase, ISaveAsOnnx, ISaveAsPfa
|
||||
{
|
||||
private NormalizingTransformer _parent;
|
||||
private readonly NormalizingTransformer _parent;
|
||||
|
||||
public bool CanSaveOnnx(OnnxContext ctx) => true;
|
||||
public bool CanSavePfa => true;
|
||||
|
|
|
@ -186,7 +186,7 @@ namespace Microsoft.ML.Transforms
|
|||
internal class DataViewHelper
|
||||
{
|
||||
/// <summary>
|
||||
/// Helper function to retrieve the Primitie type given a Type
|
||||
/// Helper function to retrieve the Primitive type given a Type
|
||||
/// </summary>
|
||||
internal static PrimitiveDataViewType GetPrimitiveType(Type rawType, out bool isVectorType)
|
||||
{
|
||||
|
@ -359,7 +359,7 @@ namespace Microsoft.ML.Transforms
|
|||
private const string DefaultMapName = "DefaultMap.idv";
|
||||
internal static string DefaultKeyColumnName = "Key";
|
||||
internal static string DefaultValueColumnName = "Value";
|
||||
private ValueMap _valueMap;
|
||||
private readonly ValueMap _valueMap;
|
||||
private readonly byte[] _dataView;
|
||||
|
||||
internal DataViewType ValueColumnType => _valueMap.ValueColumn.Type;
|
||||
|
@ -834,7 +834,7 @@ namespace Microsoft.ML.Transforms
|
|||
private static readonly FuncStaticMethodInfo1<TValue, TValue> _getValueMethodInfo
|
||||
= new FuncStaticMethodInfo1<TValue, TValue>(GetValue<int>);
|
||||
|
||||
private Dictionary<TKey, TValue> _mapping;
|
||||
private readonly Dictionary<TKey, TValue> _mapping;
|
||||
private TValue _missingValue;
|
||||
|
||||
private Dictionary<TKey, TValue> CreateDictionary()
|
||||
|
|
|
@ -46,7 +46,7 @@ namespace Microsoft.ML.Trainers.Ensemble
|
|||
public IBinaryOutputCombiner CreateComponent(IHostEnvironment env) => new WeightedAverage(env, this);
|
||||
}
|
||||
|
||||
private WeightageKind _weightageKind;
|
||||
private readonly WeightageKind _weightageKind;
|
||||
|
||||
public string WeightageMetricName { get { return _weightageKind.ToString(); } }
|
||||
|
||||
|
|
|
@ -19,7 +19,7 @@ namespace Microsoft.ML.Trainers.Ensemble
|
|||
}
|
||||
|
||||
private readonly IComponentFactory<IDiversityMeasure<TOutput>> _diversityMetricType;
|
||||
private ConcurrentDictionary<FeatureSubsetModel<TOutput>, TOutput[]> _predictions;
|
||||
private readonly ConcurrentDictionary<FeatureSubsetModel<TOutput>, TOutput[]> _predictions;
|
||||
|
||||
private protected BaseDiverseSelector(IHostEnvironment env, DiverseSelectorArguments args, string name,
|
||||
IComponentFactory<IDiversityMeasure<TOutput>> diversityMetricType)
|
||||
|
|
|
@ -420,12 +420,12 @@ namespace Microsoft.ML.Trainers.FastTree
|
|||
/// </summary>
|
||||
public sealed class DatasetSkeleton
|
||||
{
|
||||
private short[] _ratings;
|
||||
private readonly short[] _ratings;
|
||||
public readonly int[] Boundaries;
|
||||
public readonly ulong[] QueryIds;
|
||||
public readonly ulong[] DocIds;
|
||||
public double[][] MaxDcg;
|
||||
private int[] _docToQuery;
|
||||
private readonly int[] _docToQuery;
|
||||
|
||||
public Dictionary<string, DatasetSkeletonQueryDocData> AuxiliaryData { get; set; }
|
||||
|
||||
|
|
|
@ -205,7 +205,7 @@ namespace Microsoft.ML.Trainers.FastTree
|
|||
{
|
||||
private const int _bits = 10;
|
||||
private const int _mask = (1 << _bits) - 1;
|
||||
private uint[] _data;
|
||||
private readonly uint[] _data;
|
||||
|
||||
public override IntArrayBits BitsPerItem { get { return IntArrayBits.Bits10; } }
|
||||
|
||||
|
@ -381,7 +381,7 @@ namespace Microsoft.ML.Trainers.FastTree
|
|||
/// 0-bit array only represents the value -1</remarks>
|
||||
internal sealed class Dense8BitIntArray : DenseDataCallbackIntArray
|
||||
{
|
||||
private byte[] _data;
|
||||
private readonly byte[] _data;
|
||||
|
||||
public override IntArrayBits BitsPerItem { get { return IntArrayBits.Bits8; } }
|
||||
|
||||
|
@ -468,7 +468,7 @@ namespace Microsoft.ML.Trainers.FastTree
|
|||
/// <summary>
|
||||
/// For a given byte, the high 4 bits is the first value, the low 4 bits is the next value.
|
||||
/// </summary>
|
||||
private byte[] _data;
|
||||
private readonly byte[] _data;
|
||||
|
||||
public override IntArrayBits BitsPerItem { get { return IntArrayBits.Bits4; } }
|
||||
|
||||
|
@ -588,7 +588,7 @@ namespace Microsoft.ML.Trainers.FastTree
|
|||
/// </summary>
|
||||
internal sealed class Dense16BitIntArray : DenseDataCallbackIntArray
|
||||
{
|
||||
private ushort[] _data;
|
||||
private readonly ushort[] _data;
|
||||
|
||||
public override IntArrayBits BitsPerItem { get { return IntArrayBits.Bits16; } }
|
||||
|
||||
|
@ -677,7 +677,7 @@ namespace Microsoft.ML.Trainers.FastTree
|
|||
/// </summary>
|
||||
internal sealed class Dense32BitIntArray : DenseDataCallbackIntArray
|
||||
{
|
||||
private int[] _data;
|
||||
private readonly int[] _data;
|
||||
|
||||
public override IntArrayBits BitsPerItem { get { return IntArrayBits.Bits32; } }
|
||||
|
||||
|
|
|
@ -19,7 +19,7 @@ namespace Microsoft.ML.Trainers.FastTree
|
|||
{
|
||||
private readonly DenseIntArray _values;
|
||||
private readonly int _length;
|
||||
private byte[] _deltas;
|
||||
private readonly byte[] _deltas;
|
||||
private readonly int _deltasActualLength;
|
||||
|
||||
public DeltaRepeatIntArray(int length, IntArrayBits bitsPerItem, IEnumerable<int> values)
|
||||
|
|
|
@ -2616,7 +2616,7 @@ namespace Microsoft.ML.Trainers.FastTree
|
|||
#if DEBUG
|
||||
// Holds for each feature the row index that it was previously accessed on.
|
||||
// Purely for validation purposes.
|
||||
private int[] _lastRow;
|
||||
private readonly int[] _lastRow;
|
||||
#endif
|
||||
|
||||
/// <summary>
|
||||
|
|
|
@ -140,7 +140,7 @@ namespace Microsoft.ML.Trainers.FastTree
|
|||
internal const string ShortName = "ftc";
|
||||
|
||||
private bool[] _trainSetLabels;
|
||||
private double _sigmoidParameter;
|
||||
private readonly double _sigmoidParameter;
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of <see cref="FastTreeBinaryTrainer"/>
|
||||
|
@ -325,7 +325,7 @@ namespace Microsoft.ML.Trainers.FastTree
|
|||
private readonly bool _unbalancedSets; //Should we use balanced or unbalanced loss function
|
||||
private readonly long _npos;
|
||||
private readonly long _nneg;
|
||||
private IParallelTraining _parallelTraining;
|
||||
private readonly IParallelTraining _parallelTraining;
|
||||
private readonly double _sigmoidParameter; // Parameter for scaling the loss
|
||||
|
||||
public ObjectiveImpl(
|
||||
|
|
|
@ -88,7 +88,7 @@ namespace Microsoft.ML.Trainers.FastTree
|
|||
/// <param name="labelColumnName">The name of the label column.</param>
|
||||
/// <param name="featureColumnName">The name of the feature column.</param>
|
||||
/// <param name="rowGroupColumnName">The name for the column containing the group ID. </param>
|
||||
/// <param name="exampleWeightColumnName">The name for the column containing the examle weight.</param>
|
||||
/// <param name="exampleWeightColumnName">The name for the column containing the example weight.</param>
|
||||
/// <param name="numberOfLeaves">The max number of leaves in each regression tree.</param>
|
||||
/// <param name="numberOfTrees">Total number of decision trees to create in the ensemble.</param>
|
||||
/// <param name="minimumExampleCountPerLeaf">The minimal number of examples allowed in a leaf of a regression tree, out of the subsampled data.</param>
|
||||
|
@ -266,7 +266,7 @@ namespace Microsoft.ML.Trainers.FastTree
|
|||
{
|
||||
if (!FastTreeTrainerOptions.UseTolerantPruning)
|
||||
{
|
||||
//use simple eraly stopping condition
|
||||
//use simple early stopping condition
|
||||
PruningTest = new TestHistory(ValidTest, 0);
|
||||
}
|
||||
else
|
||||
|
@ -363,7 +363,7 @@ namespace Microsoft.ML.Trainers.FastTree
|
|||
|
||||
// We only print non-zero train&valid graph if earlyStoppingTruncation!=0
|
||||
// In case /es is not set, we print 0 for train and valid graph NDCG
|
||||
// Let's keeping this behaviour for backward compatibility with previous FR version
|
||||
// Let's keeping this behavior for backward compatibility with previous FR version
|
||||
// Ideally /graphtv should enforce non-zero /es in the commandline validation
|
||||
if (_specialTrainSetTest != null)
|
||||
{
|
||||
|
@ -385,7 +385,7 @@ namespace Microsoft.ML.Trainers.FastTree
|
|||
base.Train(ch);
|
||||
// Print final last iteration.
|
||||
// Note that trainNDCG printed in graph will be from copy of a value from previous iteration
|
||||
// and will diffre slightly from the proper final value computed by FullTest.
|
||||
// and will differ slightly from the proper final value computed by FullTest.
|
||||
// We cannot compute the final NDCG here due to the fact we use FastNDCGTestForTrainSet computing NDCG based on label sort saved during gradient computation (and we don;t have gradients for n+1 iteration)
|
||||
// Keeping it in sync with original FR code
|
||||
PrintTestGraph(ch);
|
||||
|
@ -507,19 +507,19 @@ namespace Microsoft.ML.Trainers.FastTree
|
|||
private readonly double[] _discount;
|
||||
private readonly int[] _oneTwoThree;
|
||||
|
||||
private int[][] _labelCounts;
|
||||
private readonly int[][] _labelCounts;
|
||||
|
||||
// reusable memory, technical stuff
|
||||
private int[][] _permutationBuffers;
|
||||
private DcgPermutationComparer[] _comparers;
|
||||
private readonly int[][] _permutationBuffers;
|
||||
private readonly DcgPermutationComparer[] _comparers;
|
||||
|
||||
//gains
|
||||
private double[] _gain;
|
||||
private readonly double[] _gain;
|
||||
private double[] _gainLabels;
|
||||
|
||||
// parameters
|
||||
private int _maxDcgTruncationLevel;
|
||||
private bool _useDcg;
|
||||
private readonly int _maxDcgTruncationLevel;
|
||||
private readonly bool _useDcg;
|
||||
// A lookup table for the sigmoid used in the lambda calculation
|
||||
// Note: Is built for a specific sigmoid parameter, so assumes this will be constant throughout computation
|
||||
private double[] _sigmoidTable;
|
||||
|
@ -533,9 +533,9 @@ namespace Microsoft.ML.Trainers.FastTree
|
|||
|
||||
// Secondary gains, currently not used in any way.
|
||||
#pragma warning disable 0649
|
||||
private double _secondaryMetricShare;
|
||||
private double[] _secondaryInverseMaxDcgt;
|
||||
private double[] _secondaryGains;
|
||||
private readonly double _secondaryMetricShare;
|
||||
private readonly double[] _secondaryInverseMaxDcgt;
|
||||
private readonly double[] _secondaryGains;
|
||||
#pragma warning restore 0649
|
||||
|
||||
// Baseline risk.
|
||||
|
@ -546,19 +546,19 @@ namespace Microsoft.ML.Trainers.FastTree
|
|||
// 1. preprocessing the scores for continuous cost function
|
||||
// 2. shifted NDCG
|
||||
// 3. max DCG per query
|
||||
private double[] _scoresCopy;
|
||||
private short[] _labelsCopy;
|
||||
private short[] _groupIdToTopLabel;
|
||||
private readonly double[] _scoresCopy;
|
||||
private readonly short[] _labelsCopy;
|
||||
private readonly short[] _groupIdToTopLabel;
|
||||
|
||||
// parameters
|
||||
private double _sigmoidParam;
|
||||
private char _costFunctionParam;
|
||||
private bool _filterZeroLambdas;
|
||||
private readonly double _sigmoidParam;
|
||||
private readonly char _costFunctionParam;
|
||||
private readonly bool _filterZeroLambdas;
|
||||
|
||||
private bool _distanceWeight2;
|
||||
private bool _normalizeQueryLambdas;
|
||||
private bool _useShiftedNdcg;
|
||||
private IParallelTraining _parallelTraining;
|
||||
private readonly bool _distanceWeight2;
|
||||
private readonly bool _normalizeQueryLambdas;
|
||||
private readonly bool _useShiftedNdcg;
|
||||
private readonly IParallelTraining _parallelTraining;
|
||||
|
||||
// Used for training NDCG calculation
|
||||
// Keeps track of labels of top 3 documents per query
|
||||
|
|
|
@ -204,7 +204,7 @@ namespace Microsoft.ML.Trainers.FastTree
|
|||
Host.CheckParam(options.NumberOfThreads == null || options.NumberOfThreads > 0, nameof(options.NumberOfThreads), "Must be positive.");
|
||||
Host.CheckParam(0 <= options.EntropyCoefficient && options.EntropyCoefficient <= 1, nameof(options.EntropyCoefficient), "Must be in [0, 1].");
|
||||
Host.CheckParam(0 <= options.GainConfidenceLevel && options.GainConfidenceLevel < 1, nameof(options.GainConfidenceLevel), "Must be in [0, 1).");
|
||||
Host.CheckParam(0 < options.MaximumBinCountPerFeature, nameof(options.MaximumBinCountPerFeature), "Must be posittive.");
|
||||
Host.CheckParam(0 < options.MaximumBinCountPerFeature, nameof(options.MaximumBinCountPerFeature), "Must be positive.");
|
||||
Host.CheckParam(0 < options.NumberOfIterations, nameof(options.NumberOfIterations), "Must be positive.");
|
||||
Host.CheckParam(0 < options.MinimumExampleCountPerLeaf, nameof(options.MinimumExampleCountPerLeaf), "Must be positive.");
|
||||
|
||||
|
@ -608,7 +608,7 @@ namespace Microsoft.ML.Trainers.FastTree
|
|||
|
||||
private class LeafSplitHelper : ILeafSplitStatisticsCalculator
|
||||
{
|
||||
private bool _hasWeights;
|
||||
private readonly bool _hasWeights;
|
||||
|
||||
public LeafSplitHelper(bool hasWeights)
|
||||
{
|
||||
|
|
|
@ -18,15 +18,15 @@ namespace Microsoft.ML.Trainers.FastTree
|
|||
private readonly int[] _oneTwoThree;
|
||||
|
||||
// reusable memory
|
||||
private int[][] _permutationBuffers;
|
||||
private double[][] _scoreBuffers;
|
||||
private DcgPermutationComparer[] _comparers;
|
||||
private readonly int[][] _permutationBuffers;
|
||||
private readonly double[][] _scoreBuffers;
|
||||
private readonly DcgPermutationComparer[] _comparers;
|
||||
|
||||
/// <summary>
|
||||
/// Contains the instances for a second Level comparer, which gets applied after the initial rank
|
||||
/// based ordering has happened. The array stores one second level comparer per thread.
|
||||
/// </summary>
|
||||
private DescendingStableIdealComparer[] _secondLevelcomparers;
|
||||
private readonly DescendingStableIdealComparer[] _secondLevelcomparers;
|
||||
|
||||
private double _result;
|
||||
|
||||
|
|
|
@ -22,7 +22,7 @@ namespace Microsoft.ML.Trainers.FastTree
|
|||
private readonly int[] _leafCount;
|
||||
private readonly int[] _documents;
|
||||
private int[] _tempDocuments;
|
||||
private int[] _initialDocuments;
|
||||
private readonly int[] _initialDocuments;
|
||||
|
||||
/// <summary>
|
||||
/// Constructor
|
||||
|
|
|
@ -33,7 +33,7 @@ namespace Microsoft.ML.Trainers.FastTree
|
|||
// Total number of passes over data
|
||||
public int NumberOfPasses;
|
||||
|
||||
private int _numFeatures;
|
||||
private readonly int _numFeatures;
|
||||
|
||||
public LassoFit(int numberOfLambdas, int maxAllowedFeaturesAlongPath, int numFeatures)
|
||||
{
|
||||
|
|
|
@ -11,14 +11,14 @@ namespace Microsoft.ML.Trainers.FastTree
|
|||
{
|
||||
internal class GradientDescent : OptimizationAlgorithm
|
||||
{
|
||||
private IGradientAdjuster _gradientWrapper;
|
||||
private readonly IGradientAdjuster _gradientWrapper;
|
||||
|
||||
/// number of trees dropped in this iteration
|
||||
private int _numberOfDroppedTrees;
|
||||
// treeScores stores for every tree the predictions it makes on every training example. This is used
|
||||
// to eliminate the need for computing the scores when we drop trees. However, it causes a horrifying
|
||||
// memory drain.
|
||||
private List<double[]> _treeScores;
|
||||
private readonly List<double[]> _treeScores;
|
||||
private double[] _droppedScores;
|
||||
private double[] _scores;
|
||||
|
||||
|
|
|
@ -11,8 +11,8 @@ namespace Microsoft.ML.Trainers.FastTree
|
|||
/// </summary>
|
||||
internal class RandomForestOptimizer : GradientDescent
|
||||
{
|
||||
private IGradientAdjuster _gradientWrapper;
|
||||
// REVIEW: When the FastTree appliation is decoupled with tree learner and boosting logic, this class should be removed.
|
||||
private readonly IGradientAdjuster _gradientWrapper;
|
||||
// REVIEW: When the FastTree application is decoupled with tree learner and boosting logic, this class should be removed.
|
||||
internal RandomForestOptimizer(InternalTreeEnsemble ensemble, Dataset trainData, double[] initTrainScores, IGradientAdjuster gradientWrapper)
|
||||
: base(ensemble, trainData, initTrainScores, gradientWrapper)
|
||||
{
|
||||
|
|
|
@ -135,13 +135,13 @@ namespace Microsoft.ML.Trainers.FastTree
|
|||
throw Contracts.ExceptNotSupp("This code should not be reachable");
|
||||
}
|
||||
|
||||
//Computes AGD specific mutiplier. Given that we have tree number t in ensemble (we count trees starting from 0)
|
||||
//Computes AGD specific multiplier. Given that we have tree number t in ensemble (we count trees starting from 0)
|
||||
//And we have total k trees in ensemble, what should be the multiplier on the tree when sum the ensemble together based on AGD formula being
|
||||
//X[k+1] = Y[k] + Tree[k]
|
||||
//Y[k+1] = X[k+1] + C[k] * (X[k+1] – X[k])
|
||||
//C[k] = (k-1) / (k+2)
|
||||
|
||||
private static Dictionary<int, Dictionary<int, double>> _treeMultiplierMap = new Dictionary<int, Dictionary<int, double>>();
|
||||
private static readonly Dictionary<int, Dictionary<int, double>> _treeMultiplierMap = new Dictionary<int, Dictionary<int, double>>();
|
||||
public static double TreeMultiplier(int t, int k)
|
||||
{
|
||||
if (_treeMultiplierMap.ContainsKey(t))
|
||||
|
|
|
@ -16,8 +16,8 @@ namespace Microsoft.ML.Trainers.FastTree
|
|||
internal sealed class LineSearch : IStepSearch, IFastTrainingScoresUpdate
|
||||
{
|
||||
private double _historicStepSize;
|
||||
private int _numPostbracketSteps;
|
||||
private double _minStepSize;
|
||||
private readonly int _numPostbracketSteps;
|
||||
private readonly double _minStepSize;
|
||||
|
||||
public LineSearch(Test lossCalculator, int lossIndex, int numPostbracketSteps, double minStepSize)
|
||||
: this(lossCalculator, lossIndex) { _numPostbracketSteps = numPostbracketSteps; _minStepSize = minStepSize; }
|
||||
|
|
|
@ -273,11 +273,11 @@ namespace Microsoft.ML.Trainers.FastTree
|
|||
private readonly int _windowSize;
|
||||
private readonly double _tolerance;
|
||||
// Queue for moving window
|
||||
private LinkedList<double> _window;
|
||||
private readonly LinkedList<double> _window;
|
||||
|
||||
// This queue keeps track of the iterations which are within tolerance from the best iteration
|
||||
// The first element of the queue is the early stopping candidate
|
||||
private LinkedList<ValueIterationPair> _toleratedQueue;
|
||||
private readonly LinkedList<ValueIterationPair> _toleratedQueue;
|
||||
|
||||
// Average validation for the current window
|
||||
private double _currentWindowSum;
|
||||
|
|
|
@ -9,8 +9,8 @@ namespace Microsoft.ML.Trainers.FastTree
|
|||
{
|
||||
internal class RandomForestLeastSquaresTreeLearner : LeastSquaresRegressionTreeLearner
|
||||
{
|
||||
private int _quantileSampleCount;
|
||||
private bool _quantileEnabled;
|
||||
private readonly int _quantileSampleCount;
|
||||
private readonly bool _quantileEnabled;
|
||||
|
||||
public RandomForestLeastSquaresTreeLearner(Dataset trainData, int numLeaves, int minDocsInLeaf, Double entropyCoefficient, Double featureFirstUsePenalty,
|
||||
Double featureReusePenalty, Double softmaxTemperature, int histogramPoolSize, int randomSeed, Double splitFraction, bool allowEmptyTrees,
|
||||
|
|
|
@ -39,7 +39,7 @@ namespace Microsoft.ML.Trainers.FastTree
|
|||
public readonly double Bias;
|
||||
|
||||
// Multithread task to find best threshold.
|
||||
private IThreadTask _calculateLeafSplitCandidates;
|
||||
private readonly IThreadTask _calculateLeafSplitCandidates;
|
||||
|
||||
protected SplitInfo[] BestSplitInfoPerLeaf;
|
||||
protected HashSet<int> CategoricalThresholds;
|
||||
|
@ -74,7 +74,7 @@ namespace Microsoft.ML.Trainers.FastTree
|
|||
// size of reserved memory
|
||||
private readonly long _sizeOfReservedMemory;
|
||||
|
||||
private IParallelTraining _parallelTraining;
|
||||
private readonly IParallelTraining _parallelTraining;
|
||||
|
||||
public int MaxCategoricalGroupsPerNode { get; }
|
||||
|
||||
|
@ -828,7 +828,7 @@ namespace Microsoft.ML.Trainers.FastTree
|
|||
private double _sumWeights;
|
||||
private double _sumSquaredTargets;
|
||||
private int[] _docIndices;
|
||||
private int[] _docIndicesCopy;
|
||||
private readonly int[] _docIndicesCopy;
|
||||
public readonly FloatType[] Targets;
|
||||
public readonly double[] Weights;
|
||||
public readonly SplitInfo[] FeatureSplitInfo;
|
||||
|
|
|
@ -23,12 +23,12 @@ namespace Microsoft.ML.Trainers.FastTree
|
|||
internal class InternalRegressionTree
|
||||
{
|
||||
private double _maxOutput;
|
||||
private double[] _splitGain;
|
||||
private double[] _gainPValue;
|
||||
private readonly double[] _splitGain;
|
||||
private readonly double[] _gainPValue;
|
||||
/// <summary>
|
||||
/// The value of this non-leaf node, prior to split when it was a leaf.
|
||||
/// </summary>
|
||||
private double[] _previousLeafValue;
|
||||
private readonly double[] _previousLeafValue;
|
||||
|
||||
// for each non-leaf, we keep the following data
|
||||
public float[] DefaultValueForMissing;
|
||||
|
@ -820,7 +820,7 @@ namespace Microsoft.ML.Trainers.FastTree
|
|||
|
||||
private float GetFeatureValue(float x, int node)
|
||||
{
|
||||
// Not need to convert missing vaules.
|
||||
// Not need to convert missing values.
|
||||
if (DefaultValueForMissing == null)
|
||||
return x;
|
||||
|
||||
|
|
|
@ -190,7 +190,7 @@ namespace Microsoft.ML.Trainers.FastTree
|
|||
public TreeEnsembleModelParameters ModelParameters;
|
||||
};
|
||||
|
||||
private TreeEnsembleModelParameters _modelParameters;
|
||||
private readonly TreeEnsembleModelParameters _modelParameters;
|
||||
|
||||
internal PretrainedTreeFeaturizationEstimator(IHostEnvironment env, Options options) : base(env, options)
|
||||
{
|
||||
|
|
|
@ -84,14 +84,14 @@ namespace Microsoft.ML.Data
|
|||
/// and the i-th vector element is the prediction value predicted by the i-th tree.
|
||||
/// If <see cref="_treesColumnName"/> is <see langword="null"/>, this output column may not be generated.
|
||||
/// </summary>
|
||||
private string _treesColumnName;
|
||||
private readonly string _treesColumnName;
|
||||
|
||||
/// <summary>
|
||||
/// The 0-1 encoding of all leaf nodes' IDs. Its type is a vector of <see cref="System.Single"/>. If the given feature
|
||||
/// vector falls into the first leaf of the first tree, the first element in the 0-1 encoding would be 1.
|
||||
/// If <see cref="_leavesColumnName"/> is <see langword="null"/>, this output column may not be generated.
|
||||
/// </summary>
|
||||
private string _leavesColumnName;
|
||||
private readonly string _leavesColumnName;
|
||||
|
||||
/// <summary>
|
||||
/// The 0-1 encoding of the paths to the leaves. If the path to the first tree's leaf is node 1 (2nd node in the first tree),
|
||||
|
@ -99,7 +99,7 @@ namespace Microsoft.ML.Data
|
|||
/// would be 1.
|
||||
/// If <see cref="_pathsColumnName"/> is <see langword="null"/>, this output column may not be generated.
|
||||
/// </summary>
|
||||
private string _pathsColumnName;
|
||||
private readonly string _pathsColumnName;
|
||||
|
||||
public BoundMapper(IExceptionContext ectx, TreeEnsembleFeaturizerBindableMapper owner, RoleMappedSchema schema,
|
||||
string treesColumnName, string leavesColumnName, string pathsColumnName)
|
||||
|
@ -230,7 +230,7 @@ namespace Microsoft.ML.Data
|
|||
private readonly int _numLeaves;
|
||||
|
||||
private VBuffer<float> _src;
|
||||
private ValueGetter<VBuffer<float>> _featureGetter;
|
||||
private readonly ValueGetter<VBuffer<float>> _featureGetter;
|
||||
private long _cachedPosition;
|
||||
private readonly int[] _leafIds;
|
||||
private readonly List<int>[] _pathIds;
|
||||
|
|
|
@ -27,7 +27,7 @@ namespace Microsoft.ML.Trainers.FastTree
|
|||
/// <summary>
|
||||
/// A dictionary containing all buffer pool types
|
||||
/// </summary>
|
||||
private static ConcurrentDictionary<Type, SortedList<int, List<Array>>> _bufferPools = new ConcurrentDictionary<Type, SortedList<int, List<Array>>>();
|
||||
private static readonly ConcurrentDictionary<Type, SortedList<int, List<Array>>> _bufferPools = new ConcurrentDictionary<Type, SortedList<int, List<Array>>>();
|
||||
|
||||
/// <summary>
|
||||
/// Gets a buffer from the pool with at least the same size as passed as input parameter
|
||||
|
|
|
@ -12,10 +12,10 @@ namespace Microsoft.ML.Trainers.FastTree
|
|||
/// </summary>
|
||||
internal class MappedObjectPool<T> where T : class
|
||||
{
|
||||
private T[] _pool;
|
||||
private int[] _map;
|
||||
private int[] _inverseMap;
|
||||
private int[] _lastAccessTime;
|
||||
private readonly T[] _pool;
|
||||
private readonly int[] _map;
|
||||
private readonly int[] _inverseMap;
|
||||
private readonly int[] _lastAccessTime;
|
||||
private int _time;
|
||||
|
||||
/// <summary>
|
||||
|
|
|
@ -12,7 +12,7 @@ namespace Microsoft.ML.Trainers.FastTree
|
|||
{
|
||||
internal static class ThreadTaskManager
|
||||
{
|
||||
private static object _lockObject = new object();
|
||||
private static readonly object _lockObject = new object();
|
||||
|
||||
// REVIEW: Should this bother with number of threads? What should it do?
|
||||
public static int NumThreads { get; private set; }
|
||||
|
|
|
@ -1,4 +1,8 @@
|
|||
using System;
|
||||
// Licensed to the .NET Foundation under one or more agreements.
|
||||
// The .NET Foundation licenses this file to you under the MIT license.
|
||||
// See the LICENSE file in the project root for more information.
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics;
|
||||
using System.Runtime.InteropServices;
|
||||
|
@ -117,7 +121,7 @@ namespace Microsoft.ML.Featurizers
|
|||
internal delegate bool DestroyTransformedDataNative(IntPtr output, out IntPtr errorHandle);
|
||||
internal class TransformedDataSafeHandle : SafeHandleZeroOrMinusOneIsInvalid
|
||||
{
|
||||
private DestroyTransformedDataNative _destroySaveDataHandler;
|
||||
private readonly DestroyTransformedDataNative _destroySaveDataHandler;
|
||||
|
||||
public TransformedDataSafeHandle(IntPtr handle, DestroyTransformedDataNative destroyCppTransformerEstimator) : base(true)
|
||||
{
|
||||
|
@ -137,7 +141,7 @@ namespace Microsoft.ML.Featurizers
|
|||
internal delegate bool DestroyNativeTransformerEstimator(IntPtr estimator, out IntPtr errorHandle);
|
||||
internal class TransformerEstimatorSafeHandle : SafeHandleZeroOrMinusOneIsInvalid
|
||||
{
|
||||
private DestroyNativeTransformerEstimator _destroyNativeTransformerEstimator;
|
||||
private readonly DestroyNativeTransformerEstimator _destroyNativeTransformerEstimator;
|
||||
public TransformerEstimatorSafeHandle(IntPtr handle, DestroyNativeTransformerEstimator destroyNativeTransformerEstimator) : base(true)
|
||||
{
|
||||
SetHandle(handle);
|
||||
|
@ -169,7 +173,7 @@ namespace Microsoft.ML.Featurizers
|
|||
|
||||
protected override bool ReleaseHandle()
|
||||
{
|
||||
// Not sure what to do with error stuff here. There shoudln't ever be one though.
|
||||
// Not sure what to do with error stuff here. There shouldn't ever be one though.
|
||||
return DestroyTransformerSaveDataNative(handle, _dataSize, out _);
|
||||
}
|
||||
}
|
||||
|
|
|
@ -245,8 +245,8 @@ namespace Microsoft.ML.Featurizers
|
|||
internal const string ShortName = "DateTimeTransform";
|
||||
internal const string LoadName = "DateTimeTransform";
|
||||
internal const string LoaderSignature = "DateTimeTransform";
|
||||
private LongTypedColumn _column;
|
||||
private DataViewSchema _schema;
|
||||
private readonly LongTypedColumn _column;
|
||||
private readonly DataViewSchema _schema;
|
||||
|
||||
#endregion
|
||||
|
||||
|
@ -554,7 +554,7 @@ namespace Microsoft.ML.Featurizers
|
|||
|
||||
internal abstract void CreateTransformerFromEstimator(DateTimeEstimator.HolidayList country);
|
||||
private protected abstract unsafe void CreateTransformerFromSavedDataHelper(byte* rawData, IntPtr dataSize);
|
||||
private protected unsafe abstract bool CreateEstimatorHelper(byte* countryName, byte* dataRootDir, out IntPtr estimator, out IntPtr errorHandle);
|
||||
private protected abstract unsafe bool CreateEstimatorHelper(byte* countryName, byte* dataRootDir, out IntPtr estimator, out IntPtr errorHandle);
|
||||
private protected abstract bool CreateTransformerFromEstimatorHelper(TransformerEstimatorSafeHandle estimator, out IntPtr transformer, out IntPtr errorHandle);
|
||||
private protected abstract bool DestroyEstimatorHelper(IntPtr estimator, out IntPtr errorHandle);
|
||||
private protected abstract bool DestroyTransformerHelper(IntPtr transformer, out IntPtr errorHandle);
|
||||
|
@ -668,7 +668,7 @@ namespace Microsoft.ML.Featurizers
|
|||
}
|
||||
|
||||
[DllImport("Featurizers", EntryPoint = "DateTimeFeaturizer_CreateEstimator"), SuppressUnmanagedCodeSecurity]
|
||||
private static unsafe extern bool CreateEstimatorNative(byte* countryName, byte* dataRootDir, out IntPtr estimator, out IntPtr errorHandle);
|
||||
private static extern unsafe bool CreateEstimatorNative(byte* countryName, byte* dataRootDir, out IntPtr estimator, out IntPtr errorHandle);
|
||||
|
||||
[DllImport("Featurizers", EntryPoint = "DateTimeFeaturizer_DestroyEstimator"), SuppressUnmanagedCodeSecurity]
|
||||
private static extern bool DestroyEstimatorNative(IntPtr estimator, out IntPtr errorHandle); // Should ONLY be called by safe handle
|
||||
|
@ -683,7 +683,7 @@ namespace Microsoft.ML.Featurizers
|
|||
}
|
||||
|
||||
[DllImport("Featurizers", EntryPoint = "DateTimeFeaturizer_CreateTransformerFromSavedDataWithDataRoot"), SuppressUnmanagedCodeSecurity]
|
||||
private static unsafe extern bool CreateTransformerFromSavedDataNative(byte* rawData, IntPtr bufferSize, byte* dataRootDir, out IntPtr transformer, out IntPtr errorHandle);
|
||||
private static extern unsafe bool CreateTransformerFromSavedDataNative(byte* rawData, IntPtr bufferSize, byte* dataRootDir, out IntPtr transformer, out IntPtr errorHandle);
|
||||
private protected override unsafe void CreateTransformerFromSavedDataHelper(byte* rawData, IntPtr dataSize)
|
||||
{
|
||||
byte[] dataRoot;
|
||||
|
@ -748,7 +748,7 @@ namespace Microsoft.ML.Featurizers
|
|||
_transformerHandler.Dispose();
|
||||
}
|
||||
|
||||
private protected unsafe override bool CreateEstimatorHelper(byte* countryName, byte* dataRootDir, out IntPtr estimator, out IntPtr errorHandle) =>
|
||||
private protected override unsafe bool CreateEstimatorHelper(byte* countryName, byte* dataRootDir, out IntPtr estimator, out IntPtr errorHandle) =>
|
||||
CreateEstimatorNative(countryName, dataRootDir, out estimator, out errorHandle);
|
||||
|
||||
private protected override bool CreateTransformerFromEstimatorHelper(TransformerEstimatorSafeHandle estimator, out IntPtr transformer, out IntPtr errorHandle) =>
|
||||
|
|
|
@ -1,4 +1,8 @@
|
|||
using System;
|
||||
// Licensed to the .NET Foundation under one or more agreements.
|
||||
// The .NET Foundation licenses this file to you under the MIT license.
|
||||
// See the LICENSE file in the project root for more information.
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics;
|
||||
using System.IO;
|
||||
|
@ -84,7 +88,7 @@ namespace Microsoft.ML.Featurizers
|
|||
/// boolean value representing if the row was created as a result of this operation or not.
|
||||
///
|
||||
/// The imputation strategies that are currently supported are ForwardFill, where the last good value is propagated forward, Backfill, where the next good value is propagated backwards,
|
||||
/// and Median, where the mathmatical median is used to fill in missing values.
|
||||
/// and Median, where the mathematical median is used to fill in missing values.
|
||||
///
|
||||
/// NOTE: It is not recommended to chain this multiple times. If a column is filtered, the default value is placed when a row is imputed, and the
|
||||
/// default value is not null. Thus any other TimeSeriesImputers will not be able to replace those values anymore causing essentially a very
|
||||
|
@ -97,7 +101,7 @@ namespace Microsoft.ML.Featurizers
|
|||
/// <seealso cref="TimeSeriesImputerExtensionClass.ReplaceMissingTimeSeriesValues(TransformsCatalog, string, string[], string[], FilterMode, ImputationStrategy, bool)"/>
|
||||
public sealed class TimeSeriesImputerEstimator : IEstimator<TimeSeriesImputerTransformer>
|
||||
{
|
||||
private Options _options;
|
||||
private readonly Options _options;
|
||||
internal const string IsRowImputedColumnName = "IsRowImputed";
|
||||
|
||||
private readonly IHost _host;
|
||||
|
@ -158,14 +162,14 @@ namespace Microsoft.ML.Featurizers
|
|||
|
||||
/// <summary>
|
||||
/// Method by which columns are selected for imputing values.
|
||||
/// NoFilter takes all of the columns so you dont have to specify anything.
|
||||
/// NoFilter takes all of the columns so you don't have to specify anything.
|
||||
/// Include only does the specified ImputationStrategy on the columns you specify. The other columns will get a default value.
|
||||
/// Exclude is the exact opposite of Include, and does the ImputationStrategy on all columns but the ones you specify, which will get the default value.
|
||||
/// </summary>
|
||||
public enum FilterMode : byte
|
||||
{
|
||||
/// <summary>
|
||||
/// Takes all of the columns so you dont have to specify anything.
|
||||
/// Takes all of the columns so you don't have to specify anything.
|
||||
/// </summary>
|
||||
NoFilter = 1,
|
||||
|
||||
|
@ -458,7 +462,7 @@ namespace Microsoft.ML.Featurizers
|
|||
if (!success)
|
||||
throw new Exception(GetErrorDetailsAndFreeNativeMemory(errorHandle));
|
||||
|
||||
// Manually dispose of the IEnumerator since we dont have a using statement;
|
||||
// Manually dispose of the IEnumerator since we don't have a using statement;
|
||||
cursor.Dispose();
|
||||
|
||||
return new TransformerEstimatorSafeHandle(transformer, DestroyTransformerNative);
|
||||
|
@ -577,7 +581,7 @@ namespace Microsoft.ML.Featurizers
|
|||
// TODO: Update entry points
|
||||
|
||||
[DllImport("Featurizers", EntryPoint = "TimeSeriesImputerFeaturizer_BinaryArchive_CreateEstimator", CallingConvention = CallingConvention.Cdecl), SuppressUnmanagedCodeSecurity]
|
||||
private static unsafe extern bool CreateEstimatorNative(TypeId* grainTypes, IntPtr grainTypesSize, TypeId* dataTypes, IntPtr dataTypesSize, TimeSeriesImputerEstimator.ImputationStrategy strategy, bool* suppressTypeErrors, out IntPtr estimator, out IntPtr errorHandle);
|
||||
private static extern unsafe bool CreateEstimatorNative(TypeId* grainTypes, IntPtr grainTypesSize, TypeId* dataTypes, IntPtr dataTypesSize, TimeSeriesImputerEstimator.ImputationStrategy strategy, bool* suppressTypeErrors, out IntPtr estimator, out IntPtr errorHandle);
|
||||
|
||||
[DllImport("Featurizers", EntryPoint = "TimeSeriesImputerFeaturizer_BinaryArchive_DestroyEstimator", CallingConvention = CallingConvention.Cdecl), SuppressUnmanagedCodeSecurity]
|
||||
private static extern bool DestroyEstimatorNative(IntPtr estimator, out IntPtr errorHandle); // Should ONLY be called by safe handle
|
||||
|
@ -589,7 +593,7 @@ namespace Microsoft.ML.Featurizers
|
|||
private static extern bool CompleteTrainingNative(TransformerEstimatorSafeHandle estimator, out IntPtr errorHandle);
|
||||
|
||||
[DllImport("Featurizers", EntryPoint = "TimeSeriesImputerFeaturizer_BinaryArchive_Fit", CallingConvention = CallingConvention.Cdecl), SuppressUnmanagedCodeSecurity]
|
||||
private static unsafe extern bool FitNative(TransformerEstimatorSafeHandle estimator, NativeBinaryArchiveData data, out FitResult fitResult, out IntPtr errorHandle);
|
||||
private static extern unsafe bool FitNative(TransformerEstimatorSafeHandle estimator, NativeBinaryArchiveData data, out FitResult fitResult, out IntPtr errorHandle);
|
||||
|
||||
[DllImport("Featurizers", EntryPoint = "TimeSeriesImputerFeaturizer_BinaryArchive_CreateTransformerFromEstimator", CallingConvention = CallingConvention.Cdecl), SuppressUnmanagedCodeSecurity]
|
||||
private static extern bool CreateTransformerFromEstimatorNative(TransformerEstimatorSafeHandle estimator, out IntPtr transformer, out IntPtr errorHandle);
|
||||
|
@ -601,13 +605,13 @@ namespace Microsoft.ML.Featurizers
|
|||
private static extern bool CreateOnnxSaveDataNative(TransformerEstimatorSafeHandle transformer, out IntPtr buffer, out IntPtr bufferSize, out IntPtr error);
|
||||
|
||||
[DllImport("Featurizers", EntryPoint = "TimeSeriesImputerFeaturizer_BinaryArchive_CreateTransformerFromSavedData"), SuppressUnmanagedCodeSecurity]
|
||||
private static unsafe extern bool CreateTransformerFromSavedDataNative(byte* rawData, IntPtr bufferSize, out IntPtr transformer, out IntPtr errorHandle);
|
||||
private static extern unsafe bool CreateTransformerFromSavedDataNative(byte* rawData, IntPtr bufferSize, out IntPtr transformer, out IntPtr errorHandle);
|
||||
|
||||
[DllImport("Featurizers", EntryPoint = "TimeSeriesImputerFeaturizer_BinaryArchive_OnDataCompleted"), SuppressUnmanagedCodeSecurity]
|
||||
private static unsafe extern bool OnDataCompletedNative(TransformerEstimatorSafeHandle estimator, out IntPtr errorHandle);
|
||||
private static extern unsafe bool OnDataCompletedNative(TransformerEstimatorSafeHandle estimator, out IntPtr errorHandle);
|
||||
|
||||
[DllImport("Featurizers", EntryPoint = "TimeSeriesImputerFeaturizer_BinaryArchive_GetState"), SuppressUnmanagedCodeSecurity]
|
||||
private static unsafe extern bool GetStateNative(TransformerEstimatorSafeHandle estimator, out TrainingState trainingState, out IntPtr errorHandle);
|
||||
private static extern unsafe bool GetStateNative(TransformerEstimatorSafeHandle estimator, out TrainingState trainingState, out IntPtr errorHandle);
|
||||
|
||||
#endregion
|
||||
|
||||
|
|
|
@ -22,7 +22,7 @@ namespace Microsoft.ML.Transforms
|
|||
internal sealed class TimeSeriesImputerDataView : IDataTransform
|
||||
{
|
||||
#region Typed Columns
|
||||
private TimeSeriesImputerTransformer _parent;
|
||||
private readonly TimeSeriesImputerTransformer _parent;
|
||||
public class SharedColumnState
|
||||
{
|
||||
public SharedColumnState()
|
||||
|
@ -269,7 +269,7 @@ namespace Microsoft.ML.Transforms
|
|||
return typeof(T).GetNativeTypeIdFromType();
|
||||
}
|
||||
|
||||
internal unsafe abstract T GetDataFromNativeBinaryArchiveData(byte* data, int offset);
|
||||
internal abstract unsafe T GetDataFromNativeBinaryArchiveData(byte* data, int offset);
|
||||
}
|
||||
|
||||
private abstract class NumericTypedColumn<T> : TypedColumn<T>
|
||||
|
@ -308,7 +308,7 @@ namespace Microsoft.ML.Transforms
|
|||
{
|
||||
}
|
||||
|
||||
internal unsafe override byte GetDataFromNativeBinaryArchiveData(byte* data, int offset)
|
||||
internal override unsafe byte GetDataFromNativeBinaryArchiveData(byte* data, int offset)
|
||||
{
|
||||
if (IsNullable)
|
||||
{
|
||||
|
@ -329,7 +329,7 @@ namespace Microsoft.ML.Transforms
|
|||
{
|
||||
}
|
||||
|
||||
internal unsafe override sbyte GetDataFromNativeBinaryArchiveData(byte* data, int offset)
|
||||
internal override unsafe sbyte GetDataFromNativeBinaryArchiveData(byte* data, int offset)
|
||||
{
|
||||
if (IsNullable)
|
||||
{
|
||||
|
@ -350,7 +350,7 @@ namespace Microsoft.ML.Transforms
|
|||
{
|
||||
}
|
||||
|
||||
internal unsafe override short GetDataFromNativeBinaryArchiveData(byte* data, int offset)
|
||||
internal override unsafe short GetDataFromNativeBinaryArchiveData(byte* data, int offset)
|
||||
{
|
||||
if (IsNullable)
|
||||
{
|
||||
|
@ -371,7 +371,7 @@ namespace Microsoft.ML.Transforms
|
|||
{
|
||||
}
|
||||
|
||||
internal unsafe override ushort GetDataFromNativeBinaryArchiveData(byte* data, int offset)
|
||||
internal override unsafe ushort GetDataFromNativeBinaryArchiveData(byte* data, int offset)
|
||||
{
|
||||
if (IsNullable)
|
||||
{
|
||||
|
@ -392,7 +392,7 @@ namespace Microsoft.ML.Transforms
|
|||
{
|
||||
}
|
||||
|
||||
internal unsafe override int GetDataFromNativeBinaryArchiveData(byte* data, int offset)
|
||||
internal override unsafe int GetDataFromNativeBinaryArchiveData(byte* data, int offset)
|
||||
{
|
||||
if (IsNullable)
|
||||
{
|
||||
|
@ -413,7 +413,7 @@ namespace Microsoft.ML.Transforms
|
|||
{
|
||||
}
|
||||
|
||||
internal unsafe override uint GetDataFromNativeBinaryArchiveData(byte* data, int offset)
|
||||
internal override unsafe uint GetDataFromNativeBinaryArchiveData(byte* data, int offset)
|
||||
{
|
||||
if (IsNullable)
|
||||
{
|
||||
|
@ -434,7 +434,7 @@ namespace Microsoft.ML.Transforms
|
|||
{
|
||||
}
|
||||
|
||||
internal unsafe override long GetDataFromNativeBinaryArchiveData(byte* data, int offset)
|
||||
internal override unsafe long GetDataFromNativeBinaryArchiveData(byte* data, int offset)
|
||||
{
|
||||
if (IsNullable)
|
||||
{
|
||||
|
@ -455,7 +455,7 @@ namespace Microsoft.ML.Transforms
|
|||
{
|
||||
}
|
||||
|
||||
internal unsafe override ulong GetDataFromNativeBinaryArchiveData(byte* data, int offset)
|
||||
internal override unsafe ulong GetDataFromNativeBinaryArchiveData(byte* data, int offset)
|
||||
{
|
||||
if (IsNullable)
|
||||
{
|
||||
|
@ -476,7 +476,7 @@ namespace Microsoft.ML.Transforms
|
|||
{
|
||||
}
|
||||
|
||||
internal unsafe override float GetDataFromNativeBinaryArchiveData(byte* data, int offset)
|
||||
internal override unsafe float GetDataFromNativeBinaryArchiveData(byte* data, int offset)
|
||||
{
|
||||
var bytes = new byte[sizeof(float)];
|
||||
Marshal.Copy((IntPtr)(data + offset), bytes, 0, sizeof(float));
|
||||
|
@ -491,7 +491,7 @@ namespace Microsoft.ML.Transforms
|
|||
{
|
||||
}
|
||||
|
||||
internal unsafe override double GetDataFromNativeBinaryArchiveData(byte* data, int offset)
|
||||
internal override unsafe double GetDataFromNativeBinaryArchiveData(byte* data, int offset)
|
||||
{
|
||||
var bytes = new byte[sizeof(double)];
|
||||
Marshal.Copy((IntPtr)(data + offset), bytes, 0, sizeof(double));
|
||||
|
@ -506,7 +506,7 @@ namespace Microsoft.ML.Transforms
|
|||
{
|
||||
}
|
||||
|
||||
internal unsafe override bool GetDataFromNativeBinaryArchiveData(byte* data, int offset)
|
||||
internal override unsafe bool GetDataFromNativeBinaryArchiveData(byte* data, int offset)
|
||||
{
|
||||
if (IsNullable)
|
||||
{
|
||||
|
@ -552,7 +552,7 @@ namespace Microsoft.ML.Transforms
|
|||
binaryWriter.Write(stringBytes);
|
||||
}
|
||||
|
||||
internal unsafe override ReadOnlyMemory<char> GetDataFromNativeBinaryArchiveData(byte* data, int offset)
|
||||
internal override unsafe ReadOnlyMemory<char> GetDataFromNativeBinaryArchiveData(byte* data, int offset)
|
||||
{
|
||||
if (_isNullable)
|
||||
{
|
||||
|
@ -608,7 +608,7 @@ namespace Microsoft.ML.Transforms
|
|||
binaryWriter.Write(value);
|
||||
}
|
||||
|
||||
internal unsafe override DateTime GetDataFromNativeBinaryArchiveData(byte* data, int offset)
|
||||
internal override unsafe DateTime GetDataFromNativeBinaryArchiveData(byte* data, int offset)
|
||||
{
|
||||
long value;
|
||||
if (_isNullable)
|
||||
|
@ -656,7 +656,7 @@ namespace Microsoft.ML.Transforms
|
|||
|
||||
internal class TransformedDataSafeHandle : SafeHandleZeroOrMinusOneIsInvalid
|
||||
{
|
||||
private IntPtr _size;
|
||||
private readonly IntPtr _size;
|
||||
public TransformedDataSafeHandle(IntPtr handle, IntPtr size) : base(true)
|
||||
{
|
||||
SetHandle(handle);
|
||||
|
@ -728,7 +728,7 @@ namespace Microsoft.ML.Transforms
|
|||
private sealed class Cursor : DataViewRowCursor
|
||||
{
|
||||
private readonly IChannelProvider _ch;
|
||||
private DataViewRowCursor _input;
|
||||
private readonly DataViewRowCursor _input;
|
||||
private long _position;
|
||||
private bool _isGood;
|
||||
private readonly Dictionary<string, TypedColumn> _allColumns;
|
||||
|
|
|
@ -160,7 +160,7 @@ namespace Microsoft.ML.Transforms.Image
|
|||
|
||||
private sealed class Mapper : OneToOneMapperBase
|
||||
{
|
||||
private ImageGrayscalingTransformer _parent;
|
||||
private readonly ImageGrayscalingTransformer _parent;
|
||||
|
||||
public Mapper(ImageGrayscalingTransformer parent, DataViewSchema inputSchema)
|
||||
: base(parent.Host.Register(nameof(Mapper)), parent, inputSchema)
|
||||
|
|
|
@ -49,8 +49,8 @@ namespace Microsoft.ML.Transforms.Onnx
|
|||
/// </summary>
|
||||
public sealed class OnnxMapTypeAttribute : DataViewTypeAttribute
|
||||
{
|
||||
private Type _keyType;
|
||||
private Type _valueType;
|
||||
private readonly Type _keyType;
|
||||
private readonly Type _valueType;
|
||||
|
||||
/// <summary>
|
||||
/// Create a map (aka dictionary) type.
|
||||
|
|
|
@ -55,7 +55,7 @@ namespace Microsoft.ML.Transforms.Onnx
|
|||
/// </summary>
|
||||
public sealed class OnnxSequenceTypeAttribute : DataViewTypeAttribute
|
||||
{
|
||||
private Type _elemType;
|
||||
private readonly Type _elemType;
|
||||
|
||||
// Make default constructor obsolete.
|
||||
// Use default constructor will left the _elemType field empty and cause exception in methods using _elemType.
|
||||
|
|
|
@ -91,7 +91,7 @@ namespace Microsoft.ML.Transforms.Onnx
|
|||
{
|
||||
var index = OutputNames.IndexOf(name);
|
||||
if (index < 0)
|
||||
throw Contracts.ExceptParamValue(name, nameof(name), $"Onput tensor, {name}, does not exist in the ONNX model. " +
|
||||
throw Contracts.ExceptParamValue(name, nameof(name), $"Ouput tensor, {name}, does not exist in the ONNX model. " +
|
||||
$"Available output names are [{string.Join(",", OutputNames)}].");
|
||||
return OutputsInfo[index];
|
||||
}
|
||||
|
@ -423,7 +423,7 @@ namespace Microsoft.ML.Transforms.Onnx
|
|||
|
||||
internal sealed class OnnxUtils
|
||||
{
|
||||
private static HashSet<Type> _onnxTypeMap =
|
||||
private static readonly HashSet<Type> _onnxTypeMap =
|
||||
new HashSet<Type>
|
||||
{
|
||||
typeof(Double),
|
||||
|
@ -439,7 +439,7 @@ namespace Microsoft.ML.Transforms.Onnx
|
|||
typeof(SByte),
|
||||
typeof(Byte)
|
||||
};
|
||||
private static Dictionary<Type, InternalDataKind> _typeToKindMap =
|
||||
private static readonly Dictionary<Type, InternalDataKind> _typeToKindMap =
|
||||
new Dictionary<Type, InternalDataKind>
|
||||
{
|
||||
{ typeof(Single) , InternalDataKind.R4},
|
||||
|
|
|
@ -441,7 +441,7 @@ namespace Microsoft.ML.Data
|
|||
private readonly ReaderOptions _readerOptions;
|
||||
private int _curDataSetRow;
|
||||
private IEnumerator<int> _dataSetEnumerator;
|
||||
private IEnumerator<int> _blockEnumerator;
|
||||
private readonly IEnumerator<int> _blockEnumerator;
|
||||
private readonly IList[] _columnValues;
|
||||
private readonly Random _rand;
|
||||
|
||||
|
|
|
@ -370,20 +370,20 @@ namespace Microsoft.ML.Data
|
|||
private static readonly FuncInstanceMethodInfo1<Cursor, int, DataViewType, Delegate> _createGetterDelegateCoreMethodInfo
|
||||
= FuncInstanceMethodInfo1<Cursor, int, DataViewType, Delegate>.Create(target => target.CreateGetterDelegateCore<int>);
|
||||
|
||||
private PartitionedFileLoader _parent;
|
||||
private readonly PartitionedFileLoader _parent;
|
||||
|
||||
private readonly bool[] _active;
|
||||
private readonly bool[] _subActive; // Active columns of the sub-cursor.
|
||||
private Delegate[] _getters;
|
||||
private Delegate[] _subGetters; // Cached getters of the sub-cursor.
|
||||
private readonly Delegate[] _getters;
|
||||
private readonly Delegate[] _subGetters; // Cached getters of the sub-cursor.
|
||||
|
||||
private readonly IEnumerable<DataViewSchema.Column> _columnsNeeded;
|
||||
private readonly IEnumerable<DataViewSchema.Column> _subActivecolumnsNeeded;
|
||||
|
||||
private ReadOnlyMemory<char>[] _colValues; // Column values cached from the file path.
|
||||
private readonly ReadOnlyMemory<char>[] _colValues; // Column values cached from the file path.
|
||||
private DataViewRowCursor _subCursor; // Sub cursor of the current file.
|
||||
|
||||
private IEnumerator<int> _fileOrder;
|
||||
private readonly IEnumerator<int> _fileOrder;
|
||||
|
||||
public Cursor(IChannelProvider provider, PartitionedFileLoader parent, IMultiStreamSource files, IEnumerable<DataViewSchema.Column> columnsNeeded, Random rand)
|
||||
: base(provider)
|
||||
|
@ -744,7 +744,7 @@ namespace Microsoft.ML.Data
|
|||
/// </summary>
|
||||
/// <param name="basepath">A base path.</param>
|
||||
/// <param name="files">A list of files under the base path.</param>
|
||||
/// <returns>A realtive file path.</returns>
|
||||
/// <returns>A relative file path.</returns>
|
||||
private string GetRelativePath(string basepath, IMultiStreamSource files)
|
||||
{
|
||||
Contracts.CheckNonEmpty(basepath, nameof(basepath));
|
||||
|
|
Некоторые файлы не были показаны из-за слишком большого количества измененных файлов Показать больше
Загрузка…
Ссылка в новой задаче